release (first build on gitea-watch / IPCR)

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

.gitea/workflows/docker-publish.yml

@@ -0,0 +1,87 @@
+# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
+# folder exists, and GitHub never reads it.
+#
+# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
+#   push  ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag>   docker login as metadec.eth with the
+#         IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
+#   pull  ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag>   anonymous
+# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
+# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
+# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
+# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
+# certificate without that CA, so the login is written straight into ~/.docker/config.json.
+# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
+# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
+
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag for the image (optional, defaults to sha)'
+        required: false
+        type: string
+
+env:
+  REGISTRY: ipcr-push-85-17-246-67.nip.io
+  IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Docker CLI
+        run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
+
+      - name: Trust the Mesh Router CA
+        run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
+        env:
+          IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          buildkitd-config-inline: |
+            [registry."${{ env.REGISTRY }}"]
+              ca = ["/tmp/ipcr-ca.pem"]
+
+      - name: Log in to IPCR
+        env:
+          IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
+        run: |
+          mkdir -p ~/.docker
+          printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -1,34 +1,42 @@
 # meta-core
 
-Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
+Standalone Go service that owns Redis, the HTTP+SSE metadata surface, the
-metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
+User Data Layer store and signing keystore, and a WebDAV file server for the
+MetaMesh dev/prod stack.
 
 ## Overview
 
 meta-core runs as its own container (`metacore-app`). Other MetaMesh services
-(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
+(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
-reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
+meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
-is not exposed outside the container — all metadata I/O is mediated by the
+its `metamesh.core` advertise carries the full `/urls` payload) and then call its
-HTTP API.
+HTTP API. `META_CORE_URL` on a client pins it instead. Redis is not published
+outside the container — all metadata I/O is mediated by the HTTP/SSE API
+(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
 
 ### Key capabilities
 
 | Feature | Description |
 |---------|-------------|
-| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
+| **Redis-volume mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two Redis writers. It is a mutex, not discovery — nothing reads the lock. |
 | **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
-| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
+| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
+| **In-memory search index** | `POST /api/metadata/search` matches against an in-memory snapshot of every record, kept current by incremental refreshes (full reconcile every 30 min) — no Redis round-trip per record on the request path (`internal/api/search_index.go`). |
 | **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
 | **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
-| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
+| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
-| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
+| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
 | **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
 | **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
+| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
+| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
+| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
 
 ### Design characteristics
 
-- **Single-leader** — flock on a shared volume; no external consensus.
+- **Single writer** — flock on the `/meta-core` volume; no external consensus.
-- **HTTP-only externally** — Redis is bound to the container, never published.
+- **HTTP-only externally** — Redis is never published by the compose stacks;
+  debug with `docker exec metacore-app redis-cli …`.
 - **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
   on-disk Redis layout predates the current build (alpha clean-wipe policy;
   see `docs/uuid-rooted-metadata.md`).
@@ -41,34 +49,31 @@ HTTP API.
 ┌──────────────────────────────────────────────────────────────────────────┐
 │                    metacore-app  (standalone container)                  │
 │                                                                          │
+│  leader-election.sh  (flock /meta-core/locks/kv-leader.lock,             │
+│                       then exec supervisord)                             │
 │  supervisord                                                             │
-│   ├─ leader-election.sh  (flock /meta-core/locks/kv-leader.lock,         │
-│   │                       writes kv-leader.info, exec supervisord)      │
 │   ├─ redis-server        (AOF + RDB on /meta-core/db/redis)              │
-│   ├─ nginx               (TLS termination, proxy_cache for /webdav)      │
+│   ├─ nginx  :80          (dashboard, /editor, API proxy,                 │
-│   ├─ rclone rcd          (mount daemon, RC API)                          │
+│   │                       proxy_cache for /webdav, rclone GUI /rclone)   │
+│   ├─ rclone rcd          (mount daemon, RC API on :5572)                 │
 │   ├─ mount-watcher.sh    (rclone mount lifecycle)                        │
 │   └─ meta-core (Go)      ─►  HTTP API + SSE + WebDAV on :9000            │
+│                          ─►  UDP advertise/listen 239.255.99.1:9099      │
 └──────────────────────────────────────────────────────────────────────────┘
                                        │
                 ┌──────────────────────┴──────────────────────┐
-                │  /meta-core (shared volume)                 │
+                │  /meta-core (meta-core's own volume)        │
-                │   locks/kv-leader.{lock,info}               │
+                │   locks/kv-leader.lock                      │
                 │   db/redis/                                 │
-                │   services/*.json                           │
+                │   identity/accounts/*.json                  │
                 │   mounts/{mounts.json, errors/}             │
                 │   watchers.json                             │
                 │   cache/ (nginx proxy_cache)                │
                 └─────────────────────────────────────────────┘
-                                       ▲
+
-        ┌──────────────────────────────┼────────────────────────────────┐
+   meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
-        │                              │                                │
+      │  1. hear the metamesh.core advertise on UDP multicast (carries /urls)
-┌─────────────┐               ┌─────────────┐                ┌─────────────┐
+      └─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never Redis, never the volume
-│  meta-sort  │               │  meta-fuse  │                │ meta-stremio│
-│             │               │             │                │             │
-│  reads      │  HTTP+SSE     │  reads      │  HTTP+SSE      │  reads      │
-│  leader.info├──────────────►│  leader.info├───────────────►│  leader.info│
-└─────────────┘               └─────────────┘                └─────────────┘
 ```
 
 ## Quick start
@@ -82,8 +87,9 @@ make test            # run Go unit tests
 ```
 
 `make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
-`:local` tag is the one every sub-stack docker-compose.yml references
+`:local` tag is the one the sub-stack docker-compose files reference
-(meta-share 10-peer federation, meta-gateway dev stacks); without it the
+(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
+dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
 sub-stacks can't see a freshly-built image by name.
 
 > **Source-change → recreate.** `make docker` rebuilds and re-tags the
@@ -94,12 +100,14 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > make docker
-> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
+> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
->   up -d --force-recreate metagateway-generic-core
+>   up -d --force-recreate metagateway-hub-metacore
-> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
->   up -d --force-recreate metagateway-torznab-core
 > docker compose -f packages/meta-share/dev/docker-compose.yml \
 >   --profile share-10node up -d --force-recreate metashare-core metashare-core-test
+> docker compose -f packages/meta-read/dev/docker-compose.yml \
+>   up -d --force-recreate metaread-dev-core
+> docker compose -f packages/meta-listen/dev/docker-compose.yml \
+>   up -d --force-recreate metalisten-dev-core
 > ```
 >
 > The symptom of a missed recreate is missing API fields on `/urls`
@@ -109,7 +117,7 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > docker inspect meta-core:local --format '{{.Id}}'                # fresh SHA
-> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
+> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'   # repeat for :latest
 > # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
 > ```
 
@@ -126,36 +134,39 @@ HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
 
 ## Configuration
 
-Environment variables consumed by `internal/config/config.go`:
+Environment variables (from `internal/config/config.go` unless noted):
 
 | Variable | Default | Description |
 |---|---|---|
-| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
+| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, Redis data, identity, mounts, watchers, cache). No other service mounts it. |
 | `FILES_PATH` | `/files` | Files volume root. |
-| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
+| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
-| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
+| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
-| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
+| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
-| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
+| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
+| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
 | `REDIS_PORT` | `6379` | Local Redis port. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
-| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind. |
+| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
-| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
+| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
-| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
+| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
-| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
+| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
-| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
+| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
-| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
+| `META_CORE_INDEX_EXCLUDE_PREFIXES` | `categories/newznab/` | Comma-separated field prefixes left out of the in-memory search index; set empty to index everything (`internal/storage/client.go`). |
-| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
+| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
-| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
-| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
 
-Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
+`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
-no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
+`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
-identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
+`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
-retry interval (`ELECTION_RETRY_SECS`, default `5`).
+them (leftovers of the file-based registry); setting them has no effect. There
+is no `META_CORE_REDIS_PORT` / `META_CORE_SERVICE_NAME` /
+`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
+configs created via `/api/watchers`.
 
 ## API reference
 
-All endpoints below are served by `internal/api/server.go`. Examples assume
+All endpoints below are registered in `internal/api/server.go` (mounts,
+watcher and watchers routes in their own packages' `handlers.go`). Examples assume
 the in-container Go port `:9000`; from the host use
 `https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
 (debug-direct).
@@ -167,12 +178,43 @@ GET  /health           # storage + role
 GET  /status           # version, uptime, etc.
 GET  /leader           # current leader info (hostname, PID, timestamps)
 GET  /urls             # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
-GET  /services         # registered services + heartbeats
+GET  /neighbors        # neighbours heard over UDP (beacon v2)
-GET  /api/services     # alias for /services
+GET  /api/neighbors    # same, under /api
-GET  /services/{name}  # one service
+GET  /api/stats        # dashboard counters (see below)
-GET  /services/cleanup/stats
 ```
 
+`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
+are **gone** — the registration-and-heartbeat registry they served was replaced
+by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
+with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
+than a 404; that is what made the dashboard fail with
+`SyntaxError: Unexpected token '<'` until 1.0.21.
+
+#### `GET /api/stats`
+
+Every headline counter in one request, split by cost:
+
+```jsonc
+{
+  "records": 88709,        // SCARD file:__index__ — metadata roots, file or not
+  "redisKeys": 3115447,    // DBSIZE — the flat per-field keys dominate
+  "redisMemory": "588.74M",
+  "identities": 3,         // signing accounts on disk
+  "files": { "count": 4864, "totalSize": 64587914424 },  // null until first sweep
+  "udlUsers": 5,           // accounts holding User Data Layer records
+  "sweptAt": 1789402401970,
+  "sweeping": false
+}
+```
+
+The first four are O(1) commands and are live on every call. `files` and
+`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
+box), so they are served from a single-flight cache refreshed in the background
+with a 60s TTL — the walk is **never** on the request path, because the
+dashboard polls and ten open tabs must not cost ten Redis walks. Prefer this
+over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
+across records that share a file.
+
 ### Metadata — primary surface
 
 These are the routes other services use. `{hash}` may be a UUID root or
@@ -198,6 +240,8 @@ GET    /api/meta/{cid}                # resolve CID → metadata document
 GET    /api/file/{cid}/info           # CID resolution metadata
 GET    /file/{cid}                    # serve file bytes (range-aware)
 POST   /file/cid                      # compute CID for an uploaded file
+POST   /api/files/register        # mint + alias one file under /files/plugin by midhash
+                                  # (the watcher's path for one file; works with the watcher OFF)
 HEAD   /data/{hash}                   # existence + size
 GET    /data/{hash}/path              # absolute path on /files
 ```
@@ -209,6 +253,7 @@ the `/file/{cid}` reverse-lookup path.)
 
 ```bash
 GET    /api/metadata/hash-ids
+GET    /api/files/tuples
 GET    /api/metadata/list
 POST   /api/metadata/search
 POST   /api/metadata/batch
@@ -248,6 +293,44 @@ signalling).
 ```bash
 GET /api/events/files     # file:events stream (watcher events)
 GET /api/events/meta      # meta:events stream (metadata mutations)
+GET /api/events/poll      # DEPRECATED long-poll over file:events — use SSE
+```
+
+### Identity (signing keystore)
+
+Reached through meta-search's identity proxy (and directly by meta-gateway,
+meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
+
+```bash
+GET    /api/identity               # status (never the private key)
+GET    /api/identity/accounts
+POST   /api/identity/generate      # the only call that returns a new private key
+POST   /api/identity/import
+POST   /api/identity/challenge     # one-shot challenge for proof-of-possession
+POST   /api/identity/reveal        # needs proof-of-possession
+DELETE /api/identity               # needs proof-of-possession; reports purged UDL counts
+POST   /api/identity/sign
+POST   /api/identity/sign-batch
+GET    /api/identity/aead-key
+```
+
+### User Data Layer
+
+Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
+writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
+cell in a batch). Redis model in `internal/storage/udl.go`.
+
+```bash
+GET  /api/udl/record?uid&cid&key
+PUT  /api/udl/record
+PUT  /api/udl/records                      # batch
+GET  /api/udl/user/{uid}/key/{key}
+GET  /api/udl/user/{uid}/cid/{cid}
+GET  /api/udl/user/{uid}/all
+GET  /api/udl/user/{uid}/count
+GET  /api/udl/cid/{cid}/users
+GET  /api/udl/cid/{cid}/aggregate          # meta-watch Aggregate wire shape
+GET  /api/udl/users/stats
 ```
 
 ### Mounts
@@ -287,38 +370,42 @@ return a deprecation pointer at the `/api/watchers/*` routes
 
 ```bash
 POST   /api/admin/migrate-dual-roots  # reunify stranded midhash-rooted entries
+POST   /api/admin/migrate-domain-screen  # domain film|tv → screen, workForm backfill (idempotent)
+POST   /api/admin/migrate-user-cids   # backfill udl:idx:user:<uid>:cids; until run, pre-index
+                                      # profiles export as EMPTY. Returns {fixed}
+POST   /api/admin/sweep-literature-echoes[?apply=true]
+       # strip query echoes off literature cards: anilistid on site cards,
+       # workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
+       # Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
 GET/PUT/DELETE /webdav/...            # mounted on /files
 ```
 
 No `/metrics` endpoint is exposed; observability is via container logs and
 the `/health` / `/status` JSON.
 
-## Leader gate
+## Redis-volume mutex
 
-Leader election is a bash flock loop in `docker/leader-election.sh`, not a
+`docker/leader-election.sh` is a bash flock loop, not a Go-level state
-Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
+machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
-script writes the leader API URL to `kv-leader.info` and `exec`s
+script `exec`s supervisord; a second container mounting the same volume
-supervisord. When the process tree dies, the kernel releases the lock and
+retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
-any standby instance can take over. There is no follower path inside the
+dies, the kernel releases the lock. There is no follower path inside the Go
-Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
+binary.
 
-`kv-leader.info` is a single plaintext line containing the API base URL:
+The lock is a **mutex, not service discovery** — it only stops two Redis
-
+writers landing on one RDB/AOF. Nothing reads it, and the `kv-leader.info`
-```
+file it used to publish is gone: siblings locate meta-core over UDP
-http://10.0.1.50:9000
+(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
-```
+`internal/leader` keeps its name for history, but only builds the `/urls`
-
+payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
-Other services read this file, then call `GET /urls` for full discovery.
+and the UDP announce share.
-
-The dev compose runs a single `metacore-app`; the flock primitive is
-retained as a correctness anchor (one process owns Redis and the
-writable state under `/meta-core`), not as live failover.
 
 ## Metadata storage shape (one-line summary)
 
 Roots are opaque UUIDv7 strings; every property is a separate Redis
-STRING at `file:<uuid>/<property>`; every known CID for a file is a
+STRING at `file:<uuid>/<property>`; every known CID is a key-set member
-reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
+`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
+(the canonical CID is derived by rank on read, never stored). Full details in
 [`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
 and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
 The authoritative registry of field semantics and value formats lives in
@@ -328,14 +415,15 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 
 | Package | Purpose |
 |---------|---------|
-| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
+| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + UDP discovery + service wiring. |
 | `internal/config` | Env-driven configuration, path helpers. |
-| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
+| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
-| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
+| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
-| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
+| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, field indexes + search-index bulk reads, UDL store (`udl.go`, `udl_admin.go`), stats, one-shot migrations/sweeps. |
-| `internal/discovery` | Service registry + dead-service cleaner. |
+| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
+| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
 | `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
-| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
+| `internal/api` | HTTP server + all handlers + SSE event endpoints + in-memory search index + `/api/stats`. |
 | `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
 | `internal/snapshot` | Snapshot export / import / wipe. |
 | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
@@ -350,10 +438,12 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 2. Build LeaderInfoProvider (no election — the bash gate already won)
 3. Connect to local Redis (retry up to 30× / 1s)
 4. EnsureSchemaVersion — abort if the existing Redis layout is stale
-5. Start service discovery + dead-service cleaner
+5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
-6. Construct API server (initialises watcher, mounts, WebDAV)
+6. Construct API server (identity keystore migration, watcher, watchers,
-7. Start API server; on first connect, start MetaPublisher + schema Indexer
+   mounts, WebDAV)
-   and republish existing metadata onto meta:events
+7. Start API server + pollers + search-index refresh loop; start
+   MetaPublisher + schema Indexer, then warm field indexes and republish
+   existing metadata onto meta:events in the background
 8. Wait for SIGINT/SIGTERM, shutdown in reverse order
 ```
 
@@ -364,11 +454,13 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 | `github.com/gorilla/mux` | HTTP router |
 | `github.com/redis/go-redis/v9` | Redis client |
 | `github.com/google/uuid` | UUIDv7 minting |
+| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
 | `golang.org/x/net` | WebDAV support |
-| Go stdlib | context, net/http, os, sync, syscall |
+| `github.com/alicebob/miniredis/v2` | In-memory Redis for tests |
 
 Runtime requirements: Go 1.21+ at build time; the container ships Redis,
-rclone, nginx, and supervisord alongside the binary.
+rclone, nginx, and supervisord alongside the binary, plus the React dashboard
+and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
 
 ## License
 

cmd/meta-core/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -10,8 +11,8 @@ import (
 
 	"github.com/metazla/meta-core/internal/api"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
@@ -60,20 +61,57 @@ func main() {
 	}
 	log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
 
-	// Create and start service discovery
+	// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
-	disc := discovery.NewService(cfg)
+	// as metamesh.core — carrying the /urls block services locate us by — plus
-	if err := disc.Start(); err != nil {
+	// its own metamesh.service/<name> nav entry. The payload is rebuilt from
-		log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
+	// leaderProvider on every tick, which keeps it and GET /urls from drifting.
-	}
+	var mesh *meshdisco.Node
-
+	if cfg.EnableUDPDiscovery {
-	// Create and start dead service cleaner
+		hostname, _ := os.Hostname()
-	cleaner := discovery.NewCleaner(cfg)
+		mesh = meshdisco.New(meshdisco.Config{
-	if err := cleaner.Start(); err != nil {
+			Name:     cfg.ServiceName,
-		log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
+			Instance: hostname,
+			Version:  Version,
+			Group:    cfg.DiscoveryGroup,
+			Port:     cfg.DiscoveryPort,
+			Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
+			Payload: func() (string, []meshdisco.Resource) {
+				service := meshdisco.Resource{
+					ID:        "service",
+					Caps:      []string{meshdisco.CapServicePrefix + cfg.ServiceName},
+					Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
+				}
+				info := leaderProvider.LeaderInfo()
+				if info == nil {
+					return "starting", []meshdisco.Resource{service}
+				}
+				data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
+					Hostname:          info.Hostname,
+					BaseUrl:           info.BaseUrl,
+					ApiUrl:            info.ApiUrl,
+					WebdavUrl:         info.WebdavUrl,
+					WebdavUrlInternal: info.WebdavUrlInternal,
+				}})
+				core := meshdisco.Resource{
+					ID:        "core",
+					Caps:      []string{meshdisco.CapCore},
+					Endpoints: map[string]string{"http": info.ApiUrl},
+					Data:      data,
+				}
+				return "running", []meshdisco.Resource{core, service}
+			},
+		})
+	if err := mesh.Start(); err != nil {
+			log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
+			mesh = nil
+		}
+	} else {
+		log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
 	}
 
 	// Create and start API server
-	apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
+	apiServer := api.NewServer(cfg, leaderProvider, storageClient)
+	apiServer.SetMeshNode(mesh)
 	if err := apiServer.Start(); err != nil {
 		log.Fatalf("[meta-core] Failed to start API server: %v", err)
 	}
@@ -90,12 +128,10 @@ func main() {
 		log.Printf("[meta-core] Error stopping API server: %v", err)
 	}
 
-	if err := cleaner.Stop(); err != nil {
+	if mesh != nil {
-		log.Printf("[meta-core] Error stopping service cleaner: %v", err)
+		if err := mesh.Stop(); err != nil {
-	}
+			log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
-
+		}
-	if err := disc.Stop(); err != nil {
-		log.Printf("[meta-core] Error stopping service discovery: %v", err)
 	}
 
 	if err := storageClient.Close(); err != nil {
@@ -112,3 +148,17 @@ func waitForShutdown() {
 	<-sigChan
 }
 
+// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
+// the three-way fallback the file registry used (discovery/service.go):
+// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
+// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
+// the container IP.
+func coreBaseURL(cfg *config.Config) string {
+	if cfg.PublicURL != "" {
+		return cfg.PublicURL
+	}
+	if cfg.BaseURL != "" {
+		return cfg.BaseURL
+	}
+	return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
+}

dashboard/dist/index.html

@@ -21,7 +21,7 @@
         min-height: 100vh;
       }
     </style>
-    <script type="module" crossorigin src="/assets/index-Cbd1uH5A.js"></script>
+    <script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
   </head>
   <body>

dashboard/package-lock.json

@@ -8,6 +8,7 @@
       "name": "@meta-core/dashboard",
       "version": "1.0.0",
       "dependencies": {
+        "@noble/secp256k1": "^3.1.0",
         "iso-639-3": "^3.0.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
@@ -744,6 +745,15 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@noble/secp256k1": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.1.0.tgz",
+      "integrity": "sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
     "node_modules/@remix-run/router": {
       "version": "1.23.2",
       "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz",

dashboard/package.json

@@ -9,10 +9,11 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "@noble/secp256k1": "^3.1.0",
+    "iso-639-3": "^3.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "react-router-dom": "^6.20.0",
+    "react-router-dom": "^6.20.0"
-    "iso-639-3": "^3.0.1"
   },
   "devDependencies": {
     "@types/react": "^18.2.38",

dashboard/src/components/ServiceNav.tsx

@@ -1,156 +1,54 @@
-import { useState, useEffect } from 'react';
+import { useEffect } from 'react';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
+// of defining the custom element.
+import '../meta-service-menu.js';
 
-interface ServiceInfo {
+/**
-  name: string;
+ * ServiceNav — the neighbouring meta-* services, as a burger menu.
-  hostname: string;
+ *
-  baseUrl: string;
+ * Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
-  status: string;
+ * a framework-free custom element shared by every meta-* UI (see
-  lastHeartbeat: string;
+ * scripts/check-mirrors.sh). The old implementation was ~165 lines of React
-}
+ * that existed in four near-identical copies and polled meta-core's
+ * /api/services; the element polls this service's own /api/neighbors instead,
+ * so the nav still renders when meta-core is down.
+ *
+ * The element lives in shadow DOM and cannot see this app's CSS variables, so
+ * the four --mm-nav-* tokens below map our palette onto it.
+ */
 
-interface ServicesResponse {
+declare global {
-  services: ServiceInfo[];
+    // eslint-disable-next-line @typescript-eslint/no-namespace
-  current: string;
+    namespace JSX {
-}
+        interface IntrinsicElements {
-
+            'meta-service-menu': React.DetailedHTMLProps<
-const serviceIcons: Record<string, string> = {
+                React.HTMLAttributes<HTMLElement>,
-  'meta-core': '🗄️',
+                HTMLElement
-  'meta-sort': '📁',
+            > & { current?: string; endpoint?: string; label?: string };
-  'meta-fuse': '🗂️',
+        }
-  'meta-stremio': '🎬',
+    }
-  'meta-orbit': '🌐',
-  'default': '📦'
-};
-
-function formatServiceName(name: string): string {
-  return name.split('-').map(word =>
-    word.charAt(0).toUpperCase() + word.slice(1)
-  ).join(' ');
 }
 
 function ServiceNav() {
-  const [services, setServices] = useState<ServiceInfo[]>([]);
+    useEffect(() => {
-  const [loading, setLoading] = useState(true);
+        // Import above is for the side effect of defining the element; this
-  const currentService = 'meta-core';
+        // keeps bundlers from tree-shaking it away in production builds.
+    }, []);
 
-  useEffect(() => {
+    return (
-    const fetchServices = async () => {
+        <>
-      try {
+            <meta-service-menu current="meta-core" />
-        const response = await fetch('/services');
+            <style>{`
-        if (response.ok) {
+                meta-service-menu {
-          const data: ServicesResponse = await response.json();
+                    --mm-nav-fg: var(--text-primary, #e0e0e0);
-          setServices(data.services || []);
+                    --mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
-        }
+                    --mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
-      } catch (error) {
+                    --mm-nav-accent: var(--accent-primary, #4ecdc4);
-        console.error('Failed to fetch services:', error);
-      } finally {
-        setLoading(false);
-      }
-    };
-
-    fetchServices();
-    const interval = setInterval(fetchServices, 30000);
-    return () => clearInterval(interval);
-  }, []);
-
-  if (loading) return null;
-
-  const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
-
-  if (sortedServices.length === 0) return null;
-
-  return (
-    <nav className="services-nav">
-      <span className="services-nav-label">Services:</span>
-      <div className="services-nav-items">
-        {sortedServices.map(service => {
-          const icon = serviceIcons[service.name] || serviceIcons.default;
-          const isActive = service.name === currentService;
-          return (
-            <a
-              key={service.name}
-              href={isActive ? '#' : service.baseUrl}
-              className={`service-link${isActive ? ' active' : ''}`}
-              onClick={(e) => {
-                e.preventDefault();
-                if (!isActive) {
-                  window.location.href = service.baseUrl;
                 }
-              }}
+            `}</style>
-            >
+        </>
-              <span className="service-icon">{icon}</span>
+    );
-              <span>{formatServiceName(service.name)}</span>
-              <span className="service-status"></span>
-            </a>
-          );
-        })}
-      </div>
-
-      <style>{`
-        .services-nav {
-          display: flex;
-          align-items: center;
-          gap: 0.5rem;
-        }
-
-        .services-nav-label {
-          color: #888;
-          font-size: 0.8rem;
-          white-space: nowrap;
-        }
-
-        .services-nav-items {
-          display: flex;
-          gap: 0.35rem;
-        }
-
-        .service-link {
-          display: inline-flex;
-          align-items: center;
-          gap: 0.3rem;
-          padding: 0.35rem 0.6rem;
-          border-radius: 6px;
-          text-decoration: none;
-          font-size: 0.8rem;
-          font-weight: 500;
-          transition: all 0.2s;
-          color: #ccc;
-          background: #0f3460;
-          border: 1px solid #1a4a7a;
-        }
-
-        .service-link:hover {
-          background: #1a4a7a;
-          border-color: rgba(78, 205, 196, 0.5);
-          text-decoration: none;
-          color: #fff;
-        }
-
-        .service-link.active {
-          background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
-          border-color: rgba(78, 205, 196, 0.5);
-          color: #fff;
-        }
-
-        .service-link .service-icon {
-          font-size: 0.9rem;
-        }
-
-        .service-link .service-status {
-          width: 5px;
-          height: 5px;
-          border-radius: 50%;
-          background: #4ecdc4;
-        }
-
-        @media (max-width: 800px) {
-          .services-nav-label {
-            display: none;
-          }
-        }
-      `}</style>
-    </nav>
-  );
 }
 
 export default ServiceNav;

dashboard/src/lib/api.ts

@@ -0,0 +1,28 @@
+// JSON fetch that fails with a sentence instead of a parser error.
+//
+// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
+// path it does not have a proxy rule for answers 200 text/html with the SPA
+// shell. A caller that does `res.ok && res.json()` on such a path gets
+// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
+// no clue which request produced it — which is exactly how a route deleted
+// from the Go router (`/services`, removed in favour of /api/neighbors) sat
+// broken on the dashboard.
+//
+// So: check the content type, and name the URL in every error.
+
+export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
+  const res = await fetch(url, init);
+  const ct = res.headers.get('content-type') || '';
+
+  if (!res.ok) {
+    throw new Error(`${url} → HTTP ${res.status}`);
+  }
+  if (!ct.includes('application/json')) {
+    // The SPA fallback is the overwhelmingly likely cause; say so.
+    throw new Error(
+      `${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
+        `missing and nginx served the dashboard shell instead)`
+    );
+  }
+  return (await res.json()) as T;
+}

dashboard/src/lib/sign.ts

@@ -0,0 +1,127 @@
+// Client-side proof of possession for account operations.
+//
+// meta-core stores every private key in plaintext, so it can sign as anybody —
+// which means a signature it produced proves nothing about who asked. The only
+// signature that carries authority is one made where the key actually is, and
+// for a browser-driven admin console that is here. meta-core enforces the other
+// half by refusing to sign anything in the challenge domain
+// (identity.ChallengeDomain), so this path cannot be short-circuited.
+//
+// The secret key entered on this page is used for exactly one signature and is
+// never sent anywhere: `/api/identity/challenge` receives only the uid, and the
+// request that follows carries only the challenge and its signature.
+//
+// Same scheme, same curve and same encodings as meta-watch's sign-in
+// (packages/meta-watch/ui/js/auth.js) — deliberately, so one implementation on
+// the server verifies both.
+
+import * as secp from '@noble/secp256k1';
+
+export type IdentityAction = 'reveal' | 'delete';
+
+function hexToBytes(hex: string): Uint8Array | null {
+  let h = hex.trim();
+  if (h.startsWith('0x') || h.startsWith('0X')) h = h.slice(2);
+  if (h.length !== 64 || /[^0-9a-fA-F]/.test(h)) return null;
+  const out = new Uint8Array(32);
+  for (let i = 0; i < 32; i++) out[i] = parseInt(h.slice(i * 2, i * 2 + 2), 16);
+  return out;
+}
+
+function bytesToBase64(bytes: Uint8Array): string {
+  let s = '';
+  for (const b of bytes) s += String.fromCharCode(b);
+  return btoa(s);
+}
+
+const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
+
+/// base58btc, the multibase "z" alphabet. Spelled out rather than pulled in:
+/// the alternative is a second dependency for thirty lines of arithmetic.
+function base58Encode(bytes: Uint8Array): string {
+  const digits: number[] = [0];
+  for (const byte of bytes) {
+    let carry = byte;
+    for (let i = 0; i < digits.length; i++) {
+      carry += digits[i] << 8;
+      digits[i] = carry % 58;
+      carry = (carry / 58) | 0;
+    }
+    while (carry) {
+      digits.push(carry % 58);
+      carry = (carry / 58) | 0;
+    }
+  }
+  // Leading zero bytes encode as leading '1's rather than being dropped.
+  let out = '';
+  for (let i = 0; i < bytes.length && bytes[i] === 0; i++) out += B58[0];
+  for (let i = digits.length - 1; i >= 0; i--) out += B58[digits[i]];
+  return out;
+}
+
+/// The uid a secret key belongs to: "z" + base58btc(33-byte compressed pubkey).
+/// Derived locally so a mistyped or wrong-account key is caught before any
+/// request goes out. Mirrors `identityFromPrivKey` in meta-core's identity.go.
+export function uidFromSecretKey(secretKeyHex: string): string | null {
+  const raw = hexToBytes(secretKeyHex);
+  if (!raw) return null;
+  try {
+    return 'z' + base58Encode(secp.getPublicKey(raw, true));
+  } catch {
+    return null; // not a valid scalar
+  }
+}
+
+/// Whether this page can sign at all.
+///
+/// noble's signAsync takes its sha256 from crypto.subtle, which browsers expose
+/// only in a secure context. The dashboard behind Caddy is https and fine; the
+/// debug-direct port (18083) is plain http, where every signature would fail
+/// with a cryptic error unless we say so first.
+export function canSign(): boolean {
+  return Boolean(globalThis.crypto?.subtle);
+}
+
+export const INSECURE_CONTEXT_MESSAGE =
+  'This page cannot sign over a plain-http connection — browsers only expose the ' +
+  'needed crypto on https (or localhost). Reach the dashboard through Caddy ' +
+  '(https://metacore-dev.localhost:8083) rather than the debug-direct port.';
+
+/// Ask meta-core for a challenge, sign it locally, and hand back the proof.
+///
+/// Throws with a message fit to show the user. The caller passes the proof
+/// straight into the reveal/delete request body.
+export async function proveOwnership(
+  uid: string,
+  action: IdentityAction,
+  secretKeyHex: string
+): Promise<{ challenge: string; signature: string }> {
+  if (!canSign()) throw new Error(INSECURE_CONTEXT_MESSAGE);
+
+  const raw = hexToBytes(secretKeyHex);
+  if (!raw) {
+    throw new Error('That is not a valid secret key: expected 64 hex characters.');
+  }
+  const derived = uidFromSecretKey(secretKeyHex);
+  if (!derived) throw new Error('That is not a valid secp256k1 secret key.');
+  if (derived !== uid) {
+    // Caught here rather than by the server, so the message can say which
+    // account the key actually is — the useful half of the answer.
+    throw new Error(`That key belongs to a different account (${derived}), not this one.`);
+  }
+
+  const cRes = await fetch('/api/identity/challenge', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ uid, action }),
+  });
+  const cBody = await cRes.json().catch(() => ({}));
+  if (!cRes.ok || !cBody.challenge) {
+    throw new Error(cBody.message ?? `Could not start verification (HTTP ${cRes.status}).`);
+  }
+
+  // signAsync prehashes with sha256 and returns a 64-byte compact (r‖s)
+  // signature — noble v3 dropped DER. meta-core's identity.Verify accepts both.
+  const sig = await secp.signAsync(new TextEncoder().encode(cBody.challenge), raw);
+  return { challenge: cBody.challenge, signature: bytesToBase64(sig) };
+}

dashboard/src/meta-service-menu.js

@@ -0,0 +1,308 @@
+/**
+ * <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
+ *
+ * Replaces the eight divergent "services bar" implementations (four copies of
+ * a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
+ * ones) with a single framework-free custom element. It drops into the React
+ * dashboards as a plain tag — no React dependency — and into the vanilla UIs
+ * as an ES module.
+ *
+ * Data comes from the host service's own `GET /api/neighbors`, same-origin.
+ * Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
+ * make a fetch to a neighbour a dead end. Links are plain navigations.
+ *
+ * ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
+ * `scripts/check-mirrors.sh`. Edit one → edit all.
+ *
+ * Shadow DOM is not optional here. The eight UIs use four incompatible token
+ * vocabularies, and two of them define *different colours under the same
+ * names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
+ * #161b22 in meta-dup). meta-search and meta-share are also the only
+ * light-mode-capable UIs, so a hardcoded dark palette would break them.
+ * The element therefore styles itself from four tokens with dark defaults:
+ *
+ *     --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
+ *
+ * Each host maps its own variables onto those, e.g.
+ *     meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
+ *
+ * Attributes:
+ *   current   name of the host service, so it can be marked and not linked
+ *   endpoint  override the default "/api/neighbors"
+ *   label     button tooltip (default "Services")
+ *
+ * Property:
+ *   rewriteUrl  optional (name, url, neighbour) => url, for hosts that must
+ *               translate internal hostnames into browser-reachable ones
+ *               (meta-gateway does this for its dev/prod split).
+ */
+
+const ICONS = {
+  "meta-core": "⚙️",
+  "meta-sort": "📁",
+  "meta-fuse": "🗂️",
+  "meta-stremio": "🎬",
+  "meta-dup": "🔍",
+  "meta-search": "🔎",
+  "meta-share": "🔗",
+  "meta-gateway": "🌉",
+  "meta-watch": "📺",
+  "meta-listen": "🎧",
+  "meta-read": "📖",
+};
+const DEFAULT_ICON = "📦";
+const POLL_MS = 30000;
+
+function titleCase(name) {
+  return String(name || "")
+    .split("-")
+    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+    .join(" ");
+}
+
+const TEMPLATE = `
+<style>
+  :host {
+    --_fg: var(--mm-nav-fg, #e8eaed);
+    --_bg: var(--mm-nav-bg, #1a1a2e);
+    --_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
+    --_accent: var(--mm-nav-accent, #4ecdc4);
+    position: relative;
+    display: inline-block;
+    font-family: inherit;
+    font-size: 14px;
+    line-height: 1.4;
+  }
+  button.trigger {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.4rem;
+    width: 2.1rem;
+    height: 2.1rem;
+    padding: 0;
+    cursor: pointer;
+    color: var(--_fg);
+    background: transparent;
+    border: 1px solid var(--_border);
+    border-radius: 8px;
+    font: inherit;
+    transition: border-color 0.15s, background 0.15s;
+  }
+  button.trigger:hover { border-color: var(--_accent); }
+  button.trigger:focus-visible {
+    outline: 2px solid var(--_accent);
+    outline-offset: 2px;
+  }
+  button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
+  .bars { display: block; width: 15px; }
+  .bars span {
+    display: block;
+    height: 2px;
+    margin: 3px 0;
+    border-radius: 2px;
+    background: currentColor;
+  }
+  .panel {
+    position: absolute;
+    top: calc(100% + 6px);
+    right: 0;
+    z-index: 1000;
+    min-width: 13rem;
+    max-height: 70vh;
+    overflow-y: auto;
+    padding: 0.3rem;
+    background: var(--_bg);
+    border: 1px solid var(--_border);
+    border-radius: 10px;
+    box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
+  }
+  .panel[hidden] { display: none; }
+  a.item, .item {
+    display: flex;
+    align-items: center;
+    gap: 0.55rem;
+    padding: 0.45rem 0.6rem;
+    border-radius: 6px;
+    color: var(--_fg);
+    text-decoration: none;
+    white-space: nowrap;
+  }
+  a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
+  .item.active { color: var(--_accent); font-weight: 600; cursor: default; }
+  .item.dead { opacity: 0.55; cursor: not-allowed; }
+  .icon { font-size: 1rem; line-height: 1; }
+  .dot {
+    width: 6px;
+    height: 6px;
+    margin-left: auto;
+    border-radius: 50%;
+    background: var(--_accent);
+  }
+  .empty {
+    padding: 0.6rem;
+    opacity: 0.7;
+    font-size: 0.85em;
+    white-space: nowrap;
+  }
+  /* Respect a host that opts out of motion. */
+  @media (prefers-reduced-motion: reduce) {
+    button.trigger { transition: none; }
+  }
+</style>
+<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
+  <span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
+</button>
+<div class="panel" role="menu" hidden></div>
+`;
+
+class MetaServiceMenu extends HTMLElement {
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot.innerHTML = TEMPLATE;
+    this._btn = this.shadowRoot.querySelector("button.trigger");
+    this._panel = this.shadowRoot.querySelector(".panel");
+    this._neighbors = [];
+    this._timer = null;
+    this._open = false;
+
+    this._onDocClick = (e) => {
+      if (!this._open) return;
+      if (e.composedPath().includes(this)) return;
+      this._setOpen(false);
+    };
+    this._onKey = (e) => {
+      if (e.key === "Escape" && this._open) {
+        this._setOpen(false);
+        this._btn.focus();
+      }
+    };
+  }
+
+  connectedCallback() {
+    this._btn.title = this.getAttribute("label") || "Services";
+    this._btn.setAttribute("aria-label", this._btn.title);
+    this._btn.addEventListener("click", () => this._setOpen(!this._open));
+    document.addEventListener("click", this._onDocClick);
+    document.addEventListener("keydown", this._onKey);
+
+    this.refresh();
+    this._timer = setInterval(() => this.refresh(), POLL_MS);
+  }
+
+  disconnectedCallback() {
+    if (this._timer) clearInterval(this._timer);
+    this._timer = null;
+    document.removeEventListener("click", this._onDocClick);
+    document.removeEventListener("keydown", this._onKey);
+  }
+
+  get endpoint() {
+    return this.getAttribute("endpoint") || "/api/neighbors";
+  }
+
+  get current() {
+    return this.getAttribute("current") || "";
+  }
+
+  async refresh() {
+    try {
+      const res = await fetch(this.endpoint, {
+        headers: { Accept: "application/json" },
+      });
+      if (!res.ok) throw new Error(String(res.status));
+      const data = await res.json();
+      // `services` is the legacy alias some backends still emit.
+      this._neighbors = data.neighbors || data.services || [];
+    } catch {
+      // Silent: the menu is navigation, not a health indicator. Keep whatever
+      // we last had rather than blanking the list on one failed poll.
+    }
+    this._render();
+  }
+
+  _setOpen(open) {
+    this._open = open;
+    this._panel.hidden = !open;
+    this._btn.setAttribute("aria-expanded", String(open));
+    if (open) this.refresh();
+  }
+
+  _render() {
+    const panel = this._panel;
+    panel.textContent = "";
+
+    const seen = new Set();
+    const rows = [];
+    for (const n of this._neighbors) {
+      const name = n.name || "";
+      if (!name || seen.has(name)) continue;
+      // Beacon v2 neighbours include plugins; the nav lists services only.
+      // (A row without `caps` comes from a pre-v2 backend — keep it.)
+      const isService = (c) => String(c).startsWith("metamesh.service/");
+      if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
+      seen.add(name);
+      rows.push(n);
+    }
+    // Always show the host itself, even before its own echo comes back.
+    if (this.current && !seen.has(this.current)) {
+      rows.push({ name: this.current, baseUrl: "" });
+    }
+    rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+
+    if (rows.length === 0) {
+      const empty = document.createElement("div");
+      empty.className = "empty";
+      empty.textContent = "No services found";
+      panel.appendChild(empty);
+      return;
+    }
+
+    for (const n of rows) {
+      const isActive = n.name === this.current;
+      let url = n.baseUrl || "";
+      if (typeof this.rewriteUrl === "function") {
+        try {
+          url = this.rewriteUrl(n.name, url, n) || "";
+        } catch {
+          /* a bad hook must not blank the menu */
+        }
+      }
+
+      // A neighbour with no reachable URL is shown but not linked — better
+      // than a link that goes nowhere.
+      const el = document.createElement(isActive || !url ? "span" : "a");
+      el.className =
+        "item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
+      if (el.tagName === "A") {
+        el.href = url;
+        el.setAttribute("role", "menuitem");
+      }
+      if (!isActive && !url) el.title = "No reachable URL advertised";
+
+      const icon = document.createElement("span");
+      icon.className = "icon";
+      icon.textContent = ICONS[n.name] || DEFAULT_ICON;
+
+      const label = document.createElement("span");
+      label.textContent = titleCase(n.name);
+
+      el.append(icon, label);
+      // Beacon v2 omits `status` when running.
+      if ((n.status || "running") === "running" || isActive) {
+        const dot = document.createElement("span");
+        dot.className = "dot";
+        dot.title = n.status || "running";
+        el.appendChild(dot);
+      }
+      panel.appendChild(el);
+    }
+  }
+}
+
+if (!customElements.get("meta-service-menu")) {
+  customElements.define("meta-service-menu", MetaServiceMenu);
+}
+
+export default MetaServiceMenu;

dashboard/src/pages/Dashboard.tsx

@@ -1,4 +1,5 @@
 import { useEffect, useState } from 'react';
+import { getJSON } from '../lib/api';
 
 interface HealthStatus {
   status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
   timestamp: string;
 }
 
+interface FileStats {
+  count: number;
+  totalSize: number;
+}
+
+interface Stats {
+  records: number;
+  redisKeys: number;
+  redisMemory: string;
+  identities: number;
+  files: FileStats | null;
+  udlUsers: number | null;
+  sweptAt: number;
+  sweeping: boolean;
+}
+
 interface WatcherStatus {
   id: string;
   path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
   count: number;
 }
 
-interface ServiceInfo {
-  name: string;
-  api: string;
-  capabilities: string[];
-  timestamp: number;
-}
-
 const cardStyle: React.CSSProperties = {
   background: '#16213e',
   borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
 
 const gridStyle: React.CSSProperties = {
   display: 'grid',
-  gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
+  // 200px, not 220: five tiles must fit one row inside the 1200px container
+  // (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
+  gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
   gap: '1rem',
+  marginBottom: '1rem',
 };
 
 const statStyle: React.CSSProperties = {
   fontSize: '2rem',
   fontWeight: 'bold',
   color: '#4ade80',
+  lineHeight: 1.2,
 };
 
+const labelStyle: React.CSSProperties = {
+  marginBottom: '0.75rem',
+  color: '#888',
+  fontSize: '0.85rem',
+  textTransform: 'uppercase',
+  letterSpacing: '0.04em',
+};
+
+const noteStyle: React.CSSProperties = {
+  color: '#6b7a99',
+  fontSize: '0.8rem',
+  marginTop: '0.4rem',
+};
+
+const numberFmt = new Intl.NumberFormat();
+
+function formatCount(n: number | null | undefined): string {
+  return n === null || n === undefined ? '—' : numberFmt.format(n);
+}
+
+// Binary units: this counts bytes on disk, and every other tool the operator
+// compares against (du, docker, the mount page) reports GiB.
+function formatBytes(bytes: number | null | undefined): string {
+  if (bytes === null || bytes === undefined) return '—';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
+}
+
+function Tile({
+  label,
+  value,
+  note,
+  pending,
+}: {
+  label: string;
+  value: string;
+  note?: string;
+  pending?: boolean;
+}) {
+  return (
+    <div style={cardStyle}>
+      <div style={labelStyle}>{label}</div>
+      <div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
+        {value}
+      </div>
+      {note && <div style={noteStyle}>{note}</div>}
+    </div>
+  );
+}
+
 export default function Dashboard() {
   const [health, setHealth] = useState<HealthStatus | null>(null);
+  const [stats, setStats] = useState<Stats | null>(null);
   const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
-  const [services, setServices] = useState<ServiceInfo[]>([]);
   const [error, setError] = useState<string | null>(null);
 
   useEffect(() => {
+    let cancelled = false;
+
     const fetchData = async () => {
-      try {
+      // Settled, not all: one dead endpoint must not blank every tile.
-        // Fetch health
+      const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
-        const healthRes = await fetch('/health');
+        getJSON<HealthStatus>('/health'),
-        if (healthRes.ok) {
+        getJSON<Stats>('/api/stats'),
-          setHealth(await healthRes.json());
+        getJSON<WatchersResponse>('/api/watchers'),
-        }
+      ]);
+      if (cancelled) return;
 
-        // Fetch watchers status
+      if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
-        const watchersRes = await fetch('/api/watchers');
+      if (statsRes.status === 'fulfilled') setStats(statsRes.value);
-        if (watchersRes.ok) {
+      if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
-          setWatchersData(await watchersRes.json());
-        }
 
-        // Fetch services
+      const failures = [healthRes, statsRes, watchersRes]
-        const servicesRes = await fetch('/services');
+        .filter((r): r is PromiseRejectedResult => r.status === 'rejected')
-        if (servicesRes.ok) {
+        .map((r) => String(r.reason?.message ?? r.reason));
-          const data = await servicesRes.json();
+      setError(failures.length ? failures.join(' · ') : null);
-          setServices(data.services || []);
-        }
-
-        setError(null);
-      } catch (err) {
-        setError(String(err));
-      }
     };
 
     fetchData();
     const interval = setInterval(fetchData, 5000);
-    return () => clearInterval(interval);
+    return () => {
+      cancelled = true;
+      clearInterval(interval);
+    };
   }, []);
 
   const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
     }
   };
 
-  // Aggregate watcher stats
+  const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
-  const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
+  const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
-  const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
+  const lastScan =
-  const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
+    watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
-  const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
+
+  // Records we hold metadata for but have no local file: gateway/cid-rooted
+  // entries, plus anything whose file went away. The interesting number on a
+  // client box, where it should be most of them.
+  const fileless =
+    stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
+
+  const sweepPending = !!stats && !stats.files;
+  const sweptNote = (() => {
+    if (!stats) return undefined;
+    if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
+    if (stats.sweeping) return 'refreshing…';
+    if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
+    return undefined;
+  })();
 
   return (
-    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
+    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
       <h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
 
       {error && (
         <div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
-          Error: {error}
+          {error}
         </div>
       )}
 
       <div style={gridStyle}>
-        {/* Health Card */}
+        <Tile
+          label="Files discovered"
+          value={formatCount(stats?.files?.count)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Library size"
+          value={formatBytes(stats?.files?.totalSize)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Metadata entries"
+          value={formatCount(stats?.records)}
+          note={
+            fileless === null
+              ? 'records in the index'
+              : `${formatCount(fileless)} with no local file`
+          }
+        />
+        <Tile
+          label="Redis keys"
+          value={formatCount(stats?.redisKeys)}
+          note={stats?.redisMemory ? `${stats.redisMemory} in use` : undefined}
+        />
+        <Tile
+          label="Identities"
+          value={formatCount(stats?.identities)}
+          note={
+            stats?.udlUsers !== null && stats?.udlUsers !== undefined
+              ? `${formatCount(stats.udlUsers)} with user data`
+              : 'signing accounts'
+          }
+        />
+      </div>
+
+      <div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
+          <div style={labelStyle}>Service health</div>
           {health ? (
             <>
               <div style={statStyle}>
                 {health.status === 'ok' ? 'Healthy' : health.status}
               </div>
-              <p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
+              <p style={noteStyle}>
+                Redis: {health.redis ? 'Connected' : 'Disconnected'}
+              </p>
             </>
           ) : (
-            <p>Loading...</p>
+            <p style={noteStyle}>Loading…</p>
           )}
         </div>
 
-        {/* Watchers Status Card */}
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
+          <div style={labelStyle}>File watchers</div>
           {watchersData ? (
             <>
-              <div style={statStyle}>{totalFiles}</div>
+              <div style={statStyle}>
-              <p>Files discovered</p>
+                {activeWatchers} / {watchersData.count}
-              <p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
+              </div>
-              <p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
+              <p style={noteStyle}>active watchers</p>
+              <p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
               {lastScan > 0 && (
-                <p>Last scan: {new Date(lastScan).toLocaleString()}</p>
+                <p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
               )}
               <button
                 onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
               </button>
             </>
           ) : (
-            <p>Loading...</p>
+            <p style={noteStyle}>Loading…</p>
-          )}
-        </div>
-
-        {/* Services Card */}
-        <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
-          <div style={statStyle}>{services.length}</div>
-          <p>Services registered</p>
-          {services.length > 0 && (
-            <ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
-              {services.map((svc) => (
-                <li key={svc.name} style={{ marginBottom: '0.5rem' }}>
-                  <strong>{svc.name}</strong>
-                  <br />
-                  <span style={{ color: '#888', fontSize: '0.9rem' }}>
-                    {svc.capabilities?.join(', ') || 'No capabilities'}
-                  </span>
-                </li>
-              ))}
-            </ul>
           )}
         </div>
       </div>

dashboard/src/pages/Identity.tsx

@@ -1,21 +1,74 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 
-// Status returned by GET /api/identity. Private key is never present here —
+import {
-// it is shown exactly once at generation time (see GeneratedKeyPanel).
+  INSECURE_CONTEXT_MESSAGE,
-interface IdentityStatus {
+  canSign,
-  hasIdentity: boolean;
+  proveOwnership,
-  uid?: string;
+  uidFromSecretKey,
-  curve?: string;
+  type IdentityAction,
-  createdAt?: number;
+} from '../lib/sign';
-}
 
-interface GeneratedKey {
+// Admin console for meta-core's account keystore.
+//
+// meta-core stores one secp256k1 keypair per file under
+// {META_CORE_PATH}/identity/accounts/<uid>.json and has NO notion of a current
+// user — which account is "you" is decided by whichever client signs in
+// (meta-watch). So this page lists accounts; it never speaks of an active one.
+//
+// The private key is returned by exactly two calls: POST /api/identity/generate
+// (once, at creation) and POST /api/identity/reveal (on demand, for an account
+// that already exists). Both land in the same exclusive SecretKeyPanel.
+//
+// Reveal and remove both require the account's own secret key, entered here and
+// used to sign a one-shot challenge locally — the key itself is never sent. So
+// this page can enumerate and create, but it holds no authority over an
+// existing account that its operator does not already have the key for. There
+// is deliberately no admin override: on this node the private key *is* the
+// account.
+
+interface Account {
   uid: string;
   curve: string;
   createdAt: number;
-  privateKeyHex: string;
 }
 
+// What one account holds in the User Data Layer. Absent from
+// /api/udl/users/stats means zero, not unknown.
+interface Stats {
+  records: number;
+  cids: number;
+  keys: number;
+}
+
+// A secret on screen. Carries no createdAt: /api/identity/reveal answers with
+// {uid, curve, privateKeyHex} only.
+interface SecretKeyView {
+  uid: string;
+  privateKeyHex: string;
+  kind: 'generated' | 'revealed';
+}
+
+// Which row has an open confirmation, and for what. One discriminated value
+// rather than a pair of ids, so at most one confirm panel can ever be open.
+interface Pending {
+  uid: string;
+  kind: 'remove' | 'reveal';
+}
+
+// Both confirmable operations map to a server-side action name, and the
+// challenge is bound to it: a challenge minted to reveal a key cannot delete
+// the account. Keeping the mapping in one place stops the two drifting.
+const ACTION_FOR: Record<Pending['kind'], IdentityAction> = {
+  reveal: 'reveal',
+  remove: 'delete',
+};
+
+// meta-watch's sentinel cell for profile records: display name and avatar are
+// about the *user*, not a content file, so they live at (uid, "self", ...).
+// meta-core stores them opaquely and knows nothing of the convention.
+const PROFILE_CID = 'self';
+const PROFILE_NAME_KEY = 'profile:name';
+
 const cardStyle: React.CSSProperties = {
   background: '#16213e',
   borderRadius: '8px',
@@ -59,6 +112,21 @@ const monoStyle: React.CSSProperties = {
   borderRadius: '4px',
 };
 
+const dangerPanelStyle: React.CSSProperties = {
+  background: '#1f0a0a',
+  border: '1px solid #7f1d1d',
+  padding: '0.75rem 1rem',
+  borderRadius: '4px',
+  marginTop: '0.75rem',
+};
+
+const errorTextStyle: React.CSSProperties = {
+  color: '#f87171',
+  background: '#1f0a0a',
+  padding: '0.5rem 0.75rem',
+  borderRadius: '4px',
+};
+
 function formatCreatedAt(unixSec?: number): string {
   if (!unixSec) return '';
   try {
@@ -79,25 +147,165 @@ async function errorMessage(res: Response, fallback: string): Promise<string> {
   return `${fallback} (HTTP ${res.status})`;
 }
 
-function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss: () => void }) {
+// Copy text, falling back to selecting the element that displays it.
-  const [copied, setCopied] = useState(false);
+//
-  const copy = async () => {
+// The fallback is load-bearing, not politeness: the dashboard is served over
-    try {
+// plain HTTP, where navigator.clipboard is undefined — without it the copy
-      await navigator.clipboard.writeText(data.privateKeyHex);
+// buttons do nothing at all on a real deployment.
-      setCopied(true);
+async function copyToClipboard(text: string, fallbackElementId: string): Promise<void> {
-      setTimeout(() => setCopied(false), 2000);
+  try {
-    } catch {
+    await navigator.clipboard.writeText(text);
-      // Clipboard API blocked (non-HTTPS context, etc.) — fall back to selection.
+  } catch {
-      const sel = window.getSelection();
+    const sel = window.getSelection();
+    const el = document.getElementById(fallbackElementId);
+    if (sel && el) {
       const range = document.createRange();
-      const el = document.getElementById('mm-privkey-box');
+      range.selectNodeContents(el);
-      if (sel && el) {
+      sel.removeAllRanges();
-        range.selectNodeContents(el);
+      sel.addRange(range);
-        sel.removeAllRanges();
-        sel.addRange(range);
-      }
     }
+  }
+}
+
+async function fetchAccounts(): Promise<Account[]> {
+  const res = await fetch('/api/identity/accounts');
+  if (!res.ok) throw new Error(await errorMessage(res, 'Failed to load accounts'));
+  const data = (await res.json()) as { accounts?: Account[] };
+  // Backend order is CreatedAt ascending then uid (identity.List) — stable, so
+  // render it as given rather than re-sorting here.
+  return data.accounts ?? [];
+}
+
+// Resolve an account's display name from the User Data Layer, or null.
+//
+// Never throws and never reports an error upward: a name is decoration on top
+// of the uid, and the account list must render identically whether Redis is up
+// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
+// — meta-core stores the plaintext beside the signed record and verifies
+// nothing — so the uid stays on screen as the real identity, and the length cap
+// bounds what a hostile writer can do to the layout.
+async function fetchDisplayName(uid: string): Promise<string | null> {
+  const url =
+    `/api/udl/record?uid=${encodeURIComponent(uid)}` +
+    `&cid=${encodeURIComponent(PROFILE_CID)}` +
+    `&key=${encodeURIComponent(PROFILE_NAME_KEY)}`;
+  try {
+    const res = await fetch(url);
+    if (!res.ok) return null; // 503 when storage is down, 500, …
+    const data = (await res.json()) as { exists?: boolean; value?: unknown };
+    // Key off `value`, not `exists`: clearing a name writes a tombstone, which
+    // leaves the cell existing with no plaintext. Private-tier cells look the
+    // same, which is exactly right — there is nothing to show for either.
+    if (typeof data.value !== 'string') return null;
+    const name = data.value.trim();
+    return name ? name.slice(0, 64) : null;
+  } catch {
+    return null;
+  }
+}
+
+// How much each account holds. One call for the whole page, not one per row:
+// meta-core answers it from a keyspace SCAN (there is no per-user cid index),
+// so per-row calls would walk the keyspace once per account.
+//
+// Never throws, for the same reason fetchDisplayName does not: the counts are
+// context on top of the uid, and the list must render identically when Redis is
+// down — accounts are file-backed, their data is not.
+//
+// Returns null on failure rather than an empty map. The difference is
+// load-bearing: an empty map renders as "no stored data" against every account,
+// which inside a delete confirmation is a false statement about what is at
+// stake. Unknown has to look like unknown.
+async function fetchStats(): Promise<Record<string, Stats> | null> {
+  try {
+    const res = await fetch('/api/udl/users/stats');
+    if (!res.ok) return null;
+    const data = (await res.json()) as { users?: Record<string, Stats> };
+    return data.users ?? {};
+  } catch {
+    return null;
+  }
+}
+
+// The secret-key box shared by the reveal and remove confirmations.
+//
+// Separate from the surrounding panel so both operations ask the same way, and
+// so the key lives in this component's state rather than the page's — it is
+// discarded with the panel instead of lingering in a parent that outlives it.
+function SecretKeyConfirm({
+  label,
+  actionLabel,
+  busy,
+  onConfirm,
+  onCancel,
+}: {
+  label: string;
+  actionLabel: string;
+  busy: boolean;
+  onConfirm: (secretKeyHex: string) => void;
+  onCancel: () => void;
+}) {
+  const [secret, setSecret] = useState('');
+  const signable = canSign();
+  // Derived locally, purely to tell the operator they have the wrong key before
+  // they hit the button. The server would refuse either way.
+  const derived = secret.trim() ? uidFromSecretKey(secret) : null;
+  const mismatch = Boolean(secret.trim() && derived && derived !== label);
+
+  return (
+    <form
+      onSubmit={(e) => {
+        e.preventDefault();
+        onConfirm(secret.trim());
+      }}
+    >
+      <p style={{ color: '#fecaca', margin: '0 0 0.5rem' }}>
+        Paste this account's secret key to prove it is yours. It is used to sign a one-time
+        challenge <em>in this browser</em> and is never sent to the server.
+      </p>
+      {!signable && <p style={errorTextStyle}>{INSECURE_CONTEXT_MESSAGE}</p>}
+      <input
+        style={inputStyle}
+        type="password"
+        autoComplete="off"
+        placeholder="64 hex characters"
+        value={secret}
+        onChange={(e) => setSecret(e.target.value)}
+        disabled={busy || !signable}
+        autoFocus
+      />
+      {mismatch && (
+        <p style={{ ...errorTextStyle, marginTop: 0 }}>
+          That key belongs to a different account ({derived}).
+        </p>
+      )}
+      <button
+        type="submit"
+        style={dangerButtonStyle}
+        disabled={busy || !signable || !secret.trim() || mismatch}
+      >
+        {busy ? 'Verifying…' : actionLabel}
+      </button>
+      <button type="button" style={buttonStyle} onClick={onCancel} disabled={busy}>
+        Cancel
+      </button>
+    </form>
+  );
+}
+
+// The one place a private key is ever displayed. Rendered exclusively (the
+// account list is hidden while it is open) so the "save it now" contract of a
+// freshly generated key is not competing with anything, and so the hardcoded
+// element id below stays unique.
+function SecretKeyPanel({ data, onDismiss }: { data: SecretKeyView; onDismiss: () => void }) {
+  const [copied, setCopied] = useState(false);
+
+  const copy = async () => {
+    await copyToClipboard(data.privateKeyHex, 'mm-privkey-box');
+    setCopied(true);
+    setTimeout(() => setCopied(false), 2000);
   };
+
   return (
     <div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
       <div
@@ -109,9 +317,19 @@ function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss:
           marginBottom: '1rem',
         }}
       >
-        <strong>This private key is shown ONCE.</strong> Save it now in a password manager.
+        {data.kind === 'generated' ? (
-        It will never be displayed again — there is no recovery path. Losing it means losing access
+          <>
-        to every record signed under this identity.
+            <strong>This private key is shown ONCE.</strong> Save it now in a password manager.
+            You can ask for it again later with <em>Reveal private key</em>, but only for as long
+            as this account exists on this node — delete it and the key is gone for good.
+          </>
+        ) : (
+          <>
+            <strong>This is the live private key for an existing account.</strong> Anyone holding
+            it can sign as this uid forever, on any machine. There is no revocation and no
+            rotation. Do not paste it into chat, tickets or screenshots.
+          </>
+        )}
       </div>
       <p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
       <div style={monoStyle}>{data.uid}</div>
@@ -124,36 +342,40 @@ function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss:
           {copied ? 'Copied' : 'Copy private key'}
         </button>
         <button style={dangerButtonStyle} onClick={onDismiss}>
-          I have saved it
+          {data.kind === 'generated' ? 'I have saved it' : 'Done — hide it'}
         </button>
       </div>
     </div>
   );
 }
 
-function NoIdentityPanel({
+// Create / import. Always available, not just on an empty keystore: this is an
+// admin console, so adding an account to a populated node has to be possible.
+function NewAccountPanel({
   onGenerated,
   onImported,
 }: {
-  onGenerated: (g: GeneratedKey) => void;
+  onGenerated: (s: SecretKeyView) => void;
   onImported: () => void;
 }) {
   const [busy, setBusy] = useState(false);
   const [importing, setImporting] = useState(false);
   const [importHex, setImportHex] = useState('');
   const [error, setError] = useState<string | null>(null);
+  const [notice, setNotice] = useState<string | null>(null);
 
   const generate = async () => {
     setBusy(true);
     setError(null);
+    setNotice(null);
     try {
       const res = await fetch('/api/identity/generate', { method: 'POST' });
       if (!res.ok) {
         setError(await errorMessage(res, 'Generate failed'));
         return;
       }
-      const data = (await res.json()) as GeneratedKey;
+      const data = (await res.json()) as { uid: string; privateKeyHex: string };
-      onGenerated(data);
+      onGenerated({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'generated' });
     } catch (e) {
       setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
     } finally {
@@ -165,6 +387,7 @@ function NoIdentityPanel({
     e.preventDefault();
     setBusy(true);
     setError(null);
+    setNotice(null);
     try {
       const res = await fetch('/api/identity/import', {
         method: 'POST',
@@ -175,6 +398,15 @@ function NoIdentityPanel({
         setError(await errorMessage(res, 'Import failed'));
         return;
       }
+      // Import is idempotent — presenting the key IS the proof of ownership, so
+      // a key this node already holds is a success, not a conflict. `created`
+      // only decides the wording.
+      const data = (await res.json()) as { created?: boolean };
+      setNotice(
+        data.created
+          ? 'Imported — a new account was added to this keystore.'
+          : 'That key was already in this keystore; nothing changed.'
+      );
       setImportHex('');
       setImporting(false);
       onImported();
@@ -187,15 +419,16 @@ function NoIdentityPanel({
 
   return (
     <div style={cardStyle}>
-      <h3 style={{ marginTop: 0 }}>No identity yet</h3>
+      <h3 style={{ marginTop: 0 }}>Add an account</h3>
-      <p style={{ color: '#9ca3af' }}>
+      <p style={{ color: '#9ca3af', marginTop: 0 }}>
-        The User Data Layer needs a signing key for your ratings, watch progress, and "liked" flags
+        Generating mints a fresh secp256k1 keypair here. Importing adds a key created elsewhere —
-        to be attributed to you across devices. Nothing happens automatically — generate one below,
+        it is also how an existing account is moved onto this node, so importing a key the
-        or import a key you already have.
+        keystore already holds succeeds and changes nothing.
       </p>
-      {error && (
+      {error && <p style={errorTextStyle}>{error}</p>}
-        <p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
+      {notice && (
-          {error}
+        <p style={{ color: '#9ca3af', background: '#0a0a18', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
+          {notice}
         </p>
       )}
       <div>
@@ -235,73 +468,156 @@ function NoIdentityPanel({
   );
 }
 
-function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onReplace: () => void }) {
+function AccountCard({
-  const [confirming, setConfirming] = useState(false);
+  account,
-  const [busy, setBusy] = useState(false);
+  name,
-  const [error, setError] = useState<string | null>(null);
+  stats,
+  busy,
+  disabled,
+  pending,
+  error,
+  onAsk,
+  onCancel,
+  onReveal,
+  onRemove,
+}: {
+  account: Account;
+  name?: string;
+  // undefined means "counted, and it holds nothing"; null means "could not be
+  // counted". The two must not read the same on a confirmation screen.
+  stats?: Stats | null;
+  busy: boolean;
+  disabled: boolean;
+  pending: Pending | null;
+  error: string | null;
+  onAsk: (p: Pending) => void;
+  onCancel: () => void;
+  onReveal: (uid: string, secretKeyHex: string) => void;
+  onRemove: (uid: string, secretKeyHex: string) => void;
+}) {
+  const [copied, setCopied] = useState(false);
+  const uidBoxId = `mm-uid-${account.uid}`;
+  const label = name ?? account.uid;
+  const counted = stats !== null;
+  const records = stats?.records ?? 0;
+  const titles = stats?.cids ?? 0;
 
-  const remove = async () => {
+  const copyUid = async () => {
-    setBusy(true);
+    await copyToClipboard(account.uid, uidBoxId);
-    setError(null);
+    setCopied(true);
-    try {
+    setTimeout(() => setCopied(false), 2000);
-      const res = await fetch('/api/identity?confirm=true', { method: 'DELETE' });
-      if (!res.ok && res.status !== 204) {
-        setError(await errorMessage(res, 'Delete failed'));
-        return;
-      }
-      onReplace();
-    } catch (e) {
-      setError(`Delete failed: ${e instanceof Error ? e.message : String(e)}`);
-    } finally {
-      setBusy(false);
-      setConfirming(false);
-    }
   };
 
   return (
     <div style={cardStyle}>
-      <h3 style={{ marginTop: 0 }}>Active identity</h3>
+      <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start', gap: '1rem' }}>
-      <p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid:</p>
+        <div style={{ minWidth: 0, flex: 1 }}>
-      <div style={monoStyle}>{status.uid}</div>
+          <h3 style={{ margin: '0 0 0.5rem', color: name ? '#fff' : '#888' }}>
-      <p style={{ color: '#888', marginTop: '1rem' }}>
+            {name ?? 'Unnamed account'}
-        Curve: {status.curve} · Created: {formatCreatedAt(status.createdAt)}
+          </h3>
-      </p>
+          <div id={uidBoxId} style={monoStyle}>
-      <p style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
+            {account.uid}
-        The private key lives at <code>/meta-core/identity/identity.json</code> on this volume and
+          </div>
-        is never returned through the API after generation. Other services on this stack (e.g.
+          <p style={{ color: '#888', margin: '0.75rem 0 0', fontSize: '0.85rem' }}>
-        meta-watch) sign records by calling <code>POST /api/identity/sign</code>.
+            Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}
-      </p>
-      {error && (
-        <p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
-          {error}
-        </p>
-      )}
-      {!confirming ? (
-        <button style={dangerButtonStyle} onClick={() => setConfirming(true)} disabled={busy}>
-          Replace identity
-        </button>
-      ) : (
-        <div
-          style={{
-            background: '#1f0a0a',
-            border: '1px solid #7f1d1d',
-            padding: '0.75rem 1rem',
-            borderRadius: '4px',
-            marginTop: '0.5rem',
-          }}
-        >
-          <p style={{ color: '#fecaca', marginTop: 0 }}>
-            Replacing the identity deletes the current keypair from disk. Records signed by the
-            current uid will keep verifying for anyone who has them cached, but this node will no
-            longer be able to write under that uid. <strong>Make sure you have backed up the key
-            first.</strong>
           </p>
-          <button style={dangerButtonStyle} onClick={remove} disabled={busy}>
+          <p style={{ color: '#888', margin: '0.25rem 0 0', fontSize: '0.85rem' }}>
-            {busy ? 'Deleting…' : 'Yes, delete identity'}
+            {/* An account that has never been used legitimately holds nothing, so
+                zero is a real answer — but only once the count actually ran. */}
+            {!counted
+              ? 'Stored data: could not be counted (storage unavailable)'
+              : records === 0
+                ? 'No stored data'
+                : `${records.toLocaleString()} stored ${records === 1 ? 'record' : 'records'}` +
+                  ` across ${titles.toLocaleString()} ${titles === 1 ? 'title' : 'titles'}`}
+          </p>
+        </div>
+        <div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem', flexShrink: 0 }}>
+          <button style={{ ...buttonStyle, marginRight: 0 }} onClick={copyUid} disabled={disabled}>
+            {copied ? 'Copied' : 'Copy uid'}
           </button>
-          <button style={buttonStyle} onClick={() => setConfirming(false)} disabled={busy}>
+          <button
-            Cancel
+            style={{ ...dangerButtonStyle, marginRight: 0 }}
+            onClick={() => onAsk({ uid: account.uid, kind: 'reveal' })}
+            disabled={disabled}
+          >
+            Reveal private key
           </button>
+          <button
+            style={{ ...dangerButtonStyle, marginRight: 0 }}
+            onClick={() => onAsk({ uid: account.uid, kind: 'remove' })}
+            disabled={disabled}
+          >
+            Remove
+          </button>
+        </div>
+      </div>
+
+      {error && <p style={{ ...errorTextStyle, marginBottom: 0 }}>{error}</p>}
+
+      {pending?.uid === account.uid && pending.kind === 'reveal' && (
+        <div style={dangerPanelStyle}>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            Show the private key for <strong>{label}</strong>?
+          </p>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            The key <em>is</em> the account. Anyone who copies it can sign records as this uid on
+            any machine, forever — there is no way to revoke or rotate it afterwards. Because
+            proving ownership already requires the key, this only confirms that the copy this node
+            holds matches yours.
+          </p>
+          <SecretKeyConfirm
+            label={account.uid}
+            actionLabel="Show the private key"
+            busy={busy}
+            onConfirm={(secret) => onReveal(account.uid, secret)}
+            onCancel={onCancel}
+          />
+        </div>
+      )}
+
+      {pending?.uid === account.uid && pending.kind === 'remove' && (
+        <div style={dangerPanelStyle}>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            Remove <strong>{label}</strong> from this node?
+          </p>
+          <ul style={{ color: '#fecaca', paddingLeft: '1.2rem', margin: '0 0 0.75rem' }}>
+            <li>
+              <strong>The key is gone for good.</strong> The uid <em>is</em> the public key, and
+              the private half exists only in this node's keystore. If nobody saved it, nothing —
+              not you, not meta-core — can ever sign as this uid again. Reveal and copy it first
+              if there is any doubt.
+            </li>
+            <li>
+              <strong>Nothing is retracted.</strong> Records already signed under this uid keep
+              verifying on every peer that replicated them. Deleting a key is not a recall.
+            </li>
+            <li>
+              <strong>Its data is destroyed, not orphaned.</strong>{' '}
+              {!counted
+                ? 'This account\u2019s records could not be counted because storage is unavailable — ' +
+                  'so meta-core will refuse this delete rather than remove the key and strand them.'
+                : records === 0
+                  ? 'This account holds no User Data Layer records, so there is nothing to purge.'
+                  : `All ${records.toLocaleString()} User Data Layer ${
+                      records === 1 ? 'record' : 'records'
+                    } — likes, ratings, watch progress across ${titles.toLocaleString()} ${
+                      titles === 1 ? 'title' : 'titles'
+                    } — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
+              This cannot be undone and there is no backup.
+            </li>
+            <li>
+              <strong>This signs nobody out elsewhere.</strong> Any device that holds this key
+              still holds it, and can import it back here at any time.
+            </li>
+          </ul>
+          <SecretKeyConfirm
+            label={account.uid}
+            actionLabel="Remove this account and all its data"
+            busy={busy}
+            onConfirm={(secret) => onRemove(account.uid, secret)}
+            onCancel={onCancel}
+          />
         </div>
       )}
     </div>
@@ -309,67 +625,235 @@ function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onRep
 }
 
 export default function Identity() {
-  const [status, setStatus] = useState<IdentityStatus | null>(null);
+  const [accounts, setAccounts] = useState<Account[] | null>(null);
-  const [generated, setGenerated] = useState<GeneratedKey | null>(null);
+  const [names, setNames] = useState<Record<string, string>>({});
+  const [stats, setStats] = useState<Record<string, Stats> | null>({});
+  const [notice, setNotice] = useState<string | null>(null);
   const [loading, setLoading] = useState(true);
   const [fetchError, setFetchError] = useState<string | null>(null);
+  const [secret, setSecret] = useState<SecretKeyView | null>(null);
+  const [pending, setPending] = useState<Pending | null>(null);
+  const [busyUid, setBusyUid] = useState<string | null>(null);
+  const [rowError, setRowError] = useState<{ uid: string; message: string } | null>(null);
+
+  // Bumped per refresh so a slow response from a superseded load cannot
+  // overwrite a newer one. The page does not poll (unlike the rest of the
+  // dashboard) — a key list has no business changing under the cursor.
+  const reqGen = useRef(0);
 
   const refresh = async () => {
+    const gen = ++reqGen.current;
     setLoading(true);
     setFetchError(null);
+    let list: Account[];
     try {
-      const res = await fetch('/api/identity');
+      list = await fetchAccounts();
-      if (!res.ok) {
-        setFetchError(await errorMessage(res, 'Failed to load identity status'));
-        return;
-      }
-      setStatus((await res.json()) as IdentityStatus);
     } catch (e) {
-      setFetchError(`Failed to load identity status: ${e instanceof Error ? e.message : String(e)}`);
+      if (gen !== reqGen.current) return;
-    } finally {
+      setFetchError(e instanceof Error ? e.message : String(e));
       setLoading(false);
+      return;
     }
+    if (gen !== reqGen.current) return;
+    // Render the list before resolving names, so a down Redis delays nothing.
+    setAccounts(list);
+    setLoading(false);
+
+    // Names and counts both come after the list is on screen, and neither can
+    // fail the page: a down Redis costs decoration, not the account list.
+    const [settled, counts] = await Promise.all([
+      Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
+      fetchStats(),
+    ]);
+    if (gen !== reqGen.current) return;
+    const next: Record<string, string> = {};
+    settled.forEach((r, i) => {
+      if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value;
+    });
+    // Replace rather than merge, so a removed account's name and counts go with
+    // it instead of lingering against a uid that no longer exists.
+    setNames(next);
+    setStats(counts);
   };
 
   useEffect(() => {
     refresh();
   }, []);
 
+  const reveal = async (uid: string, secretKeyHex: string) => {
+    setBusyUid(uid);
+    setRowError(null);
+    try {
+      const proof = await proveOwnership(uid, ACTION_FOR.reveal, secretKeyHex);
+      // POST with a body rather than ?uid=, so neither the target nor the
+      // signature reaches the proxy's access log. The handler accepts the uid
+      // either way; the proof must be in the body.
+      const res = await fetch('/api/identity/reveal', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ confirm: true, uid, ...proof }),
+      });
+      if (!res.ok) {
+        setRowError({ uid, message: await errorMessage(res, 'Reveal failed') });
+        return;
+      }
+      const data = (await res.json()) as { uid: string; privateKeyHex: string };
+      setPending(null);
+      setSecret({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'revealed' });
+    } catch (e) {
+      setRowError({ uid, message: `Reveal failed: ${e instanceof Error ? e.message : String(e)}` });
+    } finally {
+      setBusyUid(null);
+    }
+  };
+
+  const remove = async (uid: string, secretKeyHex: string) => {
+    setBusyUid(uid);
+    setRowError(null);
+    setNotice(null);
+    try {
+      const proof = await proveOwnership(uid, ACTION_FOR.remove, secretKeyHex);
+      // The uid is mandatory: the signature authorises one specific account, so
+      // the backend refuses an unqualified delete rather than guess a target.
+      const res = await fetch('/api/identity', {
+        method: 'DELETE',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ confirm: true, uid, ...proof }),
+      });
+      if (!res.ok) {
+        setRowError({ uid, message: await errorMessage(res, 'Remove failed') });
+        return;
+      }
+      // Report what the purge actually removed. "Account removed" on its own
+      // would not distinguish a clean delete from one that stranded the data.
+      const data = (await res.json().catch(() => ({}))) as { purged?: Stats };
+      const purged = data.purged;
+      setNotice(
+        purged && purged.records > 0
+          ? `Account removed. Purged ${purged.records.toLocaleString()} User Data Layer ` +
+            `${purged.records === 1 ? 'record' : 'records'} across ` +
+            `${purged.cids.toLocaleString()} ${purged.cids === 1 ? 'title' : 'titles'}.`
+          : 'Account removed. It held no User Data Layer records.'
+      );
+      setPending(null);
+      await refresh();
+    } catch (e) {
+      setRowError({ uid, message: `Remove failed: ${e instanceof Error ? e.message : String(e)}` });
+    } finally {
+      setBusyUid(null);
+    }
+  };
+
+  const dismissSecret = () => {
+    setSecret(null);
+    refresh();
+  };
+
+  const ask = (p: Pending) => {
+    setRowError(null);
+    setNotice(null);
+    setPending(p);
+  };
+
+  const showList = !loading && !fetchError && !secret;
+
   return (
+    // App.tsx's content area is `overflow: hidden`, so every page owns its own
+    // scrolling (Files.tsx does the same). Without this the account list is
+    // simply clipped at the viewport with no scrollbar — invisible on a node
+    // with two accounts, and a dead end on one with fifteen.
+    //
+    // The scroller is the outer element and the padding lives inside it, so the
+    // scrollbar sits at the edge of the content area rather than floating in
+    // 2rem of whitespace, and `margin: 0 auto` still centres the column.
+    <div style={{ height: '100%', overflowY: 'auto', minHeight: 0 }}>
     <div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
-      <h1>Identity</h1>
+      <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
+        <h1 style={{ marginBottom: 0 }}>Identity</h1>
+        <button style={{ ...buttonStyle, marginRight: 0 }} onClick={refresh} disabled={loading}>
+          {loading ? 'Loading…' : 'Refresh'}
+        </button>
+      </div>
       <p style={{ color: '#9ca3af' }}>
-        Signing key for the User Data Layer (per-user ratings, progress, "liked" flags). One
+        Every signing key in this node's keystore. meta-core has no "current user" — it stores
-        identity per node. The same key on another device makes that device count as "you".
+        accounts, and which one is <em>you</em> is decided by whichever client signs in (e.g.
+        meta-watch). Keys live one JSON file per account under{' '}
+        <code>{'{META_CORE_PATH}'}/identity/accounts/&lt;uid&gt;.json</code>, and are used to sign
+        User Data Layer records (ratings, watch progress, "liked" flags).
       </p>
 
+      <div style={{ ...dangerPanelStyle, marginTop: 0, marginBottom: '1rem' }}>
+        <p style={{ color: '#fecaca', margin: 0 }}>
+          <strong>The key is the only authority.</strong> Revealing or removing an account requires
+          that account's secret key, entered here and used to sign a one-time challenge in your
+          browser — there is no admin override, and holding this page is not one. Listing and
+          creating accounts are still open to anything that can reach this API, so the perimeter in
+          front of this port is what keeps the roster private.
+        </p>
+        <p style={{ color: '#fecaca', margin: '0.5rem 0 0' }}>
+          What this does <em>not</em> defend against: anyone with filesystem access to{' '}
+          <code>{'{META_CORE_PATH}'}/identity/accounts/</code> reads every private key directly.
+          The keys are stored in plaintext, matching the trust model of the rest of{' '}
+          <code>/meta-core</code>.
+        </p>
+      </div>
+
+      {notice && (
+        <div style={{ ...cardStyle, border: '1px solid #0f3460' }}>
+          <p style={{ color: '#9ca3af', margin: 0 }}>{notice}</p>
+        </div>
+      )}
+
       {loading && <div style={cardStyle}>Loading…</div>}
+
       {fetchError && (
         <div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
-          <p style={{ color: '#f87171', margin: 0 }}>{fetchError}</p>
+          <p style={{ color: '#f87171', margin: '0 0 1rem' }}>{fetchError}</p>
           <button style={buttonStyle} onClick={refresh}>
             Retry
           </button>
         </div>
       )}
 
-      {!loading && !fetchError && generated && (
+      {!loading && !fetchError && secret && <SecretKeyPanel data={secret} onDismiss={dismissSecret} />}
-        <GeneratedKeyPanel
-          data={generated}
-          onDismiss={() => {
-            setGenerated(null);
-            refresh();
-          }}
-        />
-      )}
 
-      {!loading && !fetchError && !generated && status && !status.hasIdentity && (
+      {showList && (
-        <NoIdentityPanel onGenerated={(g) => setGenerated(g)} onImported={refresh} />
+        <>
-      )}
+          <NewAccountPanel onGenerated={(s) => setSecret(s)} onImported={refresh} />
 
-      {!loading && !fetchError && !generated && status && status.hasIdentity && (
+          <h2 style={{ fontSize: '1.1rem', color: '#9ca3af', margin: '1.5rem 0 1rem' }}>
-        <HasIdentityPanel status={status} onReplace={refresh} />
+            Accounts ({accounts?.length ?? 0})
+          </h2>
+
+          {accounts && accounts.length === 0 && (
+            <div style={cardStyle}>
+              <p style={{ color: '#9ca3af', margin: 0 }}>
+                No accounts in this keystore. Accounts are normally created by a client when
+                someone signs up; use <em>Add an account</em> above if you are bootstrapping this
+                node.
+              </p>
+            </div>
+          )}
+
+          {accounts?.map((a) => (
+            <AccountCard
+              key={a.uid}
+              account={a}
+              name={names[a.uid]}
+              stats={stats === null ? null : stats[a.uid]}
+              busy={busyUid === a.uid}
+              disabled={busyUid !== null}
+              pending={pending}
+              error={rowError?.uid === a.uid ? rowError.message : null}
+              onAsk={ask}
+              onCancel={() => setPending(null)}
+              onReveal={reveal}
+              onRemove={remove}
+            />
+          ))}
+        </>
       )}
     </div>
+    </div>
   );
 }

docker/entrypoint.sh

@@ -10,7 +10,6 @@ set -e
 # Create required directories
 mkdir -p /meta-core/locks
 mkdir -p /meta-core/db/redis
-mkdir -p /meta-core/services
 mkdir -p /meta-core/mounts/errors
 mkdir -p /files
 mkdir -p /var/log/supervisor

docker/leader-election.sh

@@ -2,9 +2,13 @@
 # =============================================================================
 # Leader Election Script for meta-core
 #
-# Uses flock for leader election. Only the leader runs the full application
+# The flock here is a MUTEX, not service discovery. It is the only thing
-# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
+# stopping two containers that mount the same /meta-core volume from putting
-# Followers remain dormant, periodically retrying to acquire the lock.
+# two Redis writers on one RDB/AOF. Sibling services no longer read anything
+# from this directory — they locate meta-core over UDP (meta-discovery v1, see
+# docs/project-architecture/service-discovery.md), so the kv-leader.info file
+# this script used to publish is gone. Keep the lock; it costs nothing and the
+# corruption it prevents is unrecoverable.
 #
 # Environment variables:
 #   META_CORE_PATH         - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +24,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
 ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
 
 LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
-INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
 SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
 
 # Get the container's IP address
@@ -40,35 +43,13 @@ get_local_ip() {
     echo "$ip"
 }
 
-# Write leader info file (plain text API URL)
-write_leader_info() {
-    local ip=$(get_local_ip)
-    local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
-
-    # Write atomically using temp file + rename
-    local temp_file="${INFO_FILE}.tmp"
-    echo "$api_url" > "$temp_file"
-    mv "$temp_file" "$INFO_FILE"
-
-    echo "[election] Wrote leader info: $api_url"
-}
-
-# Clean up leader info file on shutdown
-cleanup() {
-    echo "[election] Cleaning up..."
-    rm -f "$INFO_FILE"
-    exit 0
-}
-
 # Set up signal handlers for graceful shutdown
-trap cleanup SIGTERM SIGINT
 
 # Ensure lock directory exists
 mkdir -p "$(dirname "$LOCK_FILE")"
 
 echo "[election] Starting leader election..."
 echo "[election] Lock file: $LOCK_FILE"
-echo "[election] Info file: $INFO_FILE"
 
 # Open lock file for flock (file descriptor 200)
 exec 200>"$LOCK_FILE"
@@ -79,9 +60,6 @@ while true; do
     if flock -n 200; then
         echo "[election] Acquired flock - transitioning to LEADER"
 
-        # Write leader info before starting services
-        write_leader_info
-
         # exec supervisord - replaces this process
         # When supervisord exits, the flock is released automatically
         echo "[election] Starting supervisord..."

docker/nginx.conf

@@ -123,14 +123,10 @@ http {
         # =========================================================================
         # Service discovery
         # =========================================================================
-        location /services {
+        # meta-discovery v1 neighbours (supersedes /services, which stays
-            proxy_pass http://meta-core-api/services;
+        # while consumers migrate).
-            proxy_http_version 1.1;
+        location /neighbors {
-            proxy_set_header Host $host;
+            proxy_pass http://meta-core-api/neighbors;
-        }
-
-        location /services/ {
-            proxy_pass http://meta-core-api/services/;
             proxy_http_version 1.1;
             proxy_set_header Host $host;
         }

docs/api-mediated-access.md

@@ -18,7 +18,7 @@ out to be load-bearing).
 
 meta-core was designed as the sidecar that mediates every metadata read and
 write — a single point that owns the Redis schema, enforces invariants
-(reverse index, canonical_cid reconciliation, schema-version sentinel),
+(reverse index, `cids/` key-set integrity, schema-version sentinel),
 and presents a stable HTTP contract. Other services were meant to talk to
 meta-core, not to Redis.
 
@@ -392,7 +392,7 @@ specifically with the Kubernetes flavour where streams are mediated too.
   coordinating releases across every other service. The UUID-root
   migration is the cautionary tale of what happens without this.
 - **Validation choke point.** Every write passes through one code path
-  that can enforce invariants (reverse index, canonical_cid, type
+  that can enforce invariants (reverse index, `cids/` key-set, type
   constraints, schema version).
 - **Observability.** Audit logs, request metrics, slow-query detection
   all live at one layer.
@@ -697,11 +697,11 @@ contract violation."
 **The fix.** Three layers in `internal/storage/cid_resolution.go`:
 
 1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
-   read/write handler, look up `cid:midhash256:<hash>` in the reverse
+   read/write handler, look up `cid:<hash>` in the reverse index (bare
-   index. If found, use the UUID as the storage root. Falls through to
+   CID — no algorithm prefix). If found, use the UUID as the storage root.
-   the bare hash for direct-UUID writes / legacy entries.
+   Falls through to the bare hash for direct-UUID writes / legacy entries.
 2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
-   register `cid:<algo>:<v>` → `<v>`. The dual-root bug's chicken-and-
+   register `cid:<v>` → `<v>`. The dual-root bug's chicken-and-
    egg is: meta-sort writes first, registers self-pointing alias,
    watcher then can't recover. Refusing the self-write keeps the
    watcher's legitimate alias intact.

docs/meta-core-architecture.md

@@ -221,12 +221,13 @@ operators wipe with `pnpm run clean:all` and restart.
 │  Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars)     │
 │  Storage: flat string keys                                         │
 │           file:<uuid>/<property>           → string value          │
-│           file:<uuid>/cids                 → SET of CID tokens     │
+│           file:<uuid>/cids/<bareCid>       → "true" (key-set)      │
-│           file:<uuid>/canonical_cid        → preferred CID         │
 │           file:<uuid>/duplicates           → SET of paths          │
 │                                                                    │
+│  No stored canonical_cid — derived by rank on read.                │
+│                                                                    │
 │  Aliases (reverse index, one Redis STRING each):                   │
-│           cid:<algorithm>:<value>          → <uuid>                │
+│           cid:<bareCid>                    → <uuid>                │
 │                                                                    │
 │  Index:   file:__index__                   → SET of all UUIDs      │
 │                                                                    │
@@ -275,9 +276,8 @@ Bootstrap / discovery
   GET  /status
   GET  /leader
   GET  /urls
-  GET  /services         (alias: /api/services)
+  GET  /neighbors        (alias: /api/neighbors)
-  GET  /services/{name}
+  GET  /api/stats
-  GET  /services/cleanup/stats
 
 Metadata (primary surface for other services)
   GET    /meta
@@ -300,6 +300,7 @@ CID-addressed (public, auth-bypassed at the perimeter)
 
 Editor / KV / schema / snapshot
   GET /api/metadata/hash-ids
+  GET /api/files/tuples
   GET /api/metadata/list
   POST /api/metadata/search
   POST /api/metadata/batch
@@ -338,6 +339,9 @@ Watchers
 
 Admin
   POST /api/admin/migrate-dual-roots
+  POST /api/admin/sweep-literature-echoes   (dry-run unless ?apply=true;
+       strips query echoes off domain=literature fileType=card records —
+       see storage/literature_echo_sweep.go)
 
 WebDAV
   /webdav/...   (GET, PUT, DELETE, MKCOL, COPY, MOVE, PROPFIND)
@@ -515,7 +519,7 @@ via `/health`; metadata I/O and SSE event streams stop.
 | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). |
 | `REDIS_PORT` | `6379` | Local Redis port. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
-| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind address. |
+| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind address. |
 | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
 | `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
 | `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is stale. |

docs/metadata-storage-structure.md

@@ -19,10 +19,13 @@ For the external-access contract (HTTP-only, SSE for events), see
   `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
 - **Properties are flat Redis STRING keys**, one per leaf:
   `file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
-- **CIDs are reverse-index aliases**: `cid:<algorithm>:<value>` → `<uuid>`
+- **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
-  (a Redis STRING per known CID).
+  (a Redis STRING per known CID; no algorithm prefix — the multicodec
-- **Per-root sets**: `file:<uuid>/cids` (every CID), `file:<uuid>/duplicates`
+  already carries it).
+- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
+  `"true"` per known CID, midhash included), `file:<uuid>/duplicates`
   (alternate paths with the same content), plus `file:__index__` (every UUID).
+  There is **no stored `canonical_cid`** — it is derived by rank on read.
 - **A schema-version sentinel** (`meta-core:schema-version`) refuses to
   boot against legacy data.
 
@@ -59,37 +62,54 @@ file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster                       = "bafkreih...
 
 ### Reverse-index aliases (CIDs)
 
-Every CID known for a file is registered as its own Redis STRING:
+Every CID known for a file is registered as its own Redis STRING, keyed by
+the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
+function in the multicodec, so naming it again in the key would be
+redundant (and would let the same digest be indexed under two spellings):
 
 ```
-cid:<algorithm>:<value>   →   <uuid>
+cid:<bareCid>   →   <uuid>
 ```
 
 Examples:
 
 ```
-cid:midhash256:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:sha256:abc...                    →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bagacbabaec7v3...     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:ipfs:bafy...                     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
 ```
 
 The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
-registered CID, independent of which hash algorithm "primary" — there is
+registered CID, independent of which hash algorithm — there is no primary.
-no primary. The auto-alias hooks in `SetProperty` and `SetMetadataFlat`
+The auto-alias hook (`maybeAddAliasFromFieldLocked` in
-(see `internal/storage/cid_resolution.go`) catch any write of a `cid_*`
+`internal/storage/cid_resolution.go`, called from `SetProperty`,
-or `midhash256` field and register the alias automatically, so plugins
+`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
-don't need to know about the reverse index.
+field whose name starts with `cids/`, so writers emit key-set members
+without knowing the reverse index exists.
+
+> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
+> `midhash256` *named* field written to a root is stored verbatim and is
+> **never indexed** — the record becomes unresolvable by that CID. Writers
+> must collapse such fields to `cids/<cid>` members before the write. See
+> `METADATA_KEYS.md` §14.13.
 
 ### Per-root sets
 
 ```
-file:<uuid>/cids              →   Redis SET of "<algorithm>:<value>" tokens
+file:<uuid>/cids/<bareCid>    →   STRING "true" — one key per known CID (a key-set, not a Redis SET)
-file:<uuid>/canonical_cid     →   STRING — the highest-ranked CID for advertising externally
 file:<uuid>/duplicates        →   Redis SET of alternate paths with same content
 ```
 
-`canonical_cid` is reconciled by `reconcileCanonicalCIDLocked` whenever
+`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
-the `cids` set changes; ranking is done by `internal/cid/rank.go`.
+which keeps it conflict-free to merge across peers (a SET would not be).
+The record's own address (the midhash) is just the `0x1000` member — it has
+no separate field.
+
+There is **no stored `canonical_cid`**. The canonical CID is *derived on
+read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the
+key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns
+the winner as the `canonical_cid` field of the JSON response only. Nothing
+reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and
+was removed with the flat `cid_*` keys.
 
 ### Global index
 
@@ -141,14 +161,15 @@ The Go API in `internal/storage/client.go`:
 | Method | Behaviour |
 |---|---|
 | `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
-| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register any `cid_*` aliases. |
+| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
 | `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
 | `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
 | `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
 | `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
 | `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
-| `AddAlias(uuid, "<algo>:<value>")` | Set `cid:<algo>:<value>` → `<uuid>`, add to `file:<uuid>/cids`, reconcile `canonical_cid`. |
+| `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
-| `ResolveCID("<algo>:<value>")` | `GET cid:<algo>:<value>` → uuid. |
+| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
+| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
 
 ## Reconstruction (flat → nested)
 
@@ -237,10 +258,14 @@ docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo
 docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
 # string
 
-# Reverse-index lookups
+# Reverse-index lookup (bare CID — no algorithm prefix)
-docker exec metacore-app redis-cli GET 'cid:midhash256:bafkreih...'      # → uuid
+docker exec metacore-app redis-cli GET 'cid:bafkreih...'      # → uuid
-docker exec metacore-app redis-cli SMEMBERS 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids'
+
-docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid'
+# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
+docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
+
+# canonical_cid is NOT stored. Derive it via the API instead:
+curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
 
 # Schema-version sentinel
 docker exec metacore-app redis-cli GET 'meta-core:schema-version'

docs/uuid-rooted-metadata.md

@@ -4,6 +4,25 @@
 
 **Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
 
+> ⚠ **Two parts of this proposal were superseded before shipping.** This doc
+> is the design record; it is *not* the schema reference. For what is actually
+> stored, see `METADATA_KEYS.md` §2/§14.13 and
+> [metadata-storage-structure.md](./metadata-storage-structure.md).
+>
+> 1. **`canonical_cid` is not a stored key.** The proposed reconciler
+>    (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't
+>    conflict-free to merge across peers. The canonical CID is **derived on
+>    read** by ranking the `cids` members (`internal/cid/rank.go` →
+>    `cid.Better`), and appears only in the `GET /api/meta/{cid}` response.
+> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not
+>    `<algo>:<value>` tokens.** One flat STRING per member:
+>    `file:<uuid>/cids/<bareCid>` = `"true"`. The reverse index is likewise
+>    `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
+>    function, so the `<algo>:` prefix was redundant.
+>
+> Text below that references either shape describes the *proposal*, not the
+> code. The rank ordering and everything else did ship as written.
+
 ## Motivation
 
 Today, every metadata entry in Redis is keyed by a content hash, specifically
@@ -79,19 +98,21 @@ today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath          = /watch/Inception.mkv
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size              = 4500000000
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime             = 1729872000000000000
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids              = SET{midhash256:abc, sha256:def, ipfs:bafy...}
+file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec…  = "true"    ← midhash (record address)
+file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd…    = "true"    ← sha2-256
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates        = SET{/watch/dup1.mkv, /watch/dup2.mkv}
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid     = ipfs:bafy...
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title             = "Inception"
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster       = bafy...
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng      = ...
 
-cid:midhash256:abc  → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bagacbabaec…  → 01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:sha256:def      → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bafybeigd…    → 01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:ipfs:bafy...    → 01JKR8XW5T4QMV7AHNJ2DEFGHK
 ```
 
+*(As shipped — the proposal originally wrote `cids` as a Redis SET of
+`<algo>:<value>` tokens plus a `canonical_cid` scalar. See the Status note.)*
+
 ### UUID format: UUIDv7
 
 UUIDv7 chosen over UUIDv4 or 64-bit random:
@@ -111,26 +132,33 @@ The metadata schema is open — any writer can put any key under
 `file:<uuid>/`. This proposal does not change that. It adds three keys whose
 sole writer is meta-core itself:
 
-| Key | Type | Purpose |
+| Key | Type (as shipped) | Purpose |
 |-----|------|---------|
-| `cids` | Redis SET | All CIDs known for this file. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. |
+| `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
 | `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
-| `canonical_cid` | STRING | The CID this entry should be presented as externally (meta-share announce, UI display). Selected by the rank function (below). |
+| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
 
 Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
 keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
 `subtitle/<lang>`, …) are unaffected.
 
-Redis SET is a departure from the all-strings convention in the current
+~~Redis SET is a departure from the all-strings convention in the current
 storage doc, but `file:__index__` is already a SET — so the precedent exists
-for using sets where they're the right structure.
+for using sets where they're the right structure.~~ **Reversed for `cids`.**
+The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
+so that two peers merging the same record can never disagree about set
+membership. `duplicates` remained a Redis SET (it is leader-local and never
+merged across peers).
 
 ### Canonical CID selection
 
-`canonical_cid` is a label, not a key. Updating it costs one write — no
+The canonical CID is a label, not a key — so nothing external breaks when it
-rename, no race, no broken external references.
+changes. **As shipped it is not written at all**: it is computed on every read
+from the `cids` members, which makes the "update" cost zero and removes the
+reconciler entirely.
 
-Selection rule, highest-wins:
+Selection rule, highest-wins (this part shipped as designed —
+`internal/cid/rank.go`):
 
 1. IPFS CID (any variant)
 2. SHA-256 / SHA-3 family
@@ -142,8 +170,10 @@ midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
 externally meaningful for content addressing. btih ranks above midhash because
 it enables swarm interop even if it's weaker as a digest.
 
-Reconciler runs on the leader, recomputes `canonical_cid` whenever
+~~Reconciler runs on the leader, recomputes `canonical_cid` whenever
-`AddAlias` registers a new CID for an entry.
+`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no
+reconciler. `GetMetadataDocument` folds the members through `cid.Better` on
+each read.
 
 ## Internal operations
 
@@ -157,8 +187,8 @@ CID-based API.
 Mint(filePath string, size int64, mtime int64) (uuid string, err error)
 
 // AddAlias registers a CID as resolving to this UUID. Updates both the
-// cid:<cid> reverse index and the file:<uuid>/cids set atomically.
+// cid:<cid> reverse index and the file:<uuid>/cids/<cid> key-set member.
-// Triggers canonical_cid reconciliation.
+// (As shipped: no reconciliation step — canonical is derived on read.)
 AddAlias(uuid string, cid string) error
 
 // GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
@@ -199,7 +229,8 @@ DeleteRoot(uuid string) error
    - Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
    - meta-core resolves midhash256:abc → uuid_Y via GetByCID.
    - meta-core calls AddAlias(uuid_Y, "sha256:def...").
-   - Reconciler updates canonical_cid if sha256 outranks midhash256.
+   - Nothing else to do: the next read of this entry derives a canonical CID
+     of sha2-256, because it outranks the midhash.
 ```
 
 ### Lifecycle: file deletion
@@ -250,13 +281,20 @@ so the impact is contained but real.
   client.
 - **meta-dup**: gains a cleaner data source — duplicates are now explicit in
   the `duplicates` set, not implicit in colliding writes.
-- **meta-share**: announces `canonical_cid` rather than midhash. Federated
+- **meta-search / meta-share**: announce the *derived* canonical CID rather
-  search responses include all CIDs from the `cids` set so any peer querying
+  than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata`
-  by any digest gets a hit.
+  rather than reading a stored field. Federated search responses include all
+  CIDs from the `cids` key-set so any peer querying by any digest gets a hit.
+  (Written before the 2026-06 discovery/transport split; discovery is now
+  meta-search's job.)
 - **Plugins**: no change. They speak CIDs only.
 
 ## Migration
 
+> **Historical — this sweep was never run.** The alpha shipped behind a
+> clean-wipe gate (Redis was flushed and the library re-ingested), so no
+> in-place migration code exists. Kept for the record.
+
 One-time sweep, idempotent, runs on first boot after upgrade:
 
 ```
@@ -285,7 +323,8 @@ roots and resumes by detecting which step completed (presence of
 | UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
 | Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
 | Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
-| `cids` and `duplicates` field type | Redis SET | Departure from "all-strings" convention but consistent with the existing `file:__index__` precedent. |
+| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/<cid>` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. |
+| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. |
 | Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
 
 ## Open questions
@@ -297,9 +336,10 @@ roots and resumes by detecting which step completed (presence of
   hashing reveals the collision. Worth doing eventually; not required for
   this iteration.
 
-- **meta-share announce strategy.** Announce only `canonical_cid`, or every
+- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every*
-  entry in `cids`? Maximum discoverability vs. DHT footprint. Decide as part
+  member of `cids` on the wire (`Record.cids`) and addresses itself by the
-  of the meta-share work, not here.
+  single derived canonical (`Record.cid`), so a peer querying by any digest
+  gets a hit without the DHT footprint of announcing each one separately.
 
 - **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
   content-streaming endpoint for external IPFS clients? Adjacent to this

go.mod

@@ -3,6 +3,7 @@ module github.com/metazla/meta-core
 go 1.21
 
 require (
+	github.com/alicebob/miniredis/v2 v2.31.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
@@ -13,6 +14,9 @@ require (
 )
 
 require (
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/yuin/gopher-lua v1.1.0 // indirect
+	golang.org/x/sys v0.28.0
 )

go.sum

@@ -1,15 +1,24 @@
+github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
+github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
 github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -18,7 +27,12 @@ github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
 github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
 github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
 github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

internal/api/admin_migrate.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"net/http"
+	"strconv"
 )
 
 // handleMigrateDualRoots triggers the one-shot sweep that re-unifies
@@ -28,3 +29,103 @@ func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request)
 		"fixed":   fixed,
 	})
 }
+
+// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the
+// retired `domain=film|tv` values to `screen` and backfills `workForm` from
+// `contentKind` (METADATA_KEYS.md §14.17). See
+// storage.MigrateDomainScreen for why the vocabulary change is paid in the
+// data rather than with a read-side alias.
+//
+// Returns {fixed: N} — records changed, not records examined. Safe to call
+// repeatedly: a record already on the current vocabulary is skipped.
+func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	fixed, err := s.storage.MigrateDomainScreenWithTimeout()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"success": true,
+		"fixed":   fixed,
+	})
+}
+
+// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
+// `anilistid` on a site card, and `workcid` / position keys on any card
+// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
+// why chapters and anchor cards' ids are deliberately left alone.
+//
+// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
+// keys, so a bare call only reports what it would delete. An unparseable
+// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
+// sweep that ran.
+//
+// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
+// records, byKey counts fields per key. Safe to call repeatedly: a swept
+// record plans nothing on the next pass.
+func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	apply := false
+	if v := r.URL.Query().Get("apply"); v != "" {
+		parsed, err := strconv.ParseBool(v)
+		if err != nil {
+			writeError(w, http.StatusBadRequest, "apply must be true or false")
+			return
+		}
+		apply = parsed
+	}
+
+	res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"success": true,
+		"dryRun":  !apply,
+		"planned": res.Planned,
+		"fixed":   res.Fixed,
+		"byKey":   res.ByKey,
+	})
+}
+
+// handleMigrateUserCids triggers the one-shot sweep that builds
+// `udl:idx:user:<uid>:cids` for User Data Layer records written before that
+// index existed. See storage.UDLBackfillUserCids.
+//
+// The index is what makes whole-profile export a set read instead of a
+// keyspace scan, and it is maintained by the upsert script from now on — so
+// this only ever matters once, for data already on disk. Until it has run, a
+// profile that predates the index exports as **empty**, which means a
+// cross-instance recovery reports success having restored nothing.
+//
+// Returns {fixed: N} — cids added, not cids examined. Safe to call repeatedly:
+// a second pass adds nothing.
+func (s *Server) handleMigrateUserCids(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	fixed, err := s.storage.UDLBackfillUserCids()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"success": true,
+		"fixed":   fixed,
+	})
+}

internal/api/admin_sweep_test.go

@@ -0,0 +1,93 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newSweepTestServer is a Server over miniredis with one literature card
+// carrying a copied `workcid`.
+func newSweepTestServer(t *testing.T) *Server {
+	t.Helper()
+	mr := miniredis.RunT(t)
+	c := storage.NewClient("")
+	if err := c.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	for k, v := range map[string]string{
+		"domain":   "literature",
+		"fileType": "card",
+		"workcid":  "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
+	} {
+		if err := c.SetProperty("card-1", k, v); err != nil {
+			t.Fatalf("seed %s: %v", k, err)
+		}
+	}
+	return &Server{storage: c}
+}
+
+type sweepResponse struct {
+	Success bool           `json:"success"`
+	DryRun  bool           `json:"dryRun"`
+	Planned int            `json:"planned"`
+	Fixed   int            `json:"fixed"`
+	ByKey   map[string]int `json:"byKey"`
+}
+
+func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
+	t.Helper()
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
+	s.handleSweepLiteratureEchoes(rr, req)
+	var body sweepResponse
+	if rr.Code == http.StatusOK {
+		if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+			t.Fatalf("decode %s: %v", rr.Body.String(), err)
+		}
+	}
+	return rr, body
+}
+
+// A bare call must never delete: the default is the dry run.
+func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
+	s := newSweepTestServer(t)
+
+	rr, body := postSweep(t, s, "")
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
+	}
+	if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
+		t.Errorf("dry run = %+v", body)
+	}
+	if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
+		t.Errorf("dry run deleted workcid")
+	}
+
+	rr, body = postSweep(t, s, "?apply=true")
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
+	}
+	if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
+		t.Errorf("apply = %+v", body)
+	}
+	if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
+		t.Errorf("apply left workcid = %q", v)
+	}
+}
+
+// A typo must not pass for a sweep that ran.
+func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
+	s := newSweepTestServer(t)
+
+	rr, _ := postSweep(t, s, "?apply=yes-please")
+	if rr.Code != http.StatusBadRequest {
+		t.Errorf("status = %d, want 400", rr.Code)
+	}
+}

internal/api/deprecated_field_test.go

@@ -0,0 +1,86 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the
+// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They
+// must be rejected at the write boundary rather than stored.
+//
+// Why a hard 400 and not a silent rewrite: the reverse-index hook
+// (maybeAddAliasFromFieldLocked) only registers an alias for a field named
+// `cids/<cid>`. A `cid_*` field was therefore written verbatim and never
+// indexed — the record silently became unresolvable by that CID. A loud
+// failure is strictly better than a record that looks correct and cannot be
+// found.
+func TestRejectDeprecatedField(t *testing.T) {
+	rejected := []string{
+		"cid_midhash256",
+		"cid_sha2-256",
+		"cid_btih_v2",
+		"cid_crc32",
+		"midhash256",
+		"canonical_cid",
+	}
+	for _, f := range rejected {
+		if !rejectDeprecatedField(f) {
+			t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f)
+		}
+	}
+
+	accepted := []string{
+		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
+		"filePath",
+		"title",
+		"fileinfo/duration",
+		"tmdb/poster",
+		// Near-misses that must NOT be caught: the prefix is `cid_`, with an
+		// underscore. A field merely starting with "cid" is fine.
+		"cids",
+		"cidrank",
+	}
+	for _, f := range accepted {
+		if rejectDeprecatedField(f) {
+			t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f)
+		}
+	}
+}
+
+func TestValidateFields_Rejects400WithSlug(t *testing.T) {
+	rr := httptest.NewRecorder()
+	blocked := validateFields(rr, map[string]string{
+		"title":          "Inception",
+		"cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
+	})
+
+	if !blocked {
+		t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed")
+	}
+	if rr.Code != http.StatusBadRequest {
+		t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest)
+	}
+
+	var body map[string]interface{}
+	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+		t.Fatalf("response is not JSON: %v", err)
+	}
+	if got := body["error"]; got != ErrDeprecatedField {
+		t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField)
+	}
+}
+
+func TestValidateFields_AllowsCidsKeyset(t *testing.T) {
+	rr := httptest.NewRecorder()
+	blocked := validateFields(rr, map[string]string{
+		"filePath": "/watch/Inception.mkv",
+		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true",
+		"cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua":     "true",
+	})
+	if blocked {
+		t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String())
+	}
+}

internal/api/files_register.go

@@ -0,0 +1,132 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/metazla/meta-core/internal/watcher"
+)
+
+// registerFileRequest is the body of POST /api/files/register.
+type registerFileRequest struct {
+	// Path relative to FILES_PATH, e.g. "plugin/subtitles/ab12….eng.srt".
+	Path string `json:"path"`
+}
+
+type registerFileResponse struct {
+	Root    string `json:"root"`
+	CID     string `json:"cid"`
+	Created bool   `json:"created"`
+	Path    string `json:"path"`
+	Size    int64  `json:"size"`
+}
+
+// pluginRelPath normalises a FILES_PATH-relative path and refuses anything
+// outside plugin/. Plugin output is the only tree a service may register this
+// way: /files/watch is the user's library and belongs to the watcher.
+func pluginRelPath(raw string) (string, bool) {
+	p := strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(raw)), "/")
+	if p == "" {
+		return "", false
+	}
+	for _, seg := range strings.Split(p, "/") {
+		if seg == ".." {
+			return "", false
+		}
+	}
+	clean := path.Clean(p)
+	if !strings.HasPrefix(clean, "plugin/") || clean == "plugin/" {
+		return "", false
+	}
+	return clean, true
+}
+
+// handleRegisterFile handles POST /api/files/register.
+//
+// Does for ONE file what the watcher does for every file it scans: midhash256
+// it, mint a root carrying filePath/sizeByte/mtimeNano, register the
+// cid:<midhash> alias, and append a file:events entry so meta-share's
+// ipfs_seed chunks, seeds and provides it.
+//
+// It exists for cores that run with ENABLE_FILE_WATCHER=false (a meta-watch
+// client box). Without it, bytes written under /files/plugin over WebDAV are
+// never resolvable by CID and never reach the swarm. Idempotent: the same
+// content at the same path returns the existing root with created=false.
+func (s *Server) handleRegisterFile(w http.ResponseWriter, r *http.Request) {
+	if s.storage == nil || !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	var req registerFileRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	rel, ok := pluginRelPath(req.Path)
+	if !ok {
+		writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
+		return
+	}
+
+	fullPath := filepath.Join(s.config.FilesPath, filepath.FromSlash(rel))
+	absFiles, _ := filepath.Abs(filepath.Join(s.config.FilesPath, "plugin"))
+	absFull, _ := filepath.Abs(fullPath)
+	if !strings.HasPrefix(absFull, absFiles+string(filepath.Separator)) {
+		writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
+		return
+	}
+
+	info, err := os.Stat(fullPath)
+	if err != nil {
+		writeError(w, http.StatusNotFound, "file not found")
+		return
+	}
+	if info.IsDir() {
+		writeError(w, http.StatusBadRequest, "path is a directory, not a file")
+		return
+	}
+
+	midhash, err := watcher.ComputeMidHash256(fullPath, info.Size())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, "failed to hash file")
+		return
+	}
+
+	root, created, err := s.storage.RegisterFileTuple(fullPath, info.Size(), info.ModTime().UnixNano(), midhash)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	if created {
+		ev := watcher.FileEvent{
+			Type:       watcher.EventTypeAdd,
+			Path:       rel,
+			Size:       info.Size(),
+			MidHash256: midhash,
+			Timestamp:  watcher.NowMS(),
+		}
+		if err := watcher.PublishEvent(s.storage, ev); err != nil {
+			// The record is written and resolvable; only seeding is late.
+			writeError(w, http.StatusInternalServerError, "registered, but file event not published: "+err.Error())
+			return
+		}
+	}
+
+	status := http.StatusOK
+	if created {
+		status = http.StatusCreated
+	}
+	writeJSON(w, status, registerFileResponse{
+		Root:    root,
+		CID:     midhash,
+		Created: created,
+		Path:    rel,
+		Size:    info.Size(),
+	})
+}

internal/api/files_register_test.go

@@ -0,0 +1,144 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/config"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
+// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
+// which is exactly the configuration this route exists for.
+func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
+	t.Helper()
+	mr := miniredis.RunT(t)
+	c := storage.NewClient("")
+	if err := c.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	files := t.TempDir()
+	return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
+}
+
+func writeFixture(t *testing.T, files, rel, body string) string {
+	t.Helper()
+	full := filepath.Join(files, filepath.FromSlash(rel))
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return full
+}
+
+func postRegister(t *testing.T, s *Server, path string) (int, registerFileResponse) {
+	t.Helper()
+	body, _ := json.Marshal(registerFileRequest{Path: path})
+	req := httptest.NewRequest(http.MethodPost, "/api/files/register", strings.NewReader(string(body)))
+	rr := httptest.NewRecorder()
+	s.handleRegisterFile(rr, req)
+	var resp registerFileResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &resp)
+	return rr.Code, resp
+}
+
+func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
+	t.Helper()
+	entries, err := mr.Stream("file:events")
+	if err != nil {
+		return 0
+	}
+	return len(entries)
+}
+
+// The whole point of the route: with the watcher off, a file written under
+// plugin/ becomes a record resolvable by its midhash, and meta-share hears
+// about it on file:events.
+func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
+	s, files, mr := newRegisterTestServer(t)
+	rel := "plugin/subtitles/ab12cd34.eng.srt"
+	full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
+
+	code, resp := postRegister(t, s, rel)
+	if code != http.StatusCreated || !resp.Created {
+		t.Fatalf("want 201 created, got %d %+v", code, resp)
+	}
+	if !strings.HasPrefix(resp.CID, "b") || resp.Root == "" {
+		t.Fatalf("want a multibase cid and a root, got %+v", resp)
+	}
+
+	root, err := s.storage.GetByCID(resp.CID)
+	if err != nil || root != resp.Root {
+		t.Fatalf("cid alias must resolve to the root: got %q err=%v, want %q", root, err, resp.Root)
+	}
+	if fp, _ := s.storage.GetProperty(resp.Root, "filePath"); fp != full {
+		t.Fatalf("filePath = %q, want %q", fp, full)
+	}
+
+	entries, err := mr.Stream("file:events")
+	if err != nil || len(entries) != 1 {
+		t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
+	}
+	vals := strings.Join(entries[0].Values, "|")
+	if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
+		t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
+	}
+}
+
+// Re-uploading the same subtitle is a no-op: same root, no second event (which
+// would make meta-share re-chunk and re-provide the file).
+func TestRegisterFile_IsIdempotent(t *testing.T) {
+	s, files, mr := newRegisterTestServer(t)
+	rel := "plugin/subtitles/same.eng.srt"
+	writeFixture(t, files, rel, "same bytes")
+
+	_, first := postRegister(t, s, rel)
+	code, second := postRegister(t, s, rel)
+	if code != http.StatusOK || second.Created {
+		t.Fatalf("second register must be 200 created=false, got %d %+v", code, second)
+	}
+	if second.Root != first.Root || second.CID != first.CID {
+		t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
+	}
+	if n := streamLen(t, mr); n != 1 {
+		t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
+	}
+}
+
+// Only plugin output may be registered here. /files/watch is the user's
+// library and belongs to the watcher; traversal must not escape either.
+func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
+	s, files, _ := newRegisterTestServer(t)
+	writeFixture(t, files, "watch/movie.mkv", "x")
+	for _, p := range []string{
+		"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
+		"plugin/../watch/movie.mkv", "../etc/passwd", "plugin/../../etc/passwd",
+	} {
+		if code, _ := postRegister(t, s, p); code != http.StatusBadRequest {
+			t.Errorf("path %q: want 400, got %d", p, code)
+		}
+	}
+}
+
+func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
+	s, files, _ := newRegisterTestServer(t)
+	if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
+		t.Errorf("missing file: want 404, got %d", code)
+	}
+	if err := os.MkdirAll(filepath.Join(files, "plugin", "subtitles", "dir"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if code, _ := postRegister(t, s, "plugin/subtitles/dir"); code != http.StatusBadRequest {
+		t.Errorf("directory: want 400, got %d", code)
+	}
+}

internal/api/files_tuples.go

@@ -0,0 +1,74 @@
+package api
+
+import "net/http"
+
+// fileTupleJSON is one file-backed root on the wire.
+type fileTupleJSON struct {
+	HashID    string `json:"hashId"`
+	FilePath  string `json:"filePath"`
+	SizeByte  int64  `json:"sizeByte"`
+	MtimeNano int64  `json:"mtimeNano"`
+}
+
+// FileTuplesResponse is the body of GET /api/files/tuples.
+//
+// Files is a pointer so the summary form can omit it while the full form
+// always carries it — an empty library is `"files": []`, never an absent key,
+// which a client could mistake for a meta-core that lacks the rows.
+type FileTuplesResponse struct {
+	Files     *[]fileTupleJSON `json:"files,omitempty"`
+	Count     int              `json:"count"`
+	TotalSize int64            `json:"totalSize"`
+}
+
+// handleGetFileTuples handles GET /api/files/tuples.
+//
+// Lists the FILE-BACKED roots — records carrying filePath, sizeByte and
+// mtimeNano, i.e. what the watcher or /api/files/register minted — with their
+// sizes, plus the count and total. Gateway / cid-rooted records carry no file
+// and are not included; neither is a locator record that only merged a cache
+// file's metadata (it has no mtimeNano).
+//
+// It exists so a client never walks every hashId to answer "how many files,
+// how big": GetTuplesForAllFiles reads the three fields for every root in
+// chunked MGETs server-side (~250 round-trips for 82k roots, no SCAN), where
+// the client-side equivalent was one HTTP request per record.
+//
+// ?summary=1 omits the rows and returns only {count, totalSize}.
+func (s *Server) handleGetFileTuples(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	tuples, err := s.storage.GetTuplesForAllFiles()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	summary := r.URL.Query().Get("summary")
+	withRows := summary == "" || summary == "0" || summary == "false"
+
+	resp := FileTuplesResponse{Count: len(tuples)}
+	var rows []fileTupleJSON
+	if withRows {
+		rows = make([]fileTupleJSON, 0, len(tuples))
+	}
+	for _, t := range tuples {
+		resp.TotalSize += t.Size
+		if withRows {
+			rows = append(rows, fileTupleJSON{
+				HashID:    t.HashID,
+				FilePath:  t.FilePath,
+				SizeByte:  t.Size,
+				MtimeNano: t.MtimeNano,
+			})
+		}
+	}
+	if withRows {
+		resp.Files = &rows
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}

internal/api/files_tuples_test.go

@@ -0,0 +1,114 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
+	t.Helper()
+	mr := miniredis.RunT(t)
+	c := storage.NewClient("")
+	if err := c.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return &Server{storage: c}, c
+}
+
+func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesResponse) {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodGet, "/api/files/tuples"+query, nil)
+	rr := httptest.NewRecorder()
+	s.handleGetFileTuples(rr, req)
+	var resp FileTuplesResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &resp)
+	return rr.Code, resp
+}
+
+func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
+	t.Helper()
+	id, err := c.Mint(path, size, mtime)
+	if err != nil {
+		t.Fatalf("mint %s: %v", path, err)
+	}
+	return id
+}
+
+// Only records that are actually files count: a gateway record and a record
+// that merely carries a path + size (no mtimeNano) are both left out.
+func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
+	s, c := newFileTuplesTestServer(t)
+	a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
+	b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
+	if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil {
+		t.Fatal(err)
+	}
+
+	code, resp := getFileTuples(t, s, "")
+	if code != http.StatusOK {
+		t.Fatalf("want 200, got %d", code)
+	}
+	if resp.Count != 2 || resp.TotalSize != 3500 {
+		t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
+	}
+	if resp.Files == nil || len(*resp.Files) != 2 {
+		t.Fatalf("want 2 rows, got %+v", resp.Files)
+	}
+	got := []string{(*resp.Files)[0].HashID, (*resp.Files)[1].HashID}
+	want := []string{a, b}
+	sort.Strings(got)
+	sort.Strings(want)
+	if got[0] != want[0] || got[1] != want[1] {
+		t.Fatalf("want roots %v, got %v", want, got)
+	}
+}
+
+func TestFileTuples_SummaryOmitsRows(t *testing.T) {
+	s, c := newFileTuplesTestServer(t)
+	mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
+	mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
+
+	code, resp := getFileTuples(t, s, "?summary=1")
+	if code != http.StatusOK {
+		t.Fatalf("want 200, got %d", code)
+	}
+	if resp.Files != nil {
+		t.Fatalf("summary must omit rows, got %d", len(*resp.Files))
+	}
+	if resp.Count != 2 || resp.TotalSize != 3500 {
+		t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
+	}
+}
+
+// An empty library still carries "files": [] — clients treat an absent key as
+// "this meta-core cannot list files", so it must not disappear when empty.
+func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
+	s, _ := newFileTuplesTestServer(t)
+
+	code, resp := getFileTuples(t, s, "")
+	if code != http.StatusOK {
+		t.Fatalf("want 200, got %d", code)
+	}
+	if resp.Files == nil || len(*resp.Files) != 0 || resp.Count != 0 {
+		t.Fatalf("want an empty files array and count 0, got %+v", resp)
+	}
+}
+
+func TestFileTuples_StorageNotConnected(t *testing.T) {
+	s := &Server{storage: storage.NewClient("")}
+
+	code, _ := getFileTuples(t, s, "")
+	if code != http.StatusServiceUnavailable {
+		t.Fatalf("want 503, got %d", code)
+	}
+}

internal/api/handlers.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/base32"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 )
 
 // contentTypeByExt maps file extensions to MIME types
@@ -78,14 +80,51 @@ type ErrorResponse struct {
 // Stable error-slug strings. Pinned in the doc; consumers may match on
 // these. Do not rename without a deprecation cycle.
 const (
-	ErrAliasCollision    = "alias_collision"
+	ErrAliasCollision     = "alias_collision"
-	ErrUnknownRoot       = "unknown_root"
+	ErrUnknownRoot        = "unknown_root"
-	ErrUnknownCID        = "unknown_cid"
+	ErrUnknownCID         = "unknown_cid"
-	ErrSchemaViolation   = "schema_violation"
+	ErrSchemaViolation    = "schema_violation"
 	ErrStorageUnavailable = "storage_unavailable"
-	ErrInternal          = "internal"
+	ErrInternal           = "internal"
+	ErrDeprecatedField    = "deprecated_field"
 )
 
+// rejectDeprecatedField reports whether a field name is one of the removed
+// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …).
+//
+// This is a tripwire, not a compatibility shim. Such a field used to be
+// accepted and written verbatim — but the reverse-index hook
+// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named
+// `cids/<cid>`, so a `cid_*` field was stored and then NEVER INDEXED: the
+// record silently became unresolvable by that CID. Failing the write loudly is
+// strictly better than a record that looks fine and cannot be found.
+//
+// Writers must emit the bare-CID key-set instead: `cids/<bareCid>` = "true".
+// See METADATA_KEYS.md §2/§14.13.
+func rejectDeprecatedField(field string) bool {
+	return strings.HasPrefix(field, "cid_") ||
+		field == "midhash256" ||
+		field == "canonical_cid"
+}
+
+// validateFields rejects a write carrying any deprecated CID field. Returns
+// true if the request was rejected (the handler must return immediately).
+func validateFields(w http.ResponseWriter, metadata map[string]string) bool {
+	for field := range metadata {
+		if rejectDeprecatedField(field) {
+			writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField,
+				fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+
+					"write the bare-CID key-set instead: \"cids/<bareCid>\": \"true\" "+
+					"(a cid_* field would be stored but never indexed, leaving the "+
+					"record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.",
+					field),
+				false)
+			return true
+		}
+	}
+	return false
+}
+
 // defaultSlugFor maps an HTTP status code to the most likely error slug.
 // Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
 // instead of "unknown_root" on 404) call writeErrorSlug directly.
@@ -255,6 +294,10 @@ func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, metadata) {
+		return
+	}
+
 	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -505,64 +548,51 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
 	http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
 }
 
-// handleListServices handles GET /services and GET /api/services
+// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
-// Supports optional ?current=<service-name> query parameter
+//
-func (s *Server) handleListServices(w http.ResponseWriter, r *http.Request) {
+// Beacon v2 (docs/project-architecture/beacon-v2.md). The list is this node's
-	services, err := s.discovery.DiscoverAll()
+// in-memory view of the UDP group, so it works across stacks that share a
-	if err != nil {
+// network but no volume. Instances of one name collapse to the most recent
-		writeError(w, http.StatusInternalServerError, err.Error())
+// (one row per service, the nav menu's shape); `?all=1` returns every
-		return
+// instance, `?cap=<pattern>` keeps only nodes with a matching capability.
-	}
+func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
-
 	response := map[string]interface{}{
-		"services": services,
+		"current":   s.config.ServiceName,
-		"count":    len(services),
+		"neighbors": []interface{}{},
+		"count":     0,
 	}
 
-	// Add 'current' field if query parameter is provided
+	if s.mesh == nil {
-	current := r.URL.Query().Get("current")
+		// Discovery off or failed to bind. Report empty rather than 503 —
-	if current != "" {
+		// the nav is decoration and callers already treat [] as "hide".
-		response["current"] = current
+		response["enabled"] = false
+		writeJSON(w, http.StatusOK, response)
+		return
 	}
 
+	var list []meshdisco.Neighbor
+	if r.URL.Query().Get("all") != "" {
+		list = s.mesh.Neighbors()
+	} else {
+		list = s.mesh.NeighborsByName()
+	}
+	if pattern := r.URL.Query().Get("cap"); pattern != "" {
+		kept := list[:0]
+		for _, nb := range list {
+			if nb.Matches(pattern) {
+				kept = append(kept, nb)
+			}
+		}
+		list = kept
+	}
+
+	response["enabled"] = true
+	response["neighbors"] = list
+	response["count"] = len(list)
+	response["self"] = s.mesh.Self()
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handleGetService handles GET /services/{name}
-func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	name := vars["name"]
-
-	if name == "" {
-		writeError(w, http.StatusBadRequest, "service name is required")
-		return
-	}
-
-	service, err := s.discovery.Discover(name)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	if service == nil {
-		writeError(w, http.StatusNotFound, "service not found")
-		return
-	}
-
-	writeJSON(w, http.StatusOK, service)
-}
-
-// handleCleanupStats handles GET /services/cleanup/stats
-func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
-	if s.cleaner == nil {
-		writeError(w, http.StatusServiceUnavailable, "cleaner not initialized")
-		return
-	}
-
-	stats := s.cleaner.Stats()
-	writeJSON(w, http.StatusOK, stats)
-}
-
 // writeJSON writes a JSON response
 func writeJSON(w http.ResponseWriter, status int, data interface{}) {
 	w.Header().Set("Content-Type", "application/json")
@@ -620,6 +650,10 @@ func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, metadata) {
+		return
+	}
+
 	updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -697,6 +731,10 @@ func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, map[string]string{key: ""}) {
+		return
+	}
+
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
@@ -790,6 +828,10 @@ func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, map[string]string{key: ""}) {
+		return
+	}
+
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
@@ -905,10 +947,10 @@ func (s *Server) handleComputeFileCID(w http.ResponseWriter, r *http.Request) {
 	// CIDv1 format: version (0x01) + codec (0x55) + multihash
 	// Multihash format: hash-code (0x12) + length (0x20) + hash
 	cidBytes := make([]byte, 0, 2+2+32)
-	cidBytes = append(cidBytes, 0x01)       // CIDv1
+	cidBytes = append(cidBytes, 0x01) // CIDv1
-	cidBytes = append(cidBytes, 0x55)       // raw codec
+	cidBytes = append(cidBytes, 0x55) // raw codec
-	cidBytes = append(cidBytes, 0x12)       // sha256 code
+	cidBytes = append(cidBytes, 0x12) // sha256 code
-	cidBytes = append(cidBytes, 0x20)       // 32 bytes
+	cidBytes = append(cidBytes, 0x20) // 32 bytes
 	cidBytes = append(cidBytes, hashBytes...)
 
 	// Encode as base32lower with 'b' prefix (multibase)

internal/api/handlers_identity.go

@@ -3,10 +3,14 @@ package api
 import (
 	"encoding/base64"
 	"encoding/json"
+	"errors"
+	"log"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/storage"
 )
 
 // identityStatusResponse is the shape of GET /api/identity. The private key
@@ -50,11 +54,68 @@ type identityImportResponse struct {
 	UID       string `json:"uid"`
 	Curve     string `json:"curve"`
 	CreatedAt int64  `json:"createdAt"`
+	// Created distinguishes "this key was new here" (201) from "this key was
+	// already in the keystore" (200). Import is the sign-in primitive for
+	// clients: presenting the secret key is the proof of ownership, so a
+	// re-import must succeed rather than conflict. Clients use the flag only
+	// to word the confirmation ("new profile" vs "welcome back").
+	Created bool `json:"created"`
+}
+
+// identityChallengeRequest asks for a one-shot challenge to sign.
+type identityChallengeRequest struct {
+	UID    string `json:"uid"`
+	Action string `json:"action"`
+}
+
+type identityChallengeResponse struct {
+	Challenge string `json:"challenge"`
+	ExpiresAt int64  `json:"expiresAt"`
+}
+
+// identityProof is the proof-of-possession carried by reveal and delete: the
+// challenge text as issued, and a signature over it by the account's key.
+//
+// The signature is produced by whoever holds the key — the browser, locally.
+// It is NOT obtainable from POST /api/identity/sign, which refuses any payload
+// in the challenge domain; that refusal is what makes this proof mean
+// something on a node that stores every private key in plaintext.
+type identityProof struct {
+	Challenge string `json:"challenge"`
+	Signature string `json:"signature"`
+}
+
+// identityRevealRequest asks for an existing account's private key back.
+//
+// Gated on proof-of-possession, which makes it a "does this node hold the same
+// key I do" check rather than the recovery path it once was. That is the
+// deliberate consequence of the rule this file enforces: the private key, and
+// nothing else, is authority over the account. An operator with root on the box
+// can still read the file; an operator with only the API cannot.
+type identityRevealRequest struct {
+	Confirm bool   `json:"confirm"`
+	UID     string `json:"uid"`
+	identityProof
+}
+
+type identityRevealResponse struct {
+	UID           string `json:"uid"`
+	Curve         string `json:"curve"`
+	PrivateKeyHex string `json:"privateKeyHex"`
 }
 
 type identityDeleteRequest struct {
 	Confirm bool   `json:"confirm"`
 	UID     string `json:"uid"`
+	identityProof
+}
+
+// identityDeleteResponse reports what the delete actually removed. The purge
+// counts are the point: "account removed" alone leaves the caller unable to
+// tell a clean removal from one that silently stranded a decade of ratings.
+type identityDeleteResponse struct {
+	UID    string               `json:"uid"`
+	Purged storage.UDLUserStats `json:"purged"`
 }
 
 type identitySignRequest struct {
@@ -68,6 +129,41 @@ type identitySignResponse struct {
 	SigB64 string `json:"sigB64"`
 }
 
+// identitySignBatchRequest signs many payloads under one account in one call.
+//
+// Exists because a User Data Layer write signs exactly one record, and the bulk
+// verbs above it ("mark this whole series seen") produce one record per episode:
+// a 300-episode show is 300 sequential sign round trips through
+// meta-watch -> meta-search -> here, which dominates the operation. The private
+// key still never leaves this process; only the transport is batched.
+type identitySignBatchRequest struct {
+	// PayloadsB64 are the canonical byte strings to sign, in order. The
+	// response's signatures line up index-for-index.
+	PayloadsB64 []string `json:"payloadsB64"`
+	// UID selects which account signs — same optional single-identity
+	// back-compat as identitySignRequest.
+	UID string `json:"uid,omitempty"`
+}
+
+type identitySignBatchResponse struct {
+	SigsB64 []string `json:"sigsB64"`
+}
+
+// maxSignBatch bounds one batch. A signature is cheap but not free, and the
+// request holds the account lock for its duration; a caller with more than this
+// should page. Sized well above the longest real series (Supernatural, 327).
+const maxSignBatch = 1000
+
+// reservedPayloadMessage explains the one refusal the signing endpoints make.
+//
+// This is the load-bearing half of proof-of-possession on a node that holds
+// every private key: without it, a caller wanting to delete someone else's
+// account just asks meta-core to sign the challenge for them, and the gate on
+// reveal/delete becomes a formality. See identity.ChallengeDomain.
+const reservedPayloadMessage = "payload is in the reserved account-authorisation domain (" +
+	identity.ChallengeDomain + "…); meta-core will not sign one. " +
+	"Signing a challenge is what proves you hold the key, so it must be done where the key is."
+
 type identityAEADKeyResponse struct {
 	AEADKeyB64 string `json:"aeadKeyB64"`
 }
@@ -172,9 +268,17 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	dir := s.config.IdentityAccountsDir()
-	if identity.ExistsByUID(dir, id.UID) {
+	// Idempotent: importing a key that is already in the keystore is how a
-		writeErrorSlug(w, http.StatusConflict, "identity_exists",
+	// second device signs in to an existing account. Answer 200 with the
-			"an account with this key already exists", false)
+	// stored account (preserving its original createdAt) instead of 409 —
+	// the caller proved ownership by presenting the key.
+	if existing, err := identity.LoadByUID(dir, id.UID); err == nil && existing != nil {
+		writeJSON(w, http.StatusOK, identityImportResponse{
+			UID:       existing.UID,
+			Curve:     existing.Curve,
+			CreatedAt: existing.CreatedAt,
+			Created:   false,
+		})
 		return
 	}
 	if err := identity.SaveAccount(dir, id); err != nil {
@@ -185,49 +289,176 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		UID:       id.UID,
 		Curve:     id.Curve,
 		CreatedAt: id.CreatedAt,
+		Created:   true,
 	})
 }
 
-func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
+// handleIdentityChallenge issues the one-shot text a client signs to prove it
-	var body identityDeleteRequest
+// holds an account's private key.
-	// Tolerate empty body — confirm flag in a query param is also acceptable
+//
-	// for shells where sending a JSON body with DELETE is awkward.
+// Unauthenticated by design: a challenge is useless without the key, and
-	_ = json.NewDecoder(r.Body).Decode(&body)
+// GET /api/identity/accounts already publishes every uid, so refusing to mint
-	if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
+// one for an unknown caller would protect nothing. Requiring the account to
-		writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
+// exist is for a clear error, not secrecy.
-			"identity deletion requires {\"confirm\": true} or ?confirm=true", false)
+func (s *Server) handleIdentityChallenge(w http.ResponseWriter, r *http.Request) {
+	var body identityChallengeRequest
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
 		return
 	}
-	dir := s.config.IdentityAccountsDir()
+	uid := strings.TrimSpace(body.UID)
+	if uid == "" {
+		writeErrorSlug(w, http.StatusBadRequest, "uid_required", "uid is required", false)
+		return
+	}
+	action := identity.Action(strings.TrimSpace(body.Action))
+	if !identity.ValidAction(action) {
+		writeErrorSlug(w, http.StatusBadRequest, "bad_action",
+			`action must be "reveal" or "delete"`, false)
+		return
+	}
+	if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
+		writeErrorSlug(w, http.StatusNotFound, "no_identity", "no identity for the given uid", false)
+		return
+	}
+	text, expires, err := s.challenges.Issue(uid, action)
+	if err != nil {
+		if errors.Is(err, identity.ErrChallengeFull) {
+			writeErrorSlug(w, http.StatusServiceUnavailable, "challenges_full", err.Error(), true)
+			return
+		}
+		writeErrorSlug(w, http.StatusBadRequest, "bad_uid", err.Error(), false)
+		return
+	}
+	writeJSON(w, http.StatusOK, identityChallengeResponse{
+		Challenge: text,
+		ExpiresAt: expires.Unix(),
+	})
+}
+
+// requireProof resolves the target account and verifies the caller's
+// proof-of-possession for `action`. status == 0 means the proof held.
+//
+// The uid is mandatory here, unlike elsewhere in this file: the old "sole
+// account" fallback would have this pick a target the signature does not name,
+// and a proof that authorises an unnamed account authorises the wrong one as
+// soon as a second profile exists.
+func (s *Server) requireProof(uid string, proof identityProof, action identity.Action) (
+	id *identity.Identity, status int, slug, msg string,
+) {
+	uid = strings.TrimSpace(uid)
+	if uid == "" {
+		return nil, http.StatusBadRequest, "uid_required",
+			"uid is required: the signature authorises one specific account"
+	}
+	if proof.Challenge == "" || proof.Signature == "" {
+		return nil, http.StatusUnauthorized, "signature_required",
+			"this operation requires proof you hold the account's private key: " +
+				"POST /api/identity/challenge, sign the returned text, and resend it as {challenge, signature}"
+	}
+	sig, err := base64.StdEncoding.DecodeString(proof.Signature)
+	if err != nil {
+		return nil, http.StatusBadRequest, "bad_signature", "signature is not valid base64"
+	}
+	// Resolve before redeeming so a request naming a missing account does not
+	// burn a challenge it could not have used anyway.
+	id, status, slug, msg = s.resolveAccount(uid)
+	if status != 0 {
+		return nil, status, slug, msg
+	}
+	// Redeem first, verify second: the challenge is single-use either way, so a
+	// wrong signature costs the attacker a fresh round trip per guess rather
+	// than letting them grind one challenge offline against the same nonce.
+	if err := s.challenges.Redeem(proof.Challenge, uid, action); err != nil {
+		return nil, http.StatusUnauthorized, "bad_challenge", err.Error()
+	}
+	if err := identity.Verify(uid, []byte(proof.Challenge), sig); err != nil {
+		return nil, http.StatusUnauthorized, "bad_signature", err.Error()
+	}
+	return id, 0, "", ""
+}
+
+// handleIdentityReveal returns an existing account's private key to a caller
+// that proves it already holds that key.
+//
+// Deliberately POST (never a bare GET) so it cannot be triggered by a link or
+// an image tag. The proof is what actually guards it: before this gate existed,
+// anything that could reach the port could read every private key on the node.
+func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
+	var body identityRevealRequest
+	_ = json.NewDecoder(r.Body).Decode(&body)
 	uid := strings.TrimSpace(body.UID)
 	if uid == "" {
 		uid = strings.TrimSpace(r.URL.Query().Get("uid"))
 	}
-	if uid == "" {
+	id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionReveal)
-		// Back-compat: with no uid, delete the sole account. Ambiguous (and so
+	if status != 0 {
-		// refused) once more than one account exists.
+		writeErrorSlug(w, status, slug, msg, false)
-		list, err := identity.List(dir)
-		if err != nil {
-			writeError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		switch len(list) {
-		case 0:
-			w.WriteHeader(http.StatusNoContent) // idempotent
-			return
-		case 1:
-			uid = list[0].UID
-		default:
-			writeErrorSlug(w, http.StatusBadRequest, "uid_required",
-				"multiple accounts exist; specify uid to delete", false)
-			return
-		}
-	}
-	if err := identity.DeleteByUID(dir, uid); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	w.WriteHeader(http.StatusNoContent)
+	writeJSON(w, http.StatusOK, identityRevealResponse{
+		UID:           id.UID,
+		Curve:         id.Curve,
+		PrivateKeyHex: id.PrivateKeyHex,
+	})
+}
+
+// handleIdentityDelete removes one account and everything it wrote.
+//
+// Two things happen, in this order and no other:
+//
+//  1. the account's User Data Layer rows are purged (cells plus every index
+//     that names the uid, including the cid-scoped ones shared with other
+//     accounts);
+//  2. the keypair file is removed.
+//
+// Purge-then-delete so a failure never strands data: if the purge errors the
+// key is still on disk, the account still exists, and the caller can retry.
+// The reverse order would leave rows nobody can ever attribute, reach or
+// rewrite — the orphaning this endpoint exists to prevent.
+//
+// What deletion still does NOT do, and cannot: records already signed under
+// this uid and replicated to other peers keep verifying there. Removing a key
+// is not a recall.
+func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
+	var body identityDeleteRequest
+	// Tolerate an empty body so the uid may ride in the query — meta-watch's
+	// proxy puts it there. The proof itself must be in the body: a signature in
+	// a URL ends up in every access log between here and the browser.
+	_ = json.NewDecoder(r.Body).Decode(&body)
+	uid := strings.TrimSpace(body.UID)
+	if uid == "" {
+		uid = strings.TrimSpace(r.URL.Query().Get("uid"))
+	}
+	id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionDelete)
+	if status != 0 {
+		writeErrorSlug(w, status, slug, msg, false)
+		return
+	}
+
+	// A purge needs storage. Refusing here — rather than deleting the key and
+	// reporting the purge as "0 records" — is the difference between a retry
+	// and permanent orphaned data: once the key is gone the account cannot be
+	// named for a second attempt through this API.
+	if !s.storage.IsConnected() {
+		writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
+			"storage is not connected; refusing to delete the key while its data cannot be purged", true)
+		return
+	}
+	purged, err := s.storage.UDLPurgeUser(id.UID)
+	if err != nil {
+		writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
+			"could not purge this account's data; the account was NOT deleted: "+err.Error(), true)
+		return
+	}
+
+	if err := identity.DeleteByUID(s.config.IdentityAccountsDir(), id.UID); err != nil {
+		writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
+		return
+	}
+	log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids",
+		id.UID, purged.Records, purged.Cids)
+	writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
 }
 
 func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
@@ -250,6 +481,10 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
 		return
 	}
+	if identity.IsChallengePayload(raw) {
+		writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
+		return
+	}
 	sig, err := identity.Sign(id, raw)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -260,6 +495,58 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+// handleIdentitySignBatch signs every payload in the request under one account.
+//
+// All-or-nothing: a single bad base64 or signing failure fails the whole
+// request rather than returning a short array. The caller pairs signatures with
+// records by index, so a partial response would silently misalign them — and a
+// misaligned signature verifies against the wrong record, which is worse than
+// an error.
+func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) {
+	var body identitySignBatchRequest
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	uid := body.UID
+	if strings.TrimSpace(uid) == "" {
+		uid = r.URL.Query().Get("uid")
+	}
+	if len(body.PayloadsB64) == 0 {
+		writeError(w, http.StatusBadRequest, "payloadsB64 is required")
+		return
+	}
+	if len(body.PayloadsB64) > maxSignBatch {
+		writeError(w, http.StatusBadRequest, "too many payloads in one batch")
+		return
+	}
+	id, status, slug, msg := s.resolveAccount(uid)
+	if status != 0 {
+		writeErrorSlug(w, status, slug, msg, false)
+		return
+	}
+	sigs := make([]string, 0, len(body.PayloadsB64))
+	for i, p := range body.PayloadsB64 {
+		raw, err := base64.StdEncoding.DecodeString(p)
+		if err != nil {
+			writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64")
+			return
+		}
+		if identity.IsChallengePayload(raw) {
+			writeErrorSlug(w, http.StatusForbidden, "reserved_payload",
+				"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
+			return
+		}
+		sig, err := identity.Sign(id, raw)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
+	}
+	writeJSON(w, http.StatusOK, identitySignBatchResponse{SigsB64: sigs})
+}
+
 func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
 	id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
 	if status != 0 {

internal/api/identity_gate_test.go

@@ -0,0 +1,257 @@
+package api
+
+// End-to-end exercise of the account gate against a live router + miniredis:
+// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
+// both the key and every row are gone. Deliberately drives the real HTTP
+// handlers rather than the packages beneath them, because the bugs that matter
+// here (a challenge accepted for the wrong action, a purge that runs after the
+// key is gone) live in the wiring.
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/gorilla/mux"
+	"github.com/metazla/meta-core/internal/config"
+	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+func newTestServer(t *testing.T) *Server {
+	t.Helper()
+	mr := miniredis.RunT(t)
+	stor := storage.NewClient("")
+	if err := stor.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	t.Cleanup(func() { _ = stor.Close() })
+
+	s := &Server{
+		config:     &config.Config{MetaCorePath: t.TempDir()},
+		storage:    stor,
+		challenges: identity.NewChallengeStore(),
+		router:     mux.NewRouter(),
+	}
+	s.setupIdentityTestRoutes()
+	return s
+}
+
+// Only the routes under test, so this does not depend on every other subsystem
+// booting.
+func (s *Server) setupIdentityTestRoutes() {
+	s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
+	s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
+	s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
+	s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
+	s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
+	s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
+	s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
+}
+
+func do(t *testing.T, s *Server, method, path string, body any) (*httptest.ResponseRecorder, map[string]any) {
+	t.Helper()
+	var rdr *bytes.Reader
+	if body != nil {
+		b, _ := json.Marshal(body)
+		rdr = bytes.NewReader(b)
+	} else {
+		rdr = bytes.NewReader(nil)
+	}
+	req := httptest.NewRequest(method, path, rdr)
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	s.router.ServeHTTP(rec, req)
+	out := map[string]any{}
+	_ = json.Unmarshal(rec.Body.Bytes(), &out)
+	return rec, out
+}
+
+// signAs produces the proof a browser would, from the raw key.
+func signAs(t *testing.T, privHex, challenge string) map[string]any {
+	t.Helper()
+	raw, err := hex.DecodeString(privHex)
+	if err != nil {
+		t.Fatalf("decode key: %v", err)
+	}
+	id, err := identity.Import(hex.EncodeToString(raw))
+	if err != nil {
+		t.Fatalf("import: %v", err)
+	}
+	sig, err := identity.Sign(id, []byte(challenge))
+	if err != nil {
+		t.Fatalf("sign: %v", err)
+	}
+	return map[string]any{
+		"challenge": challenge,
+		"signature": base64.StdEncoding.EncodeToString(sig),
+	}
+}
+
+func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
+	s := newTestServer(t)
+
+	rec, gen := do(t, s, "POST", "/api/identity/generate", nil)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("generate: %d %s", rec.Code, rec.Body)
+	}
+	uid := gen["uid"].(string)
+	priv := gen["privateKeyHex"].(string)
+
+	// Some data to lose.
+	for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} {
+		if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok {
+			t.Fatalf("seed: %v", err)
+		}
+	}
+
+	// The count the UI shows in the confirmation.
+	rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("stats: %d %s", rec.Code, rec.Body)
+	}
+	users := stats["users"].(map[string]any)
+	mine := users[uid].(map[string]any)
+	if mine["records"].(float64) != 3 || mine["cids"].(float64) != 2 {
+		t.Fatalf("stats = %v, want 3 records / 2 cids", mine)
+	}
+
+	// Unsigned delete must be refused, and must not have touched anything.
+	rec, body := do(t, s, "DELETE", "/api/identity", map[string]any{"uid": uid, "confirm": true})
+	if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
+		t.Fatalf("unsigned delete = %d %v, want 401 signature_required", rec.Code, body)
+	}
+	if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
+		t.Fatal("a refused delete removed the key anyway")
+	}
+
+	// meta-core must refuse to sign the caller's authorisation for them —
+	// without this the gate is decoration on a node holding every key.
+	ch, _, err := s.challenges.Issue(uid, identity.ActionDelete)
+	if err != nil {
+		t.Fatalf("issue: %v", err)
+	}
+	rec, body = do(t, s, "POST", "/api/identity/sign", map[string]any{
+		"uid":      uid,
+		"bytesB64": base64.StdEncoding.EncodeToString([]byte(ch)),
+	})
+	if rec.Code != http.StatusForbidden || body["error"] != "reserved_payload" {
+		t.Fatalf("sign of a challenge = %d %v, want 403 reserved_payload", rec.Code, body)
+	}
+
+	// A challenge minted for reveal must not authorise a delete.
+	rec, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
+	}
+	proof := signAs(t, priv, chBody["challenge"].(string))
+	req := map[string]any{"uid": uid, "confirm": true}
+	for k, v := range proof {
+		req[k] = v
+	}
+	rec, body = do(t, s, "DELETE", "/api/identity", req)
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("reveal challenge used for delete = %d %v, want 401", rec.Code, body)
+	}
+
+	// The real thing.
+	rec, chBody = do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
+	if rec.Code != http.StatusOK {
+		t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
+	}
+	proof = signAs(t, priv, chBody["challenge"].(string))
+	req = map[string]any{"uid": uid, "confirm": true}
+	for k, v := range proof {
+		req[k] = v
+	}
+	rec, body = do(t, s, "DELETE", "/api/identity", req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("signed delete = %d %s", rec.Code, rec.Body)
+	}
+	purged := body["purged"].(map[string]any)
+	if purged["records"].(float64) != 3 || purged["cids"].(float64) != 2 {
+		t.Fatalf("purged = %v, want 3 records / 2 cids", purged)
+	}
+
+	// Key gone.
+	if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
+		t.Fatal("the account file survived the delete")
+	}
+	// Data gone — the orphaning this whole change exists to prevent.
+	after, err := s.storage.UDLAllUserStats()
+	if err != nil {
+		t.Fatalf("stats after: %v", err)
+	}
+	if got, present := after[uid]; present {
+		t.Fatalf("deleted account still holds %+v", got)
+	}
+	for _, cid := range []string{"cidA", "cidB"} {
+		uids, err := s.storage.UDLCidUsers(cid)
+		if err != nil {
+			t.Fatalf("cid users: %v", err)
+		}
+		if len(uids) != 0 {
+			t.Fatalf("%s still names %v after the purge", cid, uids)
+		}
+	}
+
+	// The challenge is single-use, so replaying the same proof does nothing.
+	rec, _ = do(t, s, "DELETE", "/api/identity", req)
+	if rec.Code == http.StatusOK {
+		t.Fatal("a replayed proof must not succeed")
+	}
+}
+
+func TestE2E_RevealRequiresProof(t *testing.T) {
+	s := newTestServer(t)
+	_, gen := do(t, s, "POST", "/api/identity/generate", nil)
+	uid := gen["uid"].(string)
+	priv := gen["privateKeyHex"].(string)
+
+	rec, body := do(t, s, "POST", "/api/identity/reveal", map[string]any{"uid": uid, "confirm": true})
+	if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
+		t.Fatalf("unsigned reveal = %d %v, want 401 signature_required", rec.Code, body)
+	}
+
+	_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
+	proof := signAs(t, priv, chBody["challenge"].(string))
+	req := map[string]any{"uid": uid, "confirm": true}
+	for k, v := range proof {
+		req[k] = v
+	}
+	rec, body = do(t, s, "POST", "/api/identity/reveal", req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("signed reveal = %d %s", rec.Code, rec.Body)
+	}
+	if body["privateKeyHex"] != priv {
+		t.Fatal("reveal returned a different key than generate did")
+	}
+}
+
+// Another account's key must not authorise this one, even with a challenge
+// minted for the right uid and action.
+func TestE2E_AnotherAccountsKeyIsRefused(t *testing.T) {
+	s := newTestServer(t)
+	_, alice := do(t, s, "POST", "/api/identity/generate", nil)
+	_, mallory := do(t, s, "POST", "/api/identity/generate", nil)
+
+	uid := alice["uid"].(string)
+	_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
+	proof := signAs(t, mallory["privateKeyHex"].(string), chBody["challenge"].(string))
+	req := map[string]any{"uid": uid, "confirm": true}
+	for k, v := range proof {
+		req[k] = v
+	}
+	rec, body := do(t, s, "DELETE", "/api/identity", req)
+	if rec.Code != http.StatusUnauthorized || body["error"] != "bad_signature" {
+		t.Fatalf("mallory deleting alice = %d %v, want 401 bad_signature", rec.Code, body)
+	}
+	if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
+		t.Fatal("alice's account was deleted by mallory's signature")
+	}
+}

internal/api/metadata.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/gorilla/mux"
 )
@@ -39,6 +38,36 @@ type MetadataSearchRequest struct {
 	Property      string `json:"property,omitempty"`
 	PropertyValue string `json:"propertyValue,omitempty"`
 	Limit         int    `json:"limit,omitempty"`
+
+	// Filters is the structured form: AND across entries, OR within an
+	// entry's Values. Evaluated in memory against the search index alongside
+	// the free-text Query, so an id-anchored lookup costs a map probe per
+	// record rather than a Redis round-trip per record.
+	//
+	// Property/PropertyValue is the single-filter legacy shape and folds into
+	// this list (see normalizedFilters) with Exact=false, preserving its
+	// substring semantics byte-for-byte.
+	Filters []MetadataFilter `json:"filters,omitempty"`
+}
+
+// MetadataFilter is one structured constraint on a record.
+//
+// A record satisfies it when ANY of Values matches, and a search result must
+// satisfy EVERY filter (OR within, AND across). An entry with no usable value
+// constrains nothing — same rule the gateway's `genres_filter_matches` uses, so
+// a caller that lowers an empty filter doesn't accidentally empty the result.
+type MetadataFilter struct {
+	// Property is the field name, WITHOUT a key-set suffix: `genres`, not
+	// `genres/Action`. Both storage shapes are probed — see recordMatchesFilter.
+	Property string `json:"property"`
+	// Values are OR'd. Compared case-insensitively.
+	Values []string `json:"values,omitempty"`
+	// Exact selects CSV-exact comparison (the semantics meta-gateway's
+	// `any_csv_value_matches` applies at its own Layer-B re-filter) over
+	// substring containment. Callers doing identity lookups want this: with a
+	// result limit, a substring match on `tmdbid=1405` also admits `21405` and
+	// can crowd the real matches out of the page.
+	Exact bool `json:"exact,omitempty"`
 }
 
 // MetadataSearchResult is a single search result
@@ -193,6 +222,9 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Same resolution as the PUT below: callers hold a CID, not the root.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	metadata, err := s.storage.GetMetadataFlat(hashID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -227,6 +259,17 @@ func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	// The {hashId} in the URL is whatever CID the caller knows the file by —
+	// the gateway's auto-store keys this PUT by the hash it just computed
+	// (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is
+	// the dag-pb root, not the nzb-release cid the record was minted under.
+	// Without resolving, the write lands on file:<thatCid>/* and forks a
+	// second root for a file that already has one: a full duplicate record,
+	// cross-linked and separately indexed, which every downstream ingest then
+	// sees as two files. Unknown CIDs pass through unchanged, so a genuinely
+	// new gateway record still mints its root at the hash.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	var metadata map[string]string
 	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
 		writeError(w, http.StatusBadRequest, "invalid JSON body")
@@ -263,6 +306,12 @@ func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	// meta-gateway retracts a dropped hit by whichever CID it holds
+	// (meta_core.rs `delete`, which documents this resolution as a
+	// precondition). Without it, deleting by a non-root CID is a silent
+	// no-op and the retracted record lingers.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	deletedCount, err := s.storage.DeleteMetadata(hashID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -395,79 +444,18 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Get all hash IDs
+	// Everything else — free text, structured filters, both, or neither
-	allHashIDs, err := s.storage.GetAllHashIDs()
+	// (which returns the first `limit`) — is served from the in-memory search
-	if err != nil {
+	// index (search_index.go). Matching and the returned metadata both come
-		writeError(w, http.StatusInternalServerError, err.Error())
+	// from memory, so a query costs zero Redis round-trips.
-		return
+	//
-	}
+	// There is deliberately no second path. The property branch used to fall
-
+	// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which
-	var results []MetadataSearchResult
+	// is what made an id-anchored lookup 3.63s over ~20k records (vs 10ms for
-	total := len(allHashIDs)
+	// the free-text branch on the same corpus) and kept meta-core out of the
-
+	// gateway's anchored search path entirely. The index already holds every
-	// Search fields for query
+	// field of every record, so that scan was re-reading data it had in hand.
-	searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
+	s.serveIndexedSearch(w, req)
-
-	for _, hashID := range allHashIDs {
-		if len(results) >= req.Limit {
-			break
-		}
-
-		metadata, err := s.storage.GetMetadataFlat(hashID)
-		if err != nil || metadata == nil {
-			continue
-		}
-
-		matches := false
-
-		// Property search
-		if req.Property != "" && req.PropertyValue != "" {
-			value, ok := metadata[req.Property]
-			if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) {
-				matches = true
-			}
-		}
-
-		// General query search
-		if req.Query != "" && req.Property == "" {
-			queryLower := strings.ToLower(req.Query)
-
-			// Check if query matches hashId
-			if strings.Contains(strings.ToLower(hashID), queryLower) {
-				matches = true
-			}
-
-			// Check search fields
-			if !matches {
-				for _, field := range searchFields {
-					if value, ok := metadata[field]; ok {
-						if strings.Contains(strings.ToLower(value), queryLower) {
-							matches = true
-							break
-						}
-					}
-				}
-			}
-		}
-
-		// If no query or property filter, return all
-		if req.Query == "" && req.Property == "" {
-			matches = true
-		}
-
-		if matches {
-			results = append(results, MetadataSearchResult{
-				HashID:   hashID,
-				Metadata: metadata,
-			})
-		}
-	}
-
-	writeJSON(w, http.StatusOK, MetadataSearchResponse{
-		Results: results,
-		Count:   len(results),
-		Total:   total,
-	})
 }
 
 // handleBatchUpdate handles POST /api/metadata/batch

internal/api/search_filter_test.go

@@ -0,0 +1,211 @@
+package api
+
+import "testing"
+
+// Tests for the structured-filter matcher that lets meta-core answer an
+// id-anchored query from the in-memory search index (search_index.go).
+//
+// Why this matters beyond "filters work": meta-core is a *coarse pre-filter*
+// for meta-gateway, which re-evaluates every record it emits through its own
+// `query_eval::record_matches`. So the property that has to hold is that this
+// matcher stays a SUPERSET of that one — it may admit records the gateway will
+// drop, but it must never drop a record the gateway would have kept. Every
+// case below is written from that angle:
+//
+//   - key-set fields (`genres/Action`) must match a `genres:action` filter, or
+//     card-tier records vanish from a genre-scoped query;
+//   - a missing field fails CLOSED, matching record_matches;
+//   - an empty/unusable filter constrains NOTHING, so a caller that lowers a
+//     blank filter gets its results rather than an empty page.
+
+func f(property string, exact bool, values ...string) MetadataFilter {
+	return MetadataFilter{Property: property, Values: values, Exact: exact}
+}
+
+func TestRecordMatchesFilter(t *testing.T) {
+	fields := map[string]string{
+		"fileType":               "video",
+		"videoType":              "tv,anime",
+		"tmdbid":                 "1405",
+		"genres/Action":          "true",
+		"genres/Science Fiction": "true",
+		"source/gateway:nyaa.si": "true",
+		"categories/XXX":         "true",
+		"retracted":              "false",
+	}
+
+	cases := []struct {
+		name   string
+		filter MetadataFilter
+		want   bool
+	}{
+		// Flat scalar.
+		{"flat exact hit", f("fileType", true, "video"), true},
+		{"flat exact miss", f("fileType", true, "audio"), false},
+		{"flat exact is case-insensitive", f("fileType", true, "VIDEO"), true},
+
+		// csv-set field: each comma piece is compared independently.
+		{"csv member hit", f("videoType", true, "anime"), true},
+		{"csv member miss", f("videoType", true, "movie"), false},
+
+		// Exact vs contains — the reason Exact exists. A substring match on an
+		// identity field admits neighbours and, under a result limit, crowds
+		// the real matches off the page.
+		{"contains admits a superstring", f("tmdbid", false, "140"), true},
+		{"exact rejects a superstring", f("tmdbid", true, "140"), false},
+		{"exact accepts the whole value", f("tmdbid", true, "1405"), true},
+
+		// Key-set shape. The stored member is verbatim (`genres/Action`) while
+		// query values arrive lowercased, so the case-insensitive sweep is what
+		// makes a genre-scoped row work at all.
+		{"key-set exact casing", f("genres", true, "Action"), true},
+		{"key-set lowercased query", f("genres", true, "action"), true},
+		{"key-set with a space", f("genres", true, "science fiction"), true},
+		{"key-set miss", f("genres", true, "horror"), false},
+		{"key-set label containing a colon", f("source", true, "gateway:nyaa.si"), true},
+
+		// OR within one filter's values.
+		{"or hits on the second value", f("fileType", true, "audio", "video"), true},
+		{"or misses on all values", f("fileType", true, "audio", "image"), false},
+
+		// Fail-closed on an absent field — mirrors record_matches, and is what
+		// keeps a filtered query from returning unrelated records.
+		{"absent field fails closed", f("imdbid", true, "tt0903747"), false},
+
+		// A filter that constrains nothing must not empty the result.
+		{"no values constrains nothing", f("fileType", true), true},
+		{"blank value constrains nothing", f("fileType", true, "  "), true},
+		{"no property constrains nothing", f("", true, "video"), true},
+
+		// A FLAT field whose value happens to be "false" is still a value, and
+		// an exact filter for it matches. Only the key-set arm treats "false"
+		// as non-membership (pinned in the test below).
+		{"flat field valued false still matches", f("retracted", true, "false"), true},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := recordMatchesFilter(fields, &tc.filter); got != tc.want {
+				t.Fatalf("recordMatchesFilter(%+v) = %v, want %v", tc.filter, got, tc.want)
+			}
+		})
+	}
+}
+
+// `retracted = "false"` is a flat field, not a key-set member, so the flat arm
+// legitimately matches the literal string "false". The key-set arm is what must
+// treat it as non-membership — pinned separately so the two arms don't get
+// conflated if either is refactored.
+func TestFalseyKeySetMemberIsNotMembership(t *testing.T) {
+	fields := map[string]string{"genres/Action": "false", "genres/Drama": "0"}
+	for _, v := range []string{"Action", "Drama"} {
+		filter := f("genres", true, v)
+		if recordMatchesFilter(fields, &filter) {
+			t.Fatalf("genres/%s with a falsey value must not assert membership", v)
+		}
+	}
+}
+
+func TestRecordMatchesFiltersIsAndAcrossEntries(t *testing.T) {
+	fields := map[string]string{"fileType": "video", "season": "2"}
+
+	all := []MetadataFilter{f("fileType", true, "video"), f("season", true, "2")}
+	if !recordMatchesFilters(fields, all) {
+		t.Fatal("a record satisfying every filter must match")
+	}
+
+	one := []MetadataFilter{f("fileType", true, "video"), f("season", true, "3")}
+	if recordMatchesFilters(fields, one) {
+		t.Fatal("filters are AND across entries — one miss must reject the record")
+	}
+
+	if !recordMatchesFilters(fields, nil) {
+		t.Fatal("no filters must match everything")
+	}
+}
+
+// The legacy Property/PropertyValue pair keeps its substring semantics: it is
+// the shape existing callers already use, and tightening it silently would
+// change results for them. New callers pass Filters and opt into Exact.
+func TestNormalizedFiltersFoldsTheLegacyPair(t *testing.T) {
+	got := normalizedFilters(MetadataSearchRequest{Property: "tmdbid", PropertyValue: "1405"})
+	if len(got) != 1 {
+		t.Fatalf("legacy pair must fold into exactly one filter, got %d", len(got))
+	}
+	if got[0].Property != "tmdbid" || len(got[0].Values) != 1 || got[0].Values[0] != "1405" {
+		t.Fatalf("legacy pair folded incorrectly: %+v", got[0])
+	}
+	if got[0].Exact {
+		t.Fatal("the legacy pair must stay substring (Exact=false) for compatibility")
+	}
+
+	// A half-specified pair is not a filter — it must not become one that
+	// matches nothing and empties the response.
+	if n := len(normalizedFilters(MetadataSearchRequest{Property: "tmdbid"})); n != 0 {
+		t.Fatalf("a property with no value must not fold into a filter, got %d", n)
+	}
+	if n := len(normalizedFilters(MetadataSearchRequest{PropertyValue: "1405"})); n != 0 {
+		t.Fatalf("a value with no property must not fold into a filter, got %d", n)
+	}
+
+	both := normalizedFilters(MetadataSearchRequest{
+		Property:      "tmdbid",
+		PropertyValue: "1405",
+		Filters:       []MetadataFilter{f("fileType", true, "video")},
+	})
+	if len(both) != 2 {
+		t.Fatalf("the legacy pair must be appended to explicit filters, got %d", len(both))
+	}
+}
+
+// The acceptance bar for routing the anchored path through the index: a
+// full-miss filtered sweep over a realistic corpus must stay in the same class
+// as the free-text branch (measured 10 ms over 19 622 live records), not the
+// legacy per-record Redis scan it replaces (measured 3.63 s over the same
+// corpus). Full-miss is the worst case — the loop cannot short-circuit on the
+// result limit, so every record is evaluated.
+//
+// Run: go test ./internal/api/ -bench BenchmarkFilterSweep -benchtime 10x
+func BenchmarkFilterSweep(b *testing.B) {
+	const records = 20000
+	corpus := make([]map[string]string, records)
+	for i := range corpus {
+		corpus[i] = map[string]string{
+			"title":                  "Some Release",
+			"fileType":               "video",
+			"videoType":              "tv",
+			"tmdbid":                 "1405",
+			"season":                 "2",
+			"genres/Action":          "true",
+			"genres/Drama":           "true",
+			"source/gateway:nyaa.si": "true",
+			"categories/TV/Anime":    "true",
+			"indexer":                "nyaa.si",
+			"fileName":               "some.release.s02e05.mkv",
+			"description/eng":        "text",
+			"poster":                 "bafy",
+			"sizeByte":               "123456789",
+			"contentKind":            "episode",
+			"anchoredBy/tmdb:1405":   "true",
+		}
+	}
+	// A miss on the identity field: the shape an id-anchored gateway query
+	// takes, and the one that must not degrade.
+	filters := []MetadataFilter{
+		f("tmdbid", true, "999999"),
+		f("fileType", true, "video"),
+		f("genres", true, "action"),
+	}
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		hits := 0
+		for _, fields := range corpus {
+			if recordMatchesFilters(fields, filters) {
+				hits++
+			}
+		}
+		if hits != 0 {
+			b.Fatalf("expected a full miss, got %d hits", hits)
+		}
+	}
+}

internal/api/search_index.go

@@ -0,0 +1,342 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
+// from Redis in the background. Records ingest continuously, so a newly stored
+// record becomes searchable within one interval; the rebuild is a single
+// batched bulk read (storage.GetSearchIndexTuples), off the request path.
+const searchIndexRefreshInterval = 60 * time.Second
+
+// searchIndexReconcileEvery is how many refresh ticks pass between full
+// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full
+// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all.
+const searchIndexReconcileEvery = 30
+
+// searchIndex is an in-memory snapshot of every record's search haystack
+// (hashID + lowercased searchable fields), kept current from Redis. It exists
+// so POST /api/metadata/search matches in memory (string ops over a slice)
+// instead of doing a SCAN+MGET Redis round-trip per record — the latter took
+// 40s+ over ~72k records and made a live parallel search path unusable.
+//
+// # Two update paths, and why
+//
+// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a
+// complete fresh copy. That is O(corpus) regardless of how much changed, and it
+// used to run every 60 s forever. Measured on a 19 622-record staging corpus
+// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side
+// held 783 MB for data Redis stores in 135 MB — largely the double copy.
+//
+// `refreshIncremental` re-reads only the records that changed, tracked by
+// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady
+// state; the full rebuild runs once at startup (there is nothing to be
+// incremental against) and then only as an explicit repair.
+//
+// The distinction matters because meta-core is the platform's permanent,
+// additive store — every search hit on every gateway is retained — so a rebuild
+// priced against the whole corpus is a cost that only ever grows.
+type searchIndex struct {
+	mu sync.RWMutex
+	// byID is the authoritative content; tuples is a stable-order view of it
+	// rebuilt on each update. Map iteration order in Go is randomised, so
+	// serving a limited query straight off the map would return a different
+	// subset each call — the slice is what keeps results stable.
+	byID   map[string]storage.SearchTuple
+	tuples []storage.SearchTuple
+	ready  atomic.Bool
+}
+
+// snapshot returns the current tuples slice and whether the index has built at
+// least once. Every update swaps in a fresh slice under the write lock (never
+// mutates in place), so the returned slice is safe to read lock-free.
+func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return idx.tuples, idx.ready.Load()
+}
+
+// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds
+// the write lock.
+func (idx *searchIndex) rebuildViewLocked() {
+	view := make([]storage.SearchTuple, 0, len(idx.byID))
+	for _, t := range idx.byID {
+		view = append(view, t)
+	}
+	// Stable across rebuilds so a limited query returns a consistent page
+	// rather than a fresh random subset every interval.
+	sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID })
+	idx.tuples = view
+}
+
+// refresh rebuilds the whole index from Redis (a batched bulk read). Used for
+// the initial build and as an explicit repair — not the steady state.
+func (idx *searchIndex) refresh(stor *storage.Client) {
+	if stor == nil || !stor.IsConnected() {
+		return
+	}
+	tuples, err := stor.GetSearchIndexTuples()
+	if err != nil {
+		log.Printf("[API] search index refresh failed: %v", err)
+		return
+	}
+	byID := make(map[string]storage.SearchTuple, len(tuples))
+	for _, t := range tuples {
+		byID[t.HashID] = t
+	}
+	idx.mu.Lock()
+	idx.byID = byID
+	idx.rebuildViewLocked()
+	idx.mu.Unlock()
+	idx.ready.Store(true)
+}
+
+// refreshIncremental drains the dirty set and patches only those records.
+// Returns whether more dirty entries remain, so a burst is caught up over
+// consecutive passes rather than one per interval.
+//
+// A record in the dirty set but absent from storage was deleted and is evicted
+// here — otherwise it would keep answering searches until the next full
+// rebuild. A batch that fails after the (destructive) drain is re-marked by
+// storage so it is retried rather than silently lost.
+func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) {
+	if stor == nil || !stor.IsConnected() {
+		return false
+	}
+	ids, more, err := stor.DrainDirty()
+	if err != nil {
+		log.Printf("[API] search index drain failed: %v", err)
+		return false
+	}
+	if len(ids) == 0 {
+		return false
+	}
+	tuples, gone, err := stor.GetSearchIndexTuplesFor(ids)
+	if err != nil {
+		stor.MarkDirty(ids...)
+		log.Printf("[API] incremental search index refresh failed: %v", err)
+		return false
+	}
+
+	idx.mu.Lock()
+	if idx.byID == nil {
+		idx.byID = make(map[string]storage.SearchTuple, len(tuples))
+	}
+	for _, t := range tuples {
+		idx.byID[t.HashID] = t
+	}
+	for _, id := range gone {
+		delete(idx.byID, id)
+	}
+	idx.rebuildViewLocked()
+	idx.mu.Unlock()
+
+	log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples))
+	return more
+}
+
+// runRefreshLoop builds the index once in full, then keeps it current from the
+// dirty set, with a periodic full reconcile as a safety net.
+//
+// The reconcile exists because a *missed* dirty mark is silent: no error, just
+// a record whose indexed copy quietly diverges from storage until something
+// forces a rebuild. Marking is spread across eight write paths plus anything
+// added later, and the cost of getting it wrong is invisible — so the index
+// should not depend on that coverage being perfect forever. Every
+// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs
+// whatever drifted.
+//
+// That still drops the full rebuild from every 60 s to every 30 min — a ~30×
+// reduction in the steady-state cost that measured at ~45 % of all Redis CPU —
+// while keeping the self-healing property that made the naive loop safe.
+func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
+	idx.refresh(stor)
+	ticker := time.NewTicker(searchIndexRefreshInterval)
+	defer ticker.Stop()
+	ticks := 0
+	for range ticker.C {
+		ticks++
+		if ticks%searchIndexReconcileEvery == 0 {
+			idx.refresh(stor)
+			continue
+		}
+		// Drain until caught up, so a burst larger than one batch does not take
+		// one interval per batch to land.
+		for idx.refreshIncremental(stor) {
+		}
+	}
+}
+
+// serveIndexedSearch answers a search from the in-memory index: a token-AND
+// match against each record's haystack (or a whole-query hashID substring,
+// preserving the legacy behaviour) AND every structured filter, then returns
+// the full metadata for the <=limit survivors. No Redis round-trip per scanned
+// record — matching and results both come from memory.
+//
+// This is the ONLY search path. Free text, filters, both, or neither all land
+// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
+func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
+	tuples, ready := s.search.snapshot()
+	if !ready {
+		// First call before the background build finished: warm inline with the
+		// efficient batched read (seconds), never the per-record SCAN fallback.
+		s.search.refresh(s.storage)
+		tuples, _ = s.search.snapshot()
+	}
+	tokens := strings.Fields(strings.ToLower(req.Query))
+	queryLower := strings.ToLower(req.Query)
+	filters := normalizedFilters(req)
+	results := make([]MetadataSearchResult, 0, req.Limit)
+	for _, t := range tuples {
+		if len(results) >= req.Limit {
+			break
+		}
+		if req.Query != "" &&
+			!matchAllTokens(t.Haystack, tokens) &&
+			!strings.Contains(strings.ToLower(t.HashID), queryLower) {
+			continue
+		}
+		if !recordMatchesFilters(t.Fields, filters) {
+			continue
+		}
+		// Full metadata is cached in the index — return it straight from memory,
+		// no Redis round-trip per result.
+		results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
+	}
+	writeJSON(w, http.StatusOK, MetadataSearchResponse{
+		Results: results,
+		Count:   len(results),
+		Total:   len(tuples),
+	})
+}
+
+// matchAllTokens reports whether haystack contains every token (AND semantics).
+// An empty token list matches everything.
+func matchAllTokens(haystack string, tokens []string) bool {
+	for _, t := range tokens {
+		if !strings.Contains(haystack, t) {
+			return false
+		}
+	}
+	return true
+}
+
+// normalizedFilters returns the request's structured filters with the legacy
+// Property/PropertyValue pair folded in as one more entry.
+//
+// The legacy pair keeps Exact=false so its long-standing substring semantics
+// are unchanged for existing callers; new callers pass Filters and choose.
+func normalizedFilters(req MetadataSearchRequest) []MetadataFilter {
+	if req.Property == "" || req.PropertyValue == "" {
+		return req.Filters
+	}
+	out := make([]MetadataFilter, 0, len(req.Filters)+1)
+	out = append(out, req.Filters...)
+	return append(out, MetadataFilter{
+		Property: req.Property,
+		Values:   []string{req.PropertyValue},
+	})
+}
+
+// recordMatchesFilters reports whether fields satisfies every filter (AND).
+func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool {
+	for i := range filters {
+		if !recordMatchesFilter(fields, &filters[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// recordMatchesFilter reports whether fields satisfies one filter (OR within
+// its values), probing BOTH storage shapes a field can take:
+//
+//   - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`;
+//   - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"`
+//     (METADATA_KEYS "How the store is shaped").
+//
+// Probing both is what lets one filter shape serve the whole vocabulary. The
+// gateway needs a `genres:action` filter to find a record whose genre lives in
+// the key `genres/Action`, and a `fileType:video` filter to find one whose value
+// lives in the field `fileType` — without meta-core knowing which fields are
+// key-sets. It also means this stays a SUPERSET of the gateway's own
+// `record_matches`, which is the contract: meta-core pre-filters, the gateway
+// re-filters authoritatively.
+//
+// A filter with no usable value constrains nothing (matches). A filter whose
+// field is absent fails closed, matching `record_matches`.
+func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool {
+	values := make([]string, 0, len(f.Values))
+	for _, v := range f.Values {
+		if v = strings.TrimSpace(v); v != "" {
+			values = append(values, v)
+		}
+	}
+	if f.Property == "" || len(values) == 0 {
+		return true
+	}
+
+	// Key-set shape: `<property>/<value>` present and truthy. Try the exact
+	// key first — that hits whenever the caller's casing matches what was
+	// written, which is the common case and costs one map probe.
+	for _, v := range values {
+		if isTruthyMember(fields[f.Property+"/"+v]) {
+			return true
+		}
+	}
+	// Members are written verbatim (`genres/Action`) while query values arrive
+	// lowercased, so fall back to one case-insensitive sweep. Single pass over
+	// the record's fields for ALL values, not one pass per value.
+	wanted := make(map[string]struct{}, len(values))
+	for _, v := range values {
+		wanted[strings.ToLower(v)] = struct{}{}
+	}
+	prefix := strings.ToLower(f.Property) + "/"
+	for k, val := range fields {
+		if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) {
+			continue
+		}
+		if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) {
+			return true
+		}
+	}
+
+	// Flat shape: the field's own value, split on `,` for csv-set fields.
+	fieldValue, ok := fields[f.Property]
+	if !ok {
+		return false
+	}
+	for _, piece := range strings.Split(fieldValue, ",") {
+		piece = strings.TrimSpace(piece)
+		if piece == "" {
+			continue
+		}
+		for _, v := range values {
+			if f.Exact {
+				if strings.EqualFold(piece, v) {
+					return true
+				}
+			} else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// isTruthyMember reports whether a key-set member's value asserts membership.
+// Mirrors the gateway's `is_truthy_member`: presence with a non-empty,
+// non-"false"/"0" value.
+func isTruthyMember(v string) bool {
+	v = strings.TrimSpace(strings.ToLower(v))
+	return v != "" && v != "false" && v != "0"
+}

internal/api/server.go

@@ -9,10 +9,10 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
 	"github.com/metazla/meta-core/internal/events"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/mounts"
 	"github.com/metazla/meta-core/internal/schema"
 	"github.com/metazla/meta-core/internal/storage"
@@ -25,8 +25,6 @@ import (
 type Server struct {
 	config            *config.Config
 	leaderProvider    *leader.LeaderInfoProvider
-	discovery         *discovery.Service
-	cleaner           *discovery.Cleaner
 	storage           *storage.Client
 	mountsManager     *mounts.Manager
 	mountsHandlers    *mounts.Handlers
@@ -41,25 +39,41 @@ type Server struct {
 	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
-	router            *mux.Router
+	search            *searchIndex
-	server            *http.Server
+	// challenges backs proof-of-possession on reveal/delete. In-process, so a
+	// restart invalidates the ones in flight — see identity/challenge.go.
+	challenges *identity.ChallengeStore
+	// mesh is the UDP discovery node (beacon v2). nil when discovery
+	// is disabled or failed to bind, in which case /api/neighbors reports an
+	// empty list rather than erroring — a dashboard nav is decoration and
+	// must not take the API down with it.
+	mesh   *meshdisco.Node
+	router *mux.Router
+	server *http.Server
+	// stats caches the keyspace-walk half of GET /api/stats so a polling
+	// dashboard never puts a multi-second Redis walk on the request path.
+	stats *statsCache
+}
+
+// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
+// server's constructor signature (and its tests) stay untouched.
+func (s *Server) SetMeshNode(n *meshdisco.Node) {
+	s.mesh = n
 }
 
 // NewServer creates a new API server
 func NewServer(
 	cfg *config.Config,
 	leaderProvider *leader.LeaderInfoProvider,
-	disc *discovery.Service,
-	cleaner *discovery.Cleaner,
 	stor *storage.Client,
 ) *Server {
 	s := &Server{
 		config:         cfg,
 		leaderProvider: leaderProvider,
-		discovery:      disc,
-		cleaner:        cleaner,
 		storage:        stor,
+		search:         &searchIndex{},
 		router:         mux.NewRouter(),
+		stats:          &statsCache{},
 	}
 
 	// Initialize file watcher/scanner and dispatcher
@@ -137,6 +151,8 @@ func NewServer(
 	// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
 	s.webdavHandler = webdav.NewHandler(cfg)
 
+	s.challenges = identity.NewChallengeStore()
+
 	// One-time migration: fold a pre-multi-account identity.json into the
 	// per-account keystore so existing single-identity hosts keep their uid.
 	if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
@@ -157,6 +173,10 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
 	s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
 
+	// Dashboard headline counters in one request. Cheap half live, keyspace
+	// walks served from a background-refreshed cache — see stats.go.
+	s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
+
 	// Metadata Editor API routes (must be before /meta/{hash} routes)
 	s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
 	s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
@@ -168,10 +188,13 @@ func (s *Server) setupRoutes() {
 	// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
 	s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
 	s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
+	s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
 	s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
 	s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
+	s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
 	s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
 	s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
+	s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
 	s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
 
 	// Snapshot (export / import / wipe)
@@ -199,6 +222,19 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
 	s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
 
+	// User Data Layer (UDL) routes — per-user signed records (My List,
+	// Continue Watching, ratings, profile). See internal/api/udl.go.
+	s.router.HandleFunc("/api/udl/record", s.handleUDLRecordGet).Methods("GET")
+	s.router.HandleFunc("/api/udl/record", s.handleUDLRecordPut).Methods("PUT")
+	s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
+	s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
+	s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
+	s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
+	s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
+
 	// Metadata operations - base endpoints
 	s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
 	s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
@@ -222,6 +258,14 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
 	s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
 	s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
+	// Register one file under /files/plugin by midhash — the watcher's
+	// mint + alias + file:events path for a single file. Registered
+	// unconditionally: it exists for boxes whose watcher is OFF.
+	s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST")
+	// File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched
+	// server-side, so clients answer "how many files, how big" without one
+	// read per hashId. ?summary=1 returns only {count, totalSize}.
+	s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET")
 
 	// Public metadata-by-CID resolver. Reverse-index lookup → full document.
 	// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
@@ -232,6 +276,13 @@ func (s *Server) setupRoutes() {
 	// (the dual-root pattern caused by historical writes that bypassed CID
 	// resolution).
 	s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
+	// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
+	// (METADATA_KEYS.md §14.17). Idempotent.
+	s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
+	s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST")
+	// One-shot sweep of query echoes on literature cards (meta-read
+	// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
+	s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
 
 	// SSE event streams. Mediated mirror of the Redis Streams; consumers
 	// reach them over HTTP instead of touching Redis directly. Auth-bypass
@@ -240,39 +291,30 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
 	s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
 
-	// Service discovery
+	// beacon v2: neighbours heard over UDP. Supersedes /api/services,
-	// Both /services and /api/services are supported for consistency with other services
+	// which is still served above while consumers migrate.
-	s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
+	s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
-	s.router.HandleFunc("/api/services", s.handleListServices).Methods("GET")
+	s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
-	s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
-	s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
 
 	// Mount management routes (if manager initialized)
 	if s.mountsHandlers != nil {
 		s.mountsHandlers.RegisterRoutes(s.router)
-		log.Println("[API] Mount management routes registered at /api/mounts/*")
 	}
 
-	log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
-	log.Println("[API] KV Browser routes registered at /api/kv/*")
-
 	// File watcher routes (if watcher initialized)
 	if s.watcherHandlers != nil {
 		s.watcherHandlers.RegisterRoutes(s.router)
-		log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
 	}
 
 	// Watchers management routes (if watchers initialized)
 	if s.watchersHandlers != nil {
 		s.watchersHandlers.RegisterRoutes(s.router)
-		log.Println("[API] Watchers management routes registered at /api/watchers/*")
 	}
 
 	// WebDAV handler (caching is handled by nginx proxy_cache)
 	if s.webdavHandler != nil {
 		s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
 		s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
-		log.Println("[API] WebDAV routes registered at /webdav/*")
 	}
 
 	// Add middleware
@@ -321,6 +363,13 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
+	// Build + periodically refresh the in-memory search index so
+	// POST /api/metadata/search matches in memory instead of a Redis
+	// round-trip per record (see search_index.go).
+	if s.storage != nil {
+		go s.search.runRefreshLoop(s.storage)
+	}
+
 	// Start meta publisher (if storage is connected)
 	// Publishes keyspace notifications to meta:events stream
 	if s.storage != nil && s.storage.IsConnected() {
@@ -339,6 +388,21 @@ func (s *Server) Start() error {
 
 			// Republish events for all existing metadata
 			go func() {
+				// Build any missing per-record field indexes first, in ONE
+				// keyspace pass. The republish below reads every record, and
+				// an un-indexed record read costs a full-keyspace SCAN — on a
+				// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
+				// paid again on every restart). Warming first turns the whole
+				// republish into index-served reads. Idempotent, so a
+				// fully-indexed store just pays the single scan.
+				// See storage/field_index.go.
+				started := time.Now()
+				if n, err := s.storage.WarmFieldIndexes(); err != nil {
+					log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
+				} else if n > 0 {
+					log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
+				}
+
 				count, err := s.RepublishMetadata()
 				if err != nil {
 					log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)
@@ -421,12 +485,49 @@ func (s *Server) RepublishMetadata() (int, error) {
 	return s.metaPublisher.RepublishAllMetadata(republishFunc)
 }
 
-// loggingMiddleware logs all requests
+// statusRecorder captures the response status so loggingMiddleware can stay
+// silent on success and only report failures. It forwards Flush/Unwrap so SSE
+// and other streaming handlers keep working through the wrapper.
+type statusRecorder struct {
+	http.ResponseWriter
+	status  int
+	written bool
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	if !r.written {
+		r.status = code
+		r.written = true
+	}
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (r *statusRecorder) Write(b []byte) (int, error) {
+	if !r.written {
+		r.status = http.StatusOK
+		r.written = true
+	}
+	return r.ResponseWriter.Write(b)
+}
+
+func (r *statusRecorder) Flush() {
+	if f, ok := r.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
+
+// loggingMiddleware logs a request only when it fails (status >= 400), keeping
+// the nominal case quiet. Successful requests are not logged.
 func loggingMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
-		next.ServeHTTP(w, r)
+		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
-		log.Printf("[API] %s %s %s", r.Method, r.URL.Path, time.Since(start))
+		next.ServeHTTP(rec, r)
+		if rec.status >= 400 {
+			log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
+		}
 	})
 }
 

internal/api/stats.go

@@ -0,0 +1,164 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/metazla/meta-core/internal/identity"
+)
+
+// statsSweepTTL is how long a keyspace-walk result stays good. The walk is
+// never on the request path (see statsCache), so this only controls how stale
+// the library tiles may look, not how slow the endpoint is.
+const statsSweepTTL = 60 * time.Second
+
+// fileStatsJSON is the file-backed half of the library: the roots that carry
+// filePath+sizeByte+mtimeNano. Gateway / cid-rooted records have no file on
+// disk and are counted in Records, not here — the gap between the two numbers
+// is exactly "metadata we hold for files we do not have".
+type fileStatsJSON struct {
+	Count     int   `json:"count"`
+	TotalSize int64 `json:"totalSize"`
+}
+
+// StatsResponse is the body of GET /api/stats — every headline number the
+// dashboard shows, in one request.
+//
+// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
+// O(1) commands (SCARD, DBSIZE, INFO) plus a directory listing, recomputed on
+// every call. Files and UDLUsers need a keyspace walk — ~0.5s for the file
+// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
+// from a background-refreshed cache and are null until the first sweep lands.
+type StatsResponse struct {
+	Records     int64          `json:"records"`
+	RedisKeys   int64          `json:"redisKeys"`
+	RedisMemory string         `json:"redisMemory"`
+	Identities  int            `json:"identities"`
+	Files       *fileStatsJSON `json:"files"`
+	UDLUsers    *int           `json:"udlUsers"`
+	// SweptAt is when the cached half was computed (unix ms), 0 if never.
+	SweptAt int64 `json:"sweptAt"`
+	// Sweeping reports that a refresh is in flight, so the UI can show the
+	// cached numbers as provisional instead of pretending they are live.
+	Sweeping bool `json:"sweeping"`
+}
+
+// statsCache holds the expensive half of /api/stats.
+//
+// Why a cache and not just "compute it": the dashboard polls. A 3s Redis walk
+// per tick, per open tab, is a self-inflicted outage on a box that is also
+// serving playback — the same shape as meta-sort's /api/stats full sweep. So
+// the handler NEVER waits on a walk: it returns whatever the last sweep
+// produced and kicks off a new one if the old one is stale. `running` makes
+// that single-flight, so ten tabs cost one walk.
+type statsCache struct {
+	mu       sync.Mutex
+	files    *fileStatsJSON
+	udlUsers *int
+	sweptAt  time.Time
+	running  bool
+}
+
+// snapshot returns the cached values and whether a refresh is in flight.
+func (c *statsCache) snapshot() (*fileStatsJSON, *int, time.Time, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.files, c.udlUsers, c.sweptAt, c.running
+}
+
+// refreshIfStale starts a background sweep unless one is already running or
+// the last result is still fresh. Returns true if it started one.
+func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
+	c.mu.Lock()
+	if c.running || time.Since(c.sweptAt) < statsSweepTTL {
+		c.mu.Unlock()
+		return false
+	}
+	c.running = true
+	c.mu.Unlock()
+
+	go func() {
+		files, udl := sweep()
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.running = false
+		// A failed sweep leaves the previous numbers in place rather than
+		// blanking the tiles; sweptAt still advances so a hard-failing walk
+		// (Redis down) does not spin once per poll.
+		c.sweptAt = time.Now()
+		if files != nil {
+			c.files = files
+		}
+		if udl != nil {
+			c.udlUsers = udl
+		}
+	}()
+	return true
+}
+
+// handleStats handles GET /api/stats.
+func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	resp := StatsResponse{}
+
+	if n, err := s.storage.CountRoots(); err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	} else {
+		resp.Records = n
+	}
+	if n, err := s.storage.CountKeys(); err == nil {
+		resp.RedisKeys = n
+	}
+	if mem, err := s.storage.GetMemoryInfo(); err == nil {
+		resp.RedisMemory = mem
+	} else {
+		resp.RedisMemory = "N/A"
+	}
+	if list, err := identity.List(s.config.IdentityAccountsDir()); err == nil {
+		resp.Identities = len(list)
+	}
+
+	s.stats.refreshIfStale(s.sweepStats)
+	files, udl, sweptAt, sweeping := s.stats.snapshot()
+	resp.Files = files
+	resp.UDLUsers = udl
+	resp.Sweeping = sweeping
+	if !sweptAt.IsZero() {
+		resp.SweptAt = sweptAt.UnixMilli()
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// sweepStats runs the two keyspace walks behind the cached tiles. Called off
+// the request path only. A failing half returns nil so the other half still
+// lands.
+func (s *Server) sweepStats() (*fileStatsJSON, *int) {
+	var files *fileStatsJSON
+	if tuples, err := s.storage.GetTuplesForAllFiles(); err != nil {
+		log.Printf("[API] stats: file tuple sweep failed: %v", err)
+	} else {
+		fs := fileStatsJSON{Count: len(tuples)}
+		for _, t := range tuples {
+			fs.TotalSize += t.Size
+		}
+		files = &fs
+	}
+
+	var udl *int
+	if stats, err := s.storage.UDLAllUserStats(); err != nil {
+		log.Printf("[API] stats: UDL user sweep failed: %v", err)
+	} else {
+		n := len(stats)
+		udl = &n
+	}
+
+	return files, udl
+}

internal/api/udl.go

@@ -0,0 +1,454 @@
+package api
+
+// User Data Layer (UDL) HTTP surface.
+//
+// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
+// profile) now that the authoritative store lives in meta-core instead of
+// meta-watch's local op-log. Records arrive already signed (and, for
+// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
+// version-gates and indexes them but never inspects the crypto. See
+// internal/storage/udl.go for the Redis model.
+
+import (
+	"encoding/json"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// UDLPutRequest is the body of PUT /api/udl/record.
+type UDLPutRequest struct {
+	Uid     string           `json:"uid"`
+	Cid     string           `json:"cid"`
+	Key     string           `json:"key"`
+	Version int64            `json:"version"`
+	Ts      int64            `json:"ts"`
+	Record  string           `json:"record"` // base64 CBOR, opaque + signed
+	Value   *json.RawMessage `json:"value,omitempty"`
+	// Tombstone marks this write as a deletion of (uid, cid, key). The record
+	// itself is opaque here — meta-core never decodes the CBOR, and a private-
+	// tier value and a tombstone are equally "no plaintext value" — so the
+	// writer has to say. Without it the cid is never removed from
+	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
+	// account and is returned in full on every My List / Continue Watching
+	// read. Omitted by older writers, which keeps the previous behaviour.
+	Tombstone bool `json:"tombstone,omitempty"`
+}
+
+// UDLPutResponse is the response of PUT /api/udl/record.
+type UDLPutResponse struct {
+	Accepted bool `json:"accepted"`
+}
+
+// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
+// request. Each entry is exactly a UDLPutRequest, so a client can build the
+// same signed record for either endpoint.
+type UDLPutBatchRequest struct {
+	Records []UDLPutRequest `json:"records"`
+}
+
+// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
+// means the write was stale (the single-record endpoint's 409), which in a
+// batch is an ordinary outcome rather than a failure.
+type UDLPutBatchResult struct {
+	Key      string `json:"key"`
+	Accepted bool   `json:"accepted"`
+	Error    string `json:"error,omitempty"`
+}
+
+// UDLPutBatchResponse mirrors the request order index-for-index.
+type UDLPutBatchResponse struct {
+	Results []UDLPutBatchResult `json:"results"`
+}
+
+// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
+// request is held open; sized to match the sign batch on the other side of the
+// same operation (see maxSignBatch).
+const maxUDLBatch = 1000
+
+// UDLRecordResponse is the response of GET /api/udl/record.
+type UDLRecordResponse struct {
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+	Exists  bool   `json:"exists"`
+	// Value is the plaintext JSON scalar stored beside the opaque record, and
+	// is present only for PUBLIC-tier keys (like, rating, profile:name,
+	// profile:avatar). json.RawMessage rather than string because the field is
+	// generic over keys — a bool for `like`, a number for `rating`, a string
+	// for a profile name (see aggregatePublicValues) — and because it is the
+	// exact type UDLPutRequest.Value uses, so a value written through PUT reads
+	// back byte-identical.
+	//
+	// Absent, never null, when there is no plaintext: private-tier cells,
+	// tombstones and never-written cells must not read as an empty public
+	// value. Readers key off presence, not `exists`.
+	Value *json.RawMessage `json:"value,omitempty"`
+}
+
+// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
+// field names; `avg` is null for boolean keys like `like`), so the meta-watch
+// client deserializes it into the same struct it serializes to the UI.
+type UDLAggregate struct {
+	Key        string   `json:"key"`
+	Count      int      `json:"count"`
+	Avg        *float64 `json:"avg"`
+	Trues      int      `json:"trues"`
+	SampleSize int      `json:"sample_size"`
+}
+
+// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
+func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	q := r.URL.Query()
+	uid, cid, key := q.Get("uid"), q.Get("cid"), q.Get("key")
+	if uid == "" || cid == "" || key == "" {
+		writeError(w, http.StatusBadRequest, "uid, cid and key are required")
+		return
+	}
+
+	cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	resp := UDLRecordResponse{
+		Record:  cell.Record,
+		Version: cell.Version,
+		Ts:      cell.Ts,
+		Exists:  exists,
+	}
+	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
+	// Redis, a future writer bug) would otherwise fail mid-encode and emit a
+	// truncated body instead of a readable error.
+	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
+		raw := json.RawMessage(cell.Value)
+		resp.Value = &raw
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
+// (version <= stored) returns 409 stale_version so the client can re-read and
+// retry with a higher version.
+func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	var req UDLPutRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
+		writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
+		return
+	}
+
+	publicValue := ""
+	hasValue := false
+	if req.Value != nil {
+		publicValue = string(*req.Value)
+		hasValue = true
+	}
+
+	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if !accepted {
+		writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
+		return
+	}
+	writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
+}
+
+// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
+// handleUDLRecordPut.
+//
+// Exists for the bulk verbs above this layer ("mark this whole series seen"),
+// where one user action produces one cell per episode. Each cell still goes
+// through the same version-gated Lua upsert, so the CRDT semantics are
+// identical — this batches the transport only.
+//
+// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
+// replaces the single-record conflict status: in a batch, one stale cell is a
+// normal outcome (another device already marked that episode) and must not fail
+// the 299 cells beside it. Callers read the array, not the status code.
+func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	var req UDLPutBatchRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	if len(req.Records) == 0 {
+		writeError(w, http.StatusBadRequest, "records is required")
+		return
+	}
+	if len(req.Records) > maxUDLBatch {
+		writeError(w, http.StatusBadRequest, "too many records in one batch")
+		return
+	}
+	results := make([]UDLPutBatchResult, 0, len(req.Records))
+	for _, rec := range req.Records {
+		if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
+			results = append(results, UDLPutBatchResult{
+				Key:   rec.Key,
+				Error: "uid, cid, key and record are required",
+			})
+			continue
+		}
+		publicValue := ""
+		hasValue := false
+		if rec.Value != nil {
+			publicValue = string(*rec.Value)
+			hasValue = true
+		}
+		accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
+		if err != nil {
+			// A storage error is per-record here rather than fatal for the whole
+			// batch: the cells already written stay written, and reporting which
+			// ones failed lets the caller retry exactly those.
+			results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
+			continue
+		}
+		results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
+	}
+	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
+}
+
+// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
+// (active_for_user_key — My List, Continue Watching).
+func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	vars := mux.Vars(r)
+	uid, key := vars["uid"], vars["key"]
+	if uid == "" || key == "" {
+		writeError(w, http.StatusBadRequest, "uid and key are required")
+		return
+	}
+	entries, err := s.storage.UDLListUserKey(uid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
+// (list_for_user_cid).
+func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	vars := mux.Vars(r)
+	uid, cid := vars["uid"], vars["cid"]
+	if uid == "" || cid == "" {
+		writeError(w, http.StatusBadRequest, "uid and cid are required")
+		return
+	}
+	entries, err := s.storage.UDLListUserCid(uid, cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
+// — every record a uid owns, for cross-instance profile recovery.
+//
+// The other user-scoped reads answer a question with one coordinate already
+// known. Recovery knows only the uid: a box that has just imported a secret key
+// has no idea which cids or keys the profile ever touched, so it needs the
+// whole set or it gets nothing.
+//
+// Paged on a cid boundary — `nextCid` empty means the export is complete. The
+// response is opaque signed records, exactly as stored; whoever consumes it is
+// responsible for verifying each one before trusting it (meta-core has never
+// held the account key and cannot verify on anyone's behalf).
+func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	limit := maxUDLBatch
+	if v := r.URL.Query().Get("limit"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n <= 0 {
+			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
+			return
+		}
+		if n < limit {
+			limit = n
+		}
+	}
+	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"entries": entries,
+		"nextCid": nextCid,
+	})
+}
+
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
+// uid holds records for.
+//
+// Exists so a gateway mirror can enforce a per-profile quota without walking
+// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
+// disk-fill DoS that needs no account. A quota check that itself cost a scan
+// would just move the denial of service one layer up.
+func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	n, err := s.storage.UDLCountUserCids(uid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
+}
+
+// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
+func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	cid := mux.Vars(r)["cid"]
+	if cid == "" {
+		writeError(w, http.StatusBadRequest, "cid is required")
+		return
+	}
+	users, err := s.storage.UDLCidUsers(cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
+}
+
+// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
+// Computes per-key population stats from the stored public value fields
+// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
+func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	cid := mux.Vars(r)["cid"]
+	if cid == "" {
+		writeError(w, http.StatusBadRequest, "cid is required")
+		return
+	}
+	keysParam := r.URL.Query().Get("keys")
+	if keysParam == "" {
+		keysParam = "like,rating"
+	}
+	aggregates := make([]UDLAggregate, 0)
+	for _, key := range strings.Split(keysParam, ",") {
+		key = strings.TrimSpace(key)
+		if key == "" {
+			continue
+		}
+		values, err := s.storage.UDLPublicValues(cid, key)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		aggregates = append(aggregates, aggregatePublicValues(key, values))
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"aggregates": aggregates})
+}
+
+// aggregatePublicValues folds raw JSON scalar values into an UDLAggregate:
+// counts booleans as trues and averages numeric values.
+func aggregatePublicValues(key string, values []string) UDLAggregate {
+	agg := UDLAggregate{Key: key, SampleSize: len(values), Count: len(values)}
+	var sum float64
+	var numeric int
+	for _, raw := range values {
+		var v interface{}
+		if err := json.Unmarshal([]byte(raw), &v); err != nil {
+			continue
+		}
+		switch t := v.(type) {
+		case bool:
+			if t {
+				agg.Trues++
+			}
+		case float64:
+			sum += t
+			numeric++
+		}
+	}
+	if numeric > 0 {
+		avg := math.Round((sum/float64(numeric))*100) / 100
+		agg.Avg = &avg
+	}
+	return agg
+}
+
+// udlUserStatsResponse is the shape of GET /api/udl/users/stats.
+type udlUserStatsResponse struct {
+	// Users maps uid -> counts. Accounts holding nothing are absent rather than
+	// present with zeroes; a caller rendering a list treats "missing" as 0.
+	Users map[string]storage.UDLUserStats `json:"users"`
+}
+
+// handleUDLUserStats handles GET /api/udl/users/stats — how much every account
+// holds, in one keyspace walk.
+//
+// Bulk rather than per-uid because the only caller is the accounts page, which
+// needs a number for every row: the counts come from a SCAN (there is no
+// per-user cid index — see storage/udl_admin.go), and MATCH filters without
+// indexing, so N per-uid calls would walk the whole keyspace N times.
+//
+// Deliberately not on the critical path of anything. It is an admin read whose
+// cost grows with the keyspace, so callers should fetch it after rendering,
+// never before.
+func (s *Server) handleUDLUserStats(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
+			"storage not connected", true)
+		return
+	}
+	stats, err := s.storage.UDLAllUserStats()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, udlUserStatsResponse{Users: stats})
+}

internal/api/udl_test.go

@@ -0,0 +1,226 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newUDLTestServer returns a Server wired to an in-memory Redis.
+//
+// The struct is built directly rather than through NewServer: NewServer boots
+// the file watcher, mounts manager, watchers poller and WebDAV handler, none of
+// which the UDL handlers touch — they use s.storage and nothing else.
+func newUDLTestServer(t *testing.T) *Server {
+	t.Helper()
+
+	mr := miniredis.RunT(t)
+	c := storage.NewClient("")
+	if err := c.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return &Server{storage: c}
+}
+
+// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
+// fields, so a test can assert on the *encoding* of `value` (is it a quoted
+// string? is the key there at all?) rather than on what a typed struct would
+// have coerced it into.
+func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) {
+	t.Helper()
+
+	req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
+	rr := httptest.NewRecorder()
+	s.handleUDLRecordGet(rr, req)
+
+	var body map[string]json.RawMessage
+	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+	}
+	return rr.Code, body
+}
+
+// The plaintext value of a PUBLIC-tier cell must come back over the wire, and
+// come back *as it was written* — a JSON string stays a quoted string, a bool
+// stays a bool.
+//
+// Both halves matter. Without the value at all, the meta-core dashboard can
+// only ever show 44-character base58 uids: display names live in the User Data
+// Layer as the public cell (uid, "self", "profile:name"), and the plaintext
+// sits in Redis where the storage layer has always read it — the handler just
+// used to drop it. And the type has to stay json.RawMessage: this endpoint is
+// generic over keys, so a `*string` would mangle `like` (bool) and `rating`
+// (number) while looking correct for names.
+func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zAlice"
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok {
+		t.Fatalf("seed name: ok=%v err=%v", ok, err)
+	}
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok {
+		t.Fatalf("seed like: ok=%v err=%v", ok, err)
+	}
+
+	code, body := getRecord(t, s, uid, "self", "profile:name")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if got := string(body["value"]); got != `"Ada Lovelace"` {
+		t.Fatalf("value = %s, want a quoted JSON string", got)
+	}
+	var name string
+	if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" {
+		t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err)
+	}
+
+	// A boolean key survives unquoted — the assertion a *string field fails.
+	if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" {
+		t.Fatalf("like value = %s, want the bare JSON literal true", body["value"])
+	}
+}
+
+// A cell with no plaintext must omit `value` entirely rather than send null.
+//
+// That is the shape of every private-tier key (seek, watched, card, localpath —
+// their real value rides encrypted inside the opaque record) and of every
+// tombstone. A reader keys off presence: "there is no plaintext here" and "the
+// plaintext is null" are different facts, and a name cleared by its owner
+// leaves the cell existing with no value, which must not read as a name.
+func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zBob"
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok {
+		t.Fatalf("seed private cell: ok=%v err=%v", ok, err)
+	}
+
+	code, body := getRecord(t, s, uid, "bagcsaaa", "seek")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if _, ok := body["value"]; ok {
+		t.Fatalf("private cell must omit `value`, got %s", body["value"])
+	}
+	if string(body["exists"]) != "true" {
+		t.Fatalf("exists = %s, want true", body["exists"])
+	}
+}
+
+// A cell that was never written answers 200 with exists:false and no value —
+// not a 404, and not an empty-string value that would render as a blank name.
+func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	code, body := getRecord(t, s, "zNobody", "self", "profile:name")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if string(body["exists"]) != "false" {
+		t.Fatalf("exists = %s, want false", body["exists"])
+	}
+	if _, ok := body["value"]; ok {
+		t.Fatalf("missing cell must omit `value`, got %s", body["value"])
+	}
+}
+
+// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the
+// per-record results.
+func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) {
+	t.Helper()
+
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body))
+	rr := httptest.NewRecorder()
+	s.handleUDLRecordsPut(rr, req)
+
+	var out UDLPutBatchResponse
+	if rr.Code == http.StatusOK {
+		if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
+			t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+		}
+	}
+	return rr.Code, out
+}
+
+// **The load-bearing property of the batch endpoint: one stale cell does not
+// fail the batch.**
+//
+// The single-record PUT answers a stale version with 409, which is right when
+// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark
+// this whole series seen") routinely include episodes another device already
+// marked, and 409-ing there would abandon every cell after it. So staleness is
+// reported per record as accepted:false, with 200 on the request itself.
+func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zCarol"
+	const cid = "bagdsaaaseries"
+	// Seed s01e01 at version 4 — the batch below tries to write it at 2.
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok {
+		t.Fatalf("seed: ok=%v err=%v", ok, err)
+	}
+
+	code, out := putRecords(t, s, `{"records":[
+		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"},
+		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"}
+	]}`)
+
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code)
+	}
+	if len(out.Results) != 2 {
+		t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results))
+	}
+	if out.Results[0].Accepted || out.Results[0].Error != "" {
+		t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0])
+	}
+	if !out.Results[1].Accepted {
+		t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1])
+	}
+
+	// The fresh cell really landed, and the stale one did not overwrite.
+	if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" {
+		t.Fatalf("s01e02 was not written")
+	}
+	if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" {
+		t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"])
+	}
+}
+
+// A malformed record is reported in its own slot rather than rejecting the
+// request, for the same reason: the caller retries that cell, not the batch.
+func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	code, out := putRecords(t, s, `{"records":[
+		{"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"},
+		{"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"}
+	]}`)
+
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if out.Results[0].Error == "" {
+		t.Fatalf("record with no key must report an error, got %+v", out.Results[0])
+	}
+	if !out.Results[1].Accepted {
+		t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1])
+	}
+}
+
+// An empty batch is a client bug, not a no-op: it almost always means the
+// caller built its record list wrong, and answering 200 with an empty array
+// would hide that.
+func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400", code)
+	}
+}

internal/cid/card.go

@@ -0,0 +1,92 @@
+package cid
+
+import (
+	"encoding/base32"
+	"encoding/binary"
+	"fmt"
+	"strings"
+	"unicode/utf8"
+)
+
+// DecodeCard recovers the (source, id) pair a card locator (CodeCard, 0x1007)
+// wraps. Wire form:
+//
+//	0x01 ‖ varint(0x1007) ‖ 0x00 (identity mh) ‖ varint(len(digest)) ‖ digest
+//	digest = varint(len(source)) ‖ source ‖ id        (both UTF-8)
+//
+// Decode stops at the multihash code because ranking needs nothing past it;
+// reading *which work* a card addresses needs the digest too, so this is the
+// one place in Go that walks the whole thing.
+//
+// ⚠ Mirror of meta-read `card_cid::decode` and of meta-feeder-sdk
+// `compute_card_cid`. The vectors in card_test.go are pinned literals copied
+// from meta-read, not round-trips, so a drift on either side fails a test
+// instead of silently reading a different work out of the same CID.
+//
+// Identity multihash only: a 0x1007 CID carrying any other multihash is not
+// something anyone mints, and reading its digest as (source, id) would be
+// reading hash bytes as text. Bytes past the declared digest length are
+// ignored, as meta-read does.
+func DecodeCard(cidStr string) (source, id string, err error) {
+	if len(cidStr) < 2 || cidStr[0] != 'b' {
+		return "", "", fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
+	}
+	raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
+	if err != nil {
+		return "", "", fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
+	}
+
+	next := func(buf []byte, what string) (uint64, []byte, error) {
+		v, n := binary.Uvarint(buf)
+		if n <= 0 {
+			return 0, nil, fmt.Errorf("cid %q: bad %s varint", cidStr, what)
+		}
+		return v, buf[n:], nil
+	}
+
+	version, raw, err := next(raw, "version")
+	if err != nil {
+		return "", "", err
+	}
+	if version != 1 {
+		return "", "", fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
+	}
+	codec, raw, err := next(raw, "codec")
+	if err != nil {
+		return "", "", err
+	}
+	if codec != CodeCard {
+		return "", "", fmt.Errorf("cid %q: codec 0x%x is not a card locator", cidStr, codec)
+	}
+	mh, raw, err := next(raw, "multihash-code")
+	if err != nil {
+		return "", "", err
+	}
+	if mh != 0 {
+		return "", "", fmt.Errorf("cid %q: card locator with non-identity multihash 0x%x", cidStr, mh)
+	}
+	digestLen, raw, err := next(raw, "digest-length")
+	if err != nil {
+		return "", "", err
+	}
+	if digestLen > uint64(len(raw)) {
+		return "", "", fmt.Errorf("cid %q: digest truncated", cidStr)
+	}
+	digest := raw[:digestLen]
+
+	sourceLen, rest, err := next(digest, "source-length")
+	if err != nil {
+		return "", "", err
+	}
+	if sourceLen > uint64(len(rest)) {
+		return "", "", fmt.Errorf("cid %q: source truncated", cidStr)
+	}
+	source, id = string(rest[:sourceLen]), string(rest[sourceLen:])
+	if source == "" || id == "" {
+		return "", "", fmt.Errorf("cid %q: empty source or id", cidStr)
+	}
+	if !utf8.ValidString(source) || !utf8.ValidString(id) {
+		return "", "", fmt.Errorf("cid %q: source or id is not UTF-8", cidStr)
+	}
+	return source, id, nil
+}

internal/cid/card_test.go

@@ -0,0 +1,71 @@
+package cid
+
+import (
+	"encoding/base32"
+	"encoding/binary"
+	"strings"
+	"testing"
+)
+
+// Pinned from meta-read `src/card_cid.rs` / meta-feeder-sdk `compute_card_cid`
+// — literals, not round-trips (see DecodeCard).
+const (
+	goldenAnilistCard  = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy"
+	goldenMangadexCard = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
+)
+
+// wireCID builds a CIDv1 from raw parts so the negative cases below can vary
+// one field at a time.
+func wireCID(codec, mh uint64, digest []byte) string {
+	buf := []byte{0x01}
+	buf = binary.AppendUvarint(buf, codec)
+	buf = binary.AppendUvarint(buf, mh)
+	buf = binary.AppendUvarint(buf, uint64(len(digest)))
+	buf = append(buf, digest...)
+	return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf))
+}
+
+func cardDigest(source, id string) []byte {
+	d := binary.AppendUvarint(nil, uint64(len(source)))
+	d = append(d, source...)
+	return append(d, id...)
+}
+
+func TestDecodeCard_GoldenVectors(t *testing.T) {
+	for _, tc := range []struct {
+		cid, source, id string
+	}{
+		{goldenAnilistCard, "anilist", "manga:105778"},
+		{goldenMangadexCard, "mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"},
+	} {
+		source, id, err := DecodeCard(tc.cid)
+		if err != nil {
+			t.Fatalf("DecodeCard(%s): %v", tc.cid, err)
+		}
+		if source != tc.source || id != tc.id {
+			t.Errorf("DecodeCard(%s) = (%q, %q), want (%q, %q)", tc.cid, source, id, tc.source, tc.id)
+		}
+		// The test encoder must agree with the pinned literal, or the negative
+		// cases below are testing a different wire format.
+		if got := wireCID(CodeCard, 0, cardDigest(tc.source, tc.id)); got != tc.cid {
+			t.Errorf("wireCID(%q, %q) = %s, want %s", tc.source, tc.id, got, tc.cid)
+		}
+	}
+}
+
+func TestDecodeCard_RejectsEverythingElse(t *testing.T) {
+	for name, s := range map[string]string{
+		"url locator codec":      wireCID(CodeURL, 0, cardDigest("anilist", "1")),
+		"non-identity multihash": wireCID(CodeCard, CodeSha256, cardDigest("anilist", "1")),
+		"empty id":               wireCID(CodeCard, 0, cardDigest("anilist", "")),
+		"source past digest":     wireCID(CodeCard, 0, []byte{0x20, 'a'}),
+		"not a CID":              "title:berserk",
+		"CIDv0":                  "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
+		"truncated":              goldenAnilistCard[:12],
+		"empty":                  "",
+	} {
+		if source, id, err := DecodeCard(s); err == nil {
+			t.Errorf("%s: DecodeCard(%q) = (%q, %q), want error", name, s, source, id)
+		}
+	}
+}

internal/cid/cid-rank-vectors.json

@@ -0,0 +1,184 @@
+{
+  "$schema": "cid-rank-vectors/v1",
+  "$comment": [
+    "CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
+    "The rank decides which of a record's several CIDs is elected canonical - the cid a",
+    "record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
+    "they cannot share code (independent submodules, no cargo path-dep). These vectors are",
+    "the only thing holding them in sync: every implementation's test suite loads this file.",
+    "",
+    "This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
+    "own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
+    "",
+    "Ranking is by UTILITY for external retrieval, not cryptographic strength.",
+    "Ties break on the lexicographically smaller cid, so the election is deterministic",
+    "across peers and does not churn.",
+    "",
+    "See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
+  ],
+  "ladder": {
+    "40": "dag-pb (IPFS block)",
+    "30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
+    "25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
+    "20": "BitTorrent infohashes - swarm interop",
+    "10": "midhash256 - the record address",
+    "5": "opaque locators (nzb-release, url, card, yt-video, ext-play) - never outrank a real digest",
+    "0": "weak/unknown digests, unparseable input"
+  },
+  "tieBreak": "lexicographically smaller cid wins",
+  "vectors": [
+    {
+      "name": "ipfs-dagpb",
+      "cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 40,
+      "note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
+    },
+    {
+      "name": "sha2-256-raw-codec",
+      "cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 30,
+      "note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
+    },
+    {
+      "name": "sha2-256-codec-eq-mh",
+      "cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
+      "rank": 30,
+      "note": "REAL meta-hash fixture; meta-hash sets codec == mh"
+    },
+    {
+      "name": "sha3-256",
+      "cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "sha3-384",
+      "cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "nzb-posting",
+      "cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 25,
+      "note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
+    },
+    {
+      "name": "btih-v2-codec-eq-mh",
+      "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "meta-hash encoder convention"
+    },
+    {
+      "name": "btih-v2-raw-codec",
+      "cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "fullhash encoder convention - SAME tier, both conventions must match"
+    },
+    {
+      "name": "btih-v1-file",
+      "cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v1"
+    },
+    {
+      "name": "btih-v2-file",
+      "cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v2"
+    },
+    {
+      "name": "midhash256-codec-eq-mh",
+      "cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10,
+      "note": "the record address; internally meaningful, externally invisible"
+    },
+    {
+      "name": "midhash256-raw-codec",
+      "cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10
+    },
+    {
+      "name": "nzb-release-locator",
+      "cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
+      "rank": 5,
+      "note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
+    },
+    {
+      "name": "url-locator",
+      "cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
+      "rank": 5,
+      "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
+    },
+    {
+      "name": "card-locator",
+      "cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
+      "rank": 5,
+      "note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
+    },
+    {
+      "name": "yt-video-locator",
+      "cid": "bagecaaarav3gszdfn42ein3vgvfumn2tka4a",
+      "rank": 5,
+      "note": "codec 0x1008 + IDENTITY multihash 0x00, digest = varint(len(kind)) || kind || id — here (\"video\",\"4D7u5KF7SP8\"). DELEGATED PLAYBACK: the limit case beyond card. A card addresses a work with no bytes anywhere; this addresses a *rendition* whose bytes exist but are permanently someone else's — an external player renders it, nothing is transported, seeded or content-addressed. Framing is byte-identical to card (0x1007) on purpose, so each implementation extends the parser it already wrote. Same codec-slot rule: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5 is load-bearing for the on-ramp: a delegated reference and a real archived copy can sit on ONE record, and that sha2-256 must win the canonical election over this locator with no addressing change. NOTE the byte path must REFUSE this (400), not 404 — see docs/cid-formats.md §7.4."
+    },
+    {
+      "name": "ext-play-locator",
+      "cid": "bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa",
+      "rank": 5,
+      "note": "codec 0x1009 + IDENTITY multihash 0x00, digest = the verbatim UTF-8 URL. WIRE-IDENTICAL to url (0x1006) apart from the codec, and that difference is the entire contract: 0x1006 means 'fetch these bytes once and seed them', 0x1009 means 'open this in a browser, there are no bytes for us'. A decoder that matched on SHAPE rather than on the codec slot would see an identity multihash wrapping an http(s) URL, take the 0x1006 path, fetch a watch/Bandcamp page, chunk it, seed the HTML and link that content CID onto the record AS THE FILE — the MSR1 poisoning shape, except permanent and replicated. Same tier/reasoning as the rest of the locator family."
+    },
+    {
+      "name": "provider-file-locator",
+      "cid": "bagfcaaa2bvxxazloon2we5djorwgk43gnfwgkorxga3dcobtgq",
+      "rank": 5,
+      "note": "codec 0x100A + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id - here (\"opensubtitles\",\"file:7061834\"). The GENERIC fetchable locator: one codec for every provider whose bytes are fetched the same way; the provider is inside the digest. Framing byte-identical to card (0x1007), but the byte path RESOLVES it (via source's fetch tier) instead of 404ing. Same codec-slot rule as the rest of the family: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5: once the fetched bytes' sha2-256 is linked onto the record it wins the canonical election with no addressing change."
+    },
+    {
+      "name": "sha1",
+      "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "md5",
+      "cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "crc32",
+      "cid": "bagzafmqcathsbnrk",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "bt-pieces-root",
+      "cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "torrent-internal merkle root, NOT a file-identity cid"
+    },
+    {
+      "name": "cidv0-rejected",
+      "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
+      "rank": 0,
+      "note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
+    },
+    {
+      "name": "token-form-rejected",
+      "cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
+    },
+    {
+      "name": "garbage",
+      "cid": "not-a-cid",
+      "rank": 0
+    },
+    {
+      "name": "empty",
+      "cid": "",
+      "rank": 0
+    }
+  ]
+}

internal/cid/rank.go

@@ -20,6 +20,14 @@ import (
 // Multicodec / multihash codes used for ranking. Source of truth for the
 // digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
 // METADATA_KEYS.md §2.
+//
+// The tier every code maps to is pinned by the golden fixture
+// `cid-rank-vectors.json` (vendored next to this package). This is one of
+// seven implementations of the same ladder across Go, Rust and TypeScript;
+// the fixture is the only thing keeping them in agreement. If you add a code
+// here, add a vector — an unranked code silently falls to tier 0, which is
+// exactly how the nzb-release/url locators came to be ranked 20 in meta-share
+// and 0 here and in meta-search.
 const (
 	CodecRaw   = 0x55   // raw — IPFS content-addressed block
 	CodecDagPB = 0x70   // dag-pb — IPFS content-addressed block
@@ -34,6 +42,69 @@ const (
 	CodeBtihV2      = 0x10B7  // custom — BitTorrent v2 info hash (BEP 52)
 	CodeBtihV1File  = 0x1001  // custom — per-file BitTorrent v1 CID
 	CodeBtihV2File  = 0x1002  // custom — per-file BitTorrent v2 CID
+
+	// Opaque locators. These are NOT digests: the payload rides in an identity
+	// multihash (0x00), so the code identifying them sits in the *content
+	// codec* slot, never the multihash slot. Rank must match on the codec for
+	// these — matching on mh would see 0x00 and rank them 0.
+	CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
+	CodeURL        = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
+	// CodeCard is the limit case of the family: the only CID with NO bytes
+	// behind it anywhere, ever. It addresses a *work* (a series, a film) via
+	// varint(len(source))‖source‖id — ("tmdb","tv:95479") — so `/api/file/{cid}`
+	// correctly 404s and bitswap never offers it. Ranking it here (rather than
+	// letting it fall to RankUnknown) is what makes the planned "one card, two
+	// CIDs" cross-source merge safe: a TMDB and a MyAnimeList locator can sit on
+	// one record, and a real digest added later still wins the election.
+	CodeCard = 0x1007 // custom — identity-multihash CID wrapping a card (work) id
+
+	// CodeYtVideo and CodeExtPlay are the DELEGATED-PLAYBACK locators — the
+	// limit case beyond CodeCard. A card addresses a work with no bytes
+	// anywhere; these address a *rendition* whose bytes exist but are
+	// permanently someone else's: an external player renders the media, nothing
+	// is transported, seeded or content-addressed.
+	//
+	// They rank on the CODEC like the rest of the family — their multihash is
+	// identity (0x00), so a rank that dispatched on the multihash would see 0x00,
+	// fall through and silently score 0. That is the exact drift that once made
+	// meta-share and meta-search elect different canonical CIDs for one record.
+	//
+	// Tier 5 is also what keeps the on-ramp safe: a delegated reference and a
+	// real CID can sit on one record, and if the media is ever archived to the
+	// mesh its sha2-256 wins the canonical election with no addressing change.
+	CodeYtVideo = 0x1008 // custom — identity-multihash CID wrapping a YouTube (kind, id)
+	CodeExtPlay = 0x1009 // custom — identity-multihash CID wrapping a URL to OPEN, never fetch
+
+	// CodeProviderFile is the FETCHABLE twin of CodeExtPlay: a provider-held
+	// file addressed as varint(len(source))‖source‖id — ("opensubtitles",
+	// "file:7061834"). Framing is byte-identical to CodeCard; only the byte
+	// path differs (it redeems the id through that source's feeder instead of
+	// 404ing). Same codec-slot rule and same tier: once the fetched bytes'
+	// sha2-256 is linked onto the record it wins with no addressing change.
+	CodeProviderFile = 0x100A // custom — identity-multihash CID wrapping a (source, id) provider file
+
+	// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
+	// REAL sha2-256 digest (over the normalised, sorted article Message-ID
+	// set of a Usenet posting we scanned ourselves — never a locator, no
+	// indexer host embedded), so it would silently fall into RankDigest via
+	// the mh switch unless matched on the codec FIRST. It is content-derived
+	// (so it outranks the opaque locators) but addresses no fetchable block —
+	// the digest is over an id set, not over any block's bytes, so it can
+	// never be bitswap-fetched under itself — so it must stay BELOW a real
+	// digest (once the media is materialised and re-seeded, that genuine
+	// sha2-256 cid has to win the canonical election, not this one).
+	CodeNzbPosting = 0x1003 // custom — self-scanned Usenet posting identity
+)
+
+// Rank tiers, named so the ladder reads as a ladder.
+const (
+	RankIPFS       = 40 // dag-pb — broadest network reach
+	RankDigest     = 30 // sha2-256 / sha3-* — real content digests
+	RankNzbPosting = 25 // content-derived, but the digest addresses no fetchable block
+	RankBtih       = 20 // BitTorrent infohashes — swarm interop
+	RankMidhash    = 10 // the record address — internally meaningful, externally invisible
+	RankLocator    = 5  // opaque locators — never outrank a real digest
+	RankUnknown    = 0  // weak/unknown digests, unparseable input
 )
 
 // Decode parses a multibase-base32 CIDv1 string ("b…") into its content
@@ -87,33 +158,56 @@ func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
 //   - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
 //   - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
 //     block of the file IS its sha2-256, so they share this tier).
+//   - nzb-posting is content-derived (a digest over a Usenet posting's article
+//     Message-ID set) so it outranks the opaque locators below, but the
+//     digest addresses no fetchable block, so it stays below a real content
+//     digest — once the media is materialised and re-seeded under a genuine
+//     sha2-256 cid, that cid must win the election, not this one.
 //   - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
 //     swarm interop, not strength.
-//   - midhash256 is the floor among real options: internally meaningful,
+//   - midhash256 is the floor among real *digests*: internally meaningful,
 //     externally invisible.
+//   - opaque locators (nzb-release, url, card) sit BELOW midhash. They are not
+//     content-derived — two URLs to the same bytes yield two different locator
+//     CIDs — so a locator must never displace a real digest as the identity a
+//     record is advertised under. Tier 5 means a locator wins the election only
+//     when it is the record's sole CID, which is exactly the guarantee
+//     METADATA_KEYS.md §2 states.
 //   - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
 func Rank(cidStr string) int {
 	codec, mh, err := Decode(cidStr)
 	if err != nil {
-		return 0
+		return RankUnknown
 	}
 	if codec == CodecDagPB {
-		return 40
+		return RankIPFS
 	}
 	// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
 	// multihash code, in either encoder convention.
 	if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
 		mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
-		return 20
+		return RankBtih
+	}
+	// Opaque locators. Codec-only: their multihash is identity (0x00), so
+	// there is nothing to match on the mh side.
+	if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard ||
+		codec == CodeYtVideo || codec == CodeExtPlay || codec == CodeProviderFile {
+		return RankLocator
+	}
+	// nzb-posting: codec-only, checked BEFORE the mh switch below — its
+	// multihash is a real sha2-256 and would otherwise match CodeSha256 there
+	// and rank 30, tying with genuine content digests.
+	if codec == CodeNzbPosting {
+		return RankNzbPosting
 	}
 	switch mh {
 	case CodeSha256, CodeSha3_256, CodeSha3_384:
-		return 30
+		return RankDigest
 	case CodeMidhash256:
-		return 10
+		return RankMidhash
 	default:
 		// md5, crc32, plain sha1, pieces-root, unknown.
-		return 0
+		return RankUnknown
 	}
 }
 

internal/cid/rank_test.go

@@ -3,10 +3,56 @@ package cid
 import (
 	"encoding/base32"
 	"encoding/binary"
+	"encoding/json"
+	"os"
 	"strings"
 	"testing"
 )
 
+// goldenVectors is the CANONICAL cross-language fixture: the same file is
+// vendored into meta-search, meta-share and meta-hash, and each of their test
+// suites asserts against it. It is the only mechanism keeping the seven
+// implementations of this ladder in agreement — they live in independent
+// submodules and cannot share code. `scripts/check-cid-vectors.sh` at the
+// meta-root fails if any vendored copy drifts from the canonical one.
+const goldenVectors = "cid-rank-vectors.json"
+
+type vector struct {
+	Name string `json:"name"`
+	CID  string `json:"cid"`
+	Rank int    `json:"rank"`
+	Note string `json:"note"`
+}
+
+func loadVectors(t *testing.T) []vector {
+	t.Helper()
+	raw, err := os.ReadFile(goldenVectors)
+	if err != nil {
+		t.Fatalf("read %s: %v (run ./scripts/check-cid-vectors.sh --fix from the meta-root)", goldenVectors, err)
+	}
+	var doc struct {
+		Vectors []vector `json:"vectors"`
+	}
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		t.Fatalf("parse %s: %v", goldenVectors, err)
+	}
+	if len(doc.Vectors) == 0 {
+		t.Fatalf("%s has no vectors", goldenVectors)
+	}
+	return doc.Vectors
+}
+
+// TestRank_GoldenVectors is the load-bearing test. Every other test in this
+// file is a convenience; this one is the contract with Rust and TypeScript.
+func TestRank_GoldenVectors(t *testing.T) {
+	for _, v := range loadVectors(t) {
+		if got := Rank(v.CID); got != v.Rank {
+			t.Errorf("Rank(%s) = %d, want %d\n  cid:  %s\n  note: %s",
+				v.Name, got, v.Rank, v.CID, v.Note)
+		}
+	}
+}
+
 // buildCID assembles a multibase-base32 CIDv1 from a content codec, a
 // multihash code, and a digest length (digest bytes are zero — only the
 // header is under test). Mirrors the encoder in watcher/midhash.go and the
@@ -104,12 +150,31 @@ func TestRank_Tiers(t *testing.T) {
 func TestRank_StrictOrdering(t *testing.T) {
 	ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
 	sha := Rank(cidSha256)
+	posting := Rank(buildCID(CodeNzbPosting, CodeSha256, 32))
 	btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
 	mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
 	unknown := Rank(cidMd5)
-	if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
+	if !(ipfs > sha && sha > posting && posting > btih && btih > mid && mid > unknown) {
-		t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
+		t.Fatalf("rank ordering broken: ipfs=%d sha=%d posting=%d btih=%d mid=%d unknown=%d",
-			ipfs, sha, btih, mid, unknown)
+			ipfs, sha, posting, btih, mid, unknown)
+	}
+}
+
+// TestRank_NzbPostingMatchesCodecNotMultihash pins the inverse trap:
+// nzb-posting's multihash is a REAL sha2-256 (unlike nzb-release/url, whose mh
+// is identity 0x00), so Rank must match on the codec BEFORE falling into the
+// mh switch — otherwise this ties with a genuine content digest at 30 instead
+// of sitting at 25.
+func TestRank_NzbPostingMatchesCodecNotMultihash(t *testing.T) {
+	posting := buildCID(CodeNzbPosting, CodeSha256, 32)
+	if got := Rank(posting); got != RankNzbPosting {
+		t.Errorf("Rank(nzb-posting) = %d, want %d (RankNzbPosting) — codec check must precede the mh switch", got, RankNzbPosting)
+	}
+	// A real sha2-256 with the SAME digest bytes must still rank 30 — the
+	// codec, not the digest, is what distinguishes the two.
+	realDigest := buildCID(CodecRaw, CodeSha256, 32)
+	if got := Rank(realDigest); got != RankDigest {
+		t.Errorf("Rank(sha2-256) regressed to %d, want %d", got, RankDigest)
 	}
 }
 

internal/config/config.go

@@ -24,45 +24,57 @@ type Config struct {
 
 	// HTTP API configuration
 	HTTPPort int    // HTTP API port (default: 9000)
-	HTTPHost string // HTTP API host (default: "127.0.0.1")
+	HTTPHost string // HTTP API host (default: "0.0.0.0" — meta-core always runs in a container and must answer siblings)
 
 	// Timing configuration
-	HealthCheckIntervalMS    int // Health check interval in ms (default: 5000)
+	HealthCheckIntervalMS  int // Health check interval in ms (default: 5000)
-	HeartbeatIntervalMS      int // Service heartbeat interval in ms (default: 30000)
+	HeartbeatIntervalMS    int // Service heartbeat interval in ms (default: 30000)
-	StaleThresholdMS         int // Stale service threshold in ms (default: 60000)
+	StaleThresholdMS       int // Stale service threshold in ms (default: 60000)
-	CleanupIntervalMS        int // Dead service cleanup interval in ms (default: 600000)
+	CleanupIntervalMS      int // Dead service cleanup interval in ms (default: 600000)
-	DeadServiceThresholdMS   int // Dead service threshold in ms (default: 600000)
+	DeadServiceThresholdMS int // Dead service threshold in ms (default: 600000)
 
 	// File watcher configuration
-	WatchIntervalMS   int      // Polling interval for network mounts (default: 1000)
+	WatchIntervalMS   int  // Polling interval for network mounts (default: 1000)
-	DebounceMS        int      // File change debounce time (default: 30000)
+	DebounceMS        int  // File change debounce time (default: 30000)
-	EnableFileWatcher bool     // Enable file watcher (default: true)
+	EnableFileWatcher bool // Enable file watcher (default: true)
 
 	// Mount configuration
 	MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
+
+	// UDP discovery (beacon v2). Replaces the file-based registry that used
+	// to live under /meta-core/services; see
+	// docs/project-architecture/beacon-v2.md.
+	EnableUDPDiscovery  bool   // Advertise + listen on the multicast group (default: true)
+	DiscoveryGroup      string // BEACON_GROUP: IPv4 multicast group (default: 239.255.99.1)
+	DiscoveryPort       int    // BEACON_PORT: UDP port (default: 9099)
+	DiscoveryIntervalMS int    // BEACON_INTERVAL_MS: unsolicited advertise interval (default: 10000)
 }
 
 // Load creates a Config from environment variables
 func Load() *Config {
 	cfg := &Config{
-		MetaCorePath:          getEnv("META_CORE_PATH", "/meta-core"),
+		MetaCorePath:           getEnv("META_CORE_PATH", "/meta-core"),
-		FilesPath:             getEnv("FILES_PATH", "/files"),
+		FilesPath:              getEnv("FILES_PATH", "/files"),
-		ServiceName:           getEnv("SERVICE_NAME", "meta-core"),
+		ServiceName:            getEnv("SERVICE_NAME", "meta-core"),
-		ServiceVersion:        getEnv("SERVICE_VERSION", "1.0.0"),
+		ServiceVersion:         getEnv("SERVICE_VERSION", "1.0.0"),
-		APIPort:               getEnvInt("API_PORT", 8180),
+		APIPort:                getEnvInt("API_PORT", 8180),
-		BaseURL:               getEnv("BASE_URL", ""),
+		BaseURL:                getEnv("BASE_URL", ""),
-		PublicURL:             getEnv("META_CORE_PUBLIC_URL", ""),
+		PublicURL:              getEnv("META_CORE_PUBLIC_URL", ""),
-		RedisPort:             getEnvInt("REDIS_PORT", 6379),
+		RedisPort:              getEnvInt("REDIS_PORT", 6379),
-		HTTPPort:              getEnvInt("META_CORE_HTTP_PORT", 9000),
+		HTTPPort:               getEnvInt("META_CORE_HTTP_PORT", 9000),
-		HTTPHost:              getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
+		HTTPHost:               getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
 		HealthCheckIntervalMS:  getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
-		HeartbeatIntervalMS:   getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
+		HeartbeatIntervalMS:    getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
-		StaleThresholdMS:      getEnvInt("STALE_THRESHOLD_MS", 60000),
+		StaleThresholdMS:       getEnvInt("STALE_THRESHOLD_MS", 60000),
-		CleanupIntervalMS:     getEnvInt("CLEANUP_INTERVAL_MS", 600000),
+		CleanupIntervalMS:      getEnvInt("CLEANUP_INTERVAL_MS", 600000),
 		DeadServiceThresholdMS: getEnvInt("DEAD_SERVICE_THRESHOLD_MS", 600000),
-		WatchIntervalMS:       getEnvInt("WATCH_INTERVAL_MS", 1000),
+		WatchIntervalMS:        getEnvInt("WATCH_INTERVAL_MS", 1000),
-		DebounceMS:            getEnvInt("DEBOUNCE_MS", 30000),
+		DebounceMS:             getEnvInt("DEBOUNCE_MS", 30000),
-		EnableFileWatcher:     getEnvBool("ENABLE_FILE_WATCHER", true),
+		EnableFileWatcher:      getEnvBool("ENABLE_FILE_WATCHER", true),
+		EnableUDPDiscovery:     getEnvBool("ENABLE_UDP_DISCOVERY", true),
+		DiscoveryGroup:         getEnv("BEACON_GROUP", "239.255.99.1"),
+		DiscoveryPort:          getEnvInt("BEACON_PORT", 9099),
+		DiscoveryIntervalMS:    getEnvInt("BEACON_INTERVAL_MS", 10000),
 	}
 
 	// Set mounts directory
@@ -71,26 +83,18 @@ func Load() *Config {
 	return cfg
 }
 
-// LockFilePath returns the path to the leader lock file
+// LockFilePath returns the path to the Redis-volume mutex. Held by
+// docker/leader-election.sh; nothing reads it, and it is NOT service
+// discovery — see docs/project-architecture/beacon-v2.md.
 func (c *Config) LockFilePath() string {
 	return c.MetaCorePath + "/locks/kv-leader.lock"
 }
 
-// InfoFilePath returns the path to the leader info file
-func (c *Config) InfoFilePath() string {
-	return c.MetaCorePath + "/locks/kv-leader.info"
-}
-
 // RedisDataDir returns the path to Redis data directory
 func (c *Config) RedisDataDir() string {
 	return c.MetaCorePath + "/db/redis"
 }
 
-// ServicesDir returns the path to services directory
-func (c *Config) ServicesDir() string {
-	return c.MetaCorePath + "/services"
-}
-
 // MountsFilePath returns the path to the mounts configuration file
 func (c *Config) MountsFilePath() string {
 	return c.MountsDir + "/mounts.json"
@@ -161,11 +165,11 @@ func (c *Config) DefaultWatchPath() string {
 //
 //   - /files/watch  — user-curated media. The main library.
 //   - /files/plugin — plugin output (TMDB posters/backdrops, extracted
-//                     subtitles, etc.). Indexed so each artefact gets a
+//     subtitles, etc.). Indexed so each artefact gets a
-//                     midhash256 alias and CID-based references from
+//     midhash256 alias and CID-based references from
-//                     other files (e.g. `backdrop=<cid>`) resolve through
+//     other files (e.g. `backdrop=<cid>`) resolve through
-//                     the standard reverse index instead of needing a
+//     the standard reverse index instead of needing a
-//                     path companion field.
+//     path companion field.
 func (c *Config) DefaultWatcherPaths() []string {
 	return []string{
 		c.FilesPath + "/watch",

internal/discovery/cleaner.go

@@ -1,196 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// CleanupStats holds statistics about the cleanup process
-type CleanupStats struct {
-	LastRunTime     string `json:"lastRunTime,omitempty"`
-	ServicesRemoved int    `json:"servicesRemoved"`
-	TotalRuns       int    `json:"totalRuns"`
-	Errors          int    `json:"errors"`
-}
-
-// Cleaner handles periodic cleanup of dead service registration files
-type Cleaner struct {
-	config      *config.Config
-	servicesDir string
-	ownFile     string // Our own service file (never delete)
-
-	stats    CleanupStats
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-	mu       sync.RWMutex
-}
-
-// NewCleaner creates a new service cleaner
-func NewCleaner(cfg *config.Config) *Cleaner {
-	hostname, _ := os.Hostname()
-	return &Cleaner{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		ownFile:     filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins the cleanup loop
-func (c *Cleaner) Start() error {
-	log.Printf("[Cleaner] Starting dead service cleanup (interval: %dms, threshold: %dms)",
-		c.config.CleanupIntervalMS, c.config.DeadServiceThresholdMS)
-
-	c.wg.Add(1)
-	go c.cleanupLoop()
-
-	return nil
-}
-
-// Stop stops the cleanup loop
-func (c *Cleaner) Stop() error {
-	log.Println("[Cleaner] Stopping dead service cleanup...")
-	close(c.stopChan)
-	c.wg.Wait()
-	return nil
-}
-
-// cleanupLoop runs the periodic cleanup
-func (c *Cleaner) cleanupLoop() {
-	defer c.wg.Done()
-
-	// Run an initial cleanup shortly after starting
-	initialDelay := time.NewTimer(30 * time.Second)
-	select {
-	case <-c.stopChan:
-		initialDelay.Stop()
-		return
-	case <-initialDelay.C:
-		c.runCleanup()
-	}
-
-	ticker := time.NewTicker(time.Duration(c.config.CleanupIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-c.stopChan:
-			return
-		case <-ticker.C:
-			c.runCleanup()
-		}
-	}
-}
-
-// runCleanup scans the services directory and removes dead services
-func (c *Cleaner) runCleanup() {
-	c.mu.Lock()
-	c.stats.TotalRuns++
-	c.stats.LastRunTime = time.Now().UTC().Format(time.RFC3339)
-	c.mu.Unlock()
-
-	entries, err := os.ReadDir(c.servicesDir)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Printf("[Cleaner] Failed to read services directory: %v", err)
-			c.mu.Lock()
-			c.stats.Errors++
-			c.mu.Unlock()
-		}
-		return
-	}
-
-	threshold := time.Duration(c.config.DeadServiceThresholdMS) * time.Millisecond
-	removedCount := 0
-
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(c.servicesDir, entry.Name())
-
-		// Never delete our own service file
-		if filePath == c.ownFile {
-			continue
-		}
-
-		// Check if service is dead
-		isDead, serviceName, age := c.isServiceDead(filePath, threshold)
-		if isDead {
-			if err := os.Remove(filePath); err != nil {
-				log.Printf("[Cleaner] Failed to remove dead service file %s: %v", entry.Name(), err)
-				c.mu.Lock()
-				c.stats.Errors++
-				c.mu.Unlock()
-			} else {
-				ageMinutes := age.Minutes()
-				log.Printf("[Cleaner] Removed dead service: %s (last heartbeat: %.1f minutes ago)", serviceName, ageMinutes)
-				removedCount++
-			}
-		}
-	}
-
-	if removedCount > 0 {
-		c.mu.Lock()
-		c.stats.ServicesRemoved += removedCount
-		c.mu.Unlock()
-		log.Printf("[Cleaner] Cleanup complete: removed %d dead service(s)", removedCount)
-	}
-}
-
-// isServiceDead checks if a service file represents a dead service
-// Returns: isDead, serviceName, age since last heartbeat
-func (c *Cleaner) isServiceDead(filePath string, threshold time.Duration) (bool, string, time.Duration) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		// Can't read file - treat as dead to clean up corrupted files
-		log.Printf("[Cleaner] Cannot read service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, filepath.Base(filePath), threshold
-	}
-
-	var info ServiceInfo
-	if err := json.Unmarshal(data, &info); err != nil {
-		// Corrupted JSON - skip rather than delete (might be recoverable)
-		log.Printf("[Cleaner] Skipping corrupted service file %s: %v", filepath.Base(filePath), err)
-		return false, "", 0
-	}
-
-	// Parse last heartbeat timestamp
-	if info.LastHeartbeat == "" {
-		// No heartbeat timestamp - treat as dead
-		return true, info.Name, threshold
-	}
-
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err != nil {
-		// Invalid timestamp - treat as dead
-		log.Printf("[Cleaner] Invalid timestamp in service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, info.Name, threshold
-	}
-
-	age := time.Since(lastHeartbeat)
-	if age > threshold {
-		return true, info.Name, age
-	}
-
-	return false, info.Name, age
-}
-
-// Stats returns a copy of the current cleanup statistics
-func (c *Cleaner) Stats() CleanupStats {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return CleanupStats{
-		LastRunTime:     c.stats.LastRunTime,
-		ServicesRemoved: c.stats.ServicesRemoved,
-		TotalRuns:       c.stats.TotalRuns,
-		Errors:          c.stats.Errors,
-	}
-}

internal/discovery/service.go

@@ -1,286 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// ServiceInfo contains simplified service registration data
-// URLs are obtained via the /urls API endpoint
-type ServiceInfo struct {
-	Name          string `json:"name"`
-	Hostname      string `json:"hostname"`
-	BaseUrl       string `json:"baseUrl"`
-	Status        string `json:"status"`
-	LastHeartbeat string `json:"lastHeartbeat"`
-}
-
-// Service handles service registration and discovery
-type Service struct {
-	config      *config.Config
-	servicesDir string
-	serviceFile string
-	info        *ServiceInfo
-	mu          sync.RWMutex
-
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-}
-
-// NewService creates a new service discovery instance
-func NewService(cfg *config.Config) *Service {
-	hostname, _ := os.Hostname()
-	return &Service{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		serviceFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins service registration and heartbeat
-func (s *Service) Start() error {
-	log.Printf("[Discovery] Starting service discovery for %s", s.config.ServiceName)
-
-	// Ensure services directory exists
-	if err := os.MkdirAll(s.servicesDir, 0755); err != nil {
-		return fmt.Errorf("failed to create services directory: %w", err)
-	}
-
-	// Build and register service info
-	s.info = s.buildServiceInfo()
-	if err := s.register(); err != nil {
-		return fmt.Errorf("failed to register service: %w", err)
-	}
-
-	// Start heartbeat loop
-	s.wg.Add(1)
-	go s.heartbeatLoop()
-
-	return nil
-}
-
-// Stop stops service discovery and unregisters
-func (s *Service) Stop() error {
-	log.Println("[Discovery] Stopping service discovery...")
-	close(s.stopChan)
-	s.wg.Wait()
-
-	// Unregister by removing service file
-	if err := os.Remove(s.serviceFile); err != nil && !os.IsNotExist(err) {
-		log.Printf("[Discovery] Failed to remove service file: %v", err)
-	}
-
-	return nil
-}
-
-// buildServiceInfo creates the service info for this instance
-func (s *Service) buildServiceInfo() *ServiceInfo {
-	hostname, _ := os.Hostname()
-	ip := getLocalIP()
-
-	// Browser-facing nav URL: prefer META_CORE_PUBLIC_URL (the reachable URL
-	// when there's no Caddy perimeter in front — e.g. a debug-direct port),
-	// then BASE_URL (the Caddy perimeter URL), then the internal IP.
-	var baseUrl string
-	if s.config.PublicURL != "" {
-		baseUrl = s.config.PublicURL
-	} else if s.config.BaseURL != "" {
-		baseUrl = s.config.BaseURL
-	} else {
-		baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
-	}
-
-	info := &ServiceInfo{
-		Name:          s.config.ServiceName,
-		Hostname:      hostname,
-		BaseUrl:       baseUrl,
-		Status:        "running",
-		LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
-	}
-
-	return info
-}
-
-// register writes service info to file
-func (s *Service) register() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.writeServiceInfo(s.info)
-}
-
-// writeServiceInfo atomically writes service info to file
-func (s *Service) writeServiceInfo(info *ServiceInfo) error {
-	tempPath := s.serviceFile + ".tmp"
-
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	if err := os.WriteFile(tempPath, data, 0644); err != nil {
-		return err
-	}
-
-	return os.Rename(tempPath, s.serviceFile)
-}
-
-// heartbeatLoop periodically updates the last heartbeat
-func (s *Service) heartbeatLoop() {
-	defer s.wg.Done()
-
-	ticker := time.NewTicker(time.Duration(s.config.HeartbeatIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-s.stopChan:
-			return
-		case <-ticker.C:
-			s.heartbeat()
-		}
-	}
-}
-
-// heartbeat updates the last heartbeat timestamp
-func (s *Service) heartbeat() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return
-	}
-
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-
-	if err := s.writeServiceInfo(s.info); err != nil {
-		log.Printf("[Discovery] Failed to update heartbeat: %v", err)
-	}
-}
-
-// UpdateStatus updates the service status
-func (s *Service) UpdateStatus(status string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return fmt.Errorf("service not registered")
-	}
-
-	s.info.Status = status
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-	return s.writeServiceInfo(s.info)
-}
-
-// Discover finds a service by name
-// Looks for files matching pattern: name-*.json (hostname-based naming)
-func (s *Service) Discover(name string) (*ServiceInfo, error) {
-	// First try exact match for backward compatibility
-	exactPath := filepath.Join(s.servicesDir, name+".json")
-	if data, err := os.ReadFile(exactPath); err == nil {
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err == nil {
-			return s.checkStale(&info), nil
-		}
-	}
-
-	// Search for hostname-based files: name-*.json
-	pattern := filepath.Join(s.servicesDir, name+"-*.json")
-	matches, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-
-	// Return the first valid service found
-	for _, match := range matches {
-		data, err := os.ReadFile(match)
-		if err != nil {
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			continue
-		}
-
-		return s.checkStale(&info), nil
-	}
-
-	return nil, nil
-}
-
-// checkStale marks a service as stale if heartbeat is too old
-func (s *Service) checkStale(info *ServiceInfo) *ServiceInfo {
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err == nil {
-		staleThreshold := time.Duration(s.config.StaleThresholdMS) * time.Millisecond
-		if time.Since(lastHeartbeat) > staleThreshold {
-			info.Status = "stale"
-		}
-	}
-	return info
-}
-
-// DiscoverAll finds all registered services
-func (s *Service) DiscoverAll() ([]*ServiceInfo, error) {
-	entries, err := os.ReadDir(s.servicesDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return []*ServiceInfo{}, nil
-		}
-		return nil, err
-	}
-
-	var services []*ServiceInfo
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(s.servicesDir, entry.Name())
-		data, err := os.ReadFile(filePath)
-		if err != nil {
-			log.Printf("[Discovery] Failed to read service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			log.Printf("[Discovery] Failed to parse service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		services = append(services, s.checkStale(&info))
-	}
-
-	return services, nil
-}
-
-// getLocalIP returns the local IP address
-func getLocalIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		hostname, _ := os.Hostname()
-		return hostname
-	}
-
-	for _, addr := range addrs {
-		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				return ipnet.IP.String()
-			}
-		}
-	}
-
-	hostname, _ := os.Hostname()
-	return hostname
-}

internal/events/meta_publisher.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/redis/go-redis/v9"
+
+	"github.com/metazla/meta-core/internal/storage"
 )
 
 const (
@@ -126,6 +128,14 @@ func (p *MetaPublisher) publishEvent(channel, operation string) {
 	if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
 		return
 	}
+	// Skip the per-record field index. It is bookkeeping (see
+	// storage.FieldsField), not a metadata field — publishing it would make
+	// every downstream consumer that parses `file:<id>/<field>` off this
+	// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
+	// field change on a key they can never read.
+	if strings.HasSuffix(key, "/"+storage.FieldsField) {
+		return
+	}
 
 	// Build event fields
 	fields := map[string]interface{}{

internal/identity/challenge.go

@@ -0,0 +1,126 @@
+package identity
+
+// In-memory, single-use challenge store backing proof-of-possession on the
+// account operations that need it (reveal, delete).
+//
+// In memory on purpose: a challenge is worthless once redeemed and worthless
+// once expired, so persisting it would only widen the window in which a leaked
+// one is useful. A meta-core restart invalidates every challenge in flight,
+// which costs a client one retry.
+//
+// Mirrors the sign-in challenge in meta-watch's src/auth.rs, down to the TTL —
+// the browser signs immediately, so seconds is the right order and a narrow
+// window bounds replay if one ever surfaces in a proxy log.
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+// ChallengeTTL is how long an issued challenge stays answerable.
+const ChallengeTTL = 120 * time.Second
+
+// maxChallenges caps the store. An unauthenticated caller can mint challenges
+// for any uid it can name, so the map is attacker-growable; without a ceiling
+// that is a memory-exhaustion lever. At the cap, issuing fails rather than
+// evicting: dropping someone else's live challenge on demand would let an
+// attacker deny sign-outs, which is worse than making them retry.
+const maxChallenges = 4096
+
+var (
+	// ErrChallengeUnknown covers absent, already-redeemed and expired alike.
+	// One error for all three deliberately: distinguishing them tells a prober
+	// whether a challenge string was ever real.
+	ErrChallengeUnknown = errors.New("challenge is unknown, already used, or expired")
+	ErrChallengeFull    = errors.New("too many challenges in flight; try again shortly")
+)
+
+type challengeEntry struct {
+	uid     string
+	action  Action
+	expires time.Time
+}
+
+// ChallengeStore issues and redeems challenges. Safe for concurrent use.
+type ChallengeStore struct {
+	mu      sync.Mutex
+	entries map[string]challengeEntry
+	ttl     time.Duration
+	// now is swappable so the tests can age entries without sleeping.
+	now func() time.Time
+}
+
+func NewChallengeStore() *ChallengeStore {
+	return &ChallengeStore{
+		entries: make(map[string]challengeEntry),
+		ttl:     ChallengeTTL,
+		now:     time.Now,
+	}
+}
+
+// Issue mints a challenge for (uid, action) and remembers it until redeemed or
+// expired. Returns the literal text the client signs and its expiry.
+func (s *ChallengeStore) Issue(uid string, action Action) (string, time.Time, error) {
+	if !validUID(uid) {
+		return "", time.Time{}, errors.New("invalid uid")
+	}
+	if !ValidAction(action) {
+		return "", time.Time{}, errors.New("unknown action")
+	}
+	text, err := NewChallengeText(uid, action)
+	if err != nil {
+		return "", time.Time{}, err
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.sweepLocked()
+	if len(s.entries) >= maxChallenges {
+		return "", time.Time{}, ErrChallengeFull
+	}
+	expires := s.now().Add(s.ttl)
+	s.entries[text] = challengeEntry{uid: uid, action: action, expires: expires}
+	return text, expires, nil
+}
+
+// Redeem consumes a challenge, asserting it was issued for this exact uid and
+// action. Single-use: the entry is dropped on any lookup that finds it, so a
+// mismatched redemption burns the challenge rather than leaving it to be
+// retried against a different account.
+func (s *ChallengeStore) Redeem(text, uid string, action Action) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	entry, ok := s.entries[text]
+	if !ok {
+		return ErrChallengeUnknown
+	}
+	delete(s.entries, text)
+	if s.now().After(entry.expires) {
+		return ErrChallengeUnknown
+	}
+	if entry.uid != uid || entry.action != action {
+		return ErrChallengeUnknown
+	}
+	// Re-read the signed bytes rather than trusting the map key alone: the
+	// signature covers `text`, so `text` is what has to name this uid+action.
+	if !ChallengeMatches(text, uid, action) {
+		return ErrChallengeUnknown
+	}
+	return nil
+}
+
+// sweepLocked drops expired entries. Caller holds the lock.
+//
+// Called only from Issue: with no background goroutine, expiry is enforced
+// lazily on the one path that grows the map, which is exactly where the bound
+// matters.
+func (s *ChallengeStore) sweepLocked() {
+	now := s.now()
+	for text, e := range s.entries {
+		if now.After(e.expires) {
+			delete(s.entries, text)
+		}
+	}
+}

internal/identity/noble_interop_test.go

@@ -0,0 +1,43 @@
+package identity
+
+import (
+	"encoding/base64"
+	"testing"
+)
+
+// A real signature produced by @noble/secp256k1 v3 — the library the dashboard
+// and meta-watch both sign with — pinned as a fixture and verified here.
+//
+// This is the one seam neither side's own tests can cover: Go signs DER, the
+// browser signs 64-byte compact, and every unit test above verifies signatures
+// this package produced itself. If the encodings ever disagree, the gate fails
+// closed in production (nobody can delete their own account) while every other
+// test stays green. Regenerate with:
+//
+//	node -e 'import("@noble/secp256k1").then(async s => { … })'
+//
+// against the same fixed scalar (32 zero bytes with 42 in the last).
+func TestVerify_AcceptsARealNobleV3Signature(t *testing.T) {
+	const (
+		uid       = "ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY"
+		challenge = "metamesh-identity-v1:delete:ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY:deadbeef"
+		sigB64    = "lIiZGL0j9l/w2WXwzwe56cICnusuYdD3oaTuYfFTtNJBR9MolXFPk3014WQeAlvmbX4LfnW+TQz4asqoSwmSAQ=="
+	)
+
+	sig, err := base64.StdEncoding.DecodeString(sigB64)
+	if err != nil {
+		t.Fatalf("decode fixture: %v", err)
+	}
+	if len(sig) != 64 {
+		t.Fatalf("fixture is %d bytes, expected noble's 64-byte compact form", len(sig))
+	}
+	if err := Verify(uid, []byte(challenge), sig); err != nil {
+		t.Fatalf("a genuine browser signature must verify: %v", err)
+	}
+
+	// And the same signature must not verify over a different challenge — a
+	// verifier that ignored the payload would pass the line above too.
+	if err := Verify(uid, []byte(challenge+"x"), sig); err == nil {
+		t.Fatal("the fixture verified over the wrong payload")
+	}
+}

internal/identity/verify.go

@@ -0,0 +1,160 @@
+package identity
+
+// Signature verification against a uid, and the challenge vocabulary that
+// makes a signature mean "I hold this account's private key" rather than
+// "meta-core signed something for me".
+//
+// The asymmetry that motivates this file: meta-core *holds* every private key,
+// so it can sign as anybody. A signature is therefore only proof of possession
+// if the payload is one meta-core refuses to sign on request — hence
+// ChallengeDomain, which POST /api/identity/sign rejects outright. Without that
+// refusal an attacker mints their own authorisation in one extra call and the
+// whole gate is decoration.
+//
+// Verification itself needs no stored state: a uid *is* the public key
+// ("z" + base58btc(33-byte compressed sec1)), so the pubkey is recovered from
+// the uid the caller names. Mirrors `verify_local` / `verify_local_compact` in
+// meta-watch's src/record/sign.rs — same curve, same digest, both signature
+// encodings.
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/decred/dcrd/dcrec/secp256k1/v4"
+	"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
+	"github.com/mr-tron/base58"
+)
+
+// ChallengeDomain prefixes every byte string that authorises an account
+// operation. POST /api/identity/sign and /sign-batch refuse payloads carrying
+// it, so the only way to produce one is to hold the key yourself.
+//
+// Versioned because it is a wire commitment: changing it invalidates every
+// challenge in flight, which is harmless, but changing it *silently* would let
+// an old-domain payload keep working. Bump it deliberately.
+const ChallengeDomain = "metamesh-identity-v1:"
+
+// Action is the operation a challenge authorises. A challenge minted to reveal
+// a key must not also delete the account, so the action is inside the signed
+// bytes and is re-checked at redemption.
+type Action string
+
+const (
+	ActionReveal Action = "reveal"
+	ActionDelete Action = "delete"
+)
+
+// ValidAction reports whether a is an action a challenge may be issued for.
+func ValidAction(a Action) bool {
+	return a == ActionReveal || a == ActionDelete
+}
+
+// ValidUID exposes the uid shape check to callers outside this package (the
+// API layer, which must reject a hostile uid before it reaches a Redis SCAN
+// pattern or a file path).
+func ValidUID(uid string) bool { return validUID(uid) }
+
+// IsChallengePayload reports whether b is (or claims to be) an account
+// authorisation payload. Used by the signing endpoints to refuse minting one.
+//
+// Deliberately a prefix test on raw bytes rather than a parse: anything that
+// merely *looks* like a challenge is refused, so a malformed or future-version
+// payload fails closed instead of being signed.
+func IsChallengePayload(b []byte) bool {
+	return strings.HasPrefix(string(b), ChallengeDomain)
+}
+
+// PubkeyFromUID recovers the compressed public key a uid encodes.
+func PubkeyFromUID(uid string) (*secp256k1.PublicKey, error) {
+	if !validUID(uid) {
+		return nil, fmt.Errorf("invalid uid: %q", uid)
+	}
+	if !strings.HasPrefix(uid, "z") {
+		return nil, errors.New("uid is not multibase base58btc (missing 'z' prefix)")
+	}
+	raw, err := base58.Decode(uid[1:])
+	if err != nil {
+		return nil, fmt.Errorf("uid is not valid base58: %w", err)
+	}
+	if len(raw) != 33 {
+		return nil, fmt.Errorf("uid decodes to %d bytes, want a 33-byte compressed pubkey", len(raw))
+	}
+	pub, err := secp256k1.ParsePubKey(raw)
+	if err != nil {
+		return nil, fmt.Errorf("uid is not a point on secp256k1: %w", err)
+	}
+	return pub, nil
+}
+
+// Verify checks sig over SHA-256(payload) against the public key inside uid.
+//
+// Accepts both encodings in circulation: DER (what Sign emits, and what User
+// Data Layer records carry) and 64-byte compact r‖s (what the browser emits —
+// noble-secp256k1 v3 dropped DER, and meta-watch's sign-in already sends
+// compact). Rejecting either would break one of the two callers.
+func Verify(uid string, payload, sig []byte) error {
+	pub, err := PubkeyFromUID(uid)
+	if err != nil {
+		return err
+	}
+	parsed, err := parseSignature(sig)
+	if err != nil {
+		return err
+	}
+	digest := sha256.Sum256(payload)
+	if !parsed.Verify(digest[:], pub) {
+		return errors.New("signature does not verify for this uid")
+	}
+	return nil
+}
+
+func parseSignature(sig []byte) (*ecdsa.Signature, error) {
+	if len(sig) == 64 {
+		var r, s secp256k1.ModNScalar
+		// SetByteSlice reports overflow (>= curve order), which is not a valid
+		// scalar. A zero r or s is likewise never produced by a real signer.
+		if r.SetByteSlice(sig[:32]) || r.IsZero() {
+			return nil, errors.New("compact signature has an invalid r")
+		}
+		if s.SetByteSlice(sig[32:]) || s.IsZero() {
+			return nil, errors.New("compact signature has an invalid s")
+		}
+		return ecdsa.NewSignature(&r, &s), nil
+	}
+	parsed, err := ecdsa.ParseDERSignature(sig)
+	if err != nil {
+		return nil, fmt.Errorf("signature is neither 64-byte compact nor valid DER: %w", err)
+	}
+	return parsed, nil
+}
+
+// NewChallengeText mints the literal string a client signs. The uid and action
+// are inside it, so a signature cannot be lifted from one account or operation
+// onto another even if the nonce leaks.
+func NewChallengeText(uid string, action Action) (string, error) {
+	nonce := make([]byte, 32)
+	if _, err := rand.Read(nonce); err != nil {
+		return "", fmt.Errorf("read nonce: %w", err)
+	}
+	return ChallengeDomain + string(action) + ":" + uid + ":" + hex.EncodeToString(nonce), nil
+}
+
+// ChallengeMatches reports whether text is a well-formed challenge for exactly
+// this uid and action. The store already keys on the full text, so this is a
+// belt-and-braces re-read of the signed bytes themselves: it is the signed
+// content, not the lookup key, that a verifier must agree with.
+//
+// Compared in constant time — the text is a bearer secret until redeemed.
+func ChallengeMatches(text, uid string, action Action) bool {
+	want := ChallengeDomain + string(action) + ":" + uid + ":"
+	if len(text) <= len(want) {
+		return false
+	}
+	return subtle.ConstantTimeCompare([]byte(text[:len(want)]), []byte(want)) == 1
+}

internal/identity/verify_test.go

@@ -0,0 +1,278 @@
+package identity
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/decred/dcrd/dcrec/secp256k1/v4"
+	"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
+)
+
+// The whole gate rests on a uid being the public key: verification consults no
+// stored state, so a signature is checked against the account the caller names
+// rather than against whatever key meta-core happens to hold.
+func TestVerify_AcceptsDERFromOurOwnSigner(t *testing.T) {
+	id, err := Generate()
+	if err != nil {
+		t.Fatalf("generate: %v", err)
+	}
+	payload := []byte(ChallengeDomain + "delete:" + id.UID + ":abcdef")
+
+	sig, err := Sign(id, payload)
+	if err != nil {
+		t.Fatalf("sign: %v", err)
+	}
+	if err := Verify(id.UID, payload, sig); err != nil {
+		t.Fatalf("DER signature from Sign must verify: %v", err)
+	}
+}
+
+// The browser is the signer that matters here, and noble-secp256k1 v3 emits
+// 64-byte compact r‖s — it dropped DER entirely. A verifier that took only DER
+// would reject every real proof while passing this package's own tests.
+func TestVerify_AcceptsCompactAsTheBrowserEmitsIt(t *testing.T) {
+	id, err := Generate()
+	if err != nil {
+		t.Fatalf("generate: %v", err)
+	}
+	payload := []byte("some challenge text")
+
+	raw, _ := hex.DecodeString(id.PrivateKeyHex)
+	priv := secp256k1.PrivKeyFromBytes(raw)
+	digest := sha256.Sum256(payload)
+	der := ecdsa.Sign(priv, digest[:])
+
+	// Re-encode the same signature compactly, the way the browser would.
+	parsed, err := ecdsa.ParseDERSignature(der.Serialize())
+	if err != nil {
+		t.Fatalf("parse own DER: %v", err)
+	}
+	r := parsed.R()
+	sc := parsed.S()
+	compact := make([]byte, 0, 64)
+	rb := r.Bytes()
+	sb := sc.Bytes()
+	compact = append(compact, rb[:]...)
+	compact = append(compact, sb[:]...)
+
+	if err := Verify(id.UID, payload, compact); err != nil {
+		t.Fatalf("compact signature must verify: %v", err)
+	}
+}
+
+func TestVerify_RejectsAnotherAccountsSignature(t *testing.T) {
+	alice, _ := Generate()
+	mallory, _ := Generate()
+	payload := []byte("challenge for alice")
+
+	sig, err := Sign(mallory, payload)
+	if err != nil {
+		t.Fatalf("sign: %v", err)
+	}
+	if err := Verify(alice.UID, payload, sig); err == nil {
+		t.Fatal("a signature by mallory must not verify as alice")
+	}
+}
+
+func TestVerify_RejectsTamperedPayload(t *testing.T) {
+	id, _ := Generate()
+	sig, _ := Sign(id, []byte("delete account A"))
+	if err := Verify(id.UID, []byte("delete account B"), sig); err == nil {
+		t.Fatal("a signature must not verify over different bytes")
+	}
+}
+
+func TestVerify_RejectsGarbageSignatureEncodings(t *testing.T) {
+	id, _ := Generate()
+	for name, sig := range map[string][]byte{
+		"empty":            {},
+		"short":            {1, 2, 3},
+		"zero compact":     make([]byte, 64),
+		"not der":          []byte("this is not a signature at all......."),
+		"63 bytes":         make([]byte, 63),
+		"65 bytes non-der": make([]byte, 65),
+	} {
+		if err := Verify(id.UID, []byte("x"), sig); err == nil {
+			t.Errorf("%s: expected rejection", name)
+		}
+	}
+}
+
+func TestPubkeyFromUID_RejectsMalformedUIDs(t *testing.T) {
+	id, _ := Generate()
+	for name, uid := range map[string]string{
+		"empty":          "",
+		"no z prefix":    id.UID[1:],
+		"truncated":      id.UID[:20],
+		"path traversal": "../../etc/passwd",
+		"with slash":     "z/" + id.UID,
+		"not on curve":   "z" + strings.Repeat("1", 45),
+	} {
+		if _, err := PubkeyFromUID(uid); err == nil {
+			t.Errorf("%s (%q): expected rejection", name, uid)
+		}
+	}
+	if _, err := PubkeyFromUID(id.UID); err != nil {
+		t.Fatalf("a real uid must parse: %v", err)
+	}
+}
+
+// The refusal that makes proof-of-possession mean anything on a node holding
+// every private key. If this ever returns false for a challenge, an attacker
+// asks /api/identity/sign to authorise their own delete.
+func TestIsChallengePayload_CatchesAnythingInTheDomain(t *testing.T) {
+	yes := [][]byte{
+		[]byte(ChallengeDomain),
+		[]byte(ChallengeDomain + "delete:zAlice:beef"),
+		[]byte(ChallengeDomain + "some-future-action:whatever"),
+		[]byte("metamesh-identity-v1:"), // literal, in case the const drifts
+	}
+	for _, b := range yes {
+		if !IsChallengePayload(b) {
+			t.Errorf("must refuse to sign %q", b)
+		}
+	}
+	no := [][]byte{
+		{},
+		[]byte("metamesh-identity-v2:delete:zAlice"), // different domain, not ours to refuse
+		[]byte("a record payload"),
+		[]byte(" " + ChallengeDomain), // not a prefix
+	}
+	for _, b := range no {
+		if IsChallengePayload(b) {
+			t.Errorf("must still sign %q", b)
+		}
+	}
+}
+
+func TestChallengeMatches_TiesTheTextToUIDAndAction(t *testing.T) {
+	text, err := NewChallengeText("zAlice", ActionDelete)
+	if err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	if !ChallengeMatches(text, "zAlice", ActionDelete) {
+		t.Fatal("its own uid+action must match")
+	}
+	if ChallengeMatches(text, "zBob", ActionDelete) {
+		t.Fatal("another uid must not match")
+	}
+	if ChallengeMatches(text, "zAlice", ActionReveal) {
+		t.Fatal("another action must not match")
+	}
+	// A prefix-only uid must not slip through: "zAlic" is a prefix of "zAlice",
+	// and a naive HasPrefix on the uid alone would accept it.
+	if ChallengeMatches(text, "zAlic", ActionDelete) {
+		t.Fatal("a uid that is a prefix of the real one must not match")
+	}
+}
+
+func TestNewChallengeText_IsUnpredictable(t *testing.T) {
+	seen := make(map[string]struct{}, 64)
+	for i := 0; i < 64; i++ {
+		text, err := NewChallengeText("zAlice", ActionDelete)
+		if err != nil {
+			t.Fatalf("mint: %v", err)
+		}
+		if _, dup := seen[text]; dup {
+			t.Fatal("challenge repeated — the nonce is not random")
+		}
+		seen[text] = struct{}{}
+	}
+}
+
+// --- challenge store --------------------------------------------------------
+
+func TestChallengeStore_RedeemIsSingleUse(t *testing.T) {
+	s := NewChallengeStore()
+	text, _, err := s.Issue("zAlice", ActionDelete)
+	if err != nil {
+		t.Fatalf("issue: %v", err)
+	}
+	if err := s.Redeem(text, "zAlice", ActionDelete); err != nil {
+		t.Fatalf("first redeem: %v", err)
+	}
+	if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
+		t.Fatal("a challenge must not be redeemable twice — that is replay")
+	}
+}
+
+// A challenge minted to *look at* a key must not delete the account, and one
+// minted for Alice must not act on Bob. Both are inside the signed text, so a
+// store that ignored them would authorise the wrong operation with a perfectly
+// valid signature.
+func TestChallengeStore_RedeemRefusesWrongUIDOrAction(t *testing.T) {
+	s := NewChallengeStore()
+
+	text, _, _ := s.Issue("zAlice", ActionReveal)
+	if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
+		t.Fatal("a reveal challenge must not authorise a delete")
+	}
+
+	text2, _, _ := s.Issue("zAlice", ActionDelete)
+	if err := s.Redeem(text2, "zBob", ActionDelete); err == nil {
+		t.Fatal("alice's challenge must not authorise deleting bob")
+	}
+}
+
+// A failed redemption still consumes the challenge, so a wrong guess costs a
+// round trip rather than leaving the nonce up for another attempt.
+func TestChallengeStore_FailedRedeemStillBurnsIt(t *testing.T) {
+	s := NewChallengeStore()
+	text, _, _ := s.Issue("zAlice", ActionDelete)
+	_ = s.Redeem(text, "zBob", ActionDelete)
+	if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
+		t.Fatal("a burnt challenge must stay burnt for its rightful owner too")
+	}
+}
+
+func TestChallengeStore_Expires(t *testing.T) {
+	s := NewChallengeStore()
+	now := time.Unix(1_700_000_000, 0)
+	s.now = func() time.Time { return now }
+
+	text, expires, _ := s.Issue("zAlice", ActionDelete)
+	if !expires.Equal(now.Add(ChallengeTTL)) {
+		t.Fatalf("expiry %v, want %v", expires, now.Add(ChallengeTTL))
+	}
+	now = now.Add(ChallengeTTL + time.Second)
+	if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
+		t.Fatal("an expired challenge must not redeem")
+	}
+}
+
+// Issuing is unauthenticated, so the map is attacker-growable. The cap is what
+// stops that being a memory-exhaustion lever; expiry sweeping is what stops the
+// cap being reached by honest traffic.
+func TestChallengeStore_CapsInFlightAndSweepsExpired(t *testing.T) {
+	s := NewChallengeStore()
+	now := time.Unix(1_700_000_000, 0)
+	s.now = func() time.Time { return now }
+
+	for i := 0; i < maxChallenges; i++ {
+		if _, _, err := s.Issue("zAlice", ActionDelete); err != nil {
+			t.Fatalf("issue %d: %v", i, err)
+		}
+	}
+	if _, _, err := s.Issue("zAlice", ActionDelete); err == nil {
+		t.Fatal("expected the store to refuse once full")
+	}
+
+	// Once they age out, the next issue sweeps and succeeds again.
+	now = now.Add(ChallengeTTL + time.Second)
+	if _, _, err := s.Issue("zAlice", ActionDelete); err != nil {
+		t.Fatalf("after expiry the store must accept again: %v", err)
+	}
+}
+
+func TestChallengeStore_RefusesInvalidInputs(t *testing.T) {
+	s := NewChallengeStore()
+	if _, _, err := s.Issue("../etc", ActionDelete); err == nil {
+		t.Fatal("a uid that is not uid-shaped must be refused")
+	}
+	if _, _, err := s.Issue("zAlice", Action("sudo")); err == nil {
+		t.Fatal("an unknown action must be refused")
+	}
+}

internal/meshdisco/node.go

@@ -0,0 +1,439 @@
+package meshdisco
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"sort"
+	"sync"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/ipv4"
+	"golang.org/x/sys/unix"
+)
+
+// Config configures a Node. Payload is a function rather than a value so the
+// advertise is rebuilt from live state on every tick — that is what keeps the
+// advertised core URLs and GET /urls from drifting apart.
+type Config struct {
+	Name     string
+	Instance string
+	Version  string
+
+	Group    string
+	Port     int
+	Interval time.Duration
+
+	// Payload returns this node's status ("starting" / "running") and its
+	// resources. The envelope and NodeInfo are filled in by the Node.
+	Payload func() (status string, resources []Resource)
+}
+
+func (c *Config) applyDefaults() {
+	if c.Group == "" {
+		c.Group = DefaultGroup
+	}
+	if c.Port == 0 {
+		c.Port = DefaultPort
+	}
+	if c.Interval <= 0 {
+		c.Interval = DefaultInterval
+	}
+}
+
+type seen struct {
+	node      NodeInfo
+	resources []Resource
+	addr      string
+	lastSeen  time.Time
+}
+
+// Node is one beacon v2 participant: it advertises its resources, answers
+// probes, and keeps a TTL map of every other node it has heard from.
+type Node struct {
+	cfg   Config
+	group net.IP
+
+	pc     *ipv4.PacketConn
+	closer interface{ Close() error }
+
+	// ifaces are the multicast-capable interfaces we joined and send on.
+	// A container attached to two docker networks (metashare-app on `pcs`
+	// and `metamesh-mesh`, metawatch-* on `metawatch` and `metamesh-mesh`)
+	// has two entries here, and MUST be advertised on both — sending via the
+	// default route alone silently covers only one network.
+	ifaces []net.Interface
+
+	// sendMu serialises SetMulticastInterface + WriteTo, which are a
+	// two-step operation on one shared socket.
+	sendMu sync.Mutex
+
+	mu    sync.RWMutex
+	nodes map[string]*seen
+
+	stopChan chan struct{}
+	stopOnce sync.Once
+	wg       sync.WaitGroup
+}
+
+// New creates a Node. It does not touch the network until Start.
+func New(cfg Config) *Node {
+	cfg.applyDefaults()
+	return &Node{
+		cfg:      cfg,
+		nodes:    make(map[string]*seen),
+		stopChan: make(chan struct{}),
+	}
+}
+
+// Start opens the multicast socket, joins the group on every eligible
+// interface, and begins advertising. A join failure on one interface is logged
+// and skipped rather than fatal — a host that blocks multicast degrades
+// instead of failing to boot.
+func (n *Node) Start() error {
+	n.group = net.ParseIP(n.cfg.Group)
+	if n.group == nil || n.group.To4() == nil {
+		return fmt.Errorf("beacon: invalid multicast group %q", n.cfg.Group)
+	}
+
+	lc := net.ListenConfig{Control: reusePort}
+	conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", n.cfg.Port))
+	if err != nil {
+		return fmt.Errorf("beacon: listen on :%d: %w", n.cfg.Port, err)
+	}
+	n.closer = conn
+
+	pc := ipv4.NewPacketConn(conn)
+	// TTL 1: link-local only. Discovery never routes off the local segment;
+	// the two-host topology keeps two separate meshes by design.
+	if err := pc.SetMulticastTTL(1); err != nil {
+		log.Printf("[beacon] Warning: could not set multicast TTL: %v", err)
+	}
+	n.pc = pc
+
+	n.ifaces = eligibleInterfaces()
+	if len(n.ifaces) == 0 {
+		log.Println("[beacon] Warning: no multicast-capable interfaces found; discovery will be inert")
+	}
+
+	group := &net.UDPAddr{IP: n.group}
+	joined := 0
+	for i := range n.ifaces {
+		if err := pc.JoinGroup(&n.ifaces[i], group); err != nil {
+			log.Printf("[beacon] Warning: join %s on %s failed: %v", n.cfg.Group, n.ifaces[i].Name, err)
+			continue
+		}
+		joined++
+	}
+	log.Printf("[beacon] Listening on %s:%d as %s/%s (joined %d of %d interfaces)",
+		n.cfg.Group, n.cfg.Port, n.cfg.Name, n.cfg.Instance, joined, len(n.ifaces))
+
+	n.wg.Add(2)
+	go n.readLoop()
+	go n.advertiseLoop()
+
+	return nil
+}
+
+// Stop multicasts a bye — so neighbours drop us immediately instead of
+// waiting out the staleness window — then closes the socket and waits for the
+// loops to exit.
+func (n *Node) Stop() error {
+	var err error
+	n.stopOnce.Do(func() {
+		n.broadcast(Message{Proto: Proto, V: Version, Type: TypeBye, Node: n.nodeInfo("")})
+		close(n.stopChan)
+		if n.closer != nil {
+			err = n.closer.Close()
+		}
+		n.wg.Wait()
+	})
+	return err
+}
+
+// Probe multicasts a probe on every interface. Every node owning a resource
+// matching one of want (every node, when want is empty) replies immediately,
+// so a caller does not have to wait out an interval.
+func (n *Node) Probe(want ...string) {
+	n.broadcast(Message{Proto: Proto, V: Version, Type: TypeProbe, From: n.cfg.Instance, Want: want})
+}
+
+// Neighbors returns every node heard from within the staleness window,
+// sorted by (name, instance). Expired entries are dropped as a side effect —
+// expiry is a read-time concern, so there is no reaper goroutine.
+func (n *Node) Neighbors() []Neighbor {
+	cutoff := time.Now().Add(-n.cfg.Interval * LivenessFactor)
+
+	n.mu.Lock()
+	out := make([]Neighbor, 0, len(n.nodes))
+	for k, s := range n.nodes {
+		if s.lastSeen.Before(cutoff) {
+			delete(n.nodes, k)
+			continue
+		}
+		out = append(out, newNeighbor(s.node, s.resources, s.addr, s.lastSeen))
+	}
+	n.mu.Unlock()
+
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		return out[i].Instance < out[j].Instance
+	})
+	return out
+}
+
+// NeighborsByName collapses multiple instances of one name down to the most
+// recently seen — the shape the nav menu wants.
+func (n *Node) NeighborsByName() []Neighbor {
+	best := make(map[string]Neighbor)
+	for _, nb := range n.Neighbors() {
+		if cur, ok := best[nb.Name]; !ok || nb.LastSeen > cur.LastSeen {
+			best[nb.Name] = nb
+		}
+	}
+	out := make([]Neighbor, 0, len(best))
+	for _, nb := range best {
+		out = append(out, nb)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Self returns this node as a row, so callers can render themselves without
+// waiting to hear their own multicast echo back.
+func (n *Node) Self() Neighbor {
+	status, resources := n.payload()
+	return newNeighbor(*n.nodeInfo(status), resources, "", time.Now())
+}
+
+func (n *Node) payload() (string, []Resource) {
+	if n.cfg.Payload == nil {
+		return "running", nil
+	}
+	return n.cfg.Payload()
+}
+
+func (n *Node) nodeInfo(status string) *NodeInfo {
+	return &NodeInfo{Name: n.cfg.Name, Instance: n.cfg.Instance, Version: n.cfg.Version, Status: status}
+}
+
+func (n *Node) advertiseMsg() Message {
+	status, resources := n.payload()
+	return Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: n.nodeInfo(status), Resources: resources}
+}
+
+func (n *Node) readLoop() {
+	defer n.wg.Done()
+	buf := make([]byte, ReadBufferSize)
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		default:
+		}
+
+		nRead, _, src, err := n.pc.ReadFrom(buf)
+		if err != nil {
+			select {
+			case <-n.stopChan:
+				return
+			default:
+			}
+			log.Printf("[beacon] Read error: %v", err)
+			continue
+		}
+
+		msg, ok := Parse(buf[:nRead])
+		if !ok {
+			continue // v1 or foreign traffic on the shared group
+		}
+
+		switch msg.Type {
+		case TypeProbe:
+			if msg.From == n.cfg.Instance {
+				continue
+			}
+			ad := n.advertiseMsg()
+			wanted := len(msg.Want) == 0
+			for _, r := range ad.Resources {
+				if r.MatchesAny(msg.Want) {
+					wanted = true
+					break
+				}
+			}
+			if wanted {
+				// Unicast straight back to the prober's source address; its
+				// ephemeral source port is where it is listening.
+				n.send(ad, src)
+			}
+		case TypeAdvertise, TypeBye:
+			// Drop our own multicast echo. Loopback stays enabled so a
+			// single-container test still sees traffic; we filter by identity.
+			if msg.Node.Instance == n.cfg.Instance {
+				continue
+			}
+			n.record(msg, src)
+		}
+	}
+}
+
+func (n *Node) record(msg Message, src net.Addr) {
+	addr := ""
+	if src != nil {
+		if udp, ok := src.(*net.UDPAddr); ok {
+			addr = udp.IP.String()
+		} else {
+			addr = src.String()
+		}
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if msg.Type == TypeBye {
+		delete(n.nodes, msg.Node.Instance)
+		return
+	}
+	n.nodes[msg.Node.Instance] = &seen{node: *msg.Node, resources: msg.Resources, addr: addr, lastSeen: time.Now()}
+}
+
+func (n *Node) advertiseLoop() {
+	defer n.wg.Done()
+
+	// Advertise immediately, then probe once so the map fills without
+	// waiting a full interval for everyone else's ticker.
+	n.broadcast(n.advertiseMsg())
+	n.Probe()
+
+	ticker := time.NewTicker(n.cfg.Interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		case <-ticker.C:
+			n.broadcast(n.advertiseMsg())
+		}
+	}
+}
+
+func encode(msg Message) ([]byte, bool) {
+	body, err := json.Marshal(msg)
+	if err != nil {
+		return nil, false
+	}
+	if len(body) > MaxDatagram {
+		log.Printf("[beacon] Warning: %d-byte datagram exceeds the fragmentation-safe %d", len(body), MaxDatagram)
+	}
+	return body, true
+}
+
+// send unicasts msg to dst (a probe reply).
+func (n *Node) send(msg Message, dst net.Addr) {
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+		log.Printf("[beacon] Reply to %s failed: %v", dst, err)
+	}
+}
+
+// broadcast writes one message to the group once per eligible interface.
+// Writing once and letting the routing table choose would reach exactly one
+// network, which is the failure mode this protocol exists to avoid.
+func (n *Node) broadcast(msg Message) {
+	if n.pc == nil {
+		return
+	}
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	dst := &net.UDPAddr{IP: n.group, Port: n.cfg.Port}
+
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	for i := range n.ifaces {
+		if err := n.pc.SetMulticastInterface(&n.ifaces[i]); err != nil {
+			log.Printf("[beacon] Warning: select interface %s failed: %v", n.ifaces[i].Name, err)
+			continue
+		}
+		if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+			log.Printf("[beacon] Warning: send on %s failed: %v", n.ifaces[i].Name, err)
+		}
+	}
+}
+
+// eligibleInterfaces returns every up, non-loopback, multicast-capable
+// interface that has an IPv4 address.
+func eligibleInterfaces() []net.Interface {
+	all, err := net.Interfaces()
+	if err != nil {
+		log.Printf("[beacon] Warning: could not enumerate interfaces: %v", err)
+		return nil
+	}
+
+	var out []net.Interface
+	for _, ifi := range all {
+		if ifi.Flags&net.FlagUp == 0 ||
+			ifi.Flags&net.FlagLoopback != 0 ||
+			ifi.Flags&net.FlagMulticast == 0 {
+			continue
+		}
+		addrs, err := ifi.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if ipnet, ok := a.(*net.IPNet); ok && ipnet.IP.To4() != nil {
+				out = append(out, ifi)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// reusePort sets SO_REUSEADDR + SO_REUSEPORT so several listeners (a test, a
+// sidecar, a beacon v1 responder in the same netns) can bind 9099 on one host.
+func reusePort(_, _ string, c syscall.RawConn) error {
+	var serr error
+	err := c.Control(func(fd uintptr) {
+		serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
+		if serr == nil {
+			serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return serr
+}
+
+// LocalIPv4 returns this host's first non-loopback IPv4 address, or the
+// hostname when none can be determined. Same fallback the file-based registry
+// used (discovery/service.go), kept identical so announced URLs don't shift.
+func LocalIPv4() string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, addr := range addrs {
+			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+				if ipnet.IP.To4() != nil {
+					return ipnet.IP.String()
+				}
+			}
+		}
+	}
+	host, _ := os.Hostname()
+	return host
+}

internal/meshdisco/proto.go

@@ -0,0 +1,237 @@
+// Package meshdisco implements beacon v2: local resource advertise / discover
+// over UDP multicast (239.255.99.1:9099), the protocol every meta-* service
+// and SDK-hosted plugin uses to find its neighbours.
+//
+// A node advertises Resources, each tagged with capability strings
+// ("metamesh.core", "metamesh.service/meta-sort", "metamesh.transport/nzb@1");
+// consumers keep a live view of everyone else's and filter by capability.
+// meta-core advertises "metamesh.core" carrying its /urls block, which is how
+// services locate it without a shared volume.
+//
+// The wire protocol is documented in docs/project-architecture/beacon-v2.md.
+// Keep this package and that doc in step — the Rust (meta-feeder-sdk),
+// TypeScript and Python implementations are written against the doc.
+package meshdisco
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+)
+
+const (
+	// Proto and Version form the envelope every v2 message carries. Anything
+	// else on the group (beacon v1, junk) is dropped silently.
+	Proto   = "beacon"
+	Version = 2
+
+	// DefaultGroup / DefaultPort are beacon's endpoint, on purpose: one
+	// well-known place for every kind of local resource. v1 and v2 coexist
+	// because v2 never sends v1's "discovery" / "announce" types.
+	DefaultGroup = "239.255.99.1"
+	DefaultPort  = 9099
+
+	// DefaultInterval is how often an unsolicited advertise is emitted.
+	DefaultInterval = 10 * time.Second
+
+	// LivenessFactor multiplies the interval to get the staleness window.
+	// Expiry is evaluated at read time; there is no reaper.
+	LivenessFactor = 3
+
+	// ReadBufferSize is the receive buffer; MaxDatagram is what a sender
+	// stays under so nothing fragments.
+	ReadBufferSize = 8192
+	MaxDatagram    = 1400
+
+	TypeProbe     = "probe"
+	TypeAdvertise = "advertise"
+	TypeBye       = "bye"
+
+	// CapCore is meta-core's capability; its resource carries Data.urls.
+	CapCore = "metamesh.core"
+	// CapServicePrefix + name is every service's capability.
+	CapServicePrefix = "metamesh.service/"
+	CapAnyService    = "metamesh.service/*"
+)
+
+// URLs mirrors meta-core's GET /urls response (api.URLsResponse) minus
+// redisUrl. It travels as the metamesh.core resource's Data.urls, so a service
+// locates meta-core from the advertise alone.
+type URLs struct {
+	Hostname          string `json:"hostname"`
+	BaseUrl           string `json:"baseUrl"`
+	ApiUrl            string `json:"apiUrl"`
+	WebdavUrl         string `json:"webdavUrl"`
+	WebdavUrlInternal string `json:"webdavUrlInternal"`
+}
+
+// NodeInfo says who is advertising.
+type NodeInfo struct {
+	Name     string `json:"name"`
+	Instance string `json:"instance"`
+	Version  string `json:"version,omitempty"`
+	// Status is "starting" | "running"; absent reads as running.
+	Status string `json:"status,omitempty"`
+}
+
+// Resource is one advertised thing.
+type Resource struct {
+	ID   string   `json:"id"`
+	Caps []string `json:"caps"`
+	// Endpoints: well-known names http, ui, manifest, mcp. A value starting
+	// with "/" is relative to endpoints.http.
+	Endpoints map[string]string `json:"endpoints,omitempty"`
+	Rev       string            `json:"rev,omitempty"`
+	// Binds names the one consumer instance this resource belongs to.
+	Binds string          `json:"binds,omitempty"`
+	Data  json.RawMessage `json:"data,omitempty"`
+}
+
+// Matches reports whether any cap matches pattern.
+func (r Resource) Matches(pattern string) bool {
+	for _, c := range r.Caps {
+		if CapMatches(pattern, c) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchesAny reports whether any cap matches any pattern; an empty list matches.
+func (r Resource) MatchesAny(patterns []string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+	for _, p := range patterns {
+		if r.Matches(p) {
+			return true
+		}
+	}
+	return false
+}
+
+// Endpoint returns the named endpoint as an absolute URL.
+func (r Resource) Endpoint(name string) string {
+	v := r.Endpoints[name]
+	if strings.HasPrefix(v, "/") {
+		base := r.Endpoints["http"]
+		if base == "" {
+			return ""
+		}
+		return strings.TrimRight(base, "/") + v
+	}
+	return v
+}
+
+// Message is every v2 datagram.
+type Message struct {
+	Proto     string     `json:"proto"`
+	V         int        `json:"v"`
+	Type      string     `json:"type"`
+	Node      *NodeInfo  `json:"node,omitempty"`
+	Resources []Resource `json:"resources,omitempty"`
+	// From and Want are probe-only.
+	From string   `json:"from,omitempty"`
+	Want []string `json:"want,omitempty"`
+}
+
+// Parse decodes a datagram, returning ok=false for anything that is not a
+// well-formed beacon v2 message (v1 traffic, other versions, unknown types,
+// an advertise/bye without a node).
+func Parse(b []byte) (Message, bool) {
+	var m Message
+	if err := json.Unmarshal(b, &m); err != nil {
+		return m, false
+	}
+	if m.Proto != Proto || m.V != Version {
+		return m, false
+	}
+	switch m.Type {
+	case TypeProbe:
+		return m, true
+	case TypeAdvertise, TypeBye:
+		if m.Node == nil || m.Node.Name == "" || m.Node.Instance == "" {
+			return m, false
+		}
+		return m, true
+	}
+	return m, false
+}
+
+func splitContract(s string) (string, string, bool) {
+	if i := strings.LastIndex(s, "@"); i >= 0 {
+		return s[:i], s[i+1:], true
+	}
+	return s, "", false
+}
+
+// CapMatches reports whether capability cap satisfies pattern:
+// "*" matches everything; an "@N" on the pattern must equal the cap's
+// contract (none on the pattern ignores it); "x/*" matches any variant of x;
+// otherwise the bases must be equal.
+func CapMatches(pattern, cap string) bool {
+	if pattern == "*" {
+		return true
+	}
+	pbase, pc, phas := splitContract(pattern)
+	cbase, cc, chas := splitContract(cap)
+	if phas && (!chas || pc != cc) {
+		return false
+	}
+	if prefix, ok := strings.CutSuffix(pbase, "/*"); ok {
+		rest, ok := strings.CutPrefix(cbase, prefix)
+		return ok && strings.HasPrefix(rest, "/") && len(rest) > 1
+	}
+	return pbase == cbase
+}
+
+// Neighbor is a node heard on the wire, as served by /api/neighbors.
+type Neighbor struct {
+	NodeInfo
+	Resources []Resource `json:"resources"`
+	// Caps flattens every resource's caps, for consumers that just filter.
+	Caps []string `json:"caps"`
+	// BaseUrl is the metamesh.service/* resource's endpoints.ui — kept so
+	// v1-era readers (the nav menu) keep working.
+	BaseUrl string `json:"baseUrl,omitempty"`
+	// Addr is the packet's source address, never a payload claim.
+	Addr string `json:"addr"`
+	// LastSeen is unix seconds.
+	LastSeen float64 `json:"lastSeen"`
+}
+
+// Matches reports whether any resource matches pattern.
+func (n Neighbor) Matches(pattern string) bool {
+	for _, r := range n.Resources {
+		if r.Matches(pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+func newNeighbor(node NodeInfo, resources []Resource, addr string, seen time.Time) Neighbor {
+	nb := Neighbor{
+		NodeInfo:  node,
+		Resources: resources,
+		Caps:      []string{},
+		Addr:      addr,
+		LastSeen:  float64(seen.UnixNano()) / 1e9,
+	}
+	if nb.Resources == nil {
+		nb.Resources = []Resource{}
+	}
+	seenCap := map[string]bool{}
+	for _, r := range resources {
+		for _, c := range r.Caps {
+			if !seenCap[c] {
+				seenCap[c] = true
+				nb.Caps = append(nb.Caps, c)
+			}
+		}
+		if nb.BaseUrl == "" && r.Matches(CapAnyService) {
+			nb.BaseUrl = r.Endpoint("ui")
+		}
+	}
+	return nb
+}

internal/meshdisco/proto_test.go

@@ -0,0 +1,140 @@
+package meshdisco
+
+import (
+	"encoding/json"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestCapMatches(t *testing.T) {
+	cases := []struct {
+		pattern, cap string
+		want         bool
+	}{
+		{"metamesh.transport/*", "metamesh.transport/nzb@1", true},
+		{"metamesh.transport/*", "metamesh.transport/ipfs", true},
+		{"metamesh.transport/*", "metamesh.transport", false},
+		{"metamesh.transport/*", "metamesh.transportx/a", false},
+		{"metamesh.transport/*", "metamesh.transport/", false},
+		{"metamesh.transport/nzb", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@1", false},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb", false},
+		{"metamesh.transport/*@1", "metamesh.transport/nzb@1", true},
+		{"metamesh.core", "metamesh.core", true},
+		{"metamesh.core", "metamesh.core2", false},
+		{"mcp", "mcp/x", false},
+		{"*", "anything@3", true},
+	}
+	for _, c := range cases {
+		if got := CapMatches(c.pattern, c.cap); got != c.want {
+			t.Errorf("CapMatches(%q, %q) = %v, want %v", c.pattern, c.cap, got, c.want)
+		}
+	}
+}
+
+func TestParseRejectsV1AndForeign(t *testing.T) {
+	for _, raw := range []string{
+		`{"type":"discovery"}`,
+		`{"type":"announce","name":"x","tools":[]}`,
+		`{"v":1,"type":"announce","name":"meta-sort"}`,
+		`{"proto":"beacon","v":3,"type":"probe"}`,
+		`{"proto":"beacon","v":2,"type":"announce"}`,
+		`{"proto":"beacon","v":2,"type":"advertise"}`,
+		"\x00garbage",
+	} {
+		if _, ok := Parse([]byte(raw)); ok {
+			t.Errorf("Parse(%q) accepted", raw)
+		}
+	}
+	if _, ok := Parse([]byte(`{"proto":"beacon","v":2,"type":"probe","want":["metamesh.core"]}`)); !ok {
+		t.Error("valid probe rejected")
+	}
+}
+
+func TestNeighborRowCarriesCapsAndBaseUrl(t *testing.T) {
+	nb := newNeighbor(
+		NodeInfo{Name: "meta-sort", Instance: "metasort-app"},
+		[]Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}, Endpoints: map[string]string{"ui": "https://x"}}},
+		"10.0.0.1", time.Now(),
+	)
+	if nb.BaseUrl != "https://x" || len(nb.Caps) != 1 || !nb.Matches(CapAnyService) {
+		t.Fatalf("unexpected row: %+v", nb)
+	}
+}
+
+// End to end over loopback: a probe with a non-matching want gets no reply;
+// a matching one gets the advertise; advertise then bye updates the view.
+func TestProbeAndRecord(t *testing.T) {
+	port := freePort(t)
+	n := New(Config{
+		Name: "meta-core", Instance: "core-1", Port: port, Interval: time.Hour,
+		Payload: func() (string, []Resource) {
+			return "running", []Resource{{ID: "core", Caps: []string{CapCore}}}
+		},
+	})
+	if err := n.Start(); err != nil {
+		t.Fatal(err)
+	}
+	defer n.Stop()
+
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	dst := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}
+	buf := make([]byte, ReadBufferSize)
+
+	send := func(m Message) {
+		b, _ := json.Marshal(m)
+		if _, err := c.WriteToUDP(b, dst); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{"metamesh.service/*"}})
+	c.SetReadDeadline(time.Now().Add(300 * time.Millisecond))
+	if _, _, err := c.ReadFromUDP(buf); err == nil {
+		t.Fatal("non-matching probe got a reply")
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{CapCore}})
+	c.SetReadDeadline(time.Now().Add(2 * time.Second))
+	nr, _, err := c.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatalf("matching probe got no reply: %v", err)
+	}
+	if m, ok := Parse(buf[:nr]); !ok || m.Type != TypeAdvertise || m.Node.Instance != "core-1" {
+		t.Fatalf("bad reply: %s", buf[:nr])
+	}
+
+	peer := &NodeInfo{Name: "meta-sort", Instance: "sort-1"}
+	send(Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: peer,
+		Resources: []Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}}}})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 1 })
+	send(Message{Proto: Proto, V: Version, Type: TypeBye, Node: peer})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 0 })
+}
+
+func freePort(t *testing.T) int {
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	return c.LocalAddr().(*net.UDPAddr).Port
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("condition not reached")
+}

internal/mounts/poller.go

@@ -44,6 +44,8 @@ type PollingStatus struct {
 	CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
 	LastScanDurationMs   int64 `json:"lastScanDurationMs,omitempty"`   // duration of the previous scan
 	NextScanAt           int64 `json:"nextScanAt,omitempty"`           // ms; planned start of next adaptive scan
+	LastFileCount        int   `json:"-"`                              // last observed file count; used to suppress no-change scan logs
+	scanned              bool  // true once at least one scan has completed for this mount
 }
 
 // Poller manages per-mount polling goroutines
@@ -309,16 +311,22 @@ func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
 	end := time.Now()
 	duration := end.Sub(start)
 
+	// Only log a completed scan when the file count changed (or on the very
+	// first scan) so a steady mount doesn't emit one line per poll tick.
+	changed := true
 	p.mu.Lock()
 	if status, exists := p.status[mountID]; exists {
 		status.Scanning = false
 		status.CurrentScanStartedAt = 0
 		status.LastScan = end.UnixMilli()
 		status.LastScanDurationMs = duration.Milliseconds()
+		changed = !status.scanned || status.LastFileCount != fileCount
+		status.LastFileCount = fileCount
+		status.scanned = true
 	}
 	p.mu.Unlock()
 
-	if err == nil {
+	if err == nil && changed {
 		log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
 	}
 

internal/snapshot/manifest.go

@@ -5,11 +5,13 @@
 // reject archives whose schemaVersion exceeds what they know.
 //
 // v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries
-// under metadata/<root>.json carry the flat string properties; the cids
+// under metadata/<root>.json carry the flat string properties; the cid
-// and duplicates Redis SETs are NOT serialized yet — they are rebuilt
+// reverse-index and the duplicates SET are NOT serialized — they are rebuilt
-// transparently on import because SetMetadataFlat / MergeMetadataFlat
+// transparently on import because the cids/<cid> key-set members ARE flat
-// auto-register reverse-index aliases for any cid_*/midhash256 field they
+// properties, and SetMetadataFlat / MergeMetadataFlat auto-register a
-// see (see internal/storage/cid_resolution.go).
+// reverse-index alias for every one they see (see
+// internal/storage/cid_resolution.go: cidFromKeysetField). The legacy flat
+// cid_*/midhash256 named fields are not recognised.
 //
 // v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc".
 // v1 archives can no longer be imported — they would resurrect the

internal/storage/cid_resolution.go

@@ -83,6 +83,7 @@ func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
 	pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
 	pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
 	pipe.SAdd(ctx, c.buildIndexKey(), uuid)
+	c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
 	if _, err := pipe.Exec(ctx); err != nil {
 		return "", fmt.Errorf("mint pipeline: %w", err)
 	}
@@ -116,7 +117,13 @@ func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error
 		return fmt.Errorf("uuid and cid are required")
 	}
 
-	keysetKey := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + cidStr
+	keysetField := CIDsKeyPrefix + cidStr
+	keysetKey := c.buildKeyPrefix(uuid) + keysetField
+
+	// Every branch below writes the key-set member, so index the field name
+	// once here — including the two guard branches that skip the alias itself.
+	// Missing it would leave `cids/<cid>` invisible to a record read.
+	defer c.addFieldLocked(ctx, uuid, keysetField)
 
 	// Self-pointing alias guard. If the caller is trying to register
 	// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
@@ -150,6 +157,42 @@ func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error
 	return nil
 }
 
+// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index
+// entry cid:<cid> → <uuid>. The caller must already hold c.mu, and must delete
+// the cids/<cid> key-set member itself (DeleteProperty does both).
+//
+// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat
+// file:<uuid>/cids/<cid>, the __fields__ index entry, and this reverse alias —
+// while the delete path only ever removed the first two. A "deleted" CID
+// therefore still resolved: GET /meta/<cid>, ResolveRoot and GetByCID all kept
+// answering with the record. The asymmetry is not theoretical. meta-share
+// published dag-pb roots computed over RAR volumes and par2 files onto six
+// records on watch.nsl.sh; removing the member was expressible, removing the
+// claim was not, so the false identity survived its own retraction.
+//
+// Guarded, not blind: the alias is removed only when it currently points at
+// THIS uuid. Two records can legitimately race to claim a cid (see the
+// dual-root guard in addAliasLocked, which leaves the first writer's alias in
+// place); deleting a member from the loser must not unpick the winner's index.
+func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error {
+	if uuid == "" || cidStr == "" {
+		return nil
+	}
+	indexKey := c.buildCIDIndexKey(cidStr)
+	existing, err := c.client.Get(ctx, indexKey).Result()
+	if err == redis.Nil {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("read existing alias: %w", err)
+	}
+	if existing != uuid {
+		// Somebody else owns this alias. Leave it.
+		return nil
+	}
+	return c.client.Del(ctx, indexKey).Err()
+}
+
 // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
 // lookup, an SSE consumer) to the actual storage root key.
 //
@@ -233,26 +276,13 @@ func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
 	return added > 0, nil
 }
 
-// cidsForRootLocked scans the bare-CID key-set for a root and returns its
+// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
-// members (bare CIDs, with the cids/ prefix stripped). Caller must hold c.mu.
+// prefix stripped). Caller must hold c.mu.
+//
+// Reads the field index and filters by prefix; only an un-indexed root falls
+// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
 func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
-	prefix := c.buildKeyPrefix(uuid) + CIDsKeyPrefix
+	return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
-	var out []string
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan cids: %w", err)
-		}
-		for _, k := range keys {
-			out = append(out, strings.TrimPrefix(k, prefix))
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-	return out, nil
 }
 
 // DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
@@ -315,6 +345,10 @@ func (c *Client) DeleteRoot(uuid string) error {
 	if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
 		return fmt.Errorf("srem index: %w", err)
 	}
+	// Marked so the incremental search-index refresh reads it, finds it gone,
+	// and evicts it — otherwise a deleted root keeps answering searches until
+	// the next full rebuild.
+	c.markDirtyLocked(ctx, uuid)
 	return nil
 }
 
@@ -424,3 +458,50 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
 		Duplicates: dups,
 	}, nil
 }
+
+// RegisterFileTuple records one physical file under its midhash256 CID, the
+// same way the file watcher does for every file it scans: mint a UUID root
+// (filePath/sizeByte/mtimeNano) and write `cids/<midhash>`, whose hook
+// registers the cid:<midhash> reverse alias. Content already known under
+// another path is recorded as a duplicate; the same path just refreshes
+// size/mtime.
+//
+// Returns the root and whether it was newly minted. Shared by the watcher and
+// POST /api/files/register, which is how a box running with the watcher off
+// (a meta-watch client core) still gets files it writes under /files/plugin
+// resolvable by CID.
+func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
+	if midhash == "" {
+		return "", false, fmt.Errorf("midhash is required")
+	}
+	uuid, err := c.GetByCID(midhash)
+	if err != nil {
+		return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
+	}
+
+	if uuid == "" {
+		uuid, err = c.Mint(absPath, size, mtimeNano)
+		if err != nil {
+			return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
+		}
+		if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
+			return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
+		}
+		return uuid, true, nil
+	}
+
+	existing, _ := c.GetProperty(uuid, "filePath")
+	if existing != absPath {
+		if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
+			return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
+		}
+		return uuid, false, nil
+	}
+	if _, err := c.MergeMetadataFlat(uuid, map[string]string{
+		"sizeByte":  strconv.FormatInt(size, 10),
+		"mtimeNano": strconv.FormatInt(mtimeNano, 10),
+	}); err != nil {
+		return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
+	}
+	return uuid, false, nil
+}

internal/storage/cid_retraction_test.go

@@ -0,0 +1,158 @@
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+// Tests for retracting a `cids/<cid>` key-set member (cid_resolution.go's
+// removeAliasLocked, reached through DeleteProperty).
+//
+// Why this is worth its own file. Adding a member writes THREE keys — the flat
+// `file:<uuid>/cids/<cid>`, the `__fields__` index entry, and the reverse alias
+// `cid:<cid>` → `<uuid>`. Until removeAliasLocked existed, deleting removed only
+// the first two, so a retracted CID kept resolving to the record: the member was
+// gone from the document and the claim was not.
+//
+// That asymmetry had a live consequence. meta-share minted dag-pb roots and
+// midhash256s over whatever `archive::pick_media` elected, which on an
+// obfuscated posting was an arbitrary RAR volume or a par2 file. Six records on
+// watch.nsl.sh carry such members. `METADATA_KEYS.md` §2 defines a `cids/`
+// member as a digest of *this record's file* and notes that every member is a
+// reverse index back to the record — so those entries assert that an archive
+// volume is the episode. Retracting one has to remove the assertion, not just
+// the row.
+
+// aliasTarget returns the uuid that cid:<cidStr> currently resolves to, or "".
+func aliasTarget(t *testing.T, c *Client, cidStr string) string {
+	t.Helper()
+	got, err := c.client.Get(context.Background(), c.buildCIDIndexKey(cidStr)).Result()
+	if err != nil {
+		return ""
+	}
+	return got
+}
+
+func hasField(t *testing.T, c *Client, hashID, field string) bool {
+	t.Helper()
+	ok, err := c.client.SIsMember(context.Background(), c.buildFieldsKey(hashID), field).Result()
+	if err != nil {
+		t.Fatalf("sismember: %v", err)
+	}
+	return ok
+}
+
+// The real shape: `…glnrqztt…` is the nzb-release locator whose record published
+// a dag-pb root computed over volume 3 of a 35-volume RAR5 set.
+const (
+	poisonedRoot = "bafybeicicuzdvuztca2hp2g2k7scgr4nnibilzgoppljeprphybqot7v6i"
+	goodMidhash  = "bagacbabaed6uiulnv7tscpxne22eztkjvnxg5almn43dpfecppixarfe7xhlu"
+)
+
+func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const uuid = "01KSBHMTESTRECORD"
+
+		if err := c.SetMetadataFlat(uuid, map[string]string{
+			"cids/" + poisonedRoot: "true",
+			"cids/" + goodMidhash:  "true",
+			"fileType":             "video",
+		}); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+
+		// All three writes landed, and the CID resolves to the record.
+		if got := aliasTarget(t, c, poisonedRoot); got != uuid {
+			t.Fatalf("alias should point at the record before retraction, got %q", got)
+		}
+		if !hasField(t, c, uuid, "cids/"+poisonedRoot) {
+			t.Fatal("field index should list the member before retraction")
+		}
+		if got := c.ResolveRoot(poisonedRoot); got != uuid {
+			t.Fatalf("ResolveRoot should find the record before retraction, got %q", got)
+		}
+
+		if err := c.DeleteProperty(uuid, "cids/"+poisonedRoot); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+
+		// 1. the flat key
+		if v, err := c.client.Get(context.Background(),
+			c.buildKeyPrefix(uuid)+"cids/"+poisonedRoot).Result(); err == nil {
+			t.Fatalf("flat key should be gone, still %q", v)
+		}
+		// 2. the field index
+		if hasField(t, c, uuid, "cids/"+poisonedRoot) {
+			t.Fatal("field index still lists the retracted member")
+		}
+		// 3. the reverse alias — the one that used to survive
+		if got := aliasTarget(t, c, poisonedRoot); got != "" {
+			t.Fatalf("reverse alias survived retraction, still points at %q", got)
+		}
+		if got := c.ResolveRoot(poisonedRoot); got == uuid {
+			t.Fatal("a retracted cid must not resolve to the record any more")
+		}
+
+		// The siblings are untouched: retraction is surgical, not a purge.
+		if got := aliasTarget(t, c, goodMidhash); got != uuid {
+			t.Fatalf("an unrelated sibling lost its alias, got %q", got)
+		}
+		if !hasField(t, c, uuid, "cids/"+goodMidhash) {
+			t.Fatal("an unrelated sibling lost its field-index entry")
+		}
+		if v, err := c.GetProperty(uuid, "fileType"); err != nil || v != "video" {
+			t.Fatalf("an unrelated field changed: %q %v", v, err)
+		}
+	})
+}
+
+// Two records can race to claim one cid; addAliasLocked leaves the first
+// writer's alias in place. Retracting the loser's member must not unpick the
+// winner's index — which is why removeAliasLocked compares before deleting.
+func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const winner = "01KSBHMWINNER"
+		const loser = "01KSBHMLOSER"
+		const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m"
+
+		if err := c.SetMetadataFlat(winner, map[string]string{"cids/" + shared: "true"}); err != nil {
+			t.Fatalf("seed winner: %v", err)
+		}
+		if err := c.SetMetadataFlat(loser, map[string]string{"cids/" + shared: "true"}); err != nil {
+			t.Fatalf("seed loser: %v", err)
+		}
+		if got := aliasTarget(t, c, shared); got != winner {
+			t.Fatalf("the first writer should own the alias, got %q", got)
+		}
+
+		if err := c.DeleteProperty(loser, "cids/"+shared); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+
+		if got := aliasTarget(t, c, shared); got != winner {
+			t.Fatalf("retracting the loser's member stole the winner's alias, now %q", got)
+		}
+		if hasField(t, c, loser, "cids/"+shared) {
+			t.Fatal("the loser's own member should still be gone")
+		}
+	})
+}
+
+// A non-cids field has no alias to clean up, and the delete must not invent one.
+func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const uuid = "01KSBHMORDINARY"
+		if err := c.SetMetadataFlat(uuid, map[string]string{
+			"cids/" + goodMidhash: "true",
+			"quality":             "1080p",
+		}); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+		if err := c.DeleteProperty(uuid, "quality"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		if got := aliasTarget(t, c, goodMidhash); got != uuid {
+			t.Fatalf("deleting an ordinary field disturbed a cid alias, got %q", got)
+		}
+	})
+}

internal/storage/clear_test.go

@@ -0,0 +1,138 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+)
+
+// newTestClient spins an in-memory Redis and returns a connected Client with
+// the given namespace prefix. The server is torn down with the test.
+func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
+	t.Helper()
+
+	s := miniredis.RunT(t)
+	c := NewClient(prefix)
+	if err := c.Connect("redis://" + s.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return c, s
+}
+
+// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
+// leak: ClearAllMetadata used to delete every file:<root>/* key and the
+// file:__index__ set, but left the cid:<cid> → root reverse-index entries
+// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
+// a surviving entry silently redirects a later write into a deleted root.
+func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
+	for _, prefix := range []string{"", "mm:"} {
+		name := "no-prefix"
+		if prefix != "" {
+			name = "prefixed"
+		}
+		t.Run(name, func(t *testing.T) {
+			c, s := newTestClient(t, prefix)
+
+			// Two records, each with several CID aliases (a midhash plus
+			// siblings) — exactly what the watcher produces.
+			aliases := map[string][]string{}
+			for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
+				uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
+				if err != nil {
+					t.Fatalf("mint: %v", err)
+				}
+				cids := []string{
+					"bafkmid" + uuid,
+					"bafkfull" + uuid,
+				}
+				for _, cidStr := range cids {
+					if err := c.AddAlias(uuid, cidStr); err != nil {
+						t.Fatalf("add alias %s: %v", cidStr, err)
+					}
+				}
+				aliases[uuid] = cids
+			}
+
+			// An entry already orphaned by a previous (buggy) clear: a
+			// reverse-index key whose root no longer exists. The sweep is
+			// expected to collect it too.
+			if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
+				t.Fatalf("seed orphan: %v", err)
+			}
+
+			// Sanity: the aliases resolve before the clear.
+			for uuid, cids := range aliases {
+				for _, cidStr := range cids {
+					if got := c.ResolveRoot(cidStr); got != uuid {
+						t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
+					}
+				}
+			}
+
+			deleted, err := c.ClearAllMetadata()
+			if err != nil {
+				t.Fatalf("ClearAllMetadata: %v", err)
+			}
+			// deletedCount counts files, not keys.
+			if deleted != 2 {
+				t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
+			}
+
+			// No reverse-index key may survive — not for the cleared records,
+			// not the pre-existing orphan.
+			if leaked := s.Keys(); len(leaked) != 0 {
+				t.Errorf("keys survived the clear: %v", leaked)
+			}
+
+			// And the aliases must no longer resolve: ResolveRoot returns the
+			// hash unchanged when there is no index entry.
+			for _, cids := range aliases {
+				for _, cidStr := range cids {
+					if got := c.ResolveRoot(cidStr); got != cidStr {
+						t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
+					}
+				}
+			}
+			if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
+				t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
+			}
+		})
+	}
+}
+
+// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
+// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
+// so a clear on a prefixed client cannot reach another namespace's index.
+func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
+	c, s := newTestClient(t, "mm:")
+
+	uuid, err := c.Mint("watch/a.mkv", 100, 1000)
+	if err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	if err := c.AddAlias(uuid, "bafkmine"); err != nil {
+		t.Fatalf("add alias: %v", err)
+	}
+
+	// Another namespace's reverse-index entry, plus an unrelated key.
+	if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
+		t.Fatalf("seed foreign cid: %v", err)
+	}
+	if err := s.Set("unrelated", "keep-me"); err != nil {
+		t.Fatalf("seed unrelated: %v", err)
+	}
+
+	if _, err := c.ClearAllMetadata(); err != nil {
+		t.Fatalf("ClearAllMetadata: %v", err)
+	}
+
+	for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
+		if !s.Exists(k) {
+			t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
+		}
+	}
+	if s.Exists(c.buildCIDIndexKey("bafkmine")) {
+		t.Errorf("own reverse-index entry survived the clear")
+	}
+}

internal/storage/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -109,8 +110,12 @@ func (c *Client) buildIndexKey() string {
 	return c.buildKey("file:__index__")
 }
 
-// GetMetadataFlat retrieves all metadata for a file as a flat map
+// GetMetadataFlat retrieves all metadata for a file as a flat map.
-// Uses flat keys: SCAN file:{hashId}/* then MGET
+//
+// Reads the record's field-name index and MGETs those keys — two round-trips,
+// independent of keyspace size. Records written before the index existed fall
+// back to the old SCAN and are indexed on the way out. See field_index.go for
+// why the index exists and why it is safe to derive.
 func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
@@ -122,41 +127,7 @@ func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	prefix := c.buildKeyPrefix(hashID)
+	return c.readMetadataLocked(ctx, hashID)
-	result := make(map[string]string)
-
-	// Scan for all keys with this prefix
-	var cursor uint64
-	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan failed: %w", err)
-		}
-
-		if len(keys) > 0 {
-			values, err := c.client.MGet(ctx, keys...).Result()
-			if err != nil {
-				return nil, fmt.Errorf("mget failed: %w", err)
-			}
-			for i, key := range keys {
-				field := strings.TrimPrefix(key, prefix)
-				if values[i] != nil {
-					result[field] = values[i].(string)
-				}
-			}
-		}
-
-		cursor = nextCursor
-		if cursor == 0 {
-			break
-		}
-	}
-
-	if len(result) == 0 {
-		return nil, nil
-	}
-
-	return result, nil
 }
 
 // SetMetadataFlat stores metadata for a file using flat keys
@@ -179,9 +150,21 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 	prefix := c.buildKeyPrefix(hashID)
 	pipe := c.client.Pipeline()
 
+	fields := make([]string, 0, len(metadata))
 	for field, value := range metadata {
+		// A payload carrying the reserved name (a hand-made snapshot, a buggy
+		// client) would SET a string over the index SET and break every
+		// subsequent read of this record. Drop it — it is derived state, so
+		// there is never a reason to accept it from outside.
+		if isReservedFieldName(field) {
+			continue
+		}
 		pipe.Set(ctx, prefix+field, value, 0)
+		fields = append(fields, field)
 	}
+	// Same pipeline as the SETs, so indexing the field names costs no extra
+	// round-trip. See field_index.go.
+	c.addFieldsToPipe(ctx, pipe, hashID, fields)
 
 	if _, err := pipe.Exec(ctx); err != nil {
 		return fmt.Errorf("pipeline exec failed: %w", err)
@@ -191,9 +174,11 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
-	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
+	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
-	// in the payload. See cid_resolution.go for the rules.
+	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
+	// midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField).
 	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
 
 	return nil
@@ -255,6 +240,12 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
+	// See SetMetadataFlat: the reserved name addresses the derived index SET,
+	// not a metadata field.
+	if isReservedFieldName(property) {
+		return fmt.Errorf("field name %q is reserved", property)
+	}
+
 	key := c.buildKeyPrefix(hashID) + property
 
 	// Use SET to set the key
@@ -262,10 +253,13 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 		return fmt.Errorf("set failed: %w", err)
 	}
 
+	c.addFieldLocked(ctx, hashID, property)
+
 	// Add to index set
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	// Auto-register reverse-index alias if this is a CID field.
 	_ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value)
@@ -334,6 +328,10 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) {
 	if err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return deletedCount, fmt.Errorf("srem index failed: %w", err)
 	}
+	// Marked, not unmarked: the incremental refresh reads the record, finds it
+	// gone, and evicts it from the search index. Skipping this would leave a
+	// deleted record answering searches until the next full rebuild.
+	c.markDirtyLocked(ctx, hashID)
 
 	return deletedCount, nil
 }
@@ -386,32 +384,14 @@ func normalizeFilesRelativePath(p string) string {
 // getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock.
 // Callers must already hold c.mu.
 func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) {
-	prefix := c.buildKeyPrefix(hashID)
+	result, err := c.readMetadataLocked(ctx, hashID)
-	result := make(map[string]string)
+	if err != nil {
-
+		return nil, err
-	var cursor uint64
+	}
-	for {
+	// Unlike GetMetadataFlat, this variant hands back an empty map rather than
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+	// nil for a record with no fields — GetMetadataDocument writes into it.
-		if err != nil {
+	if result == nil {
-			return nil, fmt.Errorf("scan failed: %w", err)
+		result = make(map[string]string)
-		}
-		if len(keys) > 0 {
-			values, err := c.client.MGet(ctx, keys...).Result()
-			if err != nil {
-				return nil, fmt.Errorf("mget failed: %w", err)
-			}
-			for i, key := range keys {
-				if values[i] == nil {
-					continue
-				}
-				field := strings.TrimPrefix(key, prefix)
-				result[field] = values[i].(string)
-			}
-		}
-		cursor = nextCursor
-		if cursor == 0 {
-			break
-		}
 	}
 	return result, nil
 }
@@ -448,9 +428,17 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 	prefix := c.buildKeyPrefix(hashID)
 	pipe := c.client.Pipeline()
 
+	fields := make([]string, 0, len(metadata))
 	for field, value := range metadata {
+		// See SetMetadataFlat: never let an inbound payload overwrite the
+		// derived index SET with a string.
+		if isReservedFieldName(field) {
+			continue
+		}
 		pipe.Set(ctx, prefix+field, value, 0)
+		fields = append(fields, field)
 	}
+	c.addFieldsToPipe(ctx, pipe, hashID, fields)
 
 	if _, err := pipe.Exec(ctx); err != nil {
 		return 0, fmt.Errorf("pipeline exec failed: %w", err)
@@ -460,9 +448,11 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return len(metadata), fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
-	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
+	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
-	// in the payload. See cid_resolution.go for the rules.
+	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
+	// midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField).
 	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
 
 	return len(metadata), nil
@@ -470,6 +460,10 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 
 // DeleteProperty deletes a single property
 // Uses flat key: DEL file:{hashId}/{property}
+//
+// For a cids/<cid> member this is the full inverse of the write: the flat key,
+// the __fields__ entry AND the cid:<cid> reverse alias all go. Anything less
+// leaves the CID still resolving to this record — see removeAliasLocked.
 func (c *Client) DeleteProperty(hashID, property string) error {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
@@ -482,7 +476,19 @@ func (c *Client) DeleteProperty(hashID, property string) error {
 	defer cancel()
 
 	key := c.buildKeyPrefix(hashID) + property
-	return c.client.Del(ctx, key).Err()
+	if err := c.client.Del(ctx, key).Err(); err != nil {
+		return err
+	}
+	// Before the field index forgets the name, use it: a cids/ member owns a
+	// reverse-index entry that nothing else will ever clean up.
+	if cidStr := cidFromKeysetField(property); cidStr != "" {
+		if err := c.removeAliasLocked(ctx, hashID, cidStr); err != nil {
+			return fmt.Errorf("remove alias for %s: %w", cidStr, err)
+		}
+	}
+	c.removeFieldLocked(ctx, hashID, property)
+	c.markDirtyLocked(ctx, hashID)
+	return nil
 }
 
 // AddToSet adds a value to a set-type field (stored as pipe-delimited string in flat key)
@@ -528,10 +534,14 @@ func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
 		return false, err
 	}
 
+	// The SET above may have created the key, so the field name is new.
+	c.addFieldLocked(ctx, hashID, property)
+
 	// Add to index set
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return true, fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	return true, nil
 }
@@ -580,11 +590,21 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
 
 	// Save back (or delete key if empty)
 	if len(newValues) == 0 {
-		return true, c.client.Del(ctx, key).Err()
+		if err := c.client.Del(ctx, key).Err(); err != nil {
+			return true, err
+		}
+		// The key is gone, so the field name must leave the index too.
+		c.removeFieldLocked(ctx, hashID, property)
+		c.markDirtyLocked(ctx, hashID)
+		return true, nil
 	}
 
 	newValue := strings.Join(newValues, "|")
-	return true, c.client.Set(ctx, key, newValue, 0).Err()
+	err = c.client.Set(ctx, key, newValue, 0).Err()
+	// The field survives with fewer members, so no field-index change — but the
+	// VALUE changed, which is exactly what the search index caches.
+	c.markDirtyLocked(ctx, hashID)
+	return true, err
 }
 
 // FileTuple holds the (uuid, path, size, mtime, midhash) record for one
@@ -683,6 +703,170 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 	return out, nil
 }
 
+// SearchTuple is one record's search state: its hashID, a lowercased haystack of
+// its searchable text fields (for token/substring match), and the full flat
+// metadata map (so a search hit is returned straight from memory — no per-result
+// Redis fetch).
+type SearchTuple struct {
+	HashID   string
+	Haystack string
+	Fields   map[string]string
+}
+
+// searchIndexFields are the metadata fields folded into a record's search
+// haystack — must stay in sync with the fields the search handler documents.
+var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
+
+// defaultIndexExcludePrefixes are field-name prefixes kept OUT of the in-memory
+// search index. Redis still stores them and every per-record read still returns
+// them — this narrows the *index*, never the store.
+//
+// `categories/newznab/<code>` is the raw Newznab numeric category code, which
+// METADATA_KEYS documents as a mirror of the canonical `categories/<label>`
+// vocabulary that rides alongside it. Nothing in meta-gateway, meta-search,
+// meta-watch or meta-core reads it, and it is the single largest contributor to
+// the index: measured on a live 19.6k-record corpus, `categories/*` was 40% of
+// the whole keyspace and the `newznab` sub-vocabulary was 59% of that — ~24% of
+// all keys, for zero readers.
+//
+// ⚠ Excluding the WHOLE `categories/` family instead would be a security bug,
+// not an optimisation. `categories/XXX` (and `categories/XXX/<leaf>`) is the
+// server-side adult-content enforcement point — meta-search's
+// `query_eval::fields_are_adult` and meta-gateway's `cards::fields_are_adult`
+// both match on it, and the `exclude_adult` preference defaults to ON. A record
+// served from the index without its categories would read as non-adult and the
+// filter would fail OPEN. Only prefixes with no semantic readers belong here.
+var defaultIndexExcludePrefixes = []string{"categories/newznab/"}
+
+// indexExcludePrefixes resolves the exclusion list, allowing an operator to
+// override it with a comma-separated META_CORE_INDEX_EXCLUDE_PREFIXES. An
+// explicitly empty value disables exclusion entirely (index everything).
+func indexExcludePrefixes() []string {
+	raw, set := os.LookupEnv("META_CORE_INDEX_EXCLUDE_PREFIXES")
+	if !set {
+		return defaultIndexExcludePrefixes
+	}
+	out := make([]string, 0, 4)
+	for _, p := range strings.Split(raw, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+// GetSearchIndexTuples reads EVERY record's full metadata in a single keyspace
+// SCAN pass over the flat `file:{hashID}/{field}` keys plus batched MGETs,
+// grouped by record — one bulk pass instead of a SCAN+MGET per record (which is
+// O(records) Redis round-trips, ~40s over 72k field keys). It is the bulk source
+// the API folds into its in-memory search index, so per-request search does zero
+// Redis round-trips (match AND result metadata both come from memory).
+func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	filePrefix := c.buildKey("file:") // key layout: {filePrefix}{hashID}/{field}
+	indexKey := c.buildIndexKey()     // {filePrefix}__index__ (a SET, skip it)
+	pattern := filePrefix + "*"
+
+	excluded := indexExcludePrefixes()
+	records := make(map[string]map[string]string)
+	keyBatch := make([]string, 0, 1000)
+
+	flush := func() error {
+		if len(keyBatch) == 0 {
+			return nil
+		}
+		values, err := c.client.MGet(ctx, keyBatch...).Result()
+		if err != nil {
+			return fmt.Errorf("mget: %w", err)
+		}
+		for i, k := range keyBatch {
+			v, ok := values[i].(string)
+			if !ok {
+				continue
+			}
+			// {filePrefix}{hashID}/{field}; hashID has no '/', field may.
+			rest := strings.TrimPrefix(k, filePrefix)
+			slash := strings.IndexByte(rest, '/')
+			if slash < 0 {
+				continue
+			}
+			hashID, field := rest[:slash], rest[slash+1:]
+			m := records[hashID]
+			if m == nil {
+				m = make(map[string]string)
+				records[hashID] = m
+			}
+			m[field] = v
+		}
+		keyBatch = keyBatch[:0]
+		return nil
+	}
+
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
+		if err != nil {
+			return nil, fmt.Errorf("scan: %w", err)
+		}
+		for _, k := range keys {
+			if k == indexKey {
+				continue
+			}
+			// Per-record field indexes live under the same prefix and are SETs
+			// — MGET would error on them, and they are bookkeeping, not
+			// searchable metadata.
+			if strings.HasSuffix(k, "/"+FieldsField) {
+				continue
+			}
+			// Index-excluded fields are skipped here rather than after the
+			// MGET, so an excluded key costs neither the fetch nor the memory.
+			if _, field, ok := c.splitRecordKey(k); ok && hasAnyPrefix(field, excluded) {
+				continue
+			}
+			keyBatch = append(keyBatch, k)
+			if len(keyBatch) >= 1000 {
+				if err := flush(); err != nil {
+					return nil, err
+				}
+			}
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+	if err := flush(); err != nil {
+		return nil, err
+	}
+
+	out := make([]SearchTuple, 0, len(records))
+	for hashID, fields := range records {
+		// Shared with the incremental path (search_dirty.go) so the two can
+		// never disagree about what is searchable.
+		out = append(out, SearchTuple{HashID: hashID, Haystack: buildHaystack(fields), Fields: fields})
+	}
+	return out, nil
+}
+
+// hasAnyPrefix reports whether s starts with any of the given prefixes.
+func hasAnyPrefix(s string, prefixes []string) bool {
+	for _, p := range prefixes {
+		if strings.HasPrefix(s, p) {
+			return true
+		}
+	}
+	return false
+}
+
 // GetMemoryInfo returns Redis memory usage information
 func (c *Client) GetMemoryInfo() (string, error) {
 	c.mu.RLock()
@@ -710,8 +894,16 @@ func (c *Client) GetMemoryInfo() (string, error) {
 	return "N/A", nil
 }
 
-// ClearAllMetadata deletes all file metadata and index
+// ClearAllMetadata deletes all file metadata, the file index, and the CID
-// Returns the number of files deleted
+// reverse index. Returns the number of files deleted.
+//
+// The CID reverse index (cid:<cid> → root, see cid_resolution.go) MUST be
+// purged too: ResolveRoot consults it, so a surviving cid:<cid> entry pointing
+// at a root we just deleted silently redirects the next /meta/<cid> write back
+// into that dead root, resurrecting a zombie record under an old identity.
+// DeleteRoot already does this per-record; here we sweep the whole prefix
+// rather than deriving the CID list per root, which is both simpler and
+// self-healing — it also collects entries orphaned by earlier clears.
 func (c *Client) ClearAllMetadata() (int64, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -759,12 +951,40 @@ func (c *Client) ClearAllMetadata() (int64, error) {
 		deletedCount++
 	}
 
+	// Delete every CID reverse-index entry (cid:<cid> → root). Swept by
+	// prefix rather than per-root so entries orphaned by an earlier clear —
+	// which did not purge this index at all — are collected too. Built via
+	// buildCIDIndexKey so the configured namespace prefix is honoured.
+	cidIndexPattern := c.buildCIDIndexKey("*")
+	var cidIndexDeleted int64
+	var cidCursor uint64
+	for {
+		keys, nextCursor, err := c.client.Scan(ctx, cidCursor, cidIndexPattern, 1000).Result()
+		if err != nil {
+			log.Printf("[Storage] Warning: failed to scan %s: %v", cidIndexPattern, err)
+			break
+		}
+
+		if len(keys) > 0 {
+			if err := c.client.Del(ctx, keys...).Err(); err != nil {
+				log.Printf("[Storage] Warning: failed to delete cid index keys: %v", err)
+			} else {
+				cidIndexDeleted += int64(len(keys))
+			}
+		}
+
+		cidCursor = nextCursor
+		if cidCursor == 0 {
+			break
+		}
+	}
+
 	// Delete the index set
 	if err := c.client.Del(ctx, indexKey).Err(); err != nil {
 		log.Printf("[Storage] Warning: failed to delete index: %v", err)
 	}
 
-	log.Printf("[Storage] Cleared %d file metadata entries", deletedCount)
+	log.Printf("[Storage] Cleared %d file metadata entries (%d cid index entries)", deletedCount, cidIndexDeleted)
 	return deletedCount, nil
 }
 
@@ -892,6 +1112,12 @@ func (c *Client) FindByValue(contains string, fields []string, limit int) ([]Fin
 		if field == "" {
 			continue
 		}
+		// The per-record field index is a SET, so an MGET over those keys
+		// would fail the whole query with WRONGTYPE. It holds field names,
+		// not values, so there is nothing here to search anyway.
+		if isReservedFieldName(field) {
+			continue
+		}
 		if len(matches) >= limit {
 			truncated = true
 			break
@@ -1060,7 +1286,17 @@ func (c *Client) SetRawValue(key, value string) error {
 	if t != "none" && t != "string" {
 		return fmt.Errorf("key has incompatible type %q (only string is editable)", t)
 	}
-	return c.client.Set(ctx, full, value, 0).Err()
+	if err := c.client.Set(ctx, full, value, 0).Err(); err != nil {
+		return err
+	}
+	// The raw editor can mint a record property out of band, so it has to keep
+	// the field index honest like every other write path. Non-record keys
+	// (`cid:*`, `udl:*`, `file:__index__`, the index set itself) split false
+	// and are left alone.
+	if hashID, field, ok := c.splitRecordKey(full); ok {
+		c.addFieldLocked(ctx, hashID, field)
+	}
+	return nil
 }
 
 // DeleteRawKey deletes the exact key. Returns true if the key existed.
@@ -1075,10 +1311,16 @@ func (c *Client) DeleteRawKey(key string) (bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	n, err := c.client.Del(ctx, c.buildKey(key)).Result()
+	full := c.buildKey(key)
+	n, err := c.client.Del(ctx, full).Result()
 	if err != nil {
 		return false, err
 	}
+	if n > 0 {
+		if hashID, field, ok := c.splitRecordKey(full); ok {
+			c.removeFieldLocked(ctx, hashID, field)
+		}
+	}
 	return n > 0, nil
 }
 
@@ -1110,14 +1352,20 @@ func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	prefix := c.buildKeyPrefix(hashID)
+	nonEmpty, err := c.recordHasFieldsLocked(ctx, hashID)
-	keys, _, err := c.client.Scan(ctx, 0, prefix+"*", 1).Result()
 	if err != nil {
-		return false, fmt.Errorf("scan failed: %w", err)
+		return false, err
 	}
-	if len(keys) > 0 {
+	if nonEmpty {
 		return false, nil
 	}
+
+	// Nothing left but (possibly) an empty field index — drop it too, so a
+	// de-indexed root leaves no orphan key behind.
+	if err := c.client.Del(ctx, c.buildFieldsKey(hashID)).Err(); err != nil {
+		return false, fmt.Errorf("del field index: %w", err)
+	}
+
 	removed, err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Result()
 	if err != nil {
 		return false, fmt.Errorf("srem failed: %w", err)

internal/storage/domain_screen_migration.go

@@ -0,0 +1,193 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// The one-shot corpus sweep for METADATA_KEYS.md §14.17: the `film` + `tv`
+// domains merged into `screen`, and the standalone-vs-serial split they were
+// really carrying moved to the new `workForm` key.
+//
+// # Why a sweep and not a read-side alias
+//
+// `domain` is **stored, not derived on read** — that is what lets the
+// `contentKind` vocabulary stay open (a peer that has never heard of a kind
+// still routes the record correctly). The price of storing it is that a
+// vocabulary change has to be paid once, in the data: every consumer's filter
+// is an exact match, so a record left saying `film` is invisible to a
+// `domain:screen` wall, and teaching every consumer to accept both spellings
+// forever is exactly the alternation the merge exists to delete.
+//
+// So the values are rewritten at rest, here, and no alias vocabulary survives
+// in query filters. (meta-search additionally normalises on the way into its
+// index, because a record can still arrive from a peer that has not been swept
+// — that is a courtesy for un-swept peers, not a second source of truth.)
+//
+// # Contract
+//
+// Idempotent: a record already carrying `screen` and a `workForm` is skipped,
+// so the endpoint is safe to call repeatedly and safe to call on a fresh box.
+// Returns the number of records **changed**, not the number examined.
+
+// legacyDomainReplacements maps a retired `domain` value to its replacement.
+var legacyDomainReplacements = map[string]string{
+	"film": "screen",
+	"tv":   "screen",
+}
+
+// workFormByContentKind is the `contentKind` → `workForm` table from
+// METADATA_KEYS.md §1.
+//
+// ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain` (and of
+// meta-search's copy). It exists here only to backfill the pre-merge corpus —
+// meta-core never classifies a record it is given, and nothing else in Go reads
+// it. If the tables ever disagree, the writers win: this one is a one-shot.
+var workFormByContentKind = map[string]string{
+	// screen
+	"movie":   "standalone",
+	"series":  "serial",
+	"episode": "serial",
+	// music — an album is a closed work, not an ongoing run.
+	"track":      "standalone",
+	"album":      "standalone",
+	"artist":     "standalone",
+	"musicVideo": "standalone",
+	"djMix":      "standalone",
+	"liveSet":    "standalone",
+	// literature
+	"book":      "standalone",
+	"audiobook": "standalone",
+	"comic":     "serial",
+	"manga":     "serial",
+	"magazine":  "serial",
+	// science
+	"paper": "standalone",
+	// ⚠ `pack` is deliberately absent: a season pack is `serial`, an album
+	// release `standalone`, and only the writer knows which. A pack that
+	// reaches this sweep without a `workForm` keeps none — inventing one would
+	// be a guess written permanently into the record.
+}
+
+// domainScreenPlan is one record's pending rewrite. Empty strings mean "leave
+// this field alone".
+type domainScreenPlan struct {
+	hashID   string
+	domain   string
+	workForm string
+}
+
+// MigrateDomainScreen rewrites `domain=film|tv` to `domain=screen` and
+// backfills `workForm` from `contentKind` across the whole corpus.
+//
+// Reads in two batched MGET passes (never a per-record SCAN — a single record
+// read costs a full keyspace sweep on this store), then writes only the records
+// that actually change.
+func (c *Client) MigrateDomainScreen(ctx context.Context) (int, error) {
+	plans, err := c.planDomainScreen(ctx)
+	if err != nil {
+		return 0, err
+	}
+	// Writes take the client lock themselves, so they happen after the read
+	// phase has released it.
+	fixed := 0
+	for _, p := range plans {
+		if p.domain != "" {
+			if err := c.SetProperty(p.hashID, "domain", p.domain); err != nil {
+				return fixed, fmt.Errorf("set domain on %s: %w", p.hashID, err)
+			}
+		}
+		if p.workForm != "" {
+			if err := c.SetProperty(p.hashID, "workForm", p.workForm); err != nil {
+				return fixed, fmt.Errorf("set workForm on %s: %w", p.hashID, err)
+			}
+		}
+		fixed++
+	}
+	return fixed, nil
+}
+
+// planDomainScreen is the read half: what would change, without changing it.
+func (c *Client) planDomainScreen(ctx context.Context) ([]domainScreenPlan, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers index: %w", err)
+	}
+	if len(hashIDs) == 0 {
+		return nil, nil
+	}
+
+	// Three keys per record, in a fixed order so the MGET response decodes by
+	// position: keys[i*3+n] belongs to hashIDs[i].
+	const fieldsPerFile = 3
+	fields := [fieldsPerFile]string{"domain", "workForm", "contentKind"}
+	keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
+	for _, h := range hashIDs {
+		prefix := c.buildKeyPrefix(h)
+		for _, f := range fields {
+			keys = append(keys, prefix+f)
+		}
+	}
+
+	// Chunked so one MGET never carries the whole corpus.
+	const chunk = 3000
+	values := make([]interface{}, 0, len(keys))
+	for start := 0; start < len(keys); start += chunk {
+		end := start + chunk
+		if end > len(keys) {
+			end = len(keys)
+		}
+		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
+		if err != nil {
+			return nil, fmt.Errorf("mget: %w", err)
+		}
+		values = append(values, got...)
+	}
+
+	str := func(i int) string {
+		if i >= len(values) {
+			return ""
+		}
+		s, _ := values[i].(string)
+		return s
+	}
+
+	plans := make([]domainScreenPlan, 0)
+	for i, h := range hashIDs {
+		base := i * fieldsPerFile
+		domain, workForm, contentKind := str(base), str(base+1), str(base+2)
+
+		plan := domainScreenPlan{hashID: h}
+		if replacement, ok := legacyDomainReplacements[domain]; ok {
+			plan.domain = replacement
+		}
+		// A stored `workForm` is authoritative — a `pack` writer knew something
+		// the table cannot. Only an absent one is filled in.
+		if workForm == "" {
+			if form, ok := workFormByContentKind[contentKind]; ok {
+				plan.workForm = form
+			}
+		}
+		if plan.domain != "" || plan.workForm != "" {
+			plans = append(plans, plan)
+		}
+	}
+	return plans, nil
+}
+
+// MigrateDomainScreenWithTimeout is the convenience wrapper used by the API
+// handler. Generous timeout: the read phase is two batched passes, but the
+// write phase is one round-trip per changed field.
+func (c *Client) MigrateDomainScreenWithTimeout() (int, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	return c.MigrateDomainScreen(ctx)
+}

internal/storage/domain_screen_migration_test.go

@@ -0,0 +1,140 @@
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+// seedRecord writes one record's classification fields straight through
+// SetProperty, which is what a real writer uses (it maintains the field index
+// and the root index the sweep reads from).
+func seedRecord(t *testing.T, c *Client, hashID string, fields map[string]string) {
+	t.Helper()
+	for k, v := range fields {
+		if err := c.SetProperty(hashID, k, v); err != nil {
+			t.Fatalf("seed %s/%s: %v", hashID, k, err)
+		}
+	}
+}
+
+// The merge (METADATA_KEYS.md §14.17) is paid in the data: `domain` is stored,
+// not derived on read, so a record left saying `film` is invisible to a
+// `domain:screen` wall. This is the sweep that pays it.
+func TestMigrateDomainScreen_RewritesLegacyDomainsAndBackfillsWorkForm(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedRecord(t, c, "rec-movie", map[string]string{
+		"domain":      "film",
+		"contentKind": "movie",
+	})
+	seedRecord(t, c, "rec-episode", map[string]string{
+		"domain":      "tv",
+		"contentKind": "episode",
+	})
+	// Already current, and already carries both keys: must be left alone.
+	seedRecord(t, c, "rec-current", map[string]string{
+		"domain":      "screen",
+		"contentKind": "series",
+		"workForm":    "serial",
+	})
+	// A different domain entirely — the sweep must not touch it, but it is
+	// still owed the `workForm` backfill.
+	seedRecord(t, c, "rec-book", map[string]string{
+		"domain":      "literature",
+		"contentKind": "book",
+	})
+
+	fixed, err := c.MigrateDomainScreen(context.Background())
+	if err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	if fixed != 3 {
+		t.Errorf("fixed = %d, want 3 (the two legacy rows + the book's backfill)", fixed)
+	}
+
+	for _, tc := range []struct {
+		hashID   string
+		domain   string
+		workForm string
+	}{
+		{"rec-movie", "screen", "standalone"},
+		{"rec-episode", "screen", "serial"},
+		{"rec-current", "screen", "serial"},
+		{"rec-book", "literature", "standalone"},
+	} {
+		m, err := c.GetMetadataFlat(tc.hashID)
+		if err != nil {
+			t.Fatalf("read %s: %v", tc.hashID, err)
+		}
+		if m["domain"] != tc.domain {
+			t.Errorf("%s domain = %q, want %q", tc.hashID, m["domain"], tc.domain)
+		}
+		if m["workForm"] != tc.workForm {
+			t.Errorf("%s workForm = %q, want %q", tc.hashID, m["workForm"], tc.workForm)
+		}
+	}
+}
+
+// A second pass must find nothing to do — the endpoint is called by hand on
+// each box, and "run it twice" must not mean "write the whole corpus twice".
+func TestMigrateDomainScreen_IsIdempotent(t *testing.T) {
+	c, _ := newTestClient(t, "mm:")
+
+	seedRecord(t, c, "rec-1", map[string]string{
+		"domain":      "tv",
+		"contentKind": "episode",
+	})
+
+	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
+		t.Fatalf("first pass: %v", err)
+	}
+	fixed, err := c.MigrateDomainScreen(context.Background())
+	if err != nil {
+		t.Fatalf("second pass: %v", err)
+	}
+	if fixed != 0 {
+		t.Errorf("second pass fixed = %d, want 0", fixed)
+	}
+}
+
+// `pack` is the one kind neither routing table can decide — a season pack is
+// `serial`, an album release `standalone`. A pack that reaches the sweep
+// without a `workForm` must keep none: inventing one writes a guess into the
+// record permanently.
+func TestMigrateDomainScreen_NeverGuessesAPacksWorkForm(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedRecord(t, c, "rec-pack", map[string]string{
+		"domain":      "tv",
+		"contentKind": "pack",
+	})
+	// A writer that DID know keeps what it wrote.
+	seedRecord(t, c, "rec-album", map[string]string{
+		"domain":      "music",
+		"contentKind": "pack",
+		"workForm":    "standalone",
+	})
+
+	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+
+	m, err := c.GetMetadataFlat("rec-pack")
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+	if m["domain"] != "screen" {
+		t.Errorf("domain = %q, want screen (the domain WAS knowable)", m["domain"])
+	}
+	if _, ok := m["workForm"]; ok {
+		t.Errorf("workForm = %q, want absent", m["workForm"])
+	}
+
+	a, err := c.GetMetadataFlat("rec-album")
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+	if a["workForm"] != "standalone" {
+		t.Errorf("workForm = %q, want standalone (writer-supplied, never overwritten)", a["workForm"])
+	}
+}

internal/storage/field_index.go

@@ -0,0 +1,486 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// The per-record field-name index.
+//
+// A record is stored as one flat Redis STRING per leaf
+// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
+// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
+// meta-search can react to one field changing instead of diffing a whole hash
+// (see docs/project-architecture/persistent-storage.md, and
+// dev/test/suites/flat-keys.bats which asserts records are never hashes).
+//
+// The cost of that shape was that reading one record meant asking Redis "which
+// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
+// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
+// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
+// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
+// the byte path for every poster meta-share served, which is how it surfaced.
+//
+// So we keep the flat keys and index their *names*:
+//
+//	file:<uuid>/__fields__   SET{ "filePath", "sizeByte", "cids/bafk…", … }
+//
+// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
+// This mirrors `file:__index__` (the set of every root) one level down, and the
+// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
+// from the index rather than scanning per root.
+//
+// **The set is derived, never authoritative.** The flat keys remain the source
+// of truth. A record whose set is missing (anything written before this landed)
+// falls back to the old SCAN and populates the set on the way out, so existing
+// deployments heal themselves on first touch with no migration and no
+// SchemaVersion bump — which matters, because a bump is a flag day that makes
+// operators wipe Redis (see schema_version.go).
+//
+// The consistency argument that makes a derived index safe here is meta-core's
+// single-writer rule: leader election publishes an empty `redisUrl` on purpose
+// (internal/leader/election.go), so nothing outside this process can write a
+// `file:` key behind the index's back.
+
+// FieldsField is the reserved field name holding a record's field-name index.
+//
+// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
+// prefix as the record's own fields. That keeps every existing sweep correct
+// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
+// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
+// The flip side is that every reader which enumerates that prefix has to skip
+// it, exactly as they already skip `file:__index__`.
+const FieldsField = "__fields__"
+
+// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
+// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
+const mgetBatch = 1000
+
+// buildFieldsKey constructs the key for a record's field-name index set.
+func (c *Client) buildFieldsKey(hashID string) string {
+	return c.buildKeyPrefix(hashID) + FieldsField
+}
+
+// isReservedFieldName reports whether a field name is bookkeeping rather than
+// metadata, and so must never be returned to an API caller, exported into a
+// snapshot, or folded into the search haystack.
+func isReservedFieldName(field string) bool {
+	return field == FieldsField
+}
+
+// readMetadataLocked returns a record's full flat metadata. Caller must hold
+// c.mu (either mode).
+//
+// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
+// legacy full-keyspace SCAN, whose result is used to populate the index so the
+// next read takes the fast path.
+//
+// Returns (nil, nil) for a record with no fields — callers treat that as 404.
+func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
+	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers fields: %w", err)
+	}
+
+	if len(fields) > 0 {
+		return c.mgetFieldsLocked(ctx, hashID, fields)
+	}
+
+	// No index yet: a record written before the index existed, or one that
+	// genuinely has no fields. Fall back to the scan, then backfill.
+	result, scanned, err := c.scanMetadataLocked(ctx, hashID)
+	if err != nil {
+		return nil, err
+	}
+	if len(scanned) > 0 {
+		// Best-effort: a failed backfill costs the next read a scan, it does
+		// not make the read wrong. Never fail the caller's read for it.
+		if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
+			logFieldIndexBackfillFailure(hashID, err)
+		}
+	}
+	if len(result) == 0 {
+		return nil, nil
+	}
+	return result, nil
+}
+
+// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
+//
+// A field named in the index whose key is gone reads back as nil and is simply
+// skipped: a *stale* index entry is harmless. (A *missing* one would silently
+// drop data, which is why every write path maintains the set.)
+func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
+	prefix := c.buildKeyPrefix(hashID)
+
+	// Keep the reserved name out of the key list AND out of the result — it is
+	// a SET, so MGET would error on it in real Redis rather than return nil.
+	wanted := make([]string, 0, len(fields))
+	for _, f := range fields {
+		if !isReservedFieldName(f) {
+			wanted = append(wanted, f)
+		}
+	}
+	if len(wanted) == 0 {
+		return nil, nil
+	}
+
+	result := make(map[string]string, len(wanted))
+	for start := 0; start < len(wanted); start += mgetBatch {
+		end := start + mgetBatch
+		if end > len(wanted) {
+			end = len(wanted)
+		}
+		chunk := wanted[start:end]
+		keys := make([]string, len(chunk))
+		for i, f := range chunk {
+			keys[i] = prefix + f
+		}
+		values, err := c.client.MGet(ctx, keys...).Result()
+		if err != nil {
+			return nil, fmt.Errorf("mget failed: %w", err)
+		}
+		for i, v := range values {
+			if s, ok := v.(string); ok {
+				result[chunk[i]] = s
+			}
+		}
+	}
+
+	if len(result) == 0 {
+		return nil, nil
+	}
+	return result, nil
+}
+
+// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
+// MGET. Retained as the fallback for un-indexed records and as the source for
+// the backfill. Returns the metadata plus the field names it saw (which
+// includes fields whose value came back nil, so the index stays complete).
+func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
+	prefix := c.buildKeyPrefix(hashID)
+	result := make(map[string]string)
+	var fields []string
+
+	var cursor uint64
+	for {
+		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return nil, nil, fmt.Errorf("scan failed: %w", err)
+		}
+
+		if len(keys) > 0 {
+			wanted := make([]string, 0, len(keys))
+			for _, key := range keys {
+				field := strings.TrimPrefix(key, prefix)
+				if isReservedFieldName(field) {
+					continue
+				}
+				fields = append(fields, field)
+				wanted = append(wanted, key)
+			}
+			if len(wanted) > 0 {
+				values, err := c.client.MGet(ctx, wanted...).Result()
+				if err != nil {
+					return nil, nil, fmt.Errorf("mget failed: %w", err)
+				}
+				for i, key := range wanted {
+					if s, ok := values[i].(string); ok {
+						result[strings.TrimPrefix(key, prefix)] = s
+					}
+				}
+			}
+		}
+
+		cursor = nextCursor
+		if cursor == 0 {
+			break
+		}
+	}
+
+	return result, fields, nil
+}
+
+// addFieldsToPipe records field names in the index as part of an existing
+// pipeline, so a multi-field write stays one round-trip.
+func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
+	names := filterReserved(fields)
+	if len(names) == 0 {
+		return
+	}
+	pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
+}
+
+// addFieldLocked records a single field name in the index. Caller must hold
+// c.mu. Best-effort by the same argument as the backfill: the flat key is the
+// source of truth, and a lost index entry is repaired by the next scan
+// fallback — so it must not fail the write it accompanies.
+func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
+	if isReservedFieldName(field) {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
+		logFieldIndexWriteFailure(hashID, field, err)
+	}
+}
+
+// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
+//
+// A stale entry left behind by a failure is harmless (the MGET reads nil and
+// skips it), so this is best-effort too.
+func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
+	if isReservedFieldName(field) {
+		return
+	}
+	if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
+		logFieldIndexWriteFailure(hashID, field, err)
+	}
+}
+
+// fieldsWithPrefixLocked returns the record's indexed field names under a
+// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
+// no index yet. Caller must hold c.mu.
+func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
+	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers fields: %w", err)
+	}
+	if len(fields) == 0 {
+		// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
+		// rather than the whole record, and do not backfill from it — a
+		// partial view must never be mistaken for a complete index.
+		return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
+	}
+
+	var out []string
+	for _, f := range fields {
+		if isReservedFieldName(f) {
+			continue
+		}
+		if strings.HasPrefix(f, fieldPrefix) {
+			out = append(out, strings.TrimPrefix(f, fieldPrefix))
+		}
+	}
+	// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
+	// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
+	// at these sizes and makes the response byte-stable, which matters for
+	// anything diffing or caching it.
+	sort.Strings(out)
+	return out, nil
+}
+
+func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
+	prefix := c.buildKeyPrefix(hashID) + fieldPrefix
+	var out []string
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return nil, fmt.Errorf("scan fields: %w", err)
+		}
+		for _, k := range keys {
+			out = append(out, strings.TrimPrefix(k, prefix))
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
+//
+// Without it the indexes still appear — each record's first read falls back to
+// a scan and backfills — but that transition is O(records x keyspace) and it
+// runs again from scratch after any wipe. Measured on the real store (12,345
+// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
+// during which meta-core stays pinned exactly as it was before the fix. Worse,
+// `RepublishMetadata` walks every record at startup, so that tail is paid on
+// every restart.
+//
+// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
+// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
+// for the same reason.
+//
+// Idempotent, and safe to run against a partially-indexed store: SADD of a
+// member that is already there is a no-op. Returns the number of records it
+// wrote an index for.
+func (c *Client) WarmFieldIndexes() (int, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
+	indexKey := c.buildIndexKey()
+
+	byRecord := make(map[string][]string)
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
+		if err != nil {
+			return 0, fmt.Errorf("scan: %w", err)
+		}
+		for _, k := range keys {
+			if k == indexKey {
+				continue
+			}
+			// hashID has no '/'; field may contain them.
+			rest := strings.TrimPrefix(k, filePrefix)
+			slash := strings.IndexByte(rest, '/')
+			if slash < 0 {
+				continue
+			}
+			hashID, field := rest[:slash], rest[slash+1:]
+			if isReservedFieldName(field) {
+				// Already indexed — remember the record so we don't treat it
+				// as missing, but there is nothing to add for this key.
+				if _, seen := byRecord[hashID]; !seen {
+					byRecord[hashID] = nil
+				}
+				continue
+			}
+			byRecord[hashID] = append(byRecord[hashID], field)
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+
+	written := 0
+	pipe := c.client.Pipeline()
+	pending := 0
+	for hashID, fields := range byRecord {
+		if len(fields) == 0 {
+			continue
+		}
+		pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
+		written++
+		pending++
+		// Bound the pipeline so a large store doesn't buffer every command.
+		if pending >= 500 {
+			if _, err := pipe.Exec(ctx); err != nil {
+				return written, fmt.Errorf("sadd pipeline: %w", err)
+			}
+			pipe = c.client.Pipeline()
+			pending = 0
+		}
+	}
+	if pending > 0 {
+		if _, err := pipe.Exec(ctx); err != nil {
+			return written, fmt.Errorf("sadd pipeline: %w", err)
+		}
+	}
+	return written, nil
+}
+
+// recordHasFieldsLocked reports whether a record still has at least one
+// property key. Caller must hold c.mu.
+//
+// `__fields__` must not count — it lives under the record's own prefix, so a
+// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
+// whose every property has been deleted, and the root could never be
+// de-indexed.
+func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
+	n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return false, fmt.Errorf("scard fields: %w", err)
+	}
+	if n > 0 {
+		return true, nil
+	}
+
+	// Either genuinely empty, or an un-indexed record from before the index
+	// existed — the scan is the only way to tell those apart.
+	prefix := c.buildKeyPrefix(hashID)
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return false, fmt.Errorf("scan failed: %w", err)
+		}
+		for _, k := range keys {
+			if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
+				return true, nil
+			}
+		}
+		cursor = next
+		if cursor == 0 {
+			return false, nil
+		}
+	}
+}
+
+// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
+// record property key. Reports ok=false for anything else — the index set
+// itself, `file:__index__`, and any key outside the `file:` namespace.
+//
+// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
+// the index honest without its callers knowing the layout.
+func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
+	rest := key
+	if c.prefix != "" {
+		var found bool
+		rest, found = strings.CutPrefix(rest, c.prefix)
+		if !found {
+			return "", "", false
+		}
+	}
+	rest, found := strings.CutPrefix(rest, "file:")
+	if !found {
+		return "", "", false
+	}
+	hashID, field, found = strings.Cut(rest, "/")
+	if !found || hashID == "" || field == "" {
+		// `file:__index__` has no slash and lands here.
+		return "", "", false
+	}
+	if isReservedFieldName(field) {
+		return "", "", false
+	}
+	return hashID, field, true
+}
+
+// The index is derived and self-healing, so a write failure degrades
+// performance (the next read falls back to a scan and re-populates) rather
+// than correctness. Log it rather than failing the caller's write — but log it
+// loudly, because a *persistent* failure means every read is scanning again.
+func logFieldIndexWriteFailure(hashID, field string, err error) {
+	log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
+}
+
+func logFieldIndexBackfillFailure(hashID string, err error) {
+	log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
+}
+
+func filterReserved(fields []string) []string {
+	out := make([]string, 0, len(fields))
+	for _, f := range fields {
+		if !isReservedFieldName(f) {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+func toAny(ss []string) []interface{} {
+	out := make([]interface{}, len(ss))
+	for i, s := range ss {
+		out[i] = s
+	}
+	return out
+}

internal/storage/field_index_test.go

@@ -0,0 +1,464 @@
+package storage
+
+import (
+	"context"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+// Tests for the per-record field-name index (field_index.go).
+//
+// The index is what makes a record read O(fields) instead of O(keyspace). It is
+// derived state, so the properties worth pinning are: (1) a read served from
+// the index returns exactly what a scan would have, (2) a record with no index
+// still reads correctly and gets one, and (3) every write path keeps it in
+// sync — because a *missing* entry silently drops a field from every
+// subsequent read, which is the one failure mode that would be invisible.
+//
+// Every case runs with and without a namespace prefix, matching clear_test.go.
+
+func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
+	t.Helper()
+	for _, prefix := range []string{"", "mm:"} {
+		name := "no-prefix"
+		if prefix != "" {
+			name = "prefix-" + prefix
+		}
+		t.Run(name, func(t *testing.T) {
+			c, _ := newTestClient(t, prefix)
+			fn(t, c)
+		})
+	}
+}
+
+func fieldNames(t *testing.T, c *Client, hashID string) []string {
+	t.Helper()
+	got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
+	if err != nil {
+		t.Fatalf("smembers fields: %v", err)
+	}
+	sort.Strings(got)
+	return got
+}
+
+func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
+	t.Helper()
+	got, err := c.GetMetadataFlat(hashID)
+	if err != nil {
+		t.Fatalf("GetMetadataFlat: %v", err)
+	}
+	return got
+}
+
+// SetMetadataFlat must index every field it writes, and the read must come back
+// through the index rather than a scan.
+func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		want := map[string]string{
+			"filePath":                      "/files/watch/a.mkv",
+			"sizeByte":                      "123",
+			"cids/bafkreiaaa":               "true",
+			"titles/jpn/ナルト":                "true",
+			"categories/TV/UHD":             "true",
+			"info/files/0/path":             "a.mkv",
+			"provenance":                    `{"source":"gateway"}`,
+			"plot/eng":                      "a plot with / slashes",
+			"stream/3":                      "eng",
+			"genres/action":                 "true",
+			"originaltitle":                 "Naruto",
+			"deeply/nested/field/name/here": "x",
+		}
+		if err := c.SetMetadataFlat("root1", want); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
+			t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
+		}
+
+		// The index must name every field — including the ones containing '/',
+		// which is where a naive "split on the first slash" would go wrong.
+		wantNames := make([]string, 0, len(want))
+		for k := range want {
+			wantNames = append(wantNames, k)
+		}
+		sort.Strings(wantNames)
+		if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
+			t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
+		}
+	})
+}
+
+// A record written before the index existed has flat keys but no set. The read
+// must still be correct, and must leave an index behind so the next read takes
+// the fast path. This is the migration path for the ~12k records already on
+// disk: no flag day, no SchemaVersion bump.
+func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		prefix := c.buildKeyPrefix("legacy")
+		want := map[string]string{
+			"filePath":        "/files/watch/old.mkv",
+			"sizeByte":        "42",
+			"cids/bafkreiold": "true",
+		}
+		// Write the flat keys directly, bypassing every index-maintaining path
+		// — exactly the shape a pre-upgrade store is in.
+		for f, v := range want {
+			if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
+				t.Fatalf("seed %s: %v", f, err)
+			}
+		}
+		if names := fieldNames(t, c, "legacy"); len(names) != 0 {
+			t.Fatalf("precondition: expected no index, got %v", names)
+		}
+
+		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
+			t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
+		}
+
+		wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
+		if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
+			t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
+		}
+
+		// Second read, now served from the index, must agree with the first.
+		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
+			t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
+		}
+	})
+}
+
+// Each single-field write path must add to the index, and each delete path must
+// remove from it. A field written but not indexed vanishes from every later
+// read — silently — so this is the property that matters most.
+func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
+			t.Fatalf("SetProperty: %v", err)
+		}
+		if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
+			t.Fatalf("AddToSet: %v", err)
+		}
+		if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
+			t.Fatalf("MergeMetadataFlat: %v", err)
+		}
+
+		want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
+		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
+			t.Errorf("after writes\n got %v\nwant %v", got, want)
+		}
+
+		// DeleteProperty drops the key and the index entry.
+		if err := c.DeleteProperty("root2", "title"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		// RemoveFromSet deletes the key once its last member is gone.
+		if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
+			t.Fatalf("RemoveFromSet: %v", err)
+		}
+
+		want = map[string]string{"year": "2002"}
+		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
+			t.Errorf("after deletes\n got %v\nwant %v", got, want)
+		}
+		if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
+			t.Errorf("index after deletes = %v, want [year]", got)
+		}
+	})
+}
+
+// The raw KV editor can mint or drop a record property out of band. It is the
+// one write path whose caller does not know the record layout, so the storage
+// layer has to keep the index honest for it.
+func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
+			t.Fatalf("SetRawValue: %v", err)
+		}
+		if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
+			t.Errorf("index after SetRawValue = %v", got)
+		}
+		if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
+			t.Errorf("read after SetRawValue = %v", got)
+		}
+
+		if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
+			t.Fatalf("DeleteRawKey: %v", err)
+		}
+		if got := fieldNames(t, c, "root3"); len(got) != 0 {
+			t.Errorf("index after DeleteRawKey = %v, want empty", got)
+		}
+
+		// Keys outside the record namespace must be left alone entirely.
+		if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
+			t.Fatalf("SetRawValue non-record: %v", err)
+		}
+		if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
+			t.Error("cid: key was treated as a record property")
+		}
+		if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
+			t.Error("file:__index__ was treated as a record property")
+		}
+	})
+}
+
+// Mint and AddAlias write fields through their own pipelines, so they need the
+// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
+// it and a record's CID key-set becomes invisible to every read.
+func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
+		if err != nil {
+			t.Fatalf("Mint: %v", err)
+		}
+		if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
+			t.Errorf("index after Mint = %v", got)
+		}
+
+		if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
+			t.Fatalf("AddAlias: %v", err)
+		}
+		got := mustGetMetadata(t, c, uuid)
+		if got["cids/bafkreiaaa"] != "true" {
+			t.Errorf("alias key-set member missing from read: %v", got)
+		}
+
+		// And the CID enumeration, which now reads the index instead of
+		// scanning, must see it.
+		cids, err := c.cidsForRootLocked(context.Background(), uuid)
+		if err != nil {
+			t.Fatalf("cidsForRootLocked: %v", err)
+		}
+		if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
+			t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
+		}
+	})
+}
+
+// The index lives under the record's own prefix, so the "is this record empty?"
+// probe would otherwise always find it and the root could never be de-indexed.
+func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetProperty("root4", "title", "x"); err != nil {
+			t.Fatalf("SetProperty: %v", err)
+		}
+
+		removed, err := c.IndexRemoveIfEmpty("root4")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if removed {
+			t.Error("de-indexed a record that still has a field")
+		}
+
+		if err := c.DeleteProperty("root4", "title"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		removed, err = c.IndexRemoveIfEmpty("root4")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if !removed {
+			t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
+		}
+		// ...and the now-empty index key must not be left orphaned.
+		n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
+		if err != nil {
+			t.Fatalf("exists: %v", err)
+		}
+		if n != 0 {
+			t.Error("orphaned field index left behind after de-indexing")
+		}
+	})
+}
+
+// The same probe must still work for a record that predates the index.
+func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+		if err := c.IndexAdd("legacy2"); err != nil {
+			t.Fatalf("IndexAdd: %v", err)
+		}
+		removed, err := c.IndexRemoveIfEmpty("legacy2")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if removed {
+			t.Error("de-indexed an un-indexed record that still has a field")
+		}
+	})
+}
+
+// The reserved name is bookkeeping. It must never reach an API caller as a
+// metadata field, and an inbound payload must never be able to overwrite the
+// SET with a string — that would break every subsequent read of the record.
+func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		err := c.SetMetadataFlat("root5", map[string]string{
+			"title":     "x",
+			FieldsField: "malicious",
+		})
+		if err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		got := mustGetMetadata(t, c, "root5")
+		if _, present := got[FieldsField]; present {
+			t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
+		}
+		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
+			t.Errorf("read = %v, want just title", got)
+		}
+
+		// The index must still be a SET — i.e. not clobbered by the payload.
+		if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
+			t.Errorf("index = %v, want [title]", got)
+		}
+
+		if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
+			t.Error("SetProperty accepted the reserved field name")
+		}
+	})
+}
+
+// Deleting a record must take its index with it, or a recreated root would
+// inherit phantom field names.
+func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		if _, err := c.DeleteMetadata("root6"); err != nil {
+			t.Fatalf("DeleteMetadata: %v", err)
+		}
+		if got := fieldNames(t, c, "root6"); len(got) != 0 {
+			t.Errorf("index survived DeleteMetadata: %v", got)
+		}
+		got, err := c.GetMetadataFlat("root6")
+		if err != nil {
+			t.Fatalf("GetMetadataFlat: %v", err)
+		}
+		if got != nil {
+			t.Errorf("deleted record still reads back: %v", got)
+		}
+	})
+}
+
+// A stale index entry (name present, key gone) is the benign direction: the
+// MGET reads nil and the field is skipped rather than erroring.
+func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		// Name a field that has no key behind it.
+		if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
+			t.Fatalf("sadd ghost: %v", err)
+		}
+
+		got := mustGetMetadata(t, c, "root7")
+		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
+			t.Errorf("read = %v, want just title", got)
+		}
+	})
+}
+
+// WarmFieldIndexes is what keeps an upgrade from spending hours reading
+// un-indexed records one full-keyspace scan at a time. It must index every
+// record in one pass, be idempotent, and leave already-indexed records alone.
+func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+
+		// Two legacy records seeded straight into Redis (no index), plus one
+		// written through the normal path (already indexed).
+		legacy := map[string]map[string]string{
+			"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
+			"old2": {"filePath": "/b.mkv", "titles/jpn/ナルト": "true", "deep/a/b": "x"},
+		}
+		for root, fields := range legacy {
+			for f, v := range fields {
+				if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
+					t.Fatalf("seed: %v", err)
+				}
+			}
+			if err := c.IndexAdd(root); err != nil {
+				t.Fatalf("IndexAdd: %v", err)
+			}
+		}
+		if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		n, err := c.WarmFieldIndexes()
+		if err != nil {
+			t.Fatalf("WarmFieldIndexes: %v", err)
+		}
+		if n < 2 {
+			t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
+		}
+
+		if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
+			t.Errorf("old1 index = %v", got)
+		}
+		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
+			t.Errorf("old2 index = %v", got)
+		}
+		if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
+			t.Errorf("fresh index disturbed = %v", got)
+		}
+
+		// Reads must now agree with what the records actually hold.
+		for root, want := range legacy {
+			if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
+				t.Errorf("%s read = %v, want %v", root, got, want)
+			}
+		}
+
+		// Idempotent: a second pass changes nothing.
+		if _, err := c.WarmFieldIndexes(); err != nil {
+			t.Fatalf("second WarmFieldIndexes: %v", err)
+		}
+		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
+			t.Errorf("old2 index after second warm = %v", got)
+		}
+	})
+}
+
+// The bulk search read enumerates the whole file: keyspace and MGETs what it
+// finds. The index keys are SETs living in that same namespace, so it has to
+// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
+func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetMetadataFlat("root8", map[string]string{
+			"title":    "Naruto Shippuden",
+			"fileName": "naruto.mkv",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		tuples, err := c.GetSearchIndexTuples()
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuples: %v", err)
+		}
+		var found bool
+		for _, tu := range tuples {
+			if tu.HashID != "root8" {
+				continue
+			}
+			found = true
+			if _, present := tu.Fields[FieldsField]; present {
+				t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
+			}
+		}
+		if !found {
+			t.Fatal("record missing from search tuples")
+		}
+	})
+}

internal/storage/literature_echo_sweep.go

@@ -0,0 +1,326 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/metazla/meta-core/internal/cid"
+)
+
+// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
+// identity and position keys are resolved, never copied from a query, and a
+// card carries no position keys and no `workcid`.
+//
+// # Why a sweep
+//
+// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
+// own filter values onto what it returned. The gateway persists whatever passes
+// the filter, so the copy became a permanent binding — and stored records never
+// pick up the writer fix: a covered `(upstream, query)` never reaches the
+// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
+// one. So the copies already at rest are removed here, once.
+//
+// What the copies cost: an `anilistid` on a MangaDex site card claims an
+// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
+// anchor — the site card then competes with (and can hide) the real AniList
+// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
+// join and sort like a position inside one.
+//
+// # What is swept, and what deliberately is not
+//
+//   - Only `domain=literature` + `fileType=card` records. Nothing else is read
+//     for writing, whatever keys it carries.
+//   - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
+//     go from **every** such card: a card is a work, never a position in one,
+//     so no legitimate writer puts them there.
+//   - `anilistid` goes only from a **site card** — a card whose own `cids/`
+//     key-set holds a site-form card locator and no locator of any other form.
+//     On an anchor card the id *is* the card's identity (its locator is
+//     `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
+//     it: which half the id belongs to is not recoverable at rest, and leaving
+//     a stamp is the safe direction.
+//   - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
+//     Matched by rule, and a query-copied stamp is indistinguishable at rest
+//     from a title-matched one — deleting would drop real bindings with the
+//     fake ones.
+//
+// # Contract
+//
+// Dry-run unless `apply`: the sweep deletes keys, so the default call only
+// reports. Idempotent: a swept record plans nothing on the next pass, so the
+// endpoint is safe to call repeatedly and on a fresh box.
+
+// literatureEchoAnchorSources are the card-locator sources that name an id-bag
+// work (an anchor), never a site.
+//
+// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
+// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
+// excludes them and the two rules agree on every locator minted today. They are
+// listed so the rule reads on its own here.
+var literatureEchoAnchorSources = map[string]bool{
+	"anilist":     true,
+	"openlibrary": true,
+	"tmdb":        true,
+	"imdb":        true,
+	"musicbrainz": true,
+	"mal":         true,
+}
+
+// literatureCardEchoFields are removed from every literature card carrying
+// them — position keys and the work reference, none of which a card may hold.
+var literatureCardEchoFields = []string{
+	"workcid",
+	"chapterNumber",
+	"volumeNumber",
+	"chapterStart",
+	"chapterEnd",
+}
+
+// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
+// then `anilistid`, then the echo fields in literatureCardEchoFields order.
+var literatureEchoReadFields = append(
+	[]string{"domain", "fileType", "anilistid"},
+	literatureCardEchoFields...,
+)
+
+// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
+// records; ByKey counts fields per key — removed on an apply pass, that would
+// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
+type LiteratureEchoSweep struct {
+	Planned int
+	Fixed   int
+	ByKey   map[string]int
+}
+
+// literatureEchoPlan is one record's pending deletes, in a fixed order
+// (`anilistid` first, then literatureCardEchoFields order).
+type literatureEchoPlan struct {
+	hashID string
+	fields []string
+}
+
+// isSiteFormCardSource is the site-form rule on an already-decoded source: not
+// an anchor source, and shaped like a host.
+func isSiteFormCardSource(source string) bool {
+	return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
+}
+
+// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
+// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
+func isSiteFormCardLocator(cidStr string) bool {
+	source, _, err := cid.DecodeCard(cidStr)
+	return err == nil && isSiteFormCardSource(source)
+}
+
+// isSiteCard reports whether a card's field names prove it a site card: at
+// least one site-form `cids/` card locator and no card locator of another form.
+// Non-card members (a content CID) say nothing about which work the card is and
+// are ignored. An empty field list — a record with no field-name index — is
+// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
+func isSiteCard(fields []string) bool {
+	site := false
+	for _, f := range fields {
+		member := cidFromKeysetField(f)
+		if member == "" {
+			continue
+		}
+		source, _, err := cid.DecodeCard(member)
+		if err != nil {
+			continue
+		}
+		if !isSiteFormCardSource(source) {
+			return false
+		}
+		site = true
+	}
+	return site
+}
+
+// SweepLiteratureEchoes removes query echoes stored on literature cards (see
+// the file comment). With apply=false it only plans.
+//
+// Reads in batched passes (never a per-record SCAN — a single record read
+// costs a full keyspace sweep on this store), then deletes only the planned
+// fields, through DeleteProperty so the field-name index and the search-dirty
+// set stay consistent.
+func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
+	res := LiteratureEchoSweep{ByKey: map[string]int{}}
+
+	plans, err := c.planLiteratureEchoes(ctx)
+	if err != nil {
+		return res, err
+	}
+	res.Planned = len(plans)
+
+	if !apply {
+		for _, p := range plans {
+			for _, f := range p.fields {
+				res.ByKey[f]++
+			}
+		}
+		return res, nil
+	}
+
+	// Writes take the client lock themselves, so they happen after the read
+	// phase has released it. ByKey counts what was actually deleted, so a
+	// failure midway reports the true partial state.
+	for _, p := range plans {
+		// DeleteProperty carries its own per-call timeout; checking ctx here is
+		// what makes the wrapper's deadline bound the whole write phase.
+		if err := ctx.Err(); err != nil {
+			return res, err
+		}
+		for _, f := range p.fields {
+			if err := c.DeleteProperty(p.hashID, f); err != nil {
+				return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
+			}
+			res.ByKey[f]++
+		}
+		res.Fixed++
+	}
+	return res, nil
+}
+
+// planLiteratureEchoes is the read half: what would be deleted, without
+// deleting it.
+func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers index: %w", err)
+	}
+	if len(hashIDs) == 0 {
+		return nil, nil
+	}
+
+	// Pass 1: the gate and echo fields for every record, in a fixed order so
+	// the MGET response decodes by position: keys[i*per+n] belongs to
+	// hashIDs[i].
+	per := len(literatureEchoReadFields)
+	keys := make([]string, 0, len(hashIDs)*per)
+	for _, h := range hashIDs {
+		prefix := c.buildKeyPrefix(h)
+		for _, f := range literatureEchoReadFields {
+			keys = append(keys, prefix+f)
+		}
+	}
+
+	// Chunked so one MGET never carries the whole corpus.
+	const chunk = 3000
+	values := make([]interface{}, 0, len(keys))
+	for start := 0; start < len(keys); start += chunk {
+		end := start + chunk
+		if end > len(keys) {
+			end = len(keys)
+		}
+		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
+		if err != nil {
+			return nil, fmt.Errorf("mget: %w", err)
+		}
+		values = append(values, got...)
+	}
+
+	// A nil slot is an absent key. A present key holding "" is still an echo
+	// and still goes, so presence — not the value — decides.
+	get := func(i int) (string, bool) {
+		if i >= len(values) || values[i] == nil {
+			return "", false
+		}
+		s, ok := values[i].(string)
+		return s, ok
+	}
+
+	type candidate struct {
+		hashID     string
+		hasAnilist bool
+		echoes     []string
+	}
+	var cands []candidate
+	var probe []string // candidates whose anilistid needs their cids/ key-set
+	for i, h := range hashIDs {
+		base := i * per
+		domain, _ := get(base)
+		fileType, _ := get(base + 1)
+		if domain != "literature" || fileType != "card" {
+			continue
+		}
+		cand := candidate{hashID: h}
+		_, cand.hasAnilist = get(base + 2)
+		for j, f := range literatureCardEchoFields {
+			if _, ok := get(base + 3 + j); ok {
+				cand.echoes = append(cand.echoes, f)
+			}
+		}
+		if !cand.hasAnilist && len(cand.echoes) == 0 {
+			continue
+		}
+		if cand.hasAnilist {
+			probe = append(probe, h)
+		}
+		cands = append(cands, cand)
+	}
+
+	// Pass 2: the field-name index of only the cards carrying an `anilistid`
+	// — the `cids/` members are field NAMES, so the index is where they are
+	// listed. Pipelined per chunk; still no SCAN.
+	//
+	// A card with no index (never touched since before the index existed, and
+	// missed by the startup WarmFieldIndexes pass) reads as an empty set and
+	// keeps its `anilistid`: proving it a site card would need exactly the
+	// per-record SCAN this sweep refuses, and a stamp left in place is the
+	// safe direction.
+	siteCard := make(map[string]bool, len(probe))
+	for start := 0; start < len(probe); start += mgetBatch {
+		end := start + mgetBatch
+		if end > len(probe) {
+			end = len(probe)
+		}
+		pipe := c.client.Pipeline()
+		cmds := make([]*redis.StringSliceCmd, 0, end-start)
+		for _, h := range probe[start:end] {
+			cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
+		}
+		if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
+			return nil, fmt.Errorf("smembers fields: %w", err)
+		}
+		for k, cmd := range cmds {
+			fields, err := cmd.Result()
+			if err != nil && err != redis.Nil {
+				return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
+			}
+			siteCard[probe[start+k]] = isSiteCard(fields)
+		}
+	}
+
+	plans := make([]literatureEchoPlan, 0, len(cands))
+	for _, cand := range cands {
+		var fields []string
+		if cand.hasAnilist && siteCard[cand.hashID] {
+			fields = append(fields, "anilistid")
+		}
+		fields = append(fields, cand.echoes...)
+		if len(fields) > 0 {
+			plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
+		}
+	}
+	return plans, nil
+}
+
+// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
+// handler. Generous timeout: the read phase is batched, but the write phase is
+// one round-trip per deleted field.
+func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	return c.SweepLiteratureEchoes(ctx, apply)
+}

internal/storage/literature_echo_sweep_test.go

@@ -0,0 +1,249 @@
+package storage
+
+import (
+	"context"
+	"encoding/base32"
+	"encoding/binary"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Pinned from meta-read `src/card_cid.rs` — literals, not round-trips.
+const (
+	literatureAnchorCardCID = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy" // ("anilist", "manga:105778")
+	literatureSiteCardCID   = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
+	// A non-card CID a card may also carry.
+	literatureContentCID = "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
+)
+
+// mintCardCID encodes a card locator the way meta-feeder-sdk
+// `compute_card_cid` does. Asserted against the pinned literals before use.
+func mintCardCID(source, id string) string {
+	digest := binary.AppendUvarint(nil, uint64(len(source)))
+	digest = append(digest, source...)
+	digest = append(digest, id...)
+	wire := []byte{0x01}
+	wire = binary.AppendUvarint(wire, 0x1007)
+	wire = append(wire, 0x00)
+	wire = binary.AppendUvarint(wire, uint64(len(digest)))
+	wire = append(wire, digest...)
+	return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(wire))
+}
+
+func TestIsSiteFormCardLocator(t *testing.T) {
+	if got := mintCardCID("anilist", "manga:105778"); got != literatureAnchorCardCID {
+		t.Fatalf("test encoder drifted: %s != %s", got, literatureAnchorCardCID)
+	}
+	if got := mintCardCID("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"); got != literatureSiteCardCID {
+		t.Fatalf("test encoder drifted: %s != %s", got, literatureSiteCardCID)
+	}
+
+	for _, tc := range []struct {
+		name string
+		cid  string
+		want bool
+	}{
+		{"anilist anchor (golden)", literatureAnchorCardCID, false},
+		{"mangadex site (golden)", literatureSiteCardCID, true},
+		{"openlibrary anchor", mintCardCID("openlibrary", "OL45804W"), false},
+		{"another site", mintCardCID("weebcentral.com", "/series/01J76XY/chainsaw-man"), true},
+		// Not an anchor source, but not host-shaped either: meta-read reads it
+		// as an id-bag key, so it must not be treated as a site.
+		{"dotless unknown source", mintCardCID("suwayomi", "42"), false},
+		{"content CID", literatureContentCID, false},
+		{"title key", "title:chainsaw man", false},
+	} {
+		if got := isSiteFormCardLocator(tc.cid); got != tc.want {
+			t.Errorf("%s: isSiteFormCardLocator = %v, want %v", tc.name, got, tc.want)
+		}
+	}
+}
+
+// seedLiteratureCorpus writes one record per rule the sweep has to honour.
+func seedLiteratureCorpus(t *testing.T, c *Client) {
+	t.Helper()
+
+	// A site card carrying every echo: all of it goes.
+	seedRecord(t, c, "site-card", map[string]string{
+		"domain":                        "literature",
+		"fileType":                      "card",
+		"title":                         "Chainsaw Man",
+		"cids/" + literatureSiteCardCID: "true",
+		"anilistid":                     "105778",
+		"workcid":                       literatureSiteCardCID,
+		"chapterNumber":                 "12",
+		"volumeNumber":                  "2",
+		"chapterStart":                  "1",
+		"chapterEnd":                    "20",
+	})
+	// An anchor card: its anilistid is its identity and stays; the position
+	// echo and the workcid still go.
+	seedRecord(t, c, "anchor-card", map[string]string{
+		"domain":                          "literature",
+		"fileType":                        "card",
+		"title":                           "Chainsaw Man",
+		"cids/" + literatureAnchorCardCID: "true",
+		"anilistid":                       "105778",
+		"workcid":                         literatureAnchorCardCID,
+		"chapterNumber":                   "3",
+	})
+	// A site card that is already clean: nothing planned.
+	seedRecord(t, c, "clean-site-card", map[string]string{
+		"domain":                        "literature",
+		"fileType":                      "card",
+		"cids/" + literatureSiteCardCID: "true",
+	})
+	// Both locator forms: which one the id belongs to is unknowable, so it
+	// stays.
+	seedRecord(t, c, "merged-card", map[string]string{
+		"domain":                          "literature",
+		"fileType":                        "card",
+		"cids/" + literatureSiteCardCID:   "true",
+		"cids/" + literatureAnchorCardCID: "true",
+		"anilistid":                       "105778",
+	})
+	// No card locator at all: not provably a site card.
+	seedRecord(t, c, "unlocated-card", map[string]string{
+		"domain":                       "literature",
+		"fileType":                     "card",
+		"cids/" + literatureContentCID: "true",
+		"anilistid":                    "105778",
+	})
+	// A stored empty value is still an echo.
+	seedRecord(t, c, "blank-echo-card", map[string]string{
+		"domain":       "literature",
+		"fileType":     "card",
+		"volumeNumber": "",
+	})
+	// A chapter is never swept, whatever it carries.
+	seedRecord(t, c, "chapter", map[string]string{
+		"domain":                       "literature",
+		"fileType":                     "archive",
+		"cids/" + literatureContentCID: "true",
+		"anilistid":                    "105778",
+		"workcid":                      literatureSiteCardCID,
+		"chapterNumber":                "12",
+	})
+	// Not literature: out of scope even though it is a card.
+	seedRecord(t, c, "screen-card", map[string]string{
+		"domain":        "screen",
+		"fileType":      "card",
+		"workcid":       literatureSiteCardCID,
+		"chapterNumber": "1",
+	})
+}
+
+func TestPlanLiteratureEchoes_ScopesToLiteratureCards(t *testing.T) {
+	c, _ := newTestClient(t, "")
+	seedLiteratureCorpus(t, c)
+
+	plans, err := c.planLiteratureEchoes(context.Background())
+	if err != nil {
+		t.Fatalf("plan: %v", err)
+	}
+	got := map[string][]string{}
+	for _, p := range plans {
+		got[p.hashID] = p.fields
+	}
+	want := map[string][]string{
+		"site-card":       {"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"},
+		"anchor-card":     {"workcid", "chapterNumber"},
+		"blank-echo-card": {"volumeNumber"},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("plan =\n  %v\nwant\n  %v", got, want)
+	}
+}
+
+func TestSweepLiteratureEchoes_DryRunThenApply(t *testing.T) {
+	c, s := newTestClient(t, "mm:")
+	seedLiteratureCorpus(t, c)
+
+	wantByKey := map[string]int{
+		"anilistid":     1,
+		"workcid":       2,
+		"chapterNumber": 2,
+		"volumeNumber":  2,
+		"chapterStart":  1,
+		"chapterEnd":    1,
+	}
+
+	// Dry run: counts, and nothing written.
+	dry, err := c.SweepLiteratureEchoes(context.Background(), false)
+	if err != nil {
+		t.Fatalf("dry run: %v", err)
+	}
+	if dry.Planned != 3 || dry.Fixed != 0 {
+		t.Errorf("dry run planned/fixed = %d/%d, want 3/0", dry.Planned, dry.Fixed)
+	}
+	if !reflect.DeepEqual(dry.ByKey, wantByKey) {
+		t.Errorf("dry run byKey = %v, want %v", dry.ByKey, wantByKey)
+	}
+	if m, _ := c.GetMetadataFlat("site-card"); m["anilistid"] != "105778" {
+		t.Fatalf("dry run deleted anilistid: %v", m)
+	}
+
+	applied, err := c.SweepLiteratureEchoes(context.Background(), true)
+	if err != nil {
+		t.Fatalf("apply: %v", err)
+	}
+	if applied.Planned != 3 || applied.Fixed != 3 {
+		t.Errorf("apply planned/fixed = %d/%d, want 3/3", applied.Planned, applied.Fixed)
+	}
+	if !reflect.DeepEqual(applied.ByKey, wantByKey) {
+		t.Errorf("apply byKey = %v, want %v", applied.ByKey, wantByKey)
+	}
+
+	site, err := c.GetMetadataFlat("site-card")
+	if err != nil {
+		t.Fatalf("read site-card: %v", err)
+	}
+	for _, k := range []string{"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"} {
+		if _, ok := site[k]; ok {
+			t.Errorf("site-card still carries %s", k)
+		}
+		// The flat key and its field-name index entry must both be gone, or the
+		// next read would MGET a ghost field.
+		if s.Exists("mm:file:site-card/" + k) {
+			t.Errorf("site-card flat key %s survived", k)
+		}
+		if member, _ := s.IsMember("mm:file:site-card/"+FieldsField, k); member {
+			t.Errorf("site-card field index still lists %s", k)
+		}
+	}
+	if site["title"] != "Chainsaw Man" || site["cids/"+literatureSiteCardCID] != "true" {
+		t.Errorf("site-card lost fields it should keep: %v", site)
+	}
+
+	anchor, _ := c.GetMetadataFlat("anchor-card")
+	if anchor["anilistid"] != "105778" {
+		t.Errorf("anchor-card anilistid = %q, want kept", anchor["anilistid"])
+	}
+	if _, ok := anchor["workcid"]; ok {
+		t.Errorf("anchor-card still carries workcid")
+	}
+
+	chapter, _ := c.GetMetadataFlat("chapter")
+	if chapter["anilistid"] != "105778" || chapter["workcid"] != literatureSiteCardCID || chapter["chapterNumber"] != "12" {
+		t.Errorf("chapter was touched: %v", chapter)
+	}
+	screen, _ := c.GetMetadataFlat("screen-card")
+	if screen["workcid"] != literatureSiteCardCID {
+		t.Errorf("screen-card was touched: %v", screen)
+	}
+	merged, _ := c.GetMetadataFlat("merged-card")
+	if merged["anilistid"] != "105778" {
+		t.Errorf("merged-card anilistid = %q, want kept", merged["anilistid"])
+	}
+
+	// Idempotent: the endpoint is called by hand, and a second call must find
+	// nothing — with byKey still an empty map, not nil.
+	again, err := c.SweepLiteratureEchoes(context.Background(), true)
+	if err != nil {
+		t.Fatalf("second pass: %v", err)
+	}
+	if again.Planned != 0 || again.Fixed != 0 || again.ByKey == nil || len(again.ByKey) != 0 {
+		t.Errorf("second pass = %+v, want nothing planned and an empty byKey", again)
+	}
+}

internal/storage/search_dirty.go

@@ -0,0 +1,245 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Incremental search-index maintenance.
+//
+// # Why this exists
+//
+// The API's in-memory search index used to be rebuilt in full every 60 s:
+// GetSearchIndexTuples does a whole-keyspace SCAN plus batched MGETs, builds an
+// intermediate map of every record, then allocates a fresh []SearchTuple — all
+// while the previous index is still live serving reads. That cost is O(corpus),
+// not O(change): adding 200 records in a minute re-read every one of the other
+// ~620 000 field keys to notice them.
+//
+// Measured on a 19 622-record staging corpus (2026-08-30): SCAN was 57.8 % and
+// MGET 19.1 % of ALL Redis CPU, and ~90 % of the MGET calls plus ~48 % of the
+// SCAN calls were this loop — roughly 45 % of Redis's total work, spent
+// continuously whether or not anything changed. The Go side held 783 MB for
+// data Redis stores in 135 MB, largely because of the double copy.
+//
+// That was tolerable while the corpus was small and mostly static. It stops
+// being tolerable now that meta-core is the platform's permanent, additive
+// store: every search hit on every gateway is retained forever, so the rebuild
+// cost is a ratchet.
+//
+// # The shape
+//
+// Every write path marks its record in a dirty set; the refresh drains that set
+// and re-reads only those records. Cost becomes O(change).
+//
+// Marking rides along with the field-index maintenance every write path already
+// performs, so there is no new bookkeeping discipline to remember — if you are
+// adding a write path, the same line that keeps `__fields__` correct keeps this
+// correct.
+//
+// # Correctness
+//
+// The drain is a destructive SPOP, so a write that lands *during* a refresh is
+// either already in the drained batch or still in the set for the next tick —
+// never dropped. The cost of the race is one tick of staleness, which is well
+// inside the interval the index was already stale by.
+//
+// A refresh that fails after draining WOULD lose its batch, so drainDirty's
+// caller re-marks on error (see MarkDirty / the API's refreshIncremental).
+//
+// ⚠ **The destructive drain assumes ONE index consumer per Redis.** Two
+// meta-core processes sharing a keyspace would steal each other's marks, and
+// each would end up with an index missing whatever the other drained. That is
+// fine today — multi-instance leader election was deprecated and the dev/prod
+// stacks run a single meta-core — but it is the assumption to revisit first if
+// a second instance is ever pointed at the same Redis. A per-consumer dirty key
+// (`file:__dirty__:<consumer>`) is the shape that would fix it.
+//
+// The API's periodic full reconcile (searchIndexReconcileEvery) is the backstop
+// for both this and for a write path that forgets to mark: a missed mark costs
+// staleness until the next reconcile, never permanent divergence.
+
+// dirtyField is the reserved key holding the set of record ids whose metadata
+// changed since the last search-index refresh. It sits under the same `file:`
+// namespace as `__index__` and is likewise not a record.
+const dirtyField = "file:__dirty__"
+
+// dirtyDrainCap bounds one incremental refresh. A burst larger than this is
+// drained over several ticks rather than in one long-held pass — the point of
+// the change is to keep any single refresh cheap. Sized well above a normal
+// search-persist flush (which writes at most a few hundred records).
+const dirtyDrainCap = 5000
+
+// buildDirtyKey constructs the key for the dirty-record set.
+func (c *Client) buildDirtyKey() string {
+	return c.buildKey(dirtyField)
+}
+
+// markDirtyLocked records that hashID's metadata changed. Caller must hold c.mu.
+//
+// Best-effort by design: a failure here costs one record its incremental
+// refresh, and the periodic full rebuild still repairs it. Failing the caller's
+// write instead would trade a stale index entry for a lost write, which is
+// strictly worse.
+func (c *Client) markDirtyLocked(ctx context.Context, hashID string) {
+	if hashID == "" {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildDirtyKey(), hashID).Err(); err != nil {
+		log.Printf("[STORAGE] dirty-mark failed for %s: %v (index repairs on the next full rebuild)", hashID, err)
+	}
+}
+
+// MarkDirty records that hashID needs re-indexing. Exported for the API layer,
+// which re-marks a batch whose refresh failed after the drain.
+func (c *Client) MarkDirty(hashIDs ...string) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.client == nil || len(hashIDs) == 0 {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	members := make([]interface{}, 0, len(hashIDs))
+	for _, h := range hashIDs {
+		if h != "" {
+			members = append(members, h)
+		}
+	}
+	if len(members) == 0 {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildDirtyKey(), members...).Err(); err != nil {
+		log.Printf("[STORAGE] dirty re-mark failed for %d records: %v", len(members), err)
+	}
+}
+
+// DrainDirty removes and returns up to dirtyDrainCap record ids from the dirty
+// set. `more` reports whether entries remain, so the caller can schedule
+// another pass instead of waiting a full interval to catch up on a burst.
+//
+// Destructive on purpose: SPOP makes concurrent refreshes disjoint and stops a
+// record being re-read every tick forever. The caller owns the drained ids
+// until it has indexed them — on failure it must MarkDirty them again.
+func (c *Client) DrainDirty() (ids []string, more bool, err error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	key := c.buildDirtyKey()
+	ids, err = c.client.SPopN(ctx, key, dirtyDrainCap).Result()
+	if err != nil {
+		return nil, false, fmt.Errorf("spop dirty: %w", err)
+	}
+	if len(ids) < dirtyDrainCap {
+		return ids, false, nil
+	}
+	remaining, err := c.client.SCard(ctx, key).Result()
+	if err != nil {
+		// We have the batch; not knowing the remainder just means the next
+		// tick picks it up on schedule.
+		return ids, false, nil
+	}
+	return ids, remaining > 0, nil
+}
+
+// GetSearchIndexTuplesFor reads the search tuples for a specific set of records
+// — the incremental counterpart to GetSearchIndexTuples.
+//
+// Reads go through the per-record field index (GetMetadataFlat), so this is two
+// round-trips per record rather than a keyspace scan. That trade is what makes
+// it O(change): it beats the full pass whenever the changed set is a small
+// fraction of the corpus, and the caller bounds the batch (dirtyDrainCap).
+//
+// A record present in the dirty set but absent from storage was deleted; it is
+// reported in `gone` so the caller can evict it from the index rather than
+// leaving a tombstone that still answers searches.
+func (c *Client) GetSearchIndexTuplesFor(hashIDs []string) (tuples []SearchTuple, gone []string, err error) {
+	if len(hashIDs) == 0 {
+		return nil, nil, nil
+	}
+
+	excluded := indexExcludePrefixes()
+	tuples = make([]SearchTuple, 0, len(hashIDs))
+
+	for _, hashID := range hashIDs {
+		fields, ferr := c.GetMetadataFlat(hashID)
+		if ferr != nil {
+			// Re-mark so the record is retried rather than silently missing
+			// from the index until the next full rebuild.
+			c.MarkDirty(hashID)
+			log.Printf("[STORAGE] incremental index read failed for %s: %v", hashID, ferr)
+			continue
+		}
+		if len(fields) == 0 {
+			gone = append(gone, hashID)
+			continue
+		}
+		for field := range fields {
+			if hasAnyPrefix(field, excluded) {
+				delete(fields, field)
+			}
+		}
+		tuples = append(tuples, SearchTuple{
+			HashID:   hashID,
+			Haystack: buildHaystack(fields),
+			Fields:   fields,
+		})
+	}
+	return tuples, gone, nil
+}
+
+// buildHaystack folds a record's searchable text fields into one lowercased
+// blob. Shared by the full and incremental index reads so the two can never
+// disagree about what is searchable — a drift there would make a record's
+// matchability depend on which path happened to index it.
+//
+// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
+// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
+// sentinel "true", so a search for an alternative title (`3 percent`) finds the
+// record whose `title` is `3%`.
+func buildHaystack(fields map[string]string) string {
+	var sb strings.Builder
+	for _, f := range searchIndexFields {
+		if v, ok := fields[f]; ok && v != "" {
+			sb.WriteString(strings.ToLower(v))
+			sb.WriteByte('\n')
+		}
+	}
+	for _, name := range titleMemberNames(fields) {
+		sb.WriteString(strings.ToLower(name))
+		sb.WriteByte('\n')
+	}
+	return sb.String()
+}
+
+// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
+// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
+// back. Sorted: the fields map has no order, and the full and incremental
+// builds must produce byte-identical haystacks.
+func titleMemberNames(fields map[string]string) []string {
+	var names []string
+	for k, v := range fields {
+		rest, ok := strings.CutPrefix(k, "titles/")
+		if !ok || v == "" || v == "false" {
+			continue
+		}
+		_, name, ok := strings.Cut(rest, "/")
+		if !ok || name == "" {
+			continue
+		}
+		names = append(names, strings.ReplaceAll(name, "\u2215", "/"))
+	}
+	sort.Strings(names)
+	return names
+}

internal/storage/search_dirty_test.go

@@ -0,0 +1,230 @@
+package storage
+
+import (
+	"context"
+	"sort"
+	"testing"
+)
+
+// Tests for incremental search-index maintenance (search_dirty.go).
+//
+// The dirty set is derived state on the hot write path, which makes its failure
+// mode the dangerous kind: a MISSING mark doesn't error, it just leaves the
+// search index quietly serving a stale copy of that record until someone
+// triggers a full rebuild. So the property worth pinning is coverage — every
+// path that changes what a search would match must mark — and that is what the
+// bulk of these tests assert, one write path at a time.
+//
+// The second property is that a delete is a mark too, not an unmark: the
+// refresh has to LEARN the record is gone in order to evict it. Marking on
+// delete is counter-intuitive enough to be worth a test of its own.
+
+func dirtyMembers(t *testing.T, c *Client) []string {
+	t.Helper()
+	got, err := c.client.SMembers(context.Background(), c.buildDirtyKey()).Result()
+	if err != nil {
+		t.Fatalf("smembers dirty: %v", err)
+	}
+	sort.Strings(got)
+	return got
+}
+
+func clearDirty(t *testing.T, c *Client) {
+	t.Helper()
+	if err := c.client.Del(context.Background(), c.buildDirtyKey()).Err(); err != nil {
+		t.Fatalf("del dirty: %v", err)
+	}
+}
+
+// Every write path must mark. A path that doesn't leaves the index serving a
+// stale record with no error anywhere.
+func TestDirty_EveryWritePathMarks(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		cases := []struct {
+			name  string
+			id    string
+			write func(id string) error
+		}{
+			{"SetMetadataFlat", "r-set", func(id string) error {
+				return c.SetMetadataFlat(id, map[string]string{"title": "A"})
+			}},
+			{"SetProperty", "r-prop", func(id string) error {
+				return c.SetProperty(id, "title", "B")
+			}},
+			{"MergeMetadataFlat", "r-merge", func(id string) error {
+				_, err := c.MergeMetadataFlat(id, map[string]string{"title": "C"})
+				return err
+			}},
+			{"AddToSet", "r-add", func(id string) error {
+				_, err := c.AddToSet(id, "tags", "x")
+				return err
+			}},
+		}
+		for _, tc := range cases {
+			t.Run(tc.name, func(t *testing.T) {
+				clearDirty(t, c)
+				if err := tc.write(tc.id); err != nil {
+					t.Fatalf("%s: %v", tc.name, err)
+				}
+				got := dirtyMembers(t, c)
+				if len(got) != 1 || got[0] != tc.id {
+					t.Fatalf("%s must mark %q dirty, got %v", tc.name, tc.id, got)
+				}
+			})
+		}
+	})
+}
+
+// RemoveFromSet's non-empty branch changes a field's VALUE without changing the
+// field-name index, so it is the one path that a mark riding on field-index
+// maintenance would have missed. Pinned explicitly for that reason.
+func TestDirty_RemoveFromSetMarksEvenWhenTheFieldSurvives(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-rem"
+		for _, v := range []string{"x", "y"} {
+			if _, err := c.AddToSet(id, "tags", v); err != nil {
+				t.Fatalf("AddToSet: %v", err)
+			}
+		}
+		clearDirty(t, c)
+
+		if _, err := c.RemoveFromSet(id, "tags", "x"); err != nil {
+			t.Fatalf("RemoveFromSet: %v", err)
+		}
+		// `tags` still exists (y remains) — no field-index change, but the
+		// searchable value changed.
+		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
+			t.Fatalf("a value-only change must still mark dirty, got %v", got)
+		}
+	})
+}
+
+// A delete MARKS rather than unmarks: the refresh must read the record, find it
+// gone, and evict it. Unmarking would leave a deleted record answering searches
+// until the next full rebuild.
+func TestDirty_DeletesMarkSoTheRefreshCanEvict(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-del"
+		if err := c.SetMetadataFlat(id, map[string]string{"title": "Doomed"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		clearDirty(t, c)
+
+		if _, err := c.DeleteMetadata(id); err != nil {
+			t.Fatalf("DeleteMetadata: %v", err)
+		}
+		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
+			t.Fatalf("a delete must mark dirty so the index evicts it, got %v", got)
+		}
+
+		// …and the incremental read reports it as gone rather than as an error
+		// or an empty-but-present tuple.
+		tuples, gone, err := c.GetSearchIndexTuplesFor([]string{id})
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
+		}
+		if len(tuples) != 0 {
+			t.Fatalf("a deleted record must produce no tuple, got %d", len(tuples))
+		}
+		if len(gone) != 1 || gone[0] != id {
+			t.Fatalf("a deleted record must be reported gone, got %v", gone)
+		}
+	})
+}
+
+// The drain is destructive so concurrent refreshes stay disjoint and a record
+// isn't re-read every tick forever. The caller owns what it drained.
+func TestDirty_DrainIsDestructive(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		clearDirty(t, c)
+		c.MarkDirty("a", "b", "c")
+
+		ids, more, err := c.DrainDirty()
+		if err != nil {
+			t.Fatalf("DrainDirty: %v", err)
+		}
+		sort.Strings(ids)
+		if len(ids) != 3 {
+			t.Fatalf("expected 3 drained ids, got %v", ids)
+		}
+		if more {
+			t.Fatal("a fully-drained set must not report more")
+		}
+		if got := dirtyMembers(t, c); len(got) != 0 {
+			t.Fatalf("drain must empty the set, got %v", got)
+		}
+
+		// A second drain is empty, not an error — the steady state when nothing
+		// has been written since the last tick.
+		ids, _, err = c.DrainDirty()
+		if err != nil || len(ids) != 0 {
+			t.Fatalf("draining an empty set must be a clean no-op, got %v / %v", ids, err)
+		}
+	})
+}
+
+// The incremental read must produce tuples indistinguishable from the full
+// rebuild's — same haystack, same fields, same exclusions. A drift here would
+// make a record's matchability depend on which path happened to index it, which
+// is invisible until someone searches for the one term that differs.
+func TestDirty_IncrementalTuplesMatchTheFullRebuild(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-parity"
+		if err := c.SetMetadataFlat(id, map[string]string{
+			"title":                   "Parity Release",
+			"fileName":                "parity.s01e01.mkv",
+			"fileType":                "video",
+			"titles/eng/3 Percent":    "true",
+			"titles/fra/3 Pourcent":   "true",
+			"categories/XXX":          "true",
+			"categories/newznab/5070": "true",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		full := tuplesByHashID(t, c)[id]
+		incr, gone, err := c.GetSearchIndexTuplesFor([]string{id})
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
+		}
+		if len(gone) != 0 || len(incr) != 1 {
+			t.Fatalf("expected exactly one live tuple, got %d live / %d gone", len(incr), len(gone))
+		}
+
+		if incr[0].Haystack != full.Haystack {
+			t.Errorf("haystack drift:\n incremental %q\n full        %q", incr[0].Haystack, full.Haystack)
+		}
+		if len(incr[0].Fields) != len(full.Fields) {
+			t.Errorf("field-count drift: incremental %d, full %d", len(incr[0].Fields), len(full.Fields))
+		}
+		for k, v := range full.Fields {
+			if incr[0].Fields[k] != v {
+				t.Errorf("field %q drift: incremental %q, full %q", k, incr[0].Fields[k], v)
+			}
+		}
+		// The index projection must apply on both paths.
+		if _, ok := incr[0].Fields["categories/newznab/5070"]; ok {
+			t.Error("the incremental path must honour the index exclusion list too")
+		}
+		if _, ok := incr[0].Fields["categories/XXX"]; !ok {
+			t.Error("the incremental path must keep the adult-filter category")
+		}
+	})
+}
+
+// A `titles/<lang3>/<name>` member contributes its NAME (the key), never its
+// "true" value, and the `∕` a writer substituted for `/` reads back as `/`.
+func TestBuildHaystack_FoldsTitleMemberNames(t *testing.T) {
+	got := buildHaystack(map[string]string{
+		"title":                     "3%",
+		"titles/fra/3 Pourcent":     "true",
+		"titles/eng/3 Percent":      "true",
+		"titles/und/Fate\u2215Zero": "true",
+		"titles/eng":                "Retired Scalar",
+		"titles/eng/":               "true",
+	})
+	want := "3%\n3 percent\n3 pourcent\nfate/zero\n"
+	if got != want {
+		t.Errorf("haystack = %q, want %q", got, want)
+	}
+}

internal/storage/search_index_exclude_test.go

@@ -0,0 +1,116 @@
+package storage
+
+import (
+	"testing"
+)
+
+// Tests for the search-index field projection (GetSearchIndexTuples +
+// indexExcludePrefixes).
+//
+// The projection narrows the in-memory index only — Redis keeps every field and
+// every per-record read still returns it. What has to be pinned is the boundary
+// between the two, because getting it wrong in either direction is silent:
+// exclude too little and the index keeps paying for fields nothing reads;
+// exclude too much and a record served *from the index* comes back missing a
+// field a downstream consumer depends on.
+//
+// The specific hazard the default guards is adult-content filtering:
+// `categories/XXX` is the server-side enforcement point (meta-search's
+// `query_eval::fields_are_adult`, meta-gateway's `cards::fields_are_adult`),
+// and `exclude_adult` defaults to ON. Excluding the whole `categories/` family
+// would make every index-served record read as non-adult and the filter would
+// fail OPEN — so the default must keep `categories/XXX` while dropping the
+// reader-less `categories/newznab/` mirror.
+
+func tuplesByHashID(t *testing.T, c *Client) map[string]SearchTuple {
+	t.Helper()
+	got, err := c.GetSearchIndexTuples()
+	if err != nil {
+		t.Fatalf("GetSearchIndexTuples: %v", err)
+	}
+	out := make(map[string]SearchTuple, len(got))
+	for _, tu := range got {
+		out[tu.HashID] = tu
+	}
+	return out
+}
+
+func TestSearchIndex_ExcludesNewznabCategoriesButKeepsTheRest(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "root-1"
+		if err := c.SetMetadataFlat(id, map[string]string{
+			"title":                   "Some Release",
+			"fileType":                "video",
+			"categories/TV/Anime":     "true",
+			"categories/XXX":          "true",
+			"categories/newznab/5070": "true",
+			"categories/newznab/2040": "true",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		fields := tuplesByHashID(t, c)[id].Fields
+		if fields == nil {
+			t.Fatal("record missing from the search index")
+		}
+
+		for _, excluded := range []string{"categories/newznab/5070", "categories/newznab/2040"} {
+			if _, ok := fields[excluded]; ok {
+				t.Errorf("%s must not be indexed — it is a reader-less mirror vocabulary", excluded)
+			}
+		}
+		// The load-bearing ones must survive. XXX especially: without it the
+		// adult filter fails open on every index-served record.
+		for _, kept := range []string{"categories/XXX", "categories/TV/Anime", "title", "fileType"} {
+			if _, ok := fields[kept]; !ok {
+				t.Errorf("%s must stay in the index", kept)
+			}
+		}
+
+		// The store itself is untouched — the projection is index-only.
+		stored, err := c.GetMetadataFlat(id)
+		if err != nil {
+			t.Fatalf("GetMetadataFlat: %v", err)
+		}
+		if _, ok := stored["categories/newznab/5070"]; !ok {
+			t.Error("an index-excluded field must still be stored and readable per-record")
+		}
+	})
+}
+
+func TestIndexExcludePrefixes_EnvOverride(t *testing.T) {
+	if got := indexExcludePrefixes(); len(got) != 1 || got[0] != "categories/newznab/" {
+		t.Fatalf("unset env must yield the default, got %v", got)
+	}
+
+	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "foo/, bar/ ,")
+	got := indexExcludePrefixes()
+	if len(got) != 2 || got[0] != "foo/" || got[1] != "bar/" {
+		t.Fatalf("override must trim and drop blanks, got %v", got)
+	}
+
+	// An explicitly empty value is the escape hatch: index everything.
+	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "")
+	if got := indexExcludePrefixes(); len(got) != 0 {
+		t.Fatalf("an empty override must disable exclusion, got %v", got)
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	prefixes := []string{"categories/newznab/", "tmp/"}
+	cases := map[string]bool{
+		"categories/newznab/5070": true,
+		"tmp/scratch":             true,
+		"categories/XXX":          false,
+		"categories/newznab":      false, // the family root is not the mirror
+		"title":                   false,
+	}
+	for field, want := range cases {
+		if got := hasAnyPrefix(field, prefixes); got != want {
+			t.Errorf("hasAnyPrefix(%q) = %v, want %v", field, got, want)
+		}
+	}
+	if hasAnyPrefix("anything", nil) {
+		t.Error("an empty prefix list must exclude nothing")
+	}
+}

internal/storage/stats.go

@@ -0,0 +1,52 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// CountRoots returns the number of metadata roots in the file index.
+//
+// SCARD, not SMEMBERS: GetAllHashIDs ships every member over the wire (~3MB
+// and rising on an 88k-record box) just so the caller can take len(). A
+// dashboard tile that polls needs the number, never the list.
+func (c *Client) CountRoots() (int64, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	n, err := c.client.SCard(ctx, c.buildIndexKey()).Result()
+	if err != nil {
+		return 0, fmt.Errorf("scard failed: %w", err)
+	}
+	return n, nil
+}
+
+// CountKeys returns DBSIZE — every key in the Redis database, not just the
+// ones meta-core owns. It is the denominator for "how much of Redis is
+// records": ~3.1M keys against ~89k roots means the flat per-field keys
+// (file:{root}/{field}) dominate, which is expected.
+func (c *Client) CountKeys() (int64, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	n, err := c.client.DBSize(ctx).Result()
+	if err != nil {
+		return 0, fmt.Errorf("dbsize failed: %w", err)
+	}
+	return n, nil
+}

internal/storage/udl.go

@@ -0,0 +1,481 @@
+package storage
+
+// User Data Layer (UDL) storage.
+//
+// Holds per-user, identity-signed records keyed by (uid, cid, key) — the
+// subjective/per-user state that METADATA_KEYS.md rule #4 bars from the shared
+// /file/{cid} hash (likes, ratings, watch progress, local paths). Each record
+// is an opaque, client-signed (and, for private-tier keys, client-encrypted)
+// CBOR blob; meta-core stores it verbatim and never needs the account key.
+//
+// Convergence is version-gated: a write is accepted only if its `version`
+// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
+// op-log merge that used to live in meta-watch's store.rs).
+//
+// Redis model (all under the literal "udl:" prefix, mirroring the "file:" one):
+//
+//	udl:rec:<uid>/<cid>/<key>       HASH { record, version, ts, value? }
+//	udl:idx:user:<uid>:key:<key>    SET of <cid>   -> active_for_user_key (My List, Continue Watching)
+//	udl:idx:user:<uid>:cid:<cid>    SET of <key>   -> list_for_user_cid
+//	udl:idx:user:<uid>:cids         SET of <cid>   -> whole-profile export (recovery)
+//	udl:idx:cid:<cid>:key:<key>     SET of <uid>   -> aggregate
+//	udl:idx:cid:<cid>:users         SET of <uid>   -> uids_for_cid
+//
+// The `value` hash field is the plaintext JSON scalar for PUBLIC keys only
+// (like/rating), used for aggregation; it is absent for private-tier keys
+// (whose real value rides encrypted inside `record`) and for tombstones.
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// UDLCell is a decoded (uid, cid, key) record cell.
+type UDLCell struct {
+	Record   string // base64 CBOR, opaque + signed
+	Version  int64
+	Ts       int64
+	Value    string // plaintext JSON scalar, public keys only
+	HasValue bool
+}
+
+// UDLUserKeyEntry is one row of active_for_user_key (all cids for a uid+key).
+type UDLUserKeyEntry struct {
+	Cid     string `json:"cid"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+// UDLUserCidEntry is one row of list_for_user_cid (all keys for a uid+cid).
+type UDLUserCidEntry struct {
+	Key     string `json:"key"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+func udlRecKey(uid, cid, key string) string {
+	return fmt.Sprintf("udl:rec:%s/%s/%s", uid, cid, key)
+}
+func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
+func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
+func udlIdxCidKey(cid, key string) string  { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
+func udlIdxCidUsers(cid string) string      { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
+
+// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
+// index that makes "export this whole profile" a set read instead of a
+// keyspace scan.
+//
+// The other four indexes each answer a question with one coordinate already
+// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and
+// nothing else. Without this set the only ways to enumerate a profile are to
+// iterate the hardcoded key list — which silently misses any key added later —
+// or to SCAN `udl:rec:<uid>/*`, which is the full-keyspace-read pathology that
+// already made `/api/udl/users/stats` admin-only.
+func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
+
+// udlUpsertScript performs the version-gated upsert + index maintenance
+// atomically. Accepts (returns 1) only when the incoming version strictly
+// exceeds the stored one; otherwise returns 0 (stale).
+//
+//	KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
+//	      6=idx:user:cids
+//	ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
+//	      6=cid 7=key 8=uid 9=tombstone("1"/"0")
+//
+// A tombstone still writes the cell — that is the CRDT state, and dropping it
+// would let an older write resurrect the value — but it SREMs the cid from the
+// user+key index. That index is what `active_for_user_key` returns, so leaving
+// deleted cids in it means every My List / Continue Watching read carries every
+// title ever un-liked or finished, forever. The other four indexes are left
+// alone: they answer "who has an opinion about this cid", where a tombstone is
+// still an answer.
+//
+// ⚠ `idx:user:cids` (KEYS[6]) in particular must NEVER be pruned on a
+// tombstone, even though it looks symmetric with KEYS[2]. It is the export
+// index: a cid dropped from it is a cid whose tombstone never travels, and a
+// tombstone that does not travel is exactly how an un-liked title comes back
+// from the dead on the other box — the peer still holds the older `like` at a
+// lower version, and with no tombstone to beat it, nothing ever supersedes it.
+var udlUpsertScript = redis.NewScript(`
+local stored = redis.call('HGET', KEYS[1], 'version')
+if stored and tonumber(stored) >= tonumber(ARGV[1]) then
+  return 0
+end
+redis.call('HSET', KEYS[1], 'version', ARGV[1], 'ts', ARGV[2], 'record', ARGV[3])
+if ARGV[5] == '1' then
+  redis.call('HSET', KEYS[1], 'value', ARGV[4])
+else
+  redis.call('HDEL', KEYS[1], 'value')
+end
+if ARGV[9] == '1' then
+  redis.call('SREM', KEYS[2], ARGV[6])
+else
+  redis.call('SADD', KEYS[2], ARGV[6])
+end
+redis.call('SADD', KEYS[3], ARGV[7])
+redis.call('SADD', KEYS[4], ARGV[8])
+redis.call('SADD', KEYS[5], ARGV[8])
+redis.call('SADD', KEYS[6], ARGV[6])
+return 1
+`)
+
+// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
+// one. publicValue/hasValue carry the plaintext JSON scalar for public keys
+// (used by aggregation); pass hasValue=false for private-tier keys and
+// tombstones. Set tombstone=true when the record deletes the cell, which also
+// prunes the cid from the user+key index. Returns accepted=false when the write
+// is stale.
+func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client == nil {
+		return false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	keys := []string{
+		c.buildKey(udlRecKey(uid, cid, key)),
+		c.buildKey(udlIdxUserKey(uid, key)),
+		c.buildKey(udlIdxUserCid(uid, cid)),
+		c.buildKey(udlIdxCidKey(cid, key)),
+		c.buildKey(udlIdxCidUsers(cid)),
+		c.buildKey(udlIdxUserCids(uid)),
+	}
+	hv := "0"
+	if hasValue {
+		hv = "1"
+	}
+	tomb := "0"
+	if tombstone {
+		tomb = "1"
+	}
+	res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
+	if err != nil {
+		return false, fmt.Errorf("udl upsert failed: %w", err)
+	}
+	n, _ := res.(int64)
+	return n == 1, nil
+}
+
+// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
+// no record has ever been written for the cell.
+func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return UDLCell{}, false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	return c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+}
+
+// udlReadCell HMGETs a cell hash. Caller holds the lock. exists is false when
+// the version field is absent (i.e. the hash does not exist).
+func (c *Client) udlReadCell(ctx context.Context, recKey string) (UDLCell, bool, error) {
+	vals, err := c.client.HMGet(ctx, c.buildKey(recKey), "record", "version", "ts", "value").Result()
+	if err != nil {
+		return UDLCell{}, false, fmt.Errorf("hmget failed: %w", err)
+	}
+	// vals: [record, version, ts, value] — nil for absent fields.
+	if len(vals) < 4 || vals[1] == nil {
+		return UDLCell{}, false, nil
+	}
+	cell := UDLCell{}
+	if s, ok := vals[0].(string); ok {
+		cell.Record = s
+	}
+	if s, ok := vals[1].(string); ok {
+		cell.Version, _ = strconv.ParseInt(s, 10, 64)
+	}
+	if s, ok := vals[2].(string); ok {
+		cell.Ts, _ = strconv.ParseInt(s, 10, 64)
+	}
+	if s, ok := vals[3].(string); ok {
+		cell.Value = s
+		cell.HasValue = true
+	}
+	return cell, true, nil
+}
+
+// UDLListUserKey returns every cell for (uid, key) across all cids — the
+// projection behind active_for_user_key (My List, Continue Watching). Includes
+// tombstones; the caller decodes each record and filters.
+func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserKey(uid, key))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]UDLUserKeyEntry, 0, len(cids))
+	for _, cid := range cids {
+		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		out = append(out, UDLUserKeyEntry{Cid: cid, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
+	}
+	return out, nil
+}
+
+// UDLListUserCid returns every cell for (uid, cid) across all keys — the
+// projection behind list_for_user_cid.
+func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]UDLUserCidEntry, 0, len(keys))
+	for _, key := range keys {
+		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		out = append(out, UDLUserCidEntry{Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
+	}
+	return out, nil
+}
+
+// UDLUserAllEntry is one row of a whole-profile export.
+type UDLUserAllEntry struct {
+	Cid     string `json:"cid"`
+	Key     string `json:"key"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
+//
+// Ordered by cid (sorted, so paging is stable across calls even though the
+// underlying index is a Redis SET), starting strictly after `afterCid`. A cid's
+// keys are never split across a page: the page ends on a cid boundary once
+// `limit` rows have been emitted, and `nextCid` is the last cid included — pass
+// it back as `afterCid` to continue. `nextCid` is empty when the export is
+// complete.
+//
+// Tombstones are included. They are the CRDT state that stops a deleted value
+// resurrecting on the box that receives this export, so omitting them would
+// make recovery actively wrong rather than merely incomplete.
+func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, "", fmt.Errorf("not connected")
+	}
+	if limit <= 0 {
+		limit = 1000
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
+	if err != nil {
+		return nil, "", fmt.Errorf("smembers failed: %w", err)
+	}
+	sort.Strings(cids)
+
+	out := make([]UDLUserAllEntry, 0, limit)
+	nextCid := ""
+	for _, cid := range cids {
+		if afterCid != "" && cid <= afterCid {
+			continue
+		}
+		if len(out) >= limit {
+			nextCid = out[len(out)-1].Cid
+			break
+		}
+		keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
+		if err != nil {
+			return nil, "", fmt.Errorf("smembers failed: %w", err)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+			if err != nil {
+				return nil, "", err
+			}
+			if !ok {
+				continue
+			}
+			out = append(out, UDLUserAllEntry{
+				Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts,
+			})
+		}
+	}
+	return out, nextCid, nil
+}
+
+// UDLCountUserCids is the number of distinct cids a uid holds records for.
+//
+// O(1) (a Redis SCARD), which is what makes it usable as a per-push quota
+// check on the gateway mirror — a quota that cost a keyspace walk would be a
+// denial-of-service in its own right. Counts cids rather than cells because
+// that is what the index holds; a profile is bounded by the titles it has
+// touched, not by how many keys it has per title.
+func (c *Client) UDLCountUserCids(uid string) (int64, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	n, err := c.client.SCard(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
+	if err != nil {
+		return 0, fmt.Errorf("scard failed: %w", err)
+	}
+	return n, nil
+}
+
+// UDLBackfillUserCids rebuilds the `idx:user:<uid>:cids` sets for data written
+// before that index existed, and reports how many cids it added.
+//
+// One-shot: the upsert script maintains the set from now on. It is a SCAN, but
+// a targeted one (`udl:idx:user:*:cid:*`, which is one key per (uid,cid), not
+// per record) run once at boot rather than per read — the distinction that
+// makes the difference between a migration and the pathology this index exists
+// to avoid.
+func (c *Client) UDLBackfillUserCids() (int, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	pattern := c.buildKey("udl:idx:user:*:cid:*")
+	prefix := c.buildKey("udl:idx:user:")
+	added := 0
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, pattern, 500).Result()
+		if err != nil {
+			return added, fmt.Errorf("scan failed: %w", err)
+		}
+		for _, k := range keys {
+			// <prefix><uid>:cid:<cid> — split on the FIRST separator, not the
+			// last. A uid is multibase base58btc, whose alphabet contains no
+			// colon at all, so the first ":cid:" always ends the uid. A cid
+			// that happens to contain ":cid:" in its own name would make the
+			// last occurrence split in the wrong place.
+			rest := strings.TrimPrefix(k, prefix)
+			i := strings.Index(rest, ":cid:")
+			if i < 0 {
+				continue
+			}
+			uid, cid := rest[:i], rest[i+len(":cid:"):]
+			if uid == "" || cid == "" {
+				continue
+			}
+			n, err := c.client.SAdd(ctx, c.buildKey(udlIdxUserCids(uid)), cid).Result()
+			if err != nil {
+				return added, fmt.Errorf("sadd failed: %w", err)
+			}
+			added += int(n)
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+	return added, nil
+}
+
+// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
+func (c *Client) UDLCidUsers(cid string) ([]string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidUsers(cid))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	return uids, nil
+}
+
+// UDLPublicValues returns the live plaintext public JSON scalar values for
+// (cid, key) across all uids — the input to aggregation. Cells without a value
+// field (private-tier or tombstoned) are skipped.
+func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidKey(cid, key))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]string, 0, len(uids))
+	for _, uid := range uids {
+		v, err := c.client.HGet(ctx, c.buildKey(udlRecKey(uid, cid, key)), "value").Result()
+		if err == redis.Nil {
+			continue // private-tier or tombstoned
+		}
+		if err != nil {
+			return nil, fmt.Errorf("hget failed: %w", err)
+		}
+		out = append(out, v)
+	}
+	return out, nil
+}

internal/storage/udl_admin.go

@@ -0,0 +1,247 @@
+package storage
+
+// Account-scoped UDL administration: what a uid holds, and removing all of it.
+//
+// Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
+// cell — because there is no per-user cid index to consult. The Redis model
+// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
+// "what is in My List", not "everything this account ever wrote". Adding a
+// third index would put a write on every UDL upsert to serve two admin calls,
+// so these SCAN instead and accept being O(keyspace).
+//
+// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
+// one: the accounts page needs a number for every account, and MATCH filters
+// server-side without indexing, so N per-uid calls would walk the keyspace N
+// times. One walk buckets them all.
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// UDLUserStats counts what one account holds. Records is the number of cells,
+// including tombstones — a retracted rating is still a row this account owns
+// and still something a purge has to remove.
+type UDLUserStats struct {
+	Records int `json:"records"`
+	Cids    int `json:"cids"`
+	Keys    int `json:"keys"`
+}
+
+// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
+const udlScanBatch = 1000
+
+// safeUIDForPattern rejects anything that would change the meaning of a SCAN
+// glob or reach across the key structure. meta-core's uids are alphanumeric
+// (identity.validUID), so this only ever fires on a hostile or malformed one —
+// but it fires here, in the layer that builds the pattern, rather than relying
+// on every caller having validated first.
+func safeUIDForPattern(uid string) error {
+	if uid == "" {
+		return fmt.Errorf("empty uid")
+	}
+	if strings.ContainsAny(uid, "*?[]\\/:") {
+		return fmt.Errorf("uid contains characters that are not valid in a uid: %q", uid)
+	}
+	return nil
+}
+
+// splitUDLRecSuffix splits the "<uid>/<cid>/<key>" tail of a udl:rec key.
+//
+// SplitN with a limit of 3 so a key containing a slash survives intact. cids
+// never contain one; keys are a controlled vocabulary today, but a namespaced
+// one arriving later must not silently truncate here — a truncated key would
+// make the purge SREM the wrong index member and leave the real one behind.
+func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
+	parts := strings.SplitN(suffix, "/", 3)
+	if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
+		return "", "", "", false
+	}
+	return parts[0], parts[1], parts[2], true
+}
+
+// scanKeys walks the keyspace for pattern and returns every match. Caller holds
+// the lock.
+//
+// SCAN can return the same key more than once across cursor iterations (it
+// guarantees no misses, not no duplicates), so callers that count must
+// de-duplicate — both callers below collect into maps or sets for that reason.
+func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
+	var (
+		out    []string
+		cursor uint64
+	)
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
+		if err != nil {
+			return nil, fmt.Errorf("scan %q: %w", pattern, err)
+		}
+		out = append(out, keys...)
+		cursor = next
+		if cursor == 0 {
+			return out, nil
+		}
+	}
+}
+
+// UDLUserCells returns every (cid, key) pair this account holds a cell for.
+// De-duplicated, since SCAN may repeat a key.
+func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
+	// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
+	// the cid, which the purge needs grouped.
+	if err := safeUIDForPattern(uid); err != nil {
+		return nil, err
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	prefix := c.buildKey("udl:rec:")
+	keys, err := c.scanKeys(ctx, prefix+uid+"/*")
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]string, len(keys))
+	for _, full := range keys {
+		gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
+		// The glob can only match this uid, but a uid that is a prefix of
+		// another would still be caught by the '/' in the pattern; check anyway
+		// so a malformed key can never be attributed to the wrong account.
+		if !ok || gotUID != uid {
+			continue
+		}
+		out[cid+"/"+key] = cid
+	}
+	return out, nil
+}
+
+// UDLAllUserStats counts every account's cells in one keyspace walk.
+//
+// Returns only accounts that actually hold records; a uid absent from the map
+// holds none. Callers render a zero for those rather than treating it as an
+// error — an account created a minute ago legitimately has nothing.
+func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	prefix := c.buildKey("udl:rec:")
+	keys, err := c.scanKeys(ctx, prefix+"*")
+	if err != nil {
+		return nil, err
+	}
+
+	// Sets rather than counters: SCAN may hand back the same key twice, which
+	// would otherwise inflate every number on the page.
+	type acc struct {
+		cells map[string]struct{}
+		cids  map[string]struct{}
+		keys  map[string]struct{}
+	}
+	byUID := make(map[string]*acc)
+	for _, full := range keys {
+		uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
+		if !ok {
+			continue
+		}
+		a := byUID[uid]
+		if a == nil {
+			a = &acc{
+				cells: make(map[string]struct{}),
+				cids:  make(map[string]struct{}),
+				keys:  make(map[string]struct{}),
+			}
+			byUID[uid] = a
+		}
+		a.cells[cid+"/"+key] = struct{}{}
+		a.cids[cid] = struct{}{}
+		a.keys[key] = struct{}{}
+	}
+
+	out := make(map[string]UDLUserStats, len(byUID))
+	for uid, a := range byUID {
+		out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
+	}
+	return out, nil
+}
+
+// UDLPurgeUser removes every trace of one account from the User Data Layer:
+// its cells, its three user-scoped indexes, and its membership in the shared
+// cid-scoped indexes. Returns what was removed.
+//
+// The shared indexes are the part that is easy to miss and expensive to leave.
+// `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
+// not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
+// uid listed as an opinion-holder on every title it ever touched. Aggregation
+// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
+// rating average stays right, but uids_for_cid keeps naming a deleted account
+// forever and every aggregate read pays for the lookup.
+//
+// Idempotent: purging an account with nothing stored is a no-op returning
+// zeroes, so a retry after a partial failure is safe.
+func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
+	cells, err := c.UDLUserCells(uid)
+	if err != nil {
+		return UDLUserStats{}, err
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.client == nil {
+		return UDLUserStats{}, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	stats := UDLUserStats{Records: len(cells)}
+	distinctCids := make(map[string]struct{})
+	distinctKeys := make(map[string]struct{})
+
+	pipe := c.client.Pipeline()
+	for cidKey, cid := range cells {
+		key := strings.TrimPrefix(cidKey, cid+"/")
+		distinctCids[cid] = struct{}{}
+		distinctKeys[key] = struct{}{}
+		pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
+		pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
+	}
+	for cid := range distinctCids {
+		pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
+	}
+	stats.Cids = len(distinctCids)
+	stats.Keys = len(distinctKeys)
+
+	// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are enumerable
+	// directly, and are dropped whole rather than member-by-member. The glob is
+	// what keeps this correct as indexes are added — `:cids` needed no change
+	// here when it landed. Scanning them separately also catches
+	// index entries whose cell is already gone — exactly the state a previously
+	// half-finished purge leaves behind.
+	idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
+	idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
+	if err != nil {
+		return UDLUserStats{}, err
+	}
+	for _, k := range idxKeys {
+		pipe.Del(ctx, k)
+	}
+
+	if _, err := pipe.Exec(ctx); err != nil {
+		return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
+	}
+	return stats, nil
+}

internal/storage/udl_admin_test.go

@@ -0,0 +1,253 @@
+package storage
+
+import (
+	"testing"
+)
+
+// seedCell writes one live UDL cell the way a real client would.
+func seedCell(t *testing.T, c *Client, uid, cid, key string) {
+	t.Helper()
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
+		t.Fatalf("seed %s/%s/%s: ok=%v err=%v", uid, cid, key, ok, err)
+	}
+}
+
+// The purge has to reach the cid-scoped indexes, not just the user's own rows.
+//
+// This is the failure that motivated the whole endpoint: deleting an account
+// while leaving `udl:idx:cid:<cid>:users` intact means every aggregate read on
+// every title that account ever touched keeps naming a uid nobody can resolve,
+// forever — and the rows are unreachable through any API, so nothing will ever
+// clean them up.
+func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+	seedCell(t, c, "zAlice", "cidOne", "rating")
+	seedCell(t, c, "zAlice", "cidTwo", "seek")
+
+	stats, err := c.UDLPurgeUser("zAlice")
+	if err != nil {
+		t.Fatalf("purge: %v", err)
+	}
+	if stats.Records != 3 || stats.Cids != 2 || stats.Keys != 3 {
+		t.Fatalf("stats = %+v, want 3 records / 2 cids / 3 keys", stats)
+	}
+
+	// Cells gone.
+	if _, ok, err := c.UDLGetCell("zAlice", "cidOne", "like"); err != nil || ok {
+		t.Fatalf("cell survived the purge: ok=%v err=%v", ok, err)
+	}
+	// User-scoped indexes gone.
+	if entries, err := c.UDLListUserKey("zAlice", "like"); err != nil || len(entries) != 0 {
+		t.Fatalf("user+key index survived: %d entries, err=%v", len(entries), err)
+	}
+	if entries, err := c.UDLListUserCid("zAlice", "cidOne"); err != nil || len(entries) != 0 {
+		t.Fatalf("user+cid index survived: %d entries, err=%v", len(entries), err)
+	}
+	// Cid-scoped index gone — the one that is easy to miss.
+	for _, cid := range []string{"cidOne", "cidTwo"} {
+		uids, err := c.UDLCidUsers(cid)
+		if err != nil {
+			t.Fatalf("cid users %s: %v", cid, err)
+		}
+		for _, u := range uids {
+			if u == "zAlice" {
+				t.Fatalf("%s still lists the deleted account in uids_for_cid", cid)
+			}
+		}
+	}
+}
+
+// Purging one account must not touch another's rows, including on a cid they
+// both have an opinion about — where the two share an index set and a careless
+// DEL of the whole key would take the survivor's membership with it.
+func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "shared", "rating")
+	seedCell(t, c, "zBob", "shared", "rating")
+
+	if _, err := c.UDLPurgeUser("zAlice"); err != nil {
+		t.Fatalf("purge: %v", err)
+	}
+
+	if _, ok, err := c.UDLGetCell("zBob", "shared", "rating"); err != nil || !ok {
+		t.Fatalf("bob's cell must survive alice's purge: ok=%v err=%v", ok, err)
+	}
+	uids, err := c.UDLCidUsers("shared")
+	if err != nil {
+		t.Fatalf("cid users: %v", err)
+	}
+	if len(uids) != 1 || uids[0] != "zBob" {
+		t.Fatalf("uids_for_cid = %v, want exactly [zBob]", uids)
+	}
+	entries, err := c.UDLListUserKey("zBob", "rating")
+	if err != nil || len(entries) != 1 {
+		t.Fatalf("bob's user+key index must survive: %d entries, err=%v", len(entries), err)
+	}
+}
+
+// A uid that is a prefix of another must not drag its neighbour's rows out.
+// The key layout separates them with '/' and ':', so this passes only as long
+// as the scan patterns keep those separators.
+func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+	seedCell(t, c, "zAliceExtra", "cidOne", "like")
+
+	stats, err := c.UDLPurgeUser("zAlice")
+	if err != nil {
+		t.Fatalf("purge: %v", err)
+	}
+	if stats.Records != 1 {
+		t.Fatalf("purged %d records, want exactly alice's 1", stats.Records)
+	}
+	if _, ok, err := c.UDLGetCell("zAliceExtra", "cidOne", "like"); err != nil || !ok {
+		t.Fatalf("zAliceExtra's cell must survive: ok=%v err=%v", ok, err)
+	}
+}
+
+// Tombstoned cells still belong to the account. They are absent from the
+// user+key index by design (see udl.go), so a purge that enumerated *that*
+// index instead of the cells would leave every retracted rating behind — the
+// rows most likely to have accumulated over an account's life.
+func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+	if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone: ok=%v err=%v", ok, err)
+	}
+	// Precondition: the tombstone left the cell but dropped it from the index.
+	if entries, _ := c.UDLListUserKey("zAlice", "like"); len(entries) != 0 {
+		t.Fatalf("precondition: tombstone should be out of the user+key index")
+	}
+
+	stats, err := c.UDLPurgeUser("zAlice")
+	if err != nil {
+		t.Fatalf("purge: %v", err)
+	}
+	if stats.Records != 1 {
+		t.Fatalf("purged %d records, want the 1 tombstoned cell", stats.Records)
+	}
+	if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); ok {
+		t.Fatal("tombstoned cell survived the purge")
+	}
+}
+
+// Retry-safe: the delete handler purges before removing the key, so a caller
+// that retries after a partial failure runs this a second time.
+func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
+	c, _ := newTestClient(t, "")
+	seedCell(t, c, "zAlice", "cidOne", "like")
+
+	if _, err := c.UDLPurgeUser("zAlice"); err != nil {
+		t.Fatalf("first purge: %v", err)
+	}
+	stats, err := c.UDLPurgeUser("zAlice")
+	if err != nil {
+		t.Fatalf("second purge must be a no-op, got: %v", err)
+	}
+	if stats.Records != 0 || stats.Cids != 0 {
+		t.Fatalf("second purge = %+v, want zeroes", stats)
+	}
+}
+
+func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
+	c, _ := newTestClient(t, "")
+	seedCell(t, c, "zAlice", "cidOne", "like")
+
+	// "*" would otherwise match every account in the keyspace — a wildcard
+	// purge dressed up as a uid.
+	for _, uid := range []string{"*", "z*", "", "z:Alice", "z/Alice"} {
+		if _, err := c.UDLPurgeUser(uid); err == nil {
+			t.Errorf("uid %q must be refused before it reaches a SCAN pattern", uid)
+		}
+	}
+	if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); !ok {
+		t.Fatal("a refused purge must not have deleted anything")
+	}
+}
+
+func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+	seedCell(t, c, "zAlice", "cidOne", "rating")
+	seedCell(t, c, "zAlice", "cidTwo", "like")
+	seedCell(t, c, "zBob", "cidOne", "seek")
+
+	stats, err := c.UDLAllUserStats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	if got := stats["zAlice"]; got.Records != 3 || got.Cids != 2 || got.Keys != 2 {
+		t.Fatalf("alice = %+v, want 3 records / 2 cids / 2 keys", got)
+	}
+	if got := stats["zBob"]; got.Records != 1 || got.Cids != 1 || got.Keys != 1 {
+		t.Fatalf("bob = %+v, want 1/1/1", got)
+	}
+	// An account with nothing stored is simply absent — the caller renders 0.
+	if _, present := stats["zNobody"]; present {
+		t.Fatal("an account with no records must not appear in the map")
+	}
+}
+
+// The stats and the purge must agree: the number shown in the delete
+// confirmation is the number of rows the delete then removes.
+func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+	seedCell(t, c, "zAlice", "cidTwo", "rating")
+	seedCell(t, c, "zAlice", "cidTwo", "seek")
+
+	before, err := c.UDLAllUserStats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	purged, err := c.UDLPurgeUser("zAlice")
+	if err != nil {
+		t.Fatalf("purge: %v", err)
+	}
+	if before["zAlice"] != purged {
+		t.Fatalf("stats said %+v but purge removed %+v", before["zAlice"], purged)
+	}
+}
+
+// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or
+// every account on a prefixed node would be reported under a mangled name and
+// the purge would find nothing.
+func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) {
+	c, _ := newTestClient(t, "tenantA:")
+
+	seedCell(t, c, "zAlice", "cidOne", "like")
+
+	stats, err := c.UDLAllUserStats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	if got := stats["zAlice"]; got.Records != 1 {
+		t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats)
+	}
+	if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 {
+		t.Fatalf("purge under a prefix: %+v err=%v", s, err)
+	}
+}
+
+func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
+	// A namespaced key arriving later must not be truncated: the purge SREMs
+	// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong
+	// member and leaves the real one behind.
+	uid, cid, key, ok := splitUDLRecSuffix("zAlice/cidOne/ns/sub:key")
+	if !ok || uid != "zAlice" || cid != "cidOne" || key != "ns/sub:key" {
+		t.Fatalf("got (%q,%q,%q,%v)", uid, cid, key, ok)
+	}
+	for _, bad := range []string{"", "zAlice", "zAlice/cidOne", "zAlice//key", "/cid/key"} {
+		if _, _, _, ok := splitUDLRecSuffix(bad); ok {
+			t.Errorf("%q should not parse", bad)
+		}
+	}
+}

internal/storage/udl_export_test.go

@@ -0,0 +1,234 @@
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+// The whole-profile export is what cross-instance recovery reads: a box that
+// has just imported a secret key knows the uid and nothing else — not which
+// cids the profile touched, not which keys. Every one of these tests is about
+// that one caller.
+
+func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const uid = "zAlice"
+	writes := []struct{ cid, key, rec string }{
+		{"bagcsaaaaone", "like", "rec-one-like"},
+		{"bagcsaaaaone", "seek", "rec-one-seek"},
+		{"bagcsaaaatwo", "rating", "rec-two-rating"},
+		{"bagcsaaaatwo", "profile:name", "rec-two-name"},
+		{"bagcsaaathree", "watched", "rec-three-watched"},
+	}
+	for _, w := range writes {
+		if ok, err := c.UDLUpsertIfNewer(uid, w.cid, w.key, 1, 100, w.rec, "", false, false); err != nil || !ok {
+			t.Fatalf("upsert %s/%s: ok=%v err=%v", w.cid, w.key, ok, err)
+		}
+	}
+
+	entries, next, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if next != "" {
+		t.Fatalf("export of 5 rows should be complete, got nextCid=%q", next)
+	}
+	if len(entries) != len(writes) {
+		t.Fatalf("want %d rows, got %d: %+v", len(writes), len(entries), entries)
+	}
+	seen := map[string]string{}
+	for _, e := range entries {
+		seen[e.Cid+"/"+e.Key] = e.Record
+	}
+	for _, w := range writes {
+		if got := seen[w.cid+"/"+w.key]; got != w.rec {
+			t.Errorf("%s/%s: want record %q, got %q", w.cid, w.key, w.rec, got)
+		}
+	}
+}
+
+// The failure this guards is silent and total: a uid whose cids were never
+// indexed exports as empty, and recovery reports success having restored
+// nothing.
+func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
+		t.Fatalf("upsert: ok=%v err=%v", ok, err)
+	}
+	entries, _, err := c.UDLAllForUser("zBob", "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("Bob owns nothing; got %+v", entries)
+	}
+}
+
+// ⚠ The tombstone rule, and the reason the cids index is never pruned.
+//
+// A tombstone must EXPORT. It is the only thing that can beat the older live
+// value the other box still holds at a lower version — drop it from the export
+// and an un-liked title comes back from the dead on every peer that already
+// saw the `like`. This is the one place the cids index deliberately diverges
+// from the user+key index, which does prune.
+func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaaaone"
+		key = "like"
+	)
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-liked", "", false, false); err != nil || !ok {
+		t.Fatalf("live upsert: ok=%v err=%v", ok, err)
+	}
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-unliked", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
+	}
+
+	// The user+key index prunes it — that is what keeps My List short.
+	live, err := c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key: %v", err)
+	}
+	if len(live) != 0 {
+		t.Fatalf("tombstoned cid should leave the user+key index, got %+v", live)
+	}
+
+	// The export must still carry it, at the tombstone's version.
+	entries, _, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("tombstone must export, got %+v", entries)
+	}
+	if entries[0].Record != "rec-unliked" || entries[0].Version != 2 {
+		t.Fatalf("export must carry the tombstone, not the superseded value: %+v", entries[0])
+	}
+}
+
+func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const uid = "zAlice"
+	cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
+	for _, cid := range cids {
+		for _, key := range []string{"like", "seek"} {
+			if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-"+cid+"-"+key, "", false, false); err != nil || !ok {
+				t.Fatalf("upsert: ok=%v err=%v", ok, err)
+			}
+		}
+	}
+
+	// A cid's keys are never split across a page, so a limit of 1 still yields
+	// both of the first cid's rows.
+	page1, next1, err := c.UDLAllForUser(uid, "", 1)
+	if err != nil {
+		t.Fatalf("page 1: %v", err)
+	}
+	if len(page1) != 2 || page1[0].Cid != cids[0] {
+		t.Fatalf("page 1 should hold both rows of %s, got %+v", cids[0], page1)
+	}
+	if next1 != cids[0] {
+		t.Fatalf("cursor should name the last cid emitted, want %q got %q", cids[0], next1)
+	}
+
+	// Walking the cursor must visit every row exactly once.
+	all := append([]UDLUserAllEntry{}, page1...)
+	cursor := next1
+	for cursor != "" {
+		page, next, err := c.UDLAllForUser(uid, cursor, 1)
+		if err != nil {
+			t.Fatalf("page after %q: %v", cursor, err)
+		}
+		all = append(all, page...)
+		cursor = next
+	}
+	if len(all) != 6 {
+		t.Fatalf("want 6 rows across all pages, got %d: %+v", len(all), all)
+	}
+	seen := map[string]bool{}
+	for _, e := range all {
+		k := e.Cid + "/" + e.Key
+		if seen[k] {
+			t.Fatalf("%s emitted twice across pages", k)
+		}
+		seen[k] = true
+	}
+}
+
+// Data written before the cids index existed must still export, or every
+// profile that predates this feature silently recovers as empty.
+func TestUDLBackfillUserCids_MakesLegacyDataExportable(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaaaone"
+		key = "like"
+	)
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
+		t.Fatalf("upsert: ok=%v err=%v", ok, err)
+	}
+
+	// Simulate the pre-index state: the cell and the other indexes exist, the
+	// cids set does not.
+	ctx := context.Background()
+	if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
+		t.Fatalf("del cids index: %v", err)
+	}
+	if entries, _, err := c.UDLAllForUser(uid, "", 0); err != nil || len(entries) != 0 {
+		t.Fatalf("precondition: legacy data should export as empty, got %+v err=%v", entries, err)
+	}
+
+	added, err := c.UDLBackfillUserCids()
+	if err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	if added != 1 {
+		t.Fatalf("want 1 cid backfilled, got %d", added)
+	}
+	entries, _, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export after backfill: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid || entries[0].Key != key {
+		t.Fatalf("backfilled data should export, got %+v", entries)
+	}
+
+	// Idempotent: a second run adds nothing.
+	if added, err := c.UDLBackfillUserCids(); err != nil || added != 0 {
+		t.Fatalf("second backfill should be a no-op, added=%d err=%v", added, err)
+	}
+}
+
+// The backfill splits `<uid>:cid:<cid>` on the FIRST separator, because a uid
+// is multibase base58btc and its alphabet has no colon. Splitting on the last
+// occurrence — the obvious-looking choice — mis-parses a cid that contains
+// ":cid:" in its own name into uid "zAlice:cid:weird", cid "name".
+func TestUDLBackfillUserCids_SplitsOnTheFirstSeparator(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const uid = "zAlice"
+	cid := "weird:cid:name"
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, "like", 1, 100, "rec", "", false, false); err != nil || !ok {
+		t.Fatalf("upsert: ok=%v err=%v", ok, err)
+	}
+	ctx := context.Background()
+	if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
+		t.Fatalf("del cids index: %v", err)
+	}
+	if _, err := c.UDLBackfillUserCids(); err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	entries, _, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("want the cid parsed whole, got %+v", entries)
+	}
+}

internal/storage/udl_test.go

@@ -0,0 +1,79 @@
+package storage
+
+import (
+	"testing"
+)
+
+// A tombstone must leave the cell in place but drop the cid from the user+key
+// index.
+//
+// The cell has to stay: it *is* the CRDT state, and deleting it would let an
+// older write resurrect the value the tombstone retracted. The index entry has
+// to go: `active_for_user_key` returns every member of that set, so a set that
+// only ever grows means each My List / Continue Watching read carries every
+// title the user has ever un-liked or finished — with its full record blob —
+// forever. The reader filters them out again, so the cost is invisible until
+// the account is old.
+func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaalocator"
+		key = "seek"
+	)
+
+	// A live position: indexed, readable.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-v1", "", false, false); err != nil || !ok {
+		t.Fatalf("live upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err := c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("live position should be indexed, got %+v", entries)
+	}
+
+	// Finishing it tombstones the seek.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-v2-tombstone", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
+	}
+
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after tombstone: %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("tombstoned cid must leave the user+key index, got %+v", entries)
+	}
+
+	// ...but the cell itself survives, at the tombstone's version.
+	cell, exists, err := c.UDLGetCell(uid, cid, key)
+	if err != nil {
+		t.Fatalf("get cell: %v", err)
+	}
+	if !exists {
+		t.Fatal("tombstone must not delete the cell — an older write could then resurrect the value")
+	}
+	if cell.Version != 2 || cell.Record != "rec-v2-tombstone" {
+		t.Fatalf("cell should hold the tombstone record at v2, got %+v", cell)
+	}
+
+	// And the high-water mark still rejects the superseded write.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 300, "rec-v1-replay", "", false, false); err != nil || ok {
+		t.Fatalf("stale replay must be rejected: ok=%v err=%v", ok, err)
+	}
+
+	// Re-watching re-establishes the cell and puts it back in the index.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 3, 400, "rec-v3", "", false, false); err != nil || !ok {
+		t.Fatalf("re-establish upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after re-establish: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("re-established cid must return to the index, got %+v", entries)
+	}
+}

internal/watcher/dispatcher.go

@@ -50,8 +50,18 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
 	client := d.storageClient
 	d.mu.RUnlock()
 
+	if err := PublishEvent(client, event); err != nil {
+		log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
+	}
+}
+
+// PublishEvent appends one file event to the file:events stream. Exported so a
+// caller with no Dispatcher (POST /api/files/register on a box whose watcher is
+// off) emits exactly the entry the watcher would — meta-share's ipfs_seed
+// consumes this stream to chunk, seed and provide new files.
+func PublishEvent(client *storage.Client, event FileEvent) error {
 	if client == nil || !client.IsConnected() {
-		return
+		return fmt.Errorf("storage client not connected")
 	}
 
 	// Build stream fields
@@ -79,9 +89,7 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
 	// `EmitReset → ClearStream` path is still the only way the stream gets
 	// truncated to zero (signalled to consumers via the `reset` event).
 	_, err := client.XAdd(EventsStream, EventsStreamMaxLen, fields)
-	if err != nil {
+	return err
-		log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
-	}
 }
 
 // EmitReset clears the stream and emits a reset event

internal/watcher/watcher.go

@@ -4,7 +4,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -54,42 +53,8 @@ func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash
 		return
 	}
 
-	uuid, err := w.storage.GetByCID(midhash)
+	if _, _, err := w.storage.RegisterFileTuple(absPath, size, mtimeNano, midhash); err != nil {
-	if err != nil {
+		log.Printf("[Watcher] register %s: %v", absPath, err)
-		log.Printf("[Watcher] GetByCID(%s): %v", midhash, err)
-		return
-	}
-
-	if uuid == "" {
-		// New content. Mint sets filePath/sizeByte/mtimeNano; SetProperty
-		// then writes the midhash as a cids/<cid> key-set member — and the
-		// cid_resolution.go hook registers the reverse-index alias for us.
-		uuid, err = w.storage.Mint(absPath, size, mtimeNano)
-		if err != nil {
-			log.Printf("[Watcher] Mint(%s): %v", absPath, err)
-			return
-		}
-		if err := w.storage.SetProperty(uuid, storage.CIDsKeyPrefix+midhash, "true"); err != nil {
-			log.Printf("[Watcher] SetProperty cids/%s for %s: %v", midhash, uuid, err)
-		}
-		return
-	}
-
-	// Known content. If this is a new physical path for the same content,
-	// record it as a duplicate. Otherwise refresh size/mtime — same content,
-	// same path, but the file may have been rewritten with a fresh mtime.
-	existing, _ := w.storage.GetProperty(uuid, "filePath")
-	if existing != absPath {
-		if _, err := w.storage.AddDuplicatePath(uuid, absPath); err != nil {
-			log.Printf("[Watcher] AddDuplicatePath %s @ %s: %v", uuid, absPath, err)
-		}
-		return
-	}
-	if _, err := w.storage.MergeMetadataFlat(uuid, map[string]string{
-		"sizeByte":  strconv.FormatInt(size, 10),
-		"mtimeNano": strconv.FormatInt(mtimeNano, 10),
-	}); err != nil {
-		log.Printf("[Watcher] refresh tuple for %s: %v", uuid, err)
 	}
 }
 
@@ -422,10 +387,9 @@ func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
 		return 0, nil
 	}
 
-	// Scan the directory
+	// Scan the directory. The mount poller reports completed scans (only when
-	log.Printf("[Watcher] Scanning mount path: %s", mountPath)
+	// the file count changes), so nothing is logged per scan here.
 	count := w.ScanDirectory(mountPath)
-	log.Printf("[Watcher] Mount scan complete: %s (%d files)", mountPath, count)
 
 	return count, nil
 }

internal/webdav/handler.go

@@ -50,8 +50,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		path = "/"
 	}
 
-	log.Printf("[WebDAV] %s %s", r.Method, path)
-
 	switch r.Method {
 	case "GET", "HEAD":
 		h.handleRead(w, r, path)

test/config_test.go

@@ -77,15 +77,9 @@ func TestConfigPaths(t *testing.T) {
 		t.Errorf("Unexpected LockFilePath: %s", cfg.LockFilePath())
 	}
 
-	if cfg.InfoFilePath() != "/test/locks/kv-leader.info" {
-		t.Errorf("Unexpected InfoFilePath: %s", cfg.InfoFilePath())
-	}
 
 	if cfg.RedisDataDir() != "/test/db/redis" {
 		t.Errorf("Unexpected RedisDataDir: %s", cfg.RedisDataDir())
 	}
 
-	if cfg.ServicesDir() != "/test/services" {
-		t.Errorf("Unexpected ServicesDir: %s", cfg.ServicesDir())
-	}
 }

test/election_test.go

@@ -1,144 +0,0 @@
-package test
-
-import (
-	"encoding/json"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/metazla/meta-core/internal/leader"
-)
-
-func TestLeaderLockInfoJSON(t *testing.T) {
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		BaseURL:   "http://localhost:8180",
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	// Marshal to JSON
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	// Verify JSON structure matches TypeScript format
-	var parsed map[string]interface{}
-	if err := json.Unmarshal(data, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if parsed["host"] != "test-host" {
-		t.Errorf("Expected host 'test-host', got '%v'", parsed["host"])
-	}
-
-	if parsed["api"] != "redis://10.0.0.1:6379" {
-		t.Errorf("Expected api 'redis://10.0.0.1:6379', got '%v'", parsed["api"])
-	}
-
-	if parsed["http"] != "http://10.0.0.1:8180" {
-		t.Errorf("Expected http 'http://10.0.0.1:8180', got '%v'", parsed["http"])
-	}
-
-	if parsed["baseUrl"] != "http://localhost:8180" {
-		t.Errorf("Expected baseUrl 'http://localhost:8180', got '%v'", parsed["baseUrl"])
-	}
-
-	// Timestamp should be a number
-	if _, ok := parsed["timestamp"].(float64); !ok {
-		t.Errorf("Expected timestamp to be a number, got %T", parsed["timestamp"])
-	}
-
-	// PID should be a number
-	if _, ok := parsed["pid"].(float64); !ok {
-		t.Errorf("Expected pid to be a number, got %T", parsed["pid"])
-	}
-}
-
-func TestLeaderLockInfoJSONOmitEmpty(t *testing.T) {
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		BaseURL:   "", // Empty - should be omitted
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	data, err := json.Marshal(info)
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	// baseUrl should not be present when empty
-	var parsed map[string]interface{}
-	if err := json.Unmarshal(data, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if _, exists := parsed["baseUrl"]; exists {
-		t.Error("Expected baseUrl to be omitted when empty")
-	}
-}
-
-func TestLeaderInfoFileFormat(t *testing.T) {
-	// Create a temp directory
-	tmpDir, err := os.MkdirTemp("", "meta-core-test")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	// Write to file
-	infoPath := filepath.Join(tmpDir, "kv-leader.info")
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	if err := os.WriteFile(infoPath, data, 0644); err != nil {
-		t.Fatalf("Failed to write file: %v", err)
-	}
-
-	// Read and parse
-	readData, err := os.ReadFile(infoPath)
-	if err != nil {
-		t.Fatalf("Failed to read file: %v", err)
-	}
-
-	var parsed leader.LeaderLockInfo
-	if err := json.Unmarshal(readData, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if parsed.Host != info.Host {
-		t.Errorf("Host mismatch: expected '%s', got '%s'", info.Host, parsed.Host)
-	}
-
-	if parsed.API != info.API {
-		t.Errorf("API mismatch: expected '%s', got '%s'", info.API, parsed.API)
-	}
-
-	if parsed.HTTP != info.HTTP {
-		t.Errorf("HTTP mismatch: expected '%s', got '%s'", info.HTTP, parsed.HTTP)
-	}
-
-	if parsed.Timestamp != info.Timestamp {
-		t.Errorf("Timestamp mismatch: expected %d, got %d", info.Timestamp, parsed.Timestamp)
-	}
-
-	if parsed.PID != info.PID {
-		t.Errorf("PID mismatch: expected %d, got %d", info.PID, parsed.PID)
-	}
-}