CI: push to ipcr-push-85-17-246-67.ip.nsl.sh (the box's cert no longer covers nip.io)

.gitea/workflows/docker-publish.yml

@@ -0,0 +1,129 @@
+# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
+# folder exists, and GitHub never reads it.
+#
+# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
+#   push  ipcr-push-85-17-246-67.ip.nsl.sh/metadec.eth/<repo>:<tag>   docker login as metadec.eth with the
+#         IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
+#   pull  ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag>   anonymous
+# The push goes to ipcr-push-85-17-246-67.ip.nsl.sh, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
+# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
+# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
+# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
+# certificate without that CA, so the login is written straight into ~/.docker/config.json.
+# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
+# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
+
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag for the image (optional, defaults to sha)'
+        required: false
+        type: string
+
+env:
+  REGISTRY: ipcr-push-85-17-246-67.ip.nsl.sh
+  IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write    # the release step below
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Docker CLI
+        run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
+
+      - name: Trust the Mesh Router CA
+        run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
+        env:
+          IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          buildkitd-config-inline: |
+            [registry."${{ env.REGISTRY }}"]
+              ca = ["/tmp/ipcr-ca.pem"]
+
+      - name: Log in to IPCR
+        env:
+          IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
+        run: |
+          mkdir -p ~/.docker
+          printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            # A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
+            # the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
+            type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
+
+      - name: Build and push Docker image
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # Gitea's Packages tab only lists images in Gitea's own registry, so a v* tag gets a
+      # release instead, naming the image it shipped. The digest line is the one to trust:
+      # ipcr 1.7.1 can drop some of the tags pushed together (see the tag list).
+      - name: Publish the Gitea release
+        if: startsWith(github.ref, 'refs/tags/v')
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          TAG: ${{ github.ref_name }}
+          SHA: ${{ github.sha }}
+          DIGEST: ${{ steps.build.outputs.digest }}
+          TAGS: ${{ steps.meta.outputs.tags }}
+          PULL: ipcr-watch.nsl.sh/ipns/metadec.eth/${{ env.IMAGE_NAME }}
+        run: |
+          node -e '
+            const e = process.env, ver = e.TAG.replace(/^v/, "");
+            const tags = e.TAGS.split("\n").filter(Boolean).map(t => "`" + t.split(":").pop() + "`");
+            const body = [
+              "**Image** (anonymous pull):",
+              "```", `docker pull ${e.PULL}:${ver}`, "```",
+              "Pinned by digest:",
+              "```", `${e.PULL}@${e.DIGEST}`, "```",
+              `Tags pushed: ${tags.join(", ")} · commit ${e.SHA.slice(0, 7)}`,
+            ].join("\n");
+            console.log(JSON.stringify({ tag_name: e.TAG, name: e.TAG, body }));
+          ' > /tmp/release.json
+          auth="Authorization: token $TOKEN"
+          code=$(curl -s -o /tmp/existing.json -w '%{http_code}' -H "$auth" "$API/releases/tags/$TAG")
+          if [ "$code" = 200 ]; then
+            id=$(node -p 'require("/tmp/existing.json").id')
+            method=PATCH url="$API/releases/$id"
+          else
+            method=POST url="$API/releases"
+          fi
+          code=$(curl -s -o /tmp/out.json -w '%{http_code}' -X "$method" -H "$auth" \
+            -H 'Content-Type: application/json' --data @/tmp/release.json "$url")
+          case "$code" in 2*) echo "release $TAG: $method $code" ;; *) cat /tmp/out.json; exit 1 ;; esac

.github/workflows/docker-publish.yml

@@ -45,6 +45,9 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=semver,pattern={{version}}
+            # A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
+            # the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
+            type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
             type=semver,pattern={{major}}.{{minor}}
             type=sha,prefix=
             type=raw,value=latest,enable={{is_default_branch}}

Dockerfile

@@ -6,20 +6,21 @@
 # - Bash-based leader election script (flock on shared volume)
 # - meta-core Go binary (HTTP API, Mount API, Watcher)
 # - nginx (dashboard, WebDAV, API proxy)
-# - Redis (managed by supervisord on leader)
+# - SQLite metadata store, embedded in the Go binary: truth /meta-core/db/meta-core.db
+#   (signed claims, backed up) + cache /meta-core/db-cache/cache.db (rebuildable)
 # - rclone (single mount executor — SMB goes through rclone's :smb: backend)
 # - supervisord (process management - runs on leader only)
 #
 # Architecture:
 # - leader-election.sh handles flock, starts supervisord on leader
 # - Followers remain dormant, retry flock every 5s
-# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
+# - Only leader runs meta-core, nginx, rclone, mount-watcher
 # =============================================================================
 
 # -----------------------------------------------------------------------------
 # Stage 1: Build Go binary
 # -----------------------------------------------------------------------------
-FROM golang:1.21-alpine AS go-builder
+FROM golang:1.26-alpine AS go-builder
 
 RUN apk add --no-cache git
 
@@ -74,7 +75,7 @@ ARG RCLONE_VERSION=1.73.5
 # Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
 # all remote mounts go through rclone now.
 RUN apk add --no-cache \
-    redis \
+    sqlite \
     nginx \
     nginx-mod-http-dav-ext \
     supervisor \
@@ -104,7 +105,8 @@ RUN chmod +x /usr/local/bin/meta-core
 # Create directories
 RUN mkdir -p \
     /meta-core/locks \
-    /meta-core/db/redis \
+    /meta-core/db \
+    /meta-core/db-cache \
     /meta-core/services \
     /meta-core/mounts/errors \
     /files \
@@ -140,13 +142,12 @@ ENV META_CORE_PATH=/meta-core \
     SERVICE_VERSION=1.0.0 \
     META_CORE_HTTP_PORT=9000 \
     META_CORE_HTTP_HOST=0.0.0.0 \
-    REDIS_PORT=6379 \
     WATCH_INTERVAL_MS=1000
 
 # Expose ports
-# 80: nginx (dashboard + API proxy + WebDAV)
+# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port:
-# 6379: Redis (for other services)
+# it is a file inside this container, reachable only through the HTTP API.
-EXPOSE 80 6379
+EXPOSE 80
 
 # Health check
 HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \

Makefile

@@ -2,7 +2,7 @@
 
 # Variables
 BINARY := meta-core
-VERSION ?= 1.0.0
+VERSION ?= 2.0.0
 BUILD_DIR := bin
 DOCKER_IMAGE := meta-core
 

README.md

@@ -1,39 +1,51 @@
 # meta-core
 
-Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
+Standalone Go service that owns the metadata store (an embedded SQLite file),
-metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
+the HTTP+SSE metadata surface, the
+User Data Layer store and signing keystore, and a WebDAV file server for the
+MetaMesh dev/prod stack.
 
 ## Overview
 
 meta-core runs as its own container (`metacore-app`). Other MetaMesh services
-(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
+(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
-reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
+meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
-is not exposed outside the container — all metadata I/O is mediated by the
+its `metamesh.core` advertise carries the full `/urls` payload) and then call its
-HTTP API.
+HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded
+in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API
+(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
 
 ### Key capabilities
 
 | Feature | Description |
 |---------|-------------|
-| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
+| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. |
-| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
+| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). |
-| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
+| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
-| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
+| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field_v` over (field name, value) of every record key, plus an FTS5 trigram table; the smallest part drives, the other filters are probed per candidate — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). |
+| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. |
 | **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
-| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
+| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
-| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
+| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
 | **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
 | **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
+| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
+| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
+| **Report inbox** | `/api/reports` stores the bug reports / feedback the client apps' 🐛 buttons file (forwarded server-to-server by meta-watch and meta-read), in its own bounded `db/reports.db`; the dashboard's **Reports** page lists, reads and triages them. See [architecture: Report inbox](docs/meta-core-architecture.md#report-inbox). |
+| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
 
 ### Design characteristics
 
-- **Single-leader** — flock on a shared volume; no external consensus.
+- **Single writer** — flock on the `/meta-core` volume; no external consensus.
-- **HTTP-only externally** — Redis is bound to the container, never published.
+- **HTTP-only externally** — there is no database port; debug read-only with
+  `docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …`
+  (dev stacks: `/meta-core-db/meta-core.db`).
 - **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
-  on-disk Redis layout predates the current build (alpha clean-wipe policy;
+  on-disk store layout predates the current build (alpha clean-wipe policy;
   see `docs/uuid-rooted-metadata.md`).
-- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
+- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a
-  image with `redis-server`, `rclone`, nginx, and supervisord.
+  thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI
+  (for debugging).
 
 ## Architecture
 
@@ -41,34 +53,31 @@ HTTP API.
 ┌──────────────────────────────────────────────────────────────────────────┐
 │                    metacore-app  (standalone container)                  │
 │                                                                          │
+│  leader-election.sh  (flock /meta-core/locks/kv-leader.lock,             │
+│                       then exec supervisord)                             │
 │  supervisord                                                             │
-│   ├─ leader-election.sh  (flock /meta-core/locks/kv-leader.lock,         │
+│   ├─ nginx  :80          (dashboard, /editor, API proxy,                 │
-│   │                       writes kv-leader.info, exec supervisord)      │
+│   │                       proxy_cache for /webdav, rclone GUI /rclone)   │
-│   ├─ redis-server        (AOF + RDB on /meta-core/db/redis)              │
+│   ├─ rclone rcd          (mount daemon, RC API on :5572)                 │
-│   ├─ nginx               (TLS termination, proxy_cache for /webdav)      │
-│   ├─ rclone rcd          (mount daemon, RC API)                          │
 │   ├─ mount-watcher.sh    (rclone mount lifecycle)                        │
 │   └─ meta-core (Go)      ─►  HTTP API + SSE + WebDAV on :9000            │
+│                          ─►  embedded SQLite (db/meta-core.db)           │
+│                          ─►  UDP advertise/listen 239.255.99.1:9099      │
 └──────────────────────────────────────────────────────────────────────────┘
                                        │
                 ┌──────────────────────┴──────────────────────┐
-                │  /meta-core (shared volume)                 │
+                │  /meta-core (meta-core's own volume)        │
-                │   locks/kv-leader.{lock,info}               │
+                │   locks/kv-leader.lock                      │
-                │   db/redis/                                 │
+                │   db/meta-core.db (+ -wal, -shm)            │
-                │   services/*.json                           │
+                │   identity/accounts/*.json                  │
                 │   mounts/{mounts.json, errors/}             │
                 │   watchers.json                             │
                 │   cache/ (nginx proxy_cache)                │
                 └─────────────────────────────────────────────┘
-                                       ▲
+
-        ┌──────────────────────────────┼────────────────────────────────┐
+   meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
-        │                              │                                │
+      │  1. hear the metamesh.core advertise on UDP multicast (carries /urls)
-┌─────────────┐               ┌─────────────┐                ┌─────────────┐
+      └─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume
-│  meta-sort  │               │  meta-fuse  │                │ meta-stremio│
-│             │               │             │                │             │
-│  reads      │  HTTP+SSE     │  reads      │  HTTP+SSE      │  reads      │
-│  leader.info├──────────────►│  leader.info├───────────────►│  leader.info│
-└─────────────┘               └─────────────┘                └─────────────┘
 ```
 
 ## Quick start
@@ -82,8 +91,9 @@ make test            # run Go unit tests
 ```
 
 `make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
-`:local` tag is the one every sub-stack docker-compose.yml references
+`:local` tag is the one the sub-stack docker-compose files reference
-(meta-share 10-peer federation, meta-gateway dev stacks); without it the
+(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
+dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
 sub-stacks can't see a freshly-built image by name.
 
 > **Source-change → recreate.** `make docker` rebuilds and re-tags the
@@ -94,12 +104,14 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > make docker
-> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
+> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
->   up -d --force-recreate metagateway-generic-core
+>   up -d --force-recreate metagateway-hub-metacore
-> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
->   up -d --force-recreate metagateway-torznab-core
 > docker compose -f packages/meta-share/dev/docker-compose.yml \
 >   --profile share-10node up -d --force-recreate metashare-core metashare-core-test
+> docker compose -f packages/meta-read/dev/docker-compose.yml \
+>   up -d --force-recreate metaread-dev-core
+> docker compose -f packages/meta-listen/dev/docker-compose.yml \
+>   up -d --force-recreate metalisten-dev-core
 > ```
 >
 > The symptom of a missed recreate is missing API fields on `/urls`
@@ -109,7 +121,7 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > docker inspect meta-core:local --format '{{.Id}}'                # fresh SHA
-> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
+> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'   # repeat for :latest
 > # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
 > ```
 
@@ -126,36 +138,45 @@ HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
 
 ## Configuration
 
-Environment variables consumed by `internal/config/config.go`:
+Environment variables (from `internal/config/config.go` unless noted):
 
 | Variable | Default | Description |
 |---|---|---|
-| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
+| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. |
 | `FILES_PATH` | `/files` | Files volume root. |
-| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
+| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
-| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
+| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
-| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
+| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
-| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
+| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
-| `REDIS_PORT` | `6379` | Local Redis port. |
+| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
+| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). |
+| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). |
+| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
+| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
+| `META_CORE_REPORTS_DB_PATH` | `<META_CORE_PATH>/db/reports.db` | Report inbox database. |
+| `META_CORE_REPORTS_MAX` / `META_CORE_REPORTS_MAX_MB` | `5000` / `256` | Inbox bounds; closed reports are evicted first, then the oldest. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
 | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
-| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
+| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
-| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
+| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
-| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
+| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
-| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
+| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
-| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
+| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
-| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
-| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
-| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
 
-Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
+`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
-no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
+`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
-identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
+`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
-retry interval (`ELECTION_RETRY_SECS`, default `5`).
+them (leftovers of the file-based registry); setting them has no effect. There
+is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES`
+(gone with Redis and the in-memory search index; search results now include
+`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` /
+`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
+configs created via `/api/watchers`.
 
 ## API reference
 
-All endpoints below are served by `internal/api/server.go`. Examples assume
+All endpoints below are registered in `internal/api/server.go` (mounts,
+watcher and watchers routes in their own packages' `handlers.go`). Examples assume
 the in-container Go port `:9000`; from the host use
 `https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
 (debug-direct).
@@ -163,16 +184,47 @@ the in-container Go port `:9000`; from the host use
 ### Health, leader, URLs
 
 ```bash
-GET  /health           # storage + role
+GET  /health           # storage + role (`redis: bool` kept for compatibility = store open)
 GET  /status           # version, uptime, etc.
 GET  /leader           # current leader info (hostname, PID, timestamps)
-GET  /urls             # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
+GET  /urls             # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility)
-GET  /services         # registered services + heartbeats
+GET  /neighbors        # neighbours heard over UDP (beacon v2)
-GET  /api/services     # alias for /services
+GET  /api/neighbors    # same, under /api
-GET  /services/{name}  # one service
+GET  /api/stats        # dashboard counters (see below)
-GET  /services/cleanup/stats
 ```
 
+`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
+are **gone** — the registration-and-heartbeat registry they served was replaced
+by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
+with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
+than a 404; that is what made the dashboard fail with
+`SyntaxError: Unexpected token '<'` until 1.0.21.
+
+#### `GET /api/stats`
+
+Every headline counter in one request, split by cost:
+
+```jsonc
+{
+  "records": 88709,        // count of idx:roots/* — metadata roots, file or not
+  "redisKeys": 3115447,    // kv row count (name kept for compatibility; cached 60 s)
+  "redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility)
+  "identities": 3,         // signing accounts on disk
+  "files": { "count": 4864, "totalSize": 64587914424 },  // null until first sweep
+  "udlUsers": 5,           // accounts holding User Data Layer records
+  "sweptAt": 1789402401970,
+  "sweeping": false
+}
+```
+
+`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and
+`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
+box), so they are served from a single-flight cache refreshed in the background
+with a 60s TTL — the walk is **never** on the request path, because the
+dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this
+over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
+across records that share a file.
+
 ### Metadata — primary surface
 
 These are the routes other services use. `{hash}` may be a UUID root or
@@ -198,6 +250,8 @@ GET    /api/meta/{cid}                # resolve CID → metadata document
 GET    /api/file/{cid}/info           # CID resolution metadata
 GET    /file/{cid}                    # serve file bytes (range-aware)
 POST   /file/cid                      # compute CID for an uploaded file
+POST   /api/files/register        # mint + alias one file under /files/plugin by midhash
+                                  # (the watcher's path for one file; works with the watcher OFF)
 HEAD   /data/{hash}                   # existence + size
 GET    /data/{hash}/path              # absolute path on /files
 ```
@@ -222,10 +276,10 @@ PUT    /api/metadata/{hashId}/property
 
 GET    /api/kv/info
 GET    /api/kv/keys
-GET    /api/kv/tree
+GET    /api/kv/tree            # cursor is an opaque next-key ("0" when exhausted)
 GET    /api/kv/search
 GET    /api/kv/find
-GET    /api/kv/value
+GET    /api/kv/value           # type is always "string" for an existing key
 PUT    /api/kv/value
 DELETE /api/kv/value
 GET    /api/kv/key/{key...}
@@ -240,7 +294,8 @@ POST   /api/snapshot/wipe
 
 ### SSE event streams
 
-The HTTP-mirror of the Redis Streams. External services never read Redis
+The HTTP-mirror of the store's event streams (Redis Streams until 2026-10;
+IDs keep the `<ms>-<seq>` form). External services never read the store
 directly — they subscribe here with `Last-Event-ID` for resume. See
 [`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
 contract (one event per stream entry, opaque IDs, heartbeats, gap
@@ -249,6 +304,54 @@ signalling).
 ```bash
 GET /api/events/files     # file:events stream (watcher events)
 GET /api/events/meta      # meta:events stream (metadata mutations)
+GET /api/events/poll      # DEPRECATED long-poll over file:events — use SSE
+```
+
+### Identity (signing keystore)
+
+Reached through meta-search's identity proxy (and directly by meta-gateway,
+meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
+
+```bash
+GET    /api/identity               # status (never the private key)
+GET    /api/identity/accounts
+POST   /api/identity/generate      # the only call that returns a new private key
+POST   /api/identity/import
+POST   /api/identity/challenge     # one-shot challenge for proof-of-possession
+POST   /api/identity/reveal        # needs proof-of-possession
+DELETE /api/identity               # needs proof-of-possession; reports purged UDL counts
+POST   /api/identity/sign
+POST   /api/identity/sign-batch
+GET    /api/identity/aead-key
+```
+
+### User Data Layer
+
+Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
+writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
+cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec:<uid>/<cid>/<key>` → JSON `{record,version,ts,value?}`; index sets as one key per member).
+
+```bash
+GET  /api/udl/record?uid&cid&key
+PUT  /api/udl/record
+PUT  /api/udl/records                      # batch
+GET  /api/udl/user/{uid}/key/{key}
+GET  /api/udl/user/{uid}/cid/{cid}
+GET  /api/udl/user/{uid}/all
+GET  /api/udl/user/{uid}/count
+GET  /api/udl/cid/{cid}/users
+GET  /api/udl/cid/{cid}/aggregate          # meta-watch Aggregate wire shape
+GET  /api/udl/users/stats
+```
+
+### Reports (bug-report / feedback inbox)
+
+```
+POST   /api/reports                 # {app, note?, page?, route?, cid?, title?, uid?, build?, filedAt?, log?, …blocks} → 201 {id}
+GET    /api/reports                 # ?app=&status=open|triaged|closed&q=&before=<id>&limit= → {reports, apps, counts, more}
+GET    /api/reports/{id}            # full report incl. body
+PATCH  /api/reports/{id}            # {status?, triage_note?}
+DELETE /api/reports/{id}
 ```
 
 ### Mounts
@@ -284,46 +387,43 @@ The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
 return a deprecation pointer at the `/api/watchers/*` routes
 (`internal/watcher/handlers.go:32-33`).
 
-### Admin / WebDAV
+### WebDAV
 
 ```bash
-POST   /api/admin/migrate-dual-roots  # reunify stranded midhash-rooted entries
-POST   /api/admin/sweep-literature-echoes[?apply=true]
-       # strip query echoes off literature cards: anilistid on site cards,
-       # workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
-       # Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
 GET/PUT/DELETE /webdav/...            # mounted on /files
 ```
 
+The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`,
+`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were
+removed with the 2026-10 SQLite switch; they had been run once on
+watch.nsl.sh before the migration.
+
 No `/metrics` endpoint is exposed; observability is via container logs and
 the `/health` / `/status` JSON.
 
-## Leader gate
+## DB-file mutex
 
-Leader election is a bash flock loop in `docker/leader-election.sh`, not a
+`docker/leader-election.sh` is a bash flock loop, not a Go-level state
-Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
+machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
-script writes the leader API URL to `kv-leader.info` and `exec`s
+script `exec`s supervisord; a second container mounting the same volume
-supervisord. When the process tree dies, the kernel releases the lock and
+retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
-any standby instance can take over. There is no follower path inside the
+dies, the kernel releases the lock. There is no follower path inside the Go
-Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
+binary.
 
-`kv-leader.info` is a single plaintext line containing the API base URL:
+The lock is a **mutex, not service discovery** — it only stops two
-
+writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info`
-```
+file it used to publish is gone: siblings locate meta-core over UDP
-http://10.0.1.50:9000
+(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
-```
+`internal/leader` keeps its name for history, but only builds the `/urls`
-
+payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
-Other services read this file, then call `GET /urls` for full discovery.
+and the UDP announce share.
-
-The dev compose runs a single `metacore-app`; the flock primitive is
-retained as a correctness anchor (one process owns Redis and the
-writable state under `/meta-core`), not as live failover.
 
 ## Metadata storage shape (one-line summary)
 
-Roots are opaque UUIDv7 strings; every property is a separate Redis
+Roots are opaque UUIDv7 strings; every property is a separate row
-STRING at `file:<uuid>/<property>`; every known CID for a file is a
+of the `kv` table at `file:<uuid>/<property>`; every known CID is a key-set member
-reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
+`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
+(the canonical CID is derived by rank on read, never stored). Full details in
 [`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
 and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
 The authoritative registry of field semantics and value formats lives in
@@ -333,15 +433,17 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 
 | Package | Purpose |
 |---------|---------|
-| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
+| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. |
 | `internal/config` | Env-driven configuration, path helpers. |
-| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
+| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
-| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
+| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
-| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
+| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). |
-| `internal/discovery` | Service registry + dead-service cleaner. |
+| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field_v` + FTS (`search.go`), derived counters (`onChange`), UDL store (`udl.go`, `udl_admin.go`), stats. |
-| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
+| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
-| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
+| `internal/reports` | Report inbox: its own SQLite file, normalisation/bounds, list/filter/paging, prune (closed first). |
-| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
+| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
+| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. |
+| `internal/schema` | Per-field schema, folded from counters the write path keeps exact (`idx:schema/…`). |
 | `internal/snapshot` | Snapshot export / import / wipe. |
 | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
 | `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
@@ -353,13 +455,17 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 ```
 1. Load configuration from environment
 2. Build LeaderInfoProvider (no election — the bash gate already won)
-3. Connect to local Redis (retry up to 30× / 1s)
+3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
-4. EnsureSchemaVersion — abort if the existing Redis layout is stale
+4. EnsureSchemaVersion — abort if the existing store layout is stale
-5. Start service discovery + dead-service cleaner
+5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
-6. Construct API server (initialises watcher, mounts, WebDAV)
+6. Construct API server (identity keystore migration, watcher, watchers,
-7. Start API server; on first connect, start MetaPublisher + schema Indexer
+   mounts, WebDAV)
-   and republish existing metadata onto meta:events
+7. EnsureCounters — one-time build of the derived counters (record count,
-8. Wait for SIGINT/SIGTERM, shutdown in reverse order
+   per-field schema) on a store that predates them; blocking, ~1 min at 169k
+   records, then never again
+8. Start API server + pollers (no republish: consumers that want the whole
+   state ask /api/events/meta for `0-0` and get a snapshot)
+9. Wait for SIGINT/SIGTERM, shutdown in reverse order
 ```
 
 ## Dependencies
@@ -367,13 +473,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 | Dependency | Purpose |
 |------------|---------|
 | `github.com/gorilla/mux` | HTTP router |
-| `github.com/redis/go-redis/v9` | Redis client |
+| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) |
 | `github.com/google/uuid` | UUIDv7 minting |
+| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
 | `golang.org/x/net` | WebDAV support |
-| Go stdlib | context, net/http, os, sync, syscall |
 
-Runtime requirements: Go 1.21+ at build time; the container ships Redis,
+Runtime requirements: Go 1.21+ at build time; the container ships
-rclone, nginx, and supervisord alongside the binary.
+rclone, nginx, and supervisord alongside the binary, plus the React dashboard
+and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
 
 ## License
 

cmd/meta-core/main.go

@@ -1,22 +1,25 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"syscall"
 	"time"
 
 	"github.com/metazla/meta-core/internal/api"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
+	"github.com/metazla/meta-core/internal/reports"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
 // Version is set at build time
-var Version = "1.0.0"
+var Version = "2.0.0"
 
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
@@ -29,51 +32,106 @@ func main() {
 	// Create leader info provider (Go binary only runs as leader - followers loop in bash)
 	leaderProvider := leader.NewLeaderInfoProvider(cfg)
 
-	// Connect to local Redis (supervisord starts Redis before meta-core)
+	// Open the two embedded SQLite files: truth (signed claims, backed up)
-	// Retry connection since Redis may still be loading its dataset
+	// and the cache (everything derived, rebuildable). No database process.
-	redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
+	if err := os.MkdirAll(filepath.Dir(cfg.CachePath()), 0o755); err != nil {
-	log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
+		log.Fatalf("[meta-core] Failed to create the cache folder: %v", err)
-
-	storageClient := storage.NewClient("")
-	maxRetries := 30
-	retryDelay := 1 // seconds
-	var err error
-	for i := 0; i < maxRetries; i++ {
-		err = storageClient.Connect(redisURL)
-		if err == nil {
-			break
-		}
-		log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err)
-		time.Sleep(time.Duration(retryDelay) * time.Second)
 	}
-	if err != nil {
+	storageClient := storage.NewClient()
-		log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err)
+	if err := storageClient.Connect(cfg.DBPath(), cfg.CachePath()); err != nil {
+		log.Fatalf("[meta-core] Failed to open the metadata store: %v", err)
 	}
-	log.Println("[meta-core] Connected to Redis")
 
-	// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
+	// Schema-version gate: refuses a v1.x store (which the default truth path
-	// the existing Redis store predates the UUID-rooted schema, so legacy
+	// still points at) and any unknown layout. No migration code here — see
-	// data can't silently leak into the new code paths. See
+	// tools/claims-migrate.
-	// docs/uuid-rooted-metadata.md.
 	if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
 		log.Fatalf("[meta-core] %v", err)
 	}
-	log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
+	log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion)
 
-	// Create and start service discovery
+	// Bring the cache level with truth (replay after a crash, full rebuild
-	disc := discovery.NewService(cfg)
+	// when the cache folder is empty), then start the apply loop.
-	if err := disc.Start(); err != nil {
+	started := time.Now()
-		log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
+	if err := storageClient.Start(); err != nil {
+		log.Fatalf("[meta-core] %v", err)
+	}
+	log.Printf("[meta-core] Cache ready in %s", time.Since(started).Round(time.Millisecond))
+
+	// Derived counters (record count, per-field schema) are kept exact by the
+	// write path. A store that predates them gets a one-time build here,
+	// before anything else writes.
+	started = time.Now()
+	if built, err := storageClient.EnsureCounters(); err != nil {
+		log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err)
+	} else if built {
+		log.Printf("[meta-core] Built derived counters in %s", time.Since(started).Round(time.Millisecond))
 	}
 
-	// Create and start dead service cleaner
+	// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
-	cleaner := discovery.NewCleaner(cfg)
+	// as metamesh.core — carrying the /urls block services locate us by — plus
-	if err := cleaner.Start(); err != nil {
+	// its own metamesh.service/<name> nav entry. The payload is rebuilt from
-		log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
+	// leaderProvider on every tick, which keeps it and GET /urls from drifting.
+	var mesh *meshdisco.Node
+	if cfg.EnableUDPDiscovery {
+		hostname, _ := os.Hostname()
+		mesh = meshdisco.New(meshdisco.Config{
+			Name:     cfg.ServiceName,
+			Instance: hostname,
+			Version:  Version,
+			Group:    cfg.DiscoveryGroup,
+			Port:     cfg.DiscoveryPort,
+			Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
+			Payload: func() (string, []meshdisco.Resource) {
+				service := meshdisco.Resource{
+					ID:        "service",
+					Caps:      []string{meshdisco.CapServicePrefix + cfg.ServiceName},
+					Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
+				}
+				info := leaderProvider.LeaderInfo()
+				if info == nil {
+					return "starting", []meshdisco.Resource{service}
+				}
+				data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
+					Hostname:          info.Hostname,
+					BaseUrl:           info.BaseUrl,
+					ApiUrl:            info.ApiUrl,
+					WebdavUrl:         info.WebdavUrl,
+					WebdavUrlInternal: info.WebdavUrlInternal,
+				}})
+				core := meshdisco.Resource{
+					ID:        "core",
+					Caps:      []string{meshdisco.CapCore},
+					Endpoints: map[string]string{"http": info.ApiUrl},
+					Data:      data,
+				}
+				return "running", []meshdisco.Resource{core, service}
+			},
+		})
+		if err := mesh.Start(); err != nil {
+			log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
+			mesh = nil
+		}
+	} else {
+		log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
+	}
+
+	// Report inbox: its own SQLite file beside truth. Not load-bearing — a
+	// failure to open it leaves /api/reports answering 503, never the core down.
+	reportStore, err := reports.Open(cfg.ReportsDBPath(), cfg.ReportsMax, int64(cfg.ReportsMaxMB)<<20)
+	if err != nil {
+		log.Printf("[meta-core] Warning: report inbox disabled: %v", err)
+		reportStore = nil
+	} else {
+		log.Printf("[meta-core] Report inbox %s (max %d reports, %d MiB)", reportStore.Path(), cfg.ReportsMax, cfg.ReportsMaxMB)
 	}
 
 	// Create and start API server
-	apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
+	apiServer := api.NewServer(cfg, leaderProvider, storageClient)
+	apiServer.SetMeshNode(mesh)
+	if reportStore != nil {
+		apiServer.SetReports(reportStore)
+	}
 	if err := apiServer.Start(); err != nil {
 		log.Fatalf("[meta-core] Failed to start API server: %v", err)
 	}
@@ -90,12 +148,16 @@ func main() {
 		log.Printf("[meta-core] Error stopping API server: %v", err)
 	}
 
-	if err := cleaner.Stop(); err != nil {
+	if mesh != nil {
-		log.Printf("[meta-core] Error stopping service cleaner: %v", err)
+		if err := mesh.Stop(); err != nil {
+			log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
+		}
 	}
 
-	if err := disc.Stop(); err != nil {
+	if reportStore != nil {
-		log.Printf("[meta-core] Error stopping service discovery: %v", err)
+		if err := reportStore.Close(); err != nil {
+			log.Printf("[meta-core] Error closing report inbox: %v", err)
+		}
 	}
 
 	if err := storageClient.Close(); err != nil {
@@ -112,3 +174,17 @@ func waitForShutdown() {
 	<-sigChan
 }
 
+// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
+// the three-way fallback the file registry used (discovery/service.go):
+// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
+// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
+// the container IP.
+func coreBaseURL(cfg *config.Config) string {
+	if cfg.PublicURL != "" {
+		return cfg.PublicURL
+	}
+	if cfg.BaseURL != "" {
+		return cfg.BaseURL
+	}
+	return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
+}

dashboard/dist/index.html

@@ -21,7 +21,7 @@
         min-height: 100vh;
       }
     </style>
-    <script type="module" crossorigin src="/assets/index-BW9EQGCP.js"></script>
+    <script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
   </head>
   <body>

dashboard/src/App.tsx

@@ -4,6 +4,7 @@ import Mounts from './pages/Mounts';
 import Watchers from './pages/Watchers';
 import Files from './pages/Files';
 import Identity from './pages/Identity';
+import Reports from './pages/Reports';
 import ServiceNav from './components/ServiceNav';
 
 const navStyle: React.CSSProperties = {
@@ -81,6 +82,12 @@ export default function App() {
         >
           Identity
         </NavLink>
+        <NavLink
+          to="/reports"
+          style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
+        >
+          Reports
+        </NavLink>
         <NavLink
           to="/editor"
           style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
@@ -97,6 +104,7 @@ export default function App() {
           <Route path="/watchers" element={<Watchers />} />
           <Route path="/files" element={<Files />} />
           <Route path="/identity" element={<Identity />} />
+          <Route path="/reports" element={<Reports />} />
           <Route path="/editor" element={<EditorPage />} />
         </Routes>
       </div>

dashboard/src/components/ServiceNav.tsx

@@ -1,156 +1,54 @@
-import { useState, useEffect } from 'react';
+import { useEffect } from 'react';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
+// of defining the custom element.
+import '../meta-service-menu.js';
 
-interface ServiceInfo {
+/**
-  name: string;
+ * ServiceNav — the neighbouring meta-* services, as a burger menu.
-  hostname: string;
+ *
-  baseUrl: string;
+ * Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
-  status: string;
+ * a framework-free custom element shared by every meta-* UI (see
-  lastHeartbeat: string;
+ * scripts/check-mirrors.sh). The old implementation was ~165 lines of React
-}
+ * that existed in four near-identical copies and polled meta-core's
+ * /api/services; the element polls this service's own /api/neighbors instead,
+ * so the nav still renders when meta-core is down.
+ *
+ * The element lives in shadow DOM and cannot see this app's CSS variables, so
+ * the four --mm-nav-* tokens below map our palette onto it.
+ */
 
-interface ServicesResponse {
+declare global {
-  services: ServiceInfo[];
+    // eslint-disable-next-line @typescript-eslint/no-namespace
-  current: string;
+    namespace JSX {
-}
+        interface IntrinsicElements {
-
+            'meta-service-menu': React.DetailedHTMLProps<
-const serviceIcons: Record<string, string> = {
+                React.HTMLAttributes<HTMLElement>,
-  'meta-core': '🗄️',
+                HTMLElement
-  'meta-sort': '📁',
+            > & { current?: string; endpoint?: string; label?: string };
-  'meta-fuse': '🗂️',
+        }
-  'meta-stremio': '🎬',
+    }
-  'meta-orbit': '🌐',
-  'default': '📦'
-};
-
-function formatServiceName(name: string): string {
-  return name.split('-').map(word =>
-    word.charAt(0).toUpperCase() + word.slice(1)
-  ).join(' ');
 }
 
 function ServiceNav() {
-  const [services, setServices] = useState<ServiceInfo[]>([]);
+    useEffect(() => {
-  const [loading, setLoading] = useState(true);
+        // Import above is for the side effect of defining the element; this
-  const currentService = 'meta-core';
+        // keeps bundlers from tree-shaking it away in production builds.
+    }, []);
 
-  useEffect(() => {
+    return (
-    const fetchServices = async () => {
+        <>
-      try {
+            <meta-service-menu current="meta-core" />
-        const response = await fetch('/services');
+            <style>{`
-        if (response.ok) {
+                meta-service-menu {
-          const data: ServicesResponse = await response.json();
+                    --mm-nav-fg: var(--text-primary, #e0e0e0);
-          setServices(data.services || []);
+                    --mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
-        }
+                    --mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
-      } catch (error) {
+                    --mm-nav-accent: var(--accent-primary, #4ecdc4);
-        console.error('Failed to fetch services:', error);
-      } finally {
-        setLoading(false);
-      }
-    };
-
-    fetchServices();
-    const interval = setInterval(fetchServices, 30000);
-    return () => clearInterval(interval);
-  }, []);
-
-  if (loading) return null;
-
-  const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
-
-  if (sortedServices.length === 0) return null;
-
-  return (
-    <nav className="services-nav">
-      <span className="services-nav-label">Services:</span>
-      <div className="services-nav-items">
-        {sortedServices.map(service => {
-          const icon = serviceIcons[service.name] || serviceIcons.default;
-          const isActive = service.name === currentService;
-          return (
-            <a
-              key={service.name}
-              href={isActive ? '#' : service.baseUrl}
-              className={`service-link${isActive ? ' active' : ''}`}
-              onClick={(e) => {
-                e.preventDefault();
-                if (!isActive) {
-                  window.location.href = service.baseUrl;
                 }
-              }}
+            `}</style>
-            >
+        </>
-              <span className="service-icon">{icon}</span>
+    );
-              <span>{formatServiceName(service.name)}</span>
-              <span className="service-status"></span>
-            </a>
-          );
-        })}
-      </div>
-
-      <style>{`
-        .services-nav {
-          display: flex;
-          align-items: center;
-          gap: 0.5rem;
-        }
-
-        .services-nav-label {
-          color: #888;
-          font-size: 0.8rem;
-          white-space: nowrap;
-        }
-
-        .services-nav-items {
-          display: flex;
-          gap: 0.35rem;
-        }
-
-        .service-link {
-          display: inline-flex;
-          align-items: center;
-          gap: 0.3rem;
-          padding: 0.35rem 0.6rem;
-          border-radius: 6px;
-          text-decoration: none;
-          font-size: 0.8rem;
-          font-weight: 500;
-          transition: all 0.2s;
-          color: #ccc;
-          background: #0f3460;
-          border: 1px solid #1a4a7a;
-        }
-
-        .service-link:hover {
-          background: #1a4a7a;
-          border-color: rgba(78, 205, 196, 0.5);
-          text-decoration: none;
-          color: #fff;
-        }
-
-        .service-link.active {
-          background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
-          border-color: rgba(78, 205, 196, 0.5);
-          color: #fff;
-        }
-
-        .service-link .service-icon {
-          font-size: 0.9rem;
-        }
-
-        .service-link .service-status {
-          width: 5px;
-          height: 5px;
-          border-radius: 50%;
-          background: #4ecdc4;
-        }
-
-        @media (max-width: 800px) {
-          .services-nav-label {
-            display: none;
-          }
-        }
-      `}</style>
-    </nav>
-  );
 }
 
 export default ServiceNav;

dashboard/src/lib/api.ts

@@ -0,0 +1,28 @@
+// JSON fetch that fails with a sentence instead of a parser error.
+//
+// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
+// path it does not have a proxy rule for answers 200 text/html with the SPA
+// shell. A caller that does `res.ok && res.json()` on such a path gets
+// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
+// no clue which request produced it — which is exactly how a route deleted
+// from the Go router (`/services`, removed in favour of /api/neighbors) sat
+// broken on the dashboard.
+//
+// So: check the content type, and name the URL in every error.
+
+export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
+  const res = await fetch(url, init);
+  const ct = res.headers.get('content-type') || '';
+
+  if (!res.ok) {
+    throw new Error(`${url} → HTTP ${res.status}`);
+  }
+  if (!ct.includes('application/json')) {
+    // The SPA fallback is the overwhelmingly likely cause; say so.
+    throw new Error(
+      `${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
+        `missing and nginx served the dashboard shell instead)`
+    );
+  }
+  return (await res.json()) as T;
+}

dashboard/src/lib/reportText.ts

@@ -0,0 +1,271 @@
+// How a report reads at a glance (the inbox row headline) and in full (the
+// copy-as-text dump pasted into the Deck board).
+//
+// meta-core stores a report's body verbatim and knows nothing about it, so
+// the meaning lives here, per app. The meta-watch headline is ported from the
+// /info page that used to own reports (meta-watch ui/info.html,
+// reportHeadline) — its comments explain why each chip earned its place; the
+// short version is that every chip there is a stall or desync that once read
+// as "healthy" until it was measured. Anything an app adds that this file does
+// not know still renders: in the dump, and as a JSON section in the detail.
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export type Body = Record<string, any>;
+
+export interface ReportSummary {
+  id: number;
+  ts: number;
+  received: number;
+  app: string;
+  build?: string;
+  uid?: string;
+  page?: string;
+  route?: string;
+  cid?: string;
+  title?: string;
+  note?: string;
+  status: 'open' | 'triaged' | 'closed';
+  triage_note?: string;
+  updated?: number;
+  log_lines: number;
+  bytes: number;
+}
+
+export interface Report extends ReportSummary {
+  body: Body;
+}
+
+/** One headline segment; `strong` ones are what a human wrote or a failure. */
+export interface Bit {
+  text: string;
+  strong?: boolean;
+}
+
+export function fmtDur(ms: number): string {
+  if (ms == null || !isFinite(ms)) return '?';
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  const s = ms / 1000;
+  if (s < 60) return `${s.toFixed(s < 10 ? 1 : 0)}s`;
+  const m = Math.floor(s / 60);
+  const r = Math.round(s % 60);
+  if (m < 60) return `${m}m${r ? ` ${r}s` : ''}`;
+  return `${Math.floor(m / 60)}h ${m % 60}m`;
+}
+
+export function shortCid(cid: string): string {
+  return cid && cid.length > 16 ? `${cid.slice(0, 8)}…${cid.slice(-6)}` : cid || '';
+}
+
+const clip = (s: string, n: number) => (s.length > n ? s.slice(0, n - 1) + '…' : s);
+
+/** First line of the build blob, minus the app prefix: which build filed it. */
+export function buildLabel(r: ReportSummary): string {
+  const first = (r.build || '').split('\n')[0];
+  return first.replace(/^meta-(watch|read|listen)( page| server)?\s+/, '').trim();
+}
+
+/** The device a report came from, in one line. */
+export function deviceLine(d: any): string {
+  if (!d || typeof d !== 'object') return '';
+  const bits: string[] = [];
+  if (d.model) bits.push(d.model);
+  else if (d.platform) bits.push(d.platform);
+  if (d.isTv) bits.push(d.tvForced ? 'TV (forced)' : 'TV');
+  if (d.screen) bits.push(d.screen);
+  if (d.cores) bits.push(`${d.cores} cores`);
+  if (d.memoryGb) bits.push(`${d.memoryGb} GB`);
+  if (d.gpu) bits.push(d.gpu);
+  return bits.join(' · ');
+}
+
+/** What the row is about: a title, a page + route, or a cid. */
+export function subject(r: ReportSummary, body?: Body): string {
+  const c = (body && body.context) || {};
+  if (r.page === 'search') return c.query ? `search “${c.query}”` : 'search';
+  if (r.title) return r.title;
+  if (r.page && r.page !== 'watch' && r.page !== 'read') return r.page;
+  if (r.cid) return shortCid(r.cid);
+  return r.page || '—';
+}
+
+// ── Headlines ─────────────────────────────────────────────────────────────
+
+function failureBit(c: any): Bit | null {
+  const f = c && (c.failure || c.lastFailure);
+  if (!f) return null;
+  const who = c.lastFailure ? ` ${f.title || shortCid(f.cid || '')}` : '';
+  const tries = Array.isArray(f.attempts) && f.attempts.length > 1 ? ` · ${f.attempts.length} tries` : '';
+  return {
+    text: `FAILED${who}${f.elapsedMs != null ? ' after ' + fmtDur(f.elapsedMs) : ''}: ${clip(f.message || '', 90)}${tries}`,
+    strong: true,
+  };
+}
+
+function watchBits(r: ReportSummary, b: Body): string[] {
+  const bits: string[] = [];
+  const c = b.context || {};
+  if (r.page === 'search') {
+    if (c.state) bits.push(c.state === 'errored' ? 'STREAM ERRORED' : c.state);
+    if (c.anchored != null) {
+      bits.push(
+        `${c.anchored} anchored · ${c.raw || 0} raw` +
+          (c.shown != null && c.shown !== c.anchored ? ` · ${c.shown} after language filter` : '')
+      );
+    }
+    if (c.elapsedMs != null) bits.push(`stream ${fmtDur(c.elapsedMs)}`);
+    if (c.cacheSeeded) bits.push('painted from cache');
+    const ups = Array.isArray(c.upstreams) ? c.upstreams : [];
+    if (c.state !== 'idle' && c.state !== 'youtube-paste') {
+      if (ups.length) {
+        const recs = ups.reduce((n: number, u: any) => n + (u.records || 0), 0);
+        bits.push(`${ups.length} upstream${ups.length === 1 ? '' : 's'} (${recs} records)`);
+      } else bits.push('no upstream seen');
+    }
+  } else if (r.page === 'title') {
+    if (c.episodes != null) {
+      bits.push(`${c.episodes} release${c.episodes === 1 ? '' : 's'}` + (c.rawSources ? ` · ${c.rawSources} raw` : ''));
+    }
+    if (c.streamsOpen) bits.push(`${c.streamsOpen} stream${c.streamsOpen === 1 ? '' : 's'} open`);
+    else if (c.settled != null) bits.push(c.settled ? 'settled' : 'not settled');
+    if (c.placeholderCard) bits.push('placeholder hero');
+  }
+  if (c.lastPlayback) bits.push(`after playing ${c.lastPlayback.title || shortCid(c.lastPlayback.cid || '')}`);
+
+  const d = b.decode || {};
+  const p = b.pipeline || {};
+  if (d.underruns != null) bits.push(`${d.underruns} underrun${d.underruns === 1 ? '' : 's'}`);
+  if (d.minAhead != null) bits.push(`min ${(+d.minAhead).toFixed(1)}s ahead`);
+  if (d.decoded) {
+    const pct = (d.dropped / d.decoded) * 100;
+    bits.push(`${d.dropped} dropped (${pct.toFixed(pct >= 1 ? 1 : 2)}%)`);
+  }
+  if (d.capability) bits.push(d.capability);
+  if (p.pctDemuxCpu != null) bits.push(`demux CPU ${p.pctDemuxCpu}%`);
+
+  // Subtitle verdict: a stalled overlay is invisible in every other number.
+  const sub = b.subtitles;
+  if (sub && sub.up) {
+    const ago = sub.lastDrawAgoMs;
+    if (ago == null) bits.push('subs: NO DRAWS');
+    else if (ago > 4000) {
+      const why =
+        sub.paused === true ? ' while PAUSED'
+        : sub.advancing === true ? ' while playing'
+        : sub.paused === false ? ' (playing, not advancing)'
+        : '';
+      bits.push(`subs: STALLED ${fmtDur(ago)}${why}`);
+    } else bits.push(`subs ok (${sub.draws} draws)`);
+    if (sub.skipped > 0 && sub.draws > 0) {
+      const share = Math.round((100 * sub.skipped) / (sub.skipped + sub.draws));
+      if (share >= 15) bits.push(`subs: ${share}% of frames dropped`);
+    }
+    if (sub.commitMaxMs != null && sub.commitMaxMs > 250) {
+      bits.push(
+        `subs: commit ${sub.commitMs}/${sub.commitMaxMs}ms` +
+          (sub.pruned ? ` (${sub.lines}+${sub.pruned}p lines)` : ` (${sub.lines} lines)`)
+      );
+    }
+    if (sub.lastEventAgoMs != null && sub.lastEventAgoMs > 30000 && sub.advancing) {
+      bits.push(`subs: no packets for ${fmtDur(sub.lastEventAgoMs)}`);
+    }
+    if (sub.canvas && sub.canvas !== 'ok') bits.push(`subs canvas ${sub.canvas}`);
+    if (sub.videoW === 0 || sub.resizeDeferred) bits.push('subs: overlay resize owed (video has no size)');
+    else if (sub.resizesBlocked > 0) {
+      bits.push(`subs: ${sub.resizesBlocked} zero-size resize${sub.resizesBlocked > 1 ? 's' : ''} blocked`);
+    }
+  } else if (sub && sub.failed) {
+    const dd = sub.death;
+    if (!dd) bits.push('subs: fell back to WebVTT (build too old to say why)');
+    else {
+      bits.push(`subs: fell back to WebVTT at ${fmtDur(dd.mediaTime)} — ${dd.why}`);
+      if (dd.commitFailures) bits.push(`after ${dd.commitFailures} rejected commits`);
+      if (dd.strikes) bits.push(`after ${dd.strikes} stuck-loop strikes`);
+      bits.push(dd.draws ? `had drawn ${dd.draws} frames (${dd.lines}+${dd.pruned}p lines)` : 'never drew a frame');
+      if (dd.canvas && dd.canvas !== 'ok') bits.push(`canvas ${dd.canvas} at death`);
+      if (dd.resizesBlocked) bits.push(`${dd.resizesBlocked} zero-size resizes blocked`);
+      if (dd.videoW === 0) bits.push('video had no size at death');
+    }
+  }
+
+  // A/V sync: only the peak earns a chip.
+  const au = b.audio;
+  if (au && au.driftMaxMs != null && Math.abs(au.driftMaxMs) >= 40) {
+    const at = au.driftMaxAtSec != null ? ` @ ${fmtDur(au.driftMaxAtSec * 1000)}` : '';
+    bits.push(`A/V DESYNC ${au.driftMaxMs > 0 ? '+' : ''}${au.driftMaxMs}ms${at}`);
+  }
+  if (au && au.path === 'polyfill') bits.push(`audio ${au.codec || '?'} via polyfill`);
+  const rs = b.restart;
+  if (rs && rs.audioMinusVideoSec != null && rs.audioMinusVideoSec < -0.001) {
+    bits.push(`restart base from AUDIO (${rs.audioMinusVideoSec.toFixed(3)}s)`);
+  }
+  if (b.ttmvpMs != null) bits.push(`TTMVP ${fmtDur(b.ttmvpMs)}`);
+  if (b.capturedMsAgo != null && b.capturedMsAgo > 2000) bits.push(`measured ${fmtDur(b.capturedMsAgo)} before filing`);
+  return bits;
+}
+
+function readBits(r: ReportSummary, b: Body): string[] {
+  const bits: string[] = [];
+  const rd = b.reader;
+  if (rd && typeof rd === 'object') {
+    if (rd.page != null) bits.push(`page ${rd.page + 1}${rd.pages ? `/${rd.pages}` : ''}`);
+    if (rd.flow) bits.push(rd.flow);
+    if (rd.direction) bits.push(rd.direction);
+    if (rd.spread) bits.push('spread');
+    if (rd.strip && rd.strip.segments) bits.push(`strip ${rd.strip.active ?? '?'}/${rd.strip.segments}`);
+    if (rd.via) bits.push(`via ${rd.via}`);
+    if (rd.atEnd) bits.push('at end');
+  }
+  const c = b.context || {};
+  if (r.page === 'search') {
+    if (c.state) bits.push(c.state);
+    if (c.count != null) bits.push(`${c.count} shown`);
+  }
+  return bits;
+}
+
+/** The row headline: the note first (the only part a human wrote), then a
+ *  failure if there was one, then the app's own measurements. */
+export function headline(r: ReportSummary, body?: Body): Bit[] {
+  const out: Bit[] = [];
+  if (r.note) out.push({ text: `“${clip(r.note, 80)}”`, strong: true });
+  if (!body) return out;
+  const f = failureBit(body.context);
+  if (f) out.push(f);
+  const bits = r.app === 'meta-watch' ? watchBits(r, body) : r.app === 'meta-read' ? readBits(r, body) : [];
+  for (const t of bits) out.push({ text: t });
+  if (body.flags && typeof body.flags === 'object' && Object.keys(body.flags).length) {
+    out.push({ text: 'overrides: ' + Object.entries(body.flags).map(([k, v]) => `${k}=${v}`).join(' ') });
+  }
+  return out;
+}
+
+// ── Full text dump ────────────────────────────────────────────────────────
+
+const LIFTED = new Set(['app', 'build', 'uid', 'page', 'route', 'cid', 'title', 'note', 'log']);
+
+/** Every block of a report that is not one of the lifted fields or the log,
+ *  in a stable order: context first, device second, then alphabetical. */
+export function blocks(body: Body): [string, any][] {
+  const keys = Object.keys(body).filter((k) => !LIFTED.has(k));
+  const rank = (k: string) => (k === 'context' ? 0 : k === 'device' ? 1 : 2);
+  keys.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+  return keys.map((k) => [k, body[k]]);
+}
+
+/** The whole report as text, for pasting into a ticket. */
+export function dump(r: Report): string {
+  const parts: string[] = [];
+  parts.push(`REPORT #${r.id} · ${r.app} · ${new Date(r.ts).toISOString()} · ${r.status}` +
+    (r.uid ? `\nUID ${r.uid}` : ''));
+  if (r.note) parts.push('NOTE\n' + r.note);
+  if (r.page || r.route) parts.push('PAGE ' + (r.page || '—') + (r.route ? '\nROUTE ' + r.route : ''));
+  if (r.cid || r.title) parts.push(`CONTENT ${r.title || ''}${r.cid ? ` (${r.cid})` : ''}`.trim());
+  if (r.build) parts.push(r.build);
+  for (const [k, v] of blocks(r.body || {})) {
+    parts.push(k.toUpperCase() + '\n' + (typeof v === 'string' ? v : JSON.stringify(v, null, 2)));
+  }
+  const log: string[] = Array.isArray(r.body?.log) ? r.body.log : [];
+  if (log.length) parts.push(`LOG (${log.length} lines)\n` + log.join('\n'));
+  if (r.triage_note) parts.push('TRIAGE\n' + r.triage_note);
+  return parts.join('\n\n');
+}

dashboard/src/meta-service-menu.js

@@ -0,0 +1,308 @@
+/**
+ * <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
+ *
+ * Replaces the eight divergent "services bar" implementations (four copies of
+ * a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
+ * ones) with a single framework-free custom element. It drops into the React
+ * dashboards as a plain tag — no React dependency — and into the vanilla UIs
+ * as an ES module.
+ *
+ * Data comes from the host service's own `GET /api/neighbors`, same-origin.
+ * Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
+ * make a fetch to a neighbour a dead end. Links are plain navigations.
+ *
+ * ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
+ * `scripts/check-mirrors.sh`. Edit one → edit all.
+ *
+ * Shadow DOM is not optional here. The eight UIs use four incompatible token
+ * vocabularies, and two of them define *different colours under the same
+ * names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
+ * #161b22 in meta-dup). meta-search and meta-share are also the only
+ * light-mode-capable UIs, so a hardcoded dark palette would break them.
+ * The element therefore styles itself from four tokens with dark defaults:
+ *
+ *     --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
+ *
+ * Each host maps its own variables onto those, e.g.
+ *     meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
+ *
+ * Attributes:
+ *   current   name of the host service, so it can be marked and not linked
+ *   endpoint  override the default "/api/neighbors"
+ *   label     button tooltip (default "Services")
+ *
+ * Property:
+ *   rewriteUrl  optional (name, url, neighbour) => url, for hosts that must
+ *               translate internal hostnames into browser-reachable ones
+ *               (meta-gateway does this for its dev/prod split).
+ */
+
+const ICONS = {
+  "meta-core": "⚙️",
+  "meta-sort": "📁",
+  "meta-fuse": "🗂️",
+  "meta-stremio": "🎬",
+  "meta-dup": "🔍",
+  "meta-search": "🔎",
+  "meta-share": "🔗",
+  "meta-gateway": "🌉",
+  "meta-watch": "📺",
+  "meta-listen": "🎧",
+  "meta-read": "📖",
+};
+const DEFAULT_ICON = "📦";
+const POLL_MS = 30000;
+
+function titleCase(name) {
+  return String(name || "")
+    .split("-")
+    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+    .join(" ");
+}
+
+const TEMPLATE = `
+<style>
+  :host {
+    --_fg: var(--mm-nav-fg, #e8eaed);
+    --_bg: var(--mm-nav-bg, #1a1a2e);
+    --_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
+    --_accent: var(--mm-nav-accent, #4ecdc4);
+    position: relative;
+    display: inline-block;
+    font-family: inherit;
+    font-size: 14px;
+    line-height: 1.4;
+  }
+  button.trigger {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.4rem;
+    width: 2.1rem;
+    height: 2.1rem;
+    padding: 0;
+    cursor: pointer;
+    color: var(--_fg);
+    background: transparent;
+    border: 1px solid var(--_border);
+    border-radius: 8px;
+    font: inherit;
+    transition: border-color 0.15s, background 0.15s;
+  }
+  button.trigger:hover { border-color: var(--_accent); }
+  button.trigger:focus-visible {
+    outline: 2px solid var(--_accent);
+    outline-offset: 2px;
+  }
+  button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
+  .bars { display: block; width: 15px; }
+  .bars span {
+    display: block;
+    height: 2px;
+    margin: 3px 0;
+    border-radius: 2px;
+    background: currentColor;
+  }
+  .panel {
+    position: absolute;
+    top: calc(100% + 6px);
+    right: 0;
+    z-index: 1000;
+    min-width: 13rem;
+    max-height: 70vh;
+    overflow-y: auto;
+    padding: 0.3rem;
+    background: var(--_bg);
+    border: 1px solid var(--_border);
+    border-radius: 10px;
+    box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
+  }
+  .panel[hidden] { display: none; }
+  a.item, .item {
+    display: flex;
+    align-items: center;
+    gap: 0.55rem;
+    padding: 0.45rem 0.6rem;
+    border-radius: 6px;
+    color: var(--_fg);
+    text-decoration: none;
+    white-space: nowrap;
+  }
+  a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
+  .item.active { color: var(--_accent); font-weight: 600; cursor: default; }
+  .item.dead { opacity: 0.55; cursor: not-allowed; }
+  .icon { font-size: 1rem; line-height: 1; }
+  .dot {
+    width: 6px;
+    height: 6px;
+    margin-left: auto;
+    border-radius: 50%;
+    background: var(--_accent);
+  }
+  .empty {
+    padding: 0.6rem;
+    opacity: 0.7;
+    font-size: 0.85em;
+    white-space: nowrap;
+  }
+  /* Respect a host that opts out of motion. */
+  @media (prefers-reduced-motion: reduce) {
+    button.trigger { transition: none; }
+  }
+</style>
+<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
+  <span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
+</button>
+<div class="panel" role="menu" hidden></div>
+`;
+
+class MetaServiceMenu extends HTMLElement {
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot.innerHTML = TEMPLATE;
+    this._btn = this.shadowRoot.querySelector("button.trigger");
+    this._panel = this.shadowRoot.querySelector(".panel");
+    this._neighbors = [];
+    this._timer = null;
+    this._open = false;
+
+    this._onDocClick = (e) => {
+      if (!this._open) return;
+      if (e.composedPath().includes(this)) return;
+      this._setOpen(false);
+    };
+    this._onKey = (e) => {
+      if (e.key === "Escape" && this._open) {
+        this._setOpen(false);
+        this._btn.focus();
+      }
+    };
+  }
+
+  connectedCallback() {
+    this._btn.title = this.getAttribute("label") || "Services";
+    this._btn.setAttribute("aria-label", this._btn.title);
+    this._btn.addEventListener("click", () => this._setOpen(!this._open));
+    document.addEventListener("click", this._onDocClick);
+    document.addEventListener("keydown", this._onKey);
+
+    this.refresh();
+    this._timer = setInterval(() => this.refresh(), POLL_MS);
+  }
+
+  disconnectedCallback() {
+    if (this._timer) clearInterval(this._timer);
+    this._timer = null;
+    document.removeEventListener("click", this._onDocClick);
+    document.removeEventListener("keydown", this._onKey);
+  }
+
+  get endpoint() {
+    return this.getAttribute("endpoint") || "/api/neighbors";
+  }
+
+  get current() {
+    return this.getAttribute("current") || "";
+  }
+
+  async refresh() {
+    try {
+      const res = await fetch(this.endpoint, {
+        headers: { Accept: "application/json" },
+      });
+      if (!res.ok) throw new Error(String(res.status));
+      const data = await res.json();
+      // `services` is the legacy alias some backends still emit.
+      this._neighbors = data.neighbors || data.services || [];
+    } catch {
+      // Silent: the menu is navigation, not a health indicator. Keep whatever
+      // we last had rather than blanking the list on one failed poll.
+    }
+    this._render();
+  }
+
+  _setOpen(open) {
+    this._open = open;
+    this._panel.hidden = !open;
+    this._btn.setAttribute("aria-expanded", String(open));
+    if (open) this.refresh();
+  }
+
+  _render() {
+    const panel = this._panel;
+    panel.textContent = "";
+
+    const seen = new Set();
+    const rows = [];
+    for (const n of this._neighbors) {
+      const name = n.name || "";
+      if (!name || seen.has(name)) continue;
+      // Beacon v2 neighbours include plugins; the nav lists services only.
+      // (A row without `caps` comes from a pre-v2 backend — keep it.)
+      const isService = (c) => String(c).startsWith("metamesh.service/");
+      if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
+      seen.add(name);
+      rows.push(n);
+    }
+    // Always show the host itself, even before its own echo comes back.
+    if (this.current && !seen.has(this.current)) {
+      rows.push({ name: this.current, baseUrl: "" });
+    }
+    rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+
+    if (rows.length === 0) {
+      const empty = document.createElement("div");
+      empty.className = "empty";
+      empty.textContent = "No services found";
+      panel.appendChild(empty);
+      return;
+    }
+
+    for (const n of rows) {
+      const isActive = n.name === this.current;
+      let url = n.baseUrl || "";
+      if (typeof this.rewriteUrl === "function") {
+        try {
+          url = this.rewriteUrl(n.name, url, n) || "";
+        } catch {
+          /* a bad hook must not blank the menu */
+        }
+      }
+
+      // A neighbour with no reachable URL is shown but not linked — better
+      // than a link that goes nowhere.
+      const el = document.createElement(isActive || !url ? "span" : "a");
+      el.className =
+        "item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
+      if (el.tagName === "A") {
+        el.href = url;
+        el.setAttribute("role", "menuitem");
+      }
+      if (!isActive && !url) el.title = "No reachable URL advertised";
+
+      const icon = document.createElement("span");
+      icon.className = "icon";
+      icon.textContent = ICONS[n.name] || DEFAULT_ICON;
+
+      const label = document.createElement("span");
+      label.textContent = titleCase(n.name);
+
+      el.append(icon, label);
+      // Beacon v2 omits `status` when running.
+      if ((n.status || "running") === "running" || isActive) {
+        const dot = document.createElement("span");
+        dot.className = "dot";
+        dot.title = n.status || "running";
+        el.appendChild(dot);
+      }
+      panel.appendChild(el);
+    }
+  }
+}
+
+if (!customElements.get("meta-service-menu")) {
+  customElements.define("meta-service-menu", MetaServiceMenu);
+}
+
+export default MetaServiceMenu;

dashboard/src/pages/Dashboard.tsx

@@ -1,4 +1,5 @@
 import { useEffect, useState } from 'react';
+import { getJSON } from '../lib/api';
 
 interface HealthStatus {
   status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
   timestamp: string;
 }
 
+interface FileStats {
+  count: number;
+  totalSize: number;
+}
+
+interface Stats {
+  records: number;
+  redisKeys: number;
+  redisMemory: string;
+  identities: number;
+  files: FileStats | null;
+  udlUsers: number | null;
+  sweptAt: number;
+  sweeping: boolean;
+}
+
 interface WatcherStatus {
   id: string;
   path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
   count: number;
 }
 
-interface ServiceInfo {
-  name: string;
-  api: string;
-  capabilities: string[];
-  timestamp: number;
-}
-
 const cardStyle: React.CSSProperties = {
   background: '#16213e',
   borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
 
 const gridStyle: React.CSSProperties = {
   display: 'grid',
-  gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
+  // 200px, not 220: five tiles must fit one row inside the 1200px container
+  // (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
+  gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
   gap: '1rem',
+  marginBottom: '1rem',
 };
 
 const statStyle: React.CSSProperties = {
   fontSize: '2rem',
   fontWeight: 'bold',
   color: '#4ade80',
+  lineHeight: 1.2,
 };
 
+const labelStyle: React.CSSProperties = {
+  marginBottom: '0.75rem',
+  color: '#888',
+  fontSize: '0.85rem',
+  textTransform: 'uppercase',
+  letterSpacing: '0.04em',
+};
+
+const noteStyle: React.CSSProperties = {
+  color: '#6b7a99',
+  fontSize: '0.8rem',
+  marginTop: '0.4rem',
+};
+
+const numberFmt = new Intl.NumberFormat();
+
+function formatCount(n: number | null | undefined): string {
+  return n === null || n === undefined ? '—' : numberFmt.format(n);
+}
+
+// Binary units: this counts bytes on disk, and every other tool the operator
+// compares against (du, docker, the mount page) reports GiB.
+function formatBytes(bytes: number | null | undefined): string {
+  if (bytes === null || bytes === undefined) return '—';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
+}
+
+function Tile({
+  label,
+  value,
+  note,
+  pending,
+}: {
+  label: string;
+  value: string;
+  note?: string;
+  pending?: boolean;
+}) {
+  return (
+    <div style={cardStyle}>
+      <div style={labelStyle}>{label}</div>
+      <div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
+        {value}
+      </div>
+      {note && <div style={noteStyle}>{note}</div>}
+    </div>
+  );
+}
+
 export default function Dashboard() {
   const [health, setHealth] = useState<HealthStatus | null>(null);
+  const [stats, setStats] = useState<Stats | null>(null);
   const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
-  const [services, setServices] = useState<ServiceInfo[]>([]);
   const [error, setError] = useState<string | null>(null);
 
   useEffect(() => {
+    let cancelled = false;
+
     const fetchData = async () => {
-      try {
+      // Settled, not all: one dead endpoint must not blank every tile.
-        // Fetch health
+      const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
-        const healthRes = await fetch('/health');
+        getJSON<HealthStatus>('/health'),
-        if (healthRes.ok) {
+        getJSON<Stats>('/api/stats'),
-          setHealth(await healthRes.json());
+        getJSON<WatchersResponse>('/api/watchers'),
-        }
+      ]);
+      if (cancelled) return;
 
-        // Fetch watchers status
+      if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
-        const watchersRes = await fetch('/api/watchers');
+      if (statsRes.status === 'fulfilled') setStats(statsRes.value);
-        if (watchersRes.ok) {
+      if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
-          setWatchersData(await watchersRes.json());
-        }
 
-        // Fetch services
+      const failures = [healthRes, statsRes, watchersRes]
-        const servicesRes = await fetch('/services');
+        .filter((r): r is PromiseRejectedResult => r.status === 'rejected')
-        if (servicesRes.ok) {
+        .map((r) => String(r.reason?.message ?? r.reason));
-          const data = await servicesRes.json();
+      setError(failures.length ? failures.join(' · ') : null);
-          setServices(data.services || []);
-        }
-
-        setError(null);
-      } catch (err) {
-        setError(String(err));
-      }
     };
 
     fetchData();
     const interval = setInterval(fetchData, 5000);
-    return () => clearInterval(interval);
+    return () => {
+      cancelled = true;
+      clearInterval(interval);
+    };
   }, []);
 
   const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
     }
   };
 
-  // Aggregate watcher stats
+  const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
-  const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
+  const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
-  const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
+  const lastScan =
-  const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
+    watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
-  const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
+
+  // Records we hold metadata for but have no local file: gateway/cid-rooted
+  // entries, plus anything whose file went away. The interesting number on a
+  // client box, where it should be most of them.
+  const fileless =
+    stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
+
+  const sweepPending = !!stats && !stats.files;
+  const sweptNote = (() => {
+    if (!stats) return undefined;
+    if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
+    if (stats.sweeping) return 'refreshing…';
+    if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
+    return undefined;
+  })();
 
   return (
-    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
+    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
       <h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
 
       {error && (
         <div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
-          Error: {error}
+          {error}
         </div>
       )}
 
       <div style={gridStyle}>
-        {/* Health Card */}
+        <Tile
+          label="Files discovered"
+          value={formatCount(stats?.files?.count)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Library size"
+          value={formatBytes(stats?.files?.totalSize)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Metadata entries"
+          value={formatCount(stats?.records)}
+          note={
+            fileless === null
+              ? 'records in the index'
+              : `${formatCount(fileless)} with no local file`
+          }
+        />
+        <Tile
+          label="Store keys"
+          value={formatCount(stats?.redisKeys)}
+          note={stats?.redisMemory ? `${stats.redisMemory} on disk` : undefined}
+        />
+        <Tile
+          label="Identities"
+          value={formatCount(stats?.identities)}
+          note={
+            stats?.udlUsers !== null && stats?.udlUsers !== undefined
+              ? `${formatCount(stats.udlUsers)} with user data`
+              : 'signing accounts'
+          }
+        />
+      </div>
+
+      <div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
+          <div style={labelStyle}>Service health</div>
           {health ? (
             <>
               <div style={statStyle}>
                 {health.status === 'ok' ? 'Healthy' : health.status}
               </div>
-              <p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
+              <p style={noteStyle}>
+                Metadata store: {health.redis ? 'Open' : 'Unavailable'}
+              </p>
             </>
           ) : (
-            <p>Loading...</p>
+            <p style={noteStyle}>Loading…</p>
           )}
         </div>
 
-        {/* Watchers Status Card */}
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
+          <div style={labelStyle}>File watchers</div>
           {watchersData ? (
             <>
-              <div style={statStyle}>{totalFiles}</div>
+              <div style={statStyle}>
-              <p>Files discovered</p>
+                {activeWatchers} / {watchersData.count}
-              <p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
+              </div>
-              <p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
+              <p style={noteStyle}>active watchers</p>
+              <p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
               {lastScan > 0 && (
-                <p>Last scan: {new Date(lastScan).toLocaleString()}</p>
+                <p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
               )}
               <button
                 onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
               </button>
             </>
           ) : (
-            <p>Loading...</p>
+            <p style={noteStyle}>Loading…</p>
-          )}
-        </div>
-
-        {/* Services Card */}
-        <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
-          <div style={statStyle}>{services.length}</div>
-          <p>Services registered</p>
-          {services.length > 0 && (
-            <ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
-              {services.map((svc) => (
-                <li key={svc.name} style={{ marginBottom: '0.5rem' }}>
-                  <strong>{svc.name}</strong>
-                  <br />
-                  <span style={{ color: '#888', fontSize: '0.9rem' }}>
-                    {svc.capabilities?.join(', ') || 'No capabilities'}
-                  </span>
-                </li>
-              ))}
-            </ul>
           )}
         </div>
       </div>

dashboard/src/pages/Identity.tsx

@@ -179,11 +179,10 @@ async function fetchAccounts(): Promise<Account[]> {
 // Resolve an account's display name from the User Data Layer, or null.
 //
 // Never throws and never reports an error upward: a name is decoration on top
-// of the uid, and the account list must render identically whether Redis is up
+// of the uid, and the account list must render identically whether the store is up
-// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
+// (identity itself is file-backed, the UDL is not). The name is the user's
-// — meta-core stores the plaintext beside the signed record and verifies
+// signed MMUR2 claim (meta-core verified it on write); the uid stays on screen
-// nothing — so the uid stays on screen as the real identity, and the length cap
+// as the real identity, and the length cap bounds the layout.
-// bounds what a hostile writer can do to the layout.
 async function fetchDisplayName(uid: string): Promise<string | null> {
   const url =
     `/api/udl/record?uid=${encodeURIComponent(uid)}` +
@@ -192,12 +191,11 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
   try {
     const res = await fetch(url);
     if (!res.ok) return null; // 503 when storage is down, 500, …
-    const data = (await res.json()) as { exists?: boolean; value?: unknown };
+    const data = (await res.json()) as { exists?: boolean; claim?: { value?: unknown } | null };
-    // Key off `value`, not `exists`: clearing a name writes a tombstone, which
+    // A retraction (null) or ciphertext ({b64}) has nothing to show.
-    // leaves the cell existing with no plaintext. Private-tier cells look the
+    const value = data.claim?.value;
-    // same, which is exactly right — there is nothing to show for either.
+    if (typeof value !== 'string') return null;
-    if (typeof data.value !== 'string') return null;
+    const name = value.trim();
-    const name = data.value.trim();
     return name ? name.slice(0, 64) : null;
   } catch {
     return null;
@@ -209,7 +207,7 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
 // so per-row calls would walk the keyspace once per account.
 //
 // Never throws, for the same reason fetchDisplayName does not: the counts are
-// context on top of the uid, and the list must render identically when Redis is
+// context on top of the uid, and the list must render identically when the store is
 // down — accounts are file-backed, their data is not.
 //
 // Returns null on failure rather than an empty map. The difference is
@@ -603,7 +601,7 @@ function AccountCard({
                       records === 1 ? 'record' : 'records'
                     } — likes, ratings, watch progress across ${titles.toLocaleString()} ${
                       titles === 1 ? 'title' : 'titles'
-                    } — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
+                    } — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '}
               This cannot be undone and there is no backup.
             </li>
             <li>
@@ -655,12 +653,12 @@ export default function Identity() {
       return;
     }
     if (gen !== reqGen.current) return;
-    // Render the list before resolving names, so a down Redis delays nothing.
+    // Render the list before resolving names, so a down store delays nothing.
     setAccounts(list);
     setLoading(false);
 
     // Names and counts both come after the list is on screen, and neither can
-    // fail the page: a down Redis costs decoration, not the account list.
+    // fail the page: a down store costs decoration, not the account list.
     const [settled, counts] = await Promise.all([
       Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
       fetchStats(),

dashboard/src/pages/Reports.tsx

@@ -0,0 +1,406 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { getJSON } from '../lib/api';
+import {
+  blocks,
+  buildLabel,
+  deviceLine,
+  dump,
+  headline,
+  type Report,
+  type ReportSummary,
+  shortCid,
+  subject,
+} from '../lib/reportText';
+
+// The bug-report / feedback inbox. Client apps (meta-watch, meta-read, …)
+// forward what their 🐛 buttons file to POST /api/reports; this page lists,
+// reads and triages them. See internal/reports for storage and bounds.
+
+type Status = 'open' | 'triaged' | 'closed';
+const STATUSES: Status[] = ['open', 'triaged', 'closed'];
+
+interface ListPage {
+  reports: ReportSummary[];
+  apps: Record<string, number>;
+  counts: Record<string, number>;
+  more: boolean;
+}
+
+const card: React.CSSProperties = {
+  background: '#16213e',
+  borderRadius: '8px',
+  padding: '1rem',
+  display: 'flex',
+  flexDirection: 'column',
+  overflow: 'hidden',
+  minHeight: 0,
+};
+
+const chip = (active: boolean): React.CSSProperties => ({
+  padding: '0.25rem 0.7rem',
+  borderRadius: '999px',
+  border: '1px solid #0f3460',
+  background: active ? '#0f3460' : 'transparent',
+  color: active ? '#fff' : '#aaa',
+  cursor: 'pointer',
+  fontSize: '0.85rem',
+});
+
+const button: React.CSSProperties = {
+  padding: '0.35rem 0.8rem',
+  borderRadius: '4px',
+  border: '1px solid #0f3460',
+  background: '#0f3460',
+  color: '#fff',
+  cursor: 'pointer',
+  fontSize: '0.85rem',
+};
+
+const mono: React.CSSProperties = { fontFamily: 'ui-monospace, Menlo, Consolas, monospace', fontSize: '0.8rem' };
+
+const statusColor: Record<Status, string> = { open: '#b45309', triaged: '#1d4ed8', closed: '#374151' };
+
+function StatusBadge({ s }: { s: Status }) {
+  return (
+    <span style={{ background: statusColor[s], color: '#fff', borderRadius: '3px', padding: '0.05rem 0.4rem', fontSize: '0.72rem' }}>
+      {s}
+    </span>
+  );
+}
+
+const fmtTime = (ms: number) => new Date(ms).toLocaleString();
+
+export default function Reports() {
+  const [app, setApp] = useState('');
+  const [status, setStatus] = useState<Status | ''>('open');
+  const [q, setQ] = useState('');
+  const [page, setPage] = useState<ListPage | null>(null);
+  const [older, setOlder] = useState<ReportSummary[]>([]);
+  const [more, setMore] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [selected, setSelected] = useState<number | null>(null);
+
+  const query = useCallback(
+    (before?: number) => {
+      const p = new URLSearchParams();
+      if (app) p.set('app', app);
+      if (status) p.set('status', status);
+      if (q.trim()) p.set('q', q.trim());
+      if (before) p.set('before', String(before));
+      return `/api/reports?${p}`;
+    },
+    [app, status, q]
+  );
+
+  // The newest page is polled; pages loaded with "older" are kept beneath it
+  // until a filter changes, and then they own the "more" flag.
+  const olderCount = useRef(0);
+  olderCount.current = older.length;
+  const refresh = useCallback(async () => {
+    try {
+      const data = await getJSON<ListPage>(query());
+      setPage(data);
+      if (olderCount.current === 0) setMore(data.more);
+      setError(null);
+    } catch (e) {
+      setError(String((e as Error).message || e));
+    }
+  }, [query]);
+
+  useEffect(() => {
+    setOlder([]);
+    setMore(false);
+  }, [app, status, q]);
+
+  useEffect(() => {
+    refresh();
+    const t = setInterval(refresh, 10000);
+    return () => clearInterval(t);
+  }, [refresh]);
+
+  const loadOlder = async () => {
+    const all = [...(page?.reports || []), ...older];
+    const last = all[all.length - 1];
+    if (!last) return;
+    try {
+      const data = await getJSON<ListPage>(query(last.id));
+      setOlder((prev) => [...prev, ...data.reports]);
+      setMore(data.more);
+    } catch (e) {
+      setError(String((e as Error).message || e));
+    }
+  };
+
+  const rows = [...(page?.reports || []), ...older.filter((o) => !(page?.reports || []).some((r) => r.id === o.id))];
+  const apps = Object.entries(page?.apps || {}).sort((a, b) => a[0].localeCompare(b[0]));
+  const total = Object.values(page?.counts || {}).reduce((n, c) => n + c, 0);
+
+  return (
+    <div style={{ padding: '1rem', height: '100%', display: 'flex', flexDirection: 'column', overflow: 'hidden', boxSizing: 'border-box' }}>
+      <div style={{ display: 'flex', alignItems: 'baseline', gap: '1rem', flexShrink: 0, flexWrap: 'wrap' }}>
+        <h1 style={{ margin: '0 0 0.75rem 0' }}>Reports</h1>
+        <span style={{ color: '#888', fontSize: '0.9rem' }}>bug reports and feedback filed from the apps' 🐛 buttons</span>
+      </div>
+
+      <div style={{ display: 'flex', gap: '0.5rem', flexWrap: 'wrap', alignItems: 'center', marginBottom: '0.75rem', flexShrink: 0 }}>
+        <span style={chip(status === '')} onClick={() => setStatus('')}>all {total}</span>
+        {STATUSES.map((s) => (
+          <span key={s} style={chip(status === s)} onClick={() => setStatus(s)}>
+            {s} {page?.counts?.[s] ?? 0}
+          </span>
+        ))}
+        <span style={{ width: '1rem' }} />
+        <span style={chip(app === '')} onClick={() => setApp('')}>every app</span>
+        {apps.map(([a, n]) => (
+          <span key={a} style={chip(app === a)} onClick={() => setApp(a)}>
+            {a} {n}
+          </span>
+        ))}
+        <input
+          value={q}
+          onChange={(e) => setQ(e.target.value)}
+          placeholder="search note, title, cid, uid"
+          style={{ marginLeft: 'auto', padding: '0.35rem 0.6rem', borderRadius: '4px', border: '1px solid #0f3460', background: '#0b1630', color: '#eee', minWidth: '16rem' }}
+        />
+      </div>
+
+      {error && <div style={{ color: '#f87171', marginBottom: '0.5rem', flexShrink: 0 }}>Could not load reports: {error}</div>}
+
+      <div style={{ display: 'grid', gridTemplateColumns: 'minmax(22rem, 2fr) 3fr', gap: '1rem', flex: 1, minHeight: 0 }}>
+        <div style={card}>
+          {rows.length === 0 ? (
+            <p style={{ color: '#888', margin: 0 }}>{page ? 'No reports match.' : 'Loading…'}</p>
+          ) : (
+            <div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
+              {rows.map((r) => (
+                <div
+                  key={r.id}
+                  onClick={() => setSelected(r.id)}
+                  style={{
+                    padding: '0.55rem 0.5rem',
+                    borderBottom: '1px solid #0f3460',
+                    cursor: 'pointer',
+                    background: selected === r.id ? '#0f3460' : 'transparent',
+                  }}
+                >
+                  <div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', fontSize: '0.8rem', color: '#888' }}>
+                    <span style={mono}>#{r.id}</span>
+                    <StatusBadge s={r.status} />
+                    <span>{r.app}</span>
+                    <span style={{ marginLeft: 'auto' }}>{fmtTime(r.ts)}</span>
+                  </div>
+                  <div style={{ margin: '0.2rem 0', color: '#eee' }}>{subject(r)}</div>
+                  <div style={{ fontSize: '0.85rem', color: r.note ? '#ddd' : '#666' }}>
+                    {r.note ? `“${r.note.length > 120 ? r.note.slice(0, 119) + '…' : r.note}”` : 'no note'}
+                  </div>
+                  <div style={{ ...mono, color: '#666', marginTop: '0.15rem' }}>
+                    {buildLabel(r) || 'build unknown'}
+                    {r.uid ? ` · ${shortCid(r.uid)}` : ''}
+                    {r.log_lines ? ` · ${r.log_lines} log lines` : ''}
+                  </div>
+                </div>
+              ))}
+              {more && (
+                <div style={{ padding: '0.75rem', textAlign: 'center' }}>
+                  <button style={button} onClick={loadOlder}>Load older</button>
+                </div>
+              )}
+            </div>
+          )}
+        </div>
+
+        <div style={card}>
+          {selected == null ? (
+            <p style={{ color: '#888', margin: 0 }}>Pick a report to read it.</p>
+          ) : (
+            <ReportDetail
+              key={selected}
+              id={selected}
+              onChanged={refresh}
+              onDeleted={() => {
+                setSelected(null);
+                setOlder((prev) => prev.filter((r) => r.id !== selected));
+                refresh();
+              }}
+            />
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}
+
+function ReportDetail({ id, onChanged, onDeleted }: { id: number; onChanged: () => void; onDeleted: () => void }) {
+  const [r, setR] = useState<Report | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  const [note, setNote] = useState('');
+  const [busy, setBusy] = useState(false);
+  const [copied, setCopied] = useState(false);
+  const copyTimer = useRef<number>();
+
+  useEffect(() => {
+    getJSON<Report>(`/api/reports/${id}`)
+      .then((rep) => {
+        setR(rep);
+        setNote(rep.triage_note || '');
+      })
+      .catch((e) => setError(String(e.message || e)));
+    return () => window.clearTimeout(copyTimer.current);
+  }, [id]);
+
+  const patch = async (change: { status?: Status; triage_note?: string }) => {
+    setBusy(true);
+    try {
+      const rep = await getJSON<Report>(`/api/reports/${id}`, {
+        method: 'PATCH',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(change),
+      });
+      setR(rep);
+      setNote(rep.triage_note || '');
+      setError(null);
+      onChanged();
+    } catch (e) {
+      setError(String((e as Error).message || e));
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const remove = async () => {
+    if (!window.confirm(`Delete report #${id}? This cannot be undone.`)) return;
+    setBusy(true);
+    try {
+      const res = await fetch(`/api/reports/${id}`, { method: 'DELETE' });
+      if (!res.ok && res.status !== 404) throw new Error(`/api/reports/${id} → HTTP ${res.status}`);
+      onDeleted();
+    } catch (e) {
+      setError(String((e as Error).message || e));
+      setBusy(false);
+    }
+  };
+
+  const copy = async () => {
+    if (!r) return;
+    try {
+      await navigator.clipboard.writeText(dump(r));
+      setCopied(true);
+      copyTimer.current = window.setTimeout(() => setCopied(false), 1800);
+    } catch {
+      setError('Clipboard unavailable here — select the text below and copy it by hand.');
+    }
+  };
+
+  if (error && !r) return <p style={{ color: '#f87171', margin: 0 }}>Could not load report #{id}: {error}</p>;
+  if (!r) return <p style={{ color: '#888', margin: 0 }}>Loading #{id}…</p>;
+
+  const body = r.body || {};
+  const bits = headline(r, body);
+  const device = deviceLine(body.device);
+  const log: string[] = Array.isArray(body.log) ? body.log : [];
+
+  return (
+    <div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
+      <div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', flexWrap: 'wrap' }}>
+        <h2 style={{ margin: 0, fontSize: '1.15rem' }}>#{r.id} · {subject(r, body)}</h2>
+        <StatusBadge s={r.status} />
+        <span style={{ marginLeft: 'auto', display: 'flex', gap: '0.4rem' }}>
+          {STATUSES.filter((s) => s !== r.status).map((s) => (
+            <button key={s} style={button} disabled={busy} onClick={() => patch({ status: s })}>
+              Mark {s}
+            </button>
+          ))}
+          <button style={button} onClick={copy}>{copied ? 'Copied' : 'Copy as text'}</button>
+          <button style={{ ...button, background: '#7f1d1d', borderColor: '#7f1d1d' }} disabled={busy} onClick={remove}>
+            Delete
+          </button>
+        </span>
+      </div>
+
+      {error && <div style={{ color: '#f87171', marginTop: '0.5rem' }}>{error}</div>}
+
+      <div style={{ color: '#888', fontSize: '0.85rem', margin: '0.4rem 0 0.75rem' }}>
+        {r.app} · filed {fmtTime(r.ts)}
+        {r.received - r.ts > 60000 ? ` · delivered ${fmtTime(r.received)}` : ''}
+        {r.uid ? ` · uid ${r.uid}` : ''}
+      </div>
+
+      {bits.length > 0 && (
+        <div style={{ marginBottom: '0.75rem', lineHeight: 1.6 }}>
+          {bits.map((b, i) => (
+            <span key={i}>
+              {i > 0 && <span style={{ color: '#555' }}> · </span>}
+              {b.strong ? <b style={{ color: '#fff' }}>{b.text}</b> : <span style={{ color: '#ccc' }}>{b.text}</span>}
+            </span>
+          ))}
+        </div>
+      )}
+
+      <Field label="Page">
+        {r.page || '—'}
+        {r.route && <span style={{ ...mono, color: '#888' }}> {r.route}</span>}
+      </Field>
+      {(r.title || r.cid) && (
+        <Field label="Content">
+          {r.title}
+          {r.cid && <span style={{ ...mono, color: '#888' }}> {r.cid}</span>}
+        </Field>
+      )}
+      {device && <Field label="Device">{device}</Field>}
+      {r.build && (
+        <Field label="Build">
+          <pre style={{ ...mono, margin: 0, whiteSpace: 'pre-wrap', color: '#ccc' }}>{r.build}</pre>
+        </Field>
+      )}
+
+      <Field label="Triage">
+        <div style={{ display: 'flex', gap: '0.5rem' }}>
+          <input
+            value={note}
+            onChange={(e) => setNote(e.target.value)}
+            placeholder="e.g. Deck #812, duplicate of #40, fixed in 1.0.95"
+            style={{ flex: 1, padding: '0.35rem 0.6rem', borderRadius: '4px', border: '1px solid #0f3460', background: '#0b1630', color: '#eee' }}
+          />
+          <button style={button} disabled={busy || note === (r.triage_note || '')} onClick={() => patch({ triage_note: note })}>
+            Save
+          </button>
+        </div>
+      </Field>
+
+      {blocks(body)
+        .filter(([, v]) => v === null || typeof v !== 'object')
+        .map(([k, v]) => (
+          <Field key={k} label={k}>
+            <span style={mono}>{String(v)}</span>
+          </Field>
+        ))}
+
+      {blocks(body).filter(([, v]) => v !== null && typeof v === 'object').map(([k, v]) => (
+        <details key={k} open={k === 'context' || k === 'reader'} style={{ marginTop: '0.6rem' }}>
+          <summary style={{ cursor: 'pointer', color: '#9ab' }}>{k}</summary>
+          <pre style={{ ...mono, background: '#0b1630', padding: '0.6rem', borderRadius: '4px', whiteSpace: 'pre-wrap', wordBreak: 'break-word', margin: '0.3rem 0 0' }}>
+            {typeof v === 'string' ? v : JSON.stringify(v, null, 2)}
+          </pre>
+        </details>
+      ))}
+
+      {log.length > 0 && (
+        <details open style={{ marginTop: '0.6rem' }}>
+          <summary style={{ cursor: 'pointer', color: '#9ab' }}>log ({log.length} lines)</summary>
+          <pre style={{ ...mono, background: '#0b1630', padding: '0.6rem', borderRadius: '4px', whiteSpace: 'pre-wrap', wordBreak: 'break-word', margin: '0.3rem 0 0', maxHeight: '28rem', overflow: 'auto' }}>
+            {log.join('\n')}
+          </pre>
+        </details>
+      )}
+    </div>
+  );
+}
+
+function Field({ label, children }: { label: string; children: React.ReactNode }) {
+  return (
+    <div style={{ display: 'grid', gridTemplateColumns: '5.5rem 1fr', gap: '0.5rem', margin: '0.3rem 0', alignItems: 'baseline' }}>
+      <span style={{ color: '#888', fontSize: '0.85rem' }}>{label}</span>
+      <div>{children}</div>
+    </div>
+  );
+}

docker/entrypoint.sh

@@ -9,8 +9,8 @@ set -e
 
 # Create required directories
 mkdir -p /meta-core/locks
-mkdir -p /meta-core/db/redis
+mkdir -p /meta-core/db
-mkdir -p /meta-core/services
+mkdir -p /meta-core/db-cache
 mkdir -p /meta-core/mounts/errors
 mkdir -p /files
 mkdir -p /var/log/supervisor

docker/leader-election.sh

@@ -2,9 +2,14 @@
 # =============================================================================
 # Leader Election Script for meta-core
 #
-# Uses flock for leader election. Only the leader runs the full application
+# The flock here is a MUTEX, not service discovery. It is the only thing
-# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
+# stopping two containers that mount the same /meta-core volume from putting
-# Followers remain dormant, periodically retrying to acquire the lock.
+# two writers on one database file (/meta-core/db/meta-core.db). Sibling
+# services no longer read anything from this directory — they locate meta-core
+# over UDP (beacon v2, see docs/project-architecture/beacon-v2.md), so the
+# kv-leader.info file
+# this script used to publish is gone. Keep the lock; it costs nothing and the
+# corruption it prevents is unrecoverable.
 #
 # Environment variables:
 #   META_CORE_PATH         - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +25,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
 ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
 
 LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
-INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
 SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
 
 # Get the container's IP address
@@ -40,35 +44,13 @@ get_local_ip() {
     echo "$ip"
 }
 
-# Write leader info file (plain text API URL)
-write_leader_info() {
-    local ip=$(get_local_ip)
-    local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
-
-    # Write atomically using temp file + rename
-    local temp_file="${INFO_FILE}.tmp"
-    echo "$api_url" > "$temp_file"
-    mv "$temp_file" "$INFO_FILE"
-
-    echo "[election] Wrote leader info: $api_url"
-}
-
-# Clean up leader info file on shutdown
-cleanup() {
-    echo "[election] Cleaning up..."
-    rm -f "$INFO_FILE"
-    exit 0
-}
-
 # Set up signal handlers for graceful shutdown
-trap cleanup SIGTERM SIGINT
 
 # Ensure lock directory exists
 mkdir -p "$(dirname "$LOCK_FILE")"
 
 echo "[election] Starting leader election..."
 echo "[election] Lock file: $LOCK_FILE"
-echo "[election] Info file: $INFO_FILE"
 
 # Open lock file for flock (file descriptor 200)
 exec 200>"$LOCK_FILE"
@@ -79,9 +61,6 @@ while true; do
     if flock -n 200; then
         echo "[election] Acquired flock - transitioning to LEADER"
 
-        # Write leader info before starting services
-        write_leader_info
-
         # exec supervisord - replaces this process
         # When supervisord exits, the flock is released automatically
         echo "[election] Starting supervisord..."

docker/nginx.conf

@@ -123,14 +123,10 @@ http {
         # =========================================================================
         # Service discovery
         # =========================================================================
-        location /services {
+        # meta-discovery v1 neighbours (supersedes /services, which stays
-            proxy_pass http://meta-core-api/services;
+        # while consumers migrate).
-            proxy_http_version 1.1;
+        location /neighbors {
-            proxy_set_header Host $host;
+            proxy_pass http://meta-core-api/neighbors;
-        }
-
-        location /services/ {
-            proxy_pass http://meta-core-api/services/;
             proxy_http_version 1.1;
             proxy_set_header Host $host;
         }

docker/supervisord.conf

@@ -16,26 +16,13 @@ serverurl=unix:///var/run/supervisor.sock
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 # =============================================================================
-# Redis: Key-value store (managed by supervisord, started before meta-core)
+# meta-core: HTTP API, Mount API, File Watcher, metadata store (embedded SQLite)
-# Previously spawned by Go code, now managed by bash-based leader election
-# =============================================================================
-[program:redis]
-command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
-priority=1
-autostart=true
-autorestart=true
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-startsecs=2
-
-# =============================================================================
-# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
 # =============================================================================
 [program:meta-core]
 command=/usr/local/bin/meta-core
 priority=10
+; Room to finish in-flight writes and checkpoint the WAL on shutdown.
+stopwaitsecs=60
 autostart=true
 autorestart=true
 stdout_logfile=/dev/stdout
@@ -50,7 +37,6 @@ environment=
     SERVICE_VERSION="%(ENV_SERVICE_VERSION)s",
     META_CORE_HTTP_PORT="%(ENV_META_CORE_HTTP_PORT)s",
     META_CORE_HTTP_HOST="%(ENV_META_CORE_HTTP_HOST)s",
-    REDIS_PORT="%(ENV_REDIS_PORT)s",
     WATCH_INTERVAL_MS="%(ENV_WATCH_INTERVAL_MS)s",
     API_PORT="80",
     BASE_URL="%(ENV_BASE_URL)s"

docs/api-mediated-access.md

@@ -1,4 +1,14 @@
-# API-Mediated Access (Redis Lockdown)
+# API-Mediated Access (Redis Lockdown → Signed Claims)
+
+> **meta-core v2.0.0 — the claims model.** The implementation basis is
+> [`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md)
+> (meta-root). Where this file and that one disagree, that one wins. The
+> *access* contract below still holds unchanged — HTTP only, events over
+> SSE, no database reachable from outside `metacore-app`. The *write*
+> contract changed: every metadata write is a **signed claim** posted to
+> `POST /api/claims`, and the legacy unsigned write endpoints answer
+> `410 Gone`. See ["Write contract — signed claims"](#write-contract--signed-claims-v200)
+> below.
 
 ## Status
 
@@ -9,6 +19,16 @@ on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
 meta-core is the single writer to the metadata keyspace — an assumption
 that's now actually enforced rather than aspirational.
 
+> **2026-10: meta-core no longer uses Redis at all.** Its store is now an
+> embedded SQLite file (`/meta-core/db/meta-core.db`, see
+> [metadata-storage-structure.md](metadata-storage-structure.md)) and the
+> `file:events` / `meta:events` streams are rows of that store. The lockdown
+> below is now structural — there is no database port anywhere — and the
+> SSE wire contract was kept byte-for-byte, so no consumer changed. The
+> rest of this doc is the design record of the Redis-era lockdown; the
+> sections that describe *current* behaviour (SSE retention, reconnect,
+> error handling) have been updated in place.
+
 See "Implementation outcome" at the bottom for the delta between this doc's
 plan and what shipped (notably the write-path CID resolution and the
 dual-root migration sweep, which weren't in the original plan but turned
@@ -59,20 +79,23 @@ we don't fully control.
 ## The architecture in one picture
 
 ```
-                meta-core (only Redis writer + reader)
+                meta-core v2.0.0 (only store writer + reader)
                 ┌──────────────────────────────────────┐
                 │  HTTP API surface:                   │
-                │    POST/PUT/PATCH /api/metadata/*    │  ◄── writes
+                │    POST /api/claims                  │  ◄── writes (signed claims only)
-                │    GET            /api/meta/{cid}    │  ◄── reads (by CID)
+                │    POST /api/merge                   │  ◄── stateless merge, no write
-                │    GET            /api/metadata/*    │  ◄── reads (by id)
+                │    GET  /api/meta/{cid}              │  ◄── reads (by CID, merged)
-                │    GET            /api/file/{cid}    │  ◄── content
+                │    GET  /api/meta/{cid}/claims       │  ◄── reads (raw signed claims)
-                │    GET            /api/events/files  │  ◄── SSE stream
+                │    GET  /api/metadata/*              │  ◄── reads (by rid, merged)
-                │    GET            /api/events/meta   │  ◄── SSE stream
+                │    GET  /api/file/{cid}              │  ◄── content
+                │    GET  /api/events/files            │  ◄── SSE stream
+                │    GET  /api/events/meta             │  ◄── SSE stream (merged changes)
                 │     │                                │
-                │     ▼                                │
+                │     ▼  verify sig → truth → apply    │
-                │  Redis  (file:*, cid:*, streams)     │
+                │  truth  /meta-core/db/meta-core.db   │
-                │  ◄── container-local, no exposed     │
+                │  cache  /meta-core/db-cache/cache.db    │
-                │      port, no docker-network alias   │
+                │  ◄── embedded SQLite, no port,       │
+                │      no docker-network alias         │
                 └──────────────────────────────────────┘
                           ▲             ▲
                           │ HTTP        │ HTTP SSE
@@ -85,9 +108,18 @@ we don't fully control.
 ```
 
 **The single-writer rule.** meta-core has exclusive write authority over
-the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
+its store — in v2.0.0 the two SQLite databases (truth and cache); in the
-other service can write — not because of an honoured convention, but
+Redis era the `file:*`, `cid:*`, `file:__index__` (later `idx:*`), and
-because they can't reach Redis at all.
+`meta-core:*` keyspaces. No other service can write the store — not because
+of an honoured convention, but because they can't reach it at all.
+
+**What changed in v2.0.0 is who may *author* metadata.** meta-core is still
+the only process that touches the database, but it no longer accepts
+unsigned values: a writer submits signed votes (claims), meta-core verifies
+each signature and stores the vote. meta-core **never signs a claim and
+holds no writer key** — a store that could sign for everyone would prove
+nothing. The signature is the write authentication; there are no
+per-writer tokens.
 
 **Stream reads also mediated.** Events go out through SSE endpoints on
 meta-core's HTTP surface. Consumers don't see the underlying Redis
@@ -98,6 +130,147 @@ without breaking every consumer.
 
 **Redis is private.** No host port mapping, no docker-network alias, no
 URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
+(Since 2026-10 there is no Redis at all; the SQLite files are reachable only
+from inside `metacore-app`.)
+
+## Write contract — signed claims (v2.0.0)
+
+Every metadata value is a **signed vote**: *signer `uid` says `key = value`
+about content `cid`* — `claim(uid, cid, key) → value + ts + sig`. The
+record a reader sees is the **merged view** that meta-core precomputes from
+the votes; a reader's own votes always win in their view. Full model,
+schema and merge rules: the
+[implementation basis](../../../docs/implementation/metacore-claims-model.md)
+§1–§7.
+
+### Endpoint table
+
+| | Endpoint | Behaviour |
+|---|---|---|
+| **Write** | `POST /api/claims` | A batch of signed claims (a JSON array, or `{"claims": [...]}`). **The only metadata write path.** |
+| Removed | `PUT/PATCH/DELETE /meta/{hash}`, `PUT/DELETE /meta/{hash}/{key...}`, `POST /meta/{hash}/_add/{key...}`, `PUT/DELETE /api/metadata/{hashId}`, `PUT /api/metadata/{hashId}/property`, `POST /api/metadata/batch` | `410 Gone`, with a pointer to `POST /api/claims` |
+| Kept | `PUT/GET/DELETE /api/kv/value` | Service scratch KV (`btmagnet:*`, `searchcov:*`), plain rows of the cache database's `kv` table (any key outside the reserved prefixes `file:`, `cid:`, `idx:`, `dup:`, `ev:`, `evhwm:`, `meta:`, `meta-core:`) — losing it only costs a refetch |
+| Kept | `POST /api/files/register`, the file watcher | Fill `local_file` (box-local `filePath`, size, mtime, midhash — unsigned) and `cid_group` |
+| UDL facade | `/api/udl/*` (same paths) | Writes go through the claims write path; reads come from the claims (per-signer index `claim_by_pid`). Records are MMUR2 claims. Shapes: see "UDL facade shapes" below |
+| Read | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{hash}`, `GET /api/metadata/hash-ids`, `GET /api/metadata/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `GET /api/schema`, `GET /api/stats` | Unchanged paths and response shapes, served from the merged view (+ `local_file`); optional `?as=<uid>`. They carry no `nsrc` |
+| **New** | `GET /api/meta/{cid}/claims` | `{rid, cids, claims}`: the record's rid, the cids of its group and every live signed claim on them (all signers, tombstones excluded). `?key=` keeps one exact key, `?uid=` one signer. `404 unknown_cid` when no record holds the cid |
+| **New** | `GET /api/metadata/{rid}/claims` | Same body and filters, addressed by rid (a cid is also accepted); `404` for an unknown record. What meta-search ingest and the gateway's corpus search use |
+| **New** | `POST /api/merge` | Stateless: body `{claims, as?}`; verifies each claim (invalid ones dropped and counted, never a batch error), merges with the store's merge rule, overlays `as`'s own votes (in the request and held by this core for the same cids). Returns `{fields, nsrc, rejected}` — merged `key → value`, `key → agreeing signers`, rejected count. `413` above 10 000 claims. Writes nothing |
+| Events | `GET /api/events/meta` | Unchanged `{type: set\|del, key, ts}` shape, now emitted for changes of the **merged** view, `key` still the literal `file:<rid>/<field>`; `GET /api/events/files` unchanged. Both streams are rows of the cache `kv` table (`ev:meta:events/…`, `ev:file:events/…`), ~100k entries each |
+| Admin | `POST /api/metadata/clear` | Wipes both databases (truth and cache) |
+| Removed | `/api/snapshot/export`, `/api/snapshot/import`, `/api/snapshot/wipe` | No snapshot format in v2.0.0 |
+
+Endpoints the claims model does not touch keep their behaviour: discovery
+(`/health`, `/status`, `/leader`, `/urls`, `/neighbors`), the account-key
+identity service (`/api/identity/*` — its removal is a separate ticket;
+v2.0.0 adds the `format` field to `sign` / `sign-batch` — `"compact"` or
+`"der"`, default compact for `"MMUR2\n"` payloads and DER otherwise — and
+makes `DELETE /api/identity` sign and store a retraction of every live
+claim of the account before removing it),
+`/api/file/{cid}/info`, `/file/cid`, `/data/{hash}*`, mounts, watchers and
+WebDAV.
+
+### The claim on the wire (JSON form)
+
+```json
+{ "cid": "bafy…", "uid": "zQ3s…", "key": "title",
+  "value": "The Matrix",            // or {"b64": "…"} for bytes, or null
+  "ts": 1793001234567,
+  "sig": "base64(64 bytes)" }
+```
+
+| Field | Meaning |
+|---|---|
+| `cid` | The subject: a content CID as the writer knows it (bare CID string, as in `cids/*`). Never a rid — rids are local and never signed |
+| `uid` | The signer: `"z" + base58btc(33-byte compressed secp256k1 pubkey)` |
+| `key` | The field (`METADATA_KEYS.md`); a key-set member is inside the key (`genres/Drama`) |
+| `value` | Text = public plaintext; `{"b64": …}` = bytes, a private (ciphertext) envelope; `null` = retraction |
+| `ts` | Milliseconds since epoch. It **is** the version: only increases per `(uid, cid, key)` |
+| `sig` | MMUR2: secp256k1 ECDSA, compact 64 bytes, over SHA-256 of `"MMUR2\n" ‖ canonical CBOR [cid, uid, key, value, ts]` |
+
+A batch (the `POST /api/claims` body) is a JSON array of these (or
+`{"claims": [...]}`). Kamilata and
+the gateway wire carry the same five fields plus `sig`; the signed bytes
+never change on any hop, and relays (meta-search, a client persisting a
+gateway hit) forward claims verbatim — they never re-sign.
+
+### Per-claim statuses
+
+`POST /api/claims` answers with one status per claim:
+
+| Status | Meaning |
+|---|---|
+| `accepted` | Verified and stored (or a retraction stored as a tombstone) |
+| `unchanged` | The stored row for `(uid, cid, key)` already has this `ts` and `sig` — **or the same value under any `ts`** (a signer re-asserting what it holds): no verification, no write, and the stored row keeps its older `ts` |
+| `stale` | The stored `ts` is ≥ the incoming `ts`; ignored (blocks replay of old votes) |
+| `bad_signature` | The MMUR2 signature does not verify against the `uid`'s public key |
+| `bad_request` | The claim is malformed |
+
+The response is `200` with a JSON array of `{status, reason?}`, one entry
+per claim, in input order, even when some are refused. A batch above 10 000
+claims answers `413 too_many_claims`; a body that is not claims JSON
+`400 invalid_json`. The
+removed write routes answer `410 {"error":"gone","use":"POST /api/claims"}`.
+`POST /api/claims`, `POST /api/merge` and the two claims read routes are
+container-to-container routes, not routed through Caddy.
+
+Rules: every claim is signed — ciphertext and retractions included; an
+unsigned or badly signed claim is refused. A retraction (`value: null`) is
+kept as a tombstone, so a peer cannot replay the vote it retracted. Values
+are stored exactly as signed; normalisation (trim + case-fold, no NFC)
+happens only inside the merge.
+
+Accepted claims commit to the truth database in one group-committed
+transaction; an apply loop then folds them, in commit order, into the
+cache (the merged `file:<rid>/<field>` rows of its v1-layout `kv` table,
+FTS, counters, events). `POST /api/claims` answers only once the cache has
+applied the batch, so the writer reads its own write; another reader may
+briefly see the previous merged value — the cache is behind, never wrong.
+
+### `?as=<uid>` — the viewer's overlay
+
+Every merged read accepts an optional `?as=<uid>`. The merge then puts that
+signer's own live text vote first for each key (read-time overlay found
+through the per-signer index, never stored in the shared merged view). Without
+`?as=` the read returns the shared merge: most distinct signers → oldest
+`ts` → byte-wise smallest value; ciphertext never merges.
+
+### Raw claims and client-side merging
+
+The claims read routes (`GET /api/metadata/{rid}/claims` for meta-search
+ingest and the gateway's corpus hits, `GET /api/meta/{cid}/claims` by cid)
+return the raw signed votes of every signer, not a merged map. A client that receives
+claims from the swarm or a gateway displays them by calling its **local**
+meta-core's `POST /api/merge` (one merge implementation, in Go), and
+persists a hit by posting the claims **as received** to `POST /api/claims`.
+
+### UDL facade shapes
+
+Claims in the JSON form above (`internal/api/udl.go`):
+
+| Route | Request | Response |
+|---|---|---|
+| `PUT /api/udl/record` | one claim | `200 {accepted: true, status}` for `accepted` / `unchanged`; `409 stale_version` for `stale`; `400` (status as slug) for `bad_signature` / `bad_request` |
+| `PUT /api/udl/records` | `{records: [claim, …]}` | `200 {results: [{key, accepted, status, error?}, …]}`, input order; `accepted` = `accepted` or `unchanged` |
+| `GET /api/udl/record?uid=&cid=&key=` | — | `{exists, claim}` — the stored claim, tombstone included; `exists` false when absent or a tombstone |
+| `GET /api/udl/user/{uid}/key/{key}`, `/user/{uid}/cid/{cid}` | — | `{entries: [claim, …]}` |
+| `GET /api/udl/user/{uid}/all?after=&limit=` | — | `{entries: [claim, …], nextCid}` |
+| `GET /api/udl/user/{uid}/count` | — | `{cids}` |
+| `GET /api/udl/cid/{cid}/users` | — | `{users: [uid, …]}` |
+| `GET /api/udl/cid/{cid}/aggregate?keys=` | — | `{aggregates: [{key, count, avg, trues, sample_size}]}` |
+| `GET /api/udl/users/stats` | — | `{users: {uid: {records, cids, keys}}}` for **every signer** with live claims — plugin, feeder, gateway and service keys included. `/api/stats` `udlUsers` is the size of this map |
+
+### What a writer needs
+
+Each writing container (every meta-sort plugin, every self-publishing
+feeder, gateway core, meta-sort, meta-share, meta-search) signs with its
+**own** key; users sign in the browser. meta-core holds none of these keys.
+Key configuration per signer: implementation basis §9.
+
+Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are not
+claims: the watcher and `POST /api/files/register` write them into the
+cache's `local_file` table, they are joined into the record view unsigned,
+and they never appear in `/api/meta/{cid}/claims` or on the swarm.
 
 ## Eventing — HTTP Server-Sent Events
 
@@ -243,6 +416,36 @@ After PR A, the cursor contract is **"durable across meta-core restart
 and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
 when they persist a Last-Event-ID.
 
+**Current backing store (since 2026-10).** The streams are rows of
+meta-core's SQLite kv table — `ev:<stream>/<ms13>-<seq10>` → JSON of the
+entry's fields, plus a high-water mark `evhwm:<stream>` that seeds the next
+ID (so a `ClearStream` reset never reissues an ID an old cursor treats as
+seen). The contract above holds unchanged:
+
+- Entries persist across restart (they are in the database file).
+- Retention is ~100k entries per stream, trimmed every 10k appends.
+- IDs keep the Redis `<ms>-<seq>` form, so cursors persisted by meta-sort /
+  meta-share / meta-stremio before the switch still compare.
+- `meta:events` entries (`{type: set|del, key, ts}`) describe changes of
+  the **merged** view (v2.0.0): the apply loop emits `set`/`del` (keyed by
+  rid + key) only when a claim actually changed a merged value, in the same
+  cache transaction as that change. A group merge (two cid groups linked by
+  a `cids/*` vote) emits `del` for every key of the losing rid and `set` for
+  the survivor. *History:* v1.x appended them in the same transaction as the
+  `file:` key change; before 2026-10 a Redis keyspace-notification hop
+  (PSUBSCRIBE in `MetaPublisher`) could drop events when it lagged.
+- **Cache rebuild (v2.0.0).** Events live in the cache database. A cache
+  rebuild (missing cache folder, or catch-up after an unclean shutdown)
+  emits no per-key events; SSE consumers get `event: gap` and take a
+  snapshot, as they do for a trimmed cursor. Event IDs stay time-based, so
+  they keep rising across a rebuild.
+- **`Last-Event-ID: 0` / `0-0` on `/api/events/meta` returns a snapshot**
+  (since v1.2.0): one `set` per existing record key, then the live tail from
+  the head captured before the snapshot. Only the last snapshot event carries
+  an `id:`. This replaced the boot-time republish of every field, which a
+  ~100k-entry stream could never hold. A trimmed (non-zero) cursor still gets
+  the `gap` event and a replay of what is retained.
+
 ### Heartbeats
 
 Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
@@ -254,10 +457,13 @@ per the SSE spec.
 
 Per the SSE spec the client (or `EventSource`) auto-reconnects after a
 network error, sending `Last-Event-ID`. The server resumes from the next
-entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
+entry after that ID (a range read of the stream's rows; under Redis this
+was `XREAD … STREAMS file:events <id>`). With no `Last-Event-ID` the stream
+starts at the head, resolved once at connect; `0` / `0-0` go through the
+gap path and replay from the beginning.
 
-**Gap handling.** If the requested ID has been trimmed out of the Redis
+**Gap handling.** If the requested ID is older than the oldest retained
-Stream by retention, the server emits one synthetic event before resuming
+entry, the server emits one synthetic event before resuming
 from the oldest available entry:
 
 ```
@@ -271,9 +477,10 @@ react. Consumers that don't care ignore it.
 
 ### Error handling
 
-- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
+- **Store unavailable on connect**: 503 with `Retry-After: 5`.
-- **Redis fails mid-stream**: close the SSE connection. Client reconnects
+- **Store read fails mid-stream**: close the SSE connection. Client
-  with `Last-Event-ID`; the cycle resumes when Redis recovers.
+  reconnects with `Last-Event-ID`; the cycle resumes when meta-core is back
+  (a meta-core restart is the common case — SSE clients just reconnect).
 - **Client disconnects**: handler goroutine exits cleanly; no per-client
   state on meta-core.
 
@@ -307,7 +514,8 @@ inside-only until a concrete external consumer appears.
 ### Implementation footprint
 
 The SSE handler itself is ~80 lines of Go in meta-core: one handler per
-stream, each running an `XREAD … BLOCK 30s` loop and writing
+stream, each running a read-after-cursor loop (an `XREAD … BLOCK 30s` loop
+in the Redis era) and writing
 SSE-formatted lines to the response. Go's `net/http` natively supports
 streaming responses with `http.Flusher.Flush()`. Standard SSE clients
 exist for every language we use: Node `eventsource`, Rust
@@ -484,6 +692,10 @@ durability across a meta-core restart).
 
 ### PR B — meta-sort writes via HTTP
 
+> *History.* PR B moved writes onto the unsigned `/meta/{hash}` family.
+> In v2.0.0 those routes answer `410 Gone`; writers sign claims and post
+> them to `POST /api/claims` (see "Write contract — signed claims").
+
 Swap `RedisClient` for an `HttpKVClient` implementation of the existing
 `IKVClient` interface in `packages/meta-sort-core`. All writes
 (`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
@@ -562,17 +774,22 @@ metacore-app produces a connection refused. The lock is real.
 | Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
 | Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
 | Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
-| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
+| Backing store for streams? | Redis Streams at the time; SQLite kv rows since 2026-10 | SSE wraps it; the 2026-10 swap proved it — no consumer changed |
 | Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
 | Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
-| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
+| **Canonical write API path?** | **`POST /api/claims` (v2.0.0)** — signed claims only; every unsigned write route answers `410 Gone` | *History (v1.x):* the **`/meta/{hash}` family** was the canonical write surface. It had GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
-| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
+| **SSE event-id contract?** | **Opaque to consumers** | It is `<ms>-<seq>` (the Redis Stream entry form, kept after the 2026-10 SQLite switch); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
 | **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
 | **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
 | **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
 
 ## Error envelope
 
+(`410 Gone` from the removed v1.x write routes carries this envelope with a
+message pointing to `POST /api/claims`. Per-claim outcomes of
+`POST /api/claims` are statuses inside the response, not envelopes — see
+"Per-claim statuses" above.)
+
 Standard shape for all 4xx / 5xx responses from the metadata API:
 
 ```json
@@ -591,7 +808,11 @@ Stable `error` slugs the client should switch on:
 | `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
 | `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
 | `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
-| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
+| `invalid_json` | 400 | no | `POST /api/claims` / `POST /api/merge` body is not claims JSON. Caller bug. |
+| `too_many_claims` | 413 | no | More than 10 000 claims in one `POST /api/claims` or `POST /api/merge`. Split the batch. |
+| `stale_version` | 409 | yes | `PUT /api/udl/record`: an equal or newer `ts` is stored for this signer, cid and key. Re-read the record, sign with a newer `ts`, retry. |
+| `gone` | 410 | no | A removed unsigned write route. Body is `{"error":"gone","use":"POST /api/claims"}` (no `message` / `retryable`). |
+| `storage_unavailable` | 503 | yes | Store transient (Redis at the time; the embedded SQLite store since 2026-10). Caller should retry with backoff. |
 | `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
 
 **Implemented.** Every 4xx/5xx response across `internal/api`,
@@ -605,10 +826,9 @@ import cycle on the api package; consolidating it into a shared
 
 Mostly answered post-implementation:
 
-- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
+- **Deprecation timeline for `/api/metadata/{hashId}`.** Answered by
-  editor's React UI continues to call it; the surface itself was untouched
+  v2.0.0: its `PUT`/`DELETE` (and `PUT …/property`) answer `410 Gone`
-  by this work. Worth a follow-up to consolidate the editor onto
+  like every unsigned write; `GET` stays as a merged read.
-  `/meta/{hash}` and delete the parallel routes.
 
 - **Initial bootstrap flood on `/api/events/files`.** Resolved in
   practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
@@ -628,8 +848,9 @@ Mostly answered post-implementation:
 - **Authentication between internal services.** The hash-lock OIDC
   perimeter handles external clients; internal service-to-service
   trust is by network isolation (docker network for dev, equivalent
-  isolation in prod deployments). Adding service-to-service auth is a
+  isolation in prod deployments). Since v2.0.0 metadata *writes* carry
-  separate concern.
+  their own authentication — the claim signature — so meta-core needs no
+  per-writer tokens; transport-level service auth remains out of scope.
 - **Replacing the public HTTP API surface.** This proposal exercises
   what already exists; the only new endpoints are the two SSE streams.
 - **Introducing new transport, broker, proxy, or sidecar.** Everything
@@ -678,6 +899,11 @@ What actually shipped vs. what this doc planned.
 
 ### Write-path CID resolution (`storage.ResolveRoot`)
 
+> *History (v1.x).* In v2.0.0 a claim names a cid, never a rid, and the
+> hidden rid comes from `cid_group` (a `cids/<other>` vote links two
+> groups; the older rid survives). The dual-root failure below cannot
+> recur: there is no write that addresses a root.
+
 Not in the original plan but load-bearing.
 
 **The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed
@@ -709,7 +935,9 @@ contract violation."
    One-shot fixer for already-stranded entries: walks every UUID's
    `midhash256` field to build the orphan index, merges each stranded
    `file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
-   deletes the stranded root, and re-points the alias.
+   deletes the stranded root, and re-points the alias. *Removed in
+   2026-10* with the SQLite switch, together with the other one-shot Redis
+   migrations; it had been run on watch.nsl.sh before the migration.
 
 Verified on the dev stack: 48 stranded entries were reunited; the
 editor now shows every file with its full plugin metadata under one
@@ -726,9 +954,10 @@ duplicated in each to avoid an import cycle on the `api` package.
 
 - **meta-dup SSE + read migration.** Not deployed, no test harness.
   When revived, copy meta-fuse's pattern.
-- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate
+- **Editor writes (resolved in v2.0.0).** `PUT /api/metadata/{hashId}`
-  editor-internal API surface is untouched; could be retired in a
+  answers `410`. meta-sort-editor's manual edits go through meta-sort's
-  follow-up by porting the editor UI to `/meta/{hash}`.
+  backend, which signs them as meta-sort (its own key) and posts them to
+  `POST /api/claims`. The editor's reads keep using `/api/metadata/{hashId}`.
 - **Consolidated `internal/httperr` package** for the typed envelope
   helpers (currently duplicated 4× to avoid the import cycle). Code
   hygiene; functionally a no-op.

docs/meta-core-architecture.md

@@ -1,16 +1,24 @@
 # Meta-Core Architecture
 
+> **meta-core v2.0.0 — the claims model.** Implementation basis:
+> [`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md)
+> (meta-root). Where this file and that one disagree, that one wins.
+
 ## Document Purpose
 
 This document describes the architectural design of **meta-core**, the
-standalone container that owns Redis, the leader-info file, the HTTP+SSE
+standalone container that owns the metadata store (two embedded SQLite
+databases since v2.0.0 — *truth* and *cache*; a single SQLite file in
+v1.x; Redis until 2026-10), the leader-info file, the HTTP+SSE
 metadata surface, and a WebDAV file server. It covers the leader gate,
-storage data model, API design, the SSE event surface, and the integration
+the claims data model and its write/apply pipeline, API design, the SSE
-contract with other MetaMesh services.
+event surface, and the integration contract with other MetaMesh services.
 
-For the on-disk Redis layout, see
+For the engine design (SQLite, group commit, measurements), see
+[metacore-storage-engine.md](../../../docs/project-architecture/metacore-storage-engine.md);
+for the on-disk layout (tables of both databases), see
 [metadata-storage-structure.md](metadata-storage-structure.md). For the
-external-access design (Redis lockdown, SSE wire contract, auth rules),
+external-access design (store lockdown, SSE wire contract, auth rules),
 see [api-mediated-access.md](api-mediated-access.md). For the UUID-rooted
 storage rationale, see [uuid-rooted-metadata.md](uuid-rooted-metadata.md).
 
@@ -23,7 +31,7 @@ view of file metadata:
 
 | Service | Language | Role |
 |---|---|---|
-| **meta-sort** | TypeScript | Watches folders, runs plugins, writes metadata via meta-core's HTTP API. |
+| **meta-sort** | TypeScript | Watches folders, runs plugins; each plugin signs its own claims and posts them to meta-core. |
 | **meta-fuse** | TypeScript / Rust | Serves the virtual filesystem from metadata. |
 | **meta-stremio** | Python | Stremio addon for HLS streaming. |
 | **meta-dup** | TypeScript | Duplicate detection. |
@@ -31,18 +39,30 @@ view of file metadata:
 
 meta-core sits below all of these as the single owner of:
 
-1. **Redis** — the metadata store and event source.
+1. **The metadata store** — two embedded SQLite databases: **truth**
+   (`/meta-core/db/meta-core.db`, backed up) holding the signed claims, and
+   **cache** (`/meta-core/db-cache/cache.db`, never backed up) holding every
+   index and derived row — a v1-layout `kv` table (the merged view as
+   `file:<rid>/<field>` rows, lookup, counters, event streams, service
+   scratch KV) with its field index and FTS, plus the `claim_by_pid` and
+   `local_file` tables. (v2.0.0 is implemented, uncommitted and not yet
+   deployed as of 2026-10-25.)
 2. **The leader-info file** — the one place services look to discover meta-core's URLs.
 3. **The HTTP+SSE surface** — the only way external services touch the data plane.
 4. **A WebDAV server** — file access for plugins and other services.
 
 ### Core design principle
 
-> External services never speak Redis. Every read, write, and event flows
+> External services never touch the store. Every read, write, and event flows
 > through meta-core's HTTP+SSE surface.
 
 This is the contract that PR D of [api-mediated-access.md](api-mediated-access.md)
-landed and that the rest of this document assumes.
+landed and that the rest of this document assumes. v2.0.0 adds a second:
+
+> Every metadata value is a **signed vote** — `claim(uid, cid, key) → value`
+> + `ts` + `sig`. The only write is `POST /api/claims`. meta-core verifies
+> every claim and **never signs one**: it holds no writer key, and the
+> signature is the write authentication.
 
 ---
 
@@ -53,8 +73,8 @@ landed and that the rest of this document assumes.
 meta-core runs as a dedicated container (`metacore-app`). The dev compose
 runs a single instance; the legacy multi-instance topology (sibling
 followers) was deprecated. The flock primitive is retained as the
-correctness anchor — it guarantees that at most one process owns Redis and
+correctness anchor — it guarantees that at most one process writes the
-the writable state under `/meta-core`.
+database file and the writable state under `/meta-core`.
 
 ```
 ┌────────────────────────────────────────────────────────────────────────┐
@@ -62,17 +82,19 @@ the writable state under `/meta-core`.
 │                                                                        │
 │  supervisord                                                           │
 │   ├─ leader-election.sh   (flock → write kv-leader.info → exec self)   │
-│   ├─ redis-server         (AOF + RDB on /meta-core/db/redis)           │
 │   ├─ nginx                (TLS termination, proxy_cache for /webdav)   │
 │   ├─ rclone rcd           (mount daemon)                               │
 │   ├─ mount-watcher.sh     (rclone mount lifecycle)                     │
 │   └─ meta-core (Go) ──►   HTTP + SSE + WebDAV on :9000                 │
+│                           + embedded SQLite: truth (db/meta-core.db)   │
+│                             and cache (cache/cache.db)                 │
 └────────────────────────────────────────────────────────────────────────┘
                                     │
        ┌────────────────────────────┴────────────────────────────┐
        │  /meta-core (shared volume)                             │
        │    locks/kv-leader.{lock,info}                          │
-       │    db/redis/                                            │
+       │    db/meta-core.db (+ -wal, -shm)   truth, backed up    │
+       │    cache/cache.db  (+ -wal, -shm)   cache, never backed │
        │    services/*.json                                      │
        │    mounts/{mounts.json, errors/}                        │
        │    watchers.json                                        │
@@ -152,7 +174,7 @@ Container startup
        ▼
 ┌──────────────────────────────┐
 │ supervisord starts:          │
-│   redis → nginx → rclone →   │
+│   nginx → rclone →           │
 │   meta-core Go binary        │
 └──────────────────────────────┘
 ```
@@ -167,10 +189,11 @@ gate wins. Therefore:
   local hostname + IP + config and returns it on demand. There are no
   `OnBecomeLeader` / `OnBecomeFollower` / `OnLeaderLost` callbacks; those
   states do not exist inside the Go process.
-- `LeaderLockInfo.RedisUrl` is intentionally empty in the response from
+- `LeaderLockInfo.RedisUrl` is always empty in the response from
-  `/urls` — see `internal/leader/election.go:82`. PR D of
+  `/urls` (kept for wire compatibility). PR D of
-  api-mediated-access.md removed direct Redis exposure; consumers route
+  api-mediated-access.md removed direct Redis exposure, and since 2026-10
-  metadata I/O through meta-core's HTTP API and SSE event streams.
+  there is no Redis at all; consumers route metadata I/O through
+  meta-core's HTTP API and SSE event streams.
 
 ### Lock-info file
 
@@ -192,75 +215,174 @@ Other services read this file, then call `GET /urls` for full discovery
   `metacore-app` is provisioned, its bash loop will acquire on next tick.
   The dev compose runs a single instance; failover is "restart the
   container."
-- **Redis crash inside the container** → supervisord restarts redis. The
+- **The store is in-process** — there is no separate database process to
-  Go binary's connect loop retries up to 30 times (1s delay) at startup;
+  crash. A meta-core crash is a process restart; SQLite's WAL recovers the
-  at steady state, individual operations fail and clients retry.
+  last committed transaction on reopen.
 
 ---
 
 ## Schema-Version Gate
 
-Before serving any traffic, `cmd/meta-core/main.go:54-61` calls
+Before serving any traffic, `cmd/meta-core/main.go` opens both databases
-`storage.EnsureSchemaVersion`. The sentinel lives in Redis at
+and calls `storage.EnsureSchemaVersion` (before the cache catch-up). In
-`meta-core:schema-version` (`internal/storage/schema_version.go`). The
+v2.0.0 the sentinel is the `schema-version = "3"` row of the truth
-gate refuses to boot when the existing Redis layout predates the
+database's `meta` table. The gate:
-UUID-rooted schema, so legacy `file:midhash256:*` data can't leak into
+
-the current code paths. In alpha there is no automated migration —
+- **explicitly refuses a v1.x store** — a truth file holding the v1 `kv`
-operators wipe with `pnpm run clean:all` and restart.
+  table (`meta-core:schema-version = "2"`), which the default path still
+  points at — with a message naming `tools/claims-migrate`;
+- refuses a store with claims but no sentinel, or any other version;
+- stamps a fresh store with `"3"`.
+
+So no pre-claims data leaks into the current code paths. The meta-core
+**service binary contains no migration code**: watch.nsl.sh is converted
+once by the separate one-off tool `packages/meta-core/tools/claims-migrate`
+(not built into the image; implementation basis §11); every other box and
+dev stack starts empty (operators wipe with `pnpm run clean:all` and
+restart).
 
 ---
 
 ## Data Model
 
-### Two concepts: metadata vs. file bytes
+### The one concept: signed votes
 
 ```
-┌────────────────────────────────────────────────────────────────────┐
+claim(uid, cid, key) → value      + ts + sig
-│                       METADATA  (Redis)                            │
-│                                                                    │
-│  Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars)     │
-│  Storage: flat string keys                                         │
-│           file:<uuid>/<property>           → string value          │
-│           file:<uuid>/cids/<bareCid>       → "true" (key-set)      │
-│           file:<uuid>/duplicates           → SET of paths          │
-│                                                                    │
-│  No stored canonical_cid — derived by rank on read.                │
-│                                                                    │
-│  Aliases (reverse index, one Redis STRING each):                   │
-│           cid:<bareCid>                    → <uuid>                │
-│                                                                    │
-│  Index:   file:__index__                   → SET of all UUIDs      │
-│                                                                    │
-│  Mintable without a local file (remote / deleted entries OK).      │
-│  Resolvable by any registered CID in O(1).                         │
-└────────────────────────────────────────────────────────────────────┘
-
-┌────────────────────────────────────────────────────────────────────┐
-│                         FILE BYTES                                 │
-│                                                                    │
-│  Location: /files volume (shared across containers).               │
-│  Access:   path resolution via metadata, served via /webdav/*,    │
-│            /file/{cid} (reverse-lookup), or /data/{hash}.          │
-│  Path stored as filePath in the metadata, relative to /files.      │
-└────────────────────────────────────────────────────────────────────┘
 ```
 
-### Why UUID-rooted?
+- `uid` — the signer (a secp256k1 public key, `z…` base58btc). One key per
+  writing container (each meta-sort plugin, each self-publishing feeder,
+  gateway core, meta-share, meta-search) and one per user.
+- `cid` — the subject, a content CID; never a rid.
+- `key` — the field (`METADATA_KEYS.md`); key-set members live inside the
+  key (`genres/Drama`, `cids/<cid>`).
+- `value` — text (public), bytes (signed + encrypted envelope) or null
+  (retraction, kept as a tombstone).
+- `ts` — milliseconds; it **is** the version and only increases per
+  `(uid, cid, key)`.
+- `sig` — MMUR2: secp256k1 ECDSA (compact 64 bytes) over SHA-256 of
+  `"MMUR2\n" ‖ canonical CBOR [cid, uid, key, value, ts]`.
 
-Roots were previously content-hash tokens (`file:midhash256:abc/...`),
+The old shared record (`file:<rid>/<field>`) and the old User Data Layer
-which privileged one hash algorithm. UUID-rooted storage treats every CID
+store (`udl:rec:*`) both collapse into this shape. The store distinguishes
-symmetrically as an alias, makes `GET /api/meta/<cid>` O(1) for any
+only plaintext (signed) from ciphertext (signed + encrypted); "personal vs
-registered CID, and lets new digest types be added without rewriting root
+shared" is app logic.
-keys. See [uuid-rooted-metadata.md](uuid-rooted-metadata.md) for the
+
-design rationale and [metadata-storage-structure.md](metadata-storage-structure.md)
+### Two databases
-for the on-disk layout.
+
+```
+┌──────────────────────────────────────────────────────────────────────┐
+│     TRUTH  /meta-core/db/meta-core.db   (backed up; PKs only)        │
+│                                                                      │
+│  principal(pid, uid)              uid ↔ local surrogate pid          │
+│  claim(cid, key, pid → value, ts, sig, seq)                          │
+│                                   the votes; PK (cid, key, pid)      │
+│  cid_group(cid → rid)             hidden local record identity       │
+│  meta(name → value)               schema-version, last_seq           │
+└──────────────────────────────────────────────────────────────────────┘
+                     │  apply loop (seq order)
+                     ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│     CACHE  /meta-core/db-cache/cache.db   (never backed up; rebuildable)│
+│                                                                      │
+│  kv(k → v)   the v1 layout, every row derived:                       │
+│    file:<rid>/<field>             the merged view every read uses    │
+│    cid:<cid>                      cid → rid lookup                   │
+│    idx:roots/ idx:fts/ idx:count/ idx:schema/   roots, FTS rowid,    │
+│                                    counters (search totals, stats)   │
+│    ev:<stream>/…, evhwm:…         /api/events/meta + /files backlog  │
+│    meta:applied_seq, meta:rebuilt_at   apply-loop bookkeeping        │
+│    anything else                  /api/kv/value scratch KV           │
+│  kv_field_v (index), fts          filter + free-text search          │
+│  claim_by_pid(pid, cid, key)      "everything uid X wrote" (?as=,    │
+│                                    UDL facade, identity retraction)  │
+│  local_file(path → midhash, size, mtime)   watcher facts, unsigned   │
+└──────────────────────────────────────────────────────────────────────┘
+```
+
+Disaster recovery is "delete the cache, replay truth". `META_CORE_DB_PATH`
+names the truth file, `META_CORE_CACHE_PATH` the cache file. The cache keeps
+the v1 `kv` layout so every v1 reader (record reads, field index, FTS,
+counters, SSE + snapshot) runs unchanged on derived rows; `nsrc` is computed
+by the merge (`POST /api/merge`), not stored. Full schema: implementation
+basis §5.
+
+### Records: cids, groups and the hidden rid
+
+A vote never mentions a rid. A **rid** (UUIDv7, 26-char Crockford, as in
+v1.x) is a local name for a group of cids known to be the same content,
+held in `cid_group`:
+
+- A new cid with no link gets its own group and a fresh rid.
+- A `cids/<other>` vote on cid X links X's group and `<other>`'s group; the
+  group with the older rid survives, the losing rid gets a `del` event per
+  key and the survivor `set` events (no alias table). Retracting the link
+  does **not** split the group (phase 1); the record view synthesises
+  `cids/<cid> = "true"` for every cid of the group from `cid_group`.
+- Reading record R = every vote on every cid of R's group, merged per key;
+  within a group one signer counts once per key.
+- rids are never signed and never travel; consumers that hold them
+  (meta-sort, meta-fuse, meta-stremio) keep working because `cid_group`
+  lives in truth.
+
+### Merge
+
+Per `(rid, key)`, over the group's votes, each signer's newest vote counted
+once (`internal/storage/merge.go`, shared by the apply loop, `?as=` and
+`POST /api/merge`):
+
+1. the viewer's own live text vote (`?as=<uid>`, applied at read time,
+   never stored);
+2. retractions and ciphertext drop out — only text merges;
+3. most distinct signers for the normalised value (**trim + case-fold
+   only**, no NFC) — `nsrc` is that count (computed, not stored);
+4. tie → oldest `ts`;
+5. tie → byte-wise smallest value.
+
+The stored value is the winning vote's exact text. Key-sets need no special
+code (each member is a key, present while one vote supports it). A
+different kind of fact gets a different key, so a guesser (e.g.
+filename-parser) never competes with a lookup on the same key.
+
+### Box-local file facts
+
+`filePath`, size, `mtimeNano` and midhash are **not claims** — meta-core
+cannot sign, and the disk can always be rescanned. The file watcher and
+`POST /api/files/register` write them into the cache's `local_file` table
+(and the midhash into `cid_group`); they are joined into the record view
+unsigned and never reach the claims API or the swarm. When several local
+paths hold the same content, `filePath` is the lexicographically first one
+(size and mtime from the same row); a signed `sizeByte` claim wins over the
+on-disk size.
+
+### File bytes
+
+```
+Location: /files volume (shared across containers).
+Access:   path resolution via local_file, served via /webdav/*,
+          /file/{cid} (reverse-lookup), or /data/{hash}.
+Path:     filePath (from local_file) in the record view, relative to /files.
+```
+
+### History: UUID-rooted flat keys (v1.x)
+
+Until v2.0.0 a record was a set of flat keys under an opaque UUID root —
+`file:<uuid>/<property>`, `file:<uuid>/cids/<bareCid>` key-sets,
+`dup:<uuid>/<path>`, a reverse index `cid:<bareCid> → <uuid>` and a root
+index `idx:roots/<uuid>`, all in one kv table. Roots had previously been
+content-hash tokens (`file:midhash256:abc/...`); UUID roots treated every
+CID symmetrically as an alias. The rid of v2.0.0 is that UUID, kept hidden
+and local; migrated records kept their UUIDs. See
+[uuid-rooted-metadata.md](uuid-rooted-metadata.md) (the *why*) and the
+History note of [metadata-storage-structure.md](metadata-storage-structure.md).
 
 ### Path resolution
 
-`filePath` values stored in metadata are absolute paths under the `/files`
+`filePath` values are absolute paths under the `/files` volume (e.g.
-volume (e.g. `/files/watch/movies/Movie.2024.mkv`). The `/files` prefix is
+`/files/watch/movies/Movie.2024.mkv`). The `/files` prefix is itself a
-itself a per-host detail — moving the volume only requires re-mounting it
+per-host detail — moving the volume only requires re-mounting it at the
-at the same path inside the container.
+same path inside the container.
 
 ---
 
@@ -276,20 +398,19 @@ Bootstrap / discovery
   GET  /status
   GET  /leader
   GET  /urls
-  GET  /services         (alias: /api/services)
+  GET  /neighbors        (alias: /api/neighbors)
-  GET  /services/{name}
+  GET  /api/stats
-  GET  /services/cleanup/stats
 
-Metadata (primary surface for other services)
+Claims (the only metadata write path)
+  POST   /api/claims                 signed claim batch → per-claim status
+  POST   /api/merge                  {claims, as?} → {fields, nsrc, rejected}; no write
+  GET    /api/meta/{cid}/claims      {rid, cids, claims} of the cid's group (?key=, ?uid=; 404 unknown)
+  GET    /api/metadata/{rid}/claims  same, addressed by rid
+
+Metadata reads (merged view + local_file; optional ?as=<uid>)
   GET    /meta
   GET    /meta/{hash}
-  PUT    /meta/{hash}
-  PATCH  /meta/{hash}
-  DELETE /meta/{hash}
   GET    /meta/{hash}/{key...}
-  PUT    /meta/{hash}/{key...}
-  DELETE /meta/{hash}/{key...}
-  POST   /meta/{hash}/_add/{key...}
 
 CID-addressed (public, auth-bypassed at the perimeter)
   GET    /api/meta/{cid}
@@ -299,26 +420,47 @@ CID-addressed (public, auth-bypassed at the perimeter)
   HEAD   /data/{hash}
   GET    /data/{hash}/path
 
-Editor / KV / schema / snapshot
+Editor / KV / schema
   GET /api/metadata/hash-ids
   GET /api/files/tuples
+  POST /api/files/register        (fills local_file + cid_group)
   GET /api/metadata/list
   POST /api/metadata/search
-  POST /api/metadata/batch
+  POST /api/metadata/clear        (wipes both databases)
-  POST /api/metadata/clear
+  GET /api/metadata/{hashId}      (hashId = rid)
-  {GET,PUT,DELETE} /api/metadata/{hashId}
+  GET /api/metadata/{hashId}/property
-  {GET,PUT} /api/metadata/{hashId}/property
+  GET /api/kv/{info,keys,tree,search,find}   (read-only browser over the cache kv table)
-  GET /api/kv/{info,keys,tree,search,find,value}
+  {GET,PUT,DELETE} /api/kv/value  (service scratch keys only; reserved prefixes refused)
-  PUT /api/kv/value
+  GET /api/kv/key/{key...}                    (read-only)
-  DELETE /api/kv/value
-  GET /api/kv/key/{key...}
   GET /api/schema
   POST /api/schema/rescan
-  GET /api/snapshot/export
-  POST /api/snapshot/import
-  POST /api/snapshot/wipe
 
-SSE event streams (HTTP-mirror of Redis Streams)
+User Data Layer (facade over claims; records are MMUR2)
+  {GET,PUT} /api/udl/record
+  PUT /api/udl/records
+  GET /api/udl/user/{uid}/{key/{key},cid/{cid},all,count}
+  GET /api/udl/cid/{cid}/{users,aggregate}
+  GET /api/udl/users/stats        (every signer with live claims, plugin keys included)
+
+Account-key identity service (kept in v2.0.0; removal is a separate ticket)
+  {GET,DELETE} /api/identity      (DELETE signs + stores a retraction of every live claim first)
+  GET  /api/identity/{accounts,aead-key}
+  POST /api/identity/{challenge,generate,import,reveal,sign,sign-batch}
+                                  (sign/sign-batch `format`: compact | der; default
+                                   compact for "MMUR2\n" payloads, DER otherwise)
+
+Removed in v2.0.0 — answer 410 Gone, pointing to POST /api/claims
+  PUT/PATCH/DELETE /meta/{hash}
+  PUT/DELETE /meta/{hash}/{key...}
+  POST /meta/{hash}/_add/{key...}
+  PUT/DELETE /api/metadata/{hashId}
+  PUT /api/metadata/{hashId}/property
+  POST /api/metadata/batch
+
+Removed in v2.0.0 — no snapshot format
+  GET /api/snapshot/export, POST /api/snapshot/{import,wipe}
+
+SSE event streams (HTTP-mirror of the store's event streams)
   GET /api/events/files
   GET /api/events/meta
 
@@ -338,16 +480,51 @@ Watchers
   POST /api/watchers/{scan-all,reset-all}
   (deprecated: /api/scan/trigger, /api/scan/status — see watcher/handlers.go:32)
 
-Admin
+(The one-shot Redis-era admin migrations — `/api/admin/migrate-dual-roots`,
-  POST /api/admin/migrate-dual-roots
+`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes` —
-  POST /api/admin/sweep-literature-echoes   (dry-run unless ?apply=true;
+were removed with the 2026-10 SQLite switch; they had been run once on
-       strips query echoes off domain=literature fileType=card records —
+watch.nsl.sh beforehand.)
-       see storage/literature_echo_sweep.go)
+
+Report inbox (internal/reports, internal/api/reports.go) — bug reports and
+feedback the client apps' 🐛 buttons file; read on the dashboard's Reports page
+  POST   /api/reports            {app, note?, page?, route?, cid?, title?, uid?,
+                                  build?, filedAt?, log?, <any block>} → 201 {id}
+  GET    /api/reports            ?app=&status=open|triaged|closed&q=&before=<id>&limit=
+                                  → {reports: [summary…], apps, counts, more}
+  GET    /api/reports/{id}       full report incl. body
+  PATCH  /api/reports/{id}       {status?, triage_note?}
+  DELETE /api/reports/{id}
 
 WebDAV
   /webdav/...   (GET, PUT, DELETE, MKCOL, COPY, MOVE, PROPFIND)
 ```
 
+### Report inbox
+
+Client apps (meta-watch, meta-read) build a report in the browser and their
+server forwards it **server-to-server to the box's own core** — the uid is
+stamped by the app from its bearer token, never taken from the body. Apps that
+cannot reach the core spool the report locally and retry
+(`report_outbox.rs` in each app), so `filedAt` (when it was filed) and
+`received` (when meta-core stored it) can differ; the row's `ts` is `filedAt`
+when it is plausible (≤ 30 days old, ≤ 5 min in the future).
+
+- **Own file, `db/reports.db`.** Reports are diagnostics, not metadata: neither
+  the truth DB (signed claims only) nor the cache (thrown away on recovery).
+  `POST /api/metadata/clear` and a cache rebuild never touch it. A failure to
+  open it disables `/api/reports*` (503), never the core.
+- **meta-core does not interpret a report.** `app`, `build`, `uid`, `page`,
+  `route`, `cid`, `title`, `note` are lifted into columns for filtering; every
+  other top-level key is stored verbatim in `body` and rendered generically
+  (app-specific headlines live in `dashboard/src/lib/reportText.ts`). Bounds:
+  512 KiB per POST, note 1000 chars, log tail 200 × 600 chars, any other block
+  over 32 KiB replaced by `{truncated, bytes}`.
+- **Bounded.** `META_CORE_REPORTS_MAX` reports and `META_CORE_REPORTS_MAX_MB`
+  of bodies; on overflow closed reports are evicted first, then the oldest —
+  an open report is the last to go. Enforced on every insert and hourly.
+- **Auth:** perimeter-gated like the rest of `/api`; do not add it to the
+  hash-lock bypass. A report is viewing/reading history (cids, titles, a uid).
+
 No `/metrics` endpoint is exposed. Observability is via container logs +
 the `/health` and `/status` JSON.
 
@@ -365,17 +542,28 @@ peers and unauthenticated tooling can reach them):
 - (`/api/events/files`, `/api/events/meta` are also auth-bypassed but
   **should not** be exposed publicly via Caddy; they're inside-only.)
 
+Since v2.0.0 `/meta/*` serves reads only — its write verbs answer `410
+Gone` — so the bypass no longer exposes an unsigned write. Writes go to
+`POST /api/claims`, where the claim signature (not the perimeter) is the
+write authentication. `POST /api/claims`, `POST /api/merge` and
+`GET /api/meta/{cid}/claims` are container-to-container routes, not routed
+through Caddy, like the v1.x write routes were.
+
 See `docs/api-mediated-access.md` "Auth" for the rationale.
 
 ### SSE event streams — wire contract
 
 `/api/events/files` and `/api/events/meta` are SSE mirrors of the
-`file:events` and `meta:events` Redis Streams. The contract
+`file:events` and `meta:events` streams, stored as `ev:<stream>/…` rows of
+the kv table (`internal/kv/stream.go`). The contract
 (`internal/api/sse_events.go`):
 
-- One SSE event per Redis Stream entry.
+- One SSE event per stream entry.
-- `id:` is the opaque `<ms>-<seq>` Redis Stream entry ID — clients echo
+- `id:` is the opaque `<ms>-<seq>` entry ID (the Redis Stream form, kept so
-  it back via `Last-Event-ID` on reconnect.
+  cursors persisted before the 2026-10 switch still compare) — clients echo
+  it back via `Last-Event-ID` on reconnect. Without `Last-Event-ID` the
+  stream starts at the head (resolved once at connect); `0` / `0-0` replay
+  from the beginning.
 - `event:` is the `type` field of the underlying entry.
 - `data:` is the JSON of the rest of the entry.
 - Heartbeats: SSE comment `:keep-alive\n\n` every 30s of silence.
@@ -384,9 +572,46 @@ See `docs/api-mediated-access.md` "Auth" for the rationale.
   `event: gap` payload before resuming from the oldest available entry.
 
 External services use SSE with `Last-Event-ID` as the resume primitive.
-A Redis consumer-group (XREADGROUP / XACK) exists *internally* — the
+Since v2.0.0 `meta:events` describes changes of the **merged** view, same
-`MetaPublisher` uses one to drain keyspace notifications — but it is not
+`{type: set|del, key, ts}` shape: the cache engine appends an
-the external contract.
+`ev:meta:events/…` row for every change of a merged `file:` row, in the same
+cache transaction as the change (both streams are v1 stream rows of the
+cache `kv` table). A cache rebuild
+emits no per-key events — consumers get `event: gap` and take a snapshot;
+event IDs stay time-based and keep rising. *History:* v1.x appended them in
+the same transaction as the `file:` key change; the Redis-era
+keyspace-notification hop through `MetaPublisher` is long gone.
+
+### Write path and apply loop
+
+```
+POST /api/claims ──► per claim, before the writer:
+                       0. validate (required fields, reserved keys, ts range) → else bad_request
+                       1. same ts + sig as stored?       → unchanged (no verify)
+                       2. same value as stored (any ts)? → unchanged; stored row + older ts kept
+                       3. stored ts ≥ incoming ts?       → stale
+                       4. verify MMUR2 signature         → else bad_signature
+                 ──► accepted claims, one group-committed truth transaction:
+                       5. re-check 1–3 under the writer
+                       6. uid → pid (principal; insert on first sight)
+                       7. upsert claim (tombstone if null), seq = ++last_seq
+                       8. cid_group: mint rid for unknown cid; text cids/* links groups
+                          (a retraction never splits)
+                       9. commit (last_seq in truth meta); answer once the cache applied it
+                 ──► apply loop, in seq order, into the cache:
+                       sync cid:/cids/<cid>/idx:roots rows → claim_by_pid
+                       → re-merge (rid, key) from truth PK range reads
+                       → set/delete file:<rid>/<key> (+ local_file for filePath/sizeByte/mtimeNano)
+                         ⇒ meta:events row + counter delta in the same tx
+                       → FTS row on a haystack change → meta:applied_seq
+```
+
+The verifier lives in the write path (secp256k1 from `internal/identity`);
+meta-core holds **no** writer key and never signs a claim. Response: one
+status per claim — `accepted` · `unchanged` · `stale` · `bad_signature` ·
+`bad_request`. Request/response shapes and the 410 contract for removed
+routes: [api-mediated-access.md](api-mediated-access.md) "Write contract —
+signed claims".
 
 ### WebDAV URLs
 
@@ -416,7 +641,8 @@ WebDAV caching is handled by nginx's `proxy_cache` (see
 | Single binary | Static compilation |
 | Low resource | ~10-20 MB resident |
 | HTTP server | stdlib `net/http` + gorilla/mux |
-| Redis client | go-redis |
+| Metadata store | embedded SQLite via `modernc.org/sqlite` (pure Go, CGO off) — truth + cache databases |
+| Claim verification | secp256k1 ECDSA (`internal/identity`) |
 | WebDAV | `golang.org/x/net/webdav` |
 
 ### Repo layout
@@ -429,12 +655,12 @@ packages/meta-core/
 │   ├── cid/                      CID parsing + algorithm ranking
 │   ├── config/                   env-driven configuration
 │   ├── discovery/                service registry + dead-service cleaner
-│   ├── events/                   meta:events stream publisher
+│   ├── identity/                 secp256k1 (claim verification; account-key service)
+│   ├── kv/                       SQLite engine (single writer, group commit) + event streams
 │   ├── leader/                   LeaderInfoProvider (no election logic)
 │   ├── mounts/                   rclone mount manager + handlers + stats
-│   ├── schema/                   live per-field schema indexer
+│   ├── schema/                   per-field schema from transactional counters
-│   ├── snapshot/                 snapshot export / import / wipe
+│   ├── storage/                  claims write path, apply loop, merge, cid_group, search, schema gate
-│   ├── storage/                  Redis wrapper + UUID + CID reverse index + schema gate
 │   ├── watcher/                  scanner + dispatcher + midhash + state
 │   ├── watchers/                 polling-based watcher configs
 │   └── webdav/                   /webdav/* handler
@@ -455,21 +681,29 @@ packages/meta-core/
 ```
 1. Load configuration from environment
 2. Build LeaderInfoProvider          (no election; bash gate already won)
-3. Connect to local Redis            (retry up to 30× / 1s)
+3. Create the cache folder; open the truth database (META_CORE_DB_PATH or
-4. EnsureSchemaVersion               (abort if Redis layout is stale)
+   /meta-core/db/meta-core.db) and the cache (META_CORE_CACHE_PATH or
-5. Start service discovery + dead-service cleaner
+   /meta-core/db-cache/cache.db)     (storage.Connect)
-6. NewServer + Server.Start          (boots watcher, mounts, WebDAV, API)
+4. EnsureSchemaVersion               (abort on a v1 kv store or an unknown layout)
-7. On first connection: start MetaPublisher + schema Indexer,
+5. storage.Start — cache catch-up    (meta:applied_seq ≥ last_seq → nothing; behind →
-   republish existing metadata onto meta:events
+                                       replay claims with seq > applied_seq, or a full
-8. Wait for SIGINT/SIGTERM
+                                       rebuild if a cids/* link is among them; no
-9. Shutdown in reverse order
+                                       cache → full rebuild), then the apply loop
+6. Derived counters ready
+7. Start service discovery + dead-service cleaner
+8. NewServer + Server.Start          (boots watcher, mounts, WebDAV, API)
+9. Wait for SIGINT/SIGTERM
+10. Shutdown in reverse order
+
+(v1.x ran `EnsureCounters` and a background FTS rebuild here; in v2.0.0
+counters and FTS are cache rows maintained with every merged-row change.)
 ```
 
 ### Process management
 
-Redis is not spawned by the Go binary; supervisord starts it before
+There is no database process: the store is embedded in the Go binary
-meta-core (`docker/supervisord.conf`). The Go binary connects to
+(`docker/supervisord.conf` runs meta-core, nginx and rclone). Until
-`localhost:6379` and retries until Redis is reachable.
+2026-10 supervisord also ran `redis-server` on `localhost:6379`.
 
 ---
 
@@ -477,15 +711,31 @@ meta-core (`docker/supervisord.conf`). The Go binary connects to
 
 ### Container restart
 
-Supervisord restarts crashed processes (meta-core, redis, nginx, rclone)
+Supervisord restarts crashed processes (meta-core, nginx, rclone)
 individually. The bash flock is held by supervisord's process tree, so
 internal crashes don't trigger re-election.
 
-### Redis crash
+### Store durability
 
-Supervisord restarts redis. The Go binary's connect retry handles the
+Both SQLite files run in WAL mode; truth with `synchronous=NORMAL` (every
-window. Persistence is RDB + AOF on the `/meta-core` volume, so no data is
+committed transaction survives a meta-core process crash), the cache with
-lost across restarts.
+`synchronous=OFF` (it is rebuildable). A claim batch
+commits to truth in one transaction; the apply loop then commits the
+derived cache changes. **A crash between the two commits leaves the cache
+behind, never wrong**: at boot, if the cache's `applied_seq` is below
+truth's `last_seq`, meta-core scans truth's `claim` table for
+`seq > applied_seq` (a full scan — there is no seq index; it only happens
+after an unclean shutdown) and applies them — or rebuilds the cache in full
+if one of them is a `cids/*` link.
+
+### Cache loss
+
+The cache is never backed up and is fully derived (except `local_file`,
+which a rescan of `/files` refills, and the service scratch keys, whose
+loss costs a refetch). A missing cache folder (no `meta:applied_seq`) means
+meta-core rebuilds every derived row from truth and stamps
+`meta:rebuilt_at`. A rebuild emits no per-key events;
+SSE consumers get `event: gap` and re-snapshot.
 
 ### Container crash
 
@@ -496,8 +746,9 @@ the container."
 ### Schema mismatch
 
 `EnsureSchemaVersion` refuses to boot. Operators wipe with
-`pnpm run clean:all` (or equivalent) and restart. This is an alpha-stage
+`pnpm run clean:all` (or equivalent) and restart, or — once, on
-hard-fail; there is no automated migration path.
+watch.nsl.sh — convert a v1.x store with `tools/claims-migrate`. This is an
+alpha-stage hard-fail; the service never migrates automatically.
 
 ### Shared volume unavailable
 
@@ -518,7 +769,14 @@ via `/health`; metadata I/O and SSE event streams stop.
 | `SERVICE_VERSION` | `1.0.0` | Reported in `/status`. |
 | `API_PORT` | `8180` | External port baked into `baseUrl`. |
 | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). |
-| `REDIS_PORT` | `6379` | Local Redis port. |
+| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | Truth database file (backed up). Dev stacks set `/meta-core-db/meta-core.db` (named volume — SQLite WAL is unsafe on the 9p `/meta-core` bind of Windows dev boxes). |
+| `META_CORE_CACHE_PATH` | `/meta-core/db-cache/cache.db` | Cache database file (never backed up; rebuilt from truth when missing). Dev stacks keep it on a named volume too; store apps put the cache folder in Maison's `backup.exclude`. |
+| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection. |
+| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
+| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
+| `META_CORE_REPORTS_DB_PATH` | `<META_CORE_PATH>/db/reports.db` | Report inbox database (beside truth, so it is backed up with it). |
+| `META_CORE_REPORTS_MAX` | `5000` | Reports kept; closed ones are evicted first, then the oldest. |
+| `META_CORE_REPORTS_MAX_MB` | `256` | Total report body size kept, MiB. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
 | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind address. |
 | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
@@ -537,7 +795,9 @@ via `/health`; metadata I/O and SSE event streams stop.
 |---|---|
 | `locks/kv-leader.lock` | Flock target. |
 | `locks/kv-leader.info` | Leader API URL (plaintext). |
-| `db/redis/{dump.rdb, appendonly.aof}` | Redis persistence. |
+| `db/meta-core.db` (+ `-wal`, `-shm`) | Truth database (claims, principals, `cid_group`). Backed up. Replaced `db/redis/` in 2026-10; holds the claims schema since v2.0.0. |
+| `db-cache/cache.db` (+ `-wal`, `-shm`) | Cache database (v1-layout `kv` table: merged view, lookup, counters, events, service scratch keys; field index, FTS, `claim_by_pid`, `local_file`). Never backed up. |
+| `db/reports.db` (+ `-wal`, `-shm`) | Report inbox (bug reports / feedback from the client apps). Diagnostics, not metadata; bounded. |
 | `services/*.json` | Service registry. |
 | `mounts/mounts.json` | rclone mount configuration. |
 | `mounts/errors/{id}.error` | Mount error logs. |
@@ -551,7 +811,9 @@ via `/health`; metadata I/O and SSE event streams stop.
 meta-core consolidates leader gating, metadata storage, event mirroring,
 and file access for MetaMesh into one container with a typed HTTP+SSE
 surface. The bash flock gate guarantees single-leader semantics without
-an external consensus system. UUID-rooted storage with CID aliasing
+an external consensus system. Metadata is a table of signed votes in a
-treats all content hashes symmetrically and makes any-CID resolution
+backed-up truth database, merged by an apply loop into a rebuildable cache
-O(1). API-mediated access ensures no other service ever needs to speak
+that serves every read; meta-core verifies every claim and signs none.
-Redis.
+CID groups behind a hidden local rid treat all content hashes
+symmetrically. API-mediated access ensures no other service ever touches
+the store.

docs/metadata-storage-structure.md

@@ -1,180 +1,199 @@
-# Redis Metadata Storage Structure
+# Metadata Storage Structure
 
-How meta-core lays out file metadata in Redis. This document covers the
+How meta-core lays out metadata on disk since **v2.0.0 (claims model)**: the
-on-disk shape (UUID-rooted flat string keys, CID reverse index,
+two SQLite databases, their tables, how a record is assembled from signed
-per-root sets) and the schema-version gate. It does **not** enumerate
+votes, and the schema-version gate. Implementation basis:
-field semantics — that's the job of the repo-root
+[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md).
-[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key
+This document does **not** enumerate field semantics — that is the job of the
-registry for everyone who writes to or reads from metadata.
+repo-root [`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative
+key registry for everyone who writes or reads metadata.
 
-For the design rationale (why UUIDs instead of content hashes, why a
+Related: why rids are UUIDs and hidden —
-reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
+[`uuid-rooted-metadata.md`](uuid-rooted-metadata.md); why the engine is
-For the external-access contract (HTTP-only, SSE for events), see
+shaped this way, its guarantees and measurements —
+[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md);
+the external-access contract (HTTP-only, SSE for events) —
 [`api-mediated-access.md`](api-mediated-access.md).
 
+> **History.** Until 2026-10 metadata lived in Redis; v1.1.0–v1.2.x kept the
+> same flat keys (`file:<rid>/<field>`, `cid:<cid>`, `idx:*`, `dup:*`,
+> `udl:*`, `ev:*`) in one SQLite string→string `kv` table. v2.0.0 replaced
+> that keyspace entirely; the old layout is summarised at the end
+> ([Previous layout](#previous-layout-v11v12-kv-table)).
+
 ## Quick summary
 
-- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
+- **Every metadata value is a signed claim**: `claim(uid, cid, key) → value`
-  not content hashes. Examples in this doc use the placeholder
+  + `ts` + `sig` (MMUR2, secp256k1). One row per signer per field. Writers
-  `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
+  sign; meta-core never signs.
-- **Properties are flat Redis STRING keys**, one per leaf:
+- **Two databases.** *Truth* `/meta-core/db/meta-core.db`
-  `file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
+  (`META_CORE_DB_PATH`, backed up, primary keys only): `principal`, `claim`,
-- **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
+  `cid_group`, `meta`. *Cache* `/meta-core/db-cache/cache.db`
-  (a Redis STRING per known CID; no algorithm prefix — the multicodec
+  (`META_CORE_CACHE_PATH`, never backed up, rebuildable): the v1 `kv` table
-  already carries it).
+  (merged view as `file:<rid>/<field>` rows, lookup, counters, event streams,
-- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
+  bookkeeping, service scratch keys) with its `kv_field_v` index and `fts`,
-  `"true"` per known CID, midhash included), `file:<uuid>/duplicates`
+  plus two real tables, `claim_by_pid` and `local_file`.
-  (alternate paths with the same content), plus `file:__index__` (every UUID).
+- **Schema-free.** A field is a `key` string in a row; a new field never
-  There is **no stored `canonical_cid`** — it is derived by rank on read.
+  needs DDL.
-- **A schema-version sentinel** (`meta-core:schema-version`) refuses to
+- **Records are cid groups.** A vote names a cid, never a rid. `cid_group`
-  boot against legacy data.
+  maps each cid to a **rid** — a hidden, locally minted UUIDv7 (26-char
+  Crockford) naming the group. Examples use `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
+- **What readers get is the merged view**: one precomputed value per
+  `(rid, key)`, stored as the cache row `file:<rid>/<key>`. `nsrc` (how many
+  signers support it) is computed by the merge and returned by
+  `POST /api/merge`, not stored. `canonical_cid` is still **derived on read**
+  by rank, never stored.
+- **Watcher facts are not claims**: `filePath`, size, `mtimeNano`, midhash
+  live in `local_file` and are joined into the record view unsigned.
+- **A schema-version sentinel** (`schema-version = "3"` in truth `meta`).
+  meta-core explicitly refuses a truth file holding the v1 `kv` table, and any
+  store with claims but no sentinel or another version.
 
-Implementation lives in `internal/storage/client.go`,
+## Truth database — `/meta-core/db/meta-core.db`
-`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
-`internal/storage/schema_version.go`.
 
-## Key structure
+Backed up; only primary keys (plus `principal.uid`'s implicit UNIQUE index).
 
-### Roots
+| Table | Columns (PK first) | Notes |
+|---|---|---|
+| `principal` | `pid` INTEGER PK, `uid` TEXT UNIQUE | `uid` = `"z" + base58btc(33-byte compressed pubkey)`. `pid` is a local surrogate: never signed, never on the wire |
+| `claim` | `(cid, key, pid)` PK, `value` ANY, `ts`, `sig`, `seq` | `STRICT, WITHOUT ROWID`. `value`: TEXT = plaintext, BLOB = ciphertext envelope, NULL = retraction (tombstone, kept). `ts` (ms) is the version. `seq` = global change counter, not indexed |
+| `cid_group` | `cid` PK, `rid` | the cid → rid map: the record identity layer |
+| `meta` | `name` PK, `value` | `schema-version = "3"`, `last_seq` |
+
+`claim` is written **only** by `POST /api/claims`. A claim replaces the
+stored row for its `(cid, key, pid)` only if its `ts` is strictly greater;
+identical `(ts, sig)` is `unchanged`; the **same value** from the same signer
+(any `ts`) is also `unchanged` and keeps the stored row with its older `ts`;
+otherwise older or equal is `stale`, and a bad signature is refused. Why `STRICT` with an `ANY` column: a non-STRICT `ANY`
+column gets NUMERIC affinity and turns the text `"007"` into the integer 7;
+STRICT keeps the written type, which is the cipher flag.
+
+## Cache database — `/meta-core/db-cache/cache.db`
+
+Never backed up (the store app lists the folder in `backup.exclude`).
+Deleting it is the recovery procedure: meta-core rebuilds it from truth at
+boot (no `meta:applied_seq` → full rebuild), and `local_file` refills from a
+rescan.
+
+It keeps the **v1 key/value layout** (`internal/storage/client.go`): one
+`kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID` table, whose rows are
+all derived — no client writes `file:` rows any more.
+
+| `kv` rows | Holds | Written by | Read by |
+|---|---|---|---|
+| `file:<rid>/<field>` | the **merged view**, one row per `(rid, key)` — the only rows under `file:`. Includes the synthesised `cids/<cid> = "true"` members (from `cid_group`) and the box-local `filePath` / `sizeByte` / `mtimeNano` (from `local_file`) | apply loop | every record read; search |
+| `cid:<cid>` | cid → rid (copy of truth `cid_group`) | apply loop, watcher | every cid-addressed request |
+| `idx:roots/<rid>` | the set of every record | apply loop | listing, counts |
+| `idx:fts/<rid>` | rowid of the record's `fts` row | apply loop | FTS maintenance |
+| `idx:count/*`, `idx:schema/*` | record count, per-field schema counters | the cache engine's change hook, from `file:` deltas | search `total`, `/api/stats`, `/api/schema` |
+| `ev:<stream>/<ms13>-<seq10>`, `evhwm:<stream>` | `meta:events` (every `file:` change, same transaction) and `file:events` (watcher) | cache engine; watcher | `/api/events/meta`, `/api/events/files` (SSE) |
+| `meta:applied_seq`, `meta:rebuilt_at` | how far the cache has applied truth; last rebuild (older SSE cursors get `gap`) | apply loop, rebuild | boot catch-up; SSE |
+| anything else | service scratch KV (`btmagnet:*`, `searchcov:*`, `nzb:*`) | other services via `/api/kv/value` (reserved prefixes refused) | same |
+
+| Index / table | Shape | Written by | Read by |
+|---|---|---|---|
+| `kv_field_v` (index) | `kv(lower(field parsed from k), lower(v), v)` over `file:` rows only | — | filter search |
+| `fts` | FTS5 trigram `(rid UNINDEXED, hay)`, one row per record | apply loop, when a haystack field's merged value changes | free-text search |
+| `claim_by_pid` | `(pid, cid, key)` PK | apply loop | `?as=<uid>` overlay, `/api/udl/*` facade, identity retraction, `/api/udl/users/stats` |
+| `local_file` | `path` PK → `midhash`, `size`, `mtime`; index `(midhash, path)` | file watcher, `POST /api/files/register` | record view (`filePath`, `sizeByte`, `mtimeNano`) |
+
+The cache is **behind, never wrong**: after an unclean shutdown, boot finds
+`meta:applied_seq < last_seq` and re-applies every truth claim with `seq >
+applied_seq` (or rebuilds in full when one of them is a `cids/*` link). Write
+path, apply loop and durability are detailed in
+[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §4.
+
+## Keys
+
+The `key` column carries the same field paths METADATA_KEYS.md defines —
+what used to follow `file:<rid>/`:
 
 ```
-file:<uuid>/<property/path>   →   Redis STRING (always)
+claim(cid=bagacbabaec…, key=title,                          uid=<tmdb plugin>) = "Inception"
+claim(cid=bagacbabaec…, key=fileinfo/duration,              uid=<ffmpeg plugin>) = "8878.5"
+claim(cid=bagacbabaec…, key=fileinfo/streamdetails/video/0/codec, uid=<ffmpeg plugin>) = "h264"
+claim(cid=bagacbabaec…, key=genres/Drama,                   uid=<tmdb plugin>) = "true"
+claim(cid=bagacbabaec…, key=cids/bafybeigd…,                uid=<full-hash plugin>) = "true"
 ```
 
-UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
+- `/` is the path separator; nested objects are flattened on write and
-UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
+  reconstructed on read.
-time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
+- **Key-sets** put the member inside the key (`genres/<g>`, `cids/<cid>`,
-in a given window, which restored some of the debug-ability lost when
+  `titles/eng/<name>`, `anchoredBy/tmdb:123`): one claim per member, union by
-midhash256 stopped being the root.
+  the ordinary merge, removal by a signed retraction.
+- Plugins keep writing under their prefixes (`{plugin-id}/<field>`,
+  `{plugin-id}/<group>/<n>/<field>`), each signing with its own key. Two
+  sources compete on one key only when they make the same kind of statement;
+  a different fact (e.g. a filename guess vs a TMDB lookup) gets a different
+  key.
 
-Properties use `/` as the path separator. Nested objects in source code
+## Records: `cid_group` and the hidden rid
-are flattened to flat keys on write and reconstructed on read. Examples:
 
 ```
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath                          = "/files/watch/movies/Inception.2010.mkv"
+votes (truth, signed)                    cid_group (truth)     cache kv rows (merged)
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte                          = "15032385536"
+(A, M, title)    = "The Matrix"          M → R1                file:R1/title    = "The Matrix"
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano                         = "1729872000000000000"
+(A, M, cids/Q)   = "true"                Q → R1                file:R1/cids/M   = "true" (synthesised)
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title                             = "Inception"
+(B, Q, title)    = "Matrix, The"                               file:R1/cids/Q   = "true" (synthesised)
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration                 = "8878.5"
+                                                               file:R1/sizeByte = "734003200"
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264"
+(B, Q, sizeByte) = "734003200"
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster                       = "bafkreih..."
 ```
 
-### Reverse-index aliases (CIDs)
+- A text `cids/<other>` vote on cid X links X and `<other>` into one
+  group. Retracting it does **not** split the group (phase 1). In the merged
+  view, membership is synthesised from `cid_group` — `cids/<cid> = "true"` for
+  every cid of the group — and `cids/*` claims are never merged. Record R = every vote on every cid of R's group, merged per key,
+  each signer counted once per key.
+- First-seen cid with no link → new group with a fresh rid
+  (`storage.NewUUID()`, UUIDv7 → 26-char Crockford Base32, ULID layout).
+- Linking two groups → the **older rid survives**; the other group's cids
+  move to it; `/api/events/meta` emits `del` for each key of the losing rid
+  and `set` for the survivor. There is no rid alias table.
+- `GET /api/meta/<cid>` is a `cid:<cid>` lookup in the cache, then a range
+  read `[file:<rid>/, file:<rid>0)` — any known cid, any hash algorithm, no
+  primary.
+- Rids are local names: never signed, never on the swarm or gateway wire.
+  Local consumers (meta-sort, meta-fuse, meta-stremio) do hold them, which is
+  why `cid_group` lives in truth.
 
-Every CID known for a file is registered as its own Redis STRING, keyed by
+There is **no stored `canonical_cid`**. `GetMetadataDocument` ranks the
-the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
+group's cids through `internal/cid/rank.go` (`cid.Better`) and returns the
-function in the multicodec, so naming it again in the key would be
+winner as `canonical_cid` in the JSON response only.
-redundant (and would let the same digest be indexed under two spellings):
 
-```
+## Merge
-cid:<bareCid>   →   <uuid>
-```
 
-Examples:
+The cache row `file:<rid>/<key>` holds the winner over the group's votes
-
+(each signer's newest vote counts once; retractions and ciphertext drop out):
-```
+most distinct signers (values compared after **trim + case-fold only**, no
-cid:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+NFC) → oldest `ts` → byte-wise smallest. The stored value is the winner's
-cid:bagacbabaec7v3...     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+exact text; `nsrc` is not stored (`POST /api/merge` returns it). With
-```
+`?as=<uid>`, the viewer's own live text votes (found via `claim_by_pid`)
-
+override at read time; they are never written to the cache. Box-local
-The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
+fields: when several local paths hold the same content, `filePath` is the
-registered CID, independent of which hash algorithm — there is no primary.
+lexicographically first (size and mtime from the same `local_file` row); a
-The auto-alias hook (`maybeAddAliasFromFieldLocked` in
+signed `sizeByte` claim wins over the on-disk size. Full rules:
-`internal/storage/cid_resolution.go`, called from `SetProperty`,
+implementation basis §7 (`internal/storage/merge.go`).
-`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
-field whose name starts with `cids/`, so writers emit key-set members
-without knowing the reverse index exists.
-
-> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
-> `midhash256` *named* field written to a root is stored verbatim and is
-> **never indexed** — the record becomes unresolvable by that CID. Writers
-> must collapse such fields to `cids/<cid>` members before the write. See
-> `METADATA_KEYS.md` §14.13.
-
-### Per-root sets
-
-```
-file:<uuid>/cids/<bareCid>    →   STRING "true" — one key per known CID (a key-set, not a Redis SET)
-file:<uuid>/duplicates        →   Redis SET of alternate paths with same content
-```
-
-`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
-which keeps it conflict-free to merge across peers (a SET would not be).
-The record's own address (the midhash) is just the `0x1000` member — it has
-no separate field.
-
-There is **no stored `canonical_cid`**. The canonical CID is *derived on
-read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the
-key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns
-the winner as the `canonical_cid` field of the JSON response only. Nothing
-reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and
-was removed with the flat `cid_*` keys.
-
-### Global index
-
-```
-file:__index__   →   Redis SET of all UUIDs
-```
-
-`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
-every file without scanning the keyspace.
-
-```bash
-SMEMBERS file:__index__         # all UUIDs
-SCARD    file:__index__         # total file count
-SISMEMBER file:__index__ <uuid>
-```
-
-### Schema-version sentinel
-
-```
-meta-core:schema-version   →   STRING (currently "2")
-```
-
-Set at first boot, checked on every subsequent boot
-(`internal/storage/schema_version.go`). If the sentinel is missing but
-`file:__index__` is non-empty, the gate refuses to boot (legacy data).
-If the sentinel is present but doesn't match `SchemaVersion`, the gate
-also refuses to boot. Alpha-stage policy: operators wipe and restart;
-there is no automated migration.
-
-## Why flat STRING keys?
-
-The store does not use Redis Hashes. The reasons are:
-
-1. **Field-level keyspace notifications.** Redis publishes
-   `__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
-   you'd only get one notification on the hash itself. meta-core's
-   `MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
-   notifications and publishes structured events to the `meta:events`
-   Redis Stream.
-2. **Selective updates without read-modify-write.** A plugin writing
-   `tmdb/poster` doesn't have to read the rest of the document.
-3. **Compatibility with the CID alias hooks.** Field writes can trigger
-   alias-maintenance work in `SetProperty` without the plugin caring.
 
 ## Reads and writes
 
-The Go API in `internal/storage/client.go`:
+| Operation | Endpoint | Store access |
+|---|---|---|
+| write metadata | `POST /api/claims` (batch of signed claims) | truth upsert, then the apply loop updates the cache |
+| removed writes | `PUT/PATCH/DELETE /meta/*`, `POST /meta/{h}/_add/{k}`, `PUT/DELETE /api/metadata/{id}`, `PUT /api/metadata/{id}/property`, `POST /api/metadata/batch` | `410 Gone`, pointing at `POST /api/claims` |
+| read a record | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{h}` | `cid:` lookup → `file:<rid>/*` rows, optional `?as=` |
+| list / search | `GET /api/metadata/hash-ids`, `/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `/api/schema`, `/api/stats` | `file:` rows, `kv_field_v`, `fts`, `idx:count/*` / `idx:schema/*` |
+| raw votes | `GET /api/meta/{cid}/claims`, `GET /api/metadata/{rid}/claims` (`?key=`, `?uid=`) | `{rid, cids, claims}`: every live signed claim of the group, all signers, tombstones excluded; 404 for an unknown cid / rid |
+| merge without storing | `POST /api/merge` `{claims, as?}` → `{fields, nsrc, rejected}` | none — verify + merge in memory (plus `as`'s votes this core holds) |
+| watcher facts | the watcher, `POST /api/files/register` | `local_file` + `cid_group` |
+| service scratch | `PUT/GET/DELETE /api/kv/value` | non-reserved cache `kv` rows |
+| wipe | `POST /api/metadata/clear` | both databases |
 
-| Method | Behaviour |
+Truth writes go through **one writer goroutine** with group commit; reads use
-|---|---|
+a separate pool and see committed state (WAL). Read paths and shapes are
-| `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
+unchanged from v1.x — they are now served from the derived `file:` rows.
-| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
-| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
-| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
-| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
-| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
-| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
-| `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
-| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
-| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
 
 ## Reconstruction (flat → nested)
 
-When the HTTP API returns a document, it reconstructs the flattened
+The HTTP API reconstructs the merged flat keys into a nested JSON object:
-storage back into a nested JSON object:
 
 ```
 flat                                          nested
@@ -186,112 +205,122 @@ fileinfo/streamdetails/                                  streamdetails: { video:
 streams/0/language          = "eng"           streams: [ { language: "eng" }, ... ]
 ```
 
-Rules:
+- Numeric path segments become array indices on read.
+- All values are strings on the wire; type interpretation is the caller's
+  job (documented per field in `METADATA_KEYS.md`).
+- Box-local fields from `local_file` are joined in unsigned; they never
+  appear in `GET /api/meta/{cid}/claims` or on the swarm.
 
-- `/` is the path separator.
+## Search structures
-- Numeric path segments (`0`, `1`, ...) become array indices on read.
-- All values are strings on the wire; type interpretation is the
-  caller's job (and is documented per-field in `METADATA_KEYS.md`).
-- `null`/`undefined` round-trip as empty strings.
 
-## Plugin contribution flow
+Both live in the cache and are maintained in the same transaction as the
+merged-row change, so results are fresh as soon as a change is applied:
 
-Plugins don't talk to Redis. They write metadata via meta-core's HTTP
+- **`kv_field_v`** — an expression index over the field name parsed out of
-API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
+  the key, `lower(v)` and `v`, restricted to `file:` rows. Answers
-writes under a prefix:
+  "which records have field F with a value like V" for any field, including
+  ones invented tomorrow; covering for the substring test.
+- **`fts`** — FTS5 trigram `(rid UNINDEXED, hay)`, one row per record: the
+  haystack of its searchable merged fields plus the rid. Every 3+ character substring is
+  indexable.
 
-```
+`POST /api/metadata/search` builds a candidate superset in SQL and verifies
-{plugin-id}/<field>                 single value
+each hydrated merged record with the Go matchers (`recordMatchesFilter`,
-{plugin-id}/<group>/<field>         grouped
+`matchAllTokens`); results come in rid order. Plan, rejected shapes and the
-{plugin-id}/<group>/<n>/<field>     indexed (array)
+measurement status are in
-```
+[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §8.
 
-See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full,
+## Schema-version sentinel
-authoritative list of fields each plugin writes (content identification,
+
-file stats, video classification, anime detection, subtitle references,
+Truth `meta` row `schema-version` = `"3"`, set at first boot and checked on
-Jellyfin metadata, FFmpeg stream details, etc.) and the value formats
+every boot (`internal/storage/schema_version.go`), after both files are
-(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc
+opened and before the cache catch-up. A truth file holding the v1 `kv` table
-deliberately does not duplicate that registry — it would drift.
+(v1.x, `meta-core:schema-version = "2"`) is **explicitly refused**, naming the
+migration tool; claims without a sentinel, or another version, are refused
+too; a fresh store is stamped. Alpha policy: no migration code in the
+service binary; operators wipe and restart. watch.nsl.sh is converted once by
+the separate one-off tool `packages/meta-core/tools/claims-migrate` (not in
+the image; rids keep their UUIDs; legacy fields re-signed by a throwaway key;
+UDL cells re-signed by their account keys — implementation basis §11).
 
 ## Event streams
 
-Two Redis Streams are produced inside meta-core:
-
 | Stream | Producer | Contents |
 |---|---|---|
-| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
+| `/api/events/meta` | the cache engine, as `ev:meta:events/…` rows of the cache `kv` table | `{type: set\|del, key, ts}` for every change of a **merged** value, in the same cache transaction as the change |
-| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
+| `/api/events/files` | `internal/watcher` dispatcher (`ev:file:events/…` rows) | `add` / `change` / `delete` / `rename` / `reset` for files on `/files` (unchanged) |
 
-External consumers do **not** read these streams directly. They subscribe
+- A vote that does not change the merged value emits nothing.
-via SSE:
+- A group merge emits `del` for every key of the losing rid and `set` for the
-
+  survivor's keys.
-- `GET /api/events/files` — mirror of `file:events`
+- IDs keep the `<ms>-<seq>` form and stay time-based, so persisted cursors
-- `GET /api/events/meta` — mirror of `meta:events`
+  keep comparing, including across a cache rebuild.
-
+- `Last-Event-ID: 0` / `0-0` returns a snapshot (one `set` per merged key)
-The SSE wire contract uses `Last-Event-ID` for resume (the opaque
+  then the live tail; only the last snapshot event carries an `id:`.
-`<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
+- A trimmed cursor gets one `event: gap`. A cache rebuild emits no per-key
-(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
+  events; consumers get `gap` and take a snapshot.
-a client's cursor, the handler emits one `event: gap` payload before
+- Heartbeats are SSE comments (`:keep-alive\n\n`) every 30 s of silence.
-resuming. See `internal/api/sse_events.go` and
+  Contract: `internal/api/sse_events.go`, [`api-mediated-access.md`](api-mediated-access.md).
-[`api-mediated-access.md`](api-mediated-access.md) for the full
-contract.
-
-Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
-inside `MetaPublisher` to drain keyspace notifications, but this is not
-the external surface — external services use SSE.
 
 ## Debugging
 
+Paths: truth `/meta-core/db/meta-core.db`, cache `/meta-core/db-cache/cache.db`
+(image default and store app); dev stacks keep both on named volumes (the
+`/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL is unsafe).
+Always open **read-only**; writes go through `POST /api/claims`.
+
 ```bash
-# Roots
+T=/meta-core/db/meta-core.db; C=/meta-core/db-cache/cache.db   # dev: the named-volume paths
-docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
+QT() { docker exec metacore-app sqlite3 -readonly "$T" "$1"; }
-docker exec metacore-app redis-cli SCARD    'file:__index__'
+QC() { docker exec metacore-app sqlite3 -readonly "$C" "$1"; }
-docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
 
-# Full document for one root (flat scan)
+QT "SELECT rid FROM cid_group WHERE cid = 'bafkreih...'"                 # cid → rid
-docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
+QT "SELECT cid FROM cid_group WHERE rid = '01JKR8XW5T4QMV7AHNJ2DEFGHK'"  # the group (full scan: no rid index)
-
+QC "SELECT k, v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'"   # merged record
-# One property
+QT "SELECT c.key, typeof(c.value), c.value, p.uid, c.ts, c.seq
-docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
+    FROM claim c JOIN principal p USING(pid) WHERE c.cid = 'bafkreih...'" # every vote on one cid
-
+QT "SELECT name, value FROM meta"; QC "SELECT k, v FROM kv WHERE k LIKE 'meta:%'"   # last_seq vs applied_seq
-# Confirm keys are STRING (not HASH)
+QC "SELECT path, midhash FROM local_file WHERE path LIKE '/files/watch/%' LIMIT 10"
-docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
+QC "SELECT k, v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10"
-# string
-
-# Reverse-index lookup (bare CID — no algorithm prefix)
-docker exec metacore-app redis-cli GET 'cid:bafkreih...'      # → uuid
-
-# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
-docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
-
-# canonical_cid is NOT stored. Derive it via the API instead:
-curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
-
-# Schema-version sentinel
-docker exec metacore-app redis-cli GET 'meta-core:schema-version'
-
-# Streams (server-side; external consumers should use SSE)
-docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
-docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
 
 # HTTP equivalents — preferred
-curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>'      # CID → document
+curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>'          # merged document (+ canonical_cid)
-curl -k 'https://metacore-dev.localhost:8083/meta'                # list root IDs
+curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>/claims'   # signed votes
-curl -k 'https://metacore-dev.localhost:8083/meta/<uuid>'         # document by UUID
+curl -k 'https://metacore-dev.localhost:8083/api/events/meta'         # live SSE
-curl -k 'https://metacore-dev.localhost:8083/api/events/meta'     # live SSE
-curl -k 'https://metacore-dev.localhost:8083/api/kv/info'         # KV stats
 ```
 
 ## Key implementation files
 
 | Component | Path |
 |---|---|
-| Redis client + flat-key ops | `internal/storage/client.go` |
+| Single-writer, group-commit SQLite engine | `internal/kv/` |
+| Truth, apply loop, merge, local files, counters | `internal/storage/{truth,apply,merge,localfile,client}.go` |
+| Search (`kv_field_v` + FTS) | `internal/storage/search.go` |
+| MMUR2 claim encoding + verification | `internal/claim/` (secp256k1 in `internal/identity/`) |
 | UUIDv7 minting | `internal/storage/uuid.go` |
-| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
+| Canonical-CID rank | `internal/cid/rank.go` |
 | Schema-version gate | `internal/storage/schema_version.go` |
-| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
+| SSE event mirror + snapshot | `internal/api/sse_events.go` |
-| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
+| Watcher → `local_file`, `/api/events/files` | `internal/watcher/` |
-| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
-| SSE event mirror | `internal/api/sse_events.go` |
 | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
+
+## Previous layout (v1.1–v1.2 `kv` table)
+
+Kept so old notes and scripts can be read. In v2.0.0 the **truth** database
+has no `kv` table at all; the **cache** keeps the same key layout for derived
+rows (see above), but nothing in it is written by clients.
+
+| Key | Value | Replaced by |
+|---|---|---|
+| `file:<rid>/<field>` | one value per field, last writer wins | `claim` (one row per signer) → derived cache `file:` rows |
+| `cid:<cid>` | `<rid>` | `cid_group` (truth), copied to cache `cid:` rows |
+| `idx:roots/<rid>`, `idx:count/*`, `idx:schema/*`, `idx:fts/<rid>` | root set, counters, FTS rowid | same keys, derived, in the cache |
+| `dup:<rid>/<path>` | duplicate path | `local_file` rows |
+| `udl:rec:<uid>/<cid>/<key>`, `udl:idx:*` | User Data Layer cells (MMUR1, `version`) | `claim` (MMUR2, `ts`) + `claim_by_pid` |
+| `ev:<stream>/<ms13>-<seq10>`, `evhwm:<stream>` | stream rows | same keys, in the cache |
+| `btmagnet:*`, `searchcov:*`, `nzb:*` | service scratch | same keys, in the cache |
+| `meta-core:schema-version` = `"2"` | gate | truth `meta` `schema-version` = `"3"` |
+
+The flat-key idea survived — a field is still one string key, a key-set
+member is still inside the key — but the record moved from "one value per
+field" to "one value per signer per field".

docs/uuid-rooted-metadata.md

@@ -2,361 +2,275 @@
 
 ## Status
 
-**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
+**Status:** Design record. Implemented in 2026 alpha (clean-wipe gate);
+**reframed by meta-core v2.0.0 (claims model)** — see
+[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md)
+§4. This doc keeps the *why* behind UUID rids and CID symmetry; it is not
+the schema reference. For what is stored, see
+[metadata-storage-structure.md](./metadata-storage-structure.md) and
+`METADATA_KEYS.md` §2/§14.13.
 
-> ⚠ **Two parts of this proposal were superseded before shipping.** This doc
+What still holds, unchanged since the original proposal:
-> is the design record; it is *not* the schema reference. For what is actually
+
-> stored, see `METADATA_KEYS.md` §2/§14.13 and
+- **No CID is privileged.** Any known CID resolves to its record; all hash
-> [metadata-storage-structure.md](./metadata-storage-structure.md).
+  types are symmetric.
+- **Rids are UUIDv7**, 26-char Crockford Base32, **minted locally** with no
+  central allocator.
+- **`canonical_cid` is derived on read** by rank (`internal/cid/rank.go` →
+  `cid.Better`), never stored.
+- **`cids` is a key-set of bare CIDs** (`cids/<bareCid>` = `"true"`), no
+  `<algo>:` prefix — a CIDv1's multicodec already names the hash function.
+
+What changed in v2.0.0:
+
+> 1. **The rid is a hidden local name for a group of cids.** Metadata is no
+>    longer stored under `file:<uuid>/<field>`. Every value is a signed claim
+>    about a **cid** (`claim(uid, cid, key) → value`); `cid_group(cid → rid)`
+>    says which cids form one record. The rid is **derived from cid groups,
+>    never signed and never on the wire** (swarm, gateway); each box mints its
+>    own.
+> 2. **The `cid:<cid>` reverse index is gone**, replaced by the `cid_group`
+>    table (still a primary-key lookup, so still O(1)-class).
+> 3. **meta-core no longer writes `cids/*` members.** They are signed votes
+>    from the writers that learn a cid (e.g. the full-hash plugin); a non-null
+>    `cids/<other>` vote links two groups. Only the watcher's midhash enters
+>    `cid_group` directly.
+> 4. **`duplicates` / `dup:<uuid>/<path>` are gone.** Box-local paths live in
+>    the cache's `local_file(path → midhash, size, mtime)`; several paths with
+>    the same midhash are the duplicates.
 >
-> 1. **`canonical_cid` is not a stored key.** The proposed reconciler
+> Earlier superseded points, kept for the record: the proposed
->    (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't
+> `reconcileCanonicalCIDLocked` never shipped; `cids` never was a Redis SET
->    conflict-free to merge across peers. The canonical CID is **derived on
+> of `<algo>:<value>` tokens; Redis itself went in v1.1.0 (2026-10). Text
->    read** by ranking the `cids` members (`internal/cid/rank.go` →
+> below marked *(proposal)* or *(v1.x)* describes those shapes, not v2.0.0.
->    `cid.Better`), and appears only in the `GET /api/meta/{cid}` response.
-> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not
->    `<algo>:<value>` tokens.** One flat STRING per member:
->    `file:<uuid>/cids/<bareCid>` = `"true"`. The reverse index is likewise
->    `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
->    function, so the `<algo>:` prefix was redundant.
->
-> Text below that references either shape describes the *proposal*, not the
-> code. The rank ordering and everything else did ship as written.
 
-## Motivation
+## Motivation (history)
 
-Today, every metadata entry in Redis is keyed by a content hash, specifically
+Before UUID rids, every metadata entry in Redis was keyed by a content hash,
-midhash256:
+specifically midhash256:
 
 ```
 file:midhash256:abc.../filePath
 file:midhash256:abc.../title
-file:midhash256:abc.../fileinfo/duration
 file:midhash256:abc.../tmdb/poster
-...
 ```
 
-Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields
+Other digests (sha256, IPFS CIDs, BitTorrent infohashes) lived as fields
-*inside* that entry:
+*inside* that entry (`cid_sha256`, `cid_ipfs`). Two problems:
 
-```
+1. **midhash256 was privileged by accident.** It *was* the key, while every
-file:midhash256:abc.../cid_sha256 = sha256:def...
+   other CID was a secondary field. Hash-algorithm changes, or new digest
-file:midhash256:abc.../cid_ipfs   = bafy...
+   types added by plugins, forced rewrites of every root. The privilege
-```
+   itself was the issue, not which hash held it.
+2. **CID lookup was O(n).** "Given any CID, return the metadata" scanned
+   every entry (`storage/client.go::LookupPathByCID`). Unacceptable for a
+   public API, especially when peers query by any CID variant.
 
-Two problems with this:
+The requirement: **any CID a client knows must resolve to that content's
-
+metadata in O(1)**, and the storage layout must not bias toward any digest
-1. **midhash256 is privileged by accident.** It IS the key, while every other
+type. The claims model adds a second one: **what is signed must name
-   CID is a secondary field. There's nothing intrinsically special about
+something every peer agrees on** — a rid minted at random on one box does
-   midhash256 — it's just the first hash computed. Hash-algorithm changes, or
+not qualify, a cid does.
-   new digest types added by plugins, force rewrites of every root. The
-   "promotion" workarounds people reach for (re-keying entries when a stronger
-   hash arrives) solve the wrong problem; the privilege itself is the issue.
-
-2. **CID lookup is O(n).** Resolving "given any CID, return that file's
-   metadata" currently scans every indexed entry (see
-   `storage/client.go::LookupPathByCID`, which acknowledges this in a
-   comment). Acceptable for admin tooling, unacceptable for a public API
-   surface — especially when peers on meta-share may query by any CID variant.
-
-The product requirement is clear: **any CID a client knows about must resolve
-to that file's metadata in O(1).** IPFS-compatible CIDs are the externally
-meaningful identity for content retrieval; the internal storage layout should
-not bias toward any one digest type.
 
 ## Goal
 
-A single externally-meaningful contract:
-
 ```
-GET /api/meta/<cid>  →  full metadata document for that file
+GET /api/meta/<cid>  →  full (merged) metadata document for that content
 ```
 
-Where `<cid>` may be any supported type (midhash256, sha256, IPFS, btih,
+`<cid>` may be any supported type (midhash, sha256, IPFS, btih, sha3, future
-sha3, future additions). All CID types are treated symmetrically. No CID type
+additions), all treated symmetrically.
-is "the key."
 
-## Design
+## Design (v2.0.0)
 
-### Two-tier key structure
+### Votes name cids; a rid names a group
-
-The root key becomes an opaque internal UUID; CIDs are aliases that point to
-it.
 
 ```
-file:<uuid>/<property...>     ← canonical storage (UUIDv7, never disclosed externally)
+votes (signed, travel)                 derived (local, persistent: truth cid_group)
-cid:<cid> → <uuid>            ← reverse index (one Redis key per known CID)
+(A, M, title)   = "The Matrix"         M → R1
+(A, M, cids/Q)  = "true"  ← alias vote  Q → R1
+(B, Q, title)   = "Matrix, The"
+(B, Q, sizeByte)= "734003200"          record R1 = every vote on M or Q, merged per key
 ```
 
-The metadata keyspace under `file:<uuid>/` remains open and flat, exactly as
+- **First-seen cid, no link** → new group, fresh UUIDv7 rid.
-today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
+- **`cids/<other>` vote on X** → X and `<other>` join one group.
-`file:<uuid>/fileinfo/duration`. Only the root identifier changes.
+- **Link between two existing groups** → the group with the **older rid
+  survives**; the other's cids move to it; the losing rid gets a `del` event
+  for each of its keys, the survivor `set` events. No `rid_alias`: consumers
+  already handle deletes, and cids re-resolve.
+- **Reading record R** = every vote on every cid of R's group, merged per key
+  (one signer counts once per key), precomputed into the cache's
+  `file:<rid>/<field>` rows (the v1 key layout, now derived).
+- **Retracting a `cids/<other>` link does not split the group** (phase 1):
+  the tombstone drops out of the merge, `cid_group` keeps both cids under one
+  rid. In the record view, membership is synthesised from `cid_group`
+  (`cids/<cid> = "true"` for every cid of the group).
 
-**Example layout:**
+Why keep a rid at all, if votes never mention it: local consumers (meta-sort,
+meta-fuse, meta-stremio) and the merged view need one stable name per record,
+and a group's cid set grows over time. Why `cid_group` lives in **truth**
+although it could be recomputed from `cids/*` votes: the rid names are minted
+at random and consumers hold them, so a recomputation would rename every
+record.
 
-```
+### Why UUIDv7 (still valid)
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath          = /watch/Inception.mkv
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size              = 4500000000
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime             = 1729872000000000000
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec…  = "true"    ← midhash (record address)
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd…    = "true"    ← sha2-256
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates        = SET{/watch/dup1.mkv, /watch/dup2.mkv}
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title             = "Inception"
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster       = bafy...
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng      = ...
 
-cid:bagacbabaec…  → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+- **128 bits**: collision-free under distributed minting; no allocator. Each
-cid:bafybeigd…    → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+  box minting its own rids for the same content is now the normal case, not a
-```
+  corner case — which is exactly why rids never travel.
+- **Time-sortable prefix**: rids minted in a window sort together
+  (`k >= 'file:01JKR8' AND k < 'file:01JKR9'` on the cache's merged rows),
+  and "older rid
+  survives" on a group merge is a plain string comparison.
+- **Monotonic-ish**: adjacent rids cluster in the cache's `file:<rid>/…`
+  key range.
 
-*(As shipped — the proposal originally wrote `cids` as a Redis SET of
+Encoded as 26-char Crockford Base32 (ULID-compatible) for compactness and
-`<algo>:<value>` tokens plus a `canonical_cid` scalar. See the Status note.)*
+case-insensitive matching. Migrated records kept their v1.x UUIDs as rids.
 
-### UUID format: UUIDv7
+### The `cids` key-set
 
-UUIDv7 chosen over UUIDv4 or 64-bit random:
+`cids/<bareCid>` = `"true"`, one claim per member, by whichever signer knows
+the link. A key-set merges conflict-free: a member is present while at least
+one vote supports it, and removal is a signed retraction. That is the reason
+the proposal's Redis SET was reversed before shipping, and it is what lets
+the claims model treat `cids` with no special merge code.
 
-- **128 bits**: collision-free under distributed minting; no central allocator.
+Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are **not**
-- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a
+claims — meta-core cannot sign and the disk can always be rescanned. The
-  given window. Restores some of the debuggability lost by going opaque.
+watcher records them in `local_file` and puts the midhash into `cid_group`;
-- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN
+the record view joins them in unsigned. When several local paths hold the
-  locality.
+same content, `filePath` is the lexicographically first one.
 
-Encoded as a 26-character Crockford Base32 string (ULID-compatible) for
+### Canonical CID selection (unchanged)
-compactness and case-insensitive matching.
 
-### New root fields owned by meta-core
+The canonical CID is a label, not a key, so nothing external breaks when it
-
+changes. It is computed on every read from the group's cids:
-The metadata schema is open — any writer can put any key under
-`file:<uuid>/`. This proposal does not change that. It adds three keys whose
-sole writer is meta-core itself:
-
-| Key | Type (as shipped) | Purpose |
-|-----|------|---------|
-| `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
-| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
-| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
-
-Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
-keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
-`subtitle/<lang>`, …) are unaffected.
-
-~~Redis SET is a departure from the all-strings convention in the current
-storage doc, but `file:__index__` is already a SET — so the precedent exists
-for using sets where they're the right structure.~~ **Reversed for `cids`.**
-The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
-so that two peers merging the same record can never disagree about set
-membership. `duplicates` remained a Redis SET (it is leader-local and never
-merged across peers).
-
-### Canonical CID selection
-
-The canonical CID is a label, not a key — so nothing external breaks when it
-changes. **As shipped it is not written at all**: it is computed on every read
-from the `cids` members, which makes the "update" cost zero and removes the
-reconciler entirely.
-
-Selection rule, highest-wins (this part shipped as designed —
-`internal/cid/rank.go`):
 
 1. IPFS CID (any variant)
 2. SHA-256 / SHA-3 family
 3. BitTorrent infohash (v2 preferred over v1)
 4. midhash256
 
-Ranked by **utility for external retrieval**, not by cryptographic strength.
+Ranked by **utility for external retrieval**, not cryptographic strength:
 midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
-externally meaningful for content addressing. btih ranks above midhash because
+externally meaningful; btih ranks above midhash because it enables swarm
-it enables swarm interop even if it's weaker as a digest.
+interop. The full ladder (including the custom locator codecs) lives in
+`internal/cid/rank.go`, mirrored in meta-search / meta-share
+(`scripts/check-cid-vectors.sh`).
 
-~~Reconciler runs on the leader, recomputes `canonical_cid` whenever
+## Lifecycles (v2.0.0)
-`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no
-reconciler. `GetMetadataDocument` folds the members through `cid.Better` on
-each read.
 
-## Internal operations
+### File arrival
-
-The meta-core storage interface adds five operations. None are exposed
-externally — they're internal to meta-core. External callers see only the
-CID-based API.
-
-```go
-// Mint creates a new root with no aliases. Called when the watcher
-// computes a midhash for a previously-unseen file. Returns the new UUID.
-Mint(filePath string, size int64, mtime int64) (uuid string, err error)
-
-// AddAlias registers a CID as resolving to this UUID. Updates both the
-// cid:<cid> reverse index and the file:<uuid>/cids/<cid> key-set member.
-// (As shipped: no reconciliation step — canonical is derived on read.)
-AddAlias(uuid string, cid string) error
-
-// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
-GetByCID(cid string) (uuid string, err error)
-
-// MergeIfAliasExists is called when the watcher mints a fresh UUID for a
-// file path but, on hashing, discovers the resulting CID already belongs
-// to a different UUID. Merges the new (mostly-empty) UUID into the
-// existing one, transferring filePath into the duplicates set, and
-// returns the canonical UUID.
-MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error)
-
-// DeleteRoot removes the entire entry, including every cid:* alias and
-// every file:<uuid>/* key. Used for file deletion events.
-DeleteRoot(uuid string) error
-```
-
-### Lifecycle: file arrival
 
 ```
-1. Watcher sees /watch/foo.mkv created.
+1. Watcher sees /files/watch/foo.mkv, computes its midhash M.
-2. Watcher computes midhash256 → "midhash256:abc...".
+2. local_file: /files/watch/foo.mkv → (M, size, mtime).
-3. Watcher calls GetByCID("midhash256:abc...").
+3. cid_group: M already grouped → nothing; otherwise M → fresh rid.
-   → uuid_X (existing entry) or "" (new file).
+4. Plugins process the file by cid M (the rid is never disclosed to them)
-
+   and sign claims about M with their own keys.
-4a. If existing (uuid_X):
+5. The full-hash plugin computes sha256 S and signs (plugin, M, cids/S) = "true".
-    - Add "/watch/foo.mkv" to file:<uuid_X>/duplicates.
+   → S joins M's group (or the two groups merge, older rid surviving).
-    - No new root created.
+   → The next read derives canonical_cid = S, because it outranks the midhash.
-    - meta-dup is notified via the existing keyspace event channel.
-
-4b. If new:
-    - Mint("/watch/foo.mkv", size, mtime) → uuid_Y.
-    - AddAlias(uuid_Y, "midhash256:abc...").
-    - Plugins now process the file, referenced by midhash256:abc...
-      (UUID is never disclosed to them.)
-
-5. Later, the full-hash plugin computes sha256.
-   - Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
-   - meta-core resolves midhash256:abc → uuid_Y via GetByCID.
-   - meta-core calls AddAlias(uuid_Y, "sha256:def...").
-   - Nothing else to do: the next read of this entry derives a canonical CID
-     of sha2-256, because it outranks the midhash.
 ```
 
-### Lifecycle: file deletion
+A second path with the same midhash is just another `local_file` row: same
+cid, same group, same record; the record's `filePath` is the
+lexicographically first of the paths.
 
-```
+### File deletion
-1. Watcher sees /watch/foo.mkv deleted.
+
-2. Resolve which entry owns this path:
+The watcher removes the path's `local_file` row; the record view loses that
-   a. If /watch/foo.mkv is in some entry's duplicates set:
+`filePath`. Claims are signed truth from their writers and are not deleted
-      - Remove from set. Done.
+by meta-core (the store is permanent and additive,
-   b. If /watch/foo.mkv is the filePath of some entry:
+`docs/project-architecture/metacore-as-permanent-store.md`).
-      - If duplicates is non-empty: promote one duplicate to filePath.
+
-      - If duplicates is empty: DeleteRoot(uuid).
+*(v1.x)* Deletion promoted a duplicate to `filePath`, or called `DeleteRoot`,
-        - Iterates the cids set, deletes each cid:<cid> reverse-index key.
+which removed every `cid:*` alias and every `file:<uuid>/*` key.
-        - Deletes every file:<uuid>/* key.
-        - Removes uuid from file:__index__.
-```
 
 ## Public API
 
 ```
-GET /api/meta/<cid>
+GET /api/meta/<cid>          → 200 merged document { "filePath", "canonical_cid", "cids": [...], ... }
-  → 200 { "filePath": "...", "canonical_cid": "...",
+                               (optional ?as=<uid>: the viewer's own votes win)
-          "cids": [...], "title": "...", "tmdb": {...}, ... }
+                             → 404 if no such cid
-  → 404 if no such CID
+GET /api/meta/<cid>/claims   → {rid, cids, claims}: every live signed claim of the cid's
-
+                               group, all signers (?key=, ?uid=); 404 if no such cid
-GET /api/file/<cid>          (already exists — implementation now O(1))
+GET /api/metadata/<rid>/claims → the same, addressed by rid (local consumers)
-  → streams file content
 ```
 
-The UUID is **not** included in the response. Clients address everything by
+The rid is not part of the cid-addressed contract and never appears in a
-CID; meta-core's choice of internal identifier remains a private
+claim. It is visible only to local consumers that list or address records by
-implementation detail and may change.
+rid (`/api/metadata/{rid}`, `/api/metadata/{rid}/claims`, `hash-ids`, SSE event
+keys; the claims routes also echo it as `rid`), and may change on a
+group merge (delete event, then re-resolve by cid).
 
-Auth: `/api/meta/<cid>` follows the same pattern as `/api/file/<cid>` and
+Auth: `/api/meta/<cid>` bypasses the perimeter like `/api/file/<cid>` — it is
-bypasses the auth perimeter. It's the metadata twin of the content endpoint,
+the metadata twin of the content endpoint.
-and meta-share calls it from federated search responses.
 
-## Cross-service impact
+## Cross-service impact (v2.0.0)
 
-This is a meta-core schema change. Other services interact only through CIDs,
+- **Writers** (plugins, self-publishing feeders, gateway core, meta-share,
-so the impact is contained but real.
+  meta-search): address claims by cid only and sign them; they never resolve
+  a rid.
+- **meta-fuse, meta-stremio**: unchanged — merged reads and SSE keep their
+  shape; a group merge reaches them as ordinary `del` + `set` events.
+- **meta-search / meta-share**: announce the derived canonical cid (each
+  mirrors the Go rank) and relay raw claims; every cid of the group travels as
+  `cids/*` claims, so a peer querying by any digest gets a hit.
 
-- **meta-sort**: writers must resolve plugin callbacks to a UUID before
+## Migration (history)
-  writing. The existing `LeaderClient` gains a `getByCID` cache.
-- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the
-  ID format changes.
-- **meta-stremio**: queries by CID; needs no logic change beyond its storage
-  client.
-- **meta-dup**: gains a cleaner data source — duplicates are now explicit in
-  the `duplicates` set, not implicit in colliding writes.
-- **meta-search / meta-share**: announce the *derived* canonical CID rather
-  than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata`
-  rather than reading a stored field. Federated search responses include all
-  CIDs from the `cids` key-set so any peer querying by any digest gets a hit.
-  (Written before the 2026-06 discovery/transport split; discovery is now
-  meta-search's job.)
-- **Plugins**: no change. They speak CIDs only.
 
-## Migration
+> **The v1.0 sweep below was never run.** The alpha shipped behind a
+> clean-wipe gate. The v1.x → v2.0.0 conversion of watch.nsl.sh is a
+> one-off tool outside the service image, `packages/meta-core/tools/claims-migrate`
+> (canonical cid by rank per record; `cid:` aliases → `cid_group`, rids keep
+> their UUIDs, and the `cid:` index wins when a record's `cids/*` member
+> disagrees; fields re-signed by a throwaway legacy key); every other box
+> starts empty. Implementation basis §11.
 
-> **Historical — this sweep was never run.** The alpha shipped behind a
+*(proposal)* One-time sweep over `file:__index__`: mint a uuid, `AddAlias`
-> clean-wipe gate (Redis was flushed and the library re-ingested), so no
+the old hashID and every `cid_*` field, `RENAME` every property under the new
-> in-place migration code exists. Kept for the record.
+root, reconcile canonical, swap the index entry; crash-recoverable by
-
+detecting which step completed.
-One-time sweep, idempotent, runs on first boot after upgrade:
-
-```
-For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …):
-  1. Mint a fresh uuid_Y.
-  2. AddAlias(uuid_Y, hashID).  The existing primary becomes an alias.
-  3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value).
-  4. RENAME file:<hashID>/<property> → file:<uuid_Y>/<property> for every
-     property (Lua script, one entry at a time; non-atomic across entries,
-     but each entry stays internally consistent).
-  5. Reconcile canonical_cid.
-  6. Remove hashID from file:__index__, add uuid_Y.
-```
-
-Crash-recoverable: re-running the sweep finds entries with both old and new
-roots and resumes by detecting which step completed (presence of
-`file:<uuid>/cids` indicates step 3 finished; absence of any
-`file:<hashID>/*` keys indicates step 4 finished).
 
 ## Decisions recorded
 
 | Question | Decision | Notes |
 |---|---|---|
-| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. |
+| When is the rid minted? | When a cid is first seen with no link to a known group | v1.x: at first hash of a watched file. Now any first-seen cid (watcher midhash or a claim about a new cid) mints one |
-| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. |
+| Rid format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe |
-| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
+| Rid visible externally? | No — never signed, never on the swarm/gateway wire | Local HTTP consumers may hold it |
-| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
+| What do signed votes name? | A cid | Every peer agrees on a cid; rids differ per box |
-| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
+| Record identity | A group of cids (`cid_group`, in truth) | Linked by `cids/<other>` votes |
-| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/<cid>` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. |
+| Group merge | Older rid survives; losing rid → `del` events | No `rid_alias` (out of scope for phase 1) |
-| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. |
+| Duplicate paths | `local_file` rows sharing a midhash (cache) | v1.x: `duplicates` SET, then `dup:<uuid>/<path>` |
-| Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
+| Canonical CID rank | IPFS > sha-family > btih > midhash, derived on read | Utility-weighted; never stored |
+| `cids` shape | Key-set of bare CIDs, one claim per member | Merges with the generic rule; a SET would not |
+| Reverse index | `cid_group(cid PK → rid)` in truth, copied to cache `cid:<cid>` rows for reads | v1.x: flat `cid:<cid>` keys were the only copy |
+| Unlinking | A retracted `cids/*` link does not split a group | Phase 1 |
 
 ## Open questions
 
-- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints
+- **Discovery-time rid minting.** Minting at discovery (before hashing)
-  a UUID at discovery (before hashing), then `AddAlias` runs when the midhash
+  would give two same-content paths two rids until hashing links them. The
-  arrives. Requires an explicit "reconcile if alias already exists" step
+  claims model already handles that case — it is an ordinary group merge —
-  because two paths with the same content would each get their own UUID until
+  so this is now a policy choice, not a missing mechanism.
-  hashing reveals the collision. Worth doing eventually; not required for
+- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` for
-  this iteration.
+  external IPFS clients? Adjacent, not part of this design.
-
+- ~~**meta-share announce strategy.**~~ Settled: the canonical cid addresses
-- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every*
+  the record, every group cid travels as a `cids/*` claim.
-  member of `cids` on the wire (`Record.cids`) and addresses itself by the
+- ~~**Keyspace-notification consumers.**~~ Obsolete since Redis left.
-  single derived canonical (`Record.cid`), so a peer querying by any digest
-  gets a hit without the DHT footprint of announcing each one separately.
-
-- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
-  content-streaming endpoint for external IPFS clients? Adjacent to this
-  design but not part of it.
-
-- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`)
-  handle `file:<type>:<hash>/...` with a special split rule for the colon
-  inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies
-  — but any external consumer subscribed by hashID pattern needs updating.
-  Inventory those before flipping the migration.
 
 ## Non-goals
 
-- **Not** a content-replication or IPFS-pinning system. Registering an IPFS
+- **Not** a content-replication or pinning system. A `cids/<ipfs-cid>` vote
-  CID alias does not pin the file anywhere — it only states "if somebody
+  pins nothing; it states "hashing these bytes with IPFS's chunker gives this
-  hashes this file with IPFS's chunker, this is the CID they'd get."
+  CID."
-- **Not** a versioning scheme. Metadata is a snapshot of current state;
+- **Not** a history. `ts` versions each `(uid, cid, key)` and the newest vote
-  historical versions are out of scope.
+  replaces the older one; only tombstones are kept, to block replay.
-- **Not** a strong-consistency system. Reverse-index writes happen under the
+- **Not** strongly consistent across the two databases: the cache is applied
-  leader lock; reads from followers may briefly miss a freshly-added alias.
+  after the truth commit. `POST /api/claims` waits for the apply before
+  answering, so the writer reads its own write; another reader may briefly
+  miss a just-accepted claim. It is behind, never wrong.

editor/src/api/claimsApi.ts

@@ -0,0 +1,135 @@
+// Signed claims (meta-core v2.0.0). The editor no longer writes keys: an edit
+// is a claim signed as the chosen identity account — through meta-core's
+// account-key service (/api/identity/sign, compact low-S) — and posted to
+// POST /api/claims. That vote wins in that account's view; the shared merged
+// value only moves if enough signers agree.
+//
+// MMUR2 bytes (must match internal/claim and its golden vectors):
+//   "MMUR2\n" || canonical CBOR [cid, uid, key, value, ts]
+
+export interface Claim {
+  cid: string;
+  uid: string;
+  key: string;
+  value: string | { b64: string } | null;
+  ts: number;
+  sig: string;
+}
+
+export interface ClaimsResponse {
+  rid: string;
+  cids: string[];
+  claims: Claim[];
+}
+
+export interface Account {
+  uid: string;
+  curve: string;
+  createdAt: number;
+}
+
+const SIGNER_LS_KEY = 'kv-editor.signer.v1';
+
+async function jsonOrThrow<T>(res: Response): Promise<T> {
+  if (!res.ok) {
+    let msg = `HTTP ${res.status}`;
+    try {
+      const body = await res.json();
+      if (body?.message) msg = body.message;
+      else if (body?.error) msg = body.error;
+    } catch {}
+    throw new Error(msg);
+  }
+  return res.json();
+}
+
+// --- canonical CBOR for the five MMUR2 shapes -------------------------------
+
+function head(out: number[], major: number, n: number) {
+  const m = major << 5;
+  if (n < 24) out.push(m | n);
+  else if (n <= 0xff) out.push(m | 24, n);
+  else if (n <= 0xffff) out.push(m | 25, n >> 8, n & 0xff);
+  else if (n <= 0xffffffff) out.push(m | 26, (n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff);
+  else {
+    const hi = Math.floor(n / 2 ** 32);
+    const lo = n >>> 0;
+    out.push(m | 27, (hi >>> 24) & 0xff, (hi >>> 16) & 0xff, (hi >>> 8) & 0xff, hi & 0xff,
+      (lo >>> 24) & 0xff, (lo >>> 16) & 0xff, (lo >>> 8) & 0xff, lo & 0xff);
+  }
+}
+
+function text(out: number[], s: string) {
+  const b = new TextEncoder().encode(s);
+  head(out, 3, b.length);
+  for (const x of b) out.push(x);
+}
+
+export function signedBytes(cid: string, uid: string, key: string, value: string | null, ts: number): Uint8Array {
+  const out: number[] = [...new TextEncoder().encode('MMUR2\n')];
+  head(out, 4, 5);
+  text(out, cid);
+  text(out, uid);
+  text(out, key);
+  if (value === null) out.push(0xf6);
+  else text(out, value);
+  head(out, 0, ts);
+  return new Uint8Array(out);
+}
+
+function b64(bytes: Uint8Array): string {
+  let s = '';
+  for (const x of bytes) s += String.fromCharCode(x);
+  return btoa(s);
+}
+
+// --- signer -----------------------------------------------------------------
+
+export function getSigner(): string {
+  try { return localStorage.getItem(SIGNER_LS_KEY) ?? ''; } catch { return ''; }
+}
+
+export function setSigner(uid: string) {
+  try { localStorage.setItem(SIGNER_LS_KEY, uid); } catch {}
+}
+
+export const ClaimsAPI = {
+  async accounts(): Promise<Account[]> {
+    const r = await jsonOrThrow<{ accounts: Account[] }>(await fetch('/api/identity/accounts'));
+    return r.accounts ?? [];
+  },
+
+  async forRecord(rid: string, key?: string): Promise<ClaimsResponse> {
+    const q = key ? `?key=${encodeURIComponent(key)}` : '';
+    return jsonOrThrow(await fetch(`/api/metadata/${encodeURIComponent(rid)}/claims${q}`));
+  },
+
+  // vote signs value (null = retraction) for key on the record's subject cid
+  // as the chosen account and posts it.
+  async vote(rid: string, key: string, value: string | null): Promise<void> {
+    const uid = getSigner();
+    if (!uid) throw new Error('choose a signing account first (top right)');
+    const rec = await this.forRecord(rid);
+    const cid = rec.cids[0];
+    if (!cid) throw new Error(`record ${rid} has no cid`);
+    // ts must beat this account's previous vote on the key.
+    const prev = rec.claims.filter((c) => c.uid === uid && c.key === key).map((c) => c.ts);
+    const ts = Math.max(Date.now(), ...prev.map((t) => t + 1));
+    const bytes = signedBytes(cid, uid, key, value, ts);
+    const signed = await jsonOrThrow<{ sigB64: string }>(await fetch('/api/identity/sign', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ uid, bytesB64: b64(bytes), format: 'compact' }),
+    }));
+    const claim: Claim = { cid, uid, key, value, ts, sig: signed.sigB64 };
+    const res = await jsonOrThrow<{ status: string; reason?: string }[]>(await fetch('/api/claims', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify([claim]),
+    }));
+    const st = res[0]?.status;
+    if (st !== 'accepted' && st !== 'unchanged') {
+      throw new Error(`claim ${st}${res[0]?.reason ? ': ' + res[0].reason : ''}`);
+    }
+  },
+};

editor/src/api/kvApi.ts

@@ -1,8 +1,17 @@
-// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of
+// The editor's key tree. Since meta-core v2.0.0 the records are no longer raw
-// string-valued keys; this module wraps the S3-style listing, search, and
+// keys: `file:<rid>/<field>` is the MERGED view, derived from signed claims.
-// per-key get/put/delete operations.
+// This module keeps the slash-tree the components were built on, as a view:
+//
+//   - reads of `file:` keys come from the merged record API
+//     (/api/metadata/hash-ids, /api/metadata/{rid});
+//   - writes of `file:` keys are signed claims (claimsApi.ts): put = a vote,
+//     del = a retraction of the chosen account's vote;
+//   - every other key is service scratch KV, through /api/kv/*.
+
+import { ClaimsAPI } from './claimsApi';
 
 const API_BASE = '/api/kv';
+const RECORD_NS = 'file:';
 
 export interface KVTreeResponse {
   prefix: string;
@@ -61,6 +70,45 @@ async function jsonOrThrow<T>(res: Response): Promise<T> {
   return res.json();
 }
 
+// splitRecordKey decomposes file:<rid>/<field> (field may be empty).
+export function splitRecordKey(key: string): { rid: string; field: string } | null {
+  if (!key.startsWith(RECORD_NS)) return null;
+  const rest = key.slice(RECORD_NS.length);
+  const slash = rest.indexOf('/');
+  if (slash <= 0) return null;
+  return { rid: rest.slice(0, slash), field: rest.slice(slash + 1) };
+}
+
+let hashIdsCache: { at: number; ids: string[] } | null = null;
+
+async function hashIds(): Promise<string[]> {
+  if (hashIdsCache && Date.now() - hashIdsCache.at < 5000) return hashIdsCache.ids;
+  const r = await jsonOrThrow<{ hashIds: string[] }>(await fetch('/api/metadata/hash-ids'));
+  hashIdsCache = { at: Date.now(), ids: (r.hashIds ?? []).slice().sort() };
+  return hashIdsCache.ids;
+}
+
+async function recordFields(rid: string): Promise<Record<string, string>> {
+  const res = await fetch(`/api/metadata/${encodeURIComponent(rid)}`);
+  if (res.status === 404) return {};
+  const r = await jsonOrThrow<{ metadata: Record<string, string> }>(res);
+  return r.metadata ?? {};
+}
+
+// listUnder lists the children of prefix among keys, S3-style.
+function listUnder(prefix: string, delimiter: string, keys: string[]) {
+  const branches = new Set<string>();
+  const leaves: string[] = [];
+  for (const k of keys) {
+    if (!k.startsWith(prefix)) continue;
+    const rest = k.slice(prefix.length);
+    const i = delimiter ? rest.indexOf(delimiter) : -1;
+    if (i >= 0) branches.add(prefix + rest.slice(0, i + delimiter.length));
+    else leaves.push(k);
+  }
+  return { branches: [...branches].sort(), leaves: leaves.sort() };
+}
+
 export const KVAPI = {
   async info(): Promise<KVInfoResponse> {
     return jsonOrThrow(await fetch(`${API_BASE}/info`));
@@ -72,17 +120,34 @@ export const KVAPI = {
     cursor?: string;
     max?: number;
   }): Promise<KVTreeResponse> {
+    const prefix = params.prefix ?? '';
+    const delimiter = params.delimiter ?? '';
+    const done = (branches: string[], leaves: string[]): KVTreeResponse =>
+      ({ prefix, delimiter, branches, leaves, cursor: '0', hasMore: false });
+
+    if (prefix.startsWith(RECORD_NS)) {
+      const parts = splitRecordKey(prefix);
+      if (!parts) {
+        const ids = await hashIds();
+        return done(ids.map((id) => `${RECORD_NS}${id}/`), []);
+      }
+      const fields = await recordFields(parts.rid);
+      const keys = Object.keys(fields).map((f) => `${RECORD_NS}${parts.rid}/${f}`);
+      const l = listUnder(prefix, delimiter, keys);
+      return done(l.branches, l.leaves);
+    }
     const q = new URLSearchParams();
-    if (params.prefix !== undefined) q.set('prefix', params.prefix);
+    q.set('prefix', prefix);
-    if (params.delimiter !== undefined) q.set('delimiter', params.delimiter);
+    q.set('delimiter', delimiter);
     if (params.cursor) q.set('cursor', params.cursor);
     if (params.max) q.set('max', String(params.max));
-    return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`));
+    const svc = await jsonOrThrow<KVTreeResponse>(await fetch(`${API_BASE}/tree?${q}`));
+    if (prefix === '' && delimiter) {
+      svc.branches = [RECORD_NS, ...svc.branches.filter((b) => b !== RECORD_NS)];
+    }
+    return svc;
   },
 
-  // Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap
-  // is reached. Useful for branches with up to a few thousand children where
-  // pagination UX would just be a worse spinner.
   async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
     const seenLeaves = new Set<string>();
     const seenBranches = new Set<string>();
@@ -105,22 +170,73 @@ export const KVAPI = {
     };
   },
 
+  // search finds keys whose path contains the substring: records matched by
+  // free text (each matching field key), plus service keys.
   async search(contains: string, limit = 500): Promise<KVSearchResponse> {
+    const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record<string, string> }[] }>(
+      await fetch('/api/metadata/search', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ query: contains, limit: 100 }),
+      }));
+    const needle = contains.toLowerCase();
+    const keys: string[] = [];
+    for (const hit of r.results ?? []) {
+      for (const [f, v] of Object.entries(hit.metadata)) {
+        const k = `${RECORD_NS}${hit.hashId}/${f}`;
+        if (k.toLowerCase().includes(needle) || v.toLowerCase().includes(needle)) keys.push(k);
+      }
+    }
     const q = new URLSearchParams({ contains, limit: String(limit) });
-    return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`));
+    const svc = await jsonOrThrow<KVSearchResponse>(await fetch(`${API_BASE}/search?${q}`));
+    const all = [...keys, ...svc.keys].sort();
+    return { contains, keys: all.slice(0, limit), truncated: all.length > limit || svc.truncated };
   },
 
+  // find returns record fields (one of `fields`) whose value contains the
+  // substring, via the merged-record search.
   async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
-    const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) });
+    const matches: KVFindMatch[] = [];
-    return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`));
+    let truncated = false;
+    for (const field of fields) {
+      const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record<string, string> }[] }>(
+        await fetch('/api/metadata/search', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ filters: [{ property: field, values: [contains] }], limit }),
+        }));
+      for (const hit of r.results ?? []) {
+        const v = hit.metadata[field];
+        if (v === undefined) continue;
+        const entryPath = `${RECORD_NS}${hit.hashId}/`;
+        matches.push({ key: entryPath + field, value: v, entryPath, field });
+        if (matches.length >= limit) { truncated = true; break; }
+      }
+      if (truncated) break;
+    }
+    matches.sort((a, b) => a.key.localeCompare(b.key));
+    return { contains, fields, matches, truncated };
   },
 
   async get(key: string): Promise<KVValueResponse> {
+    const parts = splitRecordKey(key);
+    if (parts) {
+      const fields = await recordFields(parts.rid);
+      const value = fields[parts.field];
+      return { key, type: 'string', value: value ?? '', exists: value !== undefined };
+    }
     const q = new URLSearchParams({ key });
     return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
   },
 
+  // put on a record key is the chosen account's signed vote.
   async put(key: string, value: string): Promise<KVValueResponse> {
+    const parts = splitRecordKey(key);
+    if (parts) {
+      await ClaimsAPI.vote(parts.rid, parts.field, value);
+      hashIdsCache = null;
+      return { key, type: 'string', value, exists: true };
+    }
     const res = await fetch(`${API_BASE}/value`, {
       method: 'PUT',
       headers: { 'Content-Type': 'application/json' },
@@ -129,7 +245,14 @@ export const KVAPI = {
     return jsonOrThrow(res);
   },
 
+  // del on a record key retracts the chosen account's vote; the merged value
+  // stays while other signers still support it.
   async del(key: string): Promise<{ key: string; deleted: boolean }> {
+    const parts = splitRecordKey(key);
+    if (parts) {
+      await ClaimsAPI.vote(parts.rid, parts.field, null);
+      return { key, deleted: true };
+    }
     const q = new URLSearchParams({ key });
     return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
   },

editor/src/api/snapshotApi.ts

@@ -1,96 +0,0 @@
-const API_BASE = '/api/snapshot';
-
-export type ImportMode = 'replace' | 'merge';
-export type ImportConflict = 'mine' | 'source';
-
-export interface FileImportResult {
-  hashId: string;
-  added: number;
-  updated: number;
-  unchanged: number;
-  kept: number;
-  deleted: number;
-  error?: string;
-}
-
-export interface PluginOutputResult {
-  written: number;
-  skipped: number;
-}
-
-export interface ImportResult {
-  mode: ImportMode;
-  conflict?: ImportConflict;
-  dryRun: boolean;
-  totalFiles: number;
-  filesOk: number;
-  filesFailed: number;
-  files?: FileImportResult[];
-  pluginOutput?: PluginOutputResult;
-}
-
-export interface WipeResult {
-  metadataDeleted: number;
-}
-
-export class SnapshotAPI {
-  /**
-   * Trigger a download of a metadata snapshot ZIP.
-   * Browser handles the download via a synthetic <a> click.
-   */
-  static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
-    const parts: string[] = [];
-    if (includes.pluginOutput) parts.push('plugin-output');
-    const qs = parts.length ? `?include=${parts.join(',')}` : '';
-    const a = document.createElement('a');
-    a.href = `${API_BASE}/export${qs}`;
-    a.rel = 'noopener';
-    document.body.appendChild(a);
-    a.click();
-    a.remove();
-  }
-
-  /**
-   * Upload a snapshot ZIP and apply it.
-   * Mode replace deletes and rewrites each CID's properties.
-   * Mode merge preserves untouched fields; conflict picks who wins on overlap.
-   */
-  static async runImport(
-    file: File,
-    opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
-  ): Promise<ImportResult> {
-    const params = new URLSearchParams();
-    params.set('mode', opts.mode);
-    if (opts.mode === 'merge') {
-      params.set('conflict', opts.conflict ?? 'source');
-    }
-    if (opts.dryRun) {
-      params.set('dry_run', 'true');
-    }
-    const fd = new FormData();
-    fd.append('snapshot', file);
-    const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
-      method: 'POST',
-      body: fd,
-    });
-    if (!response.ok) {
-      const err = await response.json().catch(() => ({}));
-      throw new Error(err.message || err.error || 'Import failed');
-    }
-    return response.json();
-  }
-
-  /**
-   * Wipe Redis metadata. Pass scope=metadata.
-   */
-  static async wipe(scope: 'metadata'): Promise<WipeResult> {
-    const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
-      method: 'POST',
-    });
-    if (!response.ok) {
-      const err = await response.json().catch(() => ({}));
-      throw new Error(err.message || err.error || 'Wipe failed');
-    }
-    return response.json();
-  }
-}

editor/src/components/SnapshotPanel.css

@@ -1,209 +0,0 @@
-.snapshot-panel {
-  max-width: 920px;
-  margin: 0 auto;
-  padding: 1.5rem;
-}
-
-.snapshot-panel h2 {
-  margin: 0 0 0.5rem;
-}
-
-.snapshot-intro {
-  color: var(--text-muted, #888);
-  margin-bottom: 1.5rem;
-}
-
-.snapshot-section {
-  border: 1px solid var(--border, #e0e0e0);
-  border-radius: 6px;
-  padding: 1rem 1.25rem;
-  margin-bottom: 1.25rem;
-}
-
-.snapshot-section.danger {
-  border-color: #d04a4a;
-  background: rgba(208, 74, 74, 0.04);
-}
-
-.snapshot-section h3 {
-  margin: 0 0 0.5rem;
-}
-
-.snapshot-row {
-  display: flex;
-  gap: 1rem;
-  align-items: flex-start;
-  margin: 0.75rem 0;
-  flex-wrap: wrap;
-}
-
-.snapshot-file {
-  display: flex;
-  flex-direction: column;
-  gap: 0.25rem;
-}
-
-.snapshot-file span {
-  font-size: 0.85rem;
-  color: var(--text-muted, #888);
-}
-
-.snapshot-fieldset {
-  border: 1px solid var(--border, #e0e0e0);
-  border-radius: 4px;
-  padding: 0.5rem 0.75rem;
-  min-width: 280px;
-}
-
-.snapshot-fieldset[disabled] {
-  opacity: 0.5;
-}
-
-.snapshot-fieldset legend {
-  font-weight: 600;
-  font-size: 0.85rem;
-  padding: 0 0.25rem;
-}
-
-.snapshot-fieldset label {
-  display: block;
-  margin: 0.25rem 0;
-  font-size: 0.9rem;
-  cursor: pointer;
-}
-
-.snapshot-fieldset input[type='radio'] {
-  margin-right: 0.5rem;
-}
-
-.snapshot-btn {
-  background: var(--bg-button, #f0f0f0);
-  border: 1px solid var(--border, #c0c0c0);
-  color: var(--text, #222);
-  padding: 0.5rem 1rem;
-  border-radius: 4px;
-  cursor: pointer;
-  font-size: 0.95rem;
-}
-
-.snapshot-btn:hover:not(:disabled) {
-  background: var(--bg-button-hover, #e0e0e0);
-}
-
-.snapshot-btn:disabled {
-  opacity: 0.5;
-  cursor: not-allowed;
-}
-
-.snapshot-btn.primary {
-  background: #2a6df4;
-  border-color: #2a6df4;
-  color: white;
-}
-
-.snapshot-btn.primary:hover:not(:disabled) {
-  background: #1a5de4;
-}
-
-.snapshot-btn.danger {
-  background: #d04a4a;
-  border-color: #d04a4a;
-  color: white;
-}
-
-.snapshot-btn.danger:hover:not(:disabled) {
-  background: #b83838;
-}
-
-.snapshot-error {
-  background: rgba(208, 74, 74, 0.1);
-  color: #b83838;
-  padding: 0.5rem 0.75rem;
-  border-radius: 4px;
-  margin-top: 0.75rem;
-}
-
-.snapshot-info {
-  background: rgba(42, 109, 244, 0.08);
-  color: #1a5de4;
-  padding: 0.5rem 0.75rem;
-  border-radius: 4px;
-  margin-top: 0.75rem;
-}
-
-.snapshot-result {
-  margin-top: 0.75rem;
-  border: 1px solid var(--border, #e0e0e0);
-  border-radius: 4px;
-  padding: 0.75rem;
-}
-
-.snapshot-result-summary {
-  display: flex;
-  gap: 1rem;
-  flex-wrap: wrap;
-  margin-bottom: 0.5rem;
-  font-size: 0.9rem;
-}
-
-.snapshot-result-table {
-  width: 100%;
-  border-collapse: collapse;
-  font-size: 0.9rem;
-}
-
-.snapshot-result-table th,
-.snapshot-result-table td {
-  border: 1px solid var(--border, #e0e0e0);
-  padding: 0.35rem 0.5rem;
-  text-align: right;
-}
-
-.snapshot-result-table th {
-  background: var(--bg-subtle, #f5f5f5);
-  font-weight: 600;
-}
-
-.snapshot-result-errors {
-  margin-top: 0.5rem;
-  font-size: 0.85rem;
-}
-
-.snapshot-result-errors summary {
-  cursor: pointer;
-  color: #b83838;
-}
-
-.snapshot-result-errors ul {
-  margin: 0.5rem 0 0;
-  padding-left: 1.25rem;
-}
-
-.snapshot-result-errors code {
-  font-size: 0.8rem;
-  word-break: break-all;
-}
-
-.snapshot-checkbox {
-  display: flex;
-  align-items: flex-start;
-  gap: 0.5rem;
-  margin: 0.5rem 0 0.75rem;
-  cursor: pointer;
-  font-size: 0.95rem;
-}
-
-.snapshot-checkbox input {
-  margin-top: 0.2rem;
-}
-
-.snapshot-hint {
-  color: var(--text-muted, #888);
-  font-size: 0.9rem;
-}
-
-.snapshot-result-aside {
-  margin-top: 0.5rem;
-  font-size: 0.85rem;
-  color: var(--text-muted, #555);
-}

editor/src/components/SnapshotPanel.tsx

@@ -1,271 +0,0 @@
-import { useState } from 'react';
-import {
-  SnapshotAPI,
-  ImportMode,
-  ImportConflict,
-  ImportResult,
-} from '../api/snapshotApi';
-import './SnapshotPanel.css';
-
-export function SnapshotPanel() {
-  const [exportPluginOutput, setExportPluginOutput] = useState(false);
-  const [importFile, setImportFile] = useState<File | null>(null);
-  const [importMode, setImportMode] = useState<ImportMode>('merge');
-  const [importConflict, setImportConflict] = useState<ImportConflict>('source');
-  const [importing, setImporting] = useState(false);
-  const [importResult, setImportResult] = useState<ImportResult | null>(null);
-  const [importError, setImportError] = useState<string | null>(null);
-
-  const [wipeBusy, setWipeBusy] = useState(false);
-  const [wipeMessage, setWipeMessage] = useState<string | null>(null);
-
-  const runImport = async (dryRun: boolean) => {
-    if (!importFile) {
-      setImportError('Pick a snapshot ZIP first.');
-      return;
-    }
-    setImporting(true);
-    setImportError(null);
-    setImportResult(null);
-    try {
-      const result = await SnapshotAPI.runImport(importFile, {
-        mode: importMode,
-        conflict: importMode === 'merge' ? importConflict : undefined,
-        dryRun,
-      });
-      setImportResult(result);
-    } catch (e: any) {
-      setImportError(e.message || String(e));
-    } finally {
-      setImporting(false);
-    }
-  };
-
-  const runWipe = async () => {
-    const ok = window.confirm(
-      'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
-        'edit you made via this editor will be lost. Continue?'
-    );
-    if (!ok) return;
-    setWipeBusy(true);
-    setWipeMessage(null);
-    try {
-      const r = await SnapshotAPI.wipe('metadata');
-      setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
-    } catch (e: any) {
-      setWipeMessage(`Failed: ${e.message || e}`);
-    } finally {
-      setWipeBusy(false);
-    }
-  };
-
-  return (
-    <div className="snapshot-panel">
-      <h2>Snapshot</h2>
-      <p className="snapshot-intro">
-        Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
-        archives — one nested-JSON file per CID, plus a manifest.
-      </p>
-
-      <section className="snapshot-section">
-        <h3>Export</h3>
-        <p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
-        <label className="snapshot-checkbox">
-          <input
-            type="checkbox"
-            checked={exportPluginOutput}
-            onChange={(e) => setExportPluginOutput(e.target.checked)}
-          />
-          <span>
-            Include plugin output
-            <span className="snapshot-hint">
-              {' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
-            </span>
-          </span>
-        </label>
-        <button
-          className="snapshot-btn primary"
-          onClick={() =>
-            SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
-          }
-        >
-          Download snapshot.zip
-        </button>
-      </section>
-
-      <section className="snapshot-section">
-        <h3>Import</h3>
-        <div className="snapshot-row">
-          <label className="snapshot-file">
-            <span>Snapshot ZIP</span>
-            <input
-              type="file"
-              accept=".zip,application/zip"
-              onChange={(e) => {
-                setImportFile(e.target.files?.[0] ?? null);
-                setImportResult(null);
-                setImportError(null);
-              }}
-            />
-          </label>
-        </div>
-
-        <div className="snapshot-row">
-          <fieldset className="snapshot-fieldset">
-            <legend>Mode</legend>
-            <label>
-              <input
-                type="radio"
-                name="mode"
-                value="merge"
-                checked={importMode === 'merge'}
-                onChange={() => setImportMode('merge')}
-              />
-              Merge — keep existing fields not in the snapshot
-            </label>
-            <label>
-              <input
-                type="radio"
-                name="mode"
-                value="replace"
-                checked={importMode === 'replace'}
-                onChange={() => setImportMode('replace')}
-              />
-              Replace — wipe each imported CID and rewrite from snapshot
-            </label>
-          </fieldset>
-
-          <fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
-            <legend>Conflict</legend>
-            <label>
-              <input
-                type="radio"
-                name="conflict"
-                value="source"
-                checked={importConflict === 'source'}
-                onChange={() => setImportConflict('source')}
-              />
-              Source wins (overwrite my edits)
-            </label>
-            <label>
-              <input
-                type="radio"
-                name="conflict"
-                value="mine"
-                checked={importConflict === 'mine'}
-                onChange={() => setImportConflict('mine')}
-              />
-              Mine wins (keep my edits, fill in only missing fields)
-            </label>
-          </fieldset>
-        </div>
-
-        <div className="snapshot-row">
-          <button
-            className="snapshot-btn"
-            disabled={!importFile || importing}
-            onClick={() => runImport(true)}
-          >
-            Dry run
-          </button>
-          <button
-            className="snapshot-btn primary"
-            disabled={!importFile || importing}
-            onClick={() => runImport(false)}
-          >
-            {importing ? 'Importing…' : 'Apply import'}
-          </button>
-        </div>
-
-        {importError && <div className="snapshot-error">{importError}</div>}
-        {importResult && <ImportResultView result={importResult} />}
-      </section>
-
-      <section className="snapshot-section danger">
-        <h3>Wipe metadata</h3>
-        <p>
-          Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
-          Files on disk are untouched.
-        </p>
-        <button
-          className="snapshot-btn danger"
-          disabled={wipeBusy}
-          onClick={runWipe}
-        >
-          {wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
-        </button>
-        {wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
-      </section>
-    </div>
-  );
-}
-
-function ImportResultView({ result }: { result: ImportResult }) {
-  const totals = (result.files ?? []).reduce(
-    (acc, f) => {
-      acc.added += f.added;
-      acc.updated += f.updated;
-      acc.unchanged += f.unchanged;
-      acc.kept += f.kept;
-      acc.deleted += f.deleted;
-      return acc;
-    },
-    { added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
-  );
-  return (
-    <div className="snapshot-result">
-      <div className="snapshot-result-summary">
-        <strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
-        <span>
-          {result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
-        </span>
-        <span>
-          mode={result.mode}
-          {result.conflict ? `, conflict=${result.conflict}` : ''}
-        </span>
-      </div>
-      <table className="snapshot-result-table">
-        <thead>
-          <tr>
-            <th>added</th>
-            <th>updated</th>
-            <th>unchanged</th>
-            <th>kept (mine)</th>
-            <th>deleted (replace)</th>
-          </tr>
-        </thead>
-        <tbody>
-          <tr>
-            <td>{totals.added}</td>
-            <td>{totals.updated}</td>
-            <td>{totals.unchanged}</td>
-            <td>{totals.kept}</td>
-            <td>{totals.deleted}</td>
-          </tr>
-        </tbody>
-      </table>
-      {result.pluginOutput && (
-        <div className="snapshot-result-aside">
-          plugin-output: {result.pluginOutput.written} written
-          {result.pluginOutput.skipped > 0
-            ? `, ${result.pluginOutput.skipped} skipped (existed)`
-            : ''}
-        </div>
-      )}
-      {result.files && result.files.some((f) => f.error) && (
-        <details className="snapshot-result-errors">
-          <summary>Per-file errors</summary>
-          <ul>
-            {result.files
-              .filter((f) => f.error)
-              .map((f) => (
-                <li key={f.hashId}>
-                  <code>{f.hashId}</code>: {f.error}
-                </li>
-              ))}
-          </ul>
-        </details>
-      )}
-    </div>
-  );
-}

editor/src/kv/ClaimsPanel.tsx

@@ -0,0 +1,45 @@
+import { useEffect, useState } from 'react';
+import { Claim, ClaimsAPI, getSigner } from '../api/claimsApi';
+import { splitRecordKey } from '../api/kvApi';
+
+// ClaimsPanel lists the signed claims behind one merged field: who said what,
+// and when. The merged value is what most signers agree on.
+export function ClaimsPanel({ recordKey, reloadToken }: { recordKey: string; reloadToken: number }) {
+  const [claims, setClaims] = useState<Claim[] | null>(null);
+  const [error, setError] = useState('');
+  const parts = splitRecordKey(recordKey);
+
+  useEffect(() => {
+    if (!parts || !parts.field) return;
+    let cancelled = false;
+    ClaimsAPI.forRecord(parts.rid, parts.field)
+      .then((r) => { if (!cancelled) setClaims(r.claims); })
+      .catch((e) => { if (!cancelled) setError(e.message); });
+    return () => { cancelled = true; };
+  }, [recordKey, reloadToken]); // eslint-disable-line react-hooks/exhaustive-deps
+
+  if (!parts || !parts.field) return null;
+  if (error) return <div className="kv-claims kv-pane-status">claims: {error}</div>;
+  if (!claims) return <div className="kv-claims kv-pane-status">loading claims…</div>;
+  const me = getSigner();
+  return (
+    <div className="kv-claims">
+      <div className="kv-leaf-label">Claims ({claims.length})</div>
+      {claims.length === 0 ? (
+        <div className="kv-pane-hint">No signed claim — a box-local fact (file path, size, mtime) or a group member.</div>
+      ) : (
+        <table className="kv-claims-table">
+          <tbody>
+            {claims.map((c) => (
+              <tr key={c.uid + c.cid}>
+                <td title={c.uid}><code>{c.uid.slice(0, 10)}…</code>{c.uid === me ? ' (you)' : ''}</td>
+                <td>{c.value === null ? <em>retracted</em> : typeof c.value === 'string' ? c.value : <em>ciphertext</em>}</td>
+                <td title={new Date(c.ts).toISOString()}>{new Date(c.ts).toLocaleString()}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      )}
+    </div>
+  );
+}

editor/src/kv/KVApp.css

@@ -894,3 +894,9 @@
 .kv-modal-close:hover {
   opacity: 1;
 }
+
+.kv-signer { font-size: 12px; display: inline-flex; align-items: center; gap: 4px; }
+.kv-signer select { font-size: 12px; }
+.kv-claims { margin-top: 16px; }
+.kv-claims-table { width: 100%; border-collapse: collapse; font-size: 12px; }
+.kv-claims-table td { padding: 3px 6px; border-top: 1px solid rgba(127, 127, 127, 0.25); vertical-align: top; }

editor/src/kv/KVApp.tsx

@@ -10,7 +10,7 @@ const FILE_BRANCH_RE = /^file:[^/]+\/$/;
 import { SearchFieldChips } from './SearchFieldChips';
 import { ToastStack, makeToast } from './Toast';
 import { ToastEntry } from './types';
-import { SnapshotPanel } from '../components/SnapshotPanel';
+import { SignerPicker } from './SignerPicker';
 import './KVApp.css';
 
 const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
@@ -32,7 +32,6 @@ function loadSearchFields(): string[] {
 //
 // A sticky header carries the search bar, the breadcrumb, and basic stats.
 export default function KVApp() {
-  const [snapshotOpen, setSnapshotOpen] = useState(false);
   // Selected node. branchPath is set when a branch is selected; selectedKey
   // when a leaf is. Mutually exclusive.
   const [selectedKey, setSelectedKey] = useState<string | null>(null);
@@ -176,14 +175,7 @@ export default function KVApp() {
             onClick={() => setReloadToken((n) => n + 1)}
             title="refresh tree"
           >Refresh</button>
-          <button
+          <SignerPicker onError={(msg) => pushToast(makeToast('error', msg))} />
-            className="kv-btn-icon kv-app-gear"
-            onClick={() => setSnapshotOpen(true)}
-            title="Snapshot — export / import / wipe"
-            aria-label="Open snapshot settings"
-          >
-            ⚙
-          </button>
         </div>
         <SearchFieldChips fields={searchFields} onChange={setSearchFields} />
         {breadcrumb && breadcrumb.length > 0 && (
@@ -284,28 +276,6 @@ export default function KVApp() {
         </main>
       </div>
 
-      {snapshotOpen && (
-        <div
-          className="kv-modal-backdrop"
-          onClick={() => setSnapshotOpen(false)}
-        >
-          <div
-            className="kv-modal"
-            onClick={(e) => e.stopPropagation()}
-            role="dialog"
-            aria-modal="true"
-          >
-            <button
-              className="kv-modal-close"
-              onClick={() => setSnapshotOpen(false)}
-              aria-label="Close"
-              title="Close"
-            >×</button>
-            <SnapshotPanel />
-          </div>
-        </div>
-      )}
-
       <ToastStack toasts={toasts} onDismiss={dismissToast} />
     </div>
   );

editor/src/kv/KVLeafEditor.tsx

@@ -5,9 +5,10 @@ import { ALL_TYPES, describeType, detectType } from './typeHeuristics';
 import { lookupSchemaForKey } from './schemaLookup';
 import { LeafValueEditor } from './leafRenderers';
 import { FieldType } from './types';
+import { ClaimsPanel } from './ClaimsPanel';
 
 interface Props {
-  // Full Redis key path of the selected leaf.
+  // Full key path of the selected leaf (file:<rid>/<field> = merged view).
   selectedKey: string;
   // Bumped after a delete or external change to force refetch.
   reloadToken: number;
@@ -71,7 +72,7 @@ export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onE
   };
 
   const del = async () => {
-    if (!confirm(`Delete ${selectedKey}?`)) return;
+    if (!confirm(`Retract your vote on ${selectedKey}? Other signers' claims keep the merged value.`)) return;
     setSaving(true);
     try {
       await KVAPI.del(selectedKey);
@@ -136,9 +137,11 @@ export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onE
             className="kv-btn kv-btn-danger"
             onClick={del}
             disabled={saving}
-          >Delete</button>
+          >Retract</button>
         )}
       </div>
+
+      <ClaimsPanel recordKey={selectedKey} reloadToken={reloadToken} />
     </div>
   );
 }

editor/src/kv/SignerPicker.tsx

@@ -0,0 +1,36 @@
+import { useEffect, useState } from 'react';
+import { Account, ClaimsAPI, getSigner, setSigner } from '../api/claimsApi';
+
+// SignerPicker chooses the identity account whose key signs the editor's
+// edits. Every save is that account's claim — it wins in that account's view.
+export function SignerPicker({ onError }: { onError: (msg: string) => void }) {
+  const [accounts, setAccounts] = useState<Account[]>([]);
+  const [uid, setUid] = useState(getSigner());
+
+  useEffect(() => {
+    ClaimsAPI.accounts()
+      .then((list) => {
+        setAccounts(list);
+        if (!uid && list.length === 1) {
+          setSigner(list[0].uid);
+          setUid(list[0].uid);
+        }
+      })
+      .catch((e) => onError(`accounts: ${e.message}`));
+  }, []); // eslint-disable-line react-hooks/exhaustive-deps
+
+  return (
+    <label className="kv-signer" title="Edits are claims signed by this account">
+      sign as{' '}
+      <select
+        value={uid}
+        onChange={(e) => { setSigner(e.target.value); setUid(e.target.value); }}
+      >
+        <option value="">— choose an account —</option>
+        {accounts.map((a) => (
+          <option key={a.uid} value={a.uid}>{a.uid.slice(0, 10)}…</option>
+        ))}
+      </select>
+    </label>
+  );
+}

go.mod

@@ -1,21 +1,25 @@
 module github.com/metazla/meta-core
 
-go 1.21
+go 1.26.0
 
 require (
-	github.com/alicebob/miniredis/v2 v2.31.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/mr-tron/base58 v1.3.0
-	github.com/redis/go-redis/v9 v9.4.0
 	golang.org/x/crypto v0.31.0
 	golang.org/x/net v0.21.0
+	modernc.org/sqlite v1.60.1
 )
 
 require (
-	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
-	github.com/yuin/gopher-lua v1.1.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	modernc.org/libc v1.77.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
 )
+
+require golang.org/x/sys v0.48.0

go.sum

@@ -1,36 +1,62 @@
-github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
-github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
-github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
-github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
-github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
-github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
 github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
 github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
 github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
-github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
-github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
+golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
+modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
+modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
+modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
+modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
+modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/api/admin_migrate.go

@@ -1,101 +0,0 @@
-package api
-
-import (
-	"net/http"
-	"strconv"
-)
-
-// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
-// stranded midhash-rooted entries into their UUID roots. See
-// storage.MigrateDualRoots for the data-flow detail and
-// docs/api-mediated-access.md for the bug background.
-//
-// Returns {fixed: N} on success. Safe to call repeatedly — entries already
-// migrated are skipped on the next pass.
-func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	fixed, err := s.storage.MigrateDualRootsWithTimeout()
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"fixed":   fixed,
-	})
-}
-
-// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the
-// retired `domain=film|tv` values to `screen` and backfills `workForm` from
-// `contentKind` (METADATA_KEYS.md §14.17). See
-// storage.MigrateDomainScreen for why the vocabulary change is paid in the
-// data rather than with a read-side alias.
-//
-// Returns {fixed: N} — records changed, not records examined. Safe to call
-// repeatedly: a record already on the current vocabulary is skipped.
-func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	fixed, err := s.storage.MigrateDomainScreenWithTimeout()
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"fixed":   fixed,
-	})
-}
-
-// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
-// `anilistid` on a site card, and `workcid` / position keys on any card
-// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
-// why chapters and anchor cards' ids are deliberately left alone.
-//
-// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
-// keys, so a bare call only reports what it would delete. An unparseable
-// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
-// sweep that ran.
-//
-// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
-// records, byKey counts fields per key. Safe to call repeatedly: a swept
-// record plans nothing on the next pass.
-func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	apply := false
-	if v := r.URL.Query().Get("apply"); v != "" {
-		parsed, err := strconv.ParseBool(v)
-		if err != nil {
-			writeError(w, http.StatusBadRequest, "apply must be true or false")
-			return
-		}
-		apply = parsed
-	}
-
-	res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"dryRun":  !apply,
-		"planned": res.Planned,
-		"fixed":   res.Fixed,
-		"byKey":   res.ByKey,
-	})
-}

internal/api/admin_sweep_test.go

@@ -1,93 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/alicebob/miniredis/v2"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-// newSweepTestServer is a Server over miniredis with one literature card
-// carrying a copied `workcid`.
-func newSweepTestServer(t *testing.T) *Server {
-	t.Helper()
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
-	for k, v := range map[string]string{
-		"domain":   "literature",
-		"fileType": "card",
-		"workcid":  "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
-	} {
-		if err := c.SetProperty("card-1", k, v); err != nil {
-			t.Fatalf("seed %s: %v", k, err)
-		}
-	}
-	return &Server{storage: c}
-}
-
-type sweepResponse struct {
-	Success bool           `json:"success"`
-	DryRun  bool           `json:"dryRun"`
-	Planned int            `json:"planned"`
-	Fixed   int            `json:"fixed"`
-	ByKey   map[string]int `json:"byKey"`
-}
-
-func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
-	t.Helper()
-	rr := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
-	s.handleSweepLiteratureEchoes(rr, req)
-	var body sweepResponse
-	if rr.Code == http.StatusOK {
-		if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-			t.Fatalf("decode %s: %v", rr.Body.String(), err)
-		}
-	}
-	return rr, body
-}
-
-// A bare call must never delete: the default is the dry run.
-func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
-	s := newSweepTestServer(t)
-
-	rr, body := postSweep(t, s, "")
-	if rr.Code != http.StatusOK {
-		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
-	}
-	if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
-		t.Errorf("dry run = %+v", body)
-	}
-	if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
-		t.Errorf("dry run deleted workcid")
-	}
-
-	rr, body = postSweep(t, s, "?apply=true")
-	if rr.Code != http.StatusOK {
-		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
-	}
-	if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
-		t.Errorf("apply = %+v", body)
-	}
-	if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
-		t.Errorf("apply left workcid = %q", v)
-	}
-}
-
-// A typo must not pass for a sweep that ran.
-func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
-	s := newSweepTestServer(t)
-
-	rr, _ := postSweep(t, s, "?apply=yes-please")
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("status = %d, want 400", rr.Code)
-	}
-}

internal/api/cid_resolve.go

@@ -56,6 +56,12 @@ func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if as := r.URL.Query().Get("as"); as != "" {
+		if flat, err := s.storage.GetMetadataFlatAs(uuid, as); err == nil && flat != nil {
+			doc.Flat = flat
+		}
+	}
+
 	// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
 	// cids / canonical_cid / duplicates), with cid added so the caller can
 	// correlate when they fan out N lookups. canonical_cid is derived by

internal/api/claims.go

@@ -0,0 +1,172 @@
+package api
+
+// Signed claims (meta-core v2.0.0, docs/implementation/metacore-claims-model.md).
+//
+//	POST /api/claims                 the only metadata write path
+//	GET  /api/meta/{cid}/claims      every live claim of the cid's group
+//	GET  /api/metadata/{rid}/claims  the same, addressed by rid
+//	POST /api/merge                  stateless merge of claims the caller holds
+//
+// meta-core never signs a claim here: it verifies each one (MMUR2, low-S) and
+// the signature is the write authentication.
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// GoneResponse is the body of every removed unsigned write route.
+type GoneResponse struct {
+	Error string `json:"error"`
+	Use   string `json:"use"`
+}
+
+// handleGone answers the unsigned write routes removed in v2.0.0.
+func (s *Server) handleGone(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, http.StatusGone, GoneResponse{Error: "gone", Use: "POST /api/claims"})
+}
+
+// decodeClaims accepts a JSON array of claims or {"claims": [...]}.
+func decodeClaims(r *http.Request) ([]claim.Claim, error) {
+	var raw json.RawMessage
+	if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&raw); err != nil {
+		return nil, err
+	}
+	var claims []claim.Claim
+	if len(raw) > 0 && raw[0] == '{' {
+		var env struct {
+			Claims []claim.Claim `json:"claims"`
+		}
+		if err := json.Unmarshal(raw, &env); err != nil {
+			return nil, err
+		}
+		return env.Claims, nil
+	}
+	if err := json.Unmarshal(raw, &claims); err != nil {
+		return nil, err
+	}
+	return claims, nil
+}
+
+// handlePostClaims handles POST /api/claims: one status per claim, in input
+// order. 200 even when some claims are refused — the statuses say which.
+func (s *Server) handlePostClaims(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	claims, err := decodeClaims(r)
+	if err != nil {
+		writeErrorSlug(w, http.StatusBadRequest, "invalid_json", "body must be a JSON array of claims: "+err.Error(), false)
+		return
+	}
+	if len(claims) > storage.MaxClaimsPerBatch {
+		writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per batch", false)
+		return
+	}
+	results, err := s.storage.WriteClaims(r.Context(), claims)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, results)
+}
+
+// ClaimsResponse is the body of the claims read routes: the record's rid, the
+// cids of its group, and every live signed claim on them.
+type ClaimsResponse struct {
+	RID    string        `json:"rid"`
+	CIDs   []string      `json:"cids"`
+	Claims []claim.Claim `json:"claims"`
+}
+
+// handleClaimsByCID handles GET /api/meta/{cid}/claims[?key=…].
+func (s *Server) handleClaimsByCID(w http.ResponseWriter, r *http.Request) {
+	cid := mux.Vars(r)["cid"]
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	rid, err := s.storage.GetByCID(cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if rid == "" {
+		writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this CID", false)
+		return
+	}
+	s.writeClaims(w, r, rid)
+}
+
+// handleClaimsByRID handles GET /api/metadata/{rid}/claims[?key=…]. The rid
+// may also be a cid (resolved like every /api/metadata read).
+func (s *Server) handleClaimsByRID(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	rid := s.storage.ResolveRoot(mux.Vars(r)["hashId"])
+	s.writeClaims(w, r, rid)
+}
+
+// writeClaims answers both claims routes: ?key= keeps one exact key, ?uid=
+// one signer; 404 when the record does not exist.
+func (s *Server) writeClaims(w http.ResponseWriter, r *http.Request, rid string) {
+	cids, claims, err := s.storage.RecordClaims(rid, r.URL.Query().Get("key"))
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if len(cids) == 0 {
+		writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this rid or cid", false)
+		return
+	}
+	out := make([]claim.Claim, 0, len(claims))
+	uid := r.URL.Query().Get("uid")
+	for _, cl := range claims {
+		if uid == "" || cl.UID == uid {
+			out = append(out, cl)
+		}
+	}
+	writeJSON(w, http.StatusOK, ClaimsResponse{RID: rid, CIDs: cids, Claims: out})
+}
+
+// MergeRequest is the body of POST /api/merge.
+type MergeRequest struct {
+	Claims []claim.Claim `json:"claims"`
+	As     string        `json:"as,omitempty"`
+}
+
+// MergeResponse is the merged field map, the number of agreeing signers per
+// key, and how many claims failed verification (dropped, never an error).
+type MergeResponse struct {
+	Fields   map[string]string `json:"fields"`
+	Nsrc     map[string]int    `json:"nsrc"`
+	Rejected int               `json:"rejected"`
+}
+
+// handleMerge handles POST /api/merge: verify, merge per key with the same
+// rule the cache uses, overlay `as`'s own votes held by this core. No write.
+func (s *Server) handleMerge(w http.ResponseWriter, r *http.Request) {
+	var req MergeRequest
+	if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&req); err != nil {
+		writeErrorSlug(w, http.StatusBadRequest, "invalid_json", err.Error(), false)
+		return
+	}
+	if len(req.Claims) > storage.MaxClaimsPerBatch {
+		writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per merge", false)
+		return
+	}
+	fields, nsrc, rejected, err := s.storage.MergeClaims(req.Claims, req.As)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, MergeResponse{Fields: fields, Nsrc: nsrc, Rejected: rejected})
+}

internal/api/claims_test.go

@@ -0,0 +1,182 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+func withVars(r *http.Request, vars map[string]string) *http.Request { return mux.SetURLVars(r, vars) }
+
+// routedServer wires the real route table, so the test sees what clients see.
+func routedServer(t *testing.T) *Server {
+	t.Helper()
+	s := &Server{storage: newTestStorage(t), router: mux.NewRouter()}
+	s.setupRoutes()
+	return s
+}
+
+func call(t *testing.T, s *Server, method, path, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(method, path, strings.NewReader(body))
+	rr := httptest.NewRecorder()
+	s.router.ServeHTTP(rr, req)
+	return rr
+}
+
+// Every unsigned write route of v1.x answers 410 with a pointer to the claims.
+func TestUnsignedWritesAreGone(t *testing.T) {
+	s := routedServer(t)
+	for _, c := range []struct{ method, path string }{
+		{"PUT", "/meta/bafy"}, {"PATCH", "/meta/bafy"}, {"DELETE", "/meta/bafy"},
+		{"POST", "/meta/bafy/_add/genres"}, {"PUT", "/meta/bafy/title"}, {"DELETE", "/meta/bafy/title"},
+		{"PUT", "/api/metadata/bafy"}, {"DELETE", "/api/metadata/bafy"}, {"PUT", "/api/metadata/bafy/property"},
+		{"POST", "/api/metadata/batch"},
+	} {
+		rr := call(t, s, c.method, c.path, `{"title":"x"}`)
+		if rr.Code != http.StatusGone || !strings.Contains(rr.Body.String(), `"use":"POST /api/claims"`) {
+			t.Errorf("%s %s = %d %s", c.method, c.path, rr.Code, rr.Body)
+		}
+	}
+	if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"file:R/title","value":"x"}`); rr.Code != http.StatusConflict {
+		t.Errorf("raw record write via /api/kv/value = %d", rr.Code)
+	}
+	if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"btmagnet:abc","value":"magnet:"}`); rr.Code != http.StatusOK {
+		t.Errorf("service key write = %d %s", rr.Code, rr.Body)
+	}
+}
+
+func TestPostClaimsAndReadBack(t *testing.T) {
+	s := routedServer(t)
+	a, b := testSigner(t, 1), testSigner(t, 2)
+	bad := signed(t, b, "bafyA", "year", "1999", 5)
+	bad.Value = claim.Text("2001")
+	body, _ := json.Marshal([]claim.Claim{
+		signed(t, a, "bafyA", "title", "Matrix", 5),
+		signed(t, b, "bafyA", "title", "Matrix", 6),
+		bad,
+	})
+	rr := call(t, s, "POST", "/api/claims", string(body))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("POST /api/claims = %d %s", rr.Code, rr.Body)
+	}
+	var res []storage.ClaimResult
+	_ = json.Unmarshal(rr.Body.Bytes(), &res)
+	if len(res) != 3 || res[0].Status != "accepted" || res[1].Status != "accepted" || res[2].Status != "bad_signature" {
+		t.Fatalf("statuses %+v", res)
+	}
+
+	rr = call(t, s, "GET", "/api/meta/bafyA", "")
+	if !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
+		t.Fatalf("merged read = %s", rr.Body)
+	}
+	rr = call(t, s, "GET", "/api/meta/bafyA/claims", "")
+	var cr ClaimsResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &cr)
+	if rr.Code != http.StatusOK || len(cr.Claims) != 2 {
+		t.Fatalf("claims by cid = %d %s", rr.Code, rr.Body)
+	}
+	if cr.RID == "" || len(cr.CIDs) != 1 || cr.CIDs[0] != "bafyA" {
+		t.Fatalf("claims envelope rid/cids = %q %v", cr.RID, cr.CIDs)
+	}
+	rr = call(t, s, "GET", "/api/metadata/"+cr.RID+"/claims?key=title&uid="+a.UID, "")
+	var byRid ClaimsResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &byRid)
+	if rr.Code != http.StatusOK || len(byRid.Claims) != 1 || byRid.Claims[0].UID != a.UID || byRid.RID != cr.RID {
+		t.Fatalf("claims by rid with key+uid = %d %s", rr.Code, rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyA/claims?key=nope", ""); !strings.Contains(rr.Body.String(), `"claims":[]`) {
+		t.Fatalf("key filter = %s", rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyUnknown/claims", ""); rr.Code != http.StatusNotFound {
+		t.Fatalf("unknown cid claims = %d", rr.Code)
+	}
+	if rr := call(t, s, "GET", "/api/metadata/01NOSUCHRID/claims", ""); rr.Code != http.StatusNotFound {
+		t.Fatalf("unknown rid claims = %d", rr.Code)
+	}
+
+	// ?as= overlays the viewer's own vote.
+	me := testSigner(t, 9)
+	storeClaims(t, s.storage, signed(t, me, "bafyA", "title", "My title", 7))
+	if rr := call(t, s, "GET", "/api/meta/bafyA?as="+me.UID, ""); !strings.Contains(rr.Body.String(), `"title":"My title"`) {
+		t.Fatalf("?as overlay = %s", rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyA", ""); !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
+		t.Fatalf("shared view changed by a single viewer: %s", rr.Body)
+	}
+}
+
+func TestMergeEndpoint(t *testing.T) {
+	s := routedServer(t)
+	a, b, c := testSigner(t, 1), testSigner(t, 2), testSigner(t, 3)
+	bad := signed(t, c, "bafyX", "title", "Forged", 1)
+	bad.UID = a.UID
+	body, _ := json.Marshal(MergeRequest{Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, b, "bafyX", "title", "Dune", 3),
+		signed(t, c, "bafyX", "title", "Dune Part One", 1),
+		bad,
+	}})
+	rr := call(t, s, "POST", "/api/merge", string(body))
+	var mr MergeResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &mr)
+	if rr.Code != http.StatusOK || mr.Fields["title"] != "Dune" || mr.Nsrc["title"] != 2 || mr.Rejected != 1 {
+		t.Fatalf("merge = %d %s", rr.Code, rr.Body)
+	}
+	if !strings.Contains(rr.Body.String(), `"rejected":1`) || !strings.Contains(rr.Body.String(), `"nsrc":{"title":2}`) {
+		t.Fatalf("merge wire shape = %s", rr.Body)
+	}
+
+	// `as`: the viewer's own vote in the request wins…
+	me := testSigner(t, 9)
+	body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, b, "bafyX", "title", "Dune", 3),
+		signed(t, me, "bafyX", "title", "Dune (my cut)", 4),
+	}})
+	_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
+	if mr.Fields["title"] != "Dune (my cut)" || mr.Nsrc["title"] != 1 {
+		t.Fatalf("request overlay = %+v", mr)
+	}
+	// …and so do its votes held by this core for the same cids.
+	storeClaims(t, s.storage, signed(t, me, "bafyX", "year", "2021", 5))
+	body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, a, "bafyX", "year", "1984", 2),
+	}})
+	_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
+	if mr.Fields["year"] != "2021" || mr.Fields["title"] != "Dune" {
+		t.Fatalf("local overlay = %+v", mr)
+	}
+	// An all-invalid batch is still 200 with everything rejected.
+	bad2 := signed(t, a, "bafyX", "title", "x", 9)
+	bad2.Sig = bad2.Sig[:10]
+	body, _ = json.Marshal(MergeRequest{Claims: []claim.Claim{bad2}})
+	rr = call(t, s, "POST", "/api/merge", string(body))
+	if rr.Code != http.StatusOK || !strings.Contains(rr.Body.String(), `"rejected":1`) {
+		t.Fatalf("all-invalid merge = %d %s", rr.Code, rr.Body)
+	}
+}
+
+func TestTooManyClaimsIs413(t *testing.T) {
+	s := routedServer(t)
+	var sb strings.Builder
+	sb.WriteString("[")
+	for i := 0; i <= storage.MaxClaimsPerBatch; i++ {
+		if i > 0 {
+			sb.WriteString(",")
+		}
+		sb.WriteString(`{"cid":"c","uid":"u","key":"k","value":null,"ts":1,"sig":""}`)
+	}
+	sb.WriteString("]")
+	if rr := call(t, s, "POST", "/api/claims", sb.String()); rr.Code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized batch = %d", rr.Code)
+	}
+}

internal/api/deprecated_field_test.go

@@ -1,86 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the
-// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They
-// must be rejected at the write boundary rather than stored.
-//
-// Why a hard 400 and not a silent rewrite: the reverse-index hook
-// (maybeAddAliasFromFieldLocked) only registers an alias for a field named
-// `cids/<cid>`. A `cid_*` field was therefore written verbatim and never
-// indexed — the record silently became unresolvable by that CID. A loud
-// failure is strictly better than a record that looks correct and cannot be
-// found.
-func TestRejectDeprecatedField(t *testing.T) {
-	rejected := []string{
-		"cid_midhash256",
-		"cid_sha2-256",
-		"cid_btih_v2",
-		"cid_crc32",
-		"midhash256",
-		"canonical_cid",
-	}
-	for _, f := range rejected {
-		if !rejectDeprecatedField(f) {
-			t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f)
-		}
-	}
-
-	accepted := []string{
-		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
-		"filePath",
-		"title",
-		"fileinfo/duration",
-		"tmdb/poster",
-		// Near-misses that must NOT be caught: the prefix is `cid_`, with an
-		// underscore. A field merely starting with "cid" is fine.
-		"cids",
-		"cidrank",
-	}
-	for _, f := range accepted {
-		if rejectDeprecatedField(f) {
-			t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f)
-		}
-	}
-}
-
-func TestValidateFields_Rejects400WithSlug(t *testing.T) {
-	rr := httptest.NewRecorder()
-	blocked := validateFields(rr, map[string]string{
-		"title":          "Inception",
-		"cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
-	})
-
-	if !blocked {
-		t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed")
-	}
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest)
-	}
-
-	var body map[string]interface{}
-	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-		t.Fatalf("response is not JSON: %v", err)
-	}
-	if got := body["error"]; got != ErrDeprecatedField {
-		t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField)
-	}
-}
-
-func TestValidateFields_AllowsCidsKeyset(t *testing.T) {
-	rr := httptest.NewRecorder()
-	blocked := validateFields(rr, map[string]string{
-		"filePath": "/watch/Inception.mkv",
-		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true",
-		"cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua":     "true",
-	})
-	if blocked {
-		t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String())
-	}
-}

internal/api/files_register_test.go

@@ -9,24 +9,16 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
-// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
+// newRegisterTestServer wires handleRegisterFile to a temp store and a temp
-// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
+// FILES_PATH. Built directly, not through NewServer, so no watcher runs —
 // which is exactly the configuration this route exists for.
-func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
+func newRegisterTestServer(t *testing.T) (*Server, string) {
 	t.Helper()
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
 	files := t.TempDir()
-	return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
+	return &Server{storage: newTestStorage(t), config: &config.Config{FilesPath: files}}, files
 }
 
 func writeFixture(t *testing.T, files, rel, body string) string {
@@ -52,20 +44,20 @@ func postRegister(t *testing.T, s *Server, path string) (int, registerFileRespon
 	return rr.Code, resp
 }
 
-func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
+func streamLen(t *testing.T, s *Server) int {
 	t.Helper()
-	entries, err := mr.Stream("file:events")
+	n, err := s.storage.GetStreamLength("file:events")
 	if err != nil {
-		return 0
+		t.Fatalf("stream length: %v", err)
 	}
-	return len(entries)
+	return int(n)
 }
 
 // The whole point of the route: with the watcher off, a file written under
 // plugin/ becomes a record resolvable by its midhash, and meta-share hears
 // about it on file:events.
 func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
-	s, files, mr := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	rel := "plugin/subtitles/ab12cd34.eng.srt"
 	full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
 
@@ -85,20 +77,19 @@ func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
 		t.Fatalf("filePath = %q, want %q", fp, full)
 	}
 
-	entries, err := mr.Stream("file:events")
+	entries, err := s.storage.XRevRange("file:events", "+", "-", 0)
 	if err != nil || len(entries) != 1 {
 		t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
 	}
-	vals := strings.Join(entries[0].Values, "|")
+	if vals := entries[0].Values; vals["midhash256"] != resp.CID || vals["path"] != rel {
-	if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
+		t.Fatalf("event must carry the midhash and the relative path, got %v", vals)
-		t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
 	}
 }
 
 // Re-uploading the same subtitle is a no-op: same root, no second event (which
 // would make meta-share re-chunk and re-provide the file).
 func TestRegisterFile_IsIdempotent(t *testing.T) {
-	s, files, mr := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	rel := "plugin/subtitles/same.eng.srt"
 	writeFixture(t, files, rel, "same bytes")
 
@@ -110,7 +101,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
 	if second.Root != first.Root || second.CID != first.CID {
 		t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
 	}
-	if n := streamLen(t, mr); n != 1 {
+	if n := streamLen(t, s); n != 1 {
 		t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
 	}
 }
@@ -118,7 +109,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
 // Only plugin output may be registered here. /files/watch is the user's
 // library and belongs to the watcher; traversal must not escape either.
 func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
-	s, files, _ := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	writeFixture(t, files, "watch/movie.mkv", "x")
 	for _, p := range []string{
 		"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
@@ -131,7 +122,7 @@ func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
 }
 
 func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
-	s, files, _ := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
 		t.Errorf("missing file: want 404, got %d", code)
 	}

internal/api/files_tuples_test.go

@@ -7,18 +7,12 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
 func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
 	t.Helper()
-	mr := miniredis.RunT(t)
+	c := newTestStorage(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
 	return &Server{storage: c}, c
 }
 
@@ -34,25 +28,25 @@ func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesRespon
 
 func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
 	t.Helper()
-	id, err := c.Mint(path, size, mtime)
+	id, _, err := c.RegisterFileTuple(path, size, mtime, "bafkmid"+path)
 	if err != nil {
-		t.Fatalf("mint %s: %v", path, err)
+		t.Fatalf("register %s: %v", path, err)
 	}
 	return id
 }
 
-// Only records that are actually files count: a gateway record and a record
+// Only local files count: a gateway record — even one whose claims carry a
-// that merely carries a path + size (no mtimeNano) are both left out.
+// filePath and size — is left out.
 func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
 	s, c := newFileTuplesTestServer(t)
 	a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
 	b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
-	if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil {
+	gw := testSigner(t, 1)
-		t.Fatal(err)
+	storeClaims(t, c,
-	}
+		signed(t, gw, "bagcsgatewayrecord", "title", "Some Release", 1),
-	if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil {
+		signed(t, gw, "bafkpartial", "filePath", "/files/cache/x.mkv", 1),
-		t.Fatal(err)
+		signed(t, gw, "bafkpartial", "sizeByte", "999", 1),
-	}
+	)
 
 	code, resp := getFileTuples(t, s, "")
 	if code != http.StatusOK {
@@ -105,7 +99,7 @@ func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
 }
 
 func TestFileTuples_StorageNotConnected(t *testing.T) {
-	s := &Server{storage: storage.NewClient("")}
+	s := &Server{storage: storage.NewClient()}
 
 	code, _ := getFileTuples(t, s, "")
 	if code != http.StatusServiceUnavailable {

internal/api/handlers.go

@@ -4,7 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/base32"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -14,6 +13,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 )
 
 // contentTypeByExt maps file extensions to MIME types
@@ -79,51 +79,15 @@ type ErrorResponse struct {
 // Stable error-slug strings. Pinned in the doc; consumers may match on
 // these. Do not rename without a deprecation cycle.
 const (
-	ErrAliasCollision    = "alias_collision"
+	ErrAliasCollision     = "alias_collision"
-	ErrUnknownRoot       = "unknown_root"
+	ErrUnknownRoot        = "unknown_root"
-	ErrUnknownCID        = "unknown_cid"
+	ErrUnknownCID         = "unknown_cid"
-	ErrSchemaViolation   = "schema_violation"
+	ErrSchemaViolation    = "schema_violation"
 	ErrStorageUnavailable = "storage_unavailable"
-	ErrInternal          = "internal"
+	ErrInternal           = "internal"
-	ErrDeprecatedField   = "deprecated_field"
+	ErrDeprecatedField    = "deprecated_field"
 )
 
-// rejectDeprecatedField reports whether a field name is one of the removed
-// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …).
-//
-// This is a tripwire, not a compatibility shim. Such a field used to be
-// accepted and written verbatim — but the reverse-index hook
-// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named
-// `cids/<cid>`, so a `cid_*` field was stored and then NEVER INDEXED: the
-// record silently became unresolvable by that CID. Failing the write loudly is
-// strictly better than a record that looks fine and cannot be found.
-//
-// Writers must emit the bare-CID key-set instead: `cids/<bareCid>` = "true".
-// See METADATA_KEYS.md §2/§14.13.
-func rejectDeprecatedField(field string) bool {
-	return strings.HasPrefix(field, "cid_") ||
-		field == "midhash256" ||
-		field == "canonical_cid"
-}
-
-// validateFields rejects a write carrying any deprecated CID field. Returns
-// true if the request was rejected (the handler must return immediately).
-func validateFields(w http.ResponseWriter, metadata map[string]string) bool {
-	for field := range metadata {
-		if rejectDeprecatedField(field) {
-			writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField,
-				fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+
-					"write the bare-CID key-set instead: \"cids/<bareCid>\": \"true\" "+
-					"(a cid_* field would be stored but never indexed, leaving the "+
-					"record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.",
-					field),
-				false)
-			return true
-		}
-	}
-	return false
-}
-
 // defaultSlugFor maps an HTTP status code to the most likely error slug.
 // Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
 // instead of "unknown_root" on 404) call writeErrorSlug directly.
@@ -249,7 +213,7 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
 	// the watcher minted the root, regardless of which CID the caller knows.
 	hashID = s.storage.ResolveRoot(hashID)
 
-	metadata, err := s.storage.GetMetadataFlat(hashID)
+	metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -268,76 +232,6 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handlePutMeta handles PUT /meta/{hash}
-func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// CID → UUID resolution so writes land on the canonical root, not on
-	// a parallel midhash-rooted entry the watcher's UUID never sees.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if validateFields(w, metadata) {
-		return
-	}
-
-	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-	})
-}
-
-// handleDeleteMeta handles DELETE /meta/{hash}
-func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	deleted, err := s.storage.DeleteMetadata(hashID)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-		"deleted": deleted,
-	})
-}
-
 // handleListMeta handles GET /meta
 func (s *Server) handleListMeta(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
@@ -547,64 +441,51 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
 	http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
 }
 
-// handleListServices handles GET /services and GET /api/services
+// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
-// Supports optional ?current=<service-name> query parameter
+//
-func (s *Server) handleListServices(w http.ResponseWriter, r *http.Request) {
+// Beacon v2 (docs/project-architecture/beacon-v2.md). The list is this node's
-	services, err := s.discovery.DiscoverAll()
+// in-memory view of the UDP group, so it works across stacks that share a
-	if err != nil {
+// network but no volume. Instances of one name collapse to the most recent
-		writeError(w, http.StatusInternalServerError, err.Error())
+// (one row per service, the nav menu's shape); `?all=1` returns every
-		return
+// instance, `?cap=<pattern>` keeps only nodes with a matching capability.
-	}
+func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
-
 	response := map[string]interface{}{
-		"services": services,
+		"current":   s.config.ServiceName,
-		"count":    len(services),
+		"neighbors": []interface{}{},
+		"count":     0,
 	}
 
-	// Add 'current' field if query parameter is provided
+	if s.mesh == nil {
-	current := r.URL.Query().Get("current")
+		// Discovery off or failed to bind. Report empty rather than 503 —
-	if current != "" {
+		// the nav is decoration and callers already treat [] as "hide".
-		response["current"] = current
+		response["enabled"] = false
+		writeJSON(w, http.StatusOK, response)
+		return
 	}
 
+	var list []meshdisco.Neighbor
+	if r.URL.Query().Get("all") != "" {
+		list = s.mesh.Neighbors()
+	} else {
+		list = s.mesh.NeighborsByName()
+	}
+	if pattern := r.URL.Query().Get("cap"); pattern != "" {
+		kept := list[:0]
+		for _, nb := range list {
+			if nb.Matches(pattern) {
+				kept = append(kept, nb)
+			}
+		}
+		list = kept
+	}
+
+	response["enabled"] = true
+	response["neighbors"] = list
+	response["count"] = len(list)
+	response["self"] = s.mesh.Self()
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handleGetService handles GET /services/{name}
-func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	name := vars["name"]
-
-	if name == "" {
-		writeError(w, http.StatusBadRequest, "service name is required")
-		return
-	}
-
-	service, err := s.discovery.Discover(name)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	if service == nil {
-		writeError(w, http.StatusNotFound, "service not found")
-		return
-	}
-
-	writeJSON(w, http.StatusOK, service)
-}
-
-// handleCleanupStats handles GET /services/cleanup/stats
-func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
-	if s.cleaner == nil {
-		writeError(w, http.StatusServiceUnavailable, "cleaner not initialized")
-		return
-	}
-
-	stats := s.cleaner.Stats()
-	writeJSON(w, http.StatusOK, stats)
-}
-
 // writeJSON writes a JSON response
 func writeJSON(w http.ResponseWriter, status int, data interface{}) {
 	w.Header().Set("Content-Type", "application/json")
@@ -635,50 +516,6 @@ func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, ret
 	})
 }
 
-// handlePatchMeta handles PATCH /meta/{hash}
-// Merges the provided metadata into existing (does not delete missing keys)
-func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// PATCH is the hot write path for meta-sort. Resolve so plugin output
-	// merges into the watcher's UUID root rather than minting a parallel
-	// midhash-rooted entry.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if validateFields(w, metadata) {
-		return
-	}
-
-	updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-		"updated": updated,
-	})
-}
-
 // handleGetProperty handles GET /meta/{hash}/{key}
 // Gets a single property value
 func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
@@ -725,171 +562,6 @@ type PropertyBody struct {
 	Value string `json:"value"`
 }
 
-// handlePutProperty handles PUT /meta/{hash}/{key}
-// Sets a single property value
-// Accepts JSON body with {"value": "..."} or plain text
-func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if validateFields(w, map[string]string{key: ""}) {
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	// Read body
-	buf := make([]byte, 1024*1024) // 1MB max
-	n, err := r.Body.Read(buf)
-	if err != nil && err.Error() != "EOF" {
-		writeError(w, http.StatusBadRequest, "failed to read body")
-		return
-	}
-	body := buf[:n]
-
-	// Try to parse as JSON {"value": "..."} first
-	var propBody PropertyBody
-	var value string
-	if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
-		// JSON body with value field
-		value = propBody.Value
-	} else {
-		// Plain text body
-		value = string(body)
-	}
-
-	if err := s.storage.SetProperty(hashID, key, value); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-	})
-}
-
-// handleDeleteProperty handles DELETE /meta/{hash}/{key}
-// Deletes a single property
-func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	if err := s.storage.DeleteProperty(hashID, key); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-	})
-}
-
-// handleAddToSet handles POST /meta/{hash}/_add/{key}
-// Adds a value to a set-type field
-// Accepts JSON body with {"value": "..."} or plain text
-func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if validateFields(w, map[string]string{key: ""}) {
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	// Read body
-	buf := make([]byte, 1024*1024) // 1MB max
-	n, err := r.Body.Read(buf)
-	if err != nil && err.Error() != "EOF" {
-		writeError(w, http.StatusBadRequest, "failed to read body")
-		return
-	}
-	body := buf[:n]
-
-	// Try to parse as JSON {"value": "..."} first
-	var propBody PropertyBody
-	var value string
-	if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
-		// JSON body with value field
-		value = propBody.Value
-	} else {
-		// Plain text body
-		value = string(body)
-	}
-
-	if value == "" {
-		writeError(w, http.StatusBadRequest, "value is required")
-		return
-	}
-
-	added, err := s.storage.AddToSet(hashID, key, value)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-		"added":    added,
-	})
-}
-
 // CIDRequest is the request body for POST /file/cid
 type CIDRequest struct {
 	Path string `json:"path"`
@@ -959,10 +631,10 @@ func (s *Server) handleComputeFileCID(w http.ResponseWriter, r *http.Request) {
 	// CIDv1 format: version (0x01) + codec (0x55) + multihash
 	// Multihash format: hash-code (0x12) + length (0x20) + hash
 	cidBytes := make([]byte, 0, 2+2+32)
-	cidBytes = append(cidBytes, 0x01)       // CIDv1
+	cidBytes = append(cidBytes, 0x01) // CIDv1
-	cidBytes = append(cidBytes, 0x55)       // raw codec
+	cidBytes = append(cidBytes, 0x55) // raw codec
-	cidBytes = append(cidBytes, 0x12)       // sha256 code
+	cidBytes = append(cidBytes, 0x12) // sha256 code
-	cidBytes = append(cidBytes, 0x20)       // 32 bytes
+	cidBytes = append(cidBytes, 0x20) // 32 bytes
 	cidBytes = append(cidBytes, hashBytes...)
 
 	// Encode as base32lower with 'b' prefix (multibase)

internal/api/handlers_identity.go

@@ -4,11 +4,14 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/metazla/meta-core/internal/claim"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/storage"
 )
@@ -120,6 +123,9 @@ type identityDeleteResponse struct {
 
 type identitySignRequest struct {
 	BytesB64 string `json:"bytesB64"`
+	// Format is "der" or "compact" (64-byte r||s, low-S). Default: compact for
+	// MMUR2 claim payloads ("MMUR2\n" prefix), DER otherwise (MMUR1 back-compat).
+	Format string `json:"format,omitempty"`
 	// UID selects which account signs. Optional — when omitted and exactly one
 	// account exists, that account is used (single-identity back-compat).
 	UID string `json:"uid,omitempty"`
@@ -143,6 +149,8 @@ type identitySignBatchRequest struct {
 	// UID selects which account signs — same optional single-identity
 	// back-compat as identitySignRequest.
 	UID string `json:"uid,omitempty"`
+	// Format: see identitySignRequest.Format; applies to every payload.
+	Format string `json:"format,omitempty"`
 }
 
 type identitySignBatchResponse struct {
@@ -445,10 +453,10 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
 			"storage is not connected; refusing to delete the key while its data cannot be purged", true)
 		return
 	}
-	purged, err := s.storage.UDLPurgeUser(id.UID)
+	purged, err := s.retractAllClaims(r, id)
 	if err != nil {
 		writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
-			"could not purge this account's data; the account was NOT deleted: "+err.Error(), true)
+			"could not retract this account's claims; the account was NOT deleted: "+err.Error(), true)
 		return
 	}
 
@@ -456,7 +464,7 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
 		writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
 		return
 	}
-	log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids",
+	log.Printf("[API] Deleted account %s after retracting %d claims across %d cids",
 		id.UID, purged.Records, purged.Cids)
 	writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
 }
@@ -485,9 +493,9 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
 		writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
 		return
 	}
-	sig, err := identity.Sign(id, raw)
+	sig, err := signPayload(id, raw, body.Format)
 	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
+		writeError(w, http.StatusBadRequest, err.Error())
 		return
 	}
 	writeJSON(w, http.StatusOK, identitySignResponse{
@@ -537,9 +545,9 @@ func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request)
 				"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
 			return
 		}
-		sig, err := identity.Sign(id, raw)
+		sig, err := signPayload(id, raw, body.Format)
 		if err != nil {
-			writeError(w, http.StatusInternalServerError, err.Error())
+			writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"]: "+err.Error())
 			return
 		}
 		sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
@@ -562,3 +570,61 @@ func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
 		AEADKeyB64: base64.StdEncoding.EncodeToString(key),
 	})
 }
+
+// signPayload signs raw with the account key in the requested encoding.
+func signPayload(id *identity.Identity, raw []byte, format string) ([]byte, error) {
+	switch format {
+	case "compact":
+		return identity.SignCompact(id, raw)
+	case "der":
+		return identity.Sign(id, raw)
+	case "":
+		if strings.HasPrefix(string(raw), claim.Domain) {
+			return identity.SignCompact(id, raw)
+		}
+		return identity.Sign(id, raw)
+	}
+	return nil, fmt.Errorf("format must be \"compact\" or \"der\"")
+}
+
+// retractAllClaims signs a retraction for every live claim the account holds
+// and stores them (G11): the tombstones stay, so the votes cannot be replayed.
+func (s *Server) retractAllClaims(r *http.Request, id *identity.Identity) (storage.UDLUserStats, error) {
+	refs, err := s.storage.LiveClaimsBySigner(id.UID)
+	if err != nil {
+		return storage.UDLUserStats{}, err
+	}
+	cids, keys := map[string]bool{}, map[string]bool{}
+	now := uint64(time.Now().UnixMilli())
+	batch := make([]claim.Claim, 0, len(refs))
+	for _, ref := range refs {
+		cur, ok, err := s.storage.UDLGetClaim(id.UID, ref[0], ref[1])
+		if err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		ts := now
+		if ok && cur.TS >= ts {
+			ts = cur.TS + 1
+		}
+		tomb := claim.Claim{CID: ref[0], UID: id.UID, Key: ref[1], Value: claim.Null(), TS: ts}
+		if err := tomb.Sign(id); err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		batch = append(batch, tomb)
+		cids[ref[0]], keys[ref[1]] = true, true
+	}
+	for start := 0; start < len(batch); start += storage.MaxClaimsPerBatch {
+		end := min(start+storage.MaxClaimsPerBatch, len(batch))
+		results, err := s.storage.WriteClaims(r.Context(), batch[start:end])
+		if err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		for i, res := range results {
+			if res.Status != storage.StatusAccepted && res.Status != storage.StatusUnchanged {
+				return storage.UDLUserStats{}, fmt.Errorf("retraction of %s/%s refused: %s %s",
+					batch[start+i].CID, batch[start+i].Key, res.Status, res.Reason)
+			}
+		}
+	}
+	return storage.UDLUserStats{Records: len(batch), Cids: len(cids), Keys: len(keys)}, nil
+}

internal/api/identity_gate_test.go

@@ -1,6 +1,6 @@
 package api
 
-// End-to-end exercise of the account gate against a live router + miniredis:
+// End-to-end exercise of the account gate against a live router + a temp store:
 // generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
 // both the key and every row are gone. Deliberately drives the real HTTP
 // handlers rather than the packages beneath them, because the bugs that matter
@@ -16,21 +16,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
 	"github.com/metazla/meta-core/internal/identity"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
 func newTestServer(t *testing.T) *Server {
 	t.Helper()
-	mr := miniredis.RunT(t)
+	stor := newTestStorage(t)
-	stor := storage.NewClient("")
-	if err := stor.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect: %v", err)
-	}
-	t.Cleanup(func() { _ = stor.Close() })
 
 	s := &Server{
 		config:     &config.Config{MetaCorePath: t.TempDir()},
@@ -103,12 +96,16 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
 	uid := gen["uid"].(string)
 	priv := gen["privateKeyHex"].(string)
 
-	// Some data to lose.
+	// Some data to lose: claims signed by the account.
-	for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} {
+	acct, err := identity.Import(priv)
-		if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok {
+	if err != nil {
-			t.Fatalf("seed: %v", err)
+		t.Fatal(err)
-		}
 	}
+	storeClaims(t, s.storage,
+		signed(t, acct, "cidA", "like", "true", 100),
+		signed(t, acct, "cidA", "userRating", "5", 100),
+		signed(t, acct, "cidB", "seek", []byte("enc"), 100),
+	)
 
 	// The count the UI shows in the confirmation.
 	rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
@@ -182,7 +179,7 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
 	if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
 		t.Fatal("the account file survived the delete")
 	}
-	// Data gone — the orphaning this whole change exists to prevent.
+	// Every claim retracted (tombstones kept) — nothing left to orphan.
 	after, err := s.storage.UDLAllUserStats()
 	if err != nil {
 		t.Fatalf("stats after: %v", err)

internal/api/kv.go

@@ -206,10 +206,10 @@ type KVSearchResponse struct {
 
 // KVFindResponse is the response for GET /api/kv/find
 type KVFindResponse struct {
-	Contains  string             `json:"contains"`
+	Contains  string              `json:"contains"`
-	Fields    []string           `json:"fields"`
+	Fields    []string            `json:"fields"`
 	Matches   []storage.FindMatch `json:"matches"`
-	Truncated bool               `json:"truncated"`
+	Truncated bool                `json:"truncated"`
 }
 
 // KVValueResponse is the response for GET /api/kv/value
@@ -383,8 +383,9 @@ func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// handleKVValuePut handles PUT /api/kv/value with body {key, value}.
+// handleKVValuePut handles PUT /api/kv/value with body {key, value}: service
-// Maintains the file index when the key matches file:<hashId>/...
+// scratch KV only (btmagnet:*, searchcov:*), kept in the cache. Merged records
+// and bookkeeping are refused — metadata is written as signed claims.
 func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
@@ -401,19 +402,11 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := s.storage.SetRawValue(req.Key, req.Value); err != nil {
+	if err := s.storage.SetServiceValue(req.Key, req.Value); err != nil {
 		writeError(w, http.StatusConflict, err.Error())
 		return
 	}
 
-	if hashID := extractFileHashID(req.Key); hashID != "" {
-		if err := s.storage.IndexAdd(hashID); err != nil {
-			// Non-fatal: the value was written; index entry is best-effort.
-			// Surface in the log path of writeJSON via response header.
-			w.Header().Set("X-KV-Index-Warning", err.Error())
-		}
-	}
-
 	writeJSON(w, http.StatusOK, KVValueResponse{
 		Key:    req.Key,
 		Type:   "string",
@@ -422,9 +415,7 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// handleKVValueDelete handles DELETE /api/kv/value?key=...
+// handleKVValueDelete handles DELETE /api/kv/value?key=... (service keys only).
-// When the key matches file:<hashId>/..., removes the file index entry if no
-// sibling keys remain after the delete.
 func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
@@ -437,36 +428,14 @@ func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	deleted, err := s.storage.DeleteRawKey(key)
+	deleted, err := s.storage.DeleteServiceValue(key)
 	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
+		writeError(w, http.StatusConflict, err.Error())
 		return
 	}
 
-	if deleted {
-		if hashID := extractFileHashID(key); hashID != "" {
-			if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil {
-				w.Header().Set("X-KV-Index-Warning", err.Error())
-			}
-		}
-	}
-
 	writeJSON(w, http.StatusOK, KVValueDeleteResponse{
 		Key:     key,
 		Deleted: deleted,
 	})
 }
-
-// extractFileHashID returns the hashID for a key shaped like file:<hashId>/...
-// Returns "" when the key does not follow that convention.
-func extractFileHashID(key string) string {
-	if !strings.HasPrefix(key, "file:") {
-		return ""
-	}
-	rest := strings.TrimPrefix(key, "file:")
-	slash := strings.Index(rest, "/")
-	if slash <= 0 {
-		return ""
-	}
-	return rest[:slash]
-}

internal/api/metadata.go

@@ -38,11 +38,14 @@ type MetadataSearchRequest struct {
 	Property      string `json:"property,omitempty"`
 	PropertyValue string `json:"propertyValue,omitempty"`
 	Limit         int    `json:"limit,omitempty"`
+	// As is a viewer uid: the search sees that signer's own claims overlaid
+	// (and their records as candidates). Also accepted as ?as=.
+	As string `json:"as,omitempty"`
 
 	// Filters is the structured form: AND across entries, OR within an
-	// entry's Values. Evaluated in memory against the search index alongside
+	// entry's Values. Narrowed in SQL through the field index alongside the
-	// the free-text Query, so an id-anchored lookup costs a map probe per
+	// free-text Query, then verified per record (see serveIndexedSearch), so an
-	// record rather than a Redis round-trip per record.
+	// id-anchored lookup costs an index probe rather than a read per record.
 	//
 	// Property/PropertyValue is the single-filter legacy shape and folds into
 	// this list (see normalizedFilters) with Exact=false, preserving its
@@ -222,10 +225,10 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	// Same resolution as the PUT below: callers hold a CID, not the root.
+	// Callers may hold a CID, not the root.
 	hashID = s.storage.ResolveRoot(hashID)
 
-	metadata, err := s.storage.GetMetadataFlat(hashID)
+	metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -244,87 +247,6 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId}
-func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// The {hashId} in the URL is whatever CID the caller knows the file by —
-	// the gateway's auto-store keys this PUT by the hash it just computed
-	// (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is
-	// the dag-pb root, not the nzb-release cid the record was minted under.
-	// Without resolving, the write lands on file:<thatCid>/* and forks a
-	// second root for a file that already has one: a full duplicate record,
-	// cross-linked and separately indexed, which every downstream ingest then
-	// sees as two files. Unknown CIDs pass through unchanged, so a genuinely
-	// new gateway record still mints its root at the hash.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	// Exclude internal processing status field
-	delete(metadata, "processingStatus")
-
-	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":  "ok",
-		"hashId":  hashID,
-		"message": "Metadata updated successfully",
-	})
-}
-
-// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId}
-func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// meta-gateway retracts a dropped hit by whichever CID it holds
-	// (meta_core.rs `delete`, which documents this resolution as a
-	// precondition). Without it, deleting by a non-root CID is a silent
-	// no-op and the retracted record lingers.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	deletedCount, err := s.storage.DeleteMetadata(hashID)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":      "ok",
-		"hashId":      hashID,
-		"deletedKeys": deletedCount,
-	})
-}
-
 // handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X
 func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
@@ -364,46 +286,6 @@ func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Reques
 	})
 }
 
-// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property
-func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	var req PropertyUpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if req.Property == "" {
-		writeError(w, http.StatusBadRequest, "property is required")
-		return
-	}
-
-	if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":   "ok",
-		"hashId":   hashID,
-		"property": req.Property,
-		"value":    req.Value,
-		"message":  "Property updated successfully",
-	})
-}
-
 // handleSearchMetadata handles POST /api/metadata/search
 func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
@@ -420,6 +302,9 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	if req.Limit == 0 {
 		req.Limit = 100
 	}
+	if req.As == "" {
+		req.As = r.URL.Query().Get("as")
+	}
 
 	// If hashId is provided, do exact match
 	if req.HashID != "" {
@@ -445,9 +330,8 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Everything else — free text, structured filters, both, or neither
-	// (which returns the first `limit`) — is served from the in-memory search
+	// (which returns the first `limit`) — is served from the store's derived
-	// index (search_index.go). Matching and the returned metadata both come
+	// search indexes (search_index.go → storage.Search).
-	// from memory, so a query costs zero Redis round-trips.
 	//
 	// There is deliberately no second path. The property branch used to fall
 	// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which
@@ -458,83 +342,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	s.serveIndexedSearch(w, req)
 }
 
-// handleBatchUpdate handles POST /api/metadata/batch
-func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	var req MetadataBatchUpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if len(req.Updates) == 0 {
-		writeError(w, http.StatusBadRequest, "updates array is required")
-		return
-	}
-
-	var results []MetadataBatchResult
-	successCount := 0
-	errorCount := 0
-
-	for _, update := range req.Updates {
-		if update.HashID == "" {
-			results = append(results, MetadataBatchResult{
-				HashID: "unknown",
-				Status: "error",
-				Error:  "Missing hash ID",
-			})
-			errorCount++
-			continue
-		}
-
-		var updateErr error
-
-		// Update complete metadata
-		if update.Metadata != nil {
-			// Exclude internal processing status
-			delete(update.Metadata, "processingStatus")
-			updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata)
-		}
-
-		// Update individual properties
-		if updateErr == nil && update.Properties != nil {
-			for property, value := range update.Properties {
-				if err := s.storage.SetProperty(update.HashID, property, value); err != nil {
-					updateErr = err
-					break
-				}
-			}
-		}
-
-		if updateErr != nil {
-			results = append(results, MetadataBatchResult{
-				HashID: update.HashID,
-				Status: "error",
-				Error:  updateErr.Error(),
-			})
-			errorCount++
-		} else {
-			results = append(results, MetadataBatchResult{
-				HashID: update.HashID,
-				Status: "ok",
-			})
-			successCount++
-		}
-	}
-
-	writeJSON(w, http.StatusOK, MetadataBatchResponse{
-		Status:  "completed",
-		Total:   len(results),
-		Success: successCount,
-		Errors:  errorCount,
-		Results: results,
-	})
-}
-
 // handleClearMetadata handles POST /api/metadata/clear
 func (s *Server) handleClearMetadata(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {

internal/api/reports.go

@@ -0,0 +1,204 @@
+package api
+
+// Report inbox (/api/reports): bug reports and feedback filed from the client
+// apps. An app's browser builds the report, the app's server forwards it here
+// server-to-server, and the dashboard's Reports page reads it back. Storage
+// and bounds live in internal/reports.
+//
+// Auth: like the rest of /api, gated at the perimeter — do NOT add to the
+// nginx-hash-lock unauth bypass. Apps reach meta-core on the internal network.
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/reports"
+)
+
+// maxReportBody bounds one POST. A report is at most a few blocks of 32 KiB
+// plus a 200-line log tail; anything larger is not a report.
+const maxReportBody = 512 << 10
+
+// SetReports attaches the report inbox. Separate from NewServer so its
+// signature (and its tests) stay untouched; nil leaves the routes answering
+// 503.
+func (s *Server) SetReports(r *reports.Store) {
+	s.reports = r
+}
+
+func (s *Server) setupReportRoutes() {
+	s.router.HandleFunc("/api/reports", s.handlePostReport).Methods("POST")
+	s.router.HandleFunc("/api/reports", s.handleListReports).Methods("GET")
+	s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handleGetReport).Methods("GET")
+	s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handlePatchReport).Methods("PATCH")
+	s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handleDeleteReport).Methods("DELETE")
+}
+
+func (s *Server) reportsReady(w http.ResponseWriter) bool {
+	if s.reports == nil {
+		writeError(w, http.StatusServiceUnavailable, "report inbox not available")
+		return false
+	}
+	return true
+}
+
+func reportID(r *http.Request) int64 {
+	id, _ := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
+	return id
+}
+
+// handlePostReport handles POST /api/reports → 201 {id}.
+func (s *Server) handlePostReport(w http.ResponseWriter, r *http.Request) {
+	if !s.reportsReady(w) {
+		return
+	}
+	var in map[string]json.RawMessage
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReportBody)).Decode(&in); err != nil {
+		var tooBig *http.MaxBytesError
+		if errors.As(err, &tooBig) {
+			writeError(w, http.StatusRequestEntityTooLarge, "report larger than 512 KiB")
+			return
+		}
+		writeError(w, http.StatusBadRequest, "body must be a JSON object")
+		return
+	}
+	rep, err := reports.Normalize(in, time.Now())
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	id, err := s.reports.Insert(rep)
+	if err != nil {
+		log.Printf("[reports] insert failed: %v", err)
+		writeError(w, http.StatusInternalServerError, "could not store the report")
+		return
+	}
+	log.Printf("[reports] #%d filed by %s (%s)", id, rep.App, firstLine(rep.Note))
+	writeJSON(w, http.StatusCreated, map[string]int64{"id": id})
+}
+
+// handleListReports handles GET /api/reports?app=&status=&q=&before=&limit=.
+func (s *Server) handleListReports(w http.ResponseWriter, r *http.Request) {
+	if !s.reportsReady(w) {
+		return
+	}
+	q := r.URL.Query()
+	f := reports.Filter{App: q.Get("app"), Status: q.Get("status"), Q: q.Get("q")}
+	f.Before, _ = strconv.ParseInt(q.Get("before"), 10, 64)
+	f.Limit, _ = strconv.Atoi(q.Get("limit"))
+	if f.Status != "" && !reports.ValidStatus(f.Status) {
+		writeError(w, http.StatusBadRequest, "status must be open, triaged or closed")
+		return
+	}
+	page, err := s.reports.List(f)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, page)
+}
+
+// handleGetReport handles GET /api/reports/{id}.
+func (s *Server) handleGetReport(w http.ResponseWriter, r *http.Request) {
+	if !s.reportsReady(w) {
+		return
+	}
+	rep, err := s.reports.Get(reportID(r))
+	if errors.Is(err, reports.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "no such report")
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, rep)
+}
+
+// handlePatchReport handles PATCH /api/reports/{id} {status?, triage_note?}.
+func (s *Server) handlePatchReport(w http.ResponseWriter, r *http.Request) {
+	if !s.reportsReady(w) {
+		return
+	}
+	var in struct {
+		Status     *string `json:"status"`
+		TriageNote *string `json:"triage_note"`
+	}
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil {
+		writeError(w, http.StatusBadRequest, "body must be a JSON object")
+		return
+	}
+	if in.Status == nil && in.TriageNote == nil {
+		writeError(w, http.StatusBadRequest, "nothing to change: send status and/or triage_note")
+		return
+	}
+	if in.Status != nil && !reports.ValidStatus(*in.Status) {
+		writeError(w, http.StatusBadRequest, "status must be open, triaged or closed")
+		return
+	}
+	rep, err := s.reports.Update(reportID(r), in.Status, in.TriageNote)
+	if errors.Is(err, reports.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "no such report")
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, rep)
+}
+
+// handleDeleteReport handles DELETE /api/reports/{id}.
+func (s *Server) handleDeleteReport(w http.ResponseWriter, r *http.Request) {
+	if !s.reportsReady(w) {
+		return
+	}
+	err := s.reports.Delete(reportID(r))
+	if errors.Is(err, reports.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "no such report")
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// pruneReports re-applies the inbox bounds hourly. Insert already prunes, so
+// this only matters when a bound was lowered between restarts.
+func (s *Server) pruneReports() {
+	run := func() {
+		if n, err := s.reports.Prune(); err != nil {
+			log.Printf("[reports] prune failed: %v", err)
+		} else if n > 0 {
+			log.Printf("[reports] pruned %d report(s) over the inbox bounds", n)
+		}
+	}
+	run()
+	ticker := time.NewTicker(time.Hour)
+	defer ticker.Stop()
+	for range ticker.C {
+		run()
+	}
+}
+
+func firstLine(s string) string {
+	if i := strings.IndexByte(s, '\n'); i >= 0 {
+		s = s[:i]
+	}
+	if r := []rune(s); len(r) > 80 {
+		s = string(r[:80]) + "…"
+	}
+	if s == "" {
+		return "no note"
+	}
+	return strconv.Quote(s)
+}

internal/api/reports_test.go

@@ -0,0 +1,121 @@
+package api
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/reports"
+)
+
+func newReportServer(t *testing.T) *Server {
+	t.Helper()
+	store, err := reports.Open(filepath.Join(t.TempDir(), "reports.db"), 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = store.Close() })
+	s := &Server{router: mux.NewRouter(), reports: store}
+	s.setupReportRoutes()
+	return s
+}
+
+func TestReportsRoundTrip(t *testing.T) {
+	s := newReportServer(t)
+
+	rec, out := do(t, s, "POST", "/api/reports", map[string]any{
+		"app": "meta-read", "note": "blank page", "uid": "u1",
+		"reader": map[string]any{"page": 3}, "log": []string{"a", "b"},
+	})
+	if rec.Code != http.StatusCreated || out["id"] == nil {
+		t.Fatalf("post: %d %s", rec.Code, rec.Body)
+	}
+	id := int64(out["id"].(float64))
+
+	rec, out = do(t, s, "GET", "/api/reports", nil)
+	list, _ := out["reports"].([]any)
+	if rec.Code != 200 || len(list) != 1 {
+		t.Fatalf("list: %d %s", rec.Code, rec.Body)
+	}
+	first := list[0].(map[string]any)
+	if _, hasBody := first["body"]; hasBody || first["log_lines"].(float64) != 2 {
+		t.Fatalf("summary should carry log_lines, not the body: %v", first)
+	}
+
+	path := "/api/reports/" + itoa(id)
+	rec, out = do(t, s, "GET", path, nil)
+	body, _ := out["body"].(map[string]any)
+	if rec.Code != 200 || body["reader"] == nil || out["uid"] != "u1" {
+		t.Fatalf("get: %d %s", rec.Code, rec.Body)
+	}
+
+	rec, out = do(t, s, "PATCH", path, map[string]any{"status": "triaged", "triage_note": "Deck #1"})
+	if rec.Code != 200 || out["status"] != "triaged" || out["triage_note"] != "Deck #1" {
+		t.Fatalf("patch: %d %s", rec.Code, rec.Body)
+	}
+	rec, _ = do(t, s, "GET", "/api/reports?status=triaged&app=meta-read", nil)
+	if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"triaged":1`) {
+		t.Fatalf("filtered list: %d %s", rec.Code, rec.Body)
+	}
+
+	if rec, _ = do(t, s, "DELETE", path, nil); rec.Code != http.StatusNoContent {
+		t.Fatalf("delete: %d", rec.Code)
+	}
+	if rec, _ = do(t, s, "GET", path, nil); rec.Code != http.StatusNotFound {
+		t.Fatalf("get after delete: %d", rec.Code)
+	}
+}
+
+func TestReportsRejects(t *testing.T) {
+	s := newReportServer(t)
+	cases := []struct {
+		name, method, path string
+		body               any
+		want               int
+	}{
+		{"no app", "POST", "/api/reports", map[string]any{"note": "x"}, 400},
+		{"bad status filter", "GET", "/api/reports?status=done", nil, 400},
+		{"patch nothing", "PATCH", "/api/reports/1", map[string]any{}, 400},
+		{"patch bad status", "PATCH", "/api/reports/1", map[string]any{"status": "done"}, 400},
+		{"patch missing", "PATCH", "/api/reports/1", map[string]any{"status": "closed"}, 404},
+		{"delete missing", "DELETE", "/api/reports/1", nil, 404},
+	}
+	for _, c := range cases {
+		if rec, _ := do(t, s, c.method, c.path, c.body); rec.Code != c.want {
+			t.Errorf("%s: got %d want %d (%s)", c.name, rec.Code, c.want, rec.Body)
+		}
+	}
+
+	// Not a JSON object.
+	req := httptest.NewRequest("POST", "/api/reports", strings.NewReader(`[1,2]`))
+	rec := httptest.NewRecorder()
+	s.router.ServeHTTP(rec, req)
+	if rec.Code != 400 {
+		t.Errorf("array body: %d", rec.Code)
+	}
+
+	// Over the body cap.
+	big := bytes.Repeat([]byte("x"), maxReportBody+1)
+	req = httptest.NewRequest("POST", "/api/reports", strings.NewReader(`{"app":"a","note":"`+string(big)+`"}`))
+	rec = httptest.NewRecorder()
+	s.router.ServeHTTP(rec, req)
+	if rec.Code != http.StatusRequestEntityTooLarge {
+		t.Errorf("oversized body: %d", rec.Code)
+	}
+}
+
+func TestReportsUnavailableWithoutStore(t *testing.T) {
+	s := &Server{router: mux.NewRouter()}
+	s.setupReportRoutes()
+	if rec, _ := do(t, s, "GET", "/api/reports", nil); rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("no store: %d", rec.Code)
+	}
+}
+
+func itoa(n int64) string { return strconv.FormatInt(n, 10) }

internal/api/schema_handlers.go

@@ -3,43 +3,46 @@ package api
 import (
 	"encoding/json"
 	"net/http"
-	"time"
 )
 
-// handleSchemaGet returns the live per-field schema derived from meta:events.
+// handleSchemaGet returns the per-field schema, folded from counters the write
+// path keeps exact (see schema.Indexer).
 func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
 	if s.schemaIndexer == nil {
 		http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
 		return
 	}
-	snap := s.schemaIndexer.Snapshot()
+	snap, err := s.schemaIndexer.Snapshot(r.Context())
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(snap)
 }
 
-// handleSchemaRescan re-emits a set event for every existing property via the
+// handleSchemaRescan rebuilds the schema counters from a full read of the
-// meta publisher, waits for the indexer to drain, and returns the refreshed
+// records and returns the result. The counters are exact by construction, so
-// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
+// this is a repair tool, not routine: it runs as one write transaction, and
+// every other write waits for it (minutes on a large store).
 func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
 	if s.schemaIndexer == nil {
 		http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
 		return
 	}
-
+	if err := s.storage.RebuildCounters(); err != nil {
-	count, err := s.RepublishMetadata()
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	snap, err := s.schemaIndexer.Snapshot(r.Context())
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-
-	s.schemaIndexer.WaitForDrain(10 * time.Second)
-
-	snap := s.schemaIndexer.Snapshot()
 	snap.Source = "rescan"
-
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(map[string]any{
-		"published_events": count,
+		"published_events": 0,
 		"schema":           snap,
 	})
 }

internal/api/search_index.go

@@ -1,221 +1,59 @@
 package api
 
 import (
-	"log"
 	"net/http"
-	"sort"
 	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
 
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
+// serveIndexedSearch answers a search from the database's derived indexes
-// from Redis in the background. Records ingest continuously, so a newly stored
+// (storage.Search): SQL narrows to a superset of candidates through the field
-// record becomes searchable within one interval; the rebuild is a single
+// expression index and the FTS5 trigram table, then every candidate is checked
-// batched bulk read (storage.GetSearchIndexTuples), off the request path.
+// here with the exact matchers below — a token-AND match against the record's
-const searchIndexRefreshInterval = 60 * time.Second
+// haystack (or a whole-query hashID substring, preserving the legacy
-
+// behaviour) AND every structured filter.
-// searchIndexReconcileEvery is how many refresh ticks pass between full
-// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full
-// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all.
-const searchIndexReconcileEvery = 30
-
-// searchIndex is an in-memory snapshot of every record's search haystack
-// (hashID + lowercased searchable fields), kept current from Redis. It exists
-// so POST /api/metadata/search matches in memory (string ops over a slice)
-// instead of doing a SCAN+MGET Redis round-trip per record — the latter took
-// 40s+ over ~72k records and made a live parallel search path unusable.
 //
-// # Two update paths, and why
+// Until 2026-10 this ran against an in-memory copy of every record, rebuilt
-//
+// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh)
-// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a
+// and is gone. The matchers did not change, so neither did the semantics.
-// complete fresh copy. That is O(corpus) regardless of how much changed, and it
-// used to run every 60 s forever. Measured on a 19 622-record staging corpus
-// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side
-// held 783 MB for data Redis stores in 135 MB — largely the double copy.
-//
-// `refreshIncremental` re-reads only the records that changed, tracked by
-// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady
-// state; the full rebuild runs once at startup (there is nothing to be
-// incremental against) and then only as an explicit repair.
-//
-// The distinction matters because meta-core is the platform's permanent,
-// additive store — every search hit on every gateway is retained — so a rebuild
-// priced against the whole corpus is a cost that only ever grows.
-type searchIndex struct {
-	mu sync.RWMutex
-	// byID is the authoritative content; tuples is a stable-order view of it
-	// rebuilt on each update. Map iteration order in Go is randomised, so
-	// serving a limited query straight off the map would return a different
-	// subset each call — the slice is what keeps results stable.
-	byID   map[string]storage.SearchTuple
-	tuples []storage.SearchTuple
-	ready  atomic.Bool
-}
-
-// snapshot returns the current tuples slice and whether the index has built at
-// least once. Every update swaps in a fresh slice under the write lock (never
-// mutates in place), so the returned slice is safe to read lock-free.
-func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
-	idx.mu.RLock()
-	defer idx.mu.RUnlock()
-	return idx.tuples, idx.ready.Load()
-}
-
-// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds
-// the write lock.
-func (idx *searchIndex) rebuildViewLocked() {
-	view := make([]storage.SearchTuple, 0, len(idx.byID))
-	for _, t := range idx.byID {
-		view = append(view, t)
-	}
-	// Stable across rebuilds so a limited query returns a consistent page
-	// rather than a fresh random subset every interval.
-	sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID })
-	idx.tuples = view
-}
-
-// refresh rebuilds the whole index from Redis (a batched bulk read). Used for
-// the initial build and as an explicit repair — not the steady state.
-func (idx *searchIndex) refresh(stor *storage.Client) {
-	if stor == nil || !stor.IsConnected() {
-		return
-	}
-	tuples, err := stor.GetSearchIndexTuples()
-	if err != nil {
-		log.Printf("[API] search index refresh failed: %v", err)
-		return
-	}
-	byID := make(map[string]storage.SearchTuple, len(tuples))
-	for _, t := range tuples {
-		byID[t.HashID] = t
-	}
-	idx.mu.Lock()
-	idx.byID = byID
-	idx.rebuildViewLocked()
-	idx.mu.Unlock()
-	idx.ready.Store(true)
-}
-
-// refreshIncremental drains the dirty set and patches only those records.
-// Returns whether more dirty entries remain, so a burst is caught up over
-// consecutive passes rather than one per interval.
-//
-// A record in the dirty set but absent from storage was deleted and is evicted
-// here — otherwise it would keep answering searches until the next full
-// rebuild. A batch that fails after the (destructive) drain is re-marked by
-// storage so it is retried rather than silently lost.
-func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) {
-	if stor == nil || !stor.IsConnected() {
-		return false
-	}
-	ids, more, err := stor.DrainDirty()
-	if err != nil {
-		log.Printf("[API] search index drain failed: %v", err)
-		return false
-	}
-	if len(ids) == 0 {
-		return false
-	}
-	tuples, gone, err := stor.GetSearchIndexTuplesFor(ids)
-	if err != nil {
-		stor.MarkDirty(ids...)
-		log.Printf("[API] incremental search index refresh failed: %v", err)
-		return false
-	}
-
-	idx.mu.Lock()
-	if idx.byID == nil {
-		idx.byID = make(map[string]storage.SearchTuple, len(tuples))
-	}
-	for _, t := range tuples {
-		idx.byID[t.HashID] = t
-	}
-	for _, id := range gone {
-		delete(idx.byID, id)
-	}
-	idx.rebuildViewLocked()
-	idx.mu.Unlock()
-
-	log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples))
-	return more
-}
-
-// runRefreshLoop builds the index once in full, then keeps it current from the
-// dirty set, with a periodic full reconcile as a safety net.
-//
-// The reconcile exists because a *missed* dirty mark is silent: no error, just
-// a record whose indexed copy quietly diverges from storage until something
-// forces a rebuild. Marking is spread across eight write paths plus anything
-// added later, and the cost of getting it wrong is invisible — so the index
-// should not depend on that coverage being perfect forever. Every
-// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs
-// whatever drifted.
-//
-// That still drops the full rebuild from every 60 s to every 30 min — a ~30×
-// reduction in the steady-state cost that measured at ~45 % of all Redis CPU —
-// while keeping the self-healing property that made the naive loop safe.
-func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
-	idx.refresh(stor)
-	ticker := time.NewTicker(searchIndexRefreshInterval)
-	defer ticker.Stop()
-	ticks := 0
-	for range ticker.C {
-		ticks++
-		if ticks%searchIndexReconcileEvery == 0 {
-			idx.refresh(stor)
-			continue
-		}
-		// Drain until caught up, so a burst larger than one batch does not take
-		// one interval per batch to land.
-		for idx.refreshIncremental(stor) {
-		}
-	}
-}
-
-// serveIndexedSearch answers a search from the in-memory index: a token-AND
-// match against each record's haystack (or a whole-query hashID substring,
-// preserving the legacy behaviour) AND every structured filter, then returns
-// the full metadata for the <=limit survivors. No Redis round-trip per scanned
-// record — matching and results both come from memory.
 //
 // This is the ONLY search path. Free text, filters, both, or neither all land
 // here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
 func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
-	tuples, ready := s.search.snapshot()
-	if !ready {
-		// First call before the background build finished: warm inline with the
-		// efficient batched read (seconds), never the per-record SCAN fallback.
-		s.search.refresh(s.storage)
-		tuples, _ = s.search.snapshot()
-	}
 	tokens := strings.Fields(strings.ToLower(req.Query))
 	queryLower := strings.ToLower(req.Query)
 	filters := normalizedFilters(req)
-	results := make([]MetadataSearchResult, 0, req.Limit)
+
-	for _, t := range tuples {
+	narrow := make([]storage.SearchFilter, 0, len(filters))
-		if len(results) >= req.Limit {
+	for _, f := range filters {
-			break
+		narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
-		}
+	}
+	hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool {
 		if req.Query != "" &&
-			!matchAllTokens(t.Haystack, tokens) &&
+			!matchAllTokens(storage.Haystack(fields), tokens) &&
-			!strings.Contains(strings.ToLower(t.HashID), queryLower) {
+			!strings.Contains(strings.ToLower(hashID), queryLower) {
-			continue
+			return false
 		}
-		if !recordMatchesFilters(t.Fields, filters) {
+		return recordMatchesFilters(fields, filters)
-			continue
+	})
-		}
+	if err != nil {
-		// Full metadata is cached in the index — return it straight from memory,
+		writeError(w, http.StatusInternalServerError, "search failed: "+err.Error())
-		// no Redis round-trip per result.
+		return
-		results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
+	}
+	total, err := s.storage.CountRoots()
+	if err != nil {
+		total = 0
+	}
+
+	results := make([]MetadataSearchResult, 0, len(hits))
+	for _, h := range hits {
+		results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields})
 	}
 	writeJSON(w, http.StatusOK, MetadataSearchResponse{
 		Results: results,
 		Count:   len(results),
-		Total:   len(tuples),
+		Total:   int(total),
 	})
 }
 

internal/api/server.go

@@ -9,11 +9,11 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
-	"github.com/metazla/meta-core/internal/events"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/mounts"
+	"github.com/metazla/meta-core/internal/reports"
 	"github.com/metazla/meta-core/internal/schema"
 	"github.com/metazla/meta-core/internal/storage"
 	"github.com/metazla/meta-core/internal/watcher"
@@ -25,8 +25,6 @@ import (
 type Server struct {
 	config            *config.Config
 	leaderProvider    *leader.LeaderInfoProvider
-	discovery         *discovery.Service
-	cleaner           *discovery.Cleaner
 	storage           *storage.Client
 	mountsManager     *mounts.Manager
 	mountsHandlers    *mounts.Handlers
@@ -38,40 +36,50 @@ type Server struct {
 	watchersManager   *watchers.Manager
 	watchersPoller    *watchers.Poller
 	watchersHandlers  *watchers.Handlers
-	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
-	search            *searchIndex
 	// challenges backs proof-of-possession on reveal/delete. In-process, so a
 	// restart invalidates the ones in flight — see identity/challenge.go.
 	challenges *identity.ChallengeStore
-	router     *mux.Router
+	// mesh is the UDP discovery node (beacon v2). nil when discovery
-	server     *http.Server
+	// is disabled or failed to bind, in which case /api/neighbors reports an
+	// empty list rather than erroring — a dashboard nav is decoration and
+	// must not take the API down with it.
+	mesh   *meshdisco.Node
+	router *mux.Router
+	server *http.Server
+	// reports is the bug-report inbox (reports.go); nil = routes answer 503.
+	reports *reports.Store
+	// stats caches the keyspace-walk half of GET /api/stats so a polling
+	// dashboard never puts a multi-second store walk on the request path.
+	stats *statsCache
+}
+
+// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
+// server's constructor signature (and its tests) stay untouched.
+func (s *Server) SetMeshNode(n *meshdisco.Node) {
+	s.mesh = n
 }
 
 // NewServer creates a new API server
 func NewServer(
 	cfg *config.Config,
 	leaderProvider *leader.LeaderInfoProvider,
-	disc *discovery.Service,
-	cleaner *discovery.Cleaner,
 	stor *storage.Client,
 ) *Server {
 	s := &Server{
 		config:         cfg,
 		leaderProvider: leaderProvider,
-		discovery:      disc,
-		cleaner:        cleaner,
 		storage:        stor,
-		search:         &searchIndex{},
 		router:         mux.NewRouter(),
+		stats:          &statsCache{},
 	}
 
 	// Initialize file watcher/scanner and dispatcher
 	var scanFunc mounts.ScanFunc
 	if cfg.EnableFileWatcher {
 		s.watcherDispatcher = watcher.NewDispatcher()
-		// Wire storage client for Redis Stream publishing
+		// Wire storage client for file:events publishing
 		if stor != nil {
 			s.watcherDispatcher.SetStorageClient(stor)
 		}
@@ -81,7 +89,7 @@ func NewServer(
 		} else {
 			s.fileWatcher = fileWatcher
 			// Wire storage so the watcher can persist (path,size,mtime,cid)
-			// tuples to Redis as it computes midhashes.
+			// tuples to the store as it computes midhashes.
 			if stor != nil {
 				fileWatcher.SetStorageClient(stor)
 			}
@@ -100,10 +108,9 @@ func NewServer(
 			// Create watchers poller
 			if s.fileWatcher != nil {
 				s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
-				// Pass republish callback to handlers (handles nil metaPublisher case)
+				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller)
-				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
 
-				// Hydrate the in-memory state registry from Redis tuples so
+				// Hydrate the in-memory state registry from stored tuples so
 				// the first scan tick after boot can skip ComputeMidHash256
 				// for files whose (size, mtime) match.
 				configs := watchersManager.List()
@@ -114,7 +121,7 @@ func NewServer(
 				if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
 					log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
 				} else {
-					log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
+					log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
 				}
 			}
 		}
@@ -164,11 +171,15 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
 	s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
 
+	// Dashboard headline counters in one request. Cheap half live, keyspace
+	// walks served from a background-refreshed cache — see stats.go.
+	s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
+
 	// Metadata Editor API routes (must be before /meta/{hash} routes)
 	s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
 	s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
 	s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
-	s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
+	s.router.HandleFunc("/api/metadata/batch", s.handleGone).Methods("POST")
 	s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
 
 	// User Data Layer signing identity (see internal/identity). Auth-gated
@@ -184,15 +195,18 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
 	s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
 
-	// Snapshot (export / import / wipe)
+	// Signed claims (v2.0.0): the only metadata write path, the raw claims
-	s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
+	// behind a record, and the stateless merge clients use for search hits.
-	s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
+	s.router.HandleFunc("/api/claims", s.handlePostClaims).Methods("POST")
-	s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
+	s.router.HandleFunc("/api/merge", s.handleMerge).Methods("POST")
+	s.router.HandleFunc("/api/meta/{cid}/claims", s.handleClaimsByCID).Methods("GET")
+	s.router.HandleFunc("/api/metadata/{hashId}/claims", s.handleClaimsByRID).Methods("GET")
+
 	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
-	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
+	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleGone).Methods("PUT")
 	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
-	s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
+	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("PUT")
-	s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
+	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("DELETE")
 
 	// Schema inference
 	s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
@@ -216,24 +230,23 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
 	s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
 	s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
 	s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
 
 	// Metadata operations - base endpoints
 	s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
-	s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
+	s.router.HandleFunc("/meta/{hash}", s.handleGone).Methods("PUT", "PATCH", "DELETE")
-	s.router.HandleFunc("/meta/{hash}", s.handlePatchMeta).Methods("PATCH")
-	s.router.HandleFunc("/meta/{hash}", s.handleDeleteMeta).Methods("DELETE")
 	s.router.HandleFunc("/meta", s.handleListMeta).Methods("GET")
 
 	// Metadata operations - set operations (must be before property routes)
-	s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleAddToSet).Methods("POST")
+	s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleGone).Methods("POST")
 
 	// Metadata operations - property-level (key can contain slashes)
 	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGetProperty).Methods("GET")
-	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handlePutProperty).Methods("PUT")
+	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGone).Methods("PUT", "DELETE")
-	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleDeleteProperty).Methods("DELETE")
 
 	// Data operations
 	s.router.HandleFunc("/data/{hash}/path", s.handleGetDataPath).Methods("GET")
@@ -257,30 +270,21 @@ func (s *Server) setupRoutes() {
 	// without going through Authelia.
 	s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
 
-	// Admin: reunify stranded midhash-rooted entries into their UUID roots
+	// SSE event streams. The only way consumers see meta-core's change
-	// (the dual-root pattern caused by historical writes that bypassed CID
+	// streams — they never touch the database. Auth-bypass
-	// resolution).
-	s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
-	// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
-	// (METADATA_KEYS.md §14.17). Idempotent.
-	s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
-	// One-shot sweep of query echoes on literature cards (meta-read
-	// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
-	s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
-
-	// SSE event streams. Mediated mirror of the Redis Streams; consumers
-	// reach them over HTTP instead of touching Redis directly. Auth-bypass
 	// applies but exposure should be inside-only (do NOT proxy through
 	// Caddy publicly). See docs/api-mediated-access.md "Auth".
 	s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
 	s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
 
-	// Service discovery
+	// beacon v2: neighbours heard over UDP. Supersedes /api/services,
-	// Both /services and /api/services are supported for consistency with other services
+	// which is still served above while consumers migrate.
-	s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
+	s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
-	s.router.HandleFunc("/api/services", s.handleListServices).Methods("GET")
+	s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
-	s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
+
-	s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
+	// Report inbox: bug reports / feedback forwarded by the client apps,
+	// read by the dashboard's Reports page. Perimeter-gated (reports.go).
+	s.setupReportRoutes()
 
 	// Mount management routes (if manager initialized)
 	if s.mountsHandlers != nil {
@@ -349,54 +353,25 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
-	// Build + periodically refresh the in-memory search index so
+	// Schema: served from counters the write path keeps exact; nothing runs.
-	// POST /api/metadata/search matches in memory instead of a Redis
+	// Change events need no publisher either — the storage engine appends them
-	// round-trip per record (see search_index.go).
+	// in the same transaction as each change, and a consumer that wants the
-	if s.storage != nil {
+	// whole state asks /api/events/meta for "0" and gets a snapshot (see
-		go s.search.runRefreshLoop(s.storage)
+	// streamSnapshot). The boot republish of every field is gone with it.
-	}
-
-	// Start meta publisher (if storage is connected)
-	// Publishes keyspace notifications to meta:events stream
 	if s.storage != nil && s.storage.IsConnected() {
-		s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient())
+		s.schemaIndexer = schema.NewIndexer(s.storage.Store())
-		if err := s.metaPublisher.Start(); err != nil {
+		go func() {
-			log.Printf("[API] Warning: failed to start meta publisher: %v", err)
+			started := time.Now()
-		} else {
+			if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
-			// Schema indexer consumes the same stream to derive a live
+				log.Printf("[API] Warning: search row rebuild failed: %v", err)
-			// per-field schema (type/hint/breakdown). Start it before the
+			} else if n > 0 {
-			// republish flood so it captures the bootstrap events.
+				log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
-			s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
-			if err := s.schemaIndexer.Start(); err != nil {
-				log.Printf("[API] Warning: failed to start schema indexer: %v", err)
-				s.schemaIndexer = nil
 			}
-
+		}()
-			// Republish events for all existing metadata
+		go s.logWriteStats()
-			go func() {
+	}
-				// Build any missing per-record field indexes first, in ONE
+	if s.reports != nil {
-				// keyspace pass. The republish below reads every record, and
+		go s.pruneReports()
-				// an un-indexed record read costs a full-keyspace SCAN — on a
-				// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
-				// paid again on every restart). Warming first turns the whole
-				// republish into index-served reads. Idempotent, so a
-				// fully-indexed store just pays the single scan.
-				// See storage/field_index.go.
-				started := time.Now()
-				if n, err := s.storage.WarmFieldIndexes(); err != nil {
-					log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
-				} else if n > 0 {
-					log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
-				}
-
-				count, err := s.RepublishMetadata()
-				if err != nil {
-					log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)
-				} else {
-					log.Printf("[API] Republished %d metadata events on startup", count)
-				}
-			}()
-		}
 	}
 
 	return nil
@@ -423,20 +398,6 @@ func (s *Server) Stop() error {
 		s.mountStatsPoller.Stop()
 	}
 
-	// Stop schema indexer before the publisher so the consumer drains cleanly
-	if s.schemaIndexer != nil {
-		if err := s.schemaIndexer.Stop(); err != nil {
-			log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
-		}
-	}
-
-	// Stop meta publisher
-	if s.metaPublisher != nil {
-		if err := s.metaPublisher.Stop(); err != nil {
-			log.Printf("[API] Warning: failed to stop meta publisher: %v", err)
-		}
-	}
-
 	if s.server == nil {
 		return nil
 	}
@@ -449,26 +410,22 @@ func (s *Server) Stop() error {
 	return s.server.Shutdown(ctx)
 }
 
-// RepublishMetadata republishes all existing metadata to the meta:events stream
+// logWriteStats reports, every 15 minutes, how many writes changed a value
-// This is called on startup and after reset-all
+// and how many found it already in place (kv skips those). The ratio is what
-func (s *Server) RepublishMetadata() (int, error) {
+// the no-op skip buys on a permanent store that re-persists repeated hits.
-	if s.metaPublisher == nil {
+func (s *Server) logWriteStats() {
-		return 0, fmt.Errorf("meta publisher not initialized")
+	ticker := time.NewTicker(15 * time.Minute)
-	}
+	defer ticker.Stop()
-	if s.storage == nil || !s.storage.IsConnected() {
+	for range ticker.C {
-		return 0, fmt.Errorf("storage not connected")
+		store := s.storage.Store()
-	}
+		if store == nil {
-
+			return
-	// Create the republish function that wraps storage methods
+		}
-	republishFunc := func() ([]string, func(string) (map[string]string, error), error) {
+		changed, unchanged := store.WriteStats()
-		hashIDs, err := s.storage.GetAllHashIDs()
+		if total := changed + unchanged; total > 0 {
-		if err != nil {
+			log.Printf("[Storage] writes since start: %d changed, %d unchanged skipped (%.0f%% no-op)", changed, unchanged, 100*float64(unchanged)/float64(total))
-			return nil, nil, err
 		}
-		return hashIDs, s.storage.GetMetadataFlat, nil
 	}
-
-	return s.metaPublisher.RepublishAllMetadata(republishFunc)
 }
 
 // statusRecorder captures the response status so loggingMiddleware can stay
@@ -521,7 +478,7 @@ func loggingMiddleware(next http.Handler) http.Handler {
 func corsMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
 		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
 
 		if r.Method == "OPTIONS" {

internal/api/snapshot.go

@@ -1,167 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/metazla/meta-core/internal/snapshot"
-)
-
-// Bound the upload size so a malicious client can't OOM the server.
-// 2 GiB is well above any realistic export.
-const maxImportBytes = 2 << 30
-
-// pluginOutputDir is where plugin-produced artifacts live on disk
-// (subtitles, posters, etc). Mounted from the host into the meta-core
-// container at $FILES_PATH/plugin.
-func (s *Server) pluginOutputDir() string {
-	return filepath.Join(s.config.FilesPath, "plugin")
-}
-
-// handleSnapshotExport streams a ZIP snapshot of all metadata.
-// GET /api/snapshot/export?include=hash-index,plugin-output
-func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// The server has a 30s WriteTimeout (server.go:222). Long exports
-	// blow past that, leaving a truncated ZIP without a central directory.
-	// Clear the deadline for this handler so the export can run as long as
-	// needed. ResponseController is the standard way to do this.
-	rc := http.NewResponseController(w)
-	_ = rc.SetWriteDeadline(time.Time{})
-
-	include := splitCSVParam(r.URL.Query().Get("include"))
-	opts := snapshot.ExportOptions{
-		IncludePluginOutput: include["plugin-output"],
-		PluginOutputDir:     s.pluginOutputDir(),
-	}
-
-	filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
-	w.Header().Set("Content-Type", "application/zip")
-	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
-	// We stream — no Content-Length.
-
-	exporter := snapshot.NewExporter(s.storage)
-	if _, err := exporter.Export(w, opts); err != nil {
-		// Headers may already be sent; log and bail.
-		// Client will see a truncated zip, which is the best signal we can give.
-		fmt.Printf("[Snapshot] export failed: %v\n", err)
-		return
-	}
-}
-
-// handleSnapshotImport applies an uploaded ZIP snapshot.
-// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
-// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
-func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
-	// when the snapshot is large. Clear both for this handler.
-	rc := http.NewResponseController(w)
-	_ = rc.SetReadDeadline(time.Time{})
-	_ = rc.SetWriteDeadline(time.Time{})
-
-	q := r.URL.Query()
-	opts := snapshot.ImportOptions{
-		Mode:            snapshot.Mode(q.Get("mode")),
-		Conflict:        snapshot.Conflict(q.Get("conflict")),
-		DryRun:          strings.EqualFold(q.Get("dry_run"), "true"),
-		PluginOutputDir: s.pluginOutputDir(),
-	}
-	if opts.Mode == "" {
-		opts.Mode = snapshot.ModeMerge
-	}
-	if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
-		opts.Conflict = snapshot.ConflictSource
-	}
-
-	body, err := readImportBody(r)
-	if err != nil {
-		writeError(w, http.StatusBadRequest, err.Error())
-		return
-	}
-
-	importer := snapshot.NewImporter(s.storage)
-	result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
-	if err != nil {
-		writeError(w, http.StatusBadRequest, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, result)
-}
-
-// handleSnapshotWipe clears Redis metadata.
-// POST /api/snapshot/wipe?scope=metadata
-func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	scopes := splitCSVParam(r.URL.Query().Get("scope"))
-	if len(scopes) == 0 {
-		writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
-		return
-	}
-	scope := snapshot.WipeScope{
-		Metadata: scopes["metadata"],
-	}
-	if !scope.Metadata {
-		writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
-		return
-	}
-
-	wiper := snapshot.NewWiper(s.storage)
-	res, err := wiper.Wipe(scope)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, res)
-}
-
-// readImportBody accepts either a raw application/zip body or a multipart
-// upload with a field named "snapshot". Buffers up to maxImportBytes.
-func readImportBody(r *http.Request) ([]byte, error) {
-	r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
-
-	ct := r.Header.Get("Content-Type")
-	if strings.HasPrefix(ct, "multipart/form-data") {
-		if err := r.ParseMultipartForm(32 << 20); err != nil {
-			return nil, fmt.Errorf("parse multipart: %w", err)
-		}
-		f, _, err := r.FormFile("snapshot")
-		if err != nil {
-			return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
-		}
-		defer f.Close()
-		return io.ReadAll(f)
-	}
-
-	return io.ReadAll(r.Body)
-}
-
-// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
-func splitCSVParam(v string) map[string]bool {
-	out := map[string]bool{}
-	for _, part := range strings.Split(v, ",") {
-		p := strings.TrimSpace(part)
-		if p != "" {
-			out[p] = true
-		}
-	}
-	return out
-}

internal/api/sse_events.go

@@ -3,7 +3,6 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -11,15 +10,18 @@ import (
 	"strings"
 	"time"
 
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
+	"github.com/metazla/meta-core/internal/storage"
 )
 
-// SSE event stream endpoints for the file:events and meta:events Redis Streams.
+// SSE event stream endpoints for the file:events and meta:events streams.
 //
 // Wire contract (see docs/api-mediated-access.md):
-//   - One SSE event per Redis Stream entry.
+//   - One SSE event per stream entry.
-//   - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
+//   - `id:` is the opaque stream entry ID (`<ms>-<seq>`, the Redis Streams
-//     MUST treat it as a black-box string they echo back via Last-Event-ID.
+//     form, kept across the move to SQLite so cursors consumers persisted to
+//     disk still compare correctly); consumers MUST treat it as a black-box
+//     string they echo back via Last-Event-ID.
 //   - `event:` is the `type` field from the underlying entry.
 //   - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
 //   - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
@@ -30,30 +32,30 @@ import (
 const (
 	sseHeartbeatInterval = 30 * time.Second
 
-	// sseReadBlock controls how long each XREAD blocks before returning.
+	// sseReadBlock controls how long each read waits for new entries.
 	// Much shorter than sseHeartbeatInterval so the handler stays responsive
 	// to client disconnects and the heartbeat check runs frequently enough
 	// that the keep-alive fires within a few seconds of its nominal cadence.
 	sseReadBlock = 2 * time.Second
 
-	// sseReadCount caps how many entries one XREAD returns. Small enough
+	// sseReadCount caps how many entries one read returns. Small enough
 	// to keep per-flush memory low, large enough to drain bursts cheaply.
 	sseReadCount = 100
 )
 
-// handleEventsFiles streams the file:events Redis Stream as SSE.
+// handleEventsFiles streams the file:events stream as SSE.
 func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "file:events")
 }
 
-// handleEventsMeta streams the meta:events Redis Stream as SSE.
+// handleEventsMeta streams the meta:events stream as SSE.
 func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "meta:events")
 }
 
-// streamSSE is the shared SSE pump shared by both endpoints. One handler per
+// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads
-// stream — there is no fan-out; each subscriber holds its own XREAD loop.
+// the stream itself and sleeps on the store's commit broadcast between reads —
-// That's fine at our scale (≤20 subscribers; one per consumer service).
+// fine at our scale (≤20 subscribers; one per consumer service).
 func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
 	if !s.storage.IsConnected() {
 		w.Header().Set("Retry-After", "5")
@@ -89,14 +91,37 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	w.WriteHeader(http.StatusOK)
 	flusher.Flush()
 
-	client := s.storage.GetRedisClient()
+	store := s.storage.Store()
-	if client == nil {
+	if store == nil {
 		// Storage was just disconnected — close cleanly.
 		return
 	}
 
 	ctx := r.Context()
-	cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
+	lastID := r.Header.Get("Last-Event-ID")
+	var cursor string
+	if stream == storage.MetaEventsStream && (lastID == "0" || lastID == "0-0") {
+		// "From the beginning" on meta:events means "the whole current
+		// state": stream it from the records themselves, then the live tail.
+		head, err := streamSnapshot(ctx, store, w, flusher)
+		if err != nil {
+			if ctx.Err() == nil {
+				log.Printf("[SSE %s] snapshot failed: %v", stream, err)
+			}
+			return
+		}
+		cursor = head
+	} else {
+		if rebuilt := s.storage.RebuiltAt(); stream == storage.MetaEventsStream && lastID != "" && rebuilt != "" &&
+			compareStreamIDs(lastID, rebuilt) < 0 {
+			// The cache was rebuilt after this cursor: the rebuild published no
+			// per-key events, so the consumer must take a snapshot.
+			writeGap(w, flusher, lastID, rebuilt, "rebuild")
+			cursor = "0"
+		} else {
+			cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher)
+		}
+	}
 
 	lastWrite := time.Now()
 	for {
@@ -104,37 +129,34 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 			return
 		}
 
-		// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
+		// Take the wake-up channel BEFORE reading, so an append landing
-		// context's deadline isn't propagated into XREAD's BLOCK arg —
+		// between the read and the wait still wakes us.
-		// that's a poll-style timeout on Redis itself — so we keep BLOCK
+		wake := store.Wait(stream)
-		// short.
+		entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount)
-		readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
-		result, err := client.XRead(readCtx, &redis.XReadArgs{
-			Streams: []string{stream, cursor},
-			Block:   sseReadBlock,
-			Count:   sseReadCount,
-		}).Result()
-		cancel()
-
 		if err != nil {
-			// redis.Nil = BLOCK timeout with no entries. Anything else
+			if ctx.Err() != nil {
-			// while the client is still connected is a real failure.
-			if errors.Is(err, redis.Nil) || isContextTimeout(err) {
-				// fall through to heartbeat check
-			} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
-				return
-			} else {
-				log.Printf("[SSE %s] XREAD error: %v", stream, err)
 				return
 			}
+			log.Printf("[SSE %s] read error: %v", stream, err)
+			return
 		}
 
-		for _, str := range result {
+		for _, e := range entries {
-			for _, msg := range str.Messages {
+			writeSSEEntry(w, flusher, e)
-				writeSSEEntry(w, flusher, msg)
+			cursor = e.ID
-				cursor = msg.ID
+			lastWrite = time.Now()
-				lastWrite = time.Now()
+		}
+
+		if len(entries) == 0 {
+			timer := time.NewTimer(sseReadBlock)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return
+			case <-wake:
+			case <-timer.C:
 			}
+			timer.Stop()
 		}
 
 		// Heartbeat if quiet for >= the interval. Comment line; SSE
@@ -149,87 +171,130 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	}
 }
 
+// streamSnapshot sends one `set` event per existing record key, then returns
+// the stream position the live tail continues from.
+//
+// This is how a consumer that rebuilds its state from events (meta-fuse sends
+// Last-Event-ID 0-0 on every start) gets ALL of it. The stream itself only
+// retains ~100k entries, so replaying it never could; meta-core used to
+// republish every field into it on boot instead, which took minutes and still
+// lost everything past the retention window.
+//
+// The head is captured BEFORE the snapshot is read, so a change racing the
+// snapshot appears in the tail too — re-applying a `set` is idempotent. Only
+// the final snapshot event carries an `id:`, so a consumer that disconnects
+// mid-snapshot still holds its old cursor and gets a fresh snapshot.
+func streamSnapshot(ctx context.Context, store *kv.Store, w http.ResponseWriter, flusher http.Flusher) (string, error) {
+	head, err := store.LastID(ctx, storage.MetaEventsStream)
+	if err != nil {
+		return "", err
+	}
+	ts := strconv.FormatInt(time.Now().UnixMilli(), 10)
+	pending := ""
+	err = store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
+		for _, row := range rows {
+			if pending != "" {
+				writeSnapshotEvent(w, pending, ts, "")
+			}
+			pending = row.K
+		}
+		flusher.Flush()
+		return ctx.Err()
+	})
+	if err != nil {
+		return "", err
+	}
+	if pending != "" {
+		writeSnapshotEvent(w, pending, ts, head)
+		flusher.Flush()
+	}
+	return head, nil
+}
+
+func writeSnapshotEvent(w http.ResponseWriter, key, ts, id string) {
+	payload, _ := json.Marshal(map[string]string{"key": key, "ts": ts})
+	if id != "" {
+		fmt.Fprintf(w, "id: %s\n", id)
+	}
+	fmt.Fprintf(w, "event: set\ndata: %s\n\n", payload)
+}
+
 // resolveStartCursor decides where to start reading.
-//   - No Last-Event-ID → start at "$" (only new entries).
+//   - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
+//     entries appended from now on). Redis' "$" was re-resolved on every
+//     XREAD, which dropped anything appended between two reads.
 //   - Last-Event-ID provided AND still in stream → resume from it.
 //   - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
 //     and resume from the oldest available entry's predecessor "0", so the
-//     next XREAD picks up everything still in the stream.
+//     next read picks up everything still in the stream.
 //
-// The "still in stream" check is cheap: one XRANGE for the first entry.
+// The "still in stream" check is cheap: one read of the first entry.
-func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
+func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
-	if lastID == "" {
-		return "$"
-	}
-
-	// One-entry XRANGE to find the oldest ID currently in the stream.
 	xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
-	entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
+
+	if lastID == "" {
+		head, err := store.LastID(xCtx, stream)
+		if err != nil {
+			log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err)
+			return "0"
+		}
+		return head
+	}
+
+	oldestEntry, ok, err := store.Oldest(xCtx, stream)
 	if err != nil {
 		// Treat any error as "stream not available right now" and resume
-		// from the cursor — XREAD will either find it or block until new
+		// from the cursor — the read either finds it or waits for new entries.
-		// entries arrive.
+		log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err)
-		log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
 		return lastID
 	}
 
-	if len(entries) == 0 {
+	if !ok {
 		// Stream is empty. Cursor may be from a previous incarnation; safest
-		// is to resume from the cursor — XREAD against an empty stream
+		// is to resume from the cursor — reading an empty stream returns
-		// returns nothing and blocks until new entries arrive.
+		// nothing and waits until new entries arrive.
 		return lastID
 	}
 
-	oldest := entries[0].ID
+	oldest := oldestEntry.ID
 	if compareStreamIDs(lastID, oldest) >= 0 {
 		// Cursor is at or past the oldest — fine to resume.
 		return lastID
 	}
 
 	// Cursor was trimmed. Emit a gap event so consumers that care can react.
-	gap := map[string]string{
+	writeGap(w, flusher, lastID, oldest, "retention")
-		"requested":  lastID,
-		"resumeFrom": oldest,
-		"reason":     "retention",
-	}
-	payload, _ := json.Marshal(gap)
-	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
-	flusher.Flush()
 
-	// Return "0" so the next XREAD picks up entries starting from the
+	// Return "0" so the next read picks up entries starting from the
 	// stream's true minimum — including `oldest` itself.
 	return "0"
 }
 
-// writeSSEEntry serialises one Redis Stream entry as an SSE event.
+// writeSSEEntry serialises one stream entry as an SSE event.
 //
 // `event:` comes from the entry's `type` field (no prefix munging). `data:`
-// is a JSON object containing all OTHER fields. Field values from the Redis
+// is a JSON object containing all OTHER fields.
-// stream are interface{} (typically strings); we encode whatever the stream
+func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) {
-// holds.
-func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
 	eventType := "message"
-	data := make(map[string]interface{}, len(msg.Values))
+	data := make(map[string]string, len(e.Values))
-	for k, v := range msg.Values {
+	for k, v := range e.Values {
 		if k == "type" {
-			if s, ok := v.(string); ok {
+			eventType = v
-				eventType = s
+			continue
-				continue
-			}
 		}
 		data[k] = v
 	}
 
 	payload, err := json.Marshal(data)
 	if err != nil {
-		log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
+		log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err)
 		return
 	}
 
 	// Spec: each event ends with a blank line. id/event/data lines are
 	// terminated with single \n.
-	fmt.Fprintf(w, "id: %s\n", msg.ID)
+	fmt.Fprintf(w, "id: %s\n", e.ID)
 	fmt.Fprintf(w, "event: %s\n", eventType)
 	fmt.Fprintf(w, "data: %s\n\n", string(payload))
 	flusher.Flush()
@@ -264,17 +329,14 @@ func parseStreamID(id string) (int64, int64) {
 	return ms, seq
 }
 
-// isContextTimeout returns true for the context deadline error wrapped by
+// writeGap emits the synthetic `gap` event: the consumer's cursor cannot be
-// go-redis when XREAD's read context fires. We treat it the same as
+// resumed losslessly and it should take a snapshot (Last-Event-ID: 0).
-// redis.Nil — i.e. an empty poll cycle, not a fatal error.
+func writeGap(w http.ResponseWriter, flusher http.Flusher, requested, resumeFrom, reason string) {
-func isContextTimeout(err error) bool {
+	payload, _ := json.Marshal(map[string]string{
-	if err == nil {
+		"requested":  requested,
-		return false
+		"resumeFrom": resumeFrom,
-	}
+		"reason":     reason,
-	if errors.Is(err, context.DeadlineExceeded) {
+	})
-		return true
+	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
-	}
+	flusher.Flush()
-	// go-redis sometimes wraps this in its own error type. String match is
-	// a safety net.
-	return strings.Contains(err.Error(), "context deadline exceeded")
 }

internal/api/sse_snapshot_test.go

@@ -0,0 +1,118 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// syncRecorder is a ResponseRecorder safe to read while the handler writes.
+type syncRecorder struct {
+	*httptest.ResponseRecorder
+	mu sync.Mutex
+}
+
+func (r *syncRecorder) Write(b []byte) (int, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.ResponseRecorder.Write(b)
+}
+
+func (r *syncRecorder) body() string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.Body.String()
+}
+
+// Last-Event-ID 0-0 on meta:events returns the whole current state — one set
+// per record key, only the last carrying an id — then the live tail.
+func TestMetaEventsSnapshotThenTail(t *testing.T) {
+	stor := newTestStorage(t)
+	s := &Server{storage: stor}
+	a := testSigner(t, 1)
+	for i, cid := range []string{"bafyR1", "bafyR2"} {
+		storeClaims(t, stor, signed(t, a, cid, "title", "T", 1), signed(t, a, cid, "n", string(rune('a'+i)), 1))
+	}
+	r1, r2 := stor.ResolveRoot("bafyR1"), stor.ResolveRoot("bafyR2")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
+	req.Header.Set("Last-Event-ID", "0-0")
+	rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()}
+	done := make(chan struct{})
+	go func() { s.handleEventsMeta(rec, req); close(done) }()
+
+	waitFor := func(substr string) {
+		t.Helper()
+		deadline := time.Now().Add(5 * time.Second)
+		for !strings.Contains(rec.body(), substr) {
+			if time.Now().After(deadline) {
+				t.Fatalf("timed out waiting for %q in:\n%s", substr, rec.body())
+			}
+			time.Sleep(20 * time.Millisecond)
+		}
+	}
+	waitFor(`"key":"file:` + r2 + `/title"`)
+	storeClaims(t, stor, signed(t, a, "bafyR1", "title", "changed", 2))
+	waitFor(`event: set`)
+	time.Sleep(300 * time.Millisecond)
+	cancel()
+	<-done
+
+	body := rec.body()
+	keys := []string{"file:" + r1 + "/title", "file:" + r1 + "/n", "file:" + r2 + "/title", "file:" + r2 + "/n",
+		"file:" + r1 + "/cids/bafyR1", "file:" + r2 + "/cids/bafyR2"}
+	for _, k := range keys {
+		if !strings.Contains(body, `"key":"`+k+`"`) {
+			t.Errorf("snapshot is missing %s", k)
+		}
+	}
+	events := strings.Split(strings.TrimSpace(body), "\n\n")
+	if len(events) != 7 {
+		t.Fatalf("want 6 snapshot events + 1 live event, got %d:\n%s", len(events), body)
+	}
+	for i, e := range events[:5] {
+		if strings.Contains(e, "id: ") {
+			t.Errorf("snapshot event %d carries an id: %q", i, e)
+		}
+	}
+	if !strings.HasPrefix(events[5], "id: ") || !strings.HasPrefix(events[6], "id: ") {
+		t.Fatalf("the last snapshot event and the live event must carry ids:\n%s", body)
+	}
+	if !strings.Contains(events[6], `"key":"file:`+r1+`/title"`) {
+		t.Fatalf("live tail did not follow the snapshot: %q", events[6])
+	}
+}
+
+// A cursor older than the last cache rebuild gets `gap` (reason rebuild): the
+// rebuild published no per-key events, so the consumer must re-snapshot.
+func TestMetaEventsGapAfterRebuild(t *testing.T) {
+	stor := newTestStorage(t)
+	s := &Server{storage: stor}
+	storeClaims(t, stor, signed(t, testSigner(t, 1), "bafyA", "title", "T", 1))
+	if err := stor.Rebuild(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
+	req.Header.Set("Last-Event-ID", "1000-0")
+	rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()}
+	done := make(chan struct{})
+	go func() { s.handleEventsMeta(rec, req); close(done) }()
+	deadline := time.Now().Add(5 * time.Second)
+	for !strings.Contains(rec.body(), "event: gap") {
+		if time.Now().After(deadline) {
+			t.Fatalf("no gap after rebuild:\n%s", rec.body())
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	cancel()
+	<-done
+	if !strings.Contains(rec.body(), `"reason":"rebuild"`) {
+		t.Fatalf("gap reason: %s", rec.body())
+	}
+}

internal/api/stats.go

@@ -0,0 +1,166 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/metazla/meta-core/internal/identity"
+)
+
+// statsSweepTTL is how long a keyspace-walk result stays good. The walk is
+// never on the request path (see statsCache), so this only controls how stale
+// the library tiles may look, not how slow the endpoint is.
+const statsSweepTTL = 60 * time.Second
+
+// fileStatsJSON is the file-backed half of the library: the roots that carry
+// filePath+sizeByte+mtimeNano. Gateway / cid-rooted records have no file on
+// disk and are counted in Records, not here — the gap between the two numbers
+// is exactly "metadata we hold for files we do not have".
+type fileStatsJSON struct {
+	Count     int   `json:"count"`
+	TotalSize int64 `json:"totalSize"`
+}
+
+// StatsResponse is the body of GET /api/stats — every headline number the
+// dashboard shows, in one request.
+//
+// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
+// cheap (an index range count, a cached row count, a file stat) plus a
+// directory listing, recomputed on every call. The `redis*` JSON names
+// predate the move to SQLite and are kept for dashboard compatibility:
+// redisKeys is the number of claim rows (truth), redisMemory both databases' on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
+// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
+// from a background-refreshed cache and are null until the first sweep lands.
+type StatsResponse struct {
+	Records     int64          `json:"records"`
+	RedisKeys   int64          `json:"redisKeys"`
+	RedisMemory string         `json:"redisMemory"`
+	Identities  int            `json:"identities"`
+	Files       *fileStatsJSON `json:"files"`
+	UDLUsers    *int           `json:"udlUsers"`
+	// SweptAt is when the cached half was computed (unix ms), 0 if never.
+	SweptAt int64 `json:"sweptAt"`
+	// Sweeping reports that a refresh is in flight, so the UI can show the
+	// cached numbers as provisional instead of pretending they are live.
+	Sweeping bool `json:"sweeping"`
+}
+
+// statsCache holds the expensive half of /api/stats.
+//
+// Why a cache and not just "compute it": the dashboard polls. A multi-second store walk
+// per tick, per open tab, is a self-inflicted outage on a box that is also
+// serving playback — the same shape as meta-sort's /api/stats full sweep. So
+// the handler NEVER waits on a walk: it returns whatever the last sweep
+// produced and kicks off a new one if the old one is stale. `running` makes
+// that single-flight, so ten tabs cost one walk.
+type statsCache struct {
+	mu       sync.Mutex
+	files    *fileStatsJSON
+	udlUsers *int
+	sweptAt  time.Time
+	running  bool
+}
+
+// snapshot returns the cached values and whether a refresh is in flight.
+func (c *statsCache) snapshot() (*fileStatsJSON, *int, time.Time, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.files, c.udlUsers, c.sweptAt, c.running
+}
+
+// refreshIfStale starts a background sweep unless one is already running or
+// the last result is still fresh. Returns true if it started one.
+func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
+	c.mu.Lock()
+	if c.running || time.Since(c.sweptAt) < statsSweepTTL {
+		c.mu.Unlock()
+		return false
+	}
+	c.running = true
+	c.mu.Unlock()
+
+	go func() {
+		files, udl := sweep()
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.running = false
+		// A failed sweep leaves the previous numbers in place rather than
+		// blanking the tiles; sweptAt still advances so a hard-failing walk
+		// (store down) does not spin once per poll.
+		c.sweptAt = time.Now()
+		if files != nil {
+			c.files = files
+		}
+		if udl != nil {
+			c.udlUsers = udl
+		}
+	}()
+	return true
+}
+
+// handleStats handles GET /api/stats.
+func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	resp := StatsResponse{}
+
+	if n, err := s.storage.CountRoots(); err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	} else {
+		resp.Records = n
+	}
+	if n, err := s.storage.CountClaims(); err == nil {
+		resp.RedisKeys = n
+	}
+	if mem, err := s.storage.GetMemoryInfo(); err == nil {
+		resp.RedisMemory = mem
+	} else {
+		resp.RedisMemory = "N/A"
+	}
+	if list, err := identity.List(s.config.IdentityAccountsDir()); err == nil {
+		resp.Identities = len(list)
+	}
+
+	s.stats.refreshIfStale(s.sweepStats)
+	files, udl, sweptAt, sweeping := s.stats.snapshot()
+	resp.Files = files
+	resp.UDLUsers = udl
+	resp.Sweeping = sweeping
+	if !sweptAt.IsZero() {
+		resp.SweptAt = sweptAt.UnixMilli()
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// sweepStats runs the two keyspace walks behind the cached tiles. Called off
+// the request path only. A failing half returns nil so the other half still
+// lands.
+func (s *Server) sweepStats() (*fileStatsJSON, *int) {
+	var files *fileStatsJSON
+	if tuples, err := s.storage.GetTuplesForAllFiles(); err != nil {
+		log.Printf("[API] stats: file tuple sweep failed: %v", err)
+	} else {
+		fs := fileStatsJSON{Count: len(tuples)}
+		for _, t := range tuples {
+			fs.TotalSize += t.Size
+		}
+		files = &fs
+	}
+
+	var udl *int
+	if stats, err := s.storage.UDLAllUserStats(); err != nil {
+		log.Printf("[API] stats: UDL user sweep failed: %v", err)
+	} else {
+		n := len(stats)
+		udl = &n
+	}
+
+	return files, udl
+}

internal/api/storage_helper_test.go

@@ -0,0 +1,68 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newTestStorage opens (and starts) a fresh store in the test's temp dir.
+func newTestStorage(t *testing.T) *storage.Client {
+	t.Helper()
+	dir := t.TempDir()
+	c := storage.NewClient()
+	if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
+		t.Fatalf("schema: %v", err)
+	}
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return c
+}
+
+func testSigner(t *testing.T, n byte) *identity.Identity {
+	t.Helper()
+	id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+// signed builds a claim signed by id; v is string, []byte or nil.
+func signed(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim {
+	t.Helper()
+	cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts}
+	switch x := v.(type) {
+	case string:
+		cl.Value = claim.Text(x)
+	case []byte:
+		cl.Value = claim.Bytes(x)
+	}
+	if err := cl.Sign(id); err != nil {
+		t.Fatal(err)
+	}
+	return cl
+}
+
+func storeClaims(t *testing.T, c *storage.Client, claims ...claim.Claim) {
+	t.Helper()
+	res, err := c.WriteClaims(context.Background(), claims)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, r := range res {
+		if r.Status != storage.StatusAccepted {
+			t.Fatalf("claim %d: %s %s", i, r.Status, r.Reason)
+		}
+	}
+}

internal/api/udl.go

@@ -1,97 +1,63 @@
 package api
 
-// User Data Layer (UDL) HTTP surface.
+// User Data Layer facade (/api/udl/*). Since meta-core v2.0.0 a user's data are
+// ordinary MMUR2 claims signed by the user's key (plaintext for public-social
+// keys, ciphertext for private ones); these routes keep their paths for the
+// clients (meta-watch, meta-read, meta-listen, the gateway mirror) and speak
+// claims in the §3.4 JSON form:
 //
-// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
+//	{"cid","uid","key","value": "…" | {"b64": "…"} | null,"ts","sig"}
-// profile) now that the authoritative store lives in meta-core instead of
+//
-// meta-watch's local op-log. Records arrive already signed (and, for
+// A write is the same as POST /api/claims; newest ts wins per (uid, cid, key).
-// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
-// version-gates and indexes them but never inspects the crypto. See
-// internal/storage/udl.go for the Redis model.
 
 import (
 	"encoding/json"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// UDLPutRequest is the body of PUT /api/udl/record.
+// UDLPutResponse answers PUT /api/udl/record.
-type UDLPutRequest struct {
-	Uid     string           `json:"uid"`
-	Cid     string           `json:"cid"`
-	Key     string           `json:"key"`
-	Version int64            `json:"version"`
-	Ts      int64            `json:"ts"`
-	Record  string           `json:"record"` // base64 CBOR, opaque + signed
-	Value   *json.RawMessage `json:"value,omitempty"`
-	// Tombstone marks this write as a deletion of (uid, cid, key). The record
-	// itself is opaque here — meta-core never decodes the CBOR, and a private-
-	// tier value and a tombstone are equally "no plaintext value" — so the
-	// writer has to say. Without it the cid is never removed from
-	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
-	// account and is returned in full on every My List / Continue Watching
-	// read. Omitted by older writers, which keeps the previous behaviour.
-	Tombstone bool `json:"tombstone,omitempty"`
-}
-
-// UDLPutResponse is the response of PUT /api/udl/record.
 type UDLPutResponse struct {
-	Accepted bool `json:"accepted"`
+	Accepted bool                `json:"accepted"`
+	Status   storage.ClaimStatus `json:"status"`
 }
 
-// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
+// UDLPutBatchRequest is PUT /api/udl/records.
-// request. Each entry is exactly a UDLPutRequest, so a client can build the
-// same signed record for either endpoint.
 type UDLPutBatchRequest struct {
-	Records []UDLPutRequest `json:"records"`
+	Records []claim.Claim `json:"records"`
 }
 
-// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
+// UDLPutBatchResult is one entry of the batch response.
-// means the write was stale (the single-record endpoint's 409), which in a
-// batch is an ordinary outcome rather than a failure.
 type UDLPutBatchResult struct {
-	Key      string `json:"key"`
+	Key      string              `json:"key"`
-	Accepted bool   `json:"accepted"`
+	Accepted bool                `json:"accepted"`
-	Error    string `json:"error,omitempty"`
+	Status   storage.ClaimStatus `json:"status"`
+	Error    string              `json:"error,omitempty"`
 }
 
-// UDLPutBatchResponse mirrors the request order index-for-index.
+// UDLPutBatchResponse answers PUT /api/udl/records.
 type UDLPutBatchResponse struct {
 	Results []UDLPutBatchResult `json:"results"`
 }
 
-// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
-// request is held open; sized to match the sign batch on the other side of the
-// same operation (see maxSignBatch).
 const maxUDLBatch = 1000
 
-// UDLRecordResponse is the response of GET /api/udl/record.
+// UDLRecordResponse answers GET /api/udl/record. Claim is the stored claim,
+// tombstone included (its ts is what a newer write must exceed); Exists is
+// false when it is absent or a tombstone.
 type UDLRecordResponse struct {
-	Record  string `json:"record"`
+	Exists bool         `json:"exists"`
-	Version int64  `json:"version"`
+	Claim  *claim.Claim `json:"claim"`
-	Ts      int64  `json:"ts"`
-	Exists  bool   `json:"exists"`
-	// Value is the plaintext JSON scalar stored beside the opaque record, and
-	// is present only for PUBLIC-tier keys (like, rating, profile:name,
-	// profile:avatar). json.RawMessage rather than string because the field is
-	// generic over keys — a bool for `like`, a number for `rating`, a string
-	// for a profile name (see aggregatePublicValues) — and because it is the
-	// exact type UDLPutRequest.Value uses, so a value written through PUT reads
-	// back byte-identical.
-	//
-	// Absent, never null, when there is no plaintext: private-tier cells,
-	// tombstones and never-written cells must not read as an empty public
-	// value. Readers key off presence, not `exists`.
-	Value *json.RawMessage `json:"value,omitempty"`
 }
 
-// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
+// UDLAggregate is one key's public aggregate on a cid.
-// field names; `avg` is null for boolean keys like `like`), so the meta-watch
-// client deserializes it into the same struct it serializes to the UI.
 type UDLAggregate struct {
 	Key        string   `json:"key"`
 	Count      int      `json:"count"`
@@ -100,10 +66,17 @@ type UDLAggregate struct {
 	SampleSize int      `json:"sample_size"`
 }
 
-// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
+func (s *Server) udlReady(w http.ResponseWriter) bool {
-func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return false
+	}
+	return true
+}
+
+// handleUDLRecordGet handles GET /api/udl/record?uid=&cid=&key=.
+func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+	if !s.udlReady(w) {
 		return
 	}
 	q := r.URL.Query()
@@ -112,85 +85,53 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "uid, cid and key are required")
 		return
 	}
-
+	cl, ok, err := s.storage.UDLGetClaim(uid, cid, key)
-	cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	resp := UDLRecordResponse{
+	resp := UDLRecordResponse{}
-		Record:  cell.Record,
+	if ok {
-		Version: cell.Version,
+		resp.Claim = &cl
-		Ts:      cell.Ts,
+		resp.Exists = !cl.Value.IsNull()
-		Exists:  exists,
-	}
-	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
-	// Redis, a future writer bug) would otherwise fail mid-encode and emit a
-	// truncated body instead of a readable error.
-	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
-		raw := json.RawMessage(cell.Value)
-		resp.Value = &raw
 	}
 	writeJSON(w, http.StatusOK, resp)
 }
 
-// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
+// handleUDLRecordPut handles PUT /api/udl/record with one claim as the body.
-// (version <= stored) returns 409 stale_version so the client can re-read and
-// retry with a higher version.
 func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
-	var req UDLPutRequest
+	var cl claim.Claim
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+	if err := json.NewDecoder(r.Body).Decode(&cl); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		writeError(w, http.StatusBadRequest, "body must be one claim: "+err.Error())
 		return
 	}
-	if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
+	results, err := s.storage.WriteClaims(r.Context(), []claim.Claim{cl})
-		writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
-		return
-	}
-
-	publicValue := ""
-	hasValue := false
-	if req.Value != nil {
-		publicValue = string(*req.Value)
-		hasValue = true
-	}
-
-	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	if !accepted {
+	res := results[0]
-		writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
+	switch res.Status {
-		return
+	case storage.StatusAccepted, storage.StatusUnchanged:
+		writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true, Status: res.Status})
+	case storage.StatusStale:
+		writeErrorSlug(w, http.StatusConflict, "stale_version", "a claim with an equal or newer ts already exists", true)
+	default:
+		writeErrorSlug(w, http.StatusBadRequest, string(res.Status), res.Reason, false)
 	}
-	writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
 }
 
-// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
+// handleUDLRecordsPut handles PUT /api/udl/records {records: [claim, …]}.
-// handleUDLRecordPut.
-//
-// Exists for the bulk verbs above this layer ("mark this whole series seen"),
-// where one user action produces one cell per episode. Each cell still goes
-// through the same version-gated Lua upsert, so the CRDT semantics are
-// identical — this batches the transport only.
-//
-// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
-// replaces the single-record conflict status: in a batch, one stale cell is a
-// normal outcome (another device already marked that episode) and must not fail
-// the 299 cells beside it. Callers read the array, not the status code.
 func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
 	var req UDLPutBatchRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
 		return
 	}
 	if len(req.Records) == 0 {
@@ -201,80 +142,115 @@ func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "too many records in one batch")
 		return
 	}
-	results := make([]UDLPutBatchResult, 0, len(req.Records))
+	results, err := s.storage.WriteClaims(r.Context(), req.Records)
-	for _, rec := range req.Records {
+	if err != nil {
-		if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
+		writeError(w, http.StatusInternalServerError, err.Error())
-			results = append(results, UDLPutBatchResult{
+		return
-				Key:   rec.Key,
-				Error: "uid, cid, key and record are required",
-			})
-			continue
-		}
-		publicValue := ""
-		hasValue := false
-		if rec.Value != nil {
-			publicValue = string(*rec.Value)
-			hasValue = true
-		}
-		accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
-		if err != nil {
-			// A storage error is per-record here rather than fatal for the whole
-			// batch: the cells already written stay written, and reporting which
-			// ones failed lets the caller retry exactly those.
-			results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
-			continue
-		}
-		results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
 	}
-	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
+	out := make([]UDLPutBatchResult, len(results))
+	for i, res := range results {
+		out[i] = UDLPutBatchResult{
+			Key:      req.Records[i].Key,
+			Status:   res.Status,
+			Accepted: res.Status == storage.StatusAccepted || res.Status == storage.StatusUnchanged,
+			Error:    res.Reason,
+		}
+	}
+	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: out})
 }
 
-// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
+func (s *Server) writeUDLEntries(w http.ResponseWriter, uid, cid, key string) {
-// (active_for_user_key — My List, Continue Watching).
+	entries, err := s.storage.UDLClaimsForUser(uid, cid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}: every live claim
+// of uid on key (My List, Continue Watching).
 func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
 	vars := mux.Vars(r)
-	uid, key := vars["uid"], vars["key"]
+	if vars["uid"] == "" || vars["key"] == "" {
-	if uid == "" || key == "" {
 		writeError(w, http.StatusBadRequest, "uid and key are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserKey(uid, key)
+	s.writeUDLEntries(w, vars["uid"], "", vars["key"])
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
 }
 
-// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
+// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}.
-// (list_for_user_cid).
 func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
 	vars := mux.Vars(r)
-	uid, cid := vars["uid"], vars["cid"]
+	if vars["uid"] == "" || vars["cid"] == "" {
-	if uid == "" || cid == "" {
 		writeError(w, http.StatusBadRequest, "uid and cid are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserCid(uid, cid)
+	s.writeUDLEntries(w, vars["uid"], vars["cid"], "")
+}
+
+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= — the
+// whole profile, paged on cid boundaries (recovery / export).
+func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
+	if !s.udlReady(w) {
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	limit := maxUDLBatch
+	if v := r.URL.Query().Get("limit"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n <= 0 {
+			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
+			return
+		}
+		limit = min(n, limit)
+	}
+	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries, "nextCid": nextCid})
 }
 
-// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count.
+func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
+	if !s.udlReady(w) {
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	n, err := s.storage.UDLCountUserCids(uid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
+}
+
+// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users: signers holding live
+// claims on the cid.
 func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -290,12 +266,9 @@ func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
 }
 
-// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
+// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,userRating.
-// Computes per-key population stats from the stored public value fields
-// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
 func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
+	if !s.udlReady(w) {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -305,7 +278,7 @@ func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
 	}
 	keysParam := r.URL.Query().Get("keys")
 	if keysParam == "" {
-		keysParam = "like,rating"
+		keysParam = "like,userRating"
 	}
 	aggregates := make([]UDLAggregate, 0)
 	for _, key := range strings.Split(keysParam, ",") {

internal/api/udl_test.go

@@ -7,220 +7,106 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/claim"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
-// newUDLTestServer returns a Server wired to an in-memory Redis.
-//
-// The struct is built directly rather than through NewServer: NewServer boots
-// the file watcher, mounts manager, watchers poller and WebDAV handler, none of
-// which the UDL handlers touch — they use s.storage and nothing else.
 func newUDLTestServer(t *testing.T) *Server {
 	t.Helper()
-
+	return &Server{storage: newTestStorage(t)}
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
-	return &Server{storage: c}
 }
 
-// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
+func udlGet(t *testing.T, s *Server, uid, cid, key string) UDLRecordResponse {
-// fields, so a test can assert on the *encoding* of `value` (is it a quoted
-// string? is the key there at all?) rather than on what a typed struct would
-// have coerced it into.
-func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) {
 	t.Helper()
-
 	req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
 	rr := httptest.NewRecorder()
 	s.handleUDLRecordGet(rr, req)
-
+	if rr.Code != http.StatusOK {
-	var body map[string]json.RawMessage
+		t.Fatalf("GET record = %d %s", rr.Code, rr.Body)
-	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-		t.Fatalf("decode response %q: %v", rr.Body.String(), err)
 	}
-	return rr.Code, body
+	var resp UDLRecordResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	return resp
 }
 
-// The plaintext value of a PUBLIC-tier cell must come back over the wire, and
+func udlPutBatch(t *testing.T, s *Server, claims ...claim.Claim) UDLPutBatchResponse {
-// come back *as it was written* — a JSON string stays a quoted string, a bool
-// stays a bool.
-//
-// Both halves matter. Without the value at all, the meta-core dashboard can
-// only ever show 44-character base58 uids: display names live in the User Data
-// Layer as the public cell (uid, "self", "profile:name"), and the plaintext
-// sits in Redis where the storage layer has always read it — the handler just
-// used to drop it. And the type has to stay json.RawMessage: this endpoint is
-// generic over keys, so a `*string` would mangle `like` (bool) and `rating`
-// (number) while looking correct for names.
-func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	const uid = "zAlice"
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok {
-		t.Fatalf("seed name: ok=%v err=%v", ok, err)
-	}
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok {
-		t.Fatalf("seed like: ok=%v err=%v", ok, err)
-	}
-
-	code, body := getRecord(t, s, uid, "self", "profile:name")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if got := string(body["value"]); got != `"Ada Lovelace"` {
-		t.Fatalf("value = %s, want a quoted JSON string", got)
-	}
-	var name string
-	if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" {
-		t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err)
-	}
-
-	// A boolean key survives unquoted — the assertion a *string field fails.
-	if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" {
-		t.Fatalf("like value = %s, want the bare JSON literal true", body["value"])
-	}
-}
-
-// A cell with no plaintext must omit `value` entirely rather than send null.
-//
-// That is the shape of every private-tier key (seek, watched, card, localpath —
-// their real value rides encrypted inside the opaque record) and of every
-// tombstone. A reader keys off presence: "there is no plaintext here" and "the
-// plaintext is null" are different facts, and a name cleared by its owner
-// leaves the cell existing with no value, which must not read as a name.
-func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	const uid = "zBob"
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok {
-		t.Fatalf("seed private cell: ok=%v err=%v", ok, err)
-	}
-
-	code, body := getRecord(t, s, uid, "bagcsaaa", "seek")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if _, ok := body["value"]; ok {
-		t.Fatalf("private cell must omit `value`, got %s", body["value"])
-	}
-	if string(body["exists"]) != "true" {
-		t.Fatalf("exists = %s, want true", body["exists"])
-	}
-}
-
-// A cell that was never written answers 200 with exists:false and no value —
-// not a 404, and not an empty-string value that would render as a blank name.
-func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	code, body := getRecord(t, s, "zNobody", "self", "profile:name")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if string(body["exists"]) != "false" {
-		t.Fatalf("exists = %s, want false", body["exists"])
-	}
-	if _, ok := body["value"]; ok {
-		t.Fatalf("missing cell must omit `value`, got %s", body["value"])
-	}
-}
-
-// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the
-// per-record results.
-func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) {
 	t.Helper()
-
+	body, _ := json.Marshal(UDLPutBatchRequest{Records: claims})
-	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body))
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(string(body)))
 	rr := httptest.NewRecorder()
 	s.handleUDLRecordsPut(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("PUT records = %d %s", rr.Code, rr.Body)
+	}
+	var resp UDLPutBatchResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	return resp
+}
 
-	var out UDLPutBatchResponse
+// The facade stores and returns MMUR2 claims: public values in clear,
-	if rr.Code == http.StatusOK {
+// private ones as ciphertext, tombstones with their ts.
-		if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
+func TestUDLFacadeRoundTrip(t *testing.T) {
-			t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+	s := newUDLTestServer(t)
+	me := testSigner(t, 7)
+	res := udlPutBatch(t, s,
+		signed(t, me, "self", "profile:name", "Ada Lovelace", 100),
+		signed(t, me, "bagcsaaa", "like", "true", 100),
+		signed(t, me, "bagcsaaa", "seek", []byte("ciphertext"), 100),
+	)
+	for _, r := range res.Results {
+		if !r.Accepted {
+			t.Fatalf("not accepted: %+v", r)
 		}
 	}
-	return rr.Code, out
+	got := udlGet(t, s, me.UID, "self", "profile:name")
+	if !got.Exists || got.Claim == nil || got.Claim.Value.Text != "Ada Lovelace" || got.Claim.Verify() != nil {
+		t.Fatalf("profile = %+v", got)
+	}
+	if seek := udlGet(t, s, me.UID, "bagcsaaa", "seek"); seek.Claim == nil || seek.Claim.Value.Kind != claim.KindBytes {
+		t.Fatalf("seek = %+v", seek)
+	}
+	if miss := udlGet(t, s, me.UID, "bagcsaaa", "nope"); miss.Exists || miss.Claim != nil {
+		t.Fatalf("missing cell = %+v", miss)
+	}
+
+	// Retraction: stored, returned with its ts, no longer "exists".
+	udlPutBatch(t, s, signed(t, me, "bagcsaaa", "like", nil, 200))
+	if tomb := udlGet(t, s, me.UID, "bagcsaaa", "like"); tomb.Exists || tomb.Claim == nil || tomb.Claim.TS != 200 {
+		t.Fatalf("tombstone = %+v", tomb)
+	}
+
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/api/udl/cid/self/aggregate?keys=profile:name", nil)
+	req = withVars(req, map[string]string{"cid": "self"})
+	s.handleUDLAggregate(rr, req)
+	if !strings.Contains(rr.Body.String(), `"sample_size":1`) {
+		t.Fatalf("aggregate = %s", rr.Body)
+	}
 }
 
-// **The load-bearing property of the batch endpoint: one stale cell does not
+// A stale claim in a batch is reported in its slot without failing the rest.
-// fail the batch.**
+func TestUDLRecordsPut_StaleSlotDoesNotFailTheBatch(t *testing.T) {
-//
-// The single-record PUT answers a stale version with 409, which is right when
-// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark
-// this whole series seen") routinely include episodes another device already
-// marked, and 409-ing there would abandon every cell after it. So staleness is
-// reported per record as accepted:false, with 200 on the request itself.
-func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) {
 	s := newUDLTestServer(t)
-
+	me := testSigner(t, 7)
-	const uid = "zCarol"
+	udlPutBatch(t, s, signed(t, me, "bagcsaaa", "seen:s01e01", "true", 400))
-	const cid = "bagdsaaaseries"
+	res := udlPutBatch(t, s,
-	// Seed s01e01 at version 4 — the batch below tries to write it at 2.
+		signed(t, me, "bagcsaaa", "seen:s01e01", "false", 300),
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok {
+		signed(t, me, "bagcsaaa", "seen:s01e02", "true", 300),
-		t.Fatalf("seed: ok=%v err=%v", ok, err)
+	)
-	}
+	if res.Results[0].Accepted || res.Results[0].Status != "stale" || !res.Results[1].Accepted {
-
+		t.Fatalf("results = %+v", res.Results)
-	code, out := putRecords(t, s, `{"records":[
-		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"},
-		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"}
-	]}`)
-
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code)
-	}
-	if len(out.Results) != 2 {
-		t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results))
-	}
-	if out.Results[0].Accepted || out.Results[0].Error != "" {
-		t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0])
-	}
-	if !out.Results[1].Accepted {
-		t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1])
-	}
-
-	// The fresh cell really landed, and the stale one did not overwrite.
-	if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" {
-		t.Fatalf("s01e02 was not written")
-	}
-	if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" {
-		t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"])
 	}
 }
 
-// A malformed record is reported in its own slot rather than rejecting the
-// request, for the same reason: the caller retries that cell, not the batch.
-func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	code, out := putRecords(t, s, `{"records":[
-		{"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"},
-		{"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"}
-	]}`)
-
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if out.Results[0].Error == "" {
-		t.Fatalf("record with no key must report an error, got %+v", out.Results[0])
-	}
-	if !out.Results[1].Accepted {
-		t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1])
-	}
-}
-
-// An empty batch is a client bug, not a no-op: it almost always means the
-// caller built its record list wrong, and answering 200 with an empty array
-// would hide that.
 func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
 	s := newUDLTestServer(t)
-
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(`{"records":[]}`))
-	if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest {
+	rr := httptest.NewRecorder()
-		t.Fatalf("status = %d, want 400", code)
+	s.handleUDLRecordsPut(rr, req)
+	if rr.Code != http.StatusBadRequest {
+		t.Fatalf("empty batch = %d", rr.Code)
 	}
 }

internal/cid/cid-rank-vectors.json

@@ -128,6 +128,18 @@
       "rank": 5,
       "note": "codec 0x1009 + IDENTITY multihash 0x00, digest = the verbatim UTF-8 URL. WIRE-IDENTICAL to url (0x1006) apart from the codec, and that difference is the entire contract: 0x1006 means 'fetch these bytes once and seed them', 0x1009 means 'open this in a browser, there are no bytes for us'. A decoder that matched on SHAPE rather than on the codec slot would see an identity multihash wrapping an http(s) URL, take the 0x1006 path, fetch a watch/Bandcamp page, chunk it, seed the HTML and link that content CID onto the record AS THE FILE — the MSR1 poisoning shape, except permanent and replicated. Same tier/reasoning as the rest of the locator family."
     },
+    {
+      "name": "provider-file-locator",
+      "cid": "bagfcaaa2bvxxazloon2we5djorwgk43gnfwgkorxga3dcobtgq",
+      "rank": 5,
+      "note": "codec 0x100A + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id - here (\"opensubtitles\",\"file:7061834\"). The GENERIC fetchable locator: one codec for every provider whose bytes are fetched the same way; the provider is inside the digest. Framing byte-identical to card (0x1007), but the byte path RESOLVES it (via source's fetch tier) instead of 404ing. Same codec-slot rule as the rest of the family: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5: once the fetched bytes' sha2-256 is linked onto the record it wins the canonical election with no addressing change."
+    },
+    {
+      "name": "provider-file-tachiyomi",
+      "cid": "bagfcaabebf2gcy3inf4w63ljnvwhu332gnvw423po42hg33ug4zdeytdonrtk4bsnq",
+      "rank": 5,
+      "note": "provider-file (0x100A) under the second registered source, (\"tachiyomi\", base32lower(sha256(\"https://www.mangakakalot.gg/manga/ragna-crimson/chapter-94\"))[..26] = \"mlzoz3knkow4sot722bcsc5p2l\"). A comic chapter, minted by meta-feeder-suwayomi from the chapter's address ON ITS SITE - never the Suwayomi host/port/row ids - so every box mints this same cid; redeemed into one CBZ. Same rank as any provider-file (rank ignores source); the vector proves the digest bytes (cid-formats.md section 8.3)."
+    },
     {
       "name": "sha1",
       "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",

internal/cid/rank.go

@@ -75,6 +75,14 @@ const (
 	CodeYtVideo = 0x1008 // custom — identity-multihash CID wrapping a YouTube (kind, id)
 	CodeExtPlay = 0x1009 // custom — identity-multihash CID wrapping a URL to OPEN, never fetch
 
+	// CodeProviderFile is the FETCHABLE twin of CodeExtPlay: a provider-held
+	// file addressed as varint(len(source))‖source‖id — ("opensubtitles",
+	// "file:7061834"). Framing is byte-identical to CodeCard; only the byte
+	// path differs (it redeems the id through that source's feeder instead of
+	// 404ing). Same codec-slot rule and same tier: once the fetched bytes'
+	// sha2-256 is linked onto the record it wins with no addressing change.
+	CodeProviderFile = 0x100A // custom — identity-multihash CID wrapping a (source, id) provider file
+
 	// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
 	// REAL sha2-256 digest (over the normalised, sorted article Message-ID
 	// set of a Usenet posting we scanned ourselves — never a locator, no
@@ -183,7 +191,7 @@ func Rank(cidStr string) int {
 	// Opaque locators. Codec-only: their multihash is identity (0x00), so
 	// there is nothing to match on the mh side.
 	if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard ||
-		codec == CodeYtVideo || codec == CodeExtPlay {
+		codec == CodeYtVideo || codec == CodeExtPlay || codec == CodeProviderFile {
 		return RankLocator
 	}
 	// nzb-posting: codec-only, checked BEFORE the mh switch below — its

internal/claim/claim.go

@@ -0,0 +1,240 @@
+// Package claim is the MMUR2 signed claim: signer uid says key = value about
+// content cid, at ts (milliseconds, also the version).
+//
+// Wire spec (load-bearing for every implementation — Go here, Rust in
+// meta-feeder-sdk's `claims` feature and the clients' record/ mirror, TS in
+// the plugins' meta-core-client.ts; all pinned by testdata/claim-vectors.json):
+//
+//	domain        = "MMUR2\n"
+//	payload       = canonical CBOR array [cid, uid, key, value, ts]
+//	bytes_to_sign = domain || payload
+//	sig           = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S
+//
+// value is CBOR text (public), bytes (private envelope) or null (retraction).
+// Canonical = definite lengths, smallest integer heads — the encoder below is
+// hand-rolled because those five shapes are all MMUR2 ever needs.
+package claim
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/decred/dcrd/dcrec/secp256k1/v4"
+
+	"github.com/metazla/meta-core/internal/identity"
+)
+
+// Domain separates MMUR2 bytes from every other signed payload (MMUR1 records,
+// identity challenges). Changing it invalidates every claim everywhere.
+const Domain = "MMUR2\n"
+
+// Kind of a claim value. The SQLite storage type mirrors it: TEXT, BLOB, NULL.
+type Kind uint8
+
+const (
+	KindNull Kind = iota
+	KindText
+	KindBytes
+)
+
+// Value is a claim's value. The zero Value is a retraction (null).
+type Value struct {
+	Kind  Kind
+	Text  string
+	Bytes []byte
+}
+
+func Text(s string) Value    { return Value{Kind: KindText, Text: s} }
+func Bytes(b []byte) Value   { return Value{Kind: KindBytes, Bytes: b} }
+func Null() Value            { return Value{} }
+func (v Value) IsNull() bool { return v.Kind == KindNull }
+
+// Equal reports byte-for-byte equality, kind included.
+func (v Value) Equal(o Value) bool {
+	if v.Kind != o.Kind {
+		return false
+	}
+	switch v.Kind {
+	case KindText:
+		return v.Text == o.Text
+	case KindBytes:
+		return string(v.Bytes) == string(o.Bytes)
+	}
+	return true
+}
+
+// Claim is one signed vote.
+type Claim struct {
+	CID   string
+	UID   string
+	Key   string
+	Value Value
+	TS    uint64
+	Sig   []byte // 64-byte compact r||s
+}
+
+// SignedBytes returns the exact bytes the signature covers.
+func SignedBytes(cid, uid, key string, v Value, ts uint64) []byte {
+	b := make([]byte, 0, len(Domain)+len(cid)+len(uid)+len(key)+len(v.Text)+len(v.Bytes)+32)
+	b = append(b, Domain...)
+	b = appendHead(b, 4, 5) // array(5)
+	b = appendText(b, cid)
+	b = appendText(b, uid)
+	b = appendText(b, key)
+	switch v.Kind {
+	case KindText:
+		b = appendText(b, v.Text)
+	case KindBytes:
+		b = appendHead(b, 2, uint64(len(v.Bytes)))
+		b = append(b, v.Bytes...)
+	default:
+		b = append(b, 0xf6) // null
+	}
+	return appendHead(b, 0, ts)
+}
+
+// SignedBytes of c.
+func (c *Claim) SignedBytes() []byte {
+	return SignedBytes(c.CID, c.UID, c.Key, c.Value, c.TS)
+}
+
+func appendText(b []byte, s string) []byte {
+	b = appendHead(b, 3, uint64(len(s)))
+	return append(b, s...)
+}
+
+// appendHead writes a CBOR major type + argument in its smallest form.
+func appendHead(b []byte, major byte, n uint64) []byte {
+	m := major << 5
+	switch {
+	case n < 24:
+		return append(b, m|byte(n))
+	case n <= 0xff:
+		return append(b, m|24, byte(n))
+	case n <= 0xffff:
+		return append(b, m|25, byte(n>>8), byte(n))
+	case n <= 0xffffffff:
+		return append(b, m|26, byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	default:
+		return append(b, m|27, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32),
+			byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	}
+}
+
+// ErrBadSignature is returned for a signature that is malformed, high-S, or
+// does not verify against the uid.
+var ErrBadSignature = errors.New("bad signature")
+
+// IsLowS reports whether a compact signature's S is in the lower half.
+func IsLowS(sig []byte) bool {
+	if len(sig) != 64 {
+		return false
+	}
+	var s secp256k1.ModNScalar
+	if overflow := s.SetByteSlice(sig[32:]); overflow {
+		return false
+	}
+	return !s.IsOverHalfOrder()
+}
+
+// Verify checks the claim's shape and signature.
+func (c *Claim) Verify() error {
+	if c.CID == "" || c.Key == "" || !identity.ValidUID(c.UID) {
+		return fmt.Errorf("%w: missing cid, key or valid uid", ErrBadSignature)
+	}
+	if len(c.Sig) != 64 || !IsLowS(c.Sig) {
+		return fmt.Errorf("%w: want a 64-byte low-S compact signature", ErrBadSignature)
+	}
+	if err := identity.Verify(c.UID, c.SignedBytes(), c.Sig); err != nil {
+		return fmt.Errorf("%w: %v", ErrBadSignature, err)
+	}
+	return nil
+}
+
+// Sign fills c.Sig with id's compact low-S signature. Used by the account-key
+// service (users who sign through meta-core until its separate ticket removes
+// it), tests and the one-off migration tool — never by the claims engine.
+func (c *Claim) Sign(id *identity.Identity) error {
+	if id.UID != c.UID {
+		return fmt.Errorf("claim uid %s is not the signing identity %s", c.UID, id.UID)
+	}
+	sig, err := identity.SignCompact(id, c.SignedBytes())
+	if err != nil {
+		return err
+	}
+	c.Sig = sig
+	return nil
+}
+
+// jsonClaim is the §3.4 HTTP form.
+type jsonClaim struct {
+	CID   string          `json:"cid"`
+	UID   string          `json:"uid"`
+	Key   string          `json:"key"`
+	Value json.RawMessage `json:"value"`
+	TS    uint64          `json:"ts"`
+	Sig   string          `json:"sig"`
+}
+
+type jsonBytes struct {
+	B64 string `json:"b64"`
+}
+
+// MarshalJSON emits {"cid","uid","key","value","ts","sig"}; bytes values as
+// {"b64": "..."}, retractions as null.
+func (c Claim) MarshalJSON() ([]byte, error) {
+	var v json.RawMessage
+	var err error
+	switch c.Value.Kind {
+	case KindText:
+		v, err = json.Marshal(c.Value.Text)
+	case KindBytes:
+		v, err = json.Marshal(jsonBytes{B64: base64.StdEncoding.EncodeToString(c.Value.Bytes)})
+	default:
+		v = json.RawMessage("null")
+	}
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(jsonClaim{
+		CID: c.CID, UID: c.UID, Key: c.Key, Value: v, TS: c.TS,
+		Sig: base64.StdEncoding.EncodeToString(c.Sig),
+	})
+}
+
+// UnmarshalJSON parses the §3.4 form.
+func (c *Claim) UnmarshalJSON(data []byte) error {
+	var j jsonClaim
+	if err := json.Unmarshal(data, &j); err != nil {
+		return err
+	}
+	sig, err := base64.StdEncoding.DecodeString(j.Sig)
+	if err != nil {
+		return fmt.Errorf("sig: %w", err)
+	}
+	*c = Claim{CID: j.CID, UID: j.UID, Key: j.Key, TS: j.TS, Sig: sig}
+	raw := j.Value
+	if len(raw) == 0 || string(raw) == "null" {
+		return nil
+	}
+	if raw[0] == '"' {
+		var s string
+		if err := json.Unmarshal(raw, &s); err != nil {
+			return err
+		}
+		c.Value = Text(s)
+		return nil
+	}
+	var b jsonBytes
+	if err := json.Unmarshal(raw, &b); err != nil || b.B64 == "" && string(raw) != `{"b64":""}` {
+		return fmt.Errorf("value must be a string, {\"b64\": ...} or null")
+	}
+	bs, err := base64.StdEncoding.DecodeString(b.B64)
+	if err != nil {
+		return fmt.Errorf("value.b64: %w", err)
+	}
+	c.Value = Bytes(bs)
+	return nil
+}

internal/claim/claim_test.go

@@ -0,0 +1,207 @@
+package claim
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/decred/dcrd/dcrec/secp256k1/v4"
+
+	"github.com/metazla/meta-core/internal/identity"
+)
+
+// The golden file every MMUR2 implementation is pinned to. Regenerate with
+// CLAIM_VECTORS_UPDATE=1 go test ./internal/claim/ — then re-sync the vendored
+// copies (scripts/check-claim-vectors.sh --fix). A change here is a wire break.
+const vectorsPath = "testdata/claim-vectors.json"
+
+// Fixed test key: the scalar 0x0101…01. Never used outside tests.
+const testKeyHex = "0101010101010101010101010101010101010101010101010101010101010101"
+
+type vector struct {
+	Name     string  `json:"name"`
+	CID      string  `json:"cid"`
+	UID      string  `json:"uid"`
+	Key      string  `json:"key"`
+	Value    *string `json:"value"`                   // text value, or null
+	ValueB64 *string `json:"valueBytesHex,omitempty"` // bytes value as hex
+	TS       uint64  `json:"ts"`
+	Signed   string  `json:"signedBytesHex"`
+	Sig      string  `json:"sigHex"` // deterministic RFC 6979, low-S
+}
+
+type vectorFile struct {
+	Comment       string   `json:"_comment"`
+	PrivateKeyHex string   `json:"privateKeyHex"`
+	UID           string   `json:"uid"`
+	Vectors       []vector `json:"vectors"`
+	HighS         struct {
+		Note   string `json:"note"`
+		Vector string `json:"vector"`
+		SigHex string `json:"sigHex"`
+	} `json:"highS"`
+}
+
+func (v vector) value(t *testing.T) Value {
+	t.Helper()
+	switch {
+	case v.ValueB64 != nil:
+		b, err := hex.DecodeString(*v.ValueB64)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return Bytes(b)
+	case v.Value != nil:
+		return Text(*v.Value)
+	}
+	return Null()
+}
+
+func strp(s string) *string { return &s }
+
+func testIdentity(t *testing.T) *identity.Identity {
+	t.Helper()
+	id, err := identity.Import(testKeyHex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+func cases(uid string) []vector {
+	return []vector{
+		{Name: "text", CID: "bafkreih5kapbjzq", UID: uid, Key: "title", Value: strp("The Matrix"), TS: 1793001234567},
+		{Name: "empty-text", CID: "bafkreih5kapbjzq", UID: uid, Key: "title", Value: strp(""), TS: 1},
+		{Name: "unicode", CID: "bafkreih5kapbjzq", UID: uid, Key: "titles/jpn/攻殻機動隊", Value: strp("true"), TS: 1793001234567},
+		{Name: "keyset-member", CID: "bagacbabaec7v3", UID: uid, Key: "cids/bafkreih5kapbjzq", Value: strp("true"), TS: 23},
+		{Name: "bytes", CID: "bafkreih5kapbjzq", UID: uid, Key: "seek", ValueB64: strp("00ff10ab"), TS: 24},
+		{Name: "retraction", CID: "bafkreih5kapbjzq", UID: uid, Key: "genres/Drama", TS: 255},
+		{Name: "long-value", CID: "bafkreih5kapbjzq", UID: uid, Key: "plot/eng", Value: strp(strings.Repeat("a", 300)), TS: 65536},
+		{Name: "ts-64bit", CID: "bafkreih5kapbjzq", UID: uid, Key: "year", Value: strp("1999"), TS: 1 << 40},
+	}
+}
+
+func TestVectors(t *testing.T) {
+	id := testIdentity(t)
+	if os.Getenv("CLAIM_VECTORS_UPDATE") == "1" {
+		writeVectors(t, id)
+	}
+	raw, err := os.ReadFile(vectorsPath)
+	if err != nil {
+		t.Fatalf("read vectors: %v (generate with CLAIM_VECTORS_UPDATE=1)", err)
+	}
+	var f vectorFile
+	if err := json.Unmarshal(raw, &f); err != nil {
+		t.Fatal(err)
+	}
+	if f.UID != id.UID {
+		t.Fatalf("vector uid %s != test key uid %s", f.UID, id.UID)
+	}
+	for _, v := range f.Vectors {
+		c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS}
+		if got := hex.EncodeToString(c.SignedBytes()); got != v.Signed {
+			t.Errorf("%s: signed bytes\n got %s\nwant %s", v.Name, got, v.Signed)
+		}
+		if err := c.Sign(id); err != nil {
+			t.Fatal(err)
+		}
+		if got := hex.EncodeToString(c.Sig); got != v.Sig {
+			t.Errorf("%s: sig\n got %s\nwant %s", v.Name, got, v.Sig)
+		}
+		if err := c.Verify(); err != nil {
+			t.Errorf("%s: verify: %v", v.Name, err)
+		}
+	}
+	// The high-S twin of a valid signature must be refused.
+	for _, v := range f.Vectors {
+		if v.Name != f.HighS.Vector {
+			continue
+		}
+		sig, _ := hex.DecodeString(f.HighS.SigHex)
+		c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS, Sig: sig}
+		if err := c.Verify(); !errors.Is(err, ErrBadSignature) {
+			t.Errorf("high-S signature accepted: %v", err)
+		}
+	}
+}
+
+func writeVectors(t *testing.T, id *identity.Identity) {
+	t.Helper()
+	f := vectorFile{
+		Comment:       "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
+		PrivateKeyHex: testKeyHex,
+		UID:           id.UID,
+	}
+	for _, v := range cases(id.UID) {
+		c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS}
+		if err := c.Sign(id); err != nil {
+			t.Fatal(err)
+		}
+		v.Signed = hex.EncodeToString(c.SignedBytes())
+		v.Sig = hex.EncodeToString(c.Sig)
+		f.Vectors = append(f.Vectors, v)
+	}
+	first := f.Vectors[0]
+	sig, _ := hex.DecodeString(first.Sig)
+	f.HighS.Note = "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it"
+	f.HighS.Vector = first.Name
+	f.HighS.SigHex = hex.EncodeToString(highS(sig))
+	out, err := json.MarshalIndent(f, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll("testdata", 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(vectorsPath, append(out, '\n'), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func highS(sig []byte) []byte {
+	var s secp256k1.ModNScalar
+	s.SetByteSlice(sig[32:])
+	s.Negate()
+	out := append([]byte{}, sig[:32]...)
+	b := s.Bytes()
+	return append(out, b[:]...)
+}
+
+func TestJSONRoundTrip(t *testing.T) {
+	id := testIdentity(t)
+	for _, v := range []Value{Text("x"), Bytes([]byte{0, 1, 2}), Null()} {
+		c := Claim{CID: "bafy", UID: id.UID, Key: "k", Value: v, TS: 42}
+		if err := c.Sign(id); err != nil {
+			t.Fatal(err)
+		}
+		b, err := json.Marshal(c)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var back Claim
+		if err := json.Unmarshal(b, &back); err != nil {
+			t.Fatalf("%s: %v", b, err)
+		}
+		if !back.Value.Equal(v) || back.TS != 42 || string(back.Sig) != string(c.Sig) {
+			t.Errorf("round trip changed the claim: %s", b)
+		}
+		if err := back.Verify(); err != nil {
+			t.Errorf("%s: %v", b, err)
+		}
+	}
+}
+
+func TestTamperRejected(t *testing.T) {
+	id := testIdentity(t)
+	c := Claim{CID: "bafy", UID: id.UID, Key: "title", Value: Text("A"), TS: 1}
+	if err := c.Sign(id); err != nil {
+		t.Fatal(err)
+	}
+	c.Value = Text("B")
+	if err := c.Verify(); !errors.Is(err, ErrBadSignature) {
+		t.Fatalf("tampered value verified: %v", err)
+	}
+}

internal/claim/testdata/claim-vectors.json

@@ -0,0 +1,93 @@
+{
+  "_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
+  "privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101",
+  "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+  "vectors": [
+    {
+      "name": "text",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "title",
+      "value": "The Matrix",
+      "ts": 1793001234567,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087",
+      "sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8"
+    },
+    {
+      "name": "empty-text",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "title",
+      "value": "",
+      "ts": 1,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001",
+      "sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642"
+    },
+    {
+      "name": "unicode",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "titles/jpn/攻殻機動隊",
+      "value": "true",
+      "ts": 1793001234567,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087",
+      "sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207"
+    },
+    {
+      "name": "keyset-member",
+      "cid": "bagacbabaec7v3",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "cids/bafkreih5kapbjzq",
+      "value": "true",
+      "ts": 23,
+      "signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517",
+      "sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761"
+    },
+    {
+      "name": "bytes",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "seek",
+      "value": null,
+      "valueBytesHex": "00ff10ab",
+      "ts": 24,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818",
+      "sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4"
+    },
+    {
+      "name": "retraction",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "genres/Drama",
+      "value": null,
+      "ts": 255,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff",
+      "sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac"
+    },
+    {
+      "name": "long-value",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "plot/eng",
+      "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "ts": 65536,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000",
+      "sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2"
+    },
+    {
+      "name": "ts-64bit",
+      "cid": "bafkreih5kapbjzq",
+      "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
+      "key": "year",
+      "value": "1999",
+      "ts": 1099511627776,
+      "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000",
+      "sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d"
+    }
+  ],
+  "highS": {
+    "note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it",
+    "vector": "text",
+    "sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899"
+  }
+}

internal/config/config.go

@@ -19,50 +19,72 @@ type Config struct {
 	BaseURL        string // Base URL for stable service discovery
 	PublicURL      string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
 
-	// Redis configuration
+	// Storage configuration
-	RedisPort int // Redis port (default: 6379)
+	DBPathOverride    string // META_CORE_DB_PATH; empty = <MetaCorePath>/db/meta-core.db (truth)
+	CachePathOverride string // META_CORE_CACHE_PATH; empty = <MetaCorePath>/db-cache/cache.db
+
+	// Report inbox (internal/reports): its own SQLite file, bounded.
+	ReportsPathOverride string // META_CORE_REPORTS_DB_PATH; empty = <MetaCorePath>/db/reports.db
+	ReportsMax          int    // META_CORE_REPORTS_MAX: reports kept (default 5000; closed evicted first)
+	ReportsMaxMB        int    // META_CORE_REPORTS_MAX_MB: total body size kept, MiB (default 256)
 
 	// HTTP API configuration
 	HTTPPort int    // HTTP API port (default: 9000)
 	HTTPHost string // HTTP API host (default: "0.0.0.0" — meta-core always runs in a container and must answer siblings)
 
 	// Timing configuration
-	HealthCheckIntervalMS    int // Health check interval in ms (default: 5000)
+	HealthCheckIntervalMS  int // Health check interval in ms (default: 5000)
-	HeartbeatIntervalMS      int // Service heartbeat interval in ms (default: 30000)
+	HeartbeatIntervalMS    int // Service heartbeat interval in ms (default: 30000)
-	StaleThresholdMS         int // Stale service threshold in ms (default: 60000)
+	StaleThresholdMS       int // Stale service threshold in ms (default: 60000)
-	CleanupIntervalMS        int // Dead service cleanup interval in ms (default: 600000)
+	CleanupIntervalMS      int // Dead service cleanup interval in ms (default: 600000)
-	DeadServiceThresholdMS   int // Dead service threshold in ms (default: 600000)
+	DeadServiceThresholdMS int // Dead service threshold in ms (default: 600000)
 
 	// File watcher configuration
-	WatchIntervalMS   int      // Polling interval for network mounts (default: 1000)
+	WatchIntervalMS   int  // Polling interval for network mounts (default: 1000)
-	DebounceMS        int      // File change debounce time (default: 30000)
+	DebounceMS        int  // File change debounce time (default: 30000)
-	EnableFileWatcher bool     // Enable file watcher (default: true)
+	EnableFileWatcher bool // Enable file watcher (default: true)
 
 	// Mount configuration
 	MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
+
+	// UDP discovery (beacon v2). Replaces the file-based registry that used
+	// to live under /meta-core/services; see
+	// docs/project-architecture/beacon-v2.md.
+	EnableUDPDiscovery  bool   // Advertise + listen on the multicast group (default: true)
+	DiscoveryGroup      string // BEACON_GROUP: IPv4 multicast group (default: 239.255.99.1)
+	DiscoveryPort       int    // BEACON_PORT: UDP port (default: 9099)
+	DiscoveryIntervalMS int    // BEACON_INTERVAL_MS: unsolicited advertise interval (default: 10000)
 }
 
 // Load creates a Config from environment variables
 func Load() *Config {
 	cfg := &Config{
-		MetaCorePath:          getEnv("META_CORE_PATH", "/meta-core"),
+		MetaCorePath:           getEnv("META_CORE_PATH", "/meta-core"),
-		FilesPath:             getEnv("FILES_PATH", "/files"),
+		FilesPath:              getEnv("FILES_PATH", "/files"),
-		ServiceName:           getEnv("SERVICE_NAME", "meta-core"),
+		ServiceName:            getEnv("SERVICE_NAME", "meta-core"),
-		ServiceVersion:        getEnv("SERVICE_VERSION", "1.0.0"),
+		ServiceVersion:         getEnv("SERVICE_VERSION", "1.0.0"),
-		APIPort:               getEnvInt("API_PORT", 8180),
+		APIPort:                getEnvInt("API_PORT", 8180),
-		BaseURL:               getEnv("BASE_URL", ""),
+		BaseURL:                getEnv("BASE_URL", ""),
-		PublicURL:             getEnv("META_CORE_PUBLIC_URL", ""),
+		PublicURL:              getEnv("META_CORE_PUBLIC_URL", ""),
-		RedisPort:             getEnvInt("REDIS_PORT", 6379),
+		DBPathOverride:         getEnv("META_CORE_DB_PATH", ""),
-		HTTPPort:              getEnvInt("META_CORE_HTTP_PORT", 9000),
+		CachePathOverride:      getEnv("META_CORE_CACHE_PATH", ""),
-		HTTPHost:              getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
+		ReportsPathOverride:    getEnv("META_CORE_REPORTS_DB_PATH", ""),
+		ReportsMax:             getEnvInt("META_CORE_REPORTS_MAX", 5000),
+		ReportsMaxMB:           getEnvInt("META_CORE_REPORTS_MAX_MB", 256),
+		HTTPPort:               getEnvInt("META_CORE_HTTP_PORT", 9000),
+		HTTPHost:               getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
 		HealthCheckIntervalMS:  getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
-		HeartbeatIntervalMS:   getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
+		HeartbeatIntervalMS:    getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
-		StaleThresholdMS:      getEnvInt("STALE_THRESHOLD_MS", 60000),
+		StaleThresholdMS:       getEnvInt("STALE_THRESHOLD_MS", 60000),
-		CleanupIntervalMS:     getEnvInt("CLEANUP_INTERVAL_MS", 600000),
+		CleanupIntervalMS:      getEnvInt("CLEANUP_INTERVAL_MS", 600000),
 		DeadServiceThresholdMS: getEnvInt("DEAD_SERVICE_THRESHOLD_MS", 600000),
-		WatchIntervalMS:       getEnvInt("WATCH_INTERVAL_MS", 1000),
+		WatchIntervalMS:        getEnvInt("WATCH_INTERVAL_MS", 1000),
-		DebounceMS:            getEnvInt("DEBOUNCE_MS", 30000),
+		DebounceMS:             getEnvInt("DEBOUNCE_MS", 30000),
-		EnableFileWatcher:     getEnvBool("ENABLE_FILE_WATCHER", true),
+		EnableFileWatcher:      getEnvBool("ENABLE_FILE_WATCHER", true),
+		EnableUDPDiscovery:     getEnvBool("ENABLE_UDP_DISCOVERY", true),
+		DiscoveryGroup:         getEnv("BEACON_GROUP", "239.255.99.1"),
+		DiscoveryPort:          getEnvInt("BEACON_PORT", 9099),
+		DiscoveryIntervalMS:    getEnvInt("BEACON_INTERVAL_MS", 10000),
 	}
 
 	// Set mounts directory
@@ -71,24 +93,39 @@ func Load() *Config {
 	return cfg
 }
 
-// LockFilePath returns the path to the leader lock file
+// LockFilePath returns the path to the database-volume mutex (one meta-core
+// per database file). Held by
+// docker/leader-election.sh; nothing reads it, and it is NOT service
+// discovery — see docs/project-architecture/beacon-v2.md.
 func (c *Config) LockFilePath() string {
 	return c.MetaCorePath + "/locks/kv-leader.lock"
 }
 
-// InfoFilePath returns the path to the leader info file
+// DBPath returns the path of meta-core's truth database (backed up).
-func (c *Config) InfoFilePath() string {
+func (c *Config) DBPath() string {
-	return c.MetaCorePath + "/locks/kv-leader.info"
+	if c.DBPathOverride != "" {
+		return c.DBPathOverride
+	}
+	return c.MetaCorePath + "/db/meta-core.db"
 }
 
-// RedisDataDir returns the path to Redis data directory
+// CachePath returns the path of meta-core's cache database: everything
-func (c *Config) RedisDataDir() string {
+// derived from truth, never backed up, rebuilt when missing.
-	return c.MetaCorePath + "/db/redis"
+func (c *Config) CachePath() string {
+	if c.CachePathOverride != "" {
+		return c.CachePathOverride
+	}
+	return c.MetaCorePath + "/db-cache/cache.db"
 }
 
-// ServicesDir returns the path to services directory
+// ReportsDBPath returns the path of the report inbox database. It sits beside
-func (c *Config) ServicesDir() string {
+// the truth database (so it is backed up with it) but is a separate file:
-	return c.MetaCorePath + "/services"
+// reports are diagnostics, never metadata.
+func (c *Config) ReportsDBPath() string {
+	if c.ReportsPathOverride != "" {
+		return c.ReportsPathOverride
+	}
+	return c.MetaCorePath + "/db/reports.db"
 }
 
 // MountsFilePath returns the path to the mounts configuration file
@@ -161,11 +198,11 @@ func (c *Config) DefaultWatchPath() string {
 //
 //   - /files/watch  — user-curated media. The main library.
 //   - /files/plugin — plugin output (TMDB posters/backdrops, extracted
-//                     subtitles, etc.). Indexed so each artefact gets a
+//     subtitles, etc.). Indexed so each artefact gets a
-//                     midhash256 alias and CID-based references from
+//     midhash256 alias and CID-based references from
-//                     other files (e.g. `backdrop=<cid>`) resolve through
+//     other files (e.g. `backdrop=<cid>`) resolve through
-//                     the standard reverse index instead of needing a
+//     the standard reverse index instead of needing a
-//                     path companion field.
+//     path companion field.
 func (c *Config) DefaultWatcherPaths() []string {
 	return []string{
 		c.FilesPath + "/watch",

internal/discovery/cleaner.go

@@ -1,196 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// CleanupStats holds statistics about the cleanup process
-type CleanupStats struct {
-	LastRunTime     string `json:"lastRunTime,omitempty"`
-	ServicesRemoved int    `json:"servicesRemoved"`
-	TotalRuns       int    `json:"totalRuns"`
-	Errors          int    `json:"errors"`
-}
-
-// Cleaner handles periodic cleanup of dead service registration files
-type Cleaner struct {
-	config      *config.Config
-	servicesDir string
-	ownFile     string // Our own service file (never delete)
-
-	stats    CleanupStats
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-	mu       sync.RWMutex
-}
-
-// NewCleaner creates a new service cleaner
-func NewCleaner(cfg *config.Config) *Cleaner {
-	hostname, _ := os.Hostname()
-	return &Cleaner{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		ownFile:     filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins the cleanup loop
-func (c *Cleaner) Start() error {
-	log.Printf("[Cleaner] Starting dead service cleanup (interval: %dms, threshold: %dms)",
-		c.config.CleanupIntervalMS, c.config.DeadServiceThresholdMS)
-
-	c.wg.Add(1)
-	go c.cleanupLoop()
-
-	return nil
-}
-
-// Stop stops the cleanup loop
-func (c *Cleaner) Stop() error {
-	log.Println("[Cleaner] Stopping dead service cleanup...")
-	close(c.stopChan)
-	c.wg.Wait()
-	return nil
-}
-
-// cleanupLoop runs the periodic cleanup
-func (c *Cleaner) cleanupLoop() {
-	defer c.wg.Done()
-
-	// Run an initial cleanup shortly after starting
-	initialDelay := time.NewTimer(30 * time.Second)
-	select {
-	case <-c.stopChan:
-		initialDelay.Stop()
-		return
-	case <-initialDelay.C:
-		c.runCleanup()
-	}
-
-	ticker := time.NewTicker(time.Duration(c.config.CleanupIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-c.stopChan:
-			return
-		case <-ticker.C:
-			c.runCleanup()
-		}
-	}
-}
-
-// runCleanup scans the services directory and removes dead services
-func (c *Cleaner) runCleanup() {
-	c.mu.Lock()
-	c.stats.TotalRuns++
-	c.stats.LastRunTime = time.Now().UTC().Format(time.RFC3339)
-	c.mu.Unlock()
-
-	entries, err := os.ReadDir(c.servicesDir)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Printf("[Cleaner] Failed to read services directory: %v", err)
-			c.mu.Lock()
-			c.stats.Errors++
-			c.mu.Unlock()
-		}
-		return
-	}
-
-	threshold := time.Duration(c.config.DeadServiceThresholdMS) * time.Millisecond
-	removedCount := 0
-
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(c.servicesDir, entry.Name())
-
-		// Never delete our own service file
-		if filePath == c.ownFile {
-			continue
-		}
-
-		// Check if service is dead
-		isDead, serviceName, age := c.isServiceDead(filePath, threshold)
-		if isDead {
-			if err := os.Remove(filePath); err != nil {
-				log.Printf("[Cleaner] Failed to remove dead service file %s: %v", entry.Name(), err)
-				c.mu.Lock()
-				c.stats.Errors++
-				c.mu.Unlock()
-			} else {
-				ageMinutes := age.Minutes()
-				log.Printf("[Cleaner] Removed dead service: %s (last heartbeat: %.1f minutes ago)", serviceName, ageMinutes)
-				removedCount++
-			}
-		}
-	}
-
-	if removedCount > 0 {
-		c.mu.Lock()
-		c.stats.ServicesRemoved += removedCount
-		c.mu.Unlock()
-		log.Printf("[Cleaner] Cleanup complete: removed %d dead service(s)", removedCount)
-	}
-}
-
-// isServiceDead checks if a service file represents a dead service
-// Returns: isDead, serviceName, age since last heartbeat
-func (c *Cleaner) isServiceDead(filePath string, threshold time.Duration) (bool, string, time.Duration) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		// Can't read file - treat as dead to clean up corrupted files
-		log.Printf("[Cleaner] Cannot read service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, filepath.Base(filePath), threshold
-	}
-
-	var info ServiceInfo
-	if err := json.Unmarshal(data, &info); err != nil {
-		// Corrupted JSON - skip rather than delete (might be recoverable)
-		log.Printf("[Cleaner] Skipping corrupted service file %s: %v", filepath.Base(filePath), err)
-		return false, "", 0
-	}
-
-	// Parse last heartbeat timestamp
-	if info.LastHeartbeat == "" {
-		// No heartbeat timestamp - treat as dead
-		return true, info.Name, threshold
-	}
-
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err != nil {
-		// Invalid timestamp - treat as dead
-		log.Printf("[Cleaner] Invalid timestamp in service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, info.Name, threshold
-	}
-
-	age := time.Since(lastHeartbeat)
-	if age > threshold {
-		return true, info.Name, age
-	}
-
-	return false, info.Name, age
-}
-
-// Stats returns a copy of the current cleanup statistics
-func (c *Cleaner) Stats() CleanupStats {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return CleanupStats{
-		LastRunTime:     c.stats.LastRunTime,
-		ServicesRemoved: c.stats.ServicesRemoved,
-		TotalRuns:       c.stats.TotalRuns,
-		Errors:          c.stats.Errors,
-	}
-}

internal/discovery/service.go

@@ -1,286 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// ServiceInfo contains simplified service registration data
-// URLs are obtained via the /urls API endpoint
-type ServiceInfo struct {
-	Name          string `json:"name"`
-	Hostname      string `json:"hostname"`
-	BaseUrl       string `json:"baseUrl"`
-	Status        string `json:"status"`
-	LastHeartbeat string `json:"lastHeartbeat"`
-}
-
-// Service handles service registration and discovery
-type Service struct {
-	config      *config.Config
-	servicesDir string
-	serviceFile string
-	info        *ServiceInfo
-	mu          sync.RWMutex
-
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-}
-
-// NewService creates a new service discovery instance
-func NewService(cfg *config.Config) *Service {
-	hostname, _ := os.Hostname()
-	return &Service{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		serviceFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins service registration and heartbeat
-func (s *Service) Start() error {
-	log.Printf("[Discovery] Starting service discovery for %s", s.config.ServiceName)
-
-	// Ensure services directory exists
-	if err := os.MkdirAll(s.servicesDir, 0755); err != nil {
-		return fmt.Errorf("failed to create services directory: %w", err)
-	}
-
-	// Build and register service info
-	s.info = s.buildServiceInfo()
-	if err := s.register(); err != nil {
-		return fmt.Errorf("failed to register service: %w", err)
-	}
-
-	// Start heartbeat loop
-	s.wg.Add(1)
-	go s.heartbeatLoop()
-
-	return nil
-}
-
-// Stop stops service discovery and unregisters
-func (s *Service) Stop() error {
-	log.Println("[Discovery] Stopping service discovery...")
-	close(s.stopChan)
-	s.wg.Wait()
-
-	// Unregister by removing service file
-	if err := os.Remove(s.serviceFile); err != nil && !os.IsNotExist(err) {
-		log.Printf("[Discovery] Failed to remove service file: %v", err)
-	}
-
-	return nil
-}
-
-// buildServiceInfo creates the service info for this instance
-func (s *Service) buildServiceInfo() *ServiceInfo {
-	hostname, _ := os.Hostname()
-	ip := getLocalIP()
-
-	// Browser-facing nav URL: prefer META_CORE_PUBLIC_URL (the reachable URL
-	// when there's no Caddy perimeter in front — e.g. a debug-direct port),
-	// then BASE_URL (the Caddy perimeter URL), then the internal IP.
-	var baseUrl string
-	if s.config.PublicURL != "" {
-		baseUrl = s.config.PublicURL
-	} else if s.config.BaseURL != "" {
-		baseUrl = s.config.BaseURL
-	} else {
-		baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
-	}
-
-	info := &ServiceInfo{
-		Name:          s.config.ServiceName,
-		Hostname:      hostname,
-		BaseUrl:       baseUrl,
-		Status:        "running",
-		LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
-	}
-
-	return info
-}
-
-// register writes service info to file
-func (s *Service) register() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.writeServiceInfo(s.info)
-}
-
-// writeServiceInfo atomically writes service info to file
-func (s *Service) writeServiceInfo(info *ServiceInfo) error {
-	tempPath := s.serviceFile + ".tmp"
-
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	if err := os.WriteFile(tempPath, data, 0644); err != nil {
-		return err
-	}
-
-	return os.Rename(tempPath, s.serviceFile)
-}
-
-// heartbeatLoop periodically updates the last heartbeat
-func (s *Service) heartbeatLoop() {
-	defer s.wg.Done()
-
-	ticker := time.NewTicker(time.Duration(s.config.HeartbeatIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-s.stopChan:
-			return
-		case <-ticker.C:
-			s.heartbeat()
-		}
-	}
-}
-
-// heartbeat updates the last heartbeat timestamp
-func (s *Service) heartbeat() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return
-	}
-
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-
-	if err := s.writeServiceInfo(s.info); err != nil {
-		log.Printf("[Discovery] Failed to update heartbeat: %v", err)
-	}
-}
-
-// UpdateStatus updates the service status
-func (s *Service) UpdateStatus(status string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return fmt.Errorf("service not registered")
-	}
-
-	s.info.Status = status
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-	return s.writeServiceInfo(s.info)
-}
-
-// Discover finds a service by name
-// Looks for files matching pattern: name-*.json (hostname-based naming)
-func (s *Service) Discover(name string) (*ServiceInfo, error) {
-	// First try exact match for backward compatibility
-	exactPath := filepath.Join(s.servicesDir, name+".json")
-	if data, err := os.ReadFile(exactPath); err == nil {
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err == nil {
-			return s.checkStale(&info), nil
-		}
-	}
-
-	// Search for hostname-based files: name-*.json
-	pattern := filepath.Join(s.servicesDir, name+"-*.json")
-	matches, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-
-	// Return the first valid service found
-	for _, match := range matches {
-		data, err := os.ReadFile(match)
-		if err != nil {
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			continue
-		}
-
-		return s.checkStale(&info), nil
-	}
-
-	return nil, nil
-}
-
-// checkStale marks a service as stale if heartbeat is too old
-func (s *Service) checkStale(info *ServiceInfo) *ServiceInfo {
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err == nil {
-		staleThreshold := time.Duration(s.config.StaleThresholdMS) * time.Millisecond
-		if time.Since(lastHeartbeat) > staleThreshold {
-			info.Status = "stale"
-		}
-	}
-	return info
-}
-
-// DiscoverAll finds all registered services
-func (s *Service) DiscoverAll() ([]*ServiceInfo, error) {
-	entries, err := os.ReadDir(s.servicesDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return []*ServiceInfo{}, nil
-		}
-		return nil, err
-	}
-
-	var services []*ServiceInfo
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(s.servicesDir, entry.Name())
-		data, err := os.ReadFile(filePath)
-		if err != nil {
-			log.Printf("[Discovery] Failed to read service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			log.Printf("[Discovery] Failed to parse service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		services = append(services, s.checkStale(&info))
-	}
-
-	return services, nil
-}
-
-// getLocalIP returns the local IP address
-func getLocalIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		hostname, _ := os.Hostname()
-		return hostname
-	}
-
-	for _, addr := range addrs {
-		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				return ipnet.IP.String()
-			}
-		}
-	}
-
-	hostname, _ := os.Hostname()
-	return hostname
-}

internal/events/meta_publisher.go

@@ -1,222 +0,0 @@
-package events
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/redis/go-redis/v9"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-const (
-	// MetaEventsStream is the Redis stream name for metadata events
-	MetaEventsStream = "meta:events"
-
-	// MetaEventsMaxLen is the approximate cap on stream length. Bounded
-	// retention is a contract requirement for the SSE wire — see
-	// docs/api-mediated-access.md "Backing-store retention and lifecycle".
-	// Cheap (~) trim keeps growth bounded without serialising writes.
-	MetaEventsMaxLen = 100_000
-)
-
-// MetaPublisher subscribes to Redis keyspace notifications
-// and publishes metadata change events to the meta:events stream
-type MetaPublisher struct {
-	client  *redis.Client
-	ctx     context.Context
-	cancel  context.CancelFunc
-	wg      sync.WaitGroup
-	running bool
-	mu      sync.RWMutex
-}
-
-// NewMetaPublisher creates a new MetaPublisher
-func NewMetaPublisher(client *redis.Client) *MetaPublisher {
-	ctx, cancel := context.WithCancel(context.Background())
-	return &MetaPublisher{
-		client: client,
-		ctx:    ctx,
-		cancel: cancel,
-	}
-}
-
-// Start begins subscribing to keyspace notifications.
-//
-// The stream is NOT truncated on startup — the SSE wire contract requires
-// Last-Event-ID resumption to survive a meta-core restart, which means
-// entries must persist across restart. Growth is bounded instead by
-// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
-func (p *MetaPublisher) Start() error {
-	p.mu.Lock()
-	if p.running {
-		p.mu.Unlock()
-		return nil
-	}
-	p.running = true
-	p.mu.Unlock()
-
-	p.wg.Add(1)
-	go p.subscribeLoop()
-
-	log.Println("[MetaPublisher] Started - listening for keyspace notifications")
-	return nil
-}
-
-// Stop stops the publisher
-func (p *MetaPublisher) Stop() error {
-	p.mu.Lock()
-	if !p.running {
-		p.mu.Unlock()
-		return nil
-	}
-	p.running = false
-	p.mu.Unlock()
-
-	p.cancel()
-	p.wg.Wait()
-
-	log.Println("[MetaPublisher] Stopped")
-	return nil
-}
-
-// IsRunning returns true if the publisher is running
-func (p *MetaPublisher) IsRunning() bool {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-	return p.running
-}
-
-// subscribeLoop subscribes to keyspace notifications and publishes events
-func (p *MetaPublisher) subscribeLoop() {
-	defer p.wg.Done()
-
-	// Subscribe to all file:* keyspace events
-	// Pattern: __keyspace@0__:file:*
-	pattern := "__keyspace@0__:file:*"
-	pubsub := p.client.PSubscribe(p.ctx, pattern)
-	defer pubsub.Close()
-
-	log.Printf("[MetaPublisher] Subscribed to pattern: %s", pattern)
-
-	ch := pubsub.Channel()
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		case msg := <-ch:
-			if msg == nil {
-				continue
-			}
-			// msg.Channel: __keyspace@0__:file:midhash256:abc123/tmdb
-			// msg.Payload: set, del, expire, etc.
-			p.publishEvent(msg.Channel, msg.Payload)
-		}
-	}
-}
-
-// publishEvent publishes a keyspace event to the meta:events stream
-func (p *MetaPublisher) publishEvent(channel, operation string) {
-	// Extract key from channel: __keyspace@0__:file:xxx/field -> file:xxx/field
-	key := strings.TrimPrefix(channel, "__keyspace@0__:")
-
-	// Skip index key and non-file keys
-	if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
-		return
-	}
-	// Skip the per-record field index. It is bookkeeping (see
-	// storage.FieldsField), not a metadata field — publishing it would make
-	// every downstream consumer that parses `file:<id>/<field>` off this
-	// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
-	// field change on a key they can never read.
-	if strings.HasSuffix(key, "/"+storage.FieldsField) {
-		return
-	}
-
-	// Build event fields
-	fields := map[string]interface{}{
-		"type": operation, // "set", "del", "expire"
-		"key":  key,       // "file:midhash256:abc123/tmdb"
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	// Approximate MAXLEN trim keeps the stream bounded across restarts
-	// (see Start). Cap is intentionally generous; most consumers reconnect
-	// well within retention.
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing event: %v", err)
-	}
-}
-
-// publishEventDirect publishes an event directly (not from keyspace notification)
-func (p *MetaPublisher) publishEventDirect(operation, key string) {
-	fields := map[string]interface{}{
-		"type": operation,
-		"key":  key,
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing direct event: %v", err)
-	}
-}
-
-// RepublishFunc is a function type for getting metadata that can be republished
-type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error)
-
-// RepublishAllMetadata iterates over all existing file metadata and publishes events
-// This is called on startup and can be triggered via API
-func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) {
-	p.mu.RLock()
-	running := p.running
-	p.mu.RUnlock()
-
-	if !running {
-		return 0, fmt.Errorf("publisher not running")
-	}
-
-	// Get all hash IDs and metadata getter
-	hashIDs, getMetadata, err := republishFunc()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get hash IDs: %w", err)
-	}
-
-	log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs))
-
-	publishedCount := 0
-	for _, hashID := range hashIDs {
-		// Get all properties for this file
-		metadata, err := getMetadata(hashID)
-		if err != nil || metadata == nil {
-			continue
-		}
-
-		// Publish a set event for each property
-		for property := range metadata {
-			key := fmt.Sprintf("file:%s/%s", hashID, property)
-			p.publishEventDirect("set", key)
-			publishedCount++
-		}
-	}
-
-	log.Printf("[MetaPublisher] Republished %d property events", publishedCount)
-	return publishedCount, nil
-}

internal/identity/identity.go

@@ -307,6 +307,30 @@ func Sign(id *Identity, bytes []byte) ([]byte, error) {
 	return sig.Serialize(), nil
 }
 
+// SignCompact is Sign with the 64-byte compact r||s encoding MMUR2 claims
+// use. decred's RFC 6979 signer always emits low-S, which MMUR2 requires.
+func SignCompact(id *Identity, bytes []byte) ([]byte, error) {
+	if id == nil {
+		return nil, errors.New("no identity loaded")
+	}
+	raw, err := hex.DecodeString(id.PrivateKeyHex)
+	if err != nil {
+		return nil, fmt.Errorf("decode private key: %w", err)
+	}
+	if len(raw) != 32 {
+		return nil, fmt.Errorf("private key length %d", len(raw))
+	}
+	priv := secp256k1.PrivKeyFromBytes(raw)
+	digest := sha256.Sum256(bytes)
+	sig := ecdsa.Sign(priv, digest[:])
+	r, s := sig.R(), sig.S()
+	out := make([]byte, 64)
+	rb, sb := r.Bytes(), s.Bytes()
+	copy(out[:32], rb[:])
+	copy(out[32:], sb[:])
+	return out, nil
+}
+
 // DeriveAEADKey derives the symmetric key used to encrypt private-tier
 // User Data Layer records. The KDF parameters (salt, info, length) are
 // fixed for the v1 wire format.

internal/identity/verify.go

@@ -56,7 +56,7 @@ func ValidAction(a Action) bool {
 }
 
 // ValidUID exposes the uid shape check to callers outside this package (the
-// API layer, which must reject a hostile uid before it reaches a Redis SCAN
+// API layer, which must reject a hostile uid before it reaches a key-range read
 // pattern or a file path).
 func ValidUID(uid string) bool { return validUID(uid) }
 

internal/kv/kv.go

@@ -0,0 +1,869 @@
+// Package kv is meta-core's storage engine: one string→string table in an
+// embedded SQLite file.
+//
+// # Why a key/value table and not a schema
+//
+// meta-core's data model is schema-free on purpose — a record is a set of flat
+// `file:<rid>/<field>` keys, and a new metadata field is just a new key
+// (METADATA_KEYS.md). The engine keeps exactly that model:
+//
+//	CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID
+//
+// is the only table that holds data, and it never needs a migration. What
+// Redis gave us that a bare KV would not — "list a record's fields", "find
+// records by field value", free-text search — comes from the table being
+// *ordered* (a record is a key range) and from two derived structures the
+// storage package maintains on top: an expression index over the field name
+// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv
+// rows and never authoritative.
+//
+// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a
+// 169k-record corpus (Redis plus the API's in-memory copy needed to search
+// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here
+// and the working set is a bounded page cache.
+//
+// # Concurrency
+//
+// SQLite allows one writer at a time, so every write goes through one writer
+// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED:
+// the writer runs every request waiting at that moment inside one SQLite
+// transaction, each under its own SAVEPOINT (a failing request rolls back
+// alone), and commits once. A commit — writing the dirty pages of the table,
+// the field index and the FTS shadow tables to the WAL — was ~40% of the cost
+// of a record write, so sharing it is what makes concurrent ingest scale.
+//
+// Reads use a separate pool and see the last committed state (WAL). Inside
+// Update, read through the *Tx — the reader pool cannot see uncommitted rows —
+// and never call Update from inside an Update callback (the writer would wait
+// for itself).
+//
+// # Events
+//
+// Redis keyspace notifications are replaced by the transaction itself: every
+// Put/Delete of a key the Options.EventKey predicate selects is appended, in
+// the same transaction, to Options.EventStream as {type: set|del, key, ts}.
+// A change and its event therefore commit together or not at all.
+package kv
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// Options configures a Store.
+type Options struct {
+	// EventStream receives a set/del entry for every changed key EventKey
+	// selects. Empty disables change events.
+	EventStream string
+	// EventMaxLen is the approximate retention of EventStream.
+	EventMaxLen int64
+	// EventKey selects the keys whose changes are published.
+	EventKey func(key string) bool
+	// WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection.
+	WriterCacheMB int
+	ReaderCacheMB int
+	// Readers is the size of the read connection pool.
+	Readers int
+	// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
+	// for derived indexes owned by the caller.
+	Schema string
+	// OnChange is called inside the transaction for every key whose value
+	// actually changed (created, modified or deleted), with the old and new
+	// values — the hook derived counters hang off (Tx.AddCounter). It must not
+	// write kv rows itself.
+	OnChange func(tx *Tx, key, old string, hadOld bool, new string, hasNew bool)
+	// NoKVTable opens the file without the kv table and its prepared
+	// statements: the caller owns every table through Schema and writes them
+	// with Tx.Exec / Tx.Query (meta-core's truth database). The kv methods
+	// (Get, Put, streams, counters) must not be used on such a store.
+	NoKVTable bool
+	// Synchronous is SQLite's synchronous pragma; default NORMAL. A store that
+	// is rebuildable from another (meta-core's cache) runs OFF.
+	Synchronous string
+}
+
+// Store is an open database.
+type Store struct {
+	path string
+	opts Options
+	w    *sql.DB // single connection: the one writer
+	r    *sql.DB // read pool
+
+	streamMu sync.Mutex
+	streams  map[string]*streamState
+
+	// Hot write statements, prepared once on the writer connection: parsing
+	// SQL on every Put was the single largest cost of a record write.
+	stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
+
+	// Write accounting: how many Puts changed a value vs found it unchanged.
+	puts, noops atomic.Int64
+
+	writes    chan *writeReq
+	writeMu   sync.Mutex // held by the writer while a group runs
+	closeOnce sync.Once
+	stopped   chan struct{}
+}
+
+// writeReq is one Update call waiting for the writer.
+type writeReq struct {
+	fn   func(tx *Tx) error
+	done chan error
+}
+
+// maxGroup caps how many requests share one commit, bounding how long the
+// writer holds a transaction open.
+const maxGroup = 128
+
+// Open opens (creating if needed) the database file at path.
+func Open(path string, opts Options) (*Store, error) {
+	if opts.Readers <= 0 {
+		opts.Readers = 8
+	}
+	if opts.WriterCacheMB <= 0 {
+		opts.WriterCacheMB = 64
+	}
+	if opts.ReaderCacheMB <= 0 {
+		opts.ReaderCacheMB = 16
+	}
+	if opts.EventMaxLen <= 0 {
+		opts.EventMaxLen = 100_000
+	}
+
+	if opts.Synchronous == "" {
+		opts.Synchronous = "NORMAL"
+	}
+	w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false, opts.Synchronous))
+	if err != nil {
+		return nil, fmt.Errorf("open writer: %w", err)
+	}
+	w.SetMaxOpenConns(1)
+	w.SetMaxIdleConns(1)
+	w.SetConnMaxLifetime(0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if !opts.NoKVTable {
+		if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
+			w.Close()
+			return nil, fmt.Errorf("create kv: %w", err)
+		}
+	}
+	if opts.Schema != "" {
+		// No deadline: on an existing store, the first open after a new index
+		// is added builds it over every row (minutes at millions of keys).
+		// Cancelling that just rolls it back and restarts it on the next boot.
+		started := time.Now()
+		if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil {
+			w.Close()
+			return nil, fmt.Errorf("apply schema: %w", err)
+		}
+		if d := time.Since(started); d > 5*time.Second {
+			log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second))
+		}
+	}
+
+	r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true, opts.Synchronous))
+	if err != nil {
+		w.Close()
+		return nil, fmt.Errorf("open readers: %w", err)
+	}
+	r.SetMaxOpenConns(opts.Readers)
+	r.SetMaxIdleConns(opts.Readers)
+
+	st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)}
+	// Fresh deadline: the schema step above may legitimately have taken minutes.
+	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	stmts := []struct {
+		dst   **sql.Stmt
+		query string
+	}{
+		{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
+		{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
+		{&st.stDelete, `DELETE FROM kv WHERE k = ? RETURNING v`},
+		{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
+		{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
+	}
+	if opts.NoKVTable {
+		stmts = nil
+	}
+	for _, p := range stmts {
+		if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil {
+			r.Close()
+			w.Close()
+			return nil, fmt.Errorf("prepare: %w", err)
+		}
+	}
+	st.writes = make(chan *writeReq)
+	st.stopped = make(chan struct{})
+	go st.writer()
+	return st, nil
+}
+
+// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class
+// Redis' AOF-everysec gave us: a power loss may drop the last transactions,
+// never corrupt the file.
+func dsn(path string, cacheMB int, readOnly bool, synchronous string) string {
+	q := url.Values{}
+	q.Add("_pragma", "busy_timeout(30000)")
+	q.Add("_pragma", "journal_mode(WAL)")
+	q.Add("_pragma", "synchronous("+synchronous+")")
+	q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024))
+	q.Add("_pragma", "temp_store(MEMORY)")
+	q.Add("_pragma", "journal_size_limit(268435456)")
+	if readOnly {
+		q.Add("_pragma", "query_only(1)")
+	} else {
+		q.Add("_txlock", "immediate")
+	}
+	return "file:" + path + "?" + q.Encode()
+}
+
+// Close stops the writer and closes both pools, checkpointing the WAL into the
+// main file.
+func (s *Store) Close() error {
+	s.closeOnce.Do(func() { close(s.stopped) })
+	s.writeMu.Lock() // wait for an in-flight group to finish
+	defer s.writeMu.Unlock()
+	_, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`)
+	return errors.Join(s.r.Close(), s.w.Close())
+}
+
+// Ping reports whether the database answers.
+func (s *Store) Ping(ctx context.Context) error {
+	var one int
+	return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one)
+}
+
+// Path is the database file path.
+func (s *Store) Path() string { return s.path }
+
+// DB exposes the read pool for callers that own derived indexes (search). It
+// is read-only (query_only), so it cannot bypass Update.
+func (s *Store) DB() *sql.DB { return s.r }
+
+// FileSize returns the on-disk size of the database plus its WAL.
+func (s *Store) FileSize() int64 {
+	var total int64
+	for _, suffix := range []string{"", "-wal"} {
+		if fi, err := os.Stat(s.path + suffix); err == nil {
+			total += fi.Size()
+		}
+	}
+	return total
+}
+
+// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer
+// slot and waits for readers, so callers run it when idle.
+func (s *Store) Checkpoint(ctx context.Context) error {
+	_, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
+	return err
+}
+
+// KV is one key/value pair.
+type KV struct {
+	K, V string
+}
+
+// Reader is the read surface shared by Store (committed state) and Tx (the
+// transaction's own view). Storage helpers take a Reader so they work both
+// inside and outside a write.
+type Reader interface {
+	Get(ctx context.Context, k string) (string, bool, error)
+	MGet(ctx context.Context, keys []string) (map[string]string, error)
+	Range(ctx context.Context, prefix, after string, limit int) ([]KV, error)
+	Count(ctx context.Context, prefix string) (int64, error)
+}
+
+type queryer interface {
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
+	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
+}
+
+// PrefixEnd returns the smallest string greater than every string with the
+// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range.
+// An empty result means "unbounded".
+func PrefixEnd(prefix string) string {
+	b := []byte(prefix)
+	for i := len(b) - 1; i >= 0; i-- {
+		if b[i] < 0xff {
+			b[i]++
+			return string(b[:i+1])
+		}
+	}
+	return ""
+}
+
+func get(ctx context.Context, q queryer, k string) (string, bool, error) {
+	var v string
+	err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+
+// mgetBatch stays well under SQLite's default 32766 bound-parameter cap.
+const mgetBatch = 500
+
+func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) {
+	out := make(map[string]string, len(keys))
+	for start := 0; start < len(keys); start += mgetBatch {
+		end := min(start+mgetBatch, len(keys))
+		chunk := keys[start:end]
+		args := make([]any, len(chunk))
+		for i, k := range chunk {
+			args[i] = k
+		}
+		rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...)
+		if err != nil {
+			return nil, err
+		}
+		for rows.Next() {
+			var k, v string
+			if err := rows.Scan(&k, &v); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			out[k] = v
+		}
+		if err := rows.Close(); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+// rangeQuery returns keys with prefix, strictly after `after` (when non-empty),
+// in key order, at most limit (<= 0 means all).
+func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) {
+	lo, loOp := prefix, ">="
+	if after != "" && after >= prefix {
+		lo, loOp = after, ">"
+	}
+	query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?`
+	args := []any{lo}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	query += ` ORDER BY k`
+	if limit > 0 {
+		query += ` LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := q.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		out = append(out, kv)
+	}
+	return out, rows.Err()
+}
+
+func count(ctx context.Context, q queryer, prefix string) (int64, error) {
+	query := `SELECT count(*) FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	var n int64
+	err := q.QueryRowContext(ctx, query, args...).Scan(&n)
+	return n, err
+}
+
+// Get returns the committed value of k.
+func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) }
+
+// MGet returns the committed values of the keys that exist.
+func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, s.r, keys)
+}
+
+// Range returns committed keys under prefix after `after`, in key order.
+func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, s.r, prefix, after, limit)
+}
+
+// Count returns how many committed keys sit under prefix.
+func (s *Store) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, s.r, prefix)
+}
+
+// ScanPrefix walks every committed key under prefix in pages of `page`, each
+// page its own short read so a long walk never pins a WAL snapshot (which
+// would stop checkpoints and let the WAL grow without bound).
+func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error {
+	if page <= 0 {
+		page = 5000
+	}
+	after := ""
+	for {
+		batch, err := rangeQuery(ctx, s.r, prefix, after, page)
+		if err != nil {
+			return err
+		}
+		if len(batch) == 0 {
+			return nil
+		}
+		if err := fn(batch); err != nil {
+			return err
+		}
+		if len(batch) < page {
+			return nil
+		}
+		after = batch[len(batch)-1].K
+	}
+}
+
+// CountAll counts every row. A full scan — callers cache it.
+func (s *Store) CountAll(ctx context.Context) (int64, error) {
+	var n int64
+	err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n)
+	return n, err
+}
+
+// ErrClosed is returned by Update on a closed store.
+var ErrClosed = errors.New("kv: store closed")
+
+// Update runs fn atomically: its writes (and the change events they publish)
+// commit together or not at all. fn may share a SQLite transaction with other
+// concurrent Updates (group commit) but is isolated from their failures by a
+// savepoint. Never call Update from inside fn.
+//
+// ctx bounds only the wait for the writer. Once fn starts it runs to
+// completion: interrupting a statement inside a shared transaction would roll
+// back every request in the group.
+func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error {
+	req := &writeReq{fn: fn, done: make(chan error, 1)}
+	select {
+	case s.writes <- req:
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-s.stopped:
+		return ErrClosed
+	}
+	return <-req.done
+}
+
+// writer is the single goroutine that executes writes.
+func (s *Store) writer() {
+	for {
+		var first *writeReq
+		select {
+		case first = <-s.writes:
+		case <-s.stopped:
+			return
+		}
+		group := []*writeReq{first}
+	drain:
+		for len(group) < maxGroup {
+			select {
+			case r := <-s.writes:
+				group = append(group, r)
+			default:
+				break drain
+			}
+		}
+		s.writeMu.Lock()
+		s.runGroup(group)
+		s.writeMu.Unlock()
+	}
+}
+
+// group is the state shared by the requests of one commit.
+type group struct {
+	tx       *sql.Tx
+	pending  map[string]*streamPending
+	stmts    map[*sql.Stmt]*sql.Stmt
+	counters map[string]int64
+}
+
+func (g *group) snapshot() map[string]streamPending {
+	out := make(map[string]streamPending, len(g.pending))
+	for k, p := range g.pending {
+		out[k] = *p
+	}
+	return out
+}
+
+func (g *group) restore(snap map[string]streamPending) {
+	g.pending = make(map[string]*streamPending, len(snap))
+	for k, p := range snap {
+		p := p
+		g.pending[k] = &p
+	}
+}
+
+func (s *Store) runGroup(reqs []*writeReq) {
+	ctx := context.Background()
+	results := make([]error, len(reqs))
+	fail := func(err error) {
+		for i, r := range reqs {
+			if results[i] == nil {
+				results[i] = err
+			}
+			r.done <- results[i]
+		}
+	}
+
+	sqlTx, err := s.w.BeginTx(ctx, nil)
+	if err != nil {
+		fail(fmt.Errorf("begin: %w", err))
+		return
+	}
+	g := &group{tx: sqlTx}
+	var commitHooks []func()
+	for i, r := range reqs {
+		if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("savepoint: %w", err))
+			return
+		}
+		snap := g.snapshot()
+		tx := &Tx{ctx: ctx, s: s, g: g}
+		err := r.fn(tx)
+		if err == nil {
+			err = tx.flushEvents()
+		}
+		if err == nil {
+			g.mergeCounters(tx.counters)
+			commitHooks = append(commitHooks, tx.onCommit...)
+		}
+		if err != nil {
+			results[i] = err
+			if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
+				_ = sqlTx.Rollback()
+				fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err))
+				return
+			}
+			g.restore(snap)
+		}
+		if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("release savepoint: %w", err))
+			return
+		}
+	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushCounters(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
+	if err := sqlTx.Commit(); err != nil {
+		fail(fmt.Errorf("commit: %w", err))
+		return
+	}
+	s.afterCommit(g.pending)
+	// Hooks run in the writer goroutine, in commit order, before any request
+	// returns: a caller that orders work by commit (meta-core's apply queue)
+	// sees it in exactly that order.
+	for _, h := range commitHooks {
+		h()
+	}
+	for i, r := range reqs {
+		r.done <- results[i]
+	}
+}
+
+// Tx is one Update's view of the write transaction. Its reads see its own
+// writes (and those of earlier requests in the same group).
+type Tx struct {
+	ctx      context.Context
+	s        *Store
+	g        *group
+	events   []map[string]string
+	counters map[string]int64
+	onCommit []func()
+	quiet    bool
+}
+
+// OnCommit registers fn to run after the transaction commits, only if this
+// request succeeded. Hooks run in the writer goroutine in commit order and
+// must not block or call Update.
+func (t *Tx) OnCommit(fn func()) { t.onCommit = append(t.onCommit, fn) }
+
+// Quiet stops this request's changes from publishing events (a bulk rebuild
+// whose consumers are told to take a snapshot instead). Counters and the
+// OnChange hook still run.
+func (t *Tx) Quiet() { t.quiet = true }
+
+// Query runs a raw multi-row query in the transaction.
+func (t *Tx) Query(query string, args ...any) (*sql.Rows, error) {
+	return t.sqlTx().QueryContext(t.ctx, query, args...)
+}
+
+func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
+
+// stmt binds one of the store's prepared statements to the transaction.
+func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt {
+	if t.g.stmts == nil {
+		t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5)
+	}
+	bound := t.g.stmts[st]
+	if bound == nil {
+		bound = t.g.tx.StmtContext(t.ctx, st)
+		t.g.stmts[st] = bound
+	}
+	return bound
+}
+
+// Context is the context the transaction runs under.
+func (t *Tx) Context() context.Context { return t.ctx }
+
+func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) {
+	var v string
+	err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, t.sqlTx(), keys)
+}
+func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, t.sqlTx(), prefix, after, limit)
+}
+func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, t.sqlTx(), prefix)
+}
+
+// Exec runs a raw statement in the transaction — for the caller's derived
+// tables (FTS), never for kv rows (which would skip events).
+func (t *Tx) Exec(query string, args ...any) (sql.Result, error) {
+	return t.sqlTx().ExecContext(t.ctx, query, args...)
+}
+
+// QueryRow runs a raw single-row query in the transaction.
+func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
+	return t.sqlTx().QueryRowContext(t.ctx, query, args...)
+}
+
+// Put sets k = v. A no-op — no row write, no index churn, no event — when k
+// already holds v. (Redis keyspace notifications published `set` even for an
+// unchanged value; with meta-core persisting the same gateway hits again on
+// every repeated query, those no-op writes were most of the write load.)
+func (t *Tx) Put(k, v string) error {
+	_, err := t.Set(k, v)
+	return err
+}
+
+// Set is Put that reports whether the value changed.
+func (t *Tx) Set(k, v string) (bool, error) {
+	old, had, err := t.Get(t.ctx, k)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	if had && old == v {
+		t.s.noops.Add(1)
+		return false, nil
+	}
+	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	t.s.puts.Add(1)
+	t.changed("set", k, old, had, v, true)
+	return true, nil
+}
+
+// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
+func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
+	res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	n, _ := res.RowsAffected()
+	if n > 0 {
+		t.changed("set", k, "", false, v, true)
+	}
+	return n > 0, nil
+}
+
+// Delete removes k. Reports whether it existed; publishes `del` only then,
+// as Redis did.
+func (t *Tx) Delete(k string) (bool, error) {
+	var old string
+	err := t.stmt(t.s.stDelete).QueryRowContext(t.ctx, k).Scan(&old)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("delete %q: %w", k, err)
+	}
+	t.changed("del", k, old, true, "", false)
+	return true, nil
+}
+
+// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
+// returns how many went. Bulk callers loop over separate Updates with a limit
+// so one transaction never grows to millions of rows.
+func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
+	end := PrefixEnd(prefix)
+	sel := `SELECT k FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end != "" {
+		sel += ` AND k < ?`
+		args = append(args, end)
+	}
+	if limit > 0 {
+		sel += ` ORDER BY k LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k, v`, args...)
+	if err != nil {
+		return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
+	}
+	var gone []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			rows.Close()
+			return 0, err
+		}
+		gone = append(gone, kv)
+	}
+	if err := rows.Close(); err != nil {
+		return 0, err
+	}
+	for _, kv := range gone {
+		t.changed("del", kv.K, kv.V, true, "", false)
+	}
+	return int64(len(gone)), nil
+}
+
+// changed records one real change: the event (when in scope) and the hook.
+func (t *Tx) changed(op, k, old string, hadOld bool, v string, hasNew bool) {
+	t.event(op, k)
+	if t.s.opts.OnChange != nil {
+		t.s.opts.OnChange(t, k, old, hadOld, v, hasNew)
+	}
+}
+
+// AddCounter adds delta to the integer counter stored at key. Deltas are
+// summed in memory and written once per commit (a hot counter touched by every
+// record in a group costs one row write, not one per record); they are
+// discarded with the request if it fails. A counter that reaches zero is
+// deleted.
+func (t *Tx) AddCounter(key string, delta int64) {
+	if delta == 0 {
+		return
+	}
+	if t.counters == nil {
+		t.counters = make(map[string]int64)
+	}
+	t.counters[key] += delta
+}
+
+func (g *group) mergeCounters(c map[string]int64) {
+	if len(c) == 0 {
+		return
+	}
+	if g.counters == nil {
+		g.counters = make(map[string]int64, len(c))
+	}
+	for k, d := range c {
+		g.counters[k] += d
+	}
+}
+
+// flushCounters applies the group's counter deltas, bypassing Put (counters
+// are bookkeeping: no events, no hook).
+func (t *Tx) flushCounters() error {
+	for k, d := range t.g.counters {
+		if d == 0 {
+			continue
+		}
+		cur, _, err := t.Get(t.ctx, k)
+		if err != nil {
+			return err
+		}
+		n, _ := strconv.ParseInt(cur, 10, 64)
+		n += d
+		if n <= 0 {
+			if _, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k = ?`, k); err != nil {
+				return fmt.Errorf("counter %s: %w", k, err)
+			}
+			continue
+		}
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, strconv.FormatInt(n, 10)); err != nil {
+			return fmt.Errorf("counter %s: %w", k, err)
+		}
+	}
+	return nil
+}
+
+// Counter reads a committed counter (0 when absent).
+func (s *Store) Counter(ctx context.Context, key string) (int64, bool, error) {
+	v, ok, err := s.Get(ctx, key)
+	if err != nil || !ok {
+		return 0, ok, err
+	}
+	n, err := strconv.ParseInt(v, 10, 64)
+	return n, true, err
+}
+
+// WriteStats reports how many Puts changed a value and how many found it
+// already in place, since open.
+func (s *Store) WriteStats() (changed, unchanged int64) {
+	return s.puts.Load(), s.noops.Load()
+}
+
+func (t *Tx) event(op, k string) {
+	if t.quiet || t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) {
+		return
+	}
+	t.events = append(t.events, map[string]string{
+		"type": op,
+		"key":  k,
+		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
+	})
+}
+
+// flushEvents appends the queued change events to the event stream, inside
+// the transaction, right before commit.
+func (t *Tx) flushEvents() error {
+	if len(t.events) == 0 {
+		return nil
+	}
+	evs := t.events
+	t.events = nil
+	for _, e := range evs {
+		if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/kv/kv_test.go

@@ -0,0 +1,362 @@
+package kv
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func openTest(t *testing.T, path string) *Store {
+	t.Helper()
+	if path == "" {
+		path = filepath.Join(t.TempDir(), "kv.db")
+	}
+	s, err := Open(path, Options{
+		EventStream: "meta:events",
+		EventMaxLen: 100_000,
+		EventKey:    func(k string) bool { return strings.HasPrefix(k, "file:") },
+	})
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func update(t *testing.T, s *Store, fn func(tx *Tx) error) {
+	t.Helper()
+	if err := s.Update(context.Background(), fn); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+}
+
+func events(t *testing.T, s *Store) []Entry {
+	t.Helper()
+	es, err := s.ReadAfter(context.Background(), "meta:events", "0", 0)
+	if err != nil {
+		t.Fatalf("read events: %v", err)
+	}
+	return es
+}
+
+// The replacement for keyspace notifications: a change and its event commit
+// together. `set` only for a real change (an unchanged value is a no-op: no
+// write, no event), `del` only for a key that existed, nothing for keys outside
+// the event scope.
+func TestChangesPublishEventsWithRedisSemantics(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		_ = tx.Put("file:A/title", "x") // unchanged: no write, no event
+		_ = tx.Put("cid:bafk", "A")     // outside the event scope
+		_, _ = tx.Delete("file:A/missing")
+		_, _ = tx.Delete("file:A/title")
+		return nil
+	})
+	got := events(t, s)
+	want := []string{"set file:A/title", "del file:A/title"}
+	if len(got) != len(want) {
+		t.Fatalf("events = %+v, want %v", got, want)
+	}
+	for i, e := range got {
+		if e.Values["type"]+" "+e.Values["key"] != want[i] {
+			t.Errorf("event %d = %v, want %q", i, e.Values, want[i])
+		}
+	}
+}
+
+func TestRolledBackChangesPublishNothing(t *testing.T) {
+	s := openTest(t, "")
+	err := s.Update(context.Background(), func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		return errors.New("boom")
+	})
+	if err == nil {
+		t.Fatal("want the error back")
+	}
+	if _, ok, _ := s.Get(context.Background(), "file:A/title"); ok {
+		t.Fatal("rolled-back write is visible")
+	}
+	if es := events(t, s); len(es) != 0 {
+		t.Fatalf("rolled-back write published %+v", es)
+	}
+}
+
+func TestDeletePrefixPublishesOneDelPerKey(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b", "file:AB/c"} {
+			_ = tx.Put(k, "1")
+		}
+		return nil
+	})
+	var n int64
+	update(t, s, func(tx *Tx) error {
+		var err error
+		n, err = tx.DeletePrefix("file:A/", 0)
+		return err
+	})
+	if n != 2 {
+		t.Fatalf("deleted %d, want 2 (file:AB/ is not under file:A/)", n)
+	}
+	es := events(t, s)
+	if dels := es[3:]; len(dels) != 2 || dels[0].Values["type"] != "del" || dels[1].Values["type"] != "del" {
+		t.Fatalf("want two del events, got %+v", es)
+	}
+}
+
+// IDs are Redis-shaped and strictly increasing, within one millisecond and
+// across a reopen — the SSE cursors consumers persist depend on both.
+func TestStreamIDsAreMonotonicAcrossReopen(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "kv.db")
+	s, err := Open(path, Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ids []string
+	if err := s.Update(context.Background(), func(tx *Tx) error {
+		for i := 0; i < 50; i++ {
+			id, err := tx.Append("file:events", 0, map[string]string{"n": "x"})
+			if err != nil {
+				return err
+			}
+			ids = append(ids, id)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	_ = s.Close()
+
+	s2 := openTest(t, path)
+	var after string
+	update(t, s2, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"n": "y"})
+		return err
+	})
+	ids = append(ids, after)
+	for i := 1; i < len(ids); i++ {
+		if !idLess(t, ids[i-1], ids[i]) {
+			t.Fatalf("id %d (%s) does not follow %s", i, ids[i], ids[i-1])
+		}
+	}
+	if !strings.Contains(ids[0], "-") || strings.HasPrefix(ids[0], "0") {
+		t.Fatalf("id %q is not the unpadded <ms>-<seq> form", ids[0])
+	}
+}
+
+func idLess(t *testing.T, a, b string) bool {
+	t.Helper()
+	am, as, err := ParseID(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bm, bs, err := ParseID(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return am < bm || (am == bm && as < bs)
+}
+
+// The watcher's reset clears file:events. The next ID must still exceed every
+// ID issued before the clear, or a consumer holding an old cursor would skip
+// the reset event.
+func TestClearStreamKeepsTheHighWaterMark(t *testing.T) {
+	s := openTest(t, "")
+	var before, after string
+	update(t, s, func(tx *Tx) error {
+		var err error
+		for i := 0; i < 5; i++ {
+			if before, err = tx.Append("file:events", 0, map[string]string{"type": "add"}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	update(t, s, func(tx *Tx) error { return tx.ClearStream("file:events") })
+	if n, _ := s.Len(context.Background(), "file:events"); n != 0 {
+		t.Fatalf("len after clear = %d", n)
+	}
+	update(t, s, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"type": "reset"})
+		return err
+	})
+	if !idLess(t, before, after) {
+		t.Fatalf("post-clear id %s does not follow pre-clear %s", after, before)
+	}
+	got, err := s.ReadAfter(context.Background(), "file:events", before, 10)
+	if err != nil || len(got) != 1 || got[0].Values["type"] != "reset" {
+		t.Fatalf("an old cursor must see the reset: %+v err=%v", got, err)
+	}
+}
+
+// MAXLEN ~ semantics across many small transactions (the common case: one
+// event per write).
+func TestStreamTrimsAcrossTransactions(t *testing.T) {
+	defer func(old int64) { trimEvery = old }(trimEvery)
+	trimEvery = 100
+	s := openTest(t, "")
+	const maxLen = 50
+	for i := int64(0); i < 3*trimEvery; i++ {
+		update(t, s, func(tx *Tx) error {
+			_, err := tx.Append("x", maxLen, map[string]string{"i": "1"})
+			return err
+		})
+	}
+	n, err := s.Len(context.Background(), "x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n > maxLen+trimEvery {
+		t.Fatalf("stream holds %d entries, want <= %d", n, maxLen+trimEvery)
+	}
+	oldest, ok, err := s.Oldest(context.Background(), "x")
+	if err != nil || !ok {
+		t.Fatal(err)
+	}
+	if last, _ := s.LastID(context.Background(), "x"); !idLess(t, oldest.ID, last) {
+		t.Fatalf("oldest %s should precede last %s", oldest.ID, last)
+	}
+}
+
+// A subscriber that took the wake channel before reading is woken by the next
+// commit — the property the SSE pump relies on to not miss an append.
+func TestWaitIsClosedByTheNextCommit(t *testing.T) {
+	s := openTest(t, "")
+	wake := s.Wait("meta:events")
+	update(t, s, func(tx *Tx) error { return tx.Put("file:A/x", "1") })
+	select {
+	case <-wake:
+	case <-time.After(2 * time.Second):
+		t.Fatal("commit did not wake the waiter")
+	}
+}
+
+func TestReadAfterZeroFormsReadFromTheStart(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_, err := tx.Append("x", 0, map[string]string{"n": "1"})
+		return err
+	})
+	for _, id := range []string{"", "0", "0-0"} {
+		got, err := s.ReadAfter(context.Background(), "x", id, 10)
+		if err != nil || len(got) != 1 {
+			t.Errorf("ReadAfter(%q) = %d entries, err %v; want 1", id, len(got), err)
+		}
+	}
+}
+
+func TestRangeIsExactlyThePrefix(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b/c", "file:A0", "file:AB/x", "file:A"} {
+			_ = tx.Put(k, "v")
+		}
+		return nil
+	})
+	rows, err := s.Range(context.Background(), "file:A/", "", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 || rows[0].K != "file:A/a" || rows[1].K != "file:A/b/c" {
+		t.Fatalf("range = %+v", rows)
+	}
+}
+
+// Concurrent Updates share commits; a failing one must roll back alone and
+// leave its neighbours' writes and events intact.
+func TestGroupCommitIsolatesFailures(t *testing.T) {
+	s := openTest(t, "")
+	const n = 200
+	errs := make(chan error, n)
+	for i := 0; i < n; i++ {
+		go func(i int) {
+			errs <- s.Update(context.Background(), func(tx *Tx) error {
+				k := fmt.Sprintf("file:R%03d/title", i)
+				if err := tx.Put(k, "x"); err != nil {
+					return err
+				}
+				if i%10 == 0 {
+					return errors.New("fail")
+				}
+				return nil
+			})
+		}(i)
+	}
+	failed := 0
+	for i := 0; i < n; i++ {
+		if <-errs != nil {
+			failed++
+		}
+	}
+	if failed != n/10 {
+		t.Fatalf("failed = %d, want %d", failed, n/10)
+	}
+	rows, _ := s.Range(context.Background(), "file:", "", 0)
+	if len(rows) != n-n/10 {
+		t.Fatalf("committed %d rows, want %d", len(rows), n-n/10)
+	}
+	if es := events(t, s); len(es) != n-n/10 {
+		t.Fatalf("events = %d, want %d (one per committed write, none for failures)", len(es), n-n/10)
+	}
+	for _, r := range rows {
+		var i int
+		fmt.Sscanf(r.K, "file:R%03d/title", &i)
+		if i%10 == 0 {
+			t.Fatalf("failed request's write %s committed", r.K)
+		}
+	}
+}
+
+// The change hook sees real changes only, with old and new values, and
+// counters follow the request's fate (a failed request's deltas are dropped).
+func TestOnChangeAndCounters(t *testing.T) {
+	type change struct{ k, old, new string }
+	var seen []change
+	s, err := Open(filepath.Join(t.TempDir(), "kv.db"), Options{
+		OnChange: func(tx *Tx, k, old string, hadOld bool, v string, hasNew bool) {
+			seen = append(seen, change{k, old, v})
+			if hasNew {
+				tx.AddCounter("count/"+v, 1)
+			}
+			if hadOld {
+				tx.AddCounter("count/"+old, -1)
+			}
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	update(t, s, func(tx *Tx) error {
+		_ = tx.Put("a", "x")
+		_ = tx.Put("b", "x")
+		_ = tx.Put("a", "x") // no-op
+		_ = tx.Put("a", "y")
+		return nil
+	})
+	_ = s.Update(context.Background(), func(tx *Tx) error {
+		_ = tx.Put("c", "x")
+		return errors.New("fail")
+	})
+	update(t, s, func(tx *Tx) error { _, err := tx.DeletePrefix("b", 0); return err })
+
+	if len(seen) != 5 { // a=x, b=x, a x→y, c=x (rolled back, hook still ran), del b; stats count attempts
+		t.Fatalf("changes = %+v", seen)
+	}
+	for k, want := range map[string]int64{"count/x": 0, "count/y": 1} {
+		got, _, err := s.Counter(context.Background(), k)
+		if err != nil || got != want {
+			t.Errorf("%s = %d (err %v), want %d", k, got, err, want)
+		}
+	}
+	if changed, unchanged := s.WriteStats(); changed != 4 || unchanged != 1 {
+		t.Errorf("write stats = %d changed / %d unchanged", changed, unchanged)
+	}
+}

internal/kv/stream.go

@@ -0,0 +1,356 @@
+package kv
+
+// Append-only event streams, as rows of the kv table.
+//
+// A stream entry is the row
+//
+//	ev:<stream>/<ms:13>-<seq:10>  →  JSON object of the entry's string fields
+//
+// Zero padding makes key order equal ID order, so reading "after an ID" is a
+// plain range read and trimming is a range delete. On the wire IDs are the
+// unpadded Redis form (`1700000000000-0`): meta-sort, meta-share and
+// meta-stremio persist SSE cursors to disk across the Redis → SQLite switch,
+// and those compare as <ms>-<seq> pairs.
+//
+// The last issued ID is kept in `evhwm:<stream>` and updated in the same
+// transaction as the append. That high-water mark, not the newest surviving
+// row, seeds the next ID — so Clear (the watcher's `reset`) never reissues an
+// ID an old cursor would treat as already seen.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Entry is one stream entry.
+type Entry struct {
+	ID     string
+	Values map[string]string
+}
+
+type streamID struct{ ms, seq int64 }
+
+func (id streamID) String() string {
+	return strconv.FormatInt(id.ms, 10) + "-" + strconv.FormatInt(id.seq, 10)
+}
+
+func (id streamID) padded() string { return fmt.Sprintf("%013d-%010d", id.ms, id.seq) }
+
+func (id streamID) less(o streamID) bool {
+	return id.ms < o.ms || (id.ms == o.ms && id.seq < o.seq)
+}
+
+// ParseID parses a Redis-style stream ID. "", "0", "0-0", "-" all parse as the
+// zero ID (before everything). A bare "<ms>" means "<ms>-0".
+func ParseID(s string) (ms, seq int64, err error) {
+	if s == "" || s == "-" || s == "0" {
+		return 0, 0, nil
+	}
+	msPart, seqPart, hasSeq := strings.Cut(s, "-")
+	ms, err = strconv.ParseInt(msPart, 10, 64)
+	if err != nil || ms < 0 {
+		return 0, 0, fmt.Errorf("invalid stream id %q", s)
+	}
+	if hasSeq {
+		seq, err = strconv.ParseInt(seqPart, 10, 64)
+		if err != nil || seq < 0 {
+			return 0, 0, fmt.Errorf("invalid stream id %q", s)
+		}
+	}
+	return ms, seq, nil
+}
+
+func streamPrefix(stream string) string { return "ev:" + stream + "/" }
+func hwmKey(stream string) string       { return "evhwm:" + stream }
+
+// streamState is the in-memory view of a stream's high-water mark plus the
+// broadcast channel subscribers wait on. Updated only after a commit.
+type streamState struct {
+	loaded    bool
+	hwm       streamID
+	sinceTrim int64 // committed appends since the last trim
+	notify    chan struct{}
+}
+
+// streamPending is what a transaction did to one stream, applied to
+// streamState on commit (and discarded on rollback).
+type streamPending struct {
+	hwm        streamID
+	appends    int64
+	trimmed    bool
+	lastTrimAt int64 // value of appends when this transaction last trimmed
+}
+
+func (s *Store) state(stream string) *streamState {
+	st := s.streams[stream]
+	if st == nil {
+		st = &streamState{notify: make(chan struct{})}
+		s.streams[stream] = st
+	}
+	return st
+}
+
+// Wait returns a channel closed by the next commit that appends to stream.
+// Take it BEFORE reading, so an append between the read and the wait is not
+// missed.
+func (s *Store) Wait(stream string) <-chan struct{} {
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	return s.state(stream).notify
+}
+
+func (s *Store) afterCommit(pending map[string]*streamPending) {
+	if len(pending) == 0 {
+		return
+	}
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	for name, p := range pending {
+		st := s.state(name)
+		st.loaded = true
+		st.hwm = p.hwm
+		if p.trimmed {
+			st.sinceTrim = p.appends - p.lastTrimAt
+		} else {
+			st.sinceTrim += p.appends
+		}
+		close(st.notify)
+		st.notify = make(chan struct{})
+	}
+}
+
+// hwm returns the stream's last issued ID as seen by this transaction.
+func (t *Tx) hwm(stream string) (streamID, error) {
+	if p := t.g.pending[stream]; p != nil {
+		return p.hwm, nil
+	}
+	t.s.streamMu.Lock()
+	st := t.s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	t.s.streamMu.Unlock()
+	if loaded {
+		return hwm, nil
+	}
+	v, ok, err := t.Get(t.ctx, hwmKey(stream))
+	if err != nil {
+		return streamID{}, err
+	}
+	if !ok {
+		return streamID{}, nil
+	}
+	ms, seq, err := ParseID(v)
+	if err != nil {
+		return streamID{}, err
+	}
+	return streamID{ms, seq}, nil
+}
+
+// Append adds an entry to stream and returns its ID. The stream is trimmed to
+// about maxLen entries (maxLen <= 0: unbounded).
+func (t *Tx) Append(stream string, maxLen int64, fields map[string]string) (string, error) {
+	last, err := t.hwm(stream)
+	if err != nil {
+		return "", fmt.Errorf("stream %s hwm: %w", stream, err)
+	}
+	id := streamID{ms: time.Now().UnixMilli()}
+	if !last.less(id) {
+		id = streamID{ms: last.ms, seq: last.seq + 1}
+	}
+	body, err := json.Marshal(fields)
+	if err != nil {
+		return "", err
+	}
+	if _, err := t.stmt(t.s.stInsert).ExecContext(t.ctx, streamPrefix(stream)+id.padded(), string(body)); err != nil {
+		return "", fmt.Errorf("append %s: %w", stream, err)
+	}
+	if t.g.pending == nil {
+		t.g.pending = make(map[string]*streamPending)
+	}
+	p := t.g.pending[stream]
+	if p == nil {
+		p = &streamPending{}
+		t.g.pending[stream] = p
+	}
+	p.hwm = id
+	p.appends++
+	// Trim in steps, not per append: MAXLEN ~ semantics, like the XADD it
+	// replaces. The count spans transactions (most append one event each).
+	if maxLen > 0 {
+		t.s.streamMu.Lock()
+		since := t.s.state(stream).sinceTrim
+		t.s.streamMu.Unlock()
+		if !p.trimmed {
+			since += p.appends
+		} else {
+			since = p.appends - p.lastTrimAt
+		}
+		if since >= trimEvery {
+			if err := t.trim(stream, maxLen); err != nil {
+				return "", err
+			}
+			p.trimmed, p.lastTrimAt = true, p.appends
+		}
+	}
+	return id.String(), nil
+}
+
+// trimEvery is how many appends pass between trims. A trim locates its cut
+// point by walking maxLen entries back from the newest, so it is kept rare;
+// the cost is overshooting maxLen by at most trimEvery entries.
+var trimEvery int64 = 10_000 // var so tests can lower it
+
+func (t *Tx) trim(stream string, maxLen int64) error {
+	prefix := streamPrefix(stream)
+	var cut string
+	err := t.sqlTx().QueryRowContext(t.ctx,
+		`SELECT k FROM kv WHERE k >= ? AND k < ? ORDER BY k DESC LIMIT 1 OFFSET ?`,
+		prefix, PrefixEnd(prefix), maxLen).Scan(&cut)
+	if err != nil {
+		return nil // fewer than maxLen entries (sql.ErrNoRows): nothing to trim
+	}
+	_, err = t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k <= ?`, prefix, cut)
+	return err
+}
+
+// flushHWM persists each touched stream's high-water mark — once per
+// transaction, not per append.
+func (t *Tx) flushHWM() error {
+	for stream, p := range t.g.pending {
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, hwmKey(stream), p.hwm.String()); err != nil {
+			return fmt.Errorf("stream %s hwm: %w", stream, err)
+		}
+	}
+	return nil
+}
+
+// TrimAll trims stream to maxLen inside the transaction regardless of how many
+// appends this transaction made. Used after an append burst split over many
+// short transactions.
+func (t *Tx) TrimAll(stream string, maxLen int64) error { return t.trim(stream, maxLen) }
+
+// ClearStream deletes every entry of stream, keeping its high-water mark.
+func (t *Tx) ClearStream(stream string) error {
+	prefix := streamPrefix(stream)
+	_, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k < ?`, prefix, PrefixEnd(prefix))
+	return err
+}
+
+func decodeEntry(stream, k, v string) (Entry, error) {
+	padded := strings.TrimPrefix(k, streamPrefix(stream))
+	ms, seq, err := ParseID(padded)
+	if err != nil {
+		return Entry{}, err
+	}
+	var values map[string]string
+	if err := json.Unmarshal([]byte(v), &values); err != nil {
+		return Entry{}, fmt.Errorf("decode %s: %w", k, err)
+	}
+	return Entry{ID: streamID{ms, seq}.String(), Values: values}, nil
+}
+
+func decodeEntries(stream string, rows []KV) ([]Entry, error) {
+	out := make([]Entry, 0, len(rows))
+	for _, kv := range rows {
+		e, err := decodeEntry(stream, kv.K, kv.V)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, e)
+	}
+	return out, nil
+}
+
+// ReadAfter returns up to n entries strictly after id, oldest first. "0",
+// "0-0" and "" read from the beginning.
+func (s *Store) ReadAfter(ctx context.Context, stream, id string, n int) ([]Entry, error) {
+	ms, seq, err := ParseID(id)
+	if err != nil {
+		return nil, err
+	}
+	prefix := streamPrefix(stream)
+	after := ""
+	if ms != 0 || seq != 0 {
+		after = prefix + streamID{ms, seq}.padded()
+	}
+	rows, err := rangeQuery(ctx, s.r, prefix, after, n)
+	if err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, rows)
+}
+
+// Oldest returns the oldest surviving entry, or ok=false for an empty stream.
+func (s *Store) Oldest(ctx context.Context, stream string) (Entry, bool, error) {
+	rows, err := rangeQuery(ctx, s.r, streamPrefix(stream), "", 1)
+	if err != nil || len(rows) == 0 {
+		return Entry{}, false, err
+	}
+	e, err := decodeEntry(stream, rows[0].K, rows[0].V)
+	return e, err == nil, err
+}
+
+// LastID returns the last ID ever issued on stream ("0-0" if none) — the
+// cursor meaning "only entries appended from now on".
+func (s *Store) LastID(ctx context.Context, stream string) (string, error) {
+	s.streamMu.Lock()
+	st := s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	s.streamMu.Unlock()
+	if loaded {
+		return hwm.String(), nil
+	}
+	v, ok, err := s.Get(ctx, hwmKey(stream))
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "0-0", nil
+	}
+	return v, nil
+}
+
+// Len is the number of surviving entries.
+func (s *Store) Len(ctx context.Context, stream string) (int64, error) {
+	return count(ctx, s.r, streamPrefix(stream))
+}
+
+// RevRange returns up to n entries newest first, stopping at (excluding)
+// entries at or before afterID when afterID is non-empty.
+func (s *Store) RevRange(ctx context.Context, stream, afterID string, n int) ([]Entry, error) {
+	prefix := streamPrefix(stream)
+	query := `SELECT k, v FROM kv WHERE k > ? AND k < ? ORDER BY k DESC`
+	lo := prefix
+	if afterID != "" {
+		ms, seq, err := ParseID(afterID)
+		if err != nil {
+			return nil, err
+		}
+		lo = prefix + streamID{ms, seq}.padded()
+	}
+	args := []any{lo, PrefixEnd(prefix)}
+	if n > 0 {
+		query += ` LIMIT ?`
+		args = append(args, n)
+	}
+	rows, err := s.r.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var kvs []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		kvs = append(kvs, kv)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, kvs)
+}

internal/leader/election.go

@@ -13,7 +13,7 @@ import (
 type LeaderLockInfo struct {
 	Hostname          string `json:"hostname"`
 	BaseUrl           string `json:"baseUrl"`
-	ApiUrl            string `json:"apiUrl"`            // meta-core API (port 9000)
+	ApiUrl            string `json:"apiUrl"` // meta-core API (port 9000)
 	RedisUrl          string `json:"redisUrl"`
 	WebdavUrl         string `json:"webdavUrl"`         // External WebDAV URL (via baseUrl)
 	WebdavUrlInternal string `json:"webdavUrlInternal"` // Internal WebDAV URL (direct to port 9000)
@@ -72,9 +72,9 @@ func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo {
 	webdavUrlInternal := fmt.Sprintf("http://%s:%d/webdav", hostname, p.config.HTTPPort)
 
 	return &LeaderLockInfo{
-		Hostname:          hostname,
+		Hostname: hostname,
-		BaseUrl:           baseUrl,
+		BaseUrl:  baseUrl,
-		ApiUrl:            fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
+		ApiUrl:   fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
 		// RedisUrl deliberately empty — api-mediated-access PR D removes
 		// direct Redis exposure. Consumers route metadata I/O through
 		// meta-core's HTTP API (/meta/{hash}*) and event streams through

internal/meshdisco/node.go

@@ -0,0 +1,439 @@
+package meshdisco
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"sort"
+	"sync"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/ipv4"
+	"golang.org/x/sys/unix"
+)
+
+// Config configures a Node. Payload is a function rather than a value so the
+// advertise is rebuilt from live state on every tick — that is what keeps the
+// advertised core URLs and GET /urls from drifting apart.
+type Config struct {
+	Name     string
+	Instance string
+	Version  string
+
+	Group    string
+	Port     int
+	Interval time.Duration
+
+	// Payload returns this node's status ("starting" / "running") and its
+	// resources. The envelope and NodeInfo are filled in by the Node.
+	Payload func() (status string, resources []Resource)
+}
+
+func (c *Config) applyDefaults() {
+	if c.Group == "" {
+		c.Group = DefaultGroup
+	}
+	if c.Port == 0 {
+		c.Port = DefaultPort
+	}
+	if c.Interval <= 0 {
+		c.Interval = DefaultInterval
+	}
+}
+
+type seen struct {
+	node      NodeInfo
+	resources []Resource
+	addr      string
+	lastSeen  time.Time
+}
+
+// Node is one beacon v2 participant: it advertises its resources, answers
+// probes, and keeps a TTL map of every other node it has heard from.
+type Node struct {
+	cfg   Config
+	group net.IP
+
+	pc     *ipv4.PacketConn
+	closer interface{ Close() error }
+
+	// ifaces are the multicast-capable interfaces we joined and send on.
+	// A container attached to two docker networks (metashare-app on `pcs`
+	// and `metamesh-mesh`, metawatch-* on `metawatch` and `metamesh-mesh`)
+	// has two entries here, and MUST be advertised on both — sending via the
+	// default route alone silently covers only one network.
+	ifaces []net.Interface
+
+	// sendMu serialises SetMulticastInterface + WriteTo, which are a
+	// two-step operation on one shared socket.
+	sendMu sync.Mutex
+
+	mu    sync.RWMutex
+	nodes map[string]*seen
+
+	stopChan chan struct{}
+	stopOnce sync.Once
+	wg       sync.WaitGroup
+}
+
+// New creates a Node. It does not touch the network until Start.
+func New(cfg Config) *Node {
+	cfg.applyDefaults()
+	return &Node{
+		cfg:      cfg,
+		nodes:    make(map[string]*seen),
+		stopChan: make(chan struct{}),
+	}
+}
+
+// Start opens the multicast socket, joins the group on every eligible
+// interface, and begins advertising. A join failure on one interface is logged
+// and skipped rather than fatal — a host that blocks multicast degrades
+// instead of failing to boot.
+func (n *Node) Start() error {
+	n.group = net.ParseIP(n.cfg.Group)
+	if n.group == nil || n.group.To4() == nil {
+		return fmt.Errorf("beacon: invalid multicast group %q", n.cfg.Group)
+	}
+
+	lc := net.ListenConfig{Control: reusePort}
+	conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", n.cfg.Port))
+	if err != nil {
+		return fmt.Errorf("beacon: listen on :%d: %w", n.cfg.Port, err)
+	}
+	n.closer = conn
+
+	pc := ipv4.NewPacketConn(conn)
+	// TTL 1: link-local only. Discovery never routes off the local segment;
+	// the two-host topology keeps two separate meshes by design.
+	if err := pc.SetMulticastTTL(1); err != nil {
+		log.Printf("[beacon] Warning: could not set multicast TTL: %v", err)
+	}
+	n.pc = pc
+
+	n.ifaces = eligibleInterfaces()
+	if len(n.ifaces) == 0 {
+		log.Println("[beacon] Warning: no multicast-capable interfaces found; discovery will be inert")
+	}
+
+	group := &net.UDPAddr{IP: n.group}
+	joined := 0
+	for i := range n.ifaces {
+		if err := pc.JoinGroup(&n.ifaces[i], group); err != nil {
+			log.Printf("[beacon] Warning: join %s on %s failed: %v", n.cfg.Group, n.ifaces[i].Name, err)
+			continue
+		}
+		joined++
+	}
+	log.Printf("[beacon] Listening on %s:%d as %s/%s (joined %d of %d interfaces)",
+		n.cfg.Group, n.cfg.Port, n.cfg.Name, n.cfg.Instance, joined, len(n.ifaces))
+
+	n.wg.Add(2)
+	go n.readLoop()
+	go n.advertiseLoop()
+
+	return nil
+}
+
+// Stop multicasts a bye — so neighbours drop us immediately instead of
+// waiting out the staleness window — then closes the socket and waits for the
+// loops to exit.
+func (n *Node) Stop() error {
+	var err error
+	n.stopOnce.Do(func() {
+		n.broadcast(Message{Proto: Proto, V: Version, Type: TypeBye, Node: n.nodeInfo("")})
+		close(n.stopChan)
+		if n.closer != nil {
+			err = n.closer.Close()
+		}
+		n.wg.Wait()
+	})
+	return err
+}
+
+// Probe multicasts a probe on every interface. Every node owning a resource
+// matching one of want (every node, when want is empty) replies immediately,
+// so a caller does not have to wait out an interval.
+func (n *Node) Probe(want ...string) {
+	n.broadcast(Message{Proto: Proto, V: Version, Type: TypeProbe, From: n.cfg.Instance, Want: want})
+}
+
+// Neighbors returns every node heard from within the staleness window,
+// sorted by (name, instance). Expired entries are dropped as a side effect —
+// expiry is a read-time concern, so there is no reaper goroutine.
+func (n *Node) Neighbors() []Neighbor {
+	cutoff := time.Now().Add(-n.cfg.Interval * LivenessFactor)
+
+	n.mu.Lock()
+	out := make([]Neighbor, 0, len(n.nodes))
+	for k, s := range n.nodes {
+		if s.lastSeen.Before(cutoff) {
+			delete(n.nodes, k)
+			continue
+		}
+		out = append(out, newNeighbor(s.node, s.resources, s.addr, s.lastSeen))
+	}
+	n.mu.Unlock()
+
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		return out[i].Instance < out[j].Instance
+	})
+	return out
+}
+
+// NeighborsByName collapses multiple instances of one name down to the most
+// recently seen — the shape the nav menu wants.
+func (n *Node) NeighborsByName() []Neighbor {
+	best := make(map[string]Neighbor)
+	for _, nb := range n.Neighbors() {
+		if cur, ok := best[nb.Name]; !ok || nb.LastSeen > cur.LastSeen {
+			best[nb.Name] = nb
+		}
+	}
+	out := make([]Neighbor, 0, len(best))
+	for _, nb := range best {
+		out = append(out, nb)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Self returns this node as a row, so callers can render themselves without
+// waiting to hear their own multicast echo back.
+func (n *Node) Self() Neighbor {
+	status, resources := n.payload()
+	return newNeighbor(*n.nodeInfo(status), resources, "", time.Now())
+}
+
+func (n *Node) payload() (string, []Resource) {
+	if n.cfg.Payload == nil {
+		return "running", nil
+	}
+	return n.cfg.Payload()
+}
+
+func (n *Node) nodeInfo(status string) *NodeInfo {
+	return &NodeInfo{Name: n.cfg.Name, Instance: n.cfg.Instance, Version: n.cfg.Version, Status: status}
+}
+
+func (n *Node) advertiseMsg() Message {
+	status, resources := n.payload()
+	return Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: n.nodeInfo(status), Resources: resources}
+}
+
+func (n *Node) readLoop() {
+	defer n.wg.Done()
+	buf := make([]byte, ReadBufferSize)
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		default:
+		}
+
+		nRead, _, src, err := n.pc.ReadFrom(buf)
+		if err != nil {
+			select {
+			case <-n.stopChan:
+				return
+			default:
+			}
+			log.Printf("[beacon] Read error: %v", err)
+			continue
+		}
+
+		msg, ok := Parse(buf[:nRead])
+		if !ok {
+			continue // v1 or foreign traffic on the shared group
+		}
+
+		switch msg.Type {
+		case TypeProbe:
+			if msg.From == n.cfg.Instance {
+				continue
+			}
+			ad := n.advertiseMsg()
+			wanted := len(msg.Want) == 0
+			for _, r := range ad.Resources {
+				if r.MatchesAny(msg.Want) {
+					wanted = true
+					break
+				}
+			}
+			if wanted {
+				// Unicast straight back to the prober's source address; its
+				// ephemeral source port is where it is listening.
+				n.send(ad, src)
+			}
+		case TypeAdvertise, TypeBye:
+			// Drop our own multicast echo. Loopback stays enabled so a
+			// single-container test still sees traffic; we filter by identity.
+			if msg.Node.Instance == n.cfg.Instance {
+				continue
+			}
+			n.record(msg, src)
+		}
+	}
+}
+
+func (n *Node) record(msg Message, src net.Addr) {
+	addr := ""
+	if src != nil {
+		if udp, ok := src.(*net.UDPAddr); ok {
+			addr = udp.IP.String()
+		} else {
+			addr = src.String()
+		}
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if msg.Type == TypeBye {
+		delete(n.nodes, msg.Node.Instance)
+		return
+	}
+	n.nodes[msg.Node.Instance] = &seen{node: *msg.Node, resources: msg.Resources, addr: addr, lastSeen: time.Now()}
+}
+
+func (n *Node) advertiseLoop() {
+	defer n.wg.Done()
+
+	// Advertise immediately, then probe once so the map fills without
+	// waiting a full interval for everyone else's ticker.
+	n.broadcast(n.advertiseMsg())
+	n.Probe()
+
+	ticker := time.NewTicker(n.cfg.Interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		case <-ticker.C:
+			n.broadcast(n.advertiseMsg())
+		}
+	}
+}
+
+func encode(msg Message) ([]byte, bool) {
+	body, err := json.Marshal(msg)
+	if err != nil {
+		return nil, false
+	}
+	if len(body) > MaxDatagram {
+		log.Printf("[beacon] Warning: %d-byte datagram exceeds the fragmentation-safe %d", len(body), MaxDatagram)
+	}
+	return body, true
+}
+
+// send unicasts msg to dst (a probe reply).
+func (n *Node) send(msg Message, dst net.Addr) {
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+		log.Printf("[beacon] Reply to %s failed: %v", dst, err)
+	}
+}
+
+// broadcast writes one message to the group once per eligible interface.
+// Writing once and letting the routing table choose would reach exactly one
+// network, which is the failure mode this protocol exists to avoid.
+func (n *Node) broadcast(msg Message) {
+	if n.pc == nil {
+		return
+	}
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	dst := &net.UDPAddr{IP: n.group, Port: n.cfg.Port}
+
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	for i := range n.ifaces {
+		if err := n.pc.SetMulticastInterface(&n.ifaces[i]); err != nil {
+			log.Printf("[beacon] Warning: select interface %s failed: %v", n.ifaces[i].Name, err)
+			continue
+		}
+		if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+			log.Printf("[beacon] Warning: send on %s failed: %v", n.ifaces[i].Name, err)
+		}
+	}
+}
+
+// eligibleInterfaces returns every up, non-loopback, multicast-capable
+// interface that has an IPv4 address.
+func eligibleInterfaces() []net.Interface {
+	all, err := net.Interfaces()
+	if err != nil {
+		log.Printf("[beacon] Warning: could not enumerate interfaces: %v", err)
+		return nil
+	}
+
+	var out []net.Interface
+	for _, ifi := range all {
+		if ifi.Flags&net.FlagUp == 0 ||
+			ifi.Flags&net.FlagLoopback != 0 ||
+			ifi.Flags&net.FlagMulticast == 0 {
+			continue
+		}
+		addrs, err := ifi.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if ipnet, ok := a.(*net.IPNet); ok && ipnet.IP.To4() != nil {
+				out = append(out, ifi)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// reusePort sets SO_REUSEADDR + SO_REUSEPORT so several listeners (a test, a
+// sidecar, a beacon v1 responder in the same netns) can bind 9099 on one host.
+func reusePort(_, _ string, c syscall.RawConn) error {
+	var serr error
+	err := c.Control(func(fd uintptr) {
+		serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
+		if serr == nil {
+			serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return serr
+}
+
+// LocalIPv4 returns this host's first non-loopback IPv4 address, or the
+// hostname when none can be determined. Same fallback the file-based registry
+// used (discovery/service.go), kept identical so announced URLs don't shift.
+func LocalIPv4() string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, addr := range addrs {
+			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+				if ipnet.IP.To4() != nil {
+					return ipnet.IP.String()
+				}
+			}
+		}
+	}
+	host, _ := os.Hostname()
+	return host
+}

internal/meshdisco/proto.go

@@ -0,0 +1,237 @@
+// Package meshdisco implements beacon v2: local resource advertise / discover
+// over UDP multicast (239.255.99.1:9099), the protocol every meta-* service
+// and SDK-hosted plugin uses to find its neighbours.
+//
+// A node advertises Resources, each tagged with capability strings
+// ("metamesh.core", "metamesh.service/meta-sort", "metamesh.transport/nzb@1");
+// consumers keep a live view of everyone else's and filter by capability.
+// meta-core advertises "metamesh.core" carrying its /urls block, which is how
+// services locate it without a shared volume.
+//
+// The wire protocol is documented in docs/project-architecture/beacon-v2.md.
+// Keep this package and that doc in step — the Rust (meta-feeder-sdk),
+// TypeScript and Python implementations are written against the doc.
+package meshdisco
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+)
+
+const (
+	// Proto and Version form the envelope every v2 message carries. Anything
+	// else on the group (beacon v1, junk) is dropped silently.
+	Proto   = "beacon"
+	Version = 2
+
+	// DefaultGroup / DefaultPort are beacon's endpoint, on purpose: one
+	// well-known place for every kind of local resource. v1 and v2 coexist
+	// because v2 never sends v1's "discovery" / "announce" types.
+	DefaultGroup = "239.255.99.1"
+	DefaultPort  = 9099
+
+	// DefaultInterval is how often an unsolicited advertise is emitted.
+	DefaultInterval = 10 * time.Second
+
+	// LivenessFactor multiplies the interval to get the staleness window.
+	// Expiry is evaluated at read time; there is no reaper.
+	LivenessFactor = 3
+
+	// ReadBufferSize is the receive buffer; MaxDatagram is what a sender
+	// stays under so nothing fragments.
+	ReadBufferSize = 8192
+	MaxDatagram    = 1400
+
+	TypeProbe     = "probe"
+	TypeAdvertise = "advertise"
+	TypeBye       = "bye"
+
+	// CapCore is meta-core's capability; its resource carries Data.urls.
+	CapCore = "metamesh.core"
+	// CapServicePrefix + name is every service's capability.
+	CapServicePrefix = "metamesh.service/"
+	CapAnyService    = "metamesh.service/*"
+)
+
+// URLs mirrors meta-core's GET /urls response (api.URLsResponse) minus
+// redisUrl. It travels as the metamesh.core resource's Data.urls, so a service
+// locates meta-core from the advertise alone.
+type URLs struct {
+	Hostname          string `json:"hostname"`
+	BaseUrl           string `json:"baseUrl"`
+	ApiUrl            string `json:"apiUrl"`
+	WebdavUrl         string `json:"webdavUrl"`
+	WebdavUrlInternal string `json:"webdavUrlInternal"`
+}
+
+// NodeInfo says who is advertising.
+type NodeInfo struct {
+	Name     string `json:"name"`
+	Instance string `json:"instance"`
+	Version  string `json:"version,omitempty"`
+	// Status is "starting" | "running"; absent reads as running.
+	Status string `json:"status,omitempty"`
+}
+
+// Resource is one advertised thing.
+type Resource struct {
+	ID   string   `json:"id"`
+	Caps []string `json:"caps"`
+	// Endpoints: well-known names http, ui, manifest, mcp. A value starting
+	// with "/" is relative to endpoints.http.
+	Endpoints map[string]string `json:"endpoints,omitempty"`
+	Rev       string            `json:"rev,omitempty"`
+	// Binds names the one consumer instance this resource belongs to.
+	Binds string          `json:"binds,omitempty"`
+	Data  json.RawMessage `json:"data,omitempty"`
+}
+
+// Matches reports whether any cap matches pattern.
+func (r Resource) Matches(pattern string) bool {
+	for _, c := range r.Caps {
+		if CapMatches(pattern, c) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchesAny reports whether any cap matches any pattern; an empty list matches.
+func (r Resource) MatchesAny(patterns []string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+	for _, p := range patterns {
+		if r.Matches(p) {
+			return true
+		}
+	}
+	return false
+}
+
+// Endpoint returns the named endpoint as an absolute URL.
+func (r Resource) Endpoint(name string) string {
+	v := r.Endpoints[name]
+	if strings.HasPrefix(v, "/") {
+		base := r.Endpoints["http"]
+		if base == "" {
+			return ""
+		}
+		return strings.TrimRight(base, "/") + v
+	}
+	return v
+}
+
+// Message is every v2 datagram.
+type Message struct {
+	Proto     string     `json:"proto"`
+	V         int        `json:"v"`
+	Type      string     `json:"type"`
+	Node      *NodeInfo  `json:"node,omitempty"`
+	Resources []Resource `json:"resources,omitempty"`
+	// From and Want are probe-only.
+	From string   `json:"from,omitempty"`
+	Want []string `json:"want,omitempty"`
+}
+
+// Parse decodes a datagram, returning ok=false for anything that is not a
+// well-formed beacon v2 message (v1 traffic, other versions, unknown types,
+// an advertise/bye without a node).
+func Parse(b []byte) (Message, bool) {
+	var m Message
+	if err := json.Unmarshal(b, &m); err != nil {
+		return m, false
+	}
+	if m.Proto != Proto || m.V != Version {
+		return m, false
+	}
+	switch m.Type {
+	case TypeProbe:
+		return m, true
+	case TypeAdvertise, TypeBye:
+		if m.Node == nil || m.Node.Name == "" || m.Node.Instance == "" {
+			return m, false
+		}
+		return m, true
+	}
+	return m, false
+}
+
+func splitContract(s string) (string, string, bool) {
+	if i := strings.LastIndex(s, "@"); i >= 0 {
+		return s[:i], s[i+1:], true
+	}
+	return s, "", false
+}
+
+// CapMatches reports whether capability cap satisfies pattern:
+// "*" matches everything; an "@N" on the pattern must equal the cap's
+// contract (none on the pattern ignores it); "x/*" matches any variant of x;
+// otherwise the bases must be equal.
+func CapMatches(pattern, cap string) bool {
+	if pattern == "*" {
+		return true
+	}
+	pbase, pc, phas := splitContract(pattern)
+	cbase, cc, chas := splitContract(cap)
+	if phas && (!chas || pc != cc) {
+		return false
+	}
+	if prefix, ok := strings.CutSuffix(pbase, "/*"); ok {
+		rest, ok := strings.CutPrefix(cbase, prefix)
+		return ok && strings.HasPrefix(rest, "/") && len(rest) > 1
+	}
+	return pbase == cbase
+}
+
+// Neighbor is a node heard on the wire, as served by /api/neighbors.
+type Neighbor struct {
+	NodeInfo
+	Resources []Resource `json:"resources"`
+	// Caps flattens every resource's caps, for consumers that just filter.
+	Caps []string `json:"caps"`
+	// BaseUrl is the metamesh.service/* resource's endpoints.ui — kept so
+	// v1-era readers (the nav menu) keep working.
+	BaseUrl string `json:"baseUrl,omitempty"`
+	// Addr is the packet's source address, never a payload claim.
+	Addr string `json:"addr"`
+	// LastSeen is unix seconds.
+	LastSeen float64 `json:"lastSeen"`
+}
+
+// Matches reports whether any resource matches pattern.
+func (n Neighbor) Matches(pattern string) bool {
+	for _, r := range n.Resources {
+		if r.Matches(pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+func newNeighbor(node NodeInfo, resources []Resource, addr string, seen time.Time) Neighbor {
+	nb := Neighbor{
+		NodeInfo:  node,
+		Resources: resources,
+		Caps:      []string{},
+		Addr:      addr,
+		LastSeen:  float64(seen.UnixNano()) / 1e9,
+	}
+	if nb.Resources == nil {
+		nb.Resources = []Resource{}
+	}
+	seenCap := map[string]bool{}
+	for _, r := range resources {
+		for _, c := range r.Caps {
+			if !seenCap[c] {
+				seenCap[c] = true
+				nb.Caps = append(nb.Caps, c)
+			}
+		}
+		if nb.BaseUrl == "" && r.Matches(CapAnyService) {
+			nb.BaseUrl = r.Endpoint("ui")
+		}
+	}
+	return nb
+}

internal/meshdisco/proto_test.go

@@ -0,0 +1,140 @@
+package meshdisco
+
+import (
+	"encoding/json"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestCapMatches(t *testing.T) {
+	cases := []struct {
+		pattern, cap string
+		want         bool
+	}{
+		{"metamesh.transport/*", "metamesh.transport/nzb@1", true},
+		{"metamesh.transport/*", "metamesh.transport/ipfs", true},
+		{"metamesh.transport/*", "metamesh.transport", false},
+		{"metamesh.transport/*", "metamesh.transportx/a", false},
+		{"metamesh.transport/*", "metamesh.transport/", false},
+		{"metamesh.transport/nzb", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@1", false},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb", false},
+		{"metamesh.transport/*@1", "metamesh.transport/nzb@1", true},
+		{"metamesh.core", "metamesh.core", true},
+		{"metamesh.core", "metamesh.core2", false},
+		{"mcp", "mcp/x", false},
+		{"*", "anything@3", true},
+	}
+	for _, c := range cases {
+		if got := CapMatches(c.pattern, c.cap); got != c.want {
+			t.Errorf("CapMatches(%q, %q) = %v, want %v", c.pattern, c.cap, got, c.want)
+		}
+	}
+}
+
+func TestParseRejectsV1AndForeign(t *testing.T) {
+	for _, raw := range []string{
+		`{"type":"discovery"}`,
+		`{"type":"announce","name":"x","tools":[]}`,
+		`{"v":1,"type":"announce","name":"meta-sort"}`,
+		`{"proto":"beacon","v":3,"type":"probe"}`,
+		`{"proto":"beacon","v":2,"type":"announce"}`,
+		`{"proto":"beacon","v":2,"type":"advertise"}`,
+		"\x00garbage",
+	} {
+		if _, ok := Parse([]byte(raw)); ok {
+			t.Errorf("Parse(%q) accepted", raw)
+		}
+	}
+	if _, ok := Parse([]byte(`{"proto":"beacon","v":2,"type":"probe","want":["metamesh.core"]}`)); !ok {
+		t.Error("valid probe rejected")
+	}
+}
+
+func TestNeighborRowCarriesCapsAndBaseUrl(t *testing.T) {
+	nb := newNeighbor(
+		NodeInfo{Name: "meta-sort", Instance: "metasort-app"},
+		[]Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}, Endpoints: map[string]string{"ui": "https://x"}}},
+		"10.0.0.1", time.Now(),
+	)
+	if nb.BaseUrl != "https://x" || len(nb.Caps) != 1 || !nb.Matches(CapAnyService) {
+		t.Fatalf("unexpected row: %+v", nb)
+	}
+}
+
+// End to end over loopback: a probe with a non-matching want gets no reply;
+// a matching one gets the advertise; advertise then bye updates the view.
+func TestProbeAndRecord(t *testing.T) {
+	port := freePort(t)
+	n := New(Config{
+		Name: "meta-core", Instance: "core-1", Port: port, Interval: time.Hour,
+		Payload: func() (string, []Resource) {
+			return "running", []Resource{{ID: "core", Caps: []string{CapCore}}}
+		},
+	})
+	if err := n.Start(); err != nil {
+		t.Fatal(err)
+	}
+	defer n.Stop()
+
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	dst := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}
+	buf := make([]byte, ReadBufferSize)
+
+	send := func(m Message) {
+		b, _ := json.Marshal(m)
+		if _, err := c.WriteToUDP(b, dst); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{"metamesh.service/*"}})
+	c.SetReadDeadline(time.Now().Add(300 * time.Millisecond))
+	if _, _, err := c.ReadFromUDP(buf); err == nil {
+		t.Fatal("non-matching probe got a reply")
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{CapCore}})
+	c.SetReadDeadline(time.Now().Add(2 * time.Second))
+	nr, _, err := c.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatalf("matching probe got no reply: %v", err)
+	}
+	if m, ok := Parse(buf[:nr]); !ok || m.Type != TypeAdvertise || m.Node.Instance != "core-1" {
+		t.Fatalf("bad reply: %s", buf[:nr])
+	}
+
+	peer := &NodeInfo{Name: "meta-sort", Instance: "sort-1"}
+	send(Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: peer,
+		Resources: []Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}}}})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 1 })
+	send(Message{Proto: Proto, V: Version, Type: TypeBye, Node: peer})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 0 })
+}
+
+func freePort(t *testing.T) int {
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	return c.LocalAddr().(*net.UDPAddr).Port
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("condition not reached")
+}

internal/reports/reports.go

@@ -0,0 +1,561 @@
+// Package reports is meta-core's bug-report / feedback inbox.
+//
+// Client apps (meta-watch, meta-read, …) build a report in the browser — the
+// line the person typed, the page they were on, the build, the device, the
+// tail of the client log, and whatever app-specific state explains the moment
+// (a player pipeline, a reader position) — and their server forwards it here,
+// server-to-server, to the box's own core. meta-core stores it, lists it and
+// shows it on the dashboard's Reports page, so triage reads one inbox instead
+// of one per app.
+//
+// # Why a third SQLite file
+//
+// Reports are diagnostics, not metadata. They do not belong in the truth
+// database (signed claims only) nor in the cache (unsynchronised, never backed
+// up, deleted as the recovery procedure). They live in their own file,
+// reports.db, so neither a cache rebuild nor POST /api/metadata/clear touches
+// them, and a full report table can never slow a record write.
+//
+// # What meta-core knows about a report
+//
+// Almost nothing, on purpose. A handful of top-level fields are lifted into
+// columns so the inbox can filter and headline them (app, build, uid, page,
+// route, cid, title, note); everything else is kept verbatim in body as JSON
+// and rendered generically by the dashboard. An app can add a new block of
+// state without a meta-core release.
+//
+// ⚠ A report is a piece of viewing/reading history (cids, titles, a uid, a
+// timestamp). It stays on the box, behind the same perimeter as the rest of
+// meta-core's API; nothing here is federated.
+package reports
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+const schema = `
+CREATE TABLE IF NOT EXISTS report(
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  ts          INTEGER NOT NULL,           -- when the report was filed (ms)
+  received    INTEGER NOT NULL,           -- when meta-core stored it (ms)
+  app         TEXT    NOT NULL,
+  build       TEXT    NOT NULL DEFAULT '',
+  uid         TEXT    NOT NULL DEFAULT '',
+  page        TEXT    NOT NULL DEFAULT '',
+  route       TEXT    NOT NULL DEFAULT '',
+  cid         TEXT    NOT NULL DEFAULT '',
+  title       TEXT    NOT NULL DEFAULT '',
+  note        TEXT    NOT NULL DEFAULT '',
+  status      TEXT    NOT NULL DEFAULT 'open',
+  triage_note TEXT    NOT NULL DEFAULT '',
+  updated     INTEGER NOT NULL DEFAULT 0,
+  log_lines   INTEGER NOT NULL DEFAULT 0,
+  bytes       INTEGER NOT NULL,
+  body        TEXT    NOT NULL
+) STRICT;
+CREATE INDEX IF NOT EXISTS report_app_status ON report(app, status, id DESC);
+CREATE INDEX IF NOT EXISTS report_status ON report(status, id DESC);
+`
+
+// Statuses a report moves through. "open" is where every report starts.
+const (
+	StatusOpen    = "open"
+	StatusTriaged = "triaged"
+	StatusClosed  = "closed"
+)
+
+// ValidStatus reports whether s is a status a report can be set to.
+func ValidStatus(s string) bool {
+	return s == StatusOpen || s == StatusTriaged || s == StatusClosed
+}
+
+// Bounds on what one report may carry. The string caps match the ones
+// meta-watch applied when it kept reports itself, so moving the inbox loses
+// nothing a report used to hold.
+const (
+	maxApp        = 32
+	maxBuild      = 4000
+	maxUID        = 128
+	maxPage       = 32
+	maxRoute      = 512
+	maxCID        = 128
+	maxTitle      = 200
+	maxNote       = 1000
+	maxTriageNote = 2000
+	// MaxLogLines / MaxLogLineChars bound the client log tail.
+	MaxLogLines     = 200
+	MaxLogLineChars = 600
+	// MaxBlockBytes bounds every other top-level block (context, device,
+	// app-specific state). A larger block is replaced by a marker naming its
+	// size, never dropped silently.
+	MaxBlockBytes = 32 * 1024
+	// maxBlocks bounds the number of top-level keys.
+	maxBlocks = 64
+	// filed-at plausibility window: a report spooled by an app while the core
+	// was down arrives late but keeps the time it was filed.
+	maxFiledAge    = 30 * 24 * time.Hour
+	maxFiledFuture = 5 * time.Minute
+)
+
+// ErrNotFound is returned for an id that does not exist.
+var ErrNotFound = errors.New("report not found")
+
+// Summary is a report without its body: what the inbox list shows.
+type Summary struct {
+	ID         int64  `json:"id"`
+	TS         int64  `json:"ts"`
+	Received   int64  `json:"received"`
+	App        string `json:"app"`
+	Build      string `json:"build,omitempty"`
+	UID        string `json:"uid,omitempty"`
+	Page       string `json:"page,omitempty"`
+	Route      string `json:"route,omitempty"`
+	CID        string `json:"cid,omitempty"`
+	Title      string `json:"title,omitempty"`
+	Note       string `json:"note,omitempty"`
+	Status     string `json:"status"`
+	TriageNote string `json:"triage_note,omitempty"`
+	Updated    int64  `json:"updated,omitempty"`
+	LogLines   int    `json:"log_lines"`
+	Bytes      int64  `json:"bytes"`
+}
+
+// Report is a stored report with its full body.
+type Report struct {
+	Summary
+	Body json.RawMessage `json:"body"`
+}
+
+// Filter selects reports for List.
+type Filter struct {
+	App    string
+	Status string
+	Q      string // substring of note, title or cid
+	Before int64  // only ids below this (paging); 0 = from the newest
+	Limit  int
+}
+
+// Page is one page of List.
+type Page struct {
+	Reports []Summary        `json:"reports"`
+	Apps    map[string]int64 `json:"apps"`   // every app with a report, and how many
+	Counts  map[string]int64 `json:"counts"` // per status, within the App filter
+	More    bool             `json:"more"`
+}
+
+// Store is the open reports database.
+type Store struct {
+	db       *kv.Store
+	maxRows  int
+	maxBytes int64
+	now      func() time.Time
+}
+
+// Open opens (creating if needed) the reports database at path. maxRows and
+// maxBytes bound what Prune keeps; <= 0 disables that bound.
+func Open(path string, maxRows int, maxBytes int64) (*Store, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return nil, fmt.Errorf("reports dir: %w", err)
+	}
+	db, err := kv.Open(path, kv.Options{
+		NoKVTable:     true,
+		Schema:        schema,
+		WriterCacheMB: 4,
+		ReaderCacheMB: 2,
+		Readers:       2,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("open reports %s: %w", path, err)
+	}
+	return &Store{db: db, maxRows: maxRows, maxBytes: maxBytes, now: time.Now}, nil
+}
+
+// Close closes the database.
+func (s *Store) Close() error { return s.db.Close() }
+
+// Path is the database file.
+func (s *Store) Path() string { return s.db.Path() }
+
+func opCtx() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), 30*time.Second)
+}
+
+// ── Normalisation ─────────────────────────────────────────────────────────
+
+// Normalize turns a posted envelope into a storable report: the indexed
+// fields lifted and clamped, the log tail bounded, every other block size-
+// checked. It never fails on content an app might plausibly send; the only
+// error is a report that names no app.
+func Normalize(in map[string]json.RawMessage, now time.Time) (Report, error) {
+	var r Report
+	str := func(key string, n int) string {
+		raw, ok := in[key]
+		if !ok {
+			return ""
+		}
+		var v string
+		if json.Unmarshal(raw, &v) != nil {
+			return ""
+		}
+		return clamp(strings.TrimSpace(v), n)
+	}
+
+	r.App = str("app", maxApp)
+	if r.App == "" {
+		return r, errors.New("app is required")
+	}
+	r.Build = str("build", maxBuild)
+	r.UID = str("uid", maxUID)
+	r.Page = str("page", maxPage)
+	r.Route = str("route", maxRoute)
+	r.CID = str("cid", maxCID)
+	r.Title = str("title", maxTitle)
+	r.Note = str("note", maxNote)
+	r.Status = StatusOpen
+	r.Received = now.UnixMilli()
+	r.TS = r.Received
+	if raw, ok := in["filedAt"]; ok {
+		var f float64
+		if json.Unmarshal(raw, &f) == nil {
+			t := time.UnixMilli(int64(f))
+			if t.After(now.Add(-maxFiledAge)) && t.Before(now.Add(maxFiledFuture)) {
+				r.TS = t.UnixMilli()
+			}
+		}
+	}
+
+	// The body repeats the lifted fields (clamped), so a report exported as
+	// JSON is complete on its own.
+	body := make(map[string]any, len(in))
+	for k, v := range map[string]string{
+		"app": r.App, "build": r.Build, "uid": r.UID, "page": r.Page,
+		"route": r.Route, "cid": r.CID, "title": r.Title, "note": r.Note,
+	} {
+		if v != "" {
+			body[k] = v
+		}
+	}
+	if raw, ok := in["log"]; ok {
+		var lines []string
+		if json.Unmarshal(raw, &lines) == nil {
+			if len(lines) > MaxLogLines {
+				lines = lines[len(lines)-MaxLogLines:]
+			}
+			for i, l := range lines {
+				lines[i] = clamp(l, MaxLogLineChars)
+			}
+			if len(lines) > 0 {
+				body["log"] = lines
+				r.LogLines = len(lines)
+			}
+		}
+	}
+	// Remaining blocks in key order, so which ones survive maxBlocks does not
+	// depend on map iteration.
+	keys := make([]string, 0, len(in))
+	for k := range in {
+		if k != "log" && k != "filedAt" && !isLifted(k) {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+	blocks := 0
+	for _, k := range keys {
+		raw := in[k]
+		if string(raw) == "null" {
+			continue
+		}
+		if blocks >= maxBlocks {
+			break
+		}
+		blocks++
+		key := clamp(k, 64)
+		if len(raw) > MaxBlockBytes {
+			body[key] = map[string]any{"truncated": true, "bytes": len(raw)}
+			continue
+		}
+		body[key] = raw
+	}
+
+	b, err := json.Marshal(body)
+	if err != nil {
+		return r, fmt.Errorf("encode body: %w", err)
+	}
+	r.Body = b
+	r.Bytes = int64(len(b))
+	return r, nil
+}
+
+func isLifted(k string) bool {
+	switch k {
+	case "app", "build", "uid", "page", "route", "cid", "title", "note":
+		return true
+	}
+	return false
+}
+
+// clamp keeps at most n characters (runes) of s, never splitting one.
+func clamp(s string, n int) string {
+	if utf8.RuneCountInString(s) <= n {
+		return s
+	}
+	i := 0
+	for j := range s {
+		if i == n {
+			return s[:j]
+		}
+		i++
+	}
+	return s
+}
+
+// ── Writes ────────────────────────────────────────────────────────────────
+
+// Insert stores a normalised report and returns its id. The store is pruned
+// to its bounds in the same transaction, so it never sits over them.
+func (s *Store) Insert(r Report) (int64, error) {
+	ctx, cancel := opCtx()
+	defer cancel()
+	var id int64
+	err := s.db.Update(ctx, func(tx *kv.Tx) error {
+		res, err := tx.Exec(`INSERT INTO report(ts, received, app, build, uid, page, route, cid, title, note,
+			status, log_lines, bytes, body) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+			r.TS, r.Received, r.App, r.Build, r.UID, r.Page, r.Route, r.CID, r.Title, r.Note,
+			StatusOpen, r.LogLines, r.Bytes, string(r.Body))
+		if err != nil {
+			return err
+		}
+		if id, err = res.LastInsertId(); err != nil {
+			return err
+		}
+		_, err = s.prune(tx)
+		return err
+	})
+	return id, err
+}
+
+// Update sets a report's status and/or triage note. A nil argument leaves
+// that field as it is.
+func (s *Store) Update(id int64, status, triageNote *string) (Report, error) {
+	if status != nil && !ValidStatus(*status) {
+		return Report{}, fmt.Errorf("invalid status %q", *status)
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	err := s.db.Update(ctx, func(tx *kv.Tx) error {
+		sets, args := []string{"updated = ?"}, []any{s.now().UnixMilli()}
+		if status != nil {
+			sets, args = append(sets, "status = ?"), append(args, *status)
+		}
+		if triageNote != nil {
+			sets, args = append(sets, "triage_note = ?"), append(args, clamp(strings.TrimSpace(*triageNote), maxTriageNote))
+		}
+		res, err := tx.Exec(`UPDATE report SET `+strings.Join(sets, ", ")+` WHERE id = ?`, append(args, id)...)
+		if err != nil {
+			return err
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			return ErrNotFound
+		}
+		return nil
+	})
+	if err != nil {
+		return Report{}, err
+	}
+	return s.Get(id)
+}
+
+// Delete removes a report.
+func (s *Store) Delete(id int64) error {
+	ctx, cancel := opCtx()
+	defer cancel()
+	return s.db.Update(ctx, func(tx *kv.Tx) error {
+		res, err := tx.Exec(`DELETE FROM report WHERE id = ?`, id)
+		if err != nil {
+			return err
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			return ErrNotFound
+		}
+		return nil
+	})
+}
+
+// Prune brings the store back within its bounds and returns how many reports
+// it removed. Insert already prunes; this is for a bound lowered by config.
+func (s *Store) Prune() (int, error) {
+	ctx, cancel := opCtx()
+	defer cancel()
+	var n int
+	err := s.db.Update(ctx, func(tx *kv.Tx) error {
+		var err error
+		n, err = s.prune(tx)
+		return err
+	})
+	return n, err
+}
+
+// prune evicts closed reports first, oldest first, then any report, oldest
+// first, until both the row and the byte bound hold. An open report is the
+// last thing to go: it is the one nobody has read yet.
+func (s *Store) prune(tx *kv.Tx) (int, error) {
+	if s.maxRows <= 0 && s.maxBytes <= 0 {
+		return 0, nil
+	}
+	var rows, bytes int64
+	if err := tx.QueryRow(`SELECT count(*), coalesce(sum(bytes), 0) FROM report`).Scan(&rows, &bytes); err != nil {
+		return 0, err
+	}
+	over := func() bool {
+		return (s.maxRows > 0 && rows > int64(s.maxRows)) || (s.maxBytes > 0 && bytes > s.maxBytes)
+	}
+	if !over() {
+		return 0, nil
+	}
+	q, err := tx.Query(`SELECT id, bytes FROM report ORDER BY (status = 'closed') DESC, id ASC`)
+	if err != nil {
+		return 0, err
+	}
+	var victims []int64
+	for q.Next() && over() {
+		var id, b int64
+		if err := q.Scan(&id, &b); err != nil {
+			q.Close()
+			return 0, err
+		}
+		victims = append(victims, id)
+		rows--
+		bytes -= b
+	}
+	q.Close()
+	for _, id := range victims {
+		if _, err := tx.Exec(`DELETE FROM report WHERE id = ?`, id); err != nil {
+			return 0, err
+		}
+	}
+	return len(victims), nil
+}
+
+// ── Reads ─────────────────────────────────────────────────────────────────
+
+const summaryCols = `id, ts, received, app, build, uid, page, route, cid, title, note, status, triage_note, updated, log_lines, bytes`
+
+func scanSummary(sc interface{ Scan(...any) error }, extra ...any) (Summary, error) {
+	var r Summary
+	dst := append([]any{&r.ID, &r.TS, &r.Received, &r.App, &r.Build, &r.UID, &r.Page, &r.Route,
+		&r.CID, &r.Title, &r.Note, &r.Status, &r.TriageNote, &r.Updated, &r.LogLines, &r.Bytes}, extra...)
+	return r, sc.Scan(dst...)
+}
+
+// Get returns one report with its body.
+func (s *Store) Get(id int64) (Report, error) {
+	ctx, cancel := opCtx()
+	defer cancel()
+	var body string
+	row := s.db.DB().QueryRowContext(ctx, `SELECT `+summaryCols+`, body FROM report WHERE id = ?`, id)
+	sum, err := scanSummary(row, &body)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Report{}, ErrNotFound
+	}
+	if err != nil {
+		return Report{}, err
+	}
+	return Report{Summary: sum, Body: json.RawMessage(body)}, nil
+}
+
+// List returns one page of summaries, newest first, with the counts the
+// inbox's filter bar needs.
+func (s *Store) List(f Filter) (Page, error) {
+	if f.Limit <= 0 || f.Limit > 200 {
+		f.Limit = 50
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	db := s.db.DB()
+
+	var where []string
+	var args []any
+	if f.App != "" {
+		where, args = append(where, "app = ?"), append(args, f.App)
+	}
+	appWhere, appArgs := append([]string(nil), where...), append([]any(nil), args...)
+	if f.Status != "" {
+		where, args = append(where, "status = ?"), append(args, f.Status)
+	}
+	if q := strings.TrimSpace(f.Q); q != "" {
+		like := "%" + escapeLike(q) + "%"
+		where = append(where, `(note LIKE ? ESCAPE '\' OR title LIKE ? ESCAPE '\' OR cid LIKE ? ESCAPE '\' OR uid LIKE ? ESCAPE '\')`)
+		args = append(args, like, like, like, like)
+	}
+	if f.Before > 0 {
+		where, args = append(where, "id < ?"), append(args, f.Before)
+	}
+	clause := ""
+	if len(where) > 0 {
+		clause = " WHERE " + strings.Join(where, " AND ")
+	}
+
+	page := Page{Reports: []Summary{}, Apps: map[string]int64{}, Counts: map[string]int64{}}
+	rows, err := db.QueryContext(ctx, `SELECT `+summaryCols+` FROM report`+clause+` ORDER BY id DESC LIMIT ?`,
+		append(args, f.Limit+1)...)
+	if err != nil {
+		return page, err
+	}
+	for rows.Next() {
+		sum, err := scanSummary(rows)
+		if err != nil {
+			rows.Close()
+			return page, err
+		}
+		page.Reports = append(page.Reports, sum)
+	}
+	rows.Close()
+	if len(page.Reports) > f.Limit {
+		page.Reports, page.More = page.Reports[:f.Limit], true
+	}
+
+	if err := groupCount(ctx, db, `SELECT app, count(*) FROM report GROUP BY app`, nil, page.Apps); err != nil {
+		return page, err
+	}
+	statusQ := `SELECT status, count(*) FROM report`
+	if len(appWhere) > 0 {
+		statusQ += " WHERE " + strings.Join(appWhere, " AND ")
+	}
+	if err := groupCount(ctx, db, statusQ+` GROUP BY status`, appArgs, page.Counts); err != nil {
+		return page, err
+	}
+	return page, nil
+}
+
+func groupCount(ctx context.Context, db *sql.DB, q string, args []any, into map[string]int64) error {
+	rows, err := db.QueryContext(ctx, q, args...)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var k string
+		var n int64
+		if err := rows.Scan(&k, &n); err != nil {
+			return err
+		}
+		into[k] = n
+	}
+	return rows.Err()
+}
+
+func escapeLike(s string) string {
+	return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
+}

internal/reports/reports_test.go

@@ -0,0 +1,227 @@
+package reports
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func open(t *testing.T, maxRows int, maxBytes int64) *Store {
+	t.Helper()
+	s, err := Open(filepath.Join(t.TempDir(), "db", "reports.db"), maxRows, maxBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func envelope(t *testing.T, v map[string]any) map[string]json.RawMessage {
+	t.Helper()
+	b, _ := json.Marshal(v)
+	var out map[string]json.RawMessage
+	if err := json.Unmarshal(b, &out); err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+func file(t *testing.T, s *Store, v map[string]any) int64 {
+	t.Helper()
+	r, err := Normalize(envelope(t, v), time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	id, err := s.Insert(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+func TestNormalizeRequiresApp(t *testing.T) {
+	if _, err := Normalize(envelope(t, map[string]any{"note": "x"}), time.Now()); err == nil {
+		t.Fatal("a report without app must be refused")
+	}
+}
+
+func TestNormalizeClampsAndBounds(t *testing.T) {
+	log := make([]string, 250)
+	for i := range log {
+		log[i] = strings.Repeat("é", 700)
+	}
+	now := time.Now()
+	r, err := Normalize(envelope(t, map[string]any{
+		"app":     "meta-read",
+		"note":    strings.Repeat("n", 1500),
+		"title":   strings.Repeat("t", 300),
+		"log":     log,
+		"context": map[string]string{"big": strings.Repeat("x", MaxBlockBytes)},
+		"reader":  map[string]int{"page": 4},
+		"device":  nil,
+		"filedAt": now.Add(-time.Hour).UnixMilli(),
+	}), now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(r.Note) != maxNote || len(r.Title) != maxTitle {
+		t.Fatalf("note/title not clamped: %d %d", len(r.Note), len(r.Title))
+	}
+	if r.TS != now.Add(-time.Hour).UnixMilli() || r.Received != now.UnixMilli() {
+		t.Fatalf("filedAt should set ts, received stays now: %d %d", r.TS, r.Received)
+	}
+	var body map[string]json.RawMessage
+	_ = json.Unmarshal(r.Body, &body)
+	var lines []string
+	_ = json.Unmarshal(body["log"], &lines)
+	if len(lines) != MaxLogLines || r.LogLines != MaxLogLines {
+		t.Fatalf("log tail: %d lines (col %d)", len(lines), r.LogLines)
+	}
+	if n := len([]rune(lines[0])); n != MaxLogLineChars {
+		t.Fatalf("log line clamped by rune: %d", n)
+	}
+	if !strings.Contains(string(body["context"]), `"truncated":true`) {
+		t.Fatalf("oversized block not replaced by a marker: %.80s", body["context"])
+	}
+	if string(body["reader"]) != `{"page":4}` {
+		t.Fatalf("app block not kept verbatim: %s", body["reader"])
+	}
+	if _, ok := body["device"]; ok {
+		t.Fatal("null block should be dropped")
+	}
+	if _, ok := body["filedAt"]; ok {
+		t.Fatal("filedAt belongs in ts, not the body")
+	}
+}
+
+func TestNormalizeIgnoresImplausibleFiledAt(t *testing.T) {
+	now := time.Now()
+	for _, at := range []time.Time{now.Add(-60 * 24 * time.Hour), now.Add(time.Hour)} {
+		r, _ := Normalize(envelope(t, map[string]any{"app": "a", "filedAt": at.UnixMilli()}), now)
+		if r.TS != now.UnixMilli() {
+			t.Fatalf("filedAt %v should be ignored", at)
+		}
+	}
+}
+
+func TestInsertGetListFilters(t *testing.T) {
+	s := open(t, 0, 0)
+	a := file(t, s, map[string]any{"app": "meta-watch", "note": "stall at 3%", "cid": "bafyA"})
+	b := file(t, s, map[string]any{"app": "meta-read", "note": "blank page", "uid": "u1"})
+	c := file(t, s, map[string]any{"app": "meta-read", "note": "wrong cover 100%"})
+
+	got, err := s.Get(a)
+	if err != nil || got.App != "meta-watch" || got.CID != "bafyA" || got.Status != StatusOpen {
+		t.Fatalf("get: %+v %v", got.Summary, err)
+	}
+	if _, err := s.Get(999); err != ErrNotFound {
+		t.Fatalf("missing id: %v", err)
+	}
+
+	p, err := s.List(Filter{})
+	if err != nil || len(p.Reports) != 3 || p.Reports[0].ID != c {
+		t.Fatalf("list newest first: %+v %v", p.Reports, err)
+	}
+	if p.Apps["meta-read"] != 2 || p.Apps["meta-watch"] != 1 || p.Counts[StatusOpen] != 3 {
+		t.Fatalf("counts: %+v %+v", p.Apps, p.Counts)
+	}
+
+	p, _ = s.List(Filter{App: "meta-read"})
+	if len(p.Reports) != 2 || p.Counts[StatusOpen] != 2 || p.Apps["meta-watch"] != 1 {
+		t.Fatalf("app filter: %+v counts %+v apps %+v", p.Reports, p.Counts, p.Apps)
+	}
+	p, _ = s.List(Filter{Q: "100%"})
+	if len(p.Reports) != 1 || p.Reports[0].ID != c {
+		t.Fatalf("q is a literal substring (%% escaped): %+v", p.Reports)
+	}
+	p, _ = s.List(Filter{Q: "u1"})
+	if len(p.Reports) != 1 || p.Reports[0].ID != b {
+		t.Fatalf("q matches uid: %+v", p.Reports)
+	}
+
+	p, _ = s.List(Filter{Limit: 2})
+	if len(p.Reports) != 2 || !p.More {
+		t.Fatalf("page 1: %d more=%v", len(p.Reports), p.More)
+	}
+	p, _ = s.List(Filter{Limit: 2, Before: p.Reports[1].ID})
+	if len(p.Reports) != 1 || p.More || p.Reports[0].ID != a {
+		t.Fatalf("page 2: %+v more=%v", p.Reports, p.More)
+	}
+}
+
+func TestUpdateAndDelete(t *testing.T) {
+	s := open(t, 0, 0)
+	id := file(t, s, map[string]any{"app": "a"})
+	st, note := StatusTriaged, "  Deck #900  "
+	r, err := s.Update(id, &st, &note)
+	if err != nil || r.Status != StatusTriaged || r.TriageNote != "Deck #900" || r.Updated == 0 {
+		t.Fatalf("update: %+v %v", r.Summary, err)
+	}
+	bad := "done"
+	if _, err := s.Update(id, &bad, nil); err == nil {
+		t.Fatal("invalid status accepted")
+	}
+	if _, err := s.Update(42, &st, nil); err != ErrNotFound {
+		t.Fatalf("update missing: %v", err)
+	}
+	p, _ := s.List(Filter{Status: StatusTriaged})
+	if len(p.Reports) != 1 {
+		t.Fatalf("status filter: %+v", p.Reports)
+	}
+	if err := s.Delete(id); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Delete(id); err != ErrNotFound {
+		t.Fatalf("second delete: %v", err)
+	}
+}
+
+func TestPruneEvictsClosedThenOldest(t *testing.T) {
+	s := open(t, 3, 0)
+	ids := []int64{
+		file(t, s, map[string]any{"app": "a", "note": "1"}),
+		file(t, s, map[string]any{"app": "a", "note": "2"}),
+		file(t, s, map[string]any{"app": "a", "note": "3"}),
+	}
+	closed := StatusClosed
+	if _, err := s.Update(ids[2], &closed, nil); err != nil {
+		t.Fatal(err)
+	}
+	// Fourth report: the closed one (newest of the three) goes first.
+	file(t, s, map[string]any{"app": "a", "note": "4"})
+	if _, err := s.Get(ids[2]); err != ErrNotFound {
+		t.Fatal("closed report should be evicted before older open ones")
+	}
+	// Fifth: no closed left, so the oldest open goes.
+	file(t, s, map[string]any{"app": "a", "note": "5"})
+	if _, err := s.Get(ids[0]); err != ErrNotFound {
+		t.Fatal("oldest open report should be evicted next")
+	}
+	p, _ := s.List(Filter{})
+	if len(p.Reports) != 3 {
+		t.Fatalf("store over its row bound: %d", len(p.Reports))
+	}
+}
+
+func TestPruneByBytes(t *testing.T) {
+	s := open(t, 0, 3000)
+	for i := 0; i < 5; i++ {
+		file(t, s, map[string]any{"app": "a", "note": strings.Repeat("x", 900)})
+	}
+	p, _ := s.List(Filter{})
+	var total int64
+	for _, r := range p.Reports {
+		total += r.Bytes
+	}
+	if total > 3000 || len(p.Reports) == 0 {
+		t.Fatalf("byte bound not held: %d bytes in %d reports", total, len(p.Reports))
+	}
+}
+
+func TestClampKeepsRunes(t *testing.T) {
+	if got := clamp("héllo", 2); got != "hé" {
+		t.Fatalf("clamp: %q", got)
+	}
+}

internal/schema/classify.go

@@ -21,7 +21,7 @@ import (
 // (crc32: "bafk3eaqerjhiwhy").
 var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
 
-// ClassifyValue returns the (primitive, hint) for a Redis value.
+// ClassifyValue returns the (primitive, hint) for a stored value.
 // exists=false (key missing) and JSON null literal both map to PrimUndefined.
 // Empty string is a valid string.
 func ClassifyValue(value string, exists bool) (Primitive, Hint) {

internal/schema/indexer.go

@@ -2,21 +2,35 @@ package schema
 
 import (
 	"context"
-	"log"
 	"regexp"
+	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/metazla/meta-core/internal/events"
+	"github.com/metazla/meta-core/internal/kv"
-	"github.com/redis/go-redis/v9"
 )
 
 var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
 
-// fieldState is the per-field accumulator. perKey lets us undo a previous
+// CounterPrefix is where the schema lives: one counter per (field, key hint,
-// contribution when the same Redis key is re-SET or DELed.
+// primitive, hint), maintained by the storage write path in the same
+// transaction as the change (storage's OnChange hook → CounterKey).
+const CounterPrefix = "idx:schema/"
+
+const counterSep = "\x1f"
+
+// CounterKey is the counter a record key holding value contributes to, or
+// ok=false for keys that are not record properties.
+func CounterKey(recordKey, value string) (string, bool) {
+	fieldPath, keyHint, ok := parseFieldPath(recordKey)
+	if !ok {
+		return "", false
+	}
+	prim, hint := ClassifyValue(value, true)
+	return CounterPrefix + fieldPath + counterSep + string(keyHint) + counterSep + string(prim) + counterSep + string(hint), true
+}
+
+// fieldState is the per-field accumulator.
 type fieldState struct {
 	counts         map[string]int
 	primitives     map[Primitive]int
@@ -24,12 +38,6 @@ type fieldState struct {
 	typedCount     int
 	undefinedCount int
 	languageKeyed  bool
-	perKey         map[string]labelPair
-}
-
-type labelPair struct {
-	prim Primitive
-	hint Hint
 }
 
 func newFieldState() *fieldState {
@@ -37,160 +45,72 @@ func newFieldState() *fieldState {
 		counts:     make(map[string]int),
 		primitives: make(map[Primitive]int),
 		hintCounts: make(map[Hint]int),
-		perKey:     make(map[string]labelPair),
 	}
 }
 
-// Indexer maintains a live field-schema by consuming meta:events.
+// Indexer serves the per-field schema.
+//
+// The schema is a set of exact counters the write path keeps current in the
+// same transaction as each change (see CounterKey), so serving it is one range
+// read and nothing runs in the background. Two earlier shapes were dropped: a
+// consumer of meta:events that remembered every record key it had seen (~2 GB
+// at 6.3M keys), then a full rescan of the store every 10 minutes (~80 s of
+// CPU each time, whether or not anyone looked).
 type Indexer struct {
-	client    *redis.Client
+	store *kv.Store
-	keyPrefix string
-
-	mu     sync.RWMutex
-	fields map[string]*fieldState
-
-	processed uint64 // total events processed since start
-
-	ctx     context.Context
-	cancel  context.CancelFunc
-	wg      sync.WaitGroup
-	runMu   sync.Mutex
-	running bool
 }
 
-// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
+// NewIndexer creates an Indexer over store.
-// so the indexer can build absolute Redis keys for GET calls.
+func NewIndexer(store *kv.Store) *Indexer {
-func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
+	return &Indexer{store: store}
-	ctx, cancel := context.WithCancel(context.Background())
-	return &Indexer{
-		client:    client,
-		keyPrefix: keyPrefix,
-		fields:    make(map[string]*fieldState),
-		ctx:       ctx,
-		cancel:    cancel,
-	}
 }
 
-// Start begins consuming the meta:events stream from the earliest available ID.
+// Start and Stop exist for the server's lifecycle; there is nothing to run.
-func (i *Indexer) Start() error {
+func (i *Indexer) Start() error { return nil }
-	i.runMu.Lock()
+func (i *Indexer) Stop() error  { return nil }
-	if i.running {
-		i.runMu.Unlock()
-		return nil
-	}
-	i.running = true
-	i.runMu.Unlock()
 
-	i.wg.Add(1)
+// Snapshot reads the counters and folds them into the per-field schema.
-	go i.consumeLoop()
+func (i *Indexer) Snapshot(ctx context.Context) (*SchemaResponse, error) {
-
+	fields := make(map[string]*fieldState)
-	log.Println("[SchemaIndexer] Started - consuming meta:events stream")
+	err := i.store.ScanPrefix(ctx, CounterPrefix, 5000, func(rows []kv.KV) error {
-	return nil
+		for _, row := range rows {
-}
+			parts := strings.Split(row.K[len(CounterPrefix):], counterSep)
-
+			if len(parts) != 4 {
-// Stop terminates the consumer goroutine.
-func (i *Indexer) Stop() error {
-	i.runMu.Lock()
-	if !i.running {
-		i.runMu.Unlock()
-		return nil
-	}
-	i.running = false
-	i.runMu.Unlock()
-
-	i.cancel()
-	i.wg.Wait()
-	log.Println("[SchemaIndexer] Stopped")
-	return nil
-}
-
-// ProcessedCount returns the total number of stream events processed.
-// Used by WaitForDrain to detect when a rescan flood has been absorbed.
-func (i *Indexer) ProcessedCount() uint64 {
-	return atomic.LoadUint64(&i.processed)
-}
-
-// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
-// the timeout elapses. Suitable for use right after RepublishMetadata.
-func (i *Indexer) WaitForDrain(timeout time.Duration) {
-	deadline := time.Now().Add(timeout)
-	last := i.ProcessedCount()
-	stableSince := time.Now()
-	for time.Now().Before(deadline) {
-		time.Sleep(100 * time.Millisecond)
-		now := i.ProcessedCount()
-		if now != last {
-			last = now
-			stableSince = time.Now()
-			continue
-		}
-		if time.Since(stableSince) >= 300*time.Millisecond {
-			return
-		}
-	}
-}
-
-func (i *Indexer) consumeLoop() {
-	defer i.wg.Done()
-	lastID := "0"
-	for {
-		select {
-		case <-i.ctx.Done():
-			return
-		default:
-		}
-
-		res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
-			Streams: []string{events.MetaEventsStream, lastID},
-			Block:   2 * time.Second,
-			Count:   500,
-		}).Result()
-
-		if err != nil {
-			if err == redis.Nil || i.ctx.Err() != nil {
 				continue
 			}
-			log.Printf("[SchemaIndexer] XRead error: %v", err)
+			n, err := strconv.Atoi(row.V)
-			time.Sleep(1 * time.Second)
+			if err != nil || n <= 0 {
-			continue
+				continue
-		}
-
-		for _, stream := range res {
-			for _, msg := range stream.Messages {
-				lastID = msg.ID
-				i.processEvent(msg.Values)
-				atomic.AddUint64(&i.processed, 1)
 			}
+			fs := fields[parts[0]]
+			if fs == nil {
+				fs = newFieldState()
+				fields[parts[0]] = fs
+			}
+			if parts[1] != "" {
+				fs.languageKeyed = true
+			}
+			incContribution(fs, labelPair{prim: Primitive(parts[2]), hint: Hint(parts[3])}, n)
 		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
 	}
+	snap := &SchemaResponse{
+		Fields:      make(map[string]*FieldSchema, len(fields)),
+		GeneratedAt: time.Now().UTC(),
+		Source:      "live",
+	}
+	for fp, fs := range fields {
+		snap.Fields[fp] = computeFieldSchema(fs)
+	}
+	return snap, nil
 }
 
-func (i *Indexer) processEvent(values map[string]any) {
+type labelPair struct {
-	op, _ := values["type"].(string)
+	prim Primitive
-	key, _ := values["key"].(string)
+	hint Hint
-	if key == "" {
-		return
-	}
-	fieldPath, keyHint, ok := parseFieldPath(key)
-	if !ok {
-		return
-	}
-
-	switch op {
-	case "set":
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
-		cancel()
-		exists := err != redis.Nil
-		if err != nil && err != redis.Nil {
-			// Treat transient errors as "value disappeared" — nothing to record.
-			return
-		}
-		prim, hint := ClassifyValue(val, exists)
-		i.upsert(fieldPath, key, keyHint, prim, hint)
-	case "del", "expire":
-		i.remove(fieldPath, key)
-	}
 }
 
 // parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
@@ -222,99 +142,21 @@ func parseFieldPath(key string) (string, KeyHint, bool) {
 	return property, "", true
 }
 
-func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
+// incContribution adds n values of one (primitive, hint) to a field.
-	i.mu.Lock()
+func incContribution(fs *fieldState, lp labelPair, n int) {
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		fs = newFieldState()
-		i.fields[fieldPath] = fs
-	}
-	if keyHint != "" {
-		fs.languageKeyed = true
-	}
-
-	if old, ok := fs.perKey[redisKey]; ok {
-		decContribution(fs, old)
-	}
-
-	fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
-	incContribution(fs, labelPair{prim: prim, hint: hint})
-}
-
-func (i *Indexer) remove(fieldPath, redisKey string) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		return
-	}
-	old, ok := fs.perKey[redisKey]
-	if !ok {
-		return
-	}
-	delete(fs.perKey, redisKey)
-	decContribution(fs, old)
-	if len(fs.perKey) == 0 {
-		delete(i.fields, fieldPath)
-	}
-}
-
-func incContribution(fs *fieldState, lp labelPair) {
 	label := breakdownLabel(lp.prim, lp.hint)
-	fs.counts[label]++
+	fs.counts[label] += n
 	if lp.prim == PrimUndefined {
-		fs.undefinedCount++
+		fs.undefinedCount += n
 		return
 	}
-	fs.typedCount++
+	fs.typedCount += n
-	fs.primitives[lp.prim]++
+	fs.primitives[lp.prim] += n
 	if lp.hint != "" {
-		fs.hintCounts[lp.hint]++
+		fs.hintCounts[lp.hint] += n
 	}
 }
 
-func decContribution(fs *fieldState, lp labelPair) {
-	label := breakdownLabel(lp.prim, lp.hint)
-	fs.counts[label]--
-	if fs.counts[label] <= 0 {
-		delete(fs.counts, label)
-	}
-	if lp.prim == PrimUndefined {
-		fs.undefinedCount--
-		return
-	}
-	fs.typedCount--
-	fs.primitives[lp.prim]--
-	if fs.primitives[lp.prim] <= 0 {
-		delete(fs.primitives, lp.prim)
-	}
-	if lp.hint != "" {
-		fs.hintCounts[lp.hint]--
-		if fs.hintCounts[lp.hint] <= 0 {
-			delete(fs.hintCounts, lp.hint)
-		}
-	}
-}
-
-// Snapshot returns the current schema. Safe to call concurrently with consumption.
-func (i *Indexer) Snapshot() *SchemaResponse {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-
-	out := &SchemaResponse{
-		Fields:      make(map[string]*FieldSchema, len(i.fields)),
-		GeneratedAt: time.Now().UTC(),
-		Source:      "live",
-	}
-	for fp, fs := range i.fields {
-		out.Fields[fp] = computeFieldSchema(fs)
-	}
-	return out
-}
-
 func computeFieldSchema(fs *fieldState) *FieldSchema {
 	out := &FieldSchema{
 		Breakdown: make(map[string]int, len(fs.counts)),

internal/snapshot/cache.go

@@ -1,143 +0,0 @@
-package snapshot
-
-import (
-	"archive/zip"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
-// Trailing slash because ZIP entry names are forward-slash relative paths.
-const PluginOutputArchivePrefix = "cache/plugin-output/"
-
-// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
-// file under it into zw at PluginOutputArchivePrefix + relPath.
-//
-// Returns the number of files added. A missing srcDir is not an error: it
-// just means no plugin has produced output yet, so the section stays empty.
-func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
-	info, err := os.Stat(srcDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return 0, nil
-	}
-	if err != nil {
-		return 0, fmt.Errorf("stat %s: %w", srcDir, err)
-	}
-	if !info.IsDir() {
-		return 0, fmt.Errorf("%s is not a directory", srcDir)
-	}
-
-	count := 0
-	err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
-		if walkErr != nil {
-			return walkErr
-		}
-		if d.IsDir() {
-			return nil
-		}
-		rel, err := filepath.Rel(srcDir, path)
-		if err != nil {
-			return err
-		}
-		entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
-
-		w, err := zw.Create(entry)
-		if err != nil {
-			return err
-		}
-		f, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		if _, err := io.Copy(w, f); err != nil {
-			return err
-		}
-		count++
-		return nil
-	})
-	if err != nil {
-		return count, fmt.Errorf("walk %s: %w", srcDir, err)
-	}
-	return count, nil
-}
-
-// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
-// the ZIP and writes it under dstDir. Existing files are overwritten when
-// overwrite=true; otherwise they're left alone (treats existing files as the
-// "mine wins" choice — same toggle the metadata import uses).
-//
-// Path traversal is rejected: any entry whose relative path resolves outside
-// dstDir is skipped with a warning.
-func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
-	if dstDir == "" {
-		return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
-	}
-	if err := os.MkdirAll(dstDir, 0o755); err != nil {
-		return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
-	}
-
-	cleanRoot, err := filepath.Abs(dstDir)
-	if err != nil {
-		return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
-	}
-
-	for _, f := range zr.File {
-		if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
-			continue
-		}
-		if strings.HasSuffix(f.Name, "/") {
-			continue
-		}
-		rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
-		dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
-
-		// Reject anything that escapes the dst root.
-		if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
-			log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
-			skipped++
-			continue
-		}
-
-		if !overwrite {
-			if _, err := os.Stat(dst); err == nil {
-				skipped++
-				continue
-			}
-		}
-
-		if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
-			return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
-		}
-
-		if err := writeZipFile(f, dst); err != nil {
-			return written, skipped, err
-		}
-		written++
-	}
-	return written, skipped, nil
-}
-
-func writeZipFile(f *zip.File, dst string) error {
-	rc, err := f.Open()
-	if err != nil {
-		return fmt.Errorf("open zip entry %s: %w", f.Name, err)
-	}
-	defer rc.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return fmt.Errorf("create %s: %w", dst, err)
-	}
-	defer out.Close()
-
-	if _, err := io.Copy(out, rc); err != nil {
-		return fmt.Errorf("write %s: %w", dst, err)
-	}
-	return nil
-}

internal/snapshot/export.go

@@ -1,132 +0,0 @@
-package snapshot
-
-import (
-	"archive/zip"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"time"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-// ExportOptions controls what an export includes.
-type ExportOptions struct {
-	IncludePluginOutput bool
-	// PluginOutputDir is the absolute path on disk that holds plugin-produced
-	// artifacts (typically $FILES_PATH/plugin). Required when
-	// IncludePluginOutput is true.
-	PluginOutputDir string
-	// SourceHost is recorded in the manifest so imports can attribute
-	// where the snapshot came from (defaults to OS hostname).
-	SourceHost string
-}
-
-// Exporter writes snapshot ZIPs from a live Redis store.
-type Exporter struct {
-	storage *storage.Client
-}
-
-func NewExporter(stor *storage.Client) *Exporter {
-	return &Exporter{storage: stor}
-}
-
-// Export streams a ZIP archive to w. The archive layout is:
-//
-//	manifest.json
-//	index.json
-//	metadata/{cid}.json    (one per known CID)
-//
-// Optional sections (hash-index, plugin-output) are gated by ExportOptions
-// but are not yet implemented — they live on filesystems meta-core does not
-// own, and need cross-service collection. The flag is plumbed for forward
-// compatibility but does nothing today.
-func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
-	if e.storage == nil || !e.storage.IsConnected() {
-		return Manifest{}, fmt.Errorf("storage not connected")
-	}
-
-	hashIDs, err := e.storage.GetAllHashIDs()
-	if err != nil {
-		return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
-	}
-
-	zw := zip.NewWriter(w)
-	defer zw.Close()
-
-	// 1. manifest.json — written last after we know FileCount, but we need
-	//    the entry order to put it first in the archive for readers. Solve
-	//    by buffering in memory: it's tiny.
-	host := opts.SourceHost
-	if host == "" {
-		if h, err := os.Hostname(); err == nil {
-			host = h
-		}
-	}
-	manifest := Manifest{
-		SchemaVersion: ManifestSchemaVersion,
-		GeneratedAt:   time.Now().UTC(),
-		SourceHost:    host,
-		FileCount:     len(hashIDs),
-		Includes: Includes{
-			Metadata:     true,
-			PluginOutput: opts.IncludePluginOutput,
-		},
-	}
-	if err := writeJSON(zw, ManifestFile, manifest); err != nil {
-		return Manifest{}, fmt.Errorf("write manifest: %w", err)
-	}
-
-	// 2. index.json — flat list of CIDs. Redundant with metadata/ but
-	//    cheap and useful for readers that only need the keyset.
-	if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
-		return Manifest{}, fmt.Errorf("write index: %w", err)
-	}
-
-	// 3. metadata/{cid}.json — one nested document per CID.
-	for _, hashID := range hashIDs {
-		flat, err := e.storage.GetMetadataFlat(hashID)
-		if err != nil {
-			log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
-			continue
-		}
-		if len(flat) == 0 {
-			continue
-		}
-		nested := Reconstruct(flat)
-		entry := MetadataDir + hashID + ".json"
-		if err := writeJSON(zw, entry, nested); err != nil {
-			return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
-		}
-	}
-
-	// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
-	//    streams each file into the ZIP. Heavy (~120 MB on a populated dev
-	//    stack), so it's gated by the ExportOptions toggle.
-	if opts.IncludePluginOutput {
-		if opts.PluginOutputDir == "" {
-			return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
-		}
-		n, err := addPluginOutput(zw, opts.PluginOutputDir)
-		if err != nil {
-			return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
-		}
-		log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
-	}
-
-	log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
-		len(hashIDs), host, opts.IncludePluginOutput)
-	return manifest, nil
-}
-
-func writeJSON(zw *zip.Writer, name string, body interface{}) error {
-	w, err := zw.Create(name)
-	if err != nil {
-		return err
-	}
-	enc := json.NewEncoder(w)
-	enc.SetIndent("", "  ")
-	return enc.Encode(body)
-}

internal/snapshot/flatten.go

@@ -1,195 +0,0 @@
-package snapshot
-
-import (
-	"fmt"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-)
-
-// Flat keys use "/" as the separator. Property paths inside Redis are
-// stored as e.g. "fileinfo/streamdetails/video/0/codec".
-const flatSep = "/"
-
-// numericKey matches a path segment that is a non-negative decimal integer.
-// Used to detect array slots (".../video/0/codec").
-var numericKey = regexp.MustCompile(`^\d+$`)
-
-// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
-// Keys are property paths (no leading slash, no file:{cid}/ prefix).
-//
-// Sibling keys whose path segments are all sequential integers starting at 0
-// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
-// are coerced from string to int/float/bool/null when they parse cleanly so
-// the exported JSON is human-readable; on import we round-trip back to strings.
-func Reconstruct(flat map[string]string) interface{} {
-	root := map[string]interface{}{}
-	for path, value := range flat {
-		parts := strings.Split(path, flatSep)
-		setPath(root, parts, parseValue(value))
-	}
-	return convertNumericMapsToArrays(root)
-}
-
-// Flatten is the inverse of Reconstruct: walks a nested value and produces a
-// flat property map whose values are all strings (Redis-ready).
-//
-// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
-// values are stored as "" to preserve the field's presence — this matches
-// what flattenMetadata in the TS implementation does.
-func Flatten(value interface{}) map[string]string {
-	out := map[string]string{}
-	flattenInto(value, "", out)
-	return out
-}
-
-func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
-	if len(parts) == 0 {
-		return
-	}
-	cur := root
-	for i, part := range parts {
-		if i == len(parts)-1 {
-			cur[part] = leaf
-			return
-		}
-		next, ok := cur[part].(map[string]interface{})
-		if !ok {
-			next = map[string]interface{}{}
-			cur[part] = next
-		}
-		cur = next
-	}
-}
-
-func convertNumericMapsToArrays(v interface{}) interface{} {
-	switch t := v.(type) {
-	case map[string]interface{}:
-		// Recurse first so child arrays are detected before parent.
-		for k, child := range t {
-			t[k] = convertNumericMapsToArrays(child)
-		}
-		if !looksLikeArray(t) {
-			return t
-		}
-		// Sort numerically and emit as []interface{}.
-		idx := make([]int, 0, len(t))
-		for k := range t {
-			n, _ := strconv.Atoi(k)
-			idx = append(idx, n)
-		}
-		sort.Ints(idx)
-		arr := make([]interface{}, len(idx))
-		for i, n := range idx {
-			arr[i] = t[strconv.Itoa(n)]
-		}
-		return arr
-	case []interface{}:
-		for i := range t {
-			t[i] = convertNumericMapsToArrays(t[i])
-		}
-		return t
-	default:
-		return v
-	}
-}
-
-// looksLikeArray returns true when every key in m is a non-negative integer
-// and the keys form a contiguous range starting at 0. Empty maps return false
-// — we can't tell whether the original was {} or [], so prefer {}.
-func looksLikeArray(m map[string]interface{}) bool {
-	if len(m) == 0 {
-		return false
-	}
-	seen := make(map[int]bool, len(m))
-	for k := range m {
-		if !numericKey.MatchString(k) {
-			return false
-		}
-		n, err := strconv.Atoi(k)
-		if err != nil || n < 0 {
-			return false
-		}
-		seen[n] = true
-	}
-	for i := 0; i < len(m); i++ {
-		if !seen[i] {
-			return false
-		}
-	}
-	return true
-}
-
-func flattenInto(v interface{}, path string, out map[string]string) {
-	switch t := v.(type) {
-	case nil:
-		out[path] = ""
-	case map[string]interface{}:
-		if len(t) == 0 && path != "" {
-			// Empty objects don't have a Redis representation; skip.
-			return
-		}
-		for k, child := range t {
-			next := k
-			if path != "" {
-				next = path + flatSep + k
-			}
-			flattenInto(child, next, out)
-		}
-	case []interface{}:
-		for i, child := range t {
-			next := strconv.Itoa(i)
-			if path != "" {
-				next = path + flatSep + next
-			}
-			flattenInto(child, next, out)
-		}
-	case string:
-		out[path] = t
-	case bool:
-		if t {
-			out[path] = "true"
-		} else {
-			out[path] = "false"
-		}
-	case float64:
-		// JSON numbers always decode to float64; emit ints without decimals.
-		if t == float64(int64(t)) {
-			out[path] = strconv.FormatInt(int64(t), 10)
-		} else {
-			out[path] = strconv.FormatFloat(t, 'f', -1, 64)
-		}
-	default:
-		out[path] = fmt.Sprintf("%v", t)
-	}
-}
-
-// parseValue mirrors the TS reconstruct logic: convert obvious primitives
-// back to their typed form for nicer exported JSON, leave anything ambiguous
-// as a string so we don't lose data (e.g. "5.1" channel layouts).
-func parseValue(s string) interface{} {
-	if s == "" {
-		return nil
-	}
-	switch s {
-	case "true":
-		return true
-	case "false":
-		return false
-	case "null":
-		return nil
-	}
-	// Only treat as number when the string is a clean integer or decimal,
-	// no leading zeros / trailing junk. Matches the TS regex.
-	if isCleanNumber(s) {
-		if n, err := strconv.ParseFloat(s, 64); err == nil {
-			return n
-		}
-	}
-	return s
-}
-
-var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
-
-func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }

internal/snapshot/import.go

@@ -1,262 +0,0 @@
-package snapshot
-
-import (
-	"archive/zip"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"strings"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-// Mode controls how an import reconciles existing CIDs.
-type Mode string
-
-const (
-	// ModeReplace deletes every existing property of an imported CID and
-	// writes the snapshot's values verbatim. Untouched CIDs are left alone.
-	ModeReplace Mode = "replace"
-	// ModeMerge keeps unmentioned existing properties and only touches the
-	// fields present in the snapshot. Conflict resolution within merge is
-	// controlled by Conflict.
-	ModeMerge Mode = "merge"
-)
-
-// Conflict picks a winner for fields present on both sides during merge.
-type Conflict string
-
-const (
-	// ConflictMine keeps the existing (target) value when both sides have one.
-	ConflictMine Conflict = "mine"
-	// ConflictSource overwrites the existing value with the snapshot's value.
-	ConflictSource Conflict = "source"
-)
-
-// ImportOptions controls how Import applies a snapshot.
-type ImportOptions struct {
-	Mode     Mode
-	Conflict Conflict
-	DryRun   bool
-	// PluginOutputDir is the on-disk destination for cache/plugin-output/*
-	// entries. When empty, plugin-output is silently skipped on import.
-	PluginOutputDir string
-}
-
-// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
-type FileResult struct {
-	HashID    string `json:"hashId"`
-	Added     int    `json:"added"`     // new property keys written
-	Updated   int    `json:"updated"`   // existing property keys whose value changed
-	Unchanged int    `json:"unchanged"` // existing property keys whose value matched
-	Kept      int    `json:"kept"`      // existing property keys preserved by ConflictMine
-	Deleted   int    `json:"deleted"`   // existing property keys removed (replace mode)
-	Error     string `json:"error,omitempty"`
-}
-
-// PluginOutputResult summarizes what happened to cache/plugin-output entries.
-type PluginOutputResult struct {
-	Written int `json:"written"`
-	Skipped int `json:"skipped"`
-}
-
-// ImportResult is the response body for the import endpoint.
-type ImportResult struct {
-	Mode         Mode                `json:"mode"`
-	Conflict     Conflict            `json:"conflict,omitempty"`
-	DryRun       bool                `json:"dryRun"`
-	TotalFiles   int                 `json:"totalFiles"`
-	FilesOK      int                 `json:"filesOk"`
-	FilesFailed  int                 `json:"filesFailed"`
-	Files        []FileResult        `json:"files,omitempty"`
-	PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
-}
-
-// Importer applies snapshot ZIPs to a live Redis store.
-type Importer struct {
-	storage *storage.Client
-}
-
-func NewImporter(stor *storage.Client) *Importer {
-	return &Importer{storage: stor}
-}
-
-// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
-// *bytes.Reader after buffering the upload) and applies it to Redis.
-func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
-	if i.storage == nil || !i.storage.IsConnected() {
-		return nil, fmt.Errorf("storage not connected")
-	}
-	if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
-		return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
-	}
-	if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
-		return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
-	}
-
-	zr, err := zip.NewReader(r, size)
-	if err != nil {
-		return nil, fmt.Errorf("open zip: %w", err)
-	}
-
-	var manifest Manifest
-	if err := readJSON(zr, ManifestFile, &manifest); err != nil {
-		return nil, fmt.Errorf("read manifest: %w", err)
-	}
-	if manifest.SchemaVersion > ManifestSchemaVersion {
-		return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
-			manifest.SchemaVersion, ManifestSchemaVersion)
-	}
-	if manifest.SchemaVersion < ManifestSchemaVersion {
-		return nil, fmt.Errorf("snapshot schemaVersion %d is no longer supported (current: %d); v1 used content-hash root keys which are incompatible with the UUID-rooted schema",
-			manifest.SchemaVersion, ManifestSchemaVersion)
-	}
-
-	result := &ImportResult{
-		Mode:     opts.Mode,
-		Conflict: opts.Conflict,
-		DryRun:   opts.DryRun,
-	}
-
-	for _, f := range zr.File {
-		if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
-			continue
-		}
-		hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
-		if hashID == "" {
-			continue
-		}
-		fr := i.applyOne(f, hashID, opts)
-		result.TotalFiles++
-		if fr.Error != "" {
-			result.FilesFailed++
-		} else {
-			result.FilesOK++
-		}
-		result.Files = append(result.Files, fr)
-	}
-
-	// Plugin-output extraction. Only applied on real imports (not dry-run).
-	// If the snapshot doesn't contain a cache/plugin-output/ section, this is
-	// a no-op. If the destination is unset, we skip silently — the manifest's
-	// pluginOutput flag tells the caller it was on the source side either way.
-	if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
-		// Existing files are kept under ConflictMine / replace-doesn't-apply,
-		// overwritten under ConflictSource. Replace mode behaves like
-		// "overwrite" for cache files (it's the destructive choice for
-		// metadata; mirror that intent here).
-		overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
-		written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
-		if err != nil {
-			return result, fmt.Errorf("extract plugin-output: %w", err)
-		}
-		result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
-		log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
-	}
-
-	log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
-		opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
-	return result, nil
-}
-
-func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
-	fr := FileResult{HashID: hashID}
-
-	rc, err := f.Open()
-	if err != nil {
-		fr.Error = fmt.Sprintf("open entry: %v", err)
-		return fr
-	}
-	defer rc.Close()
-
-	var nested interface{}
-	if err := json.NewDecoder(rc).Decode(&nested); err != nil {
-		fr.Error = fmt.Sprintf("decode json: %v", err)
-		return fr
-	}
-	incoming := Flatten(nested)
-
-	existing, err := i.storage.GetMetadataFlat(hashID)
-	if err != nil {
-		fr.Error = fmt.Sprintf("read existing: %v", err)
-		return fr
-	}
-	if existing == nil {
-		existing = map[string]string{}
-	}
-
-	switch opts.Mode {
-	case ModeReplace:
-		// Diff for reporting.
-		toWrite := map[string]string{}
-		for k, v := range incoming {
-			toWrite[k] = v
-			if cur, ok := existing[k]; ok {
-				if cur == v {
-					fr.Unchanged++
-				} else {
-					fr.Updated++
-				}
-			} else {
-				fr.Added++
-			}
-		}
-		for k := range existing {
-			if _, ok := incoming[k]; !ok {
-				fr.Deleted++
-			}
-		}
-		if !opts.DryRun {
-			if _, err := i.storage.DeleteMetadata(hashID); err != nil {
-				fr.Error = fmt.Sprintf("delete existing: %v", err)
-				return fr
-			}
-			if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
-				fr.Error = fmt.Sprintf("write: %v", err)
-				return fr
-			}
-		}
-
-	case ModeMerge:
-		toWrite := map[string]string{}
-		for k, v := range incoming {
-			cur, present := existing[k]
-			switch {
-			case !present:
-				toWrite[k] = v
-				fr.Added++
-			case cur == v:
-				fr.Unchanged++
-			case opts.Conflict == ConflictSource:
-				toWrite[k] = v
-				fr.Updated++
-			case opts.Conflict == ConflictMine:
-				fr.Kept++
-			}
-		}
-		if !opts.DryRun && len(toWrite) > 0 {
-			if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
-				fr.Error = fmt.Sprintf("merge: %v", err)
-				return fr
-			}
-		}
-	}
-
-	return fr
-}
-
-func readJSON(zr *zip.Reader, name string, out interface{}) error {
-	for _, f := range zr.File {
-		if f.Name != name {
-			continue
-		}
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer rc.Close()
-		return json.NewDecoder(rc).Decode(out)
-	}
-	return fmt.Errorf("entry %q not found", name)
-}

internal/snapshot/manifest.go

@@ -1,55 +0,0 @@
-// Package snapshot implements export/import/wipe of MetaMesh metadata.
-//
-// The on-disk format is a ZIP archive containing one nested-JSON file per
-// indexed root. Schema is defined by ManifestSchemaVersion; readers must
-// reject archives whose schemaVersion exceeds what they know.
-//
-// v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries
-// under metadata/<root>.json carry the flat string properties; the cid
-// reverse-index and the duplicates SET are NOT serialized — they are rebuilt
-// transparently on import because the cids/<cid> key-set members ARE flat
-// properties, and SetMetadataFlat / MergeMetadataFlat auto-register a
-// reverse-index alias for every one they see (see
-// internal/storage/cid_resolution.go: cidFromKeysetField). The legacy flat
-// cid_*/midhash256 named fields are not recognised.
-//
-// v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc".
-// v1 archives can no longer be imported — they would resurrect the
-// privileged-hashID schema this layer was built to retire.
-package snapshot
-
-import "time"
-
-// ManifestSchemaVersion is the on-disk format version. Bump when the layout
-// of metadata/{root}.json or the manifest changes in a non-additive way.
-const ManifestSchemaVersion = 2
-
-// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
-const MetadataDir = "metadata/"
-
-// ManifestFile is the path inside the ZIP that holds the manifest JSON.
-const ManifestFile = "manifest.json"
-
-// IndexFile is the path inside the ZIP that holds the file index (CID list).
-const IndexFile = "index.json"
-
-// Manifest is the metadata header stored at the root of every snapshot ZIP.
-type Manifest struct {
-	SchemaVersion int       `json:"schemaVersion"`
-	GeneratedAt   time.Time `json:"generatedAt"`
-	SourceHost    string    `json:"sourceHost,omitempty"`
-	MetaCoreVer   string    `json:"metaCoreVersion,omitempty"`
-	FileCount     int       `json:"fileCount"`
-	Includes      Includes  `json:"includes"`
-}
-
-// Includes records which optional sections are present in a snapshot.
-type Includes struct {
-	Metadata     bool `json:"metadata"`
-	PluginOutput bool `json:"pluginOutput,omitempty"`
-}
-
-// IndexFileBody is the structure of index.json inside the ZIP.
-type IndexFileBody struct {
-	HashIDs []string `json:"hashIds"`
-}

internal/snapshot/wipe.go

@@ -1,42 +0,0 @@
-package snapshot
-
-import (
-	"fmt"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-// WipeScope picks which classes of data to clear.
-type WipeScope struct {
-	Metadata bool // file:* keys + file:__index__
-}
-
-// WipeResult reports what was cleared.
-type WipeResult struct {
-	MetadataDeleted int64 `json:"metadataDeleted"`
-}
-
-// Wiper clears metadata from Redis.
-type Wiper struct {
-	storage *storage.Client
-}
-
-func NewWiper(stor *storage.Client) *Wiper {
-	return &Wiper{storage: stor}
-}
-
-// Wipe applies the requested scope. Returns the count of files removed.
-func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
-	if w.storage == nil || !w.storage.IsConnected() {
-		return nil, fmt.Errorf("storage not connected")
-	}
-	res := &WipeResult{}
-	if scope.Metadata {
-		n, err := w.storage.ClearAllMetadata()
-		if err != nil {
-			return nil, fmt.Errorf("clear metadata: %w", err)
-		}
-		res.MetadataDeleted = n
-	}
-	return res, nil
-}

internal/storage/apply.go

@@ -0,0 +1,612 @@
+package storage
+
+// The apply loop: folds committed truth changes into the cache.
+//
+// The cache's merged view keeps the v1 record layout — `file:<rid>/<field>`
+// rows of the cache kv table — so every reader (record reads, search, the
+// field index, FTS, counters, SSE and its snapshot) is unchanged; those rows
+// are now derived, never written by a client. Each committed change re-merges
+// only the (rid, key) it touched, from the truth votes of every cid in the
+// group. Group membership is synthesised: every cid of a group appears as
+// `file:<rid>/cids/<cid> = "true"` plus the `cid:<cid> → rid` lookup, whatever
+// the cids/* claims say.
+//
+// Changes arrive in commit order (kv.Tx.OnCommit runs in the truth writer), so
+// `meta:applied_seq` only ever covers a prefix of the truth log. A crash
+// between the truth commit and the cache apply leaves the cache behind, never
+// wrong; Connect catches up from truth.
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+const (
+	appliedSeqKey = "meta:applied_seq"
+	// rebuiltAtKey holds the stream id at which the cache was last rebuilt.
+	// An SSE cursor older than it gets `gap`: the rebuild emitted no
+	// per-key events, so the consumer must take a snapshot.
+	rebuiltAtKey = "meta:rebuilt_at"
+)
+
+var localFields = []string{"filePath", "sizeByte", "mtimeNano"}
+
+type applier struct {
+	c *Client
+
+	mu      sync.Mutex
+	queue   []change
+	applied int64
+	notify  chan struct{}
+
+	wake chan struct{}
+	stop chan struct{}
+	done chan struct{}
+}
+
+func newApplier(c *Client, applied int64) *applier {
+	a := &applier{
+		c: c, applied: applied,
+		notify: make(chan struct{}),
+		wake:   make(chan struct{}, 1),
+		stop:   make(chan struct{}),
+		done:   make(chan struct{}),
+	}
+	go a.run()
+	return a
+}
+
+func (a *applier) enqueue(ch []change) {
+	a.mu.Lock()
+	a.queue = append(a.queue, ch...)
+	a.mu.Unlock()
+	select {
+	case a.wake <- struct{}{}:
+	default:
+	}
+}
+
+// wait blocks until the cache has applied every change up to seq.
+func (a *applier) wait(ctx context.Context, seq int64) error {
+	for {
+		a.mu.Lock()
+		if a.applied >= seq {
+			a.mu.Unlock()
+			return nil
+		}
+		n := a.notify
+		a.mu.Unlock()
+		select {
+		case <-n:
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-a.stop:
+			return errors.New("storage closed before the cache applied the write")
+		}
+	}
+}
+
+func (a *applier) close() {
+	close(a.stop)
+	<-a.done
+}
+
+const applyBatchMax = 2000
+
+func (a *applier) run() {
+	defer close(a.done)
+	for {
+		select {
+		case <-a.wake:
+		case <-a.stop:
+			return
+		}
+		for {
+			a.mu.Lock()
+			n := min(len(a.queue), applyBatchMax)
+			batch := append([]change(nil), a.queue[:n]...)
+			a.mu.Unlock()
+			if len(batch) == 0 {
+				break
+			}
+			if err := a.c.applyBatch(batch); err != nil {
+				log.Printf("[Storage] apply of %d changes failed, retrying: %v", len(batch), err)
+				select {
+				case <-time.After(time.Second):
+					continue
+				case <-a.stop:
+					return
+				}
+			}
+			a.mu.Lock()
+			a.queue = a.queue[n:]
+			if s := batch[len(batch)-1].seq; s > a.applied {
+				a.applied = s
+			}
+			close(a.notify)
+			a.notify = make(chan struct{})
+			a.mu.Unlock()
+		}
+	}
+}
+
+// applyBatch folds committed changes into the cache in one transaction.
+func (c *Client) applyBatch(batch []change) error {
+	s, err := c.kv()
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	return s.Update(ctx, func(tx *kv.Tx) error {
+		return c.applyTx(tx, batch)
+	})
+}
+
+func (c *Client) applyTx(tx *kv.Tx, batch []change) error {
+	ctx := tx.Context()
+	marks := map[string]map[string]bool{}
+	mark := func(rid, key string) {
+		if marks[rid] == nil {
+			marks[rid] = map[string]bool{}
+		}
+		marks[rid][key] = true
+	}
+	all := map[string]bool{}
+	dirty := map[string]bool{}
+	var maxSeq int64
+	for _, it := range batch {
+		maxSeq = max(maxSeq, it.seq)
+		if it.regroup != nil {
+			loser, survivor := it.regroup[0], it.regroup[1]
+			if _, err := tx.DeletePrefix(recordPrefix(loser), 0); err != nil {
+				return err
+			}
+			if _, err := tx.Delete(rootKey(loser)); err != nil {
+				return err
+			}
+			if err := deleteSearchRowTx(tx, loser); err != nil {
+				return err
+			}
+			delete(marks, loser)
+			cids, err := c.truthMembers(ctx, survivor)
+			if err != nil {
+				return err
+			}
+			for _, cid := range cids {
+				if _, err := syncMember(tx, cid, survivor); err != nil {
+					return err
+				}
+			}
+			all[survivor], dirty[survivor] = true, true
+			continue
+		}
+		rid, err := c.truthRid(ctx, it.cid)
+		if err != nil {
+			return err
+		}
+		if rid == "" {
+			continue
+		}
+		newRoot, err := syncMember(tx, it.cid, rid)
+		if err != nil {
+			return err
+		}
+		if newRoot {
+			dirty[rid] = true
+		}
+		if it.key == "" {
+			continue
+		}
+		if it.live {
+			_, err = tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, it.pid, it.cid, it.key)
+		} else {
+			_, err = tx.Exec(`DELETE FROM claim_by_pid WHERE pid = ? AND cid = ? AND key = ?`, it.pid, it.cid, it.key)
+		}
+		if err != nil {
+			return err
+		}
+		mark(rid, it.key)
+	}
+	for rid := range all {
+		cids, err := cidsForRoot(ctx, tx, rid)
+		if err != nil {
+			return err
+		}
+		keys, err := c.claimKeys(ctx, cids)
+		if err != nil {
+			return err
+		}
+		for _, k := range append(keys, localFields...) {
+			mark(rid, k)
+		}
+	}
+	for rid, keys := range marks {
+		cids, err := cidsForRoot(ctx, tx, rid)
+		if err != nil {
+			return err
+		}
+		for key := range keys {
+			changed, err := c.remergeTx(tx, rid, cids, key)
+			if err != nil {
+				return err
+			}
+			if changed && affectsHaystack(key) {
+				dirty[rid] = true
+			}
+		}
+	}
+	for rid := range dirty {
+		if err := refreshSearchTx(tx, rid); err != nil {
+			return err
+		}
+	}
+	if maxSeq > 0 {
+		return tx.Put(appliedSeqKey, strconv.FormatInt(maxSeq, 10))
+	}
+	return nil
+}
+
+// syncMember makes cid a member of rid in the cache: the lookup row, the
+// synthesised key-set member, and the root. Reports whether rid is new.
+func syncMember(tx *kv.Tx, cid, rid string) (bool, error) {
+	cur, _, err := tx.Get(tx.Context(), CIDIndexPrefix+cid)
+	if err != nil {
+		return false, err
+	}
+	if cur != rid {
+		if cur != "" {
+			if _, err := tx.Delete(recordPrefix(cur) + CIDsKeyPrefix + cid); err != nil {
+				return false, err
+			}
+		}
+		if err := tx.Put(CIDIndexPrefix+cid, rid); err != nil {
+			return false, err
+		}
+	}
+	if err := tx.Put(recordPrefix(rid)+CIDsKeyPrefix+cid, "true"); err != nil {
+		return false, err
+	}
+	return tx.Set(rootKey(rid), "")
+}
+
+// remergeTx recomputes file:<rid>/<key> from the group's votes (plus the
+// box-local facts for filePath / sizeByte / mtimeNano) and reports whether
+// the merged row changed.
+func (c *Client) remergeTx(tx *kv.Tx, rid string, cids []string, key string) (bool, error) {
+	if strings.HasPrefix(key, CIDsKeyPrefix) {
+		return false, nil // membership is synthesised, never merged
+	}
+	votes, err := c.votesFor(tx.Context(), cids, key)
+	if err != nil {
+		return false, err
+	}
+	res := Merge(votes, "")
+	val, ok := res.Value, res.OK
+	if isLocalField(key) {
+		lv, lok, err := localFact(tx, cids, key)
+		if err != nil {
+			return false, err
+		}
+		// The disk is the truth for where and when a file is; a signed
+		// sizeByte (a content fact) wins over the on-disk size.
+		if lok && (key != "sizeByte" || !ok) {
+			val, ok = lv, true
+		}
+	}
+	k := recordPrefix(rid) + key
+	if ok {
+		return tx.Set(k, val)
+	}
+	return tx.Delete(k)
+}
+
+// localFact reads a box-local field from local_file for the group's cids.
+func localFact(tx *kv.Tx, cids []string, key string) (string, bool, error) {
+	if len(cids) == 0 {
+		return "", false, nil
+	}
+	args := make([]any, len(cids))
+	for i, c := range cids {
+		args[i] = c
+	}
+	var path string
+	var size, mtime int64
+	err := tx.QueryRow(`SELECT path, size, mtime FROM local_file WHERE midhash IN (?`+
+		strings.Repeat(",?", len(cids)-1)+`) ORDER BY path LIMIT 1`, args...).Scan(&path, &size, &mtime)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	switch key {
+	case "filePath":
+		return path, true, nil
+	case "sizeByte":
+		return strconv.FormatInt(size, 10), true, nil
+	default:
+		return strconv.FormatInt(mtime, 10), true, nil
+	}
+}
+
+// truthMembers lists every cid of rid's group (scans cid_group: truth has no
+// rid index; only regroups and rebuilds call this).
+func (c *Client) truthMembers(ctx context.Context, rid string) ([]string, error) {
+	rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid FROM cid_group WHERE rid = ?`, rid)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var cid string
+		if err := rows.Scan(&cid); err != nil {
+			return nil, err
+		}
+		out = append(out, cid)
+	}
+	return out, rows.Err()
+}
+
+// truthLastSeq reads the newest committed seq.
+func (c *Client) truthLastSeq(ctx context.Context) (int64, error) {
+	var v string
+	err := c.truth.DB().QueryRowContext(ctx, `SELECT value FROM meta WHERE name = ?`, lastSeqName).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(v, 10, 64)
+}
+
+// catchUp brings the cache level with truth at boot: nothing to do, a replay
+// of the changes after applied_seq (found by a full scan — there is no seq
+// index, and this only runs after an unclean shutdown), or a full rebuild
+// when the cache is missing or a group changed while it was behind.
+func (c *Client) catchUp() (int64, error) {
+	s := c.store
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	last, err := c.truthLastSeq(ctx)
+	if err != nil {
+		return 0, err
+	}
+	appliedStr, has, err := s.Get(ctx, appliedSeqKey)
+	if err != nil {
+		return 0, err
+	}
+	applied, _ := strconv.ParseInt(appliedStr, 10, 64)
+	if has && applied >= last {
+		return applied, nil
+	}
+	if !has {
+		if last == 0 {
+			return 0, nil // fresh store
+		}
+		log.Printf("[Storage] cache is empty, rebuilding from truth (last_seq %d)", last)
+		return last, c.rebuild(ctx, last)
+	}
+	rows, err := c.truth.DB().QueryContext(ctx,
+		`SELECT cid, key, pid, value IS NOT NULL, seq FROM claim WHERE seq > ? ORDER BY seq`, applied)
+	if err != nil {
+		return 0, err
+	}
+	var missed []change
+	regroup := false
+	for rows.Next() {
+		var ch change
+		if err := rows.Scan(&ch.cid, &ch.key, &ch.pid, &ch.live, &ch.seq); err != nil {
+			rows.Close()
+			return 0, err
+		}
+		if strings.HasPrefix(ch.key, CIDsKeyPrefix) {
+			regroup = true
+		}
+		missed = append(missed, ch)
+	}
+	if err := rows.Close(); err != nil {
+		return 0, err
+	}
+	if regroup {
+		log.Printf("[Storage] cache behind by %d changes including a group link, rebuilding", len(missed))
+		return last, c.rebuild(ctx, last)
+	}
+	log.Printf("[Storage] cache behind (applied %d < last %d), replaying %d changes", applied, last, len(missed))
+	for start := 0; start < len(missed); start += applyBatchMax {
+		end := min(start+applyBatchMax, len(missed))
+		if err := c.applyBatch(missed[start:end]); err != nil {
+			return 0, err
+		}
+	}
+	if err := s.Update(ctx, func(tx *kv.Tx) error { return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10)) }); err != nil {
+		return 0, err
+	}
+	return last, nil
+}
+
+// Rebuild drops every derived row of the cache and recomputes it from truth.
+// Exposed for tests and operators; boot calls it when the cache is missing.
+func (c *Client) Rebuild() error {
+	if _, err := c.truthStore(); err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	last, err := c.truthLastSeq(ctx)
+	if err != nil {
+		return err
+	}
+	return c.rebuild(ctx, last)
+}
+
+const rebuildBatch = 500
+
+func (c *Client) rebuild(ctx context.Context, last int64) error {
+	s := c.store
+	started := time.Now()
+	for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
+		for {
+			var n int64
+			if err := s.Update(ctx, func(tx *kv.Tx) error {
+				tx.Quiet()
+				var err error
+				n, err = tx.DeletePrefix(prefix, bulkBatch)
+				return err
+			}); err != nil {
+				return fmt.Errorf("clear %s: %w", prefix, err)
+			}
+			if n < bulkBatch {
+				break
+			}
+		}
+	}
+	if err := s.Update(ctx, func(tx *kv.Tx) error {
+		if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
+			return err
+		}
+		_, err := tx.Exec(`DELETE FROM claim_by_pid`)
+		return err
+	}); err != nil {
+		return err
+	}
+
+	groups := map[string][]string{}
+	var order []string
+	rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid, rid FROM cid_group`)
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var cid, rid string
+		if err := rows.Scan(&cid, &rid); err != nil {
+			rows.Close()
+			return err
+		}
+		if _, ok := groups[rid]; !ok {
+			order = append(order, rid)
+		}
+		groups[rid] = append(groups[rid], cid)
+	}
+	if err := rows.Close(); err != nil {
+		return err
+	}
+	for start := 0; start < len(order); start += rebuildBatch {
+		end := min(start+rebuildBatch, len(order))
+		if err := s.Update(ctx, func(tx *kv.Tx) error {
+			tx.Quiet()
+			for _, rid := range order[start:end] {
+				cids := groups[rid]
+				for _, cid := range cids {
+					if _, err := syncMember(tx, cid, rid); err != nil {
+						return err
+					}
+				}
+				keys, err := c.claimKeys(ctx, cids)
+				if err != nil {
+					return err
+				}
+				for _, key := range append(keys, localFields...) {
+					if _, err := c.remergeTx(tx, rid, cids, key); err != nil {
+						return err
+					}
+				}
+				if err := refreshSearchTx(tx, rid); err != nil {
+					return err
+				}
+			}
+			return nil
+		}); err != nil {
+			return fmt.Errorf("rebuild records: %w", err)
+		}
+	}
+	if err := c.rebuildClaimIndex(ctx); err != nil {
+		return err
+	}
+	if err := c.RebuildCounters(); err != nil {
+		return err
+	}
+	rebuiltAt := strconv.FormatInt(time.Now().UnixMilli(), 10) + "-0"
+	if err := s.Update(ctx, func(tx *kv.Tx) error {
+		if err := tx.Put(rebuiltAtKey, rebuiltAt); err != nil {
+			return err
+		}
+		return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10))
+	}); err != nil {
+		return err
+	}
+	log.Printf("[Storage] cache rebuilt: %d records in %s", len(order), time.Since(started).Round(time.Millisecond))
+	return nil
+}
+
+// rebuildClaimIndex refills claim_by_pid from truth, a page at a time.
+func (c *Client) rebuildClaimIndex(ctx context.Context) error {
+	type row struct {
+		pid      int64
+		cid, key string
+	}
+	var after row
+	first := true
+	for {
+		q := `SELECT pid, cid, key FROM claim WHERE value IS NOT NULL`
+		var args []any
+		if !first {
+			q += ` AND (cid, key, pid) > (?, ?, ?)`
+			args = append(args, after.cid, after.key, after.pid)
+		}
+		q += ` ORDER BY cid, key, pid LIMIT 5000`
+		rs, err := c.truth.DB().QueryContext(ctx, q, args...)
+		if err != nil {
+			return err
+		}
+		var page []row
+		for rs.Next() {
+			var r row
+			if err := rs.Scan(&r.pid, &r.cid, &r.key); err != nil {
+				rs.Close()
+				return err
+			}
+			page = append(page, r)
+		}
+		if err := rs.Close(); err != nil {
+			return err
+		}
+		if len(page) == 0 {
+			return nil
+		}
+		if err := c.store.Update(ctx, func(tx *kv.Tx) error {
+			for _, r := range page {
+				if _, err := tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, r.pid, r.cid, r.key); err != nil {
+					return err
+				}
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+		after, first = page[len(page)-1], false
+	}
+}
+
+// RebuiltAt is the stream id of the last cache rebuild ("" if never).
+func (c *Client) RebuiltAt() string {
+	s, err := c.kv()
+	if err != nil {
+		return ""
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	v, _, _ := s.Get(ctx, rebuiltAtKey)
+	return v
+}

internal/storage/cid_resolution.go

@@ -3,38 +3,24 @@ package storage
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"strings"
-	"time"
 
 	"github.com/metazla/meta-core/internal/cid"
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
-// This file implements the CID-resolution layer that lives on top of the
+// The cid → record resolution layer, read side. A record (rid) is a group of
-// flat metadata storage in client.go. The short version:
+// cids known to be the same content; the group lives in truth's cid_group and
+// is mirrored into the cache as
 //
-//   - Roots are opaque UUIDv7s — file:<uuid>/<property> for every property.
+//	cid:<bareCid>               → <rid>
-//     The UUID is an internal implementation detail; the mental model is
+//	file:<rid>/cids/<bareCid>   = "true"   (synthesised membership, one per cid)
-//     "N CIDs → one record of keys→metadata".
-//   - Every known CID for a file is registered as a reverse-index entry:
-//       cid:<bareCid> → <uuid>                (Redis STRING)
-//     and as a member of the per-root bare-CID key-set:
-//       file:<uuid>/cids/<bareCid> = "true"   (flat key, sentinel value)
-//   - A CID is self-describing: its algorithm is recoverable from the CIDv1
-//     multicodec (see internal/cid), so there is no per-algorithm field name
-//     and no <algo>: token prefix. The canonical externally-visible CID is
-//     *derived by rank* from the key-set on read (GetMetadataDocument), never
-//     stored.
 //
-// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
+// by the apply loop (apply.go). A CID is self-describing (its algorithm is in
-// any write of a cids/<cid> field and register the reverse-index entry, so
+// the CIDv1 multicodec, see internal/cid); the canonical externally-visible
-// writers that just emit the key-set members don't need to know about the
+// CID is derived by rank on read (GetMetadataDocument), never stored.
-// reverse index at all. The hook helpers are *Locked — they assume c.mu is
-// already held by the caller, avoiding nested RLock (which can deadlock
-// against a waiting writer).
 
-// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
+// CIDIndexPrefix is the key prefix for the reverse index. Combined
 // with a bare CID, it forms the full key: "cid:bafk…".
 const CIDIndexPrefix = "cid:"
 
@@ -45,118 +31,6 @@ const CIDIndexPrefix = "cid:"
 // reverse-index entries that need cleanup).
 const CIDsKeyPrefix = "cids/"
 
-// DuplicatesField is the per-root SET key holding additional file paths
-// whose content hashes to the same CIDs as this root. The watcher writes
-// here when it sees a new path for content that's already indexed; meta-dup
-// reads from it.
-const DuplicatesField = "duplicates"
-
-// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
-// cidStr is a bare multibase CIDv1 ("bafk…").
-func (c *Client) buildCIDIndexKey(cidStr string) string {
-	return c.buildKey(CIDIndexPrefix + cidStr)
-}
-
-// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
-// in the file index. Returns the new UUID. The caller is expected to follow
-// up with AddAlias for at least one CID — a root without any alias is
-// reachable only via the index, which works but defeats the point.
-func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return "", fmt.Errorf("not connected")
-	}
-
-	uuid, err := NewUUID()
-	if err != nil {
-		return "", fmt.Errorf("uuid: %w", err)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	prefix := c.buildKeyPrefix(uuid)
-	pipe := c.client.Pipeline()
-	pipe.Set(ctx, prefix+"filePath", filePath, 0)
-	pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
-	pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
-	pipe.SAdd(ctx, c.buildIndexKey(), uuid)
-	c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
-	if _, err := pipe.Exec(ctx); err != nil {
-		return "", fmt.Errorf("mint pipeline: %w", err)
-	}
-	return uuid, nil
-}
-
-// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
-// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
-// Idempotent: registering the same CID twice is a no-op.
-func (c *Client) AddAlias(uuid, cidStr string) error {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	return c.addAliasLocked(ctx, uuid, cidStr)
-}
-
-// addAliasLocked is the lock-free body of AddAlias. The caller must already
-// hold c.mu (read or write). Split out so the alias-maintenance hooks
-// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
-// the work without re-entering the lock — nested RLock can deadlock against
-// a waiting writer.
-func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
-	if uuid == "" || cidStr == "" {
-		return fmt.Errorf("uuid and cid are required")
-	}
-
-	keysetField := CIDsKeyPrefix + cidStr
-	keysetKey := c.buildKeyPrefix(uuid) + keysetField
-
-	// Every branch below writes the key-set member, so index the field name
-	// once here — including the two guard branches that skip the alias itself.
-	// Missing it would leave `cids/<cid>` invisible to a record read.
-	defer c.addFieldLocked(ctx, uuid, keysetField)
-
-	// Self-pointing alias guard. If the caller is trying to register
-	// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
-	// dual-root failure mode in disguise — it means somebody wrote
-	// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
-	// write; leave any existing alias alone. Still record the key-set
-	// membership so the field-level write succeeds.
-	if uuid == cidStr {
-		return c.client.Set(ctx, keysetKey, "true", 0).Err()
-	}
-
-	// Guard against the dual-root pattern: if cid:<cid> already points at a
-	// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
-	// just record the key-set membership so it's still discoverable via the
-	// root → cids enumeration.
-	indexKey := c.buildCIDIndexKey(cidStr)
-	existing, err := c.client.Get(ctx, indexKey).Result()
-	if err != nil && err != redis.Nil {
-		return fmt.Errorf("read existing alias: %w", err)
-	}
-	if existing != "" && existing != uuid {
-		return c.client.Set(ctx, keysetKey, "true", 0).Err()
-	}
-
-	pipe := c.client.Pipeline()
-	pipe.Set(ctx, indexKey, uuid, 0)
-	pipe.Set(ctx, keysetKey, "true", 0)
-	if _, err := pipe.Exec(ctx); err != nil {
-		return fmt.Errorf("add alias pipeline: %w", err)
-	}
-	return nil
-}
-
 // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
 // lookup, an SSE consumer) to the actual storage root key.
 //
@@ -174,8 +48,8 @@ func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error
 //   - hash matches a reverse-index entry → return UUID
 //   - otherwise return hash unchanged (legacy / explicit-UUID writes)
 //
-// Errors are converted to "no resolution" — if Redis is down or the lookup
+// Errors are converted to "no resolution" — if the store is down or the
-// fails we let the caller proceed with the bare hash and surface the Redis
+// lookup fails we let the caller proceed with the bare hash and surface the
 // failure on the actual operation.
 func (c *Client) ResolveRoot(hash string) string {
 	if hash == "" {
@@ -192,155 +66,35 @@ func (c *Client) ResolveRoot(hash string) string {
 // when the CID is unknown; the empty string is the "not found" signal rather
 // than a typed error so callers can branch without errors.As.
 func (c *Client) GetByCID(cidStr string) (string, error) {
-	c.mu.RLock()
+	s, err := c.kv()
-	defer c.mu.RUnlock()
+	if err != nil {
-
+		return "", err
-	if c.client == nil {
-		return "", fmt.Errorf("not connected")
 	}
 	if cidStr == "" {
 		return "", nil
 	}
-
+	ctx, cancel := opCtx()
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
-
+	uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
-	uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
-	if err == redis.Nil {
-		return "", nil
-	}
 	if err != nil {
 		return "", fmt.Errorf("get cid index: %w", err)
 	}
 	return uuid, nil
 }
 
-// AddDuplicatePath records that path is another physical location whose
+// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
-// content matches uuid. Returns true if path was newly added (not already
+// stripped).
-// in the set). The watcher calls this when it sees a file whose midhash
+func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
-// matches an existing root but at a different filePath.
+	prefix := recordPrefix(uuid) + CIDsKeyPrefix
-func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
+	rows, err := r.Range(ctx, prefix, "", 0)
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return false, fmt.Errorf("not connected")
-	}
-	if uuid == "" || path == "" {
-		return false, nil
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
 	if err != nil {
-		return false, fmt.Errorf("sadd duplicates: %w", err)
+		return nil, err
 	}
-	return added > 0, nil
+	out := make([]string, 0, len(rows))
-}
+	for _, row := range rows {
-
+		out = append(out, row.K[len(prefix):])
-// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
-// prefix stripped). Caller must hold c.mu.
-//
-// Reads the field index and filters by prefix; only an un-indexed root falls
-// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
-func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
-	return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
-}
-
-// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
-// (including the cids/<cid> key-set), every reverse-index entry for those
-// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
-// is a successful no-op.
-func (c *Client) DeleteRoot(uuid string) error {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return fmt.Errorf("not connected")
-	}
-	if uuid == "" {
-		return nil
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	prefix := c.buildKeyPrefix(uuid)
-
-	// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
-	//    the list of reverse-index entries to clean up.
-	cids, err := c.cidsForRootLocked(ctx, uuid)
-	if err != nil {
-		return err
-	}
-
-	// 2. Delete every reverse-index entry in one shot.
-	if len(cids) > 0 {
-		keys := make([]string, 0, len(cids))
-		for _, cidStr := range cids {
-			keys = append(keys, c.buildCIDIndexKey(cidStr))
-		}
-		if err := c.client.Del(ctx, keys...).Err(); err != nil {
-			return fmt.Errorf("del cid index: %w", err)
-		}
-	}
-
-	// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return fmt.Errorf("scan root keys: %w", err)
-		}
-		if len(keys) > 0 {
-			if err := c.client.Del(ctx, keys...).Err(); err != nil {
-				return fmt.Errorf("del root keys: %w", err)
-			}
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-
-	// 4. Remove from the file index set.
-	if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
-		return fmt.Errorf("srem index: %w", err)
-	}
-	// Marked so the incremental search-index refresh reads it, finds it gone,
-	// and evicts it — otherwise a deleted root keeps answering searches until
-	// the next full rebuild.
-	c.markDirtyLocked(ctx, uuid)
-	return nil
-}
-
-// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
-// root. If it's a key-set member — field name starts with "cids/" — it
-// registers the implied CID in the reverse index. Writers emit cids/<cid>
-// members without knowing about the reverse index; this hook keeps it in
-// sync transparently.
-//
-// Caller must hold c.mu. Errors are intentionally swallowed at the call site
-// (logged, not returned): an out-of-sync reverse index is recoverable via a
-// sweep, while a failed property write is not.
-func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
-	cidStr := cidFromKeysetField(field)
-	if cidStr == "" {
-		return nil
-	}
-	return c.addAliasLocked(ctx, uuid, cidStr)
-}
-
-// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
-// over every entry in a metadata map. Used by SetMetadataFlat /
-// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
-// hold c.mu.
-func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
-	for k := range metadata {
-		_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
 	}
+	return out, nil
 }
 
 // cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
@@ -370,24 +124,23 @@ type MetadataDocument struct {
 // the duplicates SET. Returns nil if the root has no keys at all (deleted or
 // never existed).
 func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
-	c.mu.RLock()
+	s, err := c.kv()
-	defer c.mu.RUnlock()
+	if err != nil {
-
+		return nil, err
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
 	}
 	if uuid == "" {
 		return nil, nil
 	}
-
+	ctx, cancel := opCtx()
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	flat, err := c.getMetadataFlatInternal(ctx, uuid)
+	flat, err := readRecord(ctx, s, uuid)
 	if err != nil {
 		return nil, err
 	}
-	prefix := c.buildKeyPrefix(uuid)
+	if flat == nil {
+		flat = make(map[string]string)
+	}
 
 	// Hoist the cids/<cid> key-set members into a typed array and derive the
 	// canonical CID by rank. The members remain in Flat too — that's how
@@ -407,9 +160,16 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
 		}
 	}
 
-	dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
+	// Duplicates: every other local path holding the same content.
-	if err != nil && err != redis.Nil {
+	paths, err := localPaths(ctx, s.DB(), cids)
-		return nil, fmt.Errorf("smembers duplicates: %w", err)
+	if err != nil {
+		return nil, fmt.Errorf("read duplicates: %w", err)
+	}
+	dups := make([]string, 0, len(paths))
+	for _, p := range paths {
+		if p != flat["filePath"] {
+			dups = append(dups, p)
+		}
 	}
 
 	if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
@@ -422,50 +182,3 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
 		Duplicates: dups,
 	}, nil
 }
-
-// RegisterFileTuple records one physical file under its midhash256 CID, the
-// same way the file watcher does for every file it scans: mint a UUID root
-// (filePath/sizeByte/mtimeNano) and write `cids/<midhash>`, whose hook
-// registers the cid:<midhash> reverse alias. Content already known under
-// another path is recorded as a duplicate; the same path just refreshes
-// size/mtime.
-//
-// Returns the root and whether it was newly minted. Shared by the watcher and
-// POST /api/files/register, which is how a box running with the watcher off
-// (a meta-watch client core) still gets files it writes under /files/plugin
-// resolvable by CID.
-func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
-	if midhash == "" {
-		return "", false, fmt.Errorf("midhash is required")
-	}
-	uuid, err := c.GetByCID(midhash)
-	if err != nil {
-		return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
-	}
-
-	if uuid == "" {
-		uuid, err = c.Mint(absPath, size, mtimeNano)
-		if err != nil {
-			return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
-		}
-		if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
-			return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
-		}
-		return uuid, true, nil
-	}
-
-	existing, _ := c.GetProperty(uuid, "filePath")
-	if existing != absPath {
-		if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
-			return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
-		}
-		return uuid, false, nil
-	}
-	if _, err := c.MergeMetadataFlat(uuid, map[string]string{
-		"sizeByte":  strconv.FormatInt(size, 10),
-		"mtimeNano": strconv.FormatInt(mtimeNano, 10),
-	}); err != nil {
-		return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
-	}
-	return uuid, false, nil
-}

internal/storage/claims_test.go

@@ -0,0 +1,419 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+func TestWriteClaimStatuses(t *testing.T) {
+	c := newTestClient(t)
+	a := signer(t, 1)
+	good := vote(t, a, "bafyA", "title", "The Matrix", 100)
+	tampered := vote(t, a, "bafyA", "year", "1999", 100)
+	tampered.Value = claim.Text("2000")
+	highS := vote(t, a, "bafyA", "studio", "WB", 100)
+	highS.Sig = flipS(highS.Sig)
+
+	got := write(t, c,
+		good,
+		tampered,
+		highS,
+		claim.Claim{CID: "bafyA", UID: a.UID, TS: 1}, // no key
+		vote(t, a, "bafyA", "cid_sha256", "x", 100),  // removed CID shape
+	)
+	want := []ClaimStatus{StatusAccepted, StatusBadSignature, StatusBadSignature, StatusBadRequest, StatusBadRequest}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("statuses %v, want %v", got, want)
+	}
+	if st := write(t, c, good); st[0] != StatusUnchanged {
+		t.Fatalf("identical re-send: %s, want unchanged", st[0])
+	}
+	if st := write(t, c, vote(t, a, "bafyA", "title", "Older", 50)); st[0] != StatusStale {
+		t.Fatalf("older ts: %s, want stale", st[0])
+	}
+	if st := write(t, c, vote(t, a, "bafyA", "title", "Same ts other value", 100)); st[0] != StatusStale {
+		t.Fatalf("equal ts, different claim: %s, want stale", st[0])
+	}
+	if got := record(t, c, "bafyA")["title"]; got != "The Matrix" {
+		t.Fatalf("title = %q", got)
+	}
+	// A re-persist of the same value with a newer ts keeps the stored row (and
+	// its older ts): it is "unchanged", not a new version.
+	if st := write(t, c, vote(t, a, "bafyA", "title", "The Matrix", 200)); st[0] != StatusUnchanged {
+		t.Fatalf("same value, newer ts: %s, want unchanged", st[0])
+	}
+	// Proof the stored ts stayed 100: a new value at 150 (< 200) is accepted.
+	if st := write(t, c, vote(t, a, "bafyA", "title", "Updated", 150)); st[0] != StatusAccepted {
+		t.Fatalf("new value at ts 150 after a same-value re-send at 200: %s, want accepted", st[0])
+	}
+}
+
+func flipS(sig []byte) []byte {
+	// n - s for secp256k1, computed via the vectors' helper in internal/claim
+	// would be cleaner; here big-endian subtraction on bytes is enough.
+	n := []byte{
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+		0xba, 0xae, 0xdc, 0xe6, 0xaf, 0x48, 0xa0, 0x3b, 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x41,
+	}
+	out := append([]byte{}, sig...)
+	borrow := 0
+	for i := 31; i >= 0; i-- {
+		d := int(n[i]) - int(sig[32+i]) - borrow
+		borrow = 0
+		if d < 0 {
+			d += 256
+			borrow = 1
+		}
+		out[32+i] = byte(d)
+	}
+	return out
+}
+
+// A retraction is kept as a tombstone, so the vote it retracted cannot be
+// replayed — by its signer's old copy or by a peer that still holds it.
+func TestRetractionKeepsTombstoneAndBlocksReplay(t *testing.T) {
+	c := newTestClient(t)
+	a := signer(t, 1)
+	old := vote(t, a, "bafyA", "genres/Drama", "true", 10)
+	mustAccept(t, c, old, vote(t, a, "bafyA", "title", "X", 10))
+	mustAccept(t, c, vote(t, a, "bafyA", "genres/Drama", nil, 20))
+	if _, ok := record(t, c, "bafyA")["genres/Drama"]; ok {
+		t.Fatal("retracted member still merged")
+	}
+	if st := write(t, c, old); st[0] != StatusStale {
+		t.Fatalf("replay of the retracted vote: %s, want stale", st[0])
+	}
+	if n, _ := c.CountClaims(); n != 2 {
+		t.Fatalf("claims = %d, want 2 (tombstone kept)", n)
+	}
+	live, _ := c.ClaimsForCIDs([]string{"bafyA"}, "")
+	if len(live) != 1 || live[0].Key != "title" {
+		t.Fatalf("live claims = %+v, want only title", live)
+	}
+}
+
+func TestMergeRule(t *testing.T) {
+	tx := func(s string) claim.Value { return claim.Text(s) }
+	cases := []struct {
+		name   string
+		votes  []Vote
+		viewer string
+		want   string
+		nsrc   int
+		ok     bool
+	}{
+		{"support wins", []Vote{{Uid: "a", Value: tx("X"), TS: 9}, {Uid: "b", Value: tx("x "), TS: 9}, {Uid: "c", Value: tx("Y"), TS: 1}}, "", "X", 2, true},
+		{"tie: oldest wins", []Vote{{Uid: "a", Value: tx("New"), TS: 9}, {Uid: "b", Value: tx("Old"), TS: 1}}, "", "Old", 1, true},
+		{"tie: smallest wins", []Vote{{Uid: "a", Value: tx("b"), TS: 5}, {Uid: "b", Value: tx("a"), TS: 5}}, "", "a", 1, true},
+		{"signer counts once, newest", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: tx("Y"), TS: 2}, {Uid: "b", Value: tx("Y"), TS: 3}}, "", "Y", 2, true},
+		{"retraction drops", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: claim.Null(), TS: 2}}, "", "", 0, false},
+		{"ciphertext never merges", []Vote{{Uid: "a", Value: claim.Bytes([]byte{1}), TS: 1}}, "", "", 0, false},
+		{"viewer wins", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "b", Value: tx("X"), TS: 1}, {Uid: "v", Value: tx("Mine"), TS: 9}}, "v", "Mine", 1, true},
+		{"viewer retraction does not hide", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "v", Value: claim.Null(), TS: 9}}, "v", "X", 1, true},
+	}
+	for _, tc := range cases {
+		got := Merge(tc.votes, tc.viewer)
+		if got.OK != tc.ok || got.Value != tc.want || (tc.ok && got.Nsrc != tc.nsrc) {
+			t.Errorf("%s: got %+v, want %q nsrc %d ok %v", tc.name, got, tc.want, tc.nsrc, tc.ok)
+		}
+	}
+}
+
+func TestMergedViewAndClaims(t *testing.T) {
+	c := newTestClient(t)
+	gwA, gwB, gwC := signer(t, 1), signer(t, 2), signer(t, 3)
+	mustAccept(t, c,
+		vote(t, gwA, "bafyA", "title", "Matrix", 10),
+		vote(t, gwB, "bafyA", "title", "Matrix", 20),
+		vote(t, gwC, "bafyA", "title", "The Matrix (1999)", 5),
+	)
+	rec := record(t, c, "bafyA")
+	if rec["title"] != "Matrix" || rec["cids/bafyA"] != "true" {
+		t.Fatalf("merged %v", rec)
+	}
+	claims, err := c.ClaimsForRID(rid(t, c, "bafyA"), "title")
+	if err != nil || len(claims) != 3 {
+		t.Fatalf("claims %d (err %v), want 3", len(claims), err)
+	}
+	for _, cl := range claims {
+		if err := cl.Verify(); err != nil {
+			t.Fatalf("returned claim does not verify: %v", err)
+		}
+	}
+}
+
+// A cids/* link merges two groups: the older rid survives, the loser's rows
+// go with `del` events, and the survivor carries both groups' votes.
+func TestGroupLinkMergesRecords(t *testing.T) {
+	c := newTestClient(t)
+	a, b := signer(t, 1), signer(t, 2)
+	mustAccept(t, c, vote(t, a, "bafyM", "title", "Local copy", 10))
+	older := rid(t, c, "bafyM")
+	mustAccept(t, c, vote(t, b, "bafyQ", "year", "1999", 10))
+	newer := rid(t, c, "bafyQ")
+	if older == newer || older > newer {
+		t.Fatalf("expected two groups, older first: %s %s", older, newer)
+	}
+	before, _ := c.Store().LastID(context.Background(), MetaEventsStream)
+
+	mustAccept(t, c, vote(t, a, "bafyM", "cids/bafyQ", "true", 11))
+	if got := rid(t, c, "bafyQ"); got != older {
+		t.Fatalf("bafyQ → %s, want the older rid %s", got, older)
+	}
+	rec := record(t, c, "bafyM")
+	if rec["title"] != "Local copy" || rec["year"] != "1999" || rec["cids/bafyQ"] != "true" || rec["cids/bafyM"] != "true" {
+		t.Fatalf("survivor %v", rec)
+	}
+	if f, _ := c.GetMetadataFlat(newer); f != nil {
+		t.Fatalf("loser still has rows: %v", f)
+	}
+	entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 1000)
+	sawDel := false
+	for _, e := range entries {
+		if e.Values["type"] == "del" && strings.HasPrefix(e.Values["key"], "file:"+newer+"/") {
+			sawDel = true
+		}
+	}
+	if !sawDel {
+		t.Fatal("no del event for the losing rid")
+	}
+	if n, _ := c.CountRoots(); n != 1 {
+		t.Fatalf("roots = %d, want 1", n)
+	}
+}
+
+func TestViewerOverlay(t *testing.T) {
+	c := newTestClient(t)
+	a, b, me := signer(t, 1), signer(t, 2), signer(t, 9)
+	mustAccept(t, c,
+		vote(t, a, "bafyA", "title", "Shared", 1),
+		vote(t, b, "bafyA", "title", "Shared", 1),
+		vote(t, me, "bafyA", "title", "Mine", 2),
+		vote(t, me, "bafyA", "seek", []byte{1, 2, 3}, 2), // ciphertext: never shown
+	)
+	r := rid(t, c, "bafyA")
+	shared, _ := c.GetMetadataFlat(r)
+	mine, _ := c.GetMetadataFlatAs(r, me.UID)
+	if shared["title"] != "Shared" || mine["title"] != "Mine" {
+		t.Fatalf("shared %q, mine %q", shared["title"], mine["title"])
+	}
+	if _, ok := mine["seek"]; ok {
+		t.Fatal("ciphertext leaked into the view")
+	}
+	match := func(_ string, f map[string]string) bool { return strings.Contains(strings.ToLower(f["title"]), "mine") }
+	if hits, _ := c.Search("", nil, 10, match); len(hits) != 0 {
+		t.Fatalf("shared search found the viewer's value: %v", hits)
+	}
+	hits, err := c.SearchAs("", nil, 10, me.UID, match)
+	if err != nil || len(hits) != 1 {
+		t.Fatalf("viewer search hits %v (err %v), want 1", hits, err)
+	}
+	merged, _, dropped, err := c.MergeClaims([]claim.Claim{
+		vote(t, a, "bafyA", "title", "Shared", 1),
+		vote(t, b, "bafyZ", "title", "Other", 1),
+	}, me.UID)
+	if err != nil || dropped != 0 || merged["title"] != "Mine" {
+		t.Fatalf("stateless merge as viewer = %v dropped %d err %v", merged, dropped, err)
+	}
+}
+
+// A crash between the truth commit and the cache apply leaves the cache
+// behind; the next boot replays the missed changes from truth.
+func TestCrashBetweenCommitsIsCaughtUp(t *testing.T) {
+	dir := t.TempDir()
+	c := openClient(t, dir)
+	a := signer(t, 1)
+	mustAccept(t, c, vote(t, a, "bafyA", "title", "Before", 1))
+
+	// Commit straight to truth, as WriteClaims does, but never apply.
+	lost := vote(t, a, "bafyA", "title", "After", 2)
+	if err := c.truth.Update(context.Background(), func(tx *kv.Tx) error {
+		seq, err := readLastSeq(tx)
+		if err != nil {
+			return err
+		}
+		pid, err := ensurePrincipal(tx, lost.UID)
+		if err != nil {
+			return err
+		}
+		seq++
+		if _, err := tx.Exec(`UPDATE claim SET value = ?, ts = ?, sig = ?, seq = ? WHERE cid = ? AND key = ? AND pid = ?`,
+			"After", int64(lost.TS), lost.Sig, seq, lost.CID, lost.Key, pid); err != nil {
+			return err
+		}
+		return writeLastSeq(tx, seq)
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if got := record(t, c, "bafyA")["title"]; got != "Before" {
+		t.Fatalf("cache applied a change it was never given: %q", got)
+	}
+	c.Close()
+
+	c = openClient(t, dir)
+	defer c.Close()
+	if got := record(t, c, "bafyA")["title"]; got != "After" {
+		t.Fatalf("after catch-up title = %q, want After", got)
+	}
+}
+
+// The cache is a pure function of truth: deleting it and rebuilding yields
+// exactly what incremental application produced.
+func TestRebuildEqualsIncremental(t *testing.T) {
+	dir := t.TempDir()
+	c := openClient(t, dir)
+	a, b, u := signer(t, 1), signer(t, 2), signer(t, 3)
+	mustAccept(t, c,
+		vote(t, a, "bafy1", "title", "One", 1), vote(t, a, "bafy1", "genres/Action", "true", 1),
+		vote(t, b, "bafy1", "title", "Uno", 2), vote(t, b, "bafy2", "title", "Two", 2),
+		vote(t, a, "bafy2", "sizeByte", "123", 1), vote(t, u, "bafy2", "like", "true", 3),
+		vote(t, u, "bafy2", "seek", []byte{9}, 3),
+	)
+	mustAccept(t, c,
+		vote(t, a, "bafy1", "genres/Action", nil, 4), // retraction
+		vote(t, b, "bafy1", "cids/bafy2", "true", 5), // group merge
+		vote(t, a, "bafy3", "title", "Three", 5),
+	)
+	incremental := cacheState(t, c)
+	c.Close()
+	for _, f := range []string{"cache.db", "cache.db-wal", "cache.db-shm"} {
+		_ = os.Remove(filepath.Join(dir, f))
+	}
+	c = openClient(t, dir)
+	defer c.Close()
+	if rebuilt := cacheState(t, c); !reflect.DeepEqual(rebuilt, incremental) {
+		t.Fatalf("rebuild differs from incremental:\n incremental %v\n rebuilt     %v", incremental, rebuilt)
+	}
+	if c.RebuiltAt() == "" {
+		t.Fatal("rebuild did not record rebuilt_at (SSE consumers would not get a gap)")
+	}
+}
+
+// cacheState is every derived row that must be a function of truth.
+func cacheState(t *testing.T, c *Client) []string {
+	t.Helper()
+	var out []string
+	for _, k := range allKeys(t, c) {
+		if strings.HasPrefix(k, "ev:") || strings.HasPrefix(k, "evhwm:") || strings.HasPrefix(k, "meta:") ||
+			strings.HasPrefix(k, ftsNS) || k == countersReady {
+			continue
+		}
+		v, _, _ := c.Store().Get(context.Background(), k)
+		out = append(out, k+"="+v)
+	}
+	rows, err := c.Store().DB().Query(`SELECT pid, cid, key FROM claim_by_pid`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var pid int64
+		var cid, key string
+		_ = rows.Scan(&pid, &cid, &key)
+		out = append(out, "byp:"+strconv.FormatInt(pid, 10)+"/"+cid+"/"+key)
+	}
+	var hays []string
+	hr, _ := c.Store().DB().Query(`SELECT rid, hay FROM fts`)
+	for hr.Next() {
+		var r, h string
+		_ = hr.Scan(&r, &h)
+		hays = append(hays, "fts:"+r+"="+h)
+	}
+	hr.Close()
+	out = append(out, hays...)
+	sort.Strings(out)
+	return out
+}
+
+func TestLocalFileFactsAndEvents(t *testing.T) {
+	c := newTestClient(t)
+	before, _ := c.Store().LastID(context.Background(), MetaEventsStream)
+	r, created, err := c.RegisterFileTuple("/files/watch/a.mkv", 1000, 42, "bafkmid")
+	if err != nil || !created {
+		t.Fatalf("register: %v created=%v", err, created)
+	}
+	rec, _ := c.GetMetadataFlat(r)
+	if rec["filePath"] != "/files/watch/a.mkv" || rec["sizeByte"] != "1000" || rec["mtimeNano"] != "42" || rec["cids/bafkmid"] != "true" {
+		t.Fatalf("record %v", rec)
+	}
+	entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 100)
+	sawPath := false
+	for _, e := range entries {
+		if e.Values["type"] == "set" && e.Values["key"] == "file:"+r+"/filePath" {
+			sawPath = true
+		}
+	}
+	if !sawPath {
+		t.Fatal("no set event for file:<rid>/filePath (meta-fuse builds its tree from it)")
+	}
+	// A signed sizeByte (content fact) wins over the on-disk size.
+	mustAccept(t, c, vote(t, signer(t, 1), "bafkmid", "sizeByte", "999", 1))
+	if got, _ := c.GetProperty(r, "sizeByte"); got != "999" {
+		t.Fatalf("sizeByte = %q, want the signed 999", got)
+	}
+	// A second path of the same content is a duplicate.
+	if _, _, err := c.RegisterFileTuple("/files/watch/copy.mkv", 1000, 43, "bafkmid"); err != nil {
+		t.Fatal(err)
+	}
+	doc, _ := c.GetMetadataDocument(r)
+	if len(doc.Duplicates) != 1 || doc.Duplicates[0] != "/files/watch/copy.mkv" {
+		t.Fatalf("duplicates %v", doc.Duplicates)
+	}
+	tuples, _ := c.GetTuplesForAllFiles()
+	if len(tuples) != 2 || tuples[0].HashID != r {
+		t.Fatalf("tuples %+v", tuples)
+	}
+	// Deleting the files drops the local facts, keeps the claims.
+	mark, _ := c.Store().LastID(context.Background(), MetaEventsStream)
+	for _, p := range []string{"/files/watch/a.mkv", "/files/watch/copy.mkv"} {
+		if err := c.RemoveFileTuple(p); err != nil {
+			t.Fatal(err)
+		}
+	}
+	rec, _ = c.GetMetadataFlat(r)
+	if _, ok := rec["filePath"]; ok || rec["sizeByte"] != "999" {
+		t.Fatalf("after delete %v", rec)
+	}
+	entries, _ = c.Store().ReadAfter(context.Background(), MetaEventsStream, mark, 100)
+	sawDel := false
+	for _, e := range entries {
+		if e.Values["type"] == "del" && e.Values["key"] == "file:"+r+"/filePath" {
+			sawDel = true
+		}
+	}
+	if !sawDel {
+		t.Fatal("no del event for filePath")
+	}
+}
+
+func TestSchemaGateRefusesV1Store(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "meta-core.db")
+	db, err := sql.Open("sqlite", "file:"+path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID;
+		INSERT INTO kv VALUES('meta-core:schema-version','2')`); err != nil {
+		t.Fatal(err)
+	}
+	db.Close()
+	c := NewClient()
+	if err := c.Connect(path, filepath.Join(dir, "cache.db")); err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	if err := c.EnsureSchemaVersion(SchemaVersion); err == nil || !strings.Contains(err.Error(), "v1.x") {
+		t.Fatalf("v1 store accepted: %v", err)
+	}
+}

internal/storage/clear_test.go

@@ -1,138 +1,75 @@
 package storage
 
 import (
+	"strings"
 	"testing"
-
-	"github.com/alicebob/miniredis/v2"
 )
 
-// newTestClient spins an in-memory Redis and returns a connected Client with
+// ClearAllMetadata wipes BOTH databases: votes, signers and groups in truth;
-// the given namespace prefix. The server is torn down with the test.
+// merged records, the cid lookup, roots, search rows, the claim index and the
-func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
+// local files in the cache. Service scratch keys are not metadata and survive.
-	t.Helper()
+func TestClearAllMetadataWipesTruthAndCache(t *testing.T) {
-
+	c := newTestClient(t)
-	s := miniredis.RunT(t)
+	a := signer(t, 1)
-	c := NewClient(prefix)
+	mustAccept(t, c,
-	if err := c.Connect("redis://" + s.Addr()); err != nil {
+		vote(t, a, "bafkone", "title", "One", 1),
-		t.Fatalf("connect to miniredis: %v", err)
+		vote(t, a, "bafkone", "cids/bafkonefull", "true", 1),
+		vote(t, a, "bafktwo", "title", "Two", 1),
+	)
+	if _, _, err := c.RegisterFileTuple("/files/watch/a.mkv", 10, 20, "bafkmid"); err != nil {
+		t.Fatal(err)
 	}
-	t.Cleanup(func() { _ = c.Close() })
+	for k, v := range map[string]string{"btmagnet:abc": "magnet:?xt=urn:btih:abc", "searchcov:tmdb:1": "1"} {
-	return c, s
+		if err := c.SetServiceValue(k, v); err != nil {
-}
+			t.Fatalf("seed %s: %v", k, err)
-
-// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
-// leak: ClearAllMetadata used to delete every file:<root>/* key and the
-// file:__index__ set, but left the cid:<cid> → root reverse-index entries
-// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
-// a surviving entry silently redirects a later write into a deleted root.
-func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
-	for _, prefix := range []string{"", "mm:"} {
-		name := "no-prefix"
-		if prefix != "" {
-			name = "prefixed"
 		}
-		t.Run(name, func(t *testing.T) {
-			c, s := newTestClient(t, prefix)
-
-			// Two records, each with several CID aliases (a midhash plus
-			// siblings) — exactly what the watcher produces.
-			aliases := map[string][]string{}
-			for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
-				uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
-				if err != nil {
-					t.Fatalf("mint: %v", err)
-				}
-				cids := []string{
-					"bafkmid" + uuid,
-					"bafkfull" + uuid,
-				}
-				for _, cidStr := range cids {
-					if err := c.AddAlias(uuid, cidStr); err != nil {
-						t.Fatalf("add alias %s: %v", cidStr, err)
-					}
-				}
-				aliases[uuid] = cids
-			}
-
-			// An entry already orphaned by a previous (buggy) clear: a
-			// reverse-index key whose root no longer exists. The sweep is
-			// expected to collect it too.
-			if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
-				t.Fatalf("seed orphan: %v", err)
-			}
-
-			// Sanity: the aliases resolve before the clear.
-			for uuid, cids := range aliases {
-				for _, cidStr := range cids {
-					if got := c.ResolveRoot(cidStr); got != uuid {
-						t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
-					}
-				}
-			}
-
-			deleted, err := c.ClearAllMetadata()
-			if err != nil {
-				t.Fatalf("ClearAllMetadata: %v", err)
-			}
-			// deletedCount counts files, not keys.
-			if deleted != 2 {
-				t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
-			}
-
-			// No reverse-index key may survive — not for the cleared records,
-			// not the pre-existing orphan.
-			if leaked := s.Keys(); len(leaked) != 0 {
-				t.Errorf("keys survived the clear: %v", leaked)
-			}
-
-			// And the aliases must no longer resolve: ResolveRoot returns the
-			// hash unchanged when there is no index entry.
-			for _, cids := range aliases {
-				for _, cidStr := range cids {
-					if got := c.ResolveRoot(cidStr); got != cidStr {
-						t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
-					}
-				}
-			}
-			if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
-				t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
-			}
-		})
 	}
-}
 
-// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
+	deleted, err := c.ClearAllMetadata()
-// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
-// so a clear on a prefixed client cannot reach another namespace's index.
-func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
-	c, s := newTestClient(t, "mm:")
-
-	uuid, err := c.Mint("watch/a.mkv", 100, 1000)
 	if err != nil {
-		t.Fatalf("mint: %v", err)
-	}
-	if err := c.AddAlias(uuid, "bafkmine"); err != nil {
-		t.Fatalf("add alias: %v", err)
-	}
-
-	// Another namespace's reverse-index entry, plus an unrelated key.
-	if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
-		t.Fatalf("seed foreign cid: %v", err)
-	}
-	if err := s.Set("unrelated", "keep-me"); err != nil {
-		t.Fatalf("seed unrelated: %v", err)
-	}
-
-	if _, err := c.ClearAllMetadata(); err != nil {
 		t.Fatalf("ClearAllMetadata: %v", err)
 	}
-
+	if deleted != 3 {
-	for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
+		t.Errorf("deleted = %d, want 3 records", deleted)
-		if !s.Exists(k) {
+	}
-			t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
+	for _, k := range allKeys(t, c) {
+		for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
+			if strings.HasPrefix(k, ns) {
+				t.Errorf("key survived the clear: %q", k)
+			}
 		}
 	}
-	if s.Exists(c.buildCIDIndexKey("bafkmine")) {
+	for _, table := range []string{"fts", "claim_by_pid", "local_file"} {
-		t.Errorf("own reverse-index entry survived the clear")
+		var n int
+		if err := c.Store().DB().QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil || n != 0 {
+			t.Errorf("%s rows after clear = %d (err %v)", table, n, err)
+		}
+	}
+	if n, err := c.CountClaims(); err != nil || n != 0 {
+		t.Errorf("claims after clear = %d (err %v)", n, err)
+	}
+	if got := c.ResolveRoot("bafkone"); got != "bafkone" {
+		t.Errorf("post-clear ResolveRoot = %q, want the hash unchanged", got)
+	}
+	for _, k := range []string{"btmagnet:abc", "searchcov:tmdb:1"} {
+		if _, _, ok, _ := c.GetKeyInfo(k); !ok {
+			t.Errorf("clear deleted service key %q", k)
+		}
+	}
+	// The store keeps working after a wipe.
+	mustAccept(t, c, vote(t, a, "bafkone", "title", "Again", 1))
+	if got := record(t, c, "bafkone")["title"]; got != "Again" {
+		t.Errorf("title after re-write = %q", got)
+	}
+}
+
+// Raw writes are limited to service scratch keys: merged records and
+// bookkeeping cannot be written around the claims.
+func TestServiceValueRefusesRecordKeys(t *testing.T) {
+	c := newTestClient(t)
+	for _, k := range []string{"file:R1/title", "cid:bafk", "idx:roots/R1", "ev:meta:events/1", "meta:applied_seq"} {
+		if err := c.SetServiceValue(k, "x"); err == nil {
+			t.Errorf("SetServiceValue(%q) accepted", k)
+		}
 	}
 }

internal/storage/counters_test.go

@@ -0,0 +1,94 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/schema"
+)
+
+func schemaJSON(t *testing.T, c *Client) string {
+	t.Helper()
+	snap, err := schema.NewIndexer(c.Store()).Snapshot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	snap.GeneratedAt = snap.GeneratedAt.Truncate(0).UTC()
+	b, _ := json.Marshal(snap.Fields)
+	return string(b)
+}
+
+// The counters behind /api/schema and search's `total` follow the merged view.
+// Whatever sequence of claims, retractions and edits runs, they must equal what
+// a full rebuild from the records computes.
+func TestCountersStayExactAcrossWrites(t *testing.T) {
+	c := newTestClient(t)
+	a, b := signer(t, 1), signer(t, 2)
+	mustAccept(t, c,
+		vote(t, a, "bafyR1", "title", "One", 10), vote(t, a, "bafyR1", "sizeByte", "100", 10),
+		vote(t, a, "bafyR1", "title/eng", "One", 10),
+		vote(t, a, "bafyR2", "title", "Two", 10), vote(t, a, "bafyR2", "sizeByte", "x", 10),
+		vote(t, a, "bafyR2", "genres/Action", "true", 10),
+		vote(t, a, "bafyR3", "title", "Three", 10), vote(t, a, "bafyR3", "sizeByte", "300", 10),
+	)
+	mustAccept(t, c,
+		vote(t, a, "bafyR2", "sizeByte", "200", 20), // label string → int
+		vote(t, b, "bafyR1", "title", "One", 20),    // agreeing vote: merged value unchanged
+		vote(t, a, "bafyR1", "title/eng", nil, 20),  // retraction
+	)
+	if n, err := c.CountRoots(); err != nil || n != 3 {
+		t.Fatalf("CountRoots = %d (err %v), want 3", n, err)
+	}
+	incremental := schemaJSON(t, c)
+	snap, _ := schema.NewIndexer(c.Store()).Snapshot(context.Background())
+	if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 3 {
+		t.Fatalf("sizeByte = %+v, want 3 ints", f)
+	}
+	if _, ok := snap.Fields["title/eng"]; ok {
+		t.Fatalf("retracted title/eng still counted: %v", snap.Fields)
+	}
+	if err := c.RebuildCounters(); err != nil {
+		t.Fatal(err)
+	}
+	if rebuilt := schemaJSON(t, c); rebuilt != incremental {
+		t.Fatalf("incremental counters drifted from a rebuild:\n incremental %s\n rebuilt     %s", incremental, rebuilt)
+	}
+	if n, _ := c.CountRoots(); n != 3 {
+		t.Fatalf("CountRoots after rebuild = %d", n)
+	}
+}
+
+func TestEnsureCountersBuildsOnce(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "One"}})
+	built, err := c.EnsureCounters()
+	if err != nil || !built {
+		t.Fatalf("first EnsureCounters built=%v err=%v", built, err)
+	}
+	if built, _ := c.EnsureCounters(); built {
+		t.Fatal("second EnsureCounters rebuilt again")
+	}
+}
+
+// Re-persisting the same signed claims writes nothing and publishes nothing —
+// the gateway's constant repeats.
+func TestUnchangedClaimWriteIsANoOp(t *testing.T) {
+	c := newTestClient(t)
+	a := signer(t, 1)
+	batch := []claim.Claim{
+		vote(t, a, "bafkreiabc", "title", "Naruto", 5),
+		vote(t, a, "bafkreiabc", "fileType", "video", 5),
+	}
+	mustAccept(t, c, batch...)
+	before, _ := c.GetStreamLength(MetaEventsStream)
+	for _, st := range write(t, c, batch...) {
+		if st != StatusUnchanged {
+			t.Fatalf("re-sent claim status %s, want unchanged", st)
+		}
+	}
+	if after, _ := c.GetStreamLength(MetaEventsStream); after != before {
+		t.Fatalf("re-persisting unchanged claims published %d events", after-before)
+	}
+}

internal/storage/domain_screen_migration.go

@@ -1,193 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"time"
-)
-
-// The one-shot corpus sweep for METADATA_KEYS.md §14.17: the `film` + `tv`
-// domains merged into `screen`, and the standalone-vs-serial split they were
-// really carrying moved to the new `workForm` key.
-//
-// # Why a sweep and not a read-side alias
-//
-// `domain` is **stored, not derived on read** — that is what lets the
-// `contentKind` vocabulary stay open (a peer that has never heard of a kind
-// still routes the record correctly). The price of storing it is that a
-// vocabulary change has to be paid once, in the data: every consumer's filter
-// is an exact match, so a record left saying `film` is invisible to a
-// `domain:screen` wall, and teaching every consumer to accept both spellings
-// forever is exactly the alternation the merge exists to delete.
-//
-// So the values are rewritten at rest, here, and no alias vocabulary survives
-// in query filters. (meta-search additionally normalises on the way into its
-// index, because a record can still arrive from a peer that has not been swept
-// — that is a courtesy for un-swept peers, not a second source of truth.)
-//
-// # Contract
-//
-// Idempotent: a record already carrying `screen` and a `workForm` is skipped,
-// so the endpoint is safe to call repeatedly and safe to call on a fresh box.
-// Returns the number of records **changed**, not the number examined.
-
-// legacyDomainReplacements maps a retired `domain` value to its replacement.
-var legacyDomainReplacements = map[string]string{
-	"film": "screen",
-	"tv":   "screen",
-}
-
-// workFormByContentKind is the `contentKind` → `workForm` table from
-// METADATA_KEYS.md §1.
-//
-// ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain` (and of
-// meta-search's copy). It exists here only to backfill the pre-merge corpus —
-// meta-core never classifies a record it is given, and nothing else in Go reads
-// it. If the tables ever disagree, the writers win: this one is a one-shot.
-var workFormByContentKind = map[string]string{
-	// screen
-	"movie":   "standalone",
-	"series":  "serial",
-	"episode": "serial",
-	// music — an album is a closed work, not an ongoing run.
-	"track":      "standalone",
-	"album":      "standalone",
-	"artist":     "standalone",
-	"musicVideo": "standalone",
-	"djMix":      "standalone",
-	"liveSet":    "standalone",
-	// literature
-	"book":      "standalone",
-	"audiobook": "standalone",
-	"comic":     "serial",
-	"manga":     "serial",
-	"magazine":  "serial",
-	// science
-	"paper": "standalone",
-	// ⚠ `pack` is deliberately absent: a season pack is `serial`, an album
-	// release `standalone`, and only the writer knows which. A pack that
-	// reaches this sweep without a `workForm` keeps none — inventing one would
-	// be a guess written permanently into the record.
-}
-
-// domainScreenPlan is one record's pending rewrite. Empty strings mean "leave
-// this field alone".
-type domainScreenPlan struct {
-	hashID   string
-	domain   string
-	workForm string
-}
-
-// MigrateDomainScreen rewrites `domain=film|tv` to `domain=screen` and
-// backfills `workForm` from `contentKind` across the whole corpus.
-//
-// Reads in two batched MGET passes (never a per-record SCAN — a single record
-// read costs a full keyspace sweep on this store), then writes only the records
-// that actually change.
-func (c *Client) MigrateDomainScreen(ctx context.Context) (int, error) {
-	plans, err := c.planDomainScreen(ctx)
-	if err != nil {
-		return 0, err
-	}
-	// Writes take the client lock themselves, so they happen after the read
-	// phase has released it.
-	fixed := 0
-	for _, p := range plans {
-		if p.domain != "" {
-			if err := c.SetProperty(p.hashID, "domain", p.domain); err != nil {
-				return fixed, fmt.Errorf("set domain on %s: %w", p.hashID, err)
-			}
-		}
-		if p.workForm != "" {
-			if err := c.SetProperty(p.hashID, "workForm", p.workForm); err != nil {
-				return fixed, fmt.Errorf("set workForm on %s: %w", p.hashID, err)
-			}
-		}
-		fixed++
-	}
-	return fixed, nil
-}
-
-// planDomainScreen is the read half: what would change, without changing it.
-func (c *Client) planDomainScreen(ctx context.Context) ([]domainScreenPlan, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
-	if err != nil {
-		return nil, fmt.Errorf("smembers index: %w", err)
-	}
-	if len(hashIDs) == 0 {
-		return nil, nil
-	}
-
-	// Three keys per record, in a fixed order so the MGET response decodes by
-	// position: keys[i*3+n] belongs to hashIDs[i].
-	const fieldsPerFile = 3
-	fields := [fieldsPerFile]string{"domain", "workForm", "contentKind"}
-	keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
-	for _, h := range hashIDs {
-		prefix := c.buildKeyPrefix(h)
-		for _, f := range fields {
-			keys = append(keys, prefix+f)
-		}
-	}
-
-	// Chunked so one MGET never carries the whole corpus.
-	const chunk = 3000
-	values := make([]interface{}, 0, len(keys))
-	for start := 0; start < len(keys); start += chunk {
-		end := start + chunk
-		if end > len(keys) {
-			end = len(keys)
-		}
-		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
-		if err != nil {
-			return nil, fmt.Errorf("mget: %w", err)
-		}
-		values = append(values, got...)
-	}
-
-	str := func(i int) string {
-		if i >= len(values) {
-			return ""
-		}
-		s, _ := values[i].(string)
-		return s
-	}
-
-	plans := make([]domainScreenPlan, 0)
-	for i, h := range hashIDs {
-		base := i * fieldsPerFile
-		domain, workForm, contentKind := str(base), str(base+1), str(base+2)
-
-		plan := domainScreenPlan{hashID: h}
-		if replacement, ok := legacyDomainReplacements[domain]; ok {
-			plan.domain = replacement
-		}
-		// A stored `workForm` is authoritative — a `pack` writer knew something
-		// the table cannot. Only an absent one is filled in.
-		if workForm == "" {
-			if form, ok := workFormByContentKind[contentKind]; ok {
-				plan.workForm = form
-			}
-		}
-		if plan.domain != "" || plan.workForm != "" {
-			plans = append(plans, plan)
-		}
-	}
-	return plans, nil
-}
-
-// MigrateDomainScreenWithTimeout is the convenience wrapper used by the API
-// handler. Generous timeout: the read phase is two batched passes, but the
-// write phase is one round-trip per changed field.
-func (c *Client) MigrateDomainScreenWithTimeout() (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-	return c.MigrateDomainScreen(ctx)
-}

internal/storage/domain_screen_migration_test.go

@@ -1,140 +0,0 @@
-package storage
-
-import (
-	"context"
-	"testing"
-)
-
-// seedRecord writes one record's classification fields straight through
-// SetProperty, which is what a real writer uses (it maintains the field index
-// and the root index the sweep reads from).
-func seedRecord(t *testing.T, c *Client, hashID string, fields map[string]string) {
-	t.Helper()
-	for k, v := range fields {
-		if err := c.SetProperty(hashID, k, v); err != nil {
-			t.Fatalf("seed %s/%s: %v", hashID, k, err)
-		}
-	}
-}
-
-// The merge (METADATA_KEYS.md §14.17) is paid in the data: `domain` is stored,
-// not derived on read, so a record left saying `film` is invisible to a
-// `domain:screen` wall. This is the sweep that pays it.
-func TestMigrateDomainScreen_RewritesLegacyDomainsAndBackfillsWorkForm(t *testing.T) {
-	c, _ := newTestClient(t, "")
-
-	seedRecord(t, c, "rec-movie", map[string]string{
-		"domain":      "film",
-		"contentKind": "movie",
-	})
-	seedRecord(t, c, "rec-episode", map[string]string{
-		"domain":      "tv",
-		"contentKind": "episode",
-	})
-	// Already current, and already carries both keys: must be left alone.
-	seedRecord(t, c, "rec-current", map[string]string{
-		"domain":      "screen",
-		"contentKind": "series",
-		"workForm":    "serial",
-	})
-	// A different domain entirely — the sweep must not touch it, but it is
-	// still owed the `workForm` backfill.
-	seedRecord(t, c, "rec-book", map[string]string{
-		"domain":      "literature",
-		"contentKind": "book",
-	})
-
-	fixed, err := c.MigrateDomainScreen(context.Background())
-	if err != nil {
-		t.Fatalf("migrate: %v", err)
-	}
-	if fixed != 3 {
-		t.Errorf("fixed = %d, want 3 (the two legacy rows + the book's backfill)", fixed)
-	}
-
-	for _, tc := range []struct {
-		hashID   string
-		domain   string
-		workForm string
-	}{
-		{"rec-movie", "screen", "standalone"},
-		{"rec-episode", "screen", "serial"},
-		{"rec-current", "screen", "serial"},
-		{"rec-book", "literature", "standalone"},
-	} {
-		m, err := c.GetMetadataFlat(tc.hashID)
-		if err != nil {
-			t.Fatalf("read %s: %v", tc.hashID, err)
-		}
-		if m["domain"] != tc.domain {
-			t.Errorf("%s domain = %q, want %q", tc.hashID, m["domain"], tc.domain)
-		}
-		if m["workForm"] != tc.workForm {
-			t.Errorf("%s workForm = %q, want %q", tc.hashID, m["workForm"], tc.workForm)
-		}
-	}
-}
-
-// A second pass must find nothing to do — the endpoint is called by hand on
-// each box, and "run it twice" must not mean "write the whole corpus twice".
-func TestMigrateDomainScreen_IsIdempotent(t *testing.T) {
-	c, _ := newTestClient(t, "mm:")
-
-	seedRecord(t, c, "rec-1", map[string]string{
-		"domain":      "tv",
-		"contentKind": "episode",
-	})
-
-	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
-		t.Fatalf("first pass: %v", err)
-	}
-	fixed, err := c.MigrateDomainScreen(context.Background())
-	if err != nil {
-		t.Fatalf("second pass: %v", err)
-	}
-	if fixed != 0 {
-		t.Errorf("second pass fixed = %d, want 0", fixed)
-	}
-}
-
-// `pack` is the one kind neither routing table can decide — a season pack is
-// `serial`, an album release `standalone`. A pack that reaches the sweep
-// without a `workForm` must keep none: inventing one writes a guess into the
-// record permanently.
-func TestMigrateDomainScreen_NeverGuessesAPacksWorkForm(t *testing.T) {
-	c, _ := newTestClient(t, "")
-
-	seedRecord(t, c, "rec-pack", map[string]string{
-		"domain":      "tv",
-		"contentKind": "pack",
-	})
-	// A writer that DID know keeps what it wrote.
-	seedRecord(t, c, "rec-album", map[string]string{
-		"domain":      "music",
-		"contentKind": "pack",
-		"workForm":    "standalone",
-	})
-
-	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
-		t.Fatalf("migrate: %v", err)
-	}
-
-	m, err := c.GetMetadataFlat("rec-pack")
-	if err != nil {
-		t.Fatalf("read: %v", err)
-	}
-	if m["domain"] != "screen" {
-		t.Errorf("domain = %q, want screen (the domain WAS knowable)", m["domain"])
-	}
-	if _, ok := m["workForm"]; ok {
-		t.Errorf("workForm = %q, want absent", m["workForm"])
-	}
-
-	a, err := c.GetMetadataFlat("rec-album")
-	if err != nil {
-		t.Fatalf("read: %v", err)
-	}
-	if a["workForm"] != "standalone" {
-		t.Errorf("workForm = %q, want standalone (writer-supplied, never overwritten)", a["workForm"])
-	}
-}

internal/storage/dual_root_migration.go

@@ -1,41 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-)
-
-// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
-// legacy storage model: /meta/{hash} write handlers that didn't resolve
-// CID → UUID first created a parallel midhash-rooted entry alongside the
-// watcher's UUID root.
-//
-// That pattern is now structurally impossible:
-//   - writes resolve via ResolveRoot before landing (see handlers), and
-//   - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
-//     UUID roots, with the reverse index cid:<cid> → uuid as the single
-//     resolution path.
-//
-// The legacy on-disk shape this migration targeted (midhash-rooted entries,
-// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
-// cut-over wipes Redis, so there is nothing to migrate. The function is kept
-// as a no-op so the admin endpoint and its wrappers continue to compile and
-// respond.
-func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
-	if c.client == nil {
-		return 0, fmt.Errorf("not connected")
-	}
-	return 0, nil
-}
-
-// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
-// handler and the startup hook.
-func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
-	return c.MigrateDualRoots(context.Background())
-}
-
-// MigrateDualRootsReport returns a human-readable summary. With the dual-root
-// pattern retired (see MigrateDualRoots), this always reports nothing to do.
-func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
-	return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
-}

internal/storage/field_index.go

@@ -1,486 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/redis/go-redis/v9"
-)
-
-// The per-record field-name index.
-//
-// A record is stored as one flat Redis STRING per leaf
-// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
-// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
-// meta-search can react to one field changing instead of diffing a whole hash
-// (see docs/project-architecture/persistent-storage.md, and
-// dev/test/suites/flat-keys.bats which asserts records are never hashes).
-//
-// The cost of that shape was that reading one record meant asking Redis "which
-// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
-// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
-// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
-// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
-// the byte path for every poster meta-share served, which is how it surfaced.
-//
-// So we keep the flat keys and index their *names*:
-//
-//	file:<uuid>/__fields__   SET{ "filePath", "sizeByte", "cids/bafk…", … }
-//
-// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
-// This mirrors `file:__index__` (the set of every root) one level down, and the
-// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
-// from the index rather than scanning per root.
-//
-// **The set is derived, never authoritative.** The flat keys remain the source
-// of truth. A record whose set is missing (anything written before this landed)
-// falls back to the old SCAN and populates the set on the way out, so existing
-// deployments heal themselves on first touch with no migration and no
-// SchemaVersion bump — which matters, because a bump is a flag day that makes
-// operators wipe Redis (see schema_version.go).
-//
-// The consistency argument that makes a derived index safe here is meta-core's
-// single-writer rule: leader election publishes an empty `redisUrl` on purpose
-// (internal/leader/election.go), so nothing outside this process can write a
-// `file:` key behind the index's back.
-
-// FieldsField is the reserved field name holding a record's field-name index.
-//
-// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
-// prefix as the record's own fields. That keeps every existing sweep correct
-// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
-// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
-// The flip side is that every reader which enumerates that prefix has to skip
-// it, exactly as they already skip `file:__index__`.
-const FieldsField = "__fields__"
-
-// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
-// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
-const mgetBatch = 1000
-
-// buildFieldsKey constructs the key for a record's field-name index set.
-func (c *Client) buildFieldsKey(hashID string) string {
-	return c.buildKeyPrefix(hashID) + FieldsField
-}
-
-// isReservedFieldName reports whether a field name is bookkeeping rather than
-// metadata, and so must never be returned to an API caller, exported into a
-// snapshot, or folded into the search haystack.
-func isReservedFieldName(field string) bool {
-	return field == FieldsField
-}
-
-// readMetadataLocked returns a record's full flat metadata. Caller must hold
-// c.mu (either mode).
-//
-// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
-// legacy full-keyspace SCAN, whose result is used to populate the index so the
-// next read takes the fast path.
-//
-// Returns (nil, nil) for a record with no fields — callers treat that as 404.
-func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
-	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return nil, fmt.Errorf("smembers fields: %w", err)
-	}
-
-	if len(fields) > 0 {
-		return c.mgetFieldsLocked(ctx, hashID, fields)
-	}
-
-	// No index yet: a record written before the index existed, or one that
-	// genuinely has no fields. Fall back to the scan, then backfill.
-	result, scanned, err := c.scanMetadataLocked(ctx, hashID)
-	if err != nil {
-		return nil, err
-	}
-	if len(scanned) > 0 {
-		// Best-effort: a failed backfill costs the next read a scan, it does
-		// not make the read wrong. Never fail the caller's read for it.
-		if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
-			logFieldIndexBackfillFailure(hashID, err)
-		}
-	}
-	if len(result) == 0 {
-		return nil, nil
-	}
-	return result, nil
-}
-
-// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
-//
-// A field named in the index whose key is gone reads back as nil and is simply
-// skipped: a *stale* index entry is harmless. (A *missing* one would silently
-// drop data, which is why every write path maintains the set.)
-func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
-	prefix := c.buildKeyPrefix(hashID)
-
-	// Keep the reserved name out of the key list AND out of the result — it is
-	// a SET, so MGET would error on it in real Redis rather than return nil.
-	wanted := make([]string, 0, len(fields))
-	for _, f := range fields {
-		if !isReservedFieldName(f) {
-			wanted = append(wanted, f)
-		}
-	}
-	if len(wanted) == 0 {
-		return nil, nil
-	}
-
-	result := make(map[string]string, len(wanted))
-	for start := 0; start < len(wanted); start += mgetBatch {
-		end := start + mgetBatch
-		if end > len(wanted) {
-			end = len(wanted)
-		}
-		chunk := wanted[start:end]
-		keys := make([]string, len(chunk))
-		for i, f := range chunk {
-			keys[i] = prefix + f
-		}
-		values, err := c.client.MGet(ctx, keys...).Result()
-		if err != nil {
-			return nil, fmt.Errorf("mget failed: %w", err)
-		}
-		for i, v := range values {
-			if s, ok := v.(string); ok {
-				result[chunk[i]] = s
-			}
-		}
-	}
-
-	if len(result) == 0 {
-		return nil, nil
-	}
-	return result, nil
-}
-
-// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
-// MGET. Retained as the fallback for un-indexed records and as the source for
-// the backfill. Returns the metadata plus the field names it saw (which
-// includes fields whose value came back nil, so the index stays complete).
-func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
-	prefix := c.buildKeyPrefix(hashID)
-	result := make(map[string]string)
-	var fields []string
-
-	var cursor uint64
-	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, nil, fmt.Errorf("scan failed: %w", err)
-		}
-
-		if len(keys) > 0 {
-			wanted := make([]string, 0, len(keys))
-			for _, key := range keys {
-				field := strings.TrimPrefix(key, prefix)
-				if isReservedFieldName(field) {
-					continue
-				}
-				fields = append(fields, field)
-				wanted = append(wanted, key)
-			}
-			if len(wanted) > 0 {
-				values, err := c.client.MGet(ctx, wanted...).Result()
-				if err != nil {
-					return nil, nil, fmt.Errorf("mget failed: %w", err)
-				}
-				for i, key := range wanted {
-					if s, ok := values[i].(string); ok {
-						result[strings.TrimPrefix(key, prefix)] = s
-					}
-				}
-			}
-		}
-
-		cursor = nextCursor
-		if cursor == 0 {
-			break
-		}
-	}
-
-	return result, fields, nil
-}
-
-// addFieldsToPipe records field names in the index as part of an existing
-// pipeline, so a multi-field write stays one round-trip.
-func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
-	names := filterReserved(fields)
-	if len(names) == 0 {
-		return
-	}
-	pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
-}
-
-// addFieldLocked records a single field name in the index. Caller must hold
-// c.mu. Best-effort by the same argument as the backfill: the flat key is the
-// source of truth, and a lost index entry is repaired by the next scan
-// fallback — so it must not fail the write it accompanies.
-func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
-	if isReservedFieldName(field) {
-		return
-	}
-	if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
-		logFieldIndexWriteFailure(hashID, field, err)
-	}
-}
-
-// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
-//
-// A stale entry left behind by a failure is harmless (the MGET reads nil and
-// skips it), so this is best-effort too.
-func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
-	if isReservedFieldName(field) {
-		return
-	}
-	if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
-		logFieldIndexWriteFailure(hashID, field, err)
-	}
-}
-
-// fieldsWithPrefixLocked returns the record's indexed field names under a
-// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
-// no index yet. Caller must hold c.mu.
-func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
-	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return nil, fmt.Errorf("smembers fields: %w", err)
-	}
-	if len(fields) == 0 {
-		// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
-		// rather than the whole record, and do not backfill from it — a
-		// partial view must never be mistaken for a complete index.
-		return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
-	}
-
-	var out []string
-	for _, f := range fields {
-		if isReservedFieldName(f) {
-			continue
-		}
-		if strings.HasPrefix(f, fieldPrefix) {
-			out = append(out, strings.TrimPrefix(f, fieldPrefix))
-		}
-	}
-	// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
-	// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
-	// at these sizes and makes the response byte-stable, which matters for
-	// anything diffing or caching it.
-	sort.Strings(out)
-	return out, nil
-}
-
-func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
-	prefix := c.buildKeyPrefix(hashID) + fieldPrefix
-	var out []string
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan fields: %w", err)
-		}
-		for _, k := range keys {
-			out = append(out, strings.TrimPrefix(k, prefix))
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
-//
-// Without it the indexes still appear — each record's first read falls back to
-// a scan and backfills — but that transition is O(records x keyspace) and it
-// runs again from scratch after any wipe. Measured on the real store (12,345
-// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
-// during which meta-core stays pinned exactly as it was before the fix. Worse,
-// `RepublishMetadata` walks every record at startup, so that tail is paid on
-// every restart.
-//
-// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
-// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
-// for the same reason.
-//
-// Idempotent, and safe to run against a partially-indexed store: SADD of a
-// member that is already there is a no-op. Returns the number of records it
-// wrote an index for.
-func (c *Client) WarmFieldIndexes() (int, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return 0, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-
-	filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
-	indexKey := c.buildIndexKey()
-
-	byRecord := make(map[string][]string)
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
-		if err != nil {
-			return 0, fmt.Errorf("scan: %w", err)
-		}
-		for _, k := range keys {
-			if k == indexKey {
-				continue
-			}
-			// hashID has no '/'; field may contain them.
-			rest := strings.TrimPrefix(k, filePrefix)
-			slash := strings.IndexByte(rest, '/')
-			if slash < 0 {
-				continue
-			}
-			hashID, field := rest[:slash], rest[slash+1:]
-			if isReservedFieldName(field) {
-				// Already indexed — remember the record so we don't treat it
-				// as missing, but there is nothing to add for this key.
-				if _, seen := byRecord[hashID]; !seen {
-					byRecord[hashID] = nil
-				}
-				continue
-			}
-			byRecord[hashID] = append(byRecord[hashID], field)
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-
-	written := 0
-	pipe := c.client.Pipeline()
-	pending := 0
-	for hashID, fields := range byRecord {
-		if len(fields) == 0 {
-			continue
-		}
-		pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
-		written++
-		pending++
-		// Bound the pipeline so a large store doesn't buffer every command.
-		if pending >= 500 {
-			if _, err := pipe.Exec(ctx); err != nil {
-				return written, fmt.Errorf("sadd pipeline: %w", err)
-			}
-			pipe = c.client.Pipeline()
-			pending = 0
-		}
-	}
-	if pending > 0 {
-		if _, err := pipe.Exec(ctx); err != nil {
-			return written, fmt.Errorf("sadd pipeline: %w", err)
-		}
-	}
-	return written, nil
-}
-
-// recordHasFieldsLocked reports whether a record still has at least one
-// property key. Caller must hold c.mu.
-//
-// `__fields__` must not count — it lives under the record's own prefix, so a
-// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
-// whose every property has been deleted, and the root could never be
-// de-indexed.
-func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
-	n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return false, fmt.Errorf("scard fields: %w", err)
-	}
-	if n > 0 {
-		return true, nil
-	}
-
-	// Either genuinely empty, or an un-indexed record from before the index
-	// existed — the scan is the only way to tell those apart.
-	prefix := c.buildKeyPrefix(hashID)
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return false, fmt.Errorf("scan failed: %w", err)
-		}
-		for _, k := range keys {
-			if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
-				return true, nil
-			}
-		}
-		cursor = next
-		if cursor == 0 {
-			return false, nil
-		}
-	}
-}
-
-// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
-// record property key. Reports ok=false for anything else — the index set
-// itself, `file:__index__`, and any key outside the `file:` namespace.
-//
-// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
-// the index honest without its callers knowing the layout.
-func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
-	rest := key
-	if c.prefix != "" {
-		var found bool
-		rest, found = strings.CutPrefix(rest, c.prefix)
-		if !found {
-			return "", "", false
-		}
-	}
-	rest, found := strings.CutPrefix(rest, "file:")
-	if !found {
-		return "", "", false
-	}
-	hashID, field, found = strings.Cut(rest, "/")
-	if !found || hashID == "" || field == "" {
-		// `file:__index__` has no slash and lands here.
-		return "", "", false
-	}
-	if isReservedFieldName(field) {
-		return "", "", false
-	}
-	return hashID, field, true
-}
-
-// The index is derived and self-healing, so a write failure degrades
-// performance (the next read falls back to a scan and re-populates) rather
-// than correctness. Log it rather than failing the caller's write — but log it
-// loudly, because a *persistent* failure means every read is scanning again.
-func logFieldIndexWriteFailure(hashID, field string, err error) {
-	log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
-}
-
-func logFieldIndexBackfillFailure(hashID string, err error) {
-	log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
-}
-
-func filterReserved(fields []string) []string {
-	out := make([]string, 0, len(fields))
-	for _, f := range fields {
-		if !isReservedFieldName(f) {
-			out = append(out, f)
-		}
-	}
-	return out
-}
-
-func toAny(ss []string) []interface{} {
-	out := make([]interface{}, len(ss))
-	for i, s := range ss {
-		out[i] = s
-	}
-	return out
-}

internal/storage/field_index_test.go

@@ -1,464 +0,0 @@
-package storage
-
-import (
-	"context"
-	"reflect"
-	"sort"
-	"testing"
-)
-
-// Tests for the per-record field-name index (field_index.go).
-//
-// The index is what makes a record read O(fields) instead of O(keyspace). It is
-// derived state, so the properties worth pinning are: (1) a read served from
-// the index returns exactly what a scan would have, (2) a record with no index
-// still reads correctly and gets one, and (3) every write path keeps it in
-// sync — because a *missing* entry silently drops a field from every
-// subsequent read, which is the one failure mode that would be invisible.
-//
-// Every case runs with and without a namespace prefix, matching clear_test.go.
-
-func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
-	t.Helper()
-	for _, prefix := range []string{"", "mm:"} {
-		name := "no-prefix"
-		if prefix != "" {
-			name = "prefix-" + prefix
-		}
-		t.Run(name, func(t *testing.T) {
-			c, _ := newTestClient(t, prefix)
-			fn(t, c)
-		})
-	}
-}
-
-func fieldNames(t *testing.T, c *Client, hashID string) []string {
-	t.Helper()
-	got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
-	if err != nil {
-		t.Fatalf("smembers fields: %v", err)
-	}
-	sort.Strings(got)
-	return got
-}
-
-func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
-	t.Helper()
-	got, err := c.GetMetadataFlat(hashID)
-	if err != nil {
-		t.Fatalf("GetMetadataFlat: %v", err)
-	}
-	return got
-}
-
-// SetMetadataFlat must index every field it writes, and the read must come back
-// through the index rather than a scan.
-func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		want := map[string]string{
-			"filePath":                      "/files/watch/a.mkv",
-			"sizeByte":                      "123",
-			"cids/bafkreiaaa":               "true",
-			"titles/jpn":                    "ナルト",
-			"categories/TV/UHD":             "true",
-			"info/files/0/path":             "a.mkv",
-			"provenance":                    `{"source":"gateway"}`,
-			"plot/eng":                      "a plot with / slashes",
-			"stream/3":                      "eng",
-			"genres/action":                 "true",
-			"originaltitle":                 "Naruto",
-			"deeply/nested/field/name/here": "x",
-		}
-		if err := c.SetMetadataFlat("root1", want); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
-			t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
-		}
-
-		// The index must name every field — including the ones containing '/',
-		// which is where a naive "split on the first slash" would go wrong.
-		wantNames := make([]string, 0, len(want))
-		for k := range want {
-			wantNames = append(wantNames, k)
-		}
-		sort.Strings(wantNames)
-		if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
-			t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
-		}
-	})
-}
-
-// A record written before the index existed has flat keys but no set. The read
-// must still be correct, and must leave an index behind so the next read takes
-// the fast path. This is the migration path for the ~12k records already on
-// disk: no flag day, no SchemaVersion bump.
-func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		prefix := c.buildKeyPrefix("legacy")
-		want := map[string]string{
-			"filePath":        "/files/watch/old.mkv",
-			"sizeByte":        "42",
-			"cids/bafkreiold": "true",
-		}
-		// Write the flat keys directly, bypassing every index-maintaining path
-		// — exactly the shape a pre-upgrade store is in.
-		for f, v := range want {
-			if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
-				t.Fatalf("seed %s: %v", f, err)
-			}
-		}
-		if names := fieldNames(t, c, "legacy"); len(names) != 0 {
-			t.Fatalf("precondition: expected no index, got %v", names)
-		}
-
-		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
-			t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
-		}
-
-		wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
-		if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
-			t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
-		}
-
-		// Second read, now served from the index, must agree with the first.
-		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
-			t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
-		}
-	})
-}
-
-// Each single-field write path must add to the index, and each delete path must
-// remove from it. A field written but not indexed vanishes from every later
-// read — silently — so this is the property that matters most.
-func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
-			t.Fatalf("SetProperty: %v", err)
-		}
-		if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
-			t.Fatalf("AddToSet: %v", err)
-		}
-		if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
-			t.Fatalf("MergeMetadataFlat: %v", err)
-		}
-
-		want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
-		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
-			t.Errorf("after writes\n got %v\nwant %v", got, want)
-		}
-
-		// DeleteProperty drops the key and the index entry.
-		if err := c.DeleteProperty("root2", "title"); err != nil {
-			t.Fatalf("DeleteProperty: %v", err)
-		}
-		// RemoveFromSet deletes the key once its last member is gone.
-		if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
-			t.Fatalf("RemoveFromSet: %v", err)
-		}
-
-		want = map[string]string{"year": "2002"}
-		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
-			t.Errorf("after deletes\n got %v\nwant %v", got, want)
-		}
-		if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
-			t.Errorf("index after deletes = %v, want [year]", got)
-		}
-	})
-}
-
-// The raw KV editor can mint or drop a record property out of band. It is the
-// one write path whose caller does not know the record layout, so the storage
-// layer has to keep the index honest for it.
-func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
-			t.Fatalf("SetRawValue: %v", err)
-		}
-		if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
-			t.Errorf("index after SetRawValue = %v", got)
-		}
-		if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
-			t.Errorf("read after SetRawValue = %v", got)
-		}
-
-		if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
-			t.Fatalf("DeleteRawKey: %v", err)
-		}
-		if got := fieldNames(t, c, "root3"); len(got) != 0 {
-			t.Errorf("index after DeleteRawKey = %v, want empty", got)
-		}
-
-		// Keys outside the record namespace must be left alone entirely.
-		if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
-			t.Fatalf("SetRawValue non-record: %v", err)
-		}
-		if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
-			t.Error("cid: key was treated as a record property")
-		}
-		if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
-			t.Error("file:__index__ was treated as a record property")
-		}
-	})
-}
-
-// Mint and AddAlias write fields through their own pipelines, so they need the
-// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
-// it and a record's CID key-set becomes invisible to every read.
-func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
-		if err != nil {
-			t.Fatalf("Mint: %v", err)
-		}
-		if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
-			t.Errorf("index after Mint = %v", got)
-		}
-
-		if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
-			t.Fatalf("AddAlias: %v", err)
-		}
-		got := mustGetMetadata(t, c, uuid)
-		if got["cids/bafkreiaaa"] != "true" {
-			t.Errorf("alias key-set member missing from read: %v", got)
-		}
-
-		// And the CID enumeration, which now reads the index instead of
-		// scanning, must see it.
-		cids, err := c.cidsForRootLocked(context.Background(), uuid)
-		if err != nil {
-			t.Fatalf("cidsForRootLocked: %v", err)
-		}
-		if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
-			t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
-		}
-	})
-}
-
-// The index lives under the record's own prefix, so the "is this record empty?"
-// probe would otherwise always find it and the root could never be de-indexed.
-func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetProperty("root4", "title", "x"); err != nil {
-			t.Fatalf("SetProperty: %v", err)
-		}
-
-		removed, err := c.IndexRemoveIfEmpty("root4")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if removed {
-			t.Error("de-indexed a record that still has a field")
-		}
-
-		if err := c.DeleteProperty("root4", "title"); err != nil {
-			t.Fatalf("DeleteProperty: %v", err)
-		}
-		removed, err = c.IndexRemoveIfEmpty("root4")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if !removed {
-			t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
-		}
-		// ...and the now-empty index key must not be left orphaned.
-		n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
-		if err != nil {
-			t.Fatalf("exists: %v", err)
-		}
-		if n != 0 {
-			t.Error("orphaned field index left behind after de-indexing")
-		}
-	})
-}
-
-// The same probe must still work for a record that predates the index.
-func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
-			t.Fatalf("seed: %v", err)
-		}
-		if err := c.IndexAdd("legacy2"); err != nil {
-			t.Fatalf("IndexAdd: %v", err)
-		}
-		removed, err := c.IndexRemoveIfEmpty("legacy2")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if removed {
-			t.Error("de-indexed an un-indexed record that still has a field")
-		}
-	})
-}
-
-// The reserved name is bookkeeping. It must never reach an API caller as a
-// metadata field, and an inbound payload must never be able to overwrite the
-// SET with a string — that would break every subsequent read of the record.
-func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		err := c.SetMetadataFlat("root5", map[string]string{
-			"title":     "x",
-			FieldsField: "malicious",
-		})
-		if err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		got := mustGetMetadata(t, c, "root5")
-		if _, present := got[FieldsField]; present {
-			t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
-		}
-		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
-			t.Errorf("read = %v, want just title", got)
-		}
-
-		// The index must still be a SET — i.e. not clobbered by the payload.
-		if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
-			t.Errorf("index = %v, want [title]", got)
-		}
-
-		if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
-			t.Error("SetProperty accepted the reserved field name")
-		}
-	})
-}
-
-// Deleting a record must take its index with it, or a recreated root would
-// inherit phantom field names.
-func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-		if _, err := c.DeleteMetadata("root6"); err != nil {
-			t.Fatalf("DeleteMetadata: %v", err)
-		}
-		if got := fieldNames(t, c, "root6"); len(got) != 0 {
-			t.Errorf("index survived DeleteMetadata: %v", got)
-		}
-		got, err := c.GetMetadataFlat("root6")
-		if err != nil {
-			t.Fatalf("GetMetadataFlat: %v", err)
-		}
-		if got != nil {
-			t.Errorf("deleted record still reads back: %v", got)
-		}
-	})
-}
-
-// A stale index entry (name present, key gone) is the benign direction: the
-// MGET reads nil and the field is skipped rather than erroring.
-func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-		// Name a field that has no key behind it.
-		if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
-			t.Fatalf("sadd ghost: %v", err)
-		}
-
-		got := mustGetMetadata(t, c, "root7")
-		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
-			t.Errorf("read = %v, want just title", got)
-		}
-	})
-}
-
-// WarmFieldIndexes is what keeps an upgrade from spending hours reading
-// un-indexed records one full-keyspace scan at a time. It must index every
-// record in one pass, be idempotent, and leave already-indexed records alone.
-func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-
-		// Two legacy records seeded straight into Redis (no index), plus one
-		// written through the normal path (already indexed).
-		legacy := map[string]map[string]string{
-			"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
-			"old2": {"filePath": "/b.mkv", "titles/jpn": "ナルト", "deep/a/b": "x"},
-		}
-		for root, fields := range legacy {
-			for f, v := range fields {
-				if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
-					t.Fatalf("seed: %v", err)
-				}
-			}
-			if err := c.IndexAdd(root); err != nil {
-				t.Fatalf("IndexAdd: %v", err)
-			}
-		}
-		if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		n, err := c.WarmFieldIndexes()
-		if err != nil {
-			t.Fatalf("WarmFieldIndexes: %v", err)
-		}
-		if n < 2 {
-			t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
-		}
-
-		if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
-			t.Errorf("old1 index = %v", got)
-		}
-		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn"}) {
-			t.Errorf("old2 index = %v", got)
-		}
-		if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
-			t.Errorf("fresh index disturbed = %v", got)
-		}
-
-		// Reads must now agree with what the records actually hold.
-		for root, want := range legacy {
-			if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
-				t.Errorf("%s read = %v, want %v", root, got, want)
-			}
-		}
-
-		// Idempotent: a second pass changes nothing.
-		if _, err := c.WarmFieldIndexes(); err != nil {
-			t.Fatalf("second WarmFieldIndexes: %v", err)
-		}
-		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn"}) {
-			t.Errorf("old2 index after second warm = %v", got)
-		}
-	})
-}
-
-// The bulk search read enumerates the whole file: keyspace and MGETs what it
-// finds. The index keys are SETs living in that same namespace, so it has to
-// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
-func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetMetadataFlat("root8", map[string]string{
-			"title":    "Naruto Shippuden",
-			"fileName": "naruto.mkv",
-		}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		tuples, err := c.GetSearchIndexTuples()
-		if err != nil {
-			t.Fatalf("GetSearchIndexTuples: %v", err)
-		}
-		var found bool
-		for _, tu := range tuples {
-			if tu.HashID != "root8" {
-				continue
-			}
-			found = true
-			if _, present := tu.Fields[FieldsField]; present {
-				t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
-			}
-		}
-		if !found {
-			t.Fatal("record missing from search tuples")
-		}
-	})
-}

internal/storage/helpers_test.go

@@ -0,0 +1,166 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+// openClient opens (and starts) a client on dir's truth + cache files.
+func openClient(t *testing.T, dir string) *Client {
+	t.Helper()
+	c := NewClient()
+	if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	if err := c.EnsureSchemaVersion(SchemaVersion); err != nil {
+		t.Fatalf("schema: %v", err)
+	}
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	return c
+}
+
+// newTestClient opens a fresh store in the test's temp dir.
+func newTestClient(t *testing.T) *Client {
+	t.Helper()
+	c := openClient(t, t.TempDir())
+	t.Cleanup(func() { _ = c.Close() })
+	return c
+}
+
+// signer returns a deterministic test identity (scalar n repeated).
+func signer(t *testing.T, n byte) *identity.Identity {
+	t.Helper()
+	id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1))
+	if err != nil {
+		t.Fatalf("identity: %v", err)
+	}
+	return id
+}
+
+// vote builds a signed claim; v is a string (text), []byte (ciphertext) or nil
+// (retraction).
+func vote(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim {
+	t.Helper()
+	cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts}
+	switch x := v.(type) {
+	case string:
+		cl.Value = claim.Text(x)
+	case []byte:
+		cl.Value = claim.Bytes(x)
+	case nil:
+	default:
+		t.Fatalf("vote value %T", v)
+	}
+	if err := cl.Sign(id); err != nil {
+		t.Fatalf("sign: %v", err)
+	}
+	return cl
+}
+
+// write posts claims and returns their statuses.
+func write(t *testing.T, c *Client, claims ...claim.Claim) []ClaimStatus {
+	t.Helper()
+	res, err := c.WriteClaims(context.Background(), claims)
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+	out := make([]ClaimStatus, len(res))
+	for i, r := range res {
+		out[i] = r.Status
+	}
+	return out
+}
+
+// mustAccept posts claims and fails unless every one is accepted.
+func mustAccept(t *testing.T, c *Client, claims ...claim.Claim) {
+	t.Helper()
+	for i, st := range write(t, c, claims...) {
+		if st != StatusAccepted {
+			t.Fatalf("claim %d (%s/%s): %s", i, claims[i].CID, claims[i].Key, st)
+		}
+	}
+}
+
+// seed writes merged rows straight into the cache — the derived layout search
+// and the counters work on — bypassing claims. Search tests pin SQL behaviour
+// over that layout; how rows get there is the apply loop's business.
+func seed(t *testing.T, c *Client, records map[string]map[string]string) {
+	t.Helper()
+	for id, fields := range records {
+		setMerged(t, c, id, fields)
+	}
+}
+
+func setMerged(t *testing.T, c *Client, rid string, fields map[string]string) {
+	t.Helper()
+	if err := c.update(func(tx *kv.Tx) error {
+		for k, v := range fields {
+			if err := tx.Put(recordPrefix(rid)+k, v); err != nil {
+				return err
+			}
+		}
+		if err := tx.Put(rootKey(rid), ""); err != nil {
+			return err
+		}
+		return refreshSearchTx(tx, rid)
+	}); err != nil {
+		t.Fatalf("seed %s: %v", rid, err)
+	}
+}
+
+func deleteMerged(t *testing.T, c *Client, rid string) {
+	t.Helper()
+	if err := c.update(func(tx *kv.Tx) error {
+		if _, err := tx.DeletePrefix(recordPrefix(rid), 0); err != nil {
+			return err
+		}
+		if _, err := tx.Delete(rootKey(rid)); err != nil {
+			return err
+		}
+		return deleteSearchRowTx(tx, rid)
+	}); err != nil {
+		t.Fatalf("delete %s: %v", rid, err)
+	}
+}
+
+// allKeys lists every cache kv row (full scan).
+func allKeys(t *testing.T, c *Client) []string {
+	t.Helper()
+	var keys []string
+	if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
+		for _, r := range rows {
+			keys = append(keys, r.K)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+	return keys
+}
+
+// rid resolves a cid to its record (fails when unknown).
+func rid(t *testing.T, c *Client, cid string) string {
+	t.Helper()
+	r, err := c.GetByCID(cid)
+	if err != nil || r == "" {
+		t.Fatalf("cid %s unresolved (%v)", cid, err)
+	}
+	return r
+}
+
+func record(t *testing.T, c *Client, cid string) map[string]string {
+	t.Helper()
+	f, err := c.GetMetadataFlat(rid(t, c, cid))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return f
+}