From 84af2987922642327b587cd6fa14ddf283b2f433 Mon Sep 17 00:00:00 2001 From: Worph Date: Sun, 25 Oct 2026 17:35:56 +0000 Subject: [PATCH] v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410 --- Dockerfile | 4 +- Makefile | 2 +- cmd/meta-core/main.go | 29 +- dashboard/src/pages/Identity.tsx | 18 +- docker/entrypoint.sh | 1 + docs/api-mediated-access.md | 251 +++++- docs/meta-core-architecture.md | 418 +++++++--- docs/metadata-storage-structure.md | 577 ++++++------- docs/uuid-rooted-metadata.md | 497 +++++------- editor/src/api/claimsApi.ts | 135 +++ editor/src/api/kvApi.ts | 147 +++- editor/src/api/snapshotApi.ts | 96 --- editor/src/components/SnapshotPanel.css | 209 ----- editor/src/components/SnapshotPanel.tsx | 271 ------- editor/src/kv/ClaimsPanel.tsx | 45 + editor/src/kv/KVApp.css | 6 + editor/src/kv/KVApp.tsx | 34 +- editor/src/kv/KVLeafEditor.tsx | 9 +- editor/src/kv/SignerPicker.tsx | 36 + internal/api/cid_resolve.go | 6 + internal/api/claims.go | 172 ++++ internal/api/claims_test.go | 182 +++++ internal/api/deprecated_field_test.go | 86 -- internal/api/files_tuples_test.go | 20 +- internal/api/handlers.go | 318 +------- internal/api/handlers_identity.go | 80 +- internal/api/identity_gate_test.go | 16 +- internal/api/kv.go | 45 +- internal/api/metadata.go | 208 +---- internal/api/search_index.go | 2 +- internal/api/server.go | 28 +- internal/api/snapshot.go | 167 ---- internal/api/sse_events.go | 31 +- internal/api/sse_snapshot_test.go | 59 +- internal/api/stats.go | 4 +- internal/api/storage_helper_test.go | 53 +- internal/api/udl.go | 315 +++---- internal/api/udl_test.go | 252 ++---- internal/claim/claim.go | 240 ++++++ internal/claim/claim_test.go | 207 +++++ internal/claim/testdata/claim-vectors.json | 93 +++ internal/config/config.go | 15 +- internal/identity/identity.go | 24 + internal/kv/kv.go | 62 +- internal/snapshot/cache.go | 143 ---- internal/snapshot/export.go | 132 --- internal/snapshot/flatten.go | 195 ----- internal/snapshot/import.go | 262 ------ internal/snapshot/manifest.go | 55 -- internal/snapshot/wipe.go | 42 - internal/storage/apply.go | 612 ++++++++++++++ internal/storage/cid_resolution.go | 241 +----- internal/storage/cid_retraction_test.go | 159 ---- internal/storage/claims_test.go | 419 ++++++++++ internal/storage/clear_test.go | 159 +--- internal/storage/client.go | 608 +++++--------- internal/storage/counters_test.go | 74 +- internal/storage/helpers_test.go | 166 ++++ internal/storage/localfile.go | 196 +++++ internal/storage/merge.go | 102 +++ internal/storage/schema_version.go | 96 ++- internal/storage/search.go | 37 +- internal/storage/search_test.go | 17 +- internal/storage/truth.go | 701 ++++++++++++++++ internal/storage/udl.go | 506 +++--------- internal/storage/udl_admin.go | 179 ---- internal/storage/udl_admin_test.go | 233 ------ internal/storage/udl_export_test.go | 160 ---- internal/storage/udl_test.go | 79 -- internal/storage/write_bench_test.go | 137 ++-- internal/watcher/watcher.go | 7 + tools/claims-migrate/main.go | 71 ++ tools/claims-migrate/migrate.go | 902 +++++++++++++++++++++ tools/claims-migrate/migrate_test.go | 375 +++++++++ tools/claims-migrate/mmur1.go | 267 ++++++ tools/search-parity/README.md | 8 +- tools/search-parity/parity.py | 8 +- tools/search-parity/timing.py | 2 +- 78 files changed, 7203 insertions(+), 5617 deletions(-) create mode 100644 editor/src/api/claimsApi.ts delete mode 100644 editor/src/api/snapshotApi.ts delete mode 100644 editor/src/components/SnapshotPanel.css delete mode 100644 editor/src/components/SnapshotPanel.tsx create mode 100644 editor/src/kv/ClaimsPanel.tsx create mode 100644 editor/src/kv/SignerPicker.tsx create mode 100644 internal/api/claims.go create mode 100644 internal/api/claims_test.go delete mode 100644 internal/api/deprecated_field_test.go delete mode 100644 internal/api/snapshot.go create mode 100644 internal/claim/claim.go create mode 100644 internal/claim/claim_test.go create mode 100644 internal/claim/testdata/claim-vectors.json delete mode 100644 internal/snapshot/cache.go delete mode 100644 internal/snapshot/export.go delete mode 100644 internal/snapshot/flatten.go delete mode 100644 internal/snapshot/import.go delete mode 100644 internal/snapshot/manifest.go delete mode 100644 internal/snapshot/wipe.go create mode 100644 internal/storage/apply.go delete mode 100644 internal/storage/cid_retraction_test.go create mode 100644 internal/storage/claims_test.go create mode 100644 internal/storage/helpers_test.go create mode 100644 internal/storage/localfile.go create mode 100644 internal/storage/merge.go create mode 100644 internal/storage/truth.go delete mode 100644 internal/storage/udl_admin.go delete mode 100644 internal/storage/udl_admin_test.go delete mode 100644 internal/storage/udl_export_test.go delete mode 100644 internal/storage/udl_test.go create mode 100644 tools/claims-migrate/main.go create mode 100644 tools/claims-migrate/migrate.go create mode 100644 tools/claims-migrate/migrate_test.go create mode 100644 tools/claims-migrate/mmur1.go diff --git a/Dockerfile b/Dockerfile index 46aeed6..dd1d4cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ # - Bash-based leader election script (flock on shared volume) # - meta-core Go binary (HTTP API, Mount API, Watcher) # - nginx (dashboard, WebDAV, API proxy) -# - SQLite metadata store, embedded in the Go binary (/meta-core/db/meta-core.db) +# - SQLite metadata store, embedded in the Go binary: truth /meta-core/db/meta-core.db +# (signed claims, backed up) + cache /meta-core/db-cache/cache.db (rebuildable) # - rclone (single mount executor — SMB goes through rclone's :smb: backend) # - supervisord (process management - runs on leader only) # @@ -105,6 +106,7 @@ RUN chmod +x /usr/local/bin/meta-core RUN mkdir -p \ /meta-core/locks \ /meta-core/db \ + /meta-core/db-cache \ /meta-core/services \ /meta-core/mounts/errors \ /files \ diff --git a/Makefile b/Makefile index 036db51..1912719 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Variables BINARY := meta-core -VERSION ?= 1.0.0 +VERSION ?= 2.0.0 BUILD_DIR := bin DOCKER_IMAGE := meta-core diff --git a/cmd/meta-core/main.go b/cmd/meta-core/main.go index 6119c81..a809205 100644 --- a/cmd/meta-core/main.go +++ b/cmd/meta-core/main.go @@ -6,6 +6,7 @@ import ( "log" "os" "os/signal" + "path/filepath" "syscall" "time" @@ -17,7 +18,7 @@ import ( ) // Version is set at build time -var Version = "1.0.0" +var Version = "2.0.0" func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) @@ -30,26 +31,36 @@ func main() { // Create leader info provider (Go binary only runs as leader - followers loop in bash) leaderProvider := leader.NewLeaderInfoProvider(cfg) - // Open the metadata store: an embedded SQLite file (internal/kv). There - // is no separate database process to wait for. + // Open the two embedded SQLite files: truth (signed claims, backed up) + // and the cache (everything derived, rebuildable). No database process. + if err := os.MkdirAll(filepath.Dir(cfg.CachePath()), 0o755); err != nil { + log.Fatalf("[meta-core] Failed to create the cache folder: %v", err) + } storageClient := storage.NewClient() - if err := storageClient.Connect(cfg.DBPath()); err != nil { + if err := storageClient.Connect(cfg.DBPath(), cfg.CachePath()); err != nil { log.Fatalf("[meta-core] Failed to open the metadata store: %v", err) } - // Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when - // the existing store predates the UUID-rooted schema, so legacy data - // can't silently leak into the new code paths. See - // docs/uuid-rooted-metadata.md. + // Schema-version gate: refuses a v1.x store (which the default truth path + // still points at) and any unknown layout. No migration code here — see + // tools/claims-migrate. if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil { log.Fatalf("[meta-core] %v", err) } log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion) + // Bring the cache level with truth (replay after a crash, full rebuild + // when the cache folder is empty), then start the apply loop. + started := time.Now() + if err := storageClient.Start(); err != nil { + log.Fatalf("[meta-core] %v", err) + } + log.Printf("[meta-core] Cache ready in %s", time.Since(started).Round(time.Millisecond)) + // Derived counters (record count, per-field schema) are kept exact by the // write path. A store that predates them gets a one-time build here, // before anything else writes. - started := time.Now() + started = time.Now() if built, err := storageClient.EnsureCounters(); err != nil { log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err) } else if built { diff --git a/dashboard/src/pages/Identity.tsx b/dashboard/src/pages/Identity.tsx index 1fa66c7..85f6d35 100644 --- a/dashboard/src/pages/Identity.tsx +++ b/dashboard/src/pages/Identity.tsx @@ -180,10 +180,9 @@ async function fetchAccounts(): Promise { // // Never throws and never reports an error upward: a name is decoration on top // of the uid, and the account list must render identically whether the store is up -// (identity itself is file-backed, the UDL is not). The name is also *unsigned* -// — meta-core stores the plaintext beside the signed record and verifies -// nothing — so the uid stays on screen as the real identity, and the length cap -// bounds what a hostile writer can do to the layout. +// (identity itself is file-backed, the UDL is not). The name is the user's +// signed MMUR2 claim (meta-core verified it on write); the uid stays on screen +// as the real identity, and the length cap bounds the layout. async function fetchDisplayName(uid: string): Promise { const url = `/api/udl/record?uid=${encodeURIComponent(uid)}` + @@ -192,12 +191,11 @@ async function fetchDisplayName(uid: string): Promise { try { const res = await fetch(url); if (!res.ok) return null; // 503 when storage is down, 500, … - const data = (await res.json()) as { exists?: boolean; value?: unknown }; - // Key off `value`, not `exists`: clearing a name writes a tombstone, which - // leaves the cell existing with no plaintext. Private-tier cells look the - // same, which is exactly right — there is nothing to show for either. - if (typeof data.value !== 'string') return null; - const name = data.value.trim(); + const data = (await res.json()) as { exists?: boolean; claim?: { value?: unknown } | null }; + // A retraction (null) or ciphertext ({b64}) has nothing to show. + const value = data.claim?.value; + if (typeof value !== 'string') return null; + const name = value.trim(); return name ? name.slice(0, 64) : null; } catch { return null; diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7605a49..773ccec 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -10,6 +10,7 @@ set -e # Create required directories mkdir -p /meta-core/locks mkdir -p /meta-core/db +mkdir -p /meta-core/db-cache mkdir -p /meta-core/mounts/errors mkdir -p /files mkdir -p /var/log/supervisor diff --git a/docs/api-mediated-access.md b/docs/api-mediated-access.md index a1cc767..cf2b2e9 100644 --- a/docs/api-mediated-access.md +++ b/docs/api-mediated-access.md @@ -1,4 +1,14 @@ -# API-Mediated Access (Redis Lockdown) +# API-Mediated Access (Redis Lockdown → Signed Claims) + +> **meta-core v2.0.0 — the claims model.** The implementation basis is +> [`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md) +> (meta-root). Where this file and that one disagree, that one wins. The +> *access* contract below still holds unchanged — HTTP only, events over +> SSE, no database reachable from outside `metacore-app`. The *write* +> contract changed: every metadata write is a **signed claim** posted to +> `POST /api/claims`, and the legacy unsigned write endpoints answer +> `410 Gone`. See ["Write contract — signed claims"](#write-contract--signed-claims-v200) +> below. ## Status @@ -69,20 +79,23 @@ we don't fully control. ## The architecture in one picture ``` - meta-core (only Redis writer + reader) + meta-core v2.0.0 (only store writer + reader) ┌──────────────────────────────────────┐ │ HTTP API surface: │ - │ POST/PUT/PATCH /api/metadata/* │ ◄── writes - │ GET /api/meta/{cid} │ ◄── reads (by CID) - │ GET /api/metadata/* │ ◄── reads (by id) - │ GET /api/file/{cid} │ ◄── content - │ GET /api/events/files │ ◄── SSE stream - │ GET /api/events/meta │ ◄── SSE stream + │ POST /api/claims │ ◄── writes (signed claims only) + │ POST /api/merge │ ◄── stateless merge, no write + │ GET /api/meta/{cid} │ ◄── reads (by CID, merged) + │ GET /api/meta/{cid}/claims │ ◄── reads (raw signed claims) + │ GET /api/metadata/* │ ◄── reads (by rid, merged) + │ GET /api/file/{cid} │ ◄── content + │ GET /api/events/files │ ◄── SSE stream + │ GET /api/events/meta │ ◄── SSE stream (merged changes) │ │ │ - │ ▼ │ - │ Redis (file:*, cid:*, streams) │ - │ ◄── container-local, no exposed │ - │ port, no docker-network alias │ + │ ▼ verify sig → truth → apply │ + │ truth /meta-core/db/meta-core.db │ + │ cache /meta-core/db-cache/cache.db │ + │ ◄── embedded SQLite, no port, │ + │ no docker-network alias │ └──────────────────────────────────────┘ ▲ ▲ │ HTTP │ HTTP SSE @@ -95,9 +108,18 @@ we don't fully control. ``` **The single-writer rule.** meta-core has exclusive write authority over -the `file:*`, `cid:*`, `file:__index__` (now `idx:*`), and `meta-core:*` keyspaces. No -other service can write — not because of an honoured convention, but -because they can't reach Redis at all. +its store — in v2.0.0 the two SQLite databases (truth and cache); in the +Redis era the `file:*`, `cid:*`, `file:__index__` (later `idx:*`), and +`meta-core:*` keyspaces. No other service can write the store — not because +of an honoured convention, but because they can't reach it at all. + +**What changed in v2.0.0 is who may *author* metadata.** meta-core is still +the only process that touches the database, but it no longer accepts +unsigned values: a writer submits signed votes (claims), meta-core verifies +each signature and stores the vote. meta-core **never signs a claim and +holds no writer key** — a store that could sign for everyone would prove +nothing. The signature is the write authentication; there are no +per-writer tokens. **Stream reads also mediated.** Events go out through SSE endpoints on meta-core's HTTP surface. Consumers don't see the underlying Redis @@ -108,6 +130,147 @@ without breaking every consumer. **Redis is private.** No host port mapping, no docker-network alias, no URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base. +(Since 2026-10 there is no Redis at all; the SQLite files are reachable only +from inside `metacore-app`.) + +## Write contract — signed claims (v2.0.0) + +Every metadata value is a **signed vote**: *signer `uid` says `key = value` +about content `cid`* — `claim(uid, cid, key) → value + ts + sig`. The +record a reader sees is the **merged view** that meta-core precomputes from +the votes; a reader's own votes always win in their view. Full model, +schema and merge rules: the +[implementation basis](../../../docs/implementation/metacore-claims-model.md) +§1–§7. + +### Endpoint table + +| | Endpoint | Behaviour | +|---|---|---| +| **Write** | `POST /api/claims` | A batch of signed claims (a JSON array, or `{"claims": [...]}`). **The only metadata write path.** | +| Removed | `PUT/PATCH/DELETE /meta/{hash}`, `PUT/DELETE /meta/{hash}/{key...}`, `POST /meta/{hash}/_add/{key...}`, `PUT/DELETE /api/metadata/{hashId}`, `PUT /api/metadata/{hashId}/property`, `POST /api/metadata/batch` | `410 Gone`, with a pointer to `POST /api/claims` | +| Kept | `PUT/GET/DELETE /api/kv/value` | Service scratch KV (`btmagnet:*`, `searchcov:*`), plain rows of the cache database's `kv` table (any key outside the reserved prefixes `file:`, `cid:`, `idx:`, `dup:`, `ev:`, `evhwm:`, `meta:`, `meta-core:`) — losing it only costs a refetch | +| Kept | `POST /api/files/register`, the file watcher | Fill `local_file` (box-local `filePath`, size, mtime, midhash — unsigned) and `cid_group` | +| UDL facade | `/api/udl/*` (same paths) | Writes go through the claims write path; reads come from the claims (per-signer index `claim_by_pid`). Records are MMUR2 claims. Shapes: see "UDL facade shapes" below | +| Read | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{hash}`, `GET /api/metadata/hash-ids`, `GET /api/metadata/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `GET /api/schema`, `GET /api/stats` | Unchanged paths and response shapes, served from the merged view (+ `local_file`); optional `?as=`. They carry no `nsrc` | +| **New** | `GET /api/meta/{cid}/claims` | `{rid, cids, claims}`: the record's rid, the cids of its group and every live signed claim on them (all signers, tombstones excluded). `?key=` keeps one exact key, `?uid=` one signer. `404 unknown_cid` when no record holds the cid | +| **New** | `GET /api/metadata/{rid}/claims` | Same body and filters, addressed by rid (a cid is also accepted); `404` for an unknown record. What meta-search ingest and the gateway's corpus search use | +| **New** | `POST /api/merge` | Stateless: body `{claims, as?}`; verifies each claim (invalid ones dropped and counted, never a batch error), merges with the store's merge rule, overlays `as`'s own votes (in the request and held by this core for the same cids). Returns `{fields, nsrc, rejected}` — merged `key → value`, `key → agreeing signers`, rejected count. `413` above 10 000 claims. Writes nothing | +| Events | `GET /api/events/meta` | Unchanged `{type: set\|del, key, ts}` shape, now emitted for changes of the **merged** view, `key` still the literal `file:/`; `GET /api/events/files` unchanged. Both streams are rows of the cache `kv` table (`ev:meta:events/…`, `ev:file:events/…`), ~100k entries each | +| Admin | `POST /api/metadata/clear` | Wipes both databases (truth and cache) | +| Removed | `/api/snapshot/export`, `/api/snapshot/import`, `/api/snapshot/wipe` | No snapshot format in v2.0.0 | + +Endpoints the claims model does not touch keep their behaviour: discovery +(`/health`, `/status`, `/leader`, `/urls`, `/neighbors`), the account-key +identity service (`/api/identity/*` — its removal is a separate ticket; +v2.0.0 adds the `format` field to `sign` / `sign-batch` — `"compact"` or +`"der"`, default compact for `"MMUR2\n"` payloads and DER otherwise — and +makes `DELETE /api/identity` sign and store a retraction of every live +claim of the account before removing it), +`/api/file/{cid}/info`, `/file/cid`, `/data/{hash}*`, mounts, watchers and +WebDAV. + +### The claim on the wire (JSON form) + +```json +{ "cid": "bafy…", "uid": "zQ3s…", "key": "title", + "value": "The Matrix", // or {"b64": "…"} for bytes, or null + "ts": 1793001234567, + "sig": "base64(64 bytes)" } +``` + +| Field | Meaning | +|---|---| +| `cid` | The subject: a content CID as the writer knows it (bare CID string, as in `cids/*`). Never a rid — rids are local and never signed | +| `uid` | The signer: `"z" + base58btc(33-byte compressed secp256k1 pubkey)` | +| `key` | The field (`METADATA_KEYS.md`); a key-set member is inside the key (`genres/Drama`) | +| `value` | Text = public plaintext; `{"b64": …}` = bytes, a private (ciphertext) envelope; `null` = retraction | +| `ts` | Milliseconds since epoch. It **is** the version: only increases per `(uid, cid, key)` | +| `sig` | MMUR2: secp256k1 ECDSA, compact 64 bytes, over SHA-256 of `"MMUR2\n" ‖ canonical CBOR [cid, uid, key, value, ts]` | + +A batch (the `POST /api/claims` body) is a JSON array of these (or +`{"claims": [...]}`). Kamilata and +the gateway wire carry the same five fields plus `sig`; the signed bytes +never change on any hop, and relays (meta-search, a client persisting a +gateway hit) forward claims verbatim — they never re-sign. + +### Per-claim statuses + +`POST /api/claims` answers with one status per claim: + +| Status | Meaning | +|---|---| +| `accepted` | Verified and stored (or a retraction stored as a tombstone) | +| `unchanged` | The stored row for `(uid, cid, key)` already has this `ts` and `sig` — **or the same value under any `ts`** (a signer re-asserting what it holds): no verification, no write, and the stored row keeps its older `ts` | +| `stale` | The stored `ts` is ≥ the incoming `ts`; ignored (blocks replay of old votes) | +| `bad_signature` | The MMUR2 signature does not verify against the `uid`'s public key | +| `bad_request` | The claim is malformed | + +The response is `200` with a JSON array of `{status, reason?}`, one entry +per claim, in input order, even when some are refused. A batch above 10 000 +claims answers `413 too_many_claims`; a body that is not claims JSON +`400 invalid_json`. The +removed write routes answer `410 {"error":"gone","use":"POST /api/claims"}`. +`POST /api/claims`, `POST /api/merge` and the two claims read routes are +container-to-container routes, not routed through Caddy. + +Rules: every claim is signed — ciphertext and retractions included; an +unsigned or badly signed claim is refused. A retraction (`value: null`) is +kept as a tombstone, so a peer cannot replay the vote it retracted. Values +are stored exactly as signed; normalisation (trim + case-fold, no NFC) +happens only inside the merge. + +Accepted claims commit to the truth database in one group-committed +transaction; an apply loop then folds them, in commit order, into the +cache (the merged `file:/` rows of its v1-layout `kv` table, +FTS, counters, events). `POST /api/claims` answers only once the cache has +applied the batch, so the writer reads its own write; another reader may +briefly see the previous merged value — the cache is behind, never wrong. + +### `?as=` — the viewer's overlay + +Every merged read accepts an optional `?as=`. The merge then puts that +signer's own live text vote first for each key (read-time overlay found +through the per-signer index, never stored in the shared merged view). Without +`?as=` the read returns the shared merge: most distinct signers → oldest +`ts` → byte-wise smallest value; ciphertext never merges. + +### Raw claims and client-side merging + +The claims read routes (`GET /api/metadata/{rid}/claims` for meta-search +ingest and the gateway's corpus hits, `GET /api/meta/{cid}/claims` by cid) +return the raw signed votes of every signer, not a merged map. A client that receives +claims from the swarm or a gateway displays them by calling its **local** +meta-core's `POST /api/merge` (one merge implementation, in Go), and +persists a hit by posting the claims **as received** to `POST /api/claims`. + +### UDL facade shapes + +Claims in the JSON form above (`internal/api/udl.go`): + +| Route | Request | Response | +|---|---|---| +| `PUT /api/udl/record` | one claim | `200 {accepted: true, status}` for `accepted` / `unchanged`; `409 stale_version` for `stale`; `400` (status as slug) for `bad_signature` / `bad_request` | +| `PUT /api/udl/records` | `{records: [claim, …]}` | `200 {results: [{key, accepted, status, error?}, …]}`, input order; `accepted` = `accepted` or `unchanged` | +| `GET /api/udl/record?uid=&cid=&key=` | — | `{exists, claim}` — the stored claim, tombstone included; `exists` false when absent or a tombstone | +| `GET /api/udl/user/{uid}/key/{key}`, `/user/{uid}/cid/{cid}` | — | `{entries: [claim, …]}` | +| `GET /api/udl/user/{uid}/all?after=&limit=` | — | `{entries: [claim, …], nextCid}` | +| `GET /api/udl/user/{uid}/count` | — | `{cids}` | +| `GET /api/udl/cid/{cid}/users` | — | `{users: [uid, …]}` | +| `GET /api/udl/cid/{cid}/aggregate?keys=` | — | `{aggregates: [{key, count, avg, trues, sample_size}]}` | +| `GET /api/udl/users/stats` | — | `{users: {uid: {records, cids, keys}}}` for **every signer** with live claims — plugin, feeder, gateway and service keys included. `/api/stats` `udlUsers` is the size of this map | + +### What a writer needs + +Each writing container (every meta-sort plugin, every self-publishing +feeder, gateway core, meta-sort, meta-share, meta-search) signs with its +**own** key; users sign in the browser. meta-core holds none of these keys. +Key configuration per signer: implementation basis §9. + +Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are not +claims: the watcher and `POST /api/files/register` write them into the +cache's `local_file` table, they are joined into the record view unsigned, +and they never appear in `/api/meta/{cid}/claims` or on the swarm. ## Eventing — HTTP Server-Sent Events @@ -263,12 +426,19 @@ seen). The contract above holds unchanged: - Retention is ~100k entries per stream, trimmed every 10k appends. - IDs keep the Redis `-` form, so cursors persisted by meta-sort / meta-share / meta-stremio before the switch still compare. -- `meta:events` entries (`{type: set|del, key, ts}`) are appended by the - storage engine **in the same transaction** as the `file:` key change — - `del` only when the key existed, `set` only when the value actually - changed (since v1.2.0; an unchanged write is a no-op). The Redis - keyspace-notification hop (PSUBSCRIBE in `MetaPublisher`), which could - drop events when it lagged, is gone. +- `meta:events` entries (`{type: set|del, key, ts}`) describe changes of + the **merged** view (v2.0.0): the apply loop emits `set`/`del` (keyed by + rid + key) only when a claim actually changed a merged value, in the same + cache transaction as that change. A group merge (two cid groups linked by + a `cids/*` vote) emits `del` for every key of the losing rid and `set` for + the survivor. *History:* v1.x appended them in the same transaction as the + `file:` key change; before 2026-10 a Redis keyspace-notification hop + (PSUBSCRIBE in `MetaPublisher`) could drop events when it lagged. +- **Cache rebuild (v2.0.0).** Events live in the cache database. A cache + rebuild (missing cache folder, or catch-up after an unclean shutdown) + emits no per-key events; SSE consumers get `event: gap` and take a + snapshot, as they do for a trimmed cursor. Event IDs stay time-based, so + they keep rising across a rebuild. - **`Last-Event-ID: 0` / `0-0` on `/api/events/meta` returns a snapshot** (since v1.2.0): one `set` per existing record key, then the live tail from the head captured before the snapshot. Only the last snapshot event carries @@ -522,6 +692,10 @@ durability across a meta-core restart). ### PR B — meta-sort writes via HTTP +> *History.* PR B moved writes onto the unsigned `/meta/{hash}` family. +> In v2.0.0 those routes answer `410 Gone`; writers sign claims and post +> them to `POST /api/claims` (see "Write contract — signed claims"). + Swap `RedisClient` for an `HttpKVClient` implementation of the existing `IKVClient` interface in `packages/meta-sort-core`. All writes (`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP @@ -603,7 +777,7 @@ metacore-app produces a connection refused. The lock is real. | Backing store for streams? | Redis Streams at the time; SQLite kv rows since 2026-10 | SSE wraps it; the 2026-10 swap proved it — no consumer changed | | Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it | | Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha | -| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. | +| **Canonical write API path?** | **`POST /api/claims` (v2.0.0)** — signed claims only; every unsigned write route answers `410 Gone` | *History (v1.x):* the **`/meta/{hash}` family** was the canonical write surface. It had GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. | | **SSE event-id contract?** | **Opaque to consumers** | It is `-` (the Redis Stream entry form, kept after the 2026-10 SQLite switch); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. | | **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. | | **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. | @@ -611,6 +785,11 @@ metacore-app produces a connection refused. The lock is real. ## Error envelope +(`410 Gone` from the removed v1.x write routes carries this envelope with a +message pointing to `POST /api/claims`. Per-claim outcomes of +`POST /api/claims` are statuses inside the response, not envelopes — see +"Per-claim statuses" above.) + Standard shape for all 4xx / 5xx responses from the metadata API: ```json @@ -629,6 +808,10 @@ Stable `error` slugs the client should switch on: | `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. | | `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. | | `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. | +| `invalid_json` | 400 | no | `POST /api/claims` / `POST /api/merge` body is not claims JSON. Caller bug. | +| `too_many_claims` | 413 | no | More than 10 000 claims in one `POST /api/claims` or `POST /api/merge`. Split the batch. | +| `stale_version` | 409 | yes | `PUT /api/udl/record`: an equal or newer `ts` is stored for this signer, cid and key. Re-read the record, sign with a newer `ts`, retry. | +| `gone` | 410 | no | A removed unsigned write route. Body is `{"error":"gone","use":"POST /api/claims"}` (no `message` / `retryable`). | | `storage_unavailable` | 503 | yes | Store transient (Redis at the time; the embedded SQLite store since 2026-10). Caller should retry with backoff. | | `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. | @@ -643,10 +826,9 @@ import cycle on the api package; consolidating it into a shared Mostly answered post-implementation: -- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The - editor's React UI continues to call it; the surface itself was untouched - by this work. Worth a follow-up to consolidate the editor onto - `/meta/{hash}` and delete the parallel routes. +- **Deprecation timeline for `/api/metadata/{hashId}`.** Answered by + v2.0.0: its `PUT`/`DELETE` (and `PUT …/property`) answer `410 Gone` + like every unsigned write; `GET` stays as a merged read. - **Initial bootstrap flood on `/api/events/files`.** Resolved in practice: meta-fuse uses `initialCursor: '0-0'` to deliberately @@ -666,8 +848,9 @@ Mostly answered post-implementation: - **Authentication between internal services.** The hash-lock OIDC perimeter handles external clients; internal service-to-service trust is by network isolation (docker network for dev, equivalent - isolation in prod deployments). Adding service-to-service auth is a - separate concern. + isolation in prod deployments). Since v2.0.0 metadata *writes* carry + their own authentication — the claim signature — so meta-core needs no + per-writer tokens; transport-level service auth remains out of scope. - **Replacing the public HTTP API surface.** This proposal exercises what already exists; the only new endpoints are the two SSE streams. - **Introducing new transport, broker, proxy, or sidecar.** Everything @@ -716,6 +899,11 @@ What actually shipped vs. what this doc planned. ### Write-path CID resolution (`storage.ResolveRoot`) +> *History (v1.x).* In v2.0.0 a claim names a cid, never a rid, and the +> hidden rid comes from `cid_group` (a `cids/` vote links two +> groups; the older rid survives). The dual-root failure below cannot +> recur: there is no write that addresses a root. + Not in the original plan but load-bearing. **The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed @@ -766,9 +954,10 @@ duplicated in each to avoid an import cycle on the `api` package. - **meta-dup SSE + read migration.** Not deployed, no test harness. When revived, copy meta-fuse's pattern. -- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate - editor-internal API surface is untouched; could be retired in a - follow-up by porting the editor UI to `/meta/{hash}`. +- **Editor writes (resolved in v2.0.0).** `PUT /api/metadata/{hashId}` + answers `410`. meta-sort-editor's manual edits go through meta-sort's + backend, which signs them as meta-sort (its own key) and posts them to + `POST /api/claims`. The editor's reads keep using `/api/metadata/{hashId}`. - **Consolidated `internal/httperr` package** for the typed envelope helpers (currently duplicated 4× to avoid the import cycle). Code hygiene; functionally a no-op. diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index 2a3d59e..cc00075 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -1,15 +1,22 @@ # Meta-Core Architecture +> **meta-core v2.0.0 — the claims model.** Implementation basis: +> [`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md) +> (meta-root). Where this file and that one disagree, that one wins. + ## Document Purpose This document describes the architectural design of **meta-core**, the -standalone container that owns the metadata store (an embedded SQLite -file; Redis until 2026-10), the leader-info file, the HTTP+SSE +standalone container that owns the metadata store (two embedded SQLite +databases since v2.0.0 — *truth* and *cache*; a single SQLite file in +v1.x; Redis until 2026-10), the leader-info file, the HTTP+SSE metadata surface, and a WebDAV file server. It covers the leader gate, -storage data model, API design, the SSE event surface, and the integration -contract with other MetaMesh services. +the claims data model and its write/apply pipeline, API design, the SSE +event surface, and the integration contract with other MetaMesh services. -For the on-disk key layout, see +For the engine design (SQLite, group commit, measurements), see +[metacore-storage-engine.md](../../../docs/project-architecture/metacore-storage-engine.md); +for the on-disk layout (tables of both databases), see [metadata-storage-structure.md](metadata-storage-structure.md). For the external-access design (store lockdown, SSE wire contract, auth rules), see [api-mediated-access.md](api-mediated-access.md). For the UUID-rooted @@ -24,7 +31,7 @@ view of file metadata: | Service | Language | Role | |---|---|---| -| **meta-sort** | TypeScript | Watches folders, runs plugins, writes metadata via meta-core's HTTP API. | +| **meta-sort** | TypeScript | Watches folders, runs plugins; each plugin signs its own claims and posts them to meta-core. | | **meta-fuse** | TypeScript / Rust | Serves the virtual filesystem from metadata. | | **meta-stremio** | Python | Stremio addon for HLS streaming. | | **meta-dup** | TypeScript | Duplicate detection. | @@ -32,8 +39,14 @@ view of file metadata: meta-core sits below all of these as the single owner of: -1. **The metadata store** — an embedded SQLite file - (`/meta-core/db/meta-core.db`) holding metadata and the event streams. +1. **The metadata store** — two embedded SQLite databases: **truth** + (`/meta-core/db/meta-core.db`, backed up) holding the signed claims, and + **cache** (`/meta-core/db-cache/cache.db`, never backed up) holding every + index and derived row — a v1-layout `kv` table (the merged view as + `file:/` rows, lookup, counters, event streams, service + scratch KV) with its field index and FTS, plus the `claim_by_pid` and + `local_file` tables. (v2.0.0 is implemented, uncommitted and not yet + deployed as of 2026-10-25.) 2. **The leader-info file** — the one place services look to discover meta-core's URLs. 3. **The HTTP+SSE surface** — the only way external services touch the data plane. 4. **A WebDAV server** — file access for plugins and other services. @@ -44,7 +57,12 @@ meta-core sits below all of these as the single owner of: > through meta-core's HTTP+SSE surface. This is the contract that PR D of [api-mediated-access.md](api-mediated-access.md) -landed and that the rest of this document assumes. +landed and that the rest of this document assumes. v2.0.0 adds a second: + +> Every metadata value is a **signed vote** — `claim(uid, cid, key) → value` +> + `ts` + `sig`. The only write is `POST /api/claims`. meta-core verifies +> every claim and **never signs one**: it holds no writer key, and the +> signature is the write authentication. --- @@ -68,13 +86,15 @@ database file and the writable state under `/meta-core`. │ ├─ rclone rcd (mount daemon) │ │ ├─ mount-watcher.sh (rclone mount lifecycle) │ │ └─ meta-core (Go) ──► HTTP + SSE + WebDAV on :9000 │ -│ + embedded SQLite (db/meta-core.db) │ +│ + embedded SQLite: truth (db/meta-core.db) │ +│ and cache (cache/cache.db) │ └────────────────────────────────────────────────────────────────────────┘ │ ┌────────────────────────────┴────────────────────────────┐ │ /meta-core (shared volume) │ │ locks/kv-leader.{lock,info} │ - │ db/meta-core.db (+ -wal, -shm) │ + │ db/meta-core.db (+ -wal, -shm) truth, backed up │ + │ cache/cache.db (+ -wal, -shm) cache, never backed │ │ services/*.json │ │ mounts/{mounts.json, errors/} │ │ watchers.json │ @@ -203,67 +223,166 @@ Other services read this file, then call `GET /urls` for full discovery ## Schema-Version Gate -Before serving any traffic, `cmd/meta-core/main.go:54-61` calls -`storage.EnsureSchemaVersion`. The sentinel lives in the store at -`meta-core:schema-version` (`internal/storage/schema_version.go`). The -gate refuses to boot when the existing store layout predates the -UUID-rooted schema, so legacy `file:midhash256:*` data can't leak into -the current code paths. In alpha there is no automated migration — -operators wipe with `pnpm run clean:all` and restart. +Before serving any traffic, `cmd/meta-core/main.go` opens both databases +and calls `storage.EnsureSchemaVersion` (before the cache catch-up). In +v2.0.0 the sentinel is the `schema-version = "3"` row of the truth +database's `meta` table. The gate: + +- **explicitly refuses a v1.x store** — a truth file holding the v1 `kv` + table (`meta-core:schema-version = "2"`), which the default path still + points at — with a message naming `tools/claims-migrate`; +- refuses a store with claims but no sentinel, or any other version; +- stamps a fresh store with `"3"`. + +So no pre-claims data leaks into the current code paths. The meta-core +**service binary contains no migration code**: watch.nsl.sh is converted +once by the separate one-off tool `packages/meta-core/tools/claims-migrate` +(not built into the image; implementation basis §11); every other box and +dev stack starts empty (operators wipe with `pnpm run clean:all` and +restart). --- ## Data Model -### Two concepts: metadata vs. file bytes +### The one concept: signed votes ``` -┌────────────────────────────────────────────────────────────────────┐ -│ METADATA (embedded SQLite, kv table) │ -│ │ -│ Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars) │ -│ Storage: flat string keys │ -│ file:/ → string value │ -│ file:/cids/ → "true" (key-set) │ -│ dup:/ → "" (one per path) │ -│ │ -│ No stored canonical_cid — derived by rank on read. │ -│ │ -│ Aliases (reverse index, one key each): │ -│ cid: → │ -│ │ -│ Index: idx:roots/ → "" (one per root) │ -│ │ -│ Mintable without a local file (remote / deleted entries OK). │ -│ Resolvable by any registered CID in O(1). │ -└────────────────────────────────────────────────────────────────────┘ - -┌────────────────────────────────────────────────────────────────────┐ -│ FILE BYTES │ -│ │ -│ Location: /files volume (shared across containers). │ -│ Access: path resolution via metadata, served via /webdav/*, │ -│ /file/{cid} (reverse-lookup), or /data/{hash}. │ -│ Path stored as filePath in the metadata, relative to /files. │ -└────────────────────────────────────────────────────────────────────┘ +claim(uid, cid, key) → value + ts + sig ``` -### Why UUID-rooted? +- `uid` — the signer (a secp256k1 public key, `z…` base58btc). One key per + writing container (each meta-sort plugin, each self-publishing feeder, + gateway core, meta-share, meta-search) and one per user. +- `cid` — the subject, a content CID; never a rid. +- `key` — the field (`METADATA_KEYS.md`); key-set members live inside the + key (`genres/Drama`, `cids/`). +- `value` — text (public), bytes (signed + encrypted envelope) or null + (retraction, kept as a tombstone). +- `ts` — milliseconds; it **is** the version and only increases per + `(uid, cid, key)`. +- `sig` — MMUR2: secp256k1 ECDSA (compact 64 bytes) over SHA-256 of + `"MMUR2\n" ‖ canonical CBOR [cid, uid, key, value, ts]`. -Roots were previously content-hash tokens (`file:midhash256:abc/...`), -which privileged one hash algorithm. UUID-rooted storage treats every CID -symmetrically as an alias, makes `GET /api/meta/` O(1) for any -registered CID, and lets new digest types be added without rewriting root -keys. See [uuid-rooted-metadata.md](uuid-rooted-metadata.md) for the -design rationale and [metadata-storage-structure.md](metadata-storage-structure.md) -for the on-disk layout. +The old shared record (`file:/`) and the old User Data Layer +store (`udl:rec:*`) both collapse into this shape. The store distinguishes +only plaintext (signed) from ciphertext (signed + encrypted); "personal vs +shared" is app logic. + +### Two databases + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ TRUTH /meta-core/db/meta-core.db (backed up; PKs only) │ +│ │ +│ principal(pid, uid) uid ↔ local surrogate pid │ +│ claim(cid, key, pid → value, ts, sig, seq) │ +│ the votes; PK (cid, key, pid) │ +│ cid_group(cid → rid) hidden local record identity │ +│ meta(name → value) schema-version, last_seq │ +└──────────────────────────────────────────────────────────────────────┘ + │ apply loop (seq order) + ▼ +┌──────────────────────────────────────────────────────────────────────┐ +│ CACHE /meta-core/db-cache/cache.db (never backed up; rebuildable)│ +│ │ +│ kv(k → v) the v1 layout, every row derived: │ +│ file:/ the merged view every read uses │ +│ cid: cid → rid lookup │ +│ idx:roots/ idx:fts/ idx:count/ idx:schema/ roots, FTS rowid, │ +│ counters (search totals, stats) │ +│ ev:/…, evhwm:… /api/events/meta + /files backlog │ +│ meta:applied_seq, meta:rebuilt_at apply-loop bookkeeping │ +│ anything else /api/kv/value scratch KV │ +│ kv_field_v (index), fts filter + free-text search │ +│ claim_by_pid(pid, cid, key) "everything uid X wrote" (?as=, │ +│ UDL facade, identity retraction) │ +│ local_file(path → midhash, size, mtime) watcher facts, unsigned │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +Disaster recovery is "delete the cache, replay truth". `META_CORE_DB_PATH` +names the truth file, `META_CORE_CACHE_PATH` the cache file. The cache keeps +the v1 `kv` layout so every v1 reader (record reads, field index, FTS, +counters, SSE + snapshot) runs unchanged on derived rows; `nsrc` is computed +by the merge (`POST /api/merge`), not stored. Full schema: implementation +basis §5. + +### Records: cids, groups and the hidden rid + +A vote never mentions a rid. A **rid** (UUIDv7, 26-char Crockford, as in +v1.x) is a local name for a group of cids known to be the same content, +held in `cid_group`: + +- A new cid with no link gets its own group and a fresh rid. +- A `cids/` vote on cid X links X's group and ``'s group; the + group with the older rid survives, the losing rid gets a `del` event per + key and the survivor `set` events (no alias table). Retracting the link + does **not** split the group (phase 1); the record view synthesises + `cids/ = "true"` for every cid of the group from `cid_group`. +- Reading record R = every vote on every cid of R's group, merged per key; + within a group one signer counts once per key. +- rids are never signed and never travel; consumers that hold them + (meta-sort, meta-fuse, meta-stremio) keep working because `cid_group` + lives in truth. + +### Merge + +Per `(rid, key)`, over the group's votes, each signer's newest vote counted +once (`internal/storage/merge.go`, shared by the apply loop, `?as=` and +`POST /api/merge`): + +1. the viewer's own live text vote (`?as=`, applied at read time, + never stored); +2. retractions and ciphertext drop out — only text merges; +3. most distinct signers for the normalised value (**trim + case-fold + only**, no NFC) — `nsrc` is that count (computed, not stored); +4. tie → oldest `ts`; +5. tie → byte-wise smallest value. + +The stored value is the winning vote's exact text. Key-sets need no special +code (each member is a key, present while one vote supports it). A +different kind of fact gets a different key, so a guesser (e.g. +filename-parser) never competes with a lookup on the same key. + +### Box-local file facts + +`filePath`, size, `mtimeNano` and midhash are **not claims** — meta-core +cannot sign, and the disk can always be rescanned. The file watcher and +`POST /api/files/register` write them into the cache's `local_file` table +(and the midhash into `cid_group`); they are joined into the record view +unsigned and never reach the claims API or the swarm. When several local +paths hold the same content, `filePath` is the lexicographically first one +(size and mtime from the same row); a signed `sizeByte` claim wins over the +on-disk size. + +### File bytes + +``` +Location: /files volume (shared across containers). +Access: path resolution via local_file, served via /webdav/*, + /file/{cid} (reverse-lookup), or /data/{hash}. +Path: filePath (from local_file) in the record view, relative to /files. +``` + +### History: UUID-rooted flat keys (v1.x) + +Until v2.0.0 a record was a set of flat keys under an opaque UUID root — +`file:/`, `file:/cids/` key-sets, +`dup:/`, a reverse index `cid: → ` and a root +index `idx:roots/`, all in one kv table. Roots had previously been +content-hash tokens (`file:midhash256:abc/...`); UUID roots treated every +CID symmetrically as an alias. The rid of v2.0.0 is that UUID, kept hidden +and local; migrated records kept their UUIDs. See +[uuid-rooted-metadata.md](uuid-rooted-metadata.md) (the *why*) and the +History note of [metadata-storage-structure.md](metadata-storage-structure.md). ### Path resolution -`filePath` values stored in metadata are absolute paths under the `/files` -volume (e.g. `/files/watch/movies/Movie.2024.mkv`). The `/files` prefix is -itself a per-host detail — moving the volume only requires re-mounting it -at the same path inside the container. +`filePath` values are absolute paths under the `/files` volume (e.g. +`/files/watch/movies/Movie.2024.mkv`). The `/files` prefix is itself a +per-host detail — moving the volume only requires re-mounting it at the +same path inside the container. --- @@ -282,16 +401,16 @@ Bootstrap / discovery GET /neighbors (alias: /api/neighbors) GET /api/stats -Metadata (primary surface for other services) +Claims (the only metadata write path) + POST /api/claims signed claim batch → per-claim status + POST /api/merge {claims, as?} → {fields, nsrc, rejected}; no write + GET /api/meta/{cid}/claims {rid, cids, claims} of the cid's group (?key=, ?uid=; 404 unknown) + GET /api/metadata/{rid}/claims same, addressed by rid + +Metadata reads (merged view + local_file; optional ?as=) GET /meta GET /meta/{hash} - PUT /meta/{hash} - PATCH /meta/{hash} - DELETE /meta/{hash} GET /meta/{hash}/{key...} - PUT /meta/{hash}/{key...} - DELETE /meta/{hash}/{key...} - POST /meta/{hash}/_add/{key...} CID-addressed (public, auth-bypassed at the perimeter) GET /api/meta/{cid} @@ -301,24 +420,45 @@ CID-addressed (public, auth-bypassed at the perimeter) HEAD /data/{hash} GET /data/{hash}/path -Editor / KV / schema / snapshot +Editor / KV / schema GET /api/metadata/hash-ids GET /api/files/tuples + POST /api/files/register (fills local_file + cid_group) GET /api/metadata/list POST /api/metadata/search - POST /api/metadata/batch - POST /api/metadata/clear - {GET,PUT,DELETE} /api/metadata/{hashId} - {GET,PUT} /api/metadata/{hashId}/property - GET /api/kv/{info,keys,tree,search,find,value} - PUT /api/kv/value - DELETE /api/kv/value - GET /api/kv/key/{key...} + POST /api/metadata/clear (wipes both databases) + GET /api/metadata/{hashId} (hashId = rid) + GET /api/metadata/{hashId}/property + GET /api/kv/{info,keys,tree,search,find} (read-only browser over the cache kv table) + {GET,PUT,DELETE} /api/kv/value (service scratch keys only; reserved prefixes refused) + GET /api/kv/key/{key...} (read-only) GET /api/schema POST /api/schema/rescan - GET /api/snapshot/export - POST /api/snapshot/import - POST /api/snapshot/wipe + +User Data Layer (facade over claims; records are MMUR2) + {GET,PUT} /api/udl/record + PUT /api/udl/records + GET /api/udl/user/{uid}/{key/{key},cid/{cid},all,count} + GET /api/udl/cid/{cid}/{users,aggregate} + GET /api/udl/users/stats (every signer with live claims, plugin keys included) + +Account-key identity service (kept in v2.0.0; removal is a separate ticket) + {GET,DELETE} /api/identity (DELETE signs + stores a retraction of every live claim first) + GET /api/identity/{accounts,aead-key} + POST /api/identity/{challenge,generate,import,reveal,sign,sign-batch} + (sign/sign-batch `format`: compact | der; default + compact for "MMUR2\n" payloads, DER otherwise) + +Removed in v2.0.0 — answer 410 Gone, pointing to POST /api/claims + PUT/PATCH/DELETE /meta/{hash} + PUT/DELETE /meta/{hash}/{key...} + POST /meta/{hash}/_add/{key...} + PUT/DELETE /api/metadata/{hashId} + PUT /api/metadata/{hashId}/property + POST /api/metadata/batch + +Removed in v2.0.0 — no snapshot format + GET /api/snapshot/export, POST /api/snapshot/{import,wipe} SSE event streams (HTTP-mirror of the store's event streams) GET /api/events/files @@ -366,6 +506,13 @@ peers and unauthenticated tooling can reach them): - (`/api/events/files`, `/api/events/meta` are also auth-bypassed but **should not** be exposed publicly via Caddy; they're inside-only.) +Since v2.0.0 `/meta/*` serves reads only — its write verbs answer `410 +Gone` — so the bypass no longer exposes an unsigned write. Writes go to +`POST /api/claims`, where the claim signature (not the perimeter) is the +write authentication. `POST /api/claims`, `POST /api/merge` and +`GET /api/meta/{cid}/claims` are container-to-container routes, not routed +through Caddy, like the v1.x write routes were. + See `docs/api-mediated-access.md` "Auth" for the rationale. ### SSE event streams — wire contract @@ -389,9 +536,46 @@ the kv table (`internal/kv/stream.go`). The contract `event: gap` payload before resuming from the oldest available entry. External services use SSE with `Last-Event-ID` as the resume primitive. -`meta:events` entries are appended by the storage engine in the same -transaction as the change they describe; the Redis-era keyspace-notification -hop through `MetaPublisher` is gone. +Since v2.0.0 `meta:events` describes changes of the **merged** view, same +`{type: set|del, key, ts}` shape: the cache engine appends an +`ev:meta:events/…` row for every change of a merged `file:` row, in the same +cache transaction as the change (both streams are v1 stream rows of the +cache `kv` table). A cache rebuild +emits no per-key events — consumers get `event: gap` and take a snapshot; +event IDs stay time-based and keep rising. *History:* v1.x appended them in +the same transaction as the `file:` key change; the Redis-era +keyspace-notification hop through `MetaPublisher` is long gone. + +### Write path and apply loop + +``` +POST /api/claims ──► per claim, before the writer: + 0. validate (required fields, reserved keys, ts range) → else bad_request + 1. same ts + sig as stored? → unchanged (no verify) + 2. same value as stored (any ts)? → unchanged; stored row + older ts kept + 3. stored ts ≥ incoming ts? → stale + 4. verify MMUR2 signature → else bad_signature + ──► accepted claims, one group-committed truth transaction: + 5. re-check 1–3 under the writer + 6. uid → pid (principal; insert on first sight) + 7. upsert claim (tombstone if null), seq = ++last_seq + 8. cid_group: mint rid for unknown cid; text cids/* links groups + (a retraction never splits) + 9. commit (last_seq in truth meta); answer once the cache applied it + ──► apply loop, in seq order, into the cache: + sync cid:/cids//idx:roots rows → claim_by_pid + → re-merge (rid, key) from truth PK range reads + → set/delete file:/ (+ local_file for filePath/sizeByte/mtimeNano) + ⇒ meta:events row + counter delta in the same tx + → FTS row on a haystack change → meta:applied_seq +``` + +The verifier lives in the write path (secp256k1 from `internal/identity`); +meta-core holds **no** writer key and never signs a claim. Response: one +status per claim — `accepted` · `unchanged` · `stale` · `bad_signature` · +`bad_request`. Request/response shapes and the 410 contract for removed +routes: [api-mediated-access.md](api-mediated-access.md) "Write contract — +signed claims". ### WebDAV URLs @@ -421,7 +605,8 @@ WebDAV caching is handled by nginx's `proxy_cache` (see | Single binary | Static compilation | | Low resource | ~10-20 MB resident | | HTTP server | stdlib `net/http` + gorilla/mux | -| Metadata store | embedded SQLite via `modernc.org/sqlite` (pure Go, CGO off) | +| Metadata store | embedded SQLite via `modernc.org/sqlite` (pure Go, CGO off) — truth + cache databases | +| Claim verification | secp256k1 ECDSA (`internal/identity`) | | WebDAV | `golang.org/x/net/webdav` | ### Repo layout @@ -434,12 +619,12 @@ packages/meta-core/ │ ├── cid/ CID parsing + algorithm ranking │ ├── config/ env-driven configuration │ ├── discovery/ service registry + dead-service cleaner -│ ├── kv/ SQLite kv engine + event streams +│ ├── identity/ secp256k1 (claim verification; account-key service) +│ ├── kv/ SQLite engine (single writer, group commit) + event streams │ ├── leader/ LeaderInfoProvider (no election logic) │ ├── mounts/ rclone mount manager + handlers + stats │ ├── schema/ per-field schema from transactional counters -│ ├── snapshot/ snapshot export / import / wipe -│ ├── storage/ flat-key records + search + UUID + CID reverse index + schema gate +│ ├── storage/ claims write path, apply loop, merge, cid_group, search, schema gate │ ├── watcher/ scanner + dispatcher + midhash + state │ ├── watchers/ polling-based watcher configs │ └── webdav/ /webdav/* handler @@ -460,15 +645,22 @@ packages/meta-core/ ``` 1. Load configuration from environment 2. Build LeaderInfoProvider (no election; bash gate already won) -3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db) -4. EnsureSchemaVersion (abort if the store layout is stale) -5. Start service discovery + dead-service cleaner -6. NewServer + Server.Start (boots watcher, mounts, WebDAV, API) -7. EnsureCounters (one-time build of the derived counters - on a store that predates them) -8. Rebuild FTS rows in the background if the FTS table is empty +3. Create the cache folder; open the truth database (META_CORE_DB_PATH or + /meta-core/db/meta-core.db) and the cache (META_CORE_CACHE_PATH or + /meta-core/db-cache/cache.db) (storage.Connect) +4. EnsureSchemaVersion (abort on a v1 kv store or an unknown layout) +5. storage.Start — cache catch-up (meta:applied_seq ≥ last_seq → nothing; behind → + replay claims with seq > applied_seq, or a full + rebuild if a cids/* link is among them; no + cache → full rebuild), then the apply loop +6. Derived counters ready +7. Start service discovery + dead-service cleaner +8. NewServer + Server.Start (boots watcher, mounts, WebDAV, API) 9. Wait for SIGINT/SIGTERM 10. Shutdown in reverse order + +(v1.x ran `EnsureCounters` and a background FTS rebuild here; in v2.0.0 +counters and FTS are cache rows maintained with every merged-row change.) ``` ### Process management @@ -489,10 +681,25 @@ internal crashes don't trigger re-election. ### Store durability -The SQLite file runs in WAL mode with `synchronous=NORMAL`: every -committed transaction survives a meta-core process crash. A record write — fields, root index, CID aliases, -FTS row and its events — is one transaction, so a crash never leaves a -half-written record. +Both SQLite files run in WAL mode; truth with `synchronous=NORMAL` (every +committed transaction survives a meta-core process crash), the cache with +`synchronous=OFF` (it is rebuildable). A claim batch +commits to truth in one transaction; the apply loop then commits the +derived cache changes. **A crash between the two commits leaves the cache +behind, never wrong**: at boot, if the cache's `applied_seq` is below +truth's `last_seq`, meta-core scans truth's `claim` table for +`seq > applied_seq` (a full scan — there is no seq index; it only happens +after an unclean shutdown) and applies them — or rebuilds the cache in full +if one of them is a `cids/*` link. + +### Cache loss + +The cache is never backed up and is fully derived (except `local_file`, +which a rescan of `/files` refills, and the service scratch keys, whose +loss costs a refetch). A missing cache folder (no `meta:applied_seq`) means +meta-core rebuilds every derived row from truth and stamps +`meta:rebuilt_at`. A rebuild emits no per-key events; +SSE consumers get `event: gap` and re-snapshot. ### Container crash @@ -503,8 +710,9 @@ the container." ### Schema mismatch `EnsureSchemaVersion` refuses to boot. Operators wipe with -`pnpm run clean:all` (or equivalent) and restart. This is an alpha-stage -hard-fail; there is no automated migration path. +`pnpm run clean:all` (or equivalent) and restart, or — once, on +watch.nsl.sh — convert a v1.x store with `tools/claims-migrate`. This is an +alpha-stage hard-fail; the service never migrates automatically. ### Shared volume unavailable @@ -525,7 +733,8 @@ via `/health`; metadata I/O and SSE event streams stop. | `SERVICE_VERSION` | `1.0.0` | Reported in `/status`. | | `API_PORT` | `8180` | External port baked into `baseUrl`. | | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). | -| `META_CORE_DB_PATH` | `/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` (named volume — SQLite WAL is unsafe on the 9p `/meta-core` bind of Windows dev boxes). | +| `META_CORE_DB_PATH` | `/db/meta-core.db` | Truth database file (backed up). Dev stacks set `/meta-core-db/meta-core.db` (named volume — SQLite WAL is unsafe on the 9p `/meta-core` bind of Windows dev boxes). | +| `META_CORE_CACHE_PATH` | `/meta-core/db-cache/cache.db` | Cache database file (never backed up; rebuilt from truth when missing). Dev stacks keep it on a named volume too; store apps put the cache folder in Maison's `backup.exclude`. | | `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection. | | `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. | | `META_CORE_DB_READERS` | `8` | Reader connection pool size. | @@ -547,7 +756,8 @@ via `/health`; metadata I/O and SSE event streams stop. |---|---| | `locks/kv-leader.lock` | Flock target. | | `locks/kv-leader.info` | Leader API URL (plaintext). | -| `db/meta-core.db` (+ `-wal`, `-shm`) | Metadata store (SQLite). Replaced `db/redis/` in 2026-10. | +| `db/meta-core.db` (+ `-wal`, `-shm`) | Truth database (claims, principals, `cid_group`). Backed up. Replaced `db/redis/` in 2026-10; holds the claims schema since v2.0.0. | +| `db-cache/cache.db` (+ `-wal`, `-shm`) | Cache database (v1-layout `kv` table: merged view, lookup, counters, events, service scratch keys; field index, FTS, `claim_by_pid`, `local_file`). Never backed up. | | `services/*.json` | Service registry. | | `mounts/mounts.json` | rclone mount configuration. | | `mounts/errors/{id}.error` | Mount error logs. | @@ -561,7 +771,9 @@ via `/health`; metadata I/O and SSE event streams stop. meta-core consolidates leader gating, metadata storage, event mirroring, and file access for MetaMesh into one container with a typed HTTP+SSE surface. The bash flock gate guarantees single-leader semantics without -an external consensus system. UUID-rooted storage with CID aliasing -treats all content hashes symmetrically and makes any-CID resolution -a single lookup. API-mediated access ensures no other service ever touches +an external consensus system. Metadata is a table of signed votes in a +backed-up truth database, merged by an apply loop into a rebuildable cache +that serves every read; meta-core verifies every claim and signs none. +CID groups behind a hidden local rid treat all content hashes +symmetrically. API-mediated access ensures no other service ever touches the store. diff --git a/docs/metadata-storage-structure.md b/docs/metadata-storage-structure.md index 96a1e21..200422e 100644 --- a/docs/metadata-storage-structure.md +++ b/docs/metadata-storage-structure.md @@ -1,256 +1,199 @@ # Metadata Storage Structure -How meta-core lays out file metadata in its store. This document covers the -on-disk shape (UUID-rooted flat string keys, CID reverse index, -per-root sets) and the schema-version gate. It does **not** enumerate -field semantics — that's the job of the repo-root -[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key -registry for everyone who writes to or reads from metadata. +How meta-core lays out metadata on disk since **v2.0.0 (claims model)**: the +two SQLite databases, their tables, how a record is assembled from signed +votes, and the schema-version gate. Implementation basis: +[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md). +This document does **not** enumerate field semantics — that is the job of the +repo-root [`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative +key registry for everyone who writes or reads metadata. -For the design rationale (why UUIDs instead of content hashes, why a -reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md). -For the external-access contract (HTTP-only, SSE for events), see +Related: why rids are UUIDs and hidden — +[`uuid-rooted-metadata.md`](uuid-rooted-metadata.md); why the engine is +shaped this way, its guarantees and measurements — +[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md); +the external-access contract (HTTP-only, SSE for events) — [`api-mediated-access.md`](api-mediated-access.md). -> **Storage engine (since 2026-10): an embedded SQLite file, not Redis.** -> Until 2026-10 these keys lived in a Redis server spawned inside the -> container. They now live in one string→string table of an embedded SQLite -> database (pure-Go driver `modernc.org/sqlite`, CGO off). The **key names -> did not change** — only the Redis *types* that some bookkeeping used (SETs, -> HASHes, Streams) were flattened into plain keys, listed below. Holding -> everything in RAM had cost ~6.6 GB on watch.nsl.sh's 169k records (Redis -> plus the API's in-memory search copy) and caused repeated OOM kills. +> **History.** Until 2026-10 metadata lived in Redis; v1.1.0–v1.2.x kept the +> same flat keys (`file:/`, `cid:`, `idx:*`, `dup:*`, +> `udl:*`, `ev:*`) in one SQLite string→string `kv` table. v2.0.0 replaced +> that keyspace entirely; the old layout is summarised at the end +> ([Previous layout](#previous-layout-v11v12-kv-table)). ## Quick summary -- **One table**: `kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`, - schema-free — a new metadata field is just a new key, never a migration. - File: `/meta-core/db/meta-core.db` (override `META_CORE_DB_PATH`), WAL - mode, `synchronous=NORMAL`. -- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout), - not content hashes. Examples in this doc use the placeholder - `01JKR8XW5T4QMV7AHNJ2DEFGHK`. -- **Properties are flat string keys**, one per leaf: - `file:/` → string value. A record is exactly the - key range `[file:/, file:0)` — there is no per-record field - list. -- **CIDs are reverse-index aliases**: `cid:` → `` - (one key per known CID; no algorithm prefix — the multicodec - already carries it). -- **Per-root sets**: `file:/cids/` (a *key-set*: one key - `"true"` per known CID, midhash included), `dup:/` (alternate - paths with the same content), plus `idx:roots/` (every UUID). - There is **no stored `canonical_cid`** — it is derived by rank on read. -- **A schema-version sentinel** (`meta-core:schema-version`) refuses to - boot against legacy data. +- **Every metadata value is a signed claim**: `claim(uid, cid, key) → value` + + `ts` + `sig` (MMUR2, secp256k1). One row per signer per field. Writers + sign; meta-core never signs. +- **Two databases.** *Truth* `/meta-core/db/meta-core.db` + (`META_CORE_DB_PATH`, backed up, primary keys only): `principal`, `claim`, + `cid_group`, `meta`. *Cache* `/meta-core/db-cache/cache.db` + (`META_CORE_CACHE_PATH`, never backed up, rebuildable): the v1 `kv` table + (merged view as `file:/` rows, lookup, counters, event streams, + bookkeeping, service scratch keys) with its `kv_field_v` index and `fts`, + plus two real tables, `claim_by_pid` and `local_file`. +- **Schema-free.** A field is a `key` string in a row; a new field never + needs DDL. +- **Records are cid groups.** A vote names a cid, never a rid. `cid_group` + maps each cid to a **rid** — a hidden, locally minted UUIDv7 (26-char + Crockford) naming the group. Examples use `01JKR8XW5T4QMV7AHNJ2DEFGHK`. +- **What readers get is the merged view**: one precomputed value per + `(rid, key)`, stored as the cache row `file:/`. `nsrc` (how many + signers support it) is computed by the merge and returned by + `POST /api/merge`, not stored. `canonical_cid` is still **derived on read** + by rank, never stored. +- **Watcher facts are not claims**: `filePath`, size, `mtimeNano`, midhash + live in `local_file` and are joined into the record view unsigned. +- **A schema-version sentinel** (`schema-version = "3"` in truth `meta`). + meta-core explicitly refuses a truth file holding the v1 `kv` table, and any + store with claims but no sentinel or another version. -Implementation lives in `internal/kv/kv.go` (engine), -`internal/kv/stream.go` (event streams), `internal/storage/client.go`, -`internal/storage/search.go`, `internal/storage/cid_resolution.go`, -`internal/storage/uuid.go`, and `internal/storage/schema_version.go`. +## Truth database — `/meta-core/db/meta-core.db` -## Key structure +Backed up; only primary keys (plus `principal.uid`'s implicit UNIQUE index). -Everything is one string→string table. The full layout: - -| Key | Value | Notes | +| Table | Columns (PK first) | Notes | |---|---|---| -| `file:/` | field value | a record property — the **only** thing under `file:` | -| `cid:` | `` | reverse index | -| `meta-core:schema-version` | `"2"` | schema gate | -| `idx:roots/` | `""` | set of every root (was the Redis SET `file:__index__`) | -| `idx:fts/` | FTS rowid | the record's row in the FTS table | -| `idx:count/roots` | integer | number of roots — search's `total`, `/api/stats` records (v1.2.0) | -| `idx:schema/\x1f\x1f\x1f` | integer | per-field value-type counts behind `/api/schema` (v1.2.0) | -| `idx:count/ready` | `"1"` | marks the counters as built (absent → one-time build at boot) | -| `dup:/` | `""` | a duplicate path (was the SET `file:/duplicates`) | -| `udl:rec://` | JSON `{record,version,ts,value?}` | User Data Layer entry (was a HASH) | -| `udl:idx:…\x1f` | `""` | UDL index sets, one key per member | -| `ev:/-` | JSON of the entry's fields | `meta:events` / `file:events` entries (were Redis Streams) | -| `evhwm:` | last issued ID | stream high-water mark | -| `btmagnet:*`, `searchcov:*`, `nzb:*` | — | written by other services via `/api/kv/value`; unchanged | +| `principal` | `pid` INTEGER PK, `uid` TEXT UNIQUE | `uid` = `"z" + base58btc(33-byte compressed pubkey)`. `pid` is a local surrogate: never signed, never on the wire | +| `claim` | `(cid, key, pid)` PK, `value` ANY, `ts`, `sig`, `seq` | `STRICT, WITHOUT ROWID`. `value`: TEXT = plaintext, BLOB = ciphertext envelope, NULL = retraction (tombstone, kept). `ts` (ms) is the version. `seq` = global change counter, not indexed | +| `cid_group` | `cid` PK, `rid` | the cid → rid map: the record identity layer | +| `meta` | `name` PK, `value` | `schema-version = "3"`, `last_seq` | -Keeping bookkeeping **out of `file:`** is load-bearing: the `file:` range is -exactly the records, so the field index covers only real fields and every -change under `file:` is a real field change worth publishing on -`meta:events`. The per-record `file:/__fields__` SET and the -`file:__dirty__` set of the Redis era no longer exist. Roots are split from -the field at the **first** `/`, so CID-rooted and UUID-rooted records both -parse. +`claim` is written **only** by `POST /api/claims`. A claim replaces the +stored row for its `(cid, key, pid)` only if its `ts` is strictly greater; +identical `(ts, sig)` is `unchanged`; the **same value** from the same signer +(any `ts`) is also `unchanged` and keeps the stored row with its older `ts`; +otherwise older or equal is `stale`, and a bad signature is refused. Why `STRICT` with an `ANY` column: a non-STRICT `ANY` +column gets NUMERIC affinity and turns the text `"007"` into the integer 7; +STRICT keeps the written type, which is the cipher flag. -### Roots +## Cache database — `/meta-core/db-cache/cache.db` + +Never backed up (the store app lists the folder in `backup.exclude`). +Deleting it is the recovery procedure: meta-core rebuilds it from truth at +boot (no `meta:applied_seq` → full rebuild), and `local_file` refills from a +rescan. + +It keeps the **v1 key/value layout** (`internal/storage/client.go`): one +`kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID` table, whose rows are +all derived — no client writes `file:` rows any more. + +| `kv` rows | Holds | Written by | Read by | +|---|---|---|---| +| `file:/` | the **merged view**, one row per `(rid, key)` — the only rows under `file:`. Includes the synthesised `cids/ = "true"` members (from `cid_group`) and the box-local `filePath` / `sizeByte` / `mtimeNano` (from `local_file`) | apply loop | every record read; search | +| `cid:` | cid → rid (copy of truth `cid_group`) | apply loop, watcher | every cid-addressed request | +| `idx:roots/` | the set of every record | apply loop | listing, counts | +| `idx:fts/` | rowid of the record's `fts` row | apply loop | FTS maintenance | +| `idx:count/*`, `idx:schema/*` | record count, per-field schema counters | the cache engine's change hook, from `file:` deltas | search `total`, `/api/stats`, `/api/schema` | +| `ev:/-`, `evhwm:` | `meta:events` (every `file:` change, same transaction) and `file:events` (watcher) | cache engine; watcher | `/api/events/meta`, `/api/events/files` (SSE) | +| `meta:applied_seq`, `meta:rebuilt_at` | how far the cache has applied truth; last rebuild (older SSE cursors get `gap`) | apply loop, rebuild | boot catch-up; SSE | +| anything else | service scratch KV (`btmagnet:*`, `searchcov:*`, `nzb:*`) | other services via `/api/kv/value` (reserved prefixes refused) | same | + +| Index / table | Shape | Written by | Read by | +|---|---|---|---| +| `kv_field_v` (index) | `kv(lower(field parsed from k), lower(v), v)` over `file:` rows only | — | filter search | +| `fts` | FTS5 trigram `(rid UNINDEXED, hay)`, one row per record | apply loop, when a haystack field's merged value changes | free-text search | +| `claim_by_pid` | `(pid, cid, key)` PK | apply loop | `?as=` overlay, `/api/udl/*` facade, identity retraction, `/api/udl/users/stats` | +| `local_file` | `path` PK → `midhash`, `size`, `mtime`; index `(midhash, path)` | file watcher, `POST /api/files/register` | record view (`filePath`, `sizeByte`, `mtimeNano`) | + +The cache is **behind, never wrong**: after an unclean shutdown, boot finds +`meta:applied_seq < last_seq` and re-applies every truth claim with `seq > +applied_seq` (or rebuilds in full when one of them is a `cids/*` link). Write +path, apply loop and durability are detailed in +[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §4. + +## Keys + +The `key` column carries the same field paths METADATA_KEYS.md defines — +what used to follow `file:/`: ``` -file:/ → string (always) +claim(cid=bagacbabaec…, key=title, uid=) = "Inception" +claim(cid=bagacbabaec…, key=fileinfo/duration, uid=) = "8878.5" +claim(cid=bagacbabaec…, key=fileinfo/streamdetails/video/0/codec, uid=) = "h264" +claim(cid=bagacbabaec…, key=genres/Drama, uid=) = "true" +claim(cid=bagacbabaec…, key=cids/bafybeigd…, uid=) = "true" ``` -UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`): -UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The -time-sortable prefix means roots minted in a given window sort together -(`k >= 'file:01JKR8' AND k < 'file:01JKR9'`), which restored some of the -debug-ability lost when midhash256 stopped being the root. +- `/` is the path separator; nested objects are flattened on write and + reconstructed on read. +- **Key-sets** put the member inside the key (`genres/`, `cids/`, + `titles/eng/`, `anchoredBy/tmdb:123`): one claim per member, union by + the ordinary merge, removal by a signed retraction. +- Plugins keep writing under their prefixes (`{plugin-id}/`, + `{plugin-id}///`), each signing with its own key. Two + sources compete on one key only when they make the same kind of statement; + a different fact (e.g. a filename guess vs a TMDB lookup) gets a different + key. -Properties use `/` as the path separator. Nested objects in source code -are flattened to flat keys on write and reconstructed on read. Examples: +## Records: `cid_group` and the hidden rid ``` -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..." +votes (truth, signed) cid_group (truth) cache kv rows (merged) +(A, M, title) = "The Matrix" M → R1 file:R1/title = "The Matrix" +(A, M, cids/Q) = "true" Q → R1 file:R1/cids/M = "true" (synthesised) +(B, Q, title) = "Matrix, The" file:R1/cids/Q = "true" (synthesised) + file:R1/sizeByte = "734003200" +(B, Q, sizeByte) = "734003200" ``` -### Reverse-index aliases (CIDs) +- A text `cids/` vote on cid X links X and `` into one + group. Retracting it does **not** split the group (phase 1). In the merged + view, membership is synthesised from `cid_group` — `cids/ = "true"` for + every cid of the group — and `cids/*` claims are never merged. Record R = every vote on every cid of R's group, merged per key, + each signer counted once per key. +- First-seen cid with no link → new group with a fresh rid + (`storage.NewUUID()`, UUIDv7 → 26-char Crockford Base32, ULID layout). +- Linking two groups → the **older rid survives**; the other group's cids + move to it; `/api/events/meta` emits `del` for each key of the losing rid + and `set` for the survivor. There is no rid alias table. +- `GET /api/meta/` is a `cid:` lookup in the cache, then a range + read `[file:/, file:0)` — any known cid, any hash algorithm, no + primary. +- Rids are local names: never signed, never on the swarm or gateway wire. + Local consumers (meta-sort, meta-fuse, meta-stremio) do hold them, which is + why `cid_group` lives in truth. -Every CID known for a file is registered as its own key, keyed by -the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash -function in the multicodec, so naming it again in the key would be -redundant (and would let the same digest be indexed under two spellings): +There is **no stored `canonical_cid`**. `GetMetadataDocument` ranks the +group's cids through `internal/cid/rank.go` (`cid.Better`) and returns the +winner as `canonical_cid` in the JSON response only. -``` -cid: → -``` +## Merge -Examples: - -``` -cid:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK -cid:bagacbabaec7v3... → 01JKR8XW5T4QMV7AHNJ2DEFGHK -``` - -The reverse index makes `GET /api/meta/` a primary-key lookup for any -registered CID, independent of which hash algorithm — there is no primary. -The record write path (`writeFieldsTx` in `internal/storage/client.go`, used -by `SetProperty`, `SetMetadataFlat` and `MergeMetadataFlat`) registers the -alias for any field whose name starts with `cids/`, in the same transaction, -so writers emit key-set members without knowing the reverse index exists. - -> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or -> `midhash256` *named* field written to a root is stored verbatim and is -> **never indexed** — the record becomes unresolvable by that CID. Writers -> must collapse such fields to `cids/` members before the write. See -> `METADATA_KEYS.md` §14.13. - -### Per-root sets - -``` -file:/cids/ → "true" — one key per known CID (a key-set) -dup:/ → "" — one key per alternate path with same content -``` - -`cids` is a **key-set**: one flat `file:/cids/` key per member, -which keeps it conflict-free to merge across peers (a single set-valued -field would not be). -The record's own address (the midhash) is just the `0x1000` member — it has -no separate field. - -There is **no stored `canonical_cid`**. The canonical CID is *derived on -read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the -key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns -the winner as the `canonical_cid` field of the JSON response only. Nothing -reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and -was removed with the flat `cid_*` keys. - -### Global index - -``` -idx:roots/ → "" (one key per root; was the Redis SET file:__index__) -``` - -`internal/storage/client.go::GetAllHashIDs` range-reads `idx:roots/` to -enumerate every file without scanning the records. - -### Search structures (derived) - -Two derived structures are maintained on top of `kv`, in the same -transaction as every record write. Both are rebuildable from the kv rows and -never authoritative (`internal/storage/search.go`): - -- **`kv_field_v`** — a partial expression index - `ON kv(lower(substr(k,instr(k,'/')+1)), lower(v), v) WHERE k>='file:' AND k<'file;' AND instr(k,'/')>0`. - It answers "which records have field F with a value like V" for **any** - field, including ones invented tomorrow — the index is over the key itself. - It carries `v` so the substring test is answered from the index alone - (covering; 13 ms vs 1.48 s without it, for tmdbid on watch). -- **`fts`** — an FTS5 trigram table `fts(rid UNINDEXED, hay)` holding one row - per record: its search haystack plus its rid (`idx:fts/` points at the - row). Trigrams make every 3+ character substring indexable. - -`POST /api/metadata/search` uses SQL to build a superset of candidates, then -verifies each with the unchanged Go matchers (`recordMatchesFilter`, -`matchAllTokens`), so semantics are identical to the old in-memory index. -The smallest part (by a capped count) is collected once and sorted; every -other filter is probed per page as an EXISTS over the candidate's own key -range. The plan, its measurements and the rejected shapes are in -[`docs/project-architecture/metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md). -Results still come in hashID order and are fresh immediately (no refresh -lag). This replaced the API's in-memory `searchIndex` (60 s refresh, ~30 KB -RAM per record). - -### Schema-version sentinel - -``` -meta-core:schema-version → string (currently "2") -``` - -Set at first boot, checked on every subsequent boot -(`internal/storage/schema_version.go`). If the sentinel is missing but -the root index is non-empty, the gate refuses to boot (legacy data). -If the sentinel is present but doesn't match `SchemaVersion`, the gate -also refuses to boot. Alpha-stage policy: operators wipe and restart; -there is no automated migration. - -## Why flat string keys? - -The store keeps one flat key per leaf rather than a document or a -per-record row. The reasons are: - -1. **Schema-free.** A plugin inventing a field writes a new key; nothing - declares it, and the table never needs a migration. The field index - covers it automatically. -2. **Conflict-free merging across peers.** Key-sets (`cids/`, and the - flat fields in general) merge by union of keys — no read-modify-write of - a shared value, nothing to reconcile. -3. **Selective updates without read-modify-write.** A plugin writing - `tmdb/poster` doesn't have to read the rest of the document. -4. **Compatibility with the CID alias hooks.** Field writes can trigger - alias-maintenance work in the write path without the plugin caring. - -(Under Redis there was a fourth reason — field-level keyspace notifications -fed `meta:events`. That is obsolete: events are now written by the storage -engine itself, see [Event streams](#event-streams).) +The cache row `file:/` holds the winner over the group's votes +(each signer's newest vote counts once; retractions and ciphertext drop out): +most distinct signers (values compared after **trim + case-fold only**, no +NFC) → oldest `ts` → byte-wise smallest. The stored value is the winner's +exact text; `nsrc` is not stored (`POST /api/merge` returns it). With +`?as=`, the viewer's own live text votes (found via `claim_by_pid`) +override at read time; they are never written to the cache. Box-local +fields: when several local paths hold the same content, `filePath` is the +lexicographically first (size and mtime from the same `local_file` row); a +signed `sizeByte` claim wins over the on-disk size. Full rules: +implementation basis §7 (`internal/storage/merge.go`). ## Reads and writes -All writes go through **one writer goroutine** with group commit: concurrent -writes share one SQLite transaction, each under its own SAVEPOINT so a -failure rolls back alone. A record write — fields + root index + CID aliases -+ FTS row + events — is atomic. (Measured ~835 records/s with 16 concurrent -writers of 40-field records.) Reads use a separate connection pool and see -the last committed state. +| Operation | Endpoint | Store access | +|---|---|---| +| write metadata | `POST /api/claims` (batch of signed claims) | truth upsert, then the apply loop updates the cache | +| removed writes | `PUT/PATCH/DELETE /meta/*`, `POST /meta/{h}/_add/{k}`, `PUT/DELETE /api/metadata/{id}`, `PUT /api/metadata/{id}/property`, `POST /api/metadata/batch` | `410 Gone`, pointing at `POST /api/claims` | +| read a record | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{h}` | `cid:` lookup → `file:/*` rows, optional `?as=` | +| list / search | `GET /api/metadata/hash-ids`, `/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `/api/schema`, `/api/stats` | `file:` rows, `kv_field_v`, `fts`, `idx:count/*` / `idx:schema/*` | +| raw votes | `GET /api/meta/{cid}/claims`, `GET /api/metadata/{rid}/claims` (`?key=`, `?uid=`) | `{rid, cids, claims}`: every live signed claim of the group, all signers, tombstones excluded; 404 for an unknown cid / rid | +| merge without storing | `POST /api/merge` `{claims, as?}` → `{fields, nsrc, rejected}` | none — verify + merge in memory (plus `as`'s votes this core holds) | +| watcher facts | the watcher, `POST /api/files/register` | `local_file` + `cid_group` | +| service scratch | `PUT/GET/DELETE /api/kv/value` | non-reserved cache `kv` rows | +| wipe | `POST /api/metadata/clear` | both databases | -The Go API in `internal/storage/client.go`: - -| Method | Behaviour | -|---|---| -| `GetMetadataFlat(uuid)` | Range-read `[file:/, file:0)`, return `map[string]string`. | -| `SetMetadataFlat(uuid, m)` | Put each field; add `idx:roots/`; auto-register an alias for every `cids/` member; refresh the FTS row. One transaction. | -| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. | -| `GetProperty(uuid, field)` | Get `file:/`. | -| `SetProperty(uuid, field, value)` | Put `file:/`; auto-register alias if the field is a CID. | -| `DeleteRoot(uuid)` | Delete the `file:/` range, its duplicates, search row and index membership; unmap every reverse-index entry. | -| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. | -| `AddAlias(uuid, "")` | Set `cid:` → `` and the key-set member `file:/cids/` = `"true"`. Nothing to reconcile — canonical is derived on read. | -| `ResolveRoot("")` | Get `cid:` → uuid (returns the input unchanged if it is already a UUID root). | -| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. | +Truth writes go through **one writer goroutine** with group commit; reads use +a separate pool and see committed state (WAL). Read paths and shapes are +unchanged from v1.x — they are now served from the derived `file:` rows. ## Reconstruction (flat → nested) -When the HTTP API returns a document, it reconstructs the flattened -storage back into a nested JSON object: +The HTTP API reconstructs the merged flat keys into a nested JSON object: ``` flat nested @@ -262,134 +205,122 @@ fileinfo/streamdetails/ streamdetails: { video: streams/0/language = "eng" streams: [ { language: "eng" }, ... ] ``` -Rules: +- Numeric path segments become array indices on read. +- All values are strings on the wire; type interpretation is the caller's + job (documented per field in `METADATA_KEYS.md`). +- Box-local fields from `local_file` are joined in unsigned; they never + appear in `GET /api/meta/{cid}/claims` or on the swarm. -- `/` is the path separator. -- Numeric path segments (`0`, `1`, ...) become array indices on read. -- All values are strings on the wire; type interpretation is the - caller's job (and is documented per-field in `METADATA_KEYS.md`). -- `null`/`undefined` round-trip as empty strings. +## Search structures -## Plugin contribution flow +Both live in the cache and are maintained in the same transaction as the +merged-row change, so results are fresh as soon as a change is applied: -Plugins don't talk to the store directly. They write metadata via meta-core's HTTP -API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin -writes under a prefix: +- **`kv_field_v`** — an expression index over the field name parsed out of + the key, `lower(v)` and `v`, restricted to `file:` rows. Answers + "which records have field F with a value like V" for any field, including + ones invented tomorrow; covering for the substring test. +- **`fts`** — FTS5 trigram `(rid UNINDEXED, hay)`, one row per record: the + haystack of its searchable merged fields plus the rid. Every 3+ character substring is + indexable. -``` -{plugin-id}/ single value -{plugin-id}// grouped -{plugin-id}/// indexed (array) -``` +`POST /api/metadata/search` builds a candidate superset in SQL and verifies +each hydrated merged record with the Go matchers (`recordMatchesFilter`, +`matchAllTokens`); results come in rid order. Plan, rejected shapes and the +measurement status are in +[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §8. -See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full, -authoritative list of fields each plugin writes (content identification, -file stats, video classification, anime detection, subtitle references, -Jellyfin metadata, FFmpeg stream details, etc.) and the value formats -(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc -deliberately does not duplicate that registry — it would drift. +## Schema-version sentinel + +Truth `meta` row `schema-version` = `"3"`, set at first boot and checked on +every boot (`internal/storage/schema_version.go`), after both files are +opened and before the cache catch-up. A truth file holding the v1 `kv` table +(v1.x, `meta-core:schema-version = "2"`) is **explicitly refused**, naming the +migration tool; claims without a sentinel, or another version, are refused +too; a fresh store is stamped. Alpha policy: no migration code in the +service binary; operators wipe and restart. watch.nsl.sh is converted once by +the separate one-off tool `packages/meta-core/tools/claims-migrate` (not in +the image; rids keep their UUIDs; legacy fields re-signed by a throwaway key; +UDL cells re-signed by their account keys — implementation basis §11). ## Event streams -Two event streams are kept inside the store, as rows -`ev:/-` (zero-padded so key order is ID order) plus a -high-water mark `evhwm:` (`internal/kv/stream.go`): - | Stream | Producer | Contents | |---|---|---| -| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. | -| `meta:events` | the storage engine (`internal/kv`) | `{type: set\|del, key, ts}` for every Put/Delete of a `file:` key, appended **in the same transaction** as the change. | +| `/api/events/meta` | the cache engine, as `ev:meta:events/…` rows of the cache `kv` table | `{type: set\|del, key, ts}` for every change of a **merged** value, in the same cache transaction as the change | +| `/api/events/files` | `internal/watcher` dispatcher (`ev:file:events/…` rows) | `add` / `change` / `delete` / `rename` / `reset` for files on `/files` (unchanged) | -`set` is emitted only for a **real** change: a Put of the value a key -already holds is a no-op — no row write, no index churn, no event (v1.2.0; -Redis published `set` even then, and with gateway hits re-persisted on -every repeated query most writes were exactly that). `del` is emitted only -when the key existed. Retention is ~100k entries, trimmed every 10k appends. - -There is **no boot-time republish** (v1.2.0). A consumer that wants the whole -current state asks `/api/events/meta` for `Last-Event-ID: 0` / `0-0` and gets -a **snapshot**: one `set` per existing record key, streamed from the kv -range, then the live tail from the stream head captured before the snapshot -(only the last snapshot event carries an `id:`, so a consumer that drops -mid-snapshot gets a fresh one). The old republish wrote every field into a -stream that retains only ~100k entries, so a consumer replaying it later -never saw most records — and it cost ~6 minutes on every boot. - -The Redis-era path — keyspace notifications drained by `MetaPublisher` -through a PSUBSCRIBE / consumer-group hop — is gone (2026-10). - -External consumers do **not** read these streams directly. They subscribe -via SSE: - -- `GET /api/events/files` — mirror of `file:events` -- `GET /api/events/meta` — mirror of `meta:events` - -The SSE wire contract uses `Last-Event-ID` for resume. IDs keep the Redis -`-` form so cursors persisted by meta-sort / meta-share / -meta-stremio across the switch still compare. Heartbeats are SSE comments -(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past -a client's cursor, the handler emits one `event: gap` payload before -resuming. See `internal/api/sse_events.go` and -[`api-mediated-access.md`](api-mediated-access.md) for the full -contract. +- A vote that does not change the merged value emits nothing. +- A group merge emits `del` for every key of the losing rid and `set` for the + survivor's keys. +- IDs keep the `-` form and stay time-based, so persisted cursors + keep comparing, including across a cache rebuild. +- `Last-Event-ID: 0` / `0-0` returns a snapshot (one `set` per merged key) + then the live tail; only the last snapshot event carries an `id:`. +- A trimmed cursor gets one `event: gap`. A cache rebuild emits no per-key + events; consumers get `gap` and take a snapshot. +- Heartbeats are SSE comments (`:keep-alive\n\n`) every 30 s of silence. + Contract: `internal/api/sse_events.go`, [`api-mediated-access.md`](api-mediated-access.md). ## Debugging -The database path is `/meta-core/db/meta-core.db` in the image default and -on the store app; dev stacks use `/meta-core-db/meta-core.db` (a named -volume — the `/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL -is unsafe). Always open it **read-only**; writes must go through the API. +Paths: truth `/meta-core/db/meta-core.db`, cache `/meta-core/db-cache/cache.db` +(image default and store app); dev stacks keep both on named volumes (the +`/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL is unsafe). +Always open **read-only**; writes go through `POST /api/claims`. ```bash -DB=/meta-core/db/meta-core.db # dev stack: /meta-core-db/meta-core.db -Q() { docker exec metacore-app sqlite3 -readonly "$DB" "$1"; } +T=/meta-core/db/meta-core.db; C=/meta-core/db-cache/cache.db # dev: the named-volume paths +QT() { docker exec metacore-app sqlite3 -readonly "$T" "$1"; } +QC() { docker exec metacore-app sqlite3 -readonly "$C" "$1"; } -# Roots -Q "SELECT count(*) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0'" -Q "SELECT substr(k,11) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0' LIMIT 10" - -# Full document for one root -Q "SELECT k,v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'" - -# One property -Q "SELECT v FROM kv WHERE k = 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'" - -# Reverse-index lookup (bare CID — no algorithm prefix) -Q "SELECT v FROM kv WHERE k = 'cid:bafkreih...'" # → uuid - -# The cids key-set -Q "SELECT k FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids0'" - -# canonical_cid is NOT stored. Derive it via the API instead: -curl -k https://metacore-dev.localhost:8083/api/meta/ | jq '.canonical_cid, .cids' - -# Schema-version sentinel -Q "SELECT v FROM kv WHERE k = 'meta-core:schema-version'" - -# Streams (server-side; external consumers should use SSE) -Q "SELECT k,v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10" +QT "SELECT rid FROM cid_group WHERE cid = 'bafkreih...'" # cid → rid +QT "SELECT cid FROM cid_group WHERE rid = '01JKR8XW5T4QMV7AHNJ2DEFGHK'" # the group (full scan: no rid index) +QC "SELECT k, v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'" # merged record +QT "SELECT c.key, typeof(c.value), c.value, p.uid, c.ts, c.seq + FROM claim c JOIN principal p USING(pid) WHERE c.cid = 'bafkreih...'" # every vote on one cid +QT "SELECT name, value FROM meta"; QC "SELECT k, v FROM kv WHERE k LIKE 'meta:%'" # last_seq vs applied_seq +QC "SELECT path, midhash FROM local_file WHERE path LIKE '/files/watch/%' LIMIT 10" +QC "SELECT k, v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10" # HTTP equivalents — preferred -curl -k 'https://metacore-dev.localhost:8083/api/meta/' # CID → document -curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs -curl -k 'https://metacore-dev.localhost:8083/meta/' # document by UUID -curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE -curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats +curl -k 'https://metacore-dev.localhost:8083/api/meta/' # merged document (+ canonical_cid) +curl -k 'https://metacore-dev.localhost:8083/api/meta//claims' # signed votes +curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE ``` ## Key implementation files | Component | Path | |---|---| -| Storage engine (SQLite kv table, writer, group commit) | `internal/kv/kv.go` | -| Event streams as kv rows | `internal/kv/stream.go` | -| Flat-key record ops | `internal/storage/client.go` | -| Search (field index + FTS) | `internal/storage/search.go` | +| Single-writer, group-commit SQLite engine | `internal/kv/` | +| Truth, apply loop, merge, local files, counters | `internal/storage/{truth,apply,merge,localfile,client}.go` | +| Search (`kv_field_v` + FTS) | `internal/storage/search.go` | +| MMUR2 claim encoding + verification | `internal/claim/` (secp256k1 in `internal/identity/`) | | UUIDv7 minting | `internal/storage/uuid.go` | -| CID reverse index + aliases + canonical-CID derivation | `internal/storage/cid_resolution.go` | +| Canonical-CID rank | `internal/cid/rank.go` | | Schema-version gate | `internal/storage/schema_version.go` | -| `meta:events` snapshot for `0`/`0-0` | `internal/api/sse_events.go` (`streamSnapshot`) | -| Derived counters (root count, per-field schema) | `internal/storage/client.go` (`onChange`), `internal/schema/indexer.go` | -| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` | -| SSE event mirror | `internal/api/sse_events.go` | +| SSE event mirror + snapshot | `internal/api/sse_events.go` | +| Watcher → `local_file`, `/api/events/files` | `internal/watcher/` | | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) | + +## Previous layout (v1.1–v1.2 `kv` table) + +Kept so old notes and scripts can be read. In v2.0.0 the **truth** database +has no `kv` table at all; the **cache** keeps the same key layout for derived +rows (see above), but nothing in it is written by clients. + +| Key | Value | Replaced by | +|---|---|---| +| `file:/` | one value per field, last writer wins | `claim` (one row per signer) → derived cache `file:` rows | +| `cid:` | `` | `cid_group` (truth), copied to cache `cid:` rows | +| `idx:roots/`, `idx:count/*`, `idx:schema/*`, `idx:fts/` | root set, counters, FTS rowid | same keys, derived, in the cache | +| `dup:/` | duplicate path | `local_file` rows | +| `udl:rec://`, `udl:idx:*` | User Data Layer cells (MMUR1, `version`) | `claim` (MMUR2, `ts`) + `claim_by_pid` | +| `ev:/-`, `evhwm:` | stream rows | same keys, in the cache | +| `btmagnet:*`, `searchcov:*`, `nzb:*` | service scratch | same keys, in the cache | +| `meta-core:schema-version` = `"2"` | gate | truth `meta` `schema-version` = `"3"` | + +The flat-key idea survived — a field is still one string key, a key-set +member is still inside the key — but the record moved from "one value per +field" to "one value per signer per field". diff --git a/docs/uuid-rooted-metadata.md b/docs/uuid-rooted-metadata.md index 529e5d4..3283014 100644 --- a/docs/uuid-rooted-metadata.md +++ b/docs/uuid-rooted-metadata.md @@ -2,368 +2,275 @@ ## Status -**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale. +**Status:** Design record. Implemented in 2026 alpha (clean-wipe gate); +**reframed by meta-core v2.0.0 (claims model)** — see +[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md) +§4. This doc keeps the *why* behind UUID rids and CID symmetry; it is not +the schema reference. For what is stored, see +[metadata-storage-structure.md](./metadata-storage-structure.md) and +`METADATA_KEYS.md` §2/§14.13. -> ⚠ **Two parts of this proposal were superseded before shipping.** This doc -> is the design record; it is *not* the schema reference. For what is actually -> stored, see `METADATA_KEYS.md` §2/§14.13 and -> [metadata-storage-structure.md](./metadata-storage-structure.md). -> -> 1. **`canonical_cid` is not a stored key.** The proposed reconciler -> (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't -> conflict-free to merge across peers. The canonical CID is **derived on -> read** by ranking the `cids` members (`internal/cid/rank.go` → -> `cid.Better`), and appears only in the `GET /api/meta/{cid}` response. -> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not -> `:` tokens.** One flat STRING per member: -> `file:/cids/` = `"true"`. The reverse index is likewise -> `cid:` → uuid. A CIDv1's multicodec already names its hash -> function, so the `:` prefix was redundant. -> -> 3. **The store is no longer Redis (2026-10).** The same keys now live in -> one string→string table of an embedded SQLite file; Redis SETs became -> one key per member (`file:__index__` → `idx:roots/`, -> `file:/duplicates` → `dup:/`). Redis types named -> below are the shape at the time. -> -> Text below that references either shape describes the *proposal*, not the -> code. The rank ordering and everything else did ship as written. +What still holds, unchanged since the original proposal: -## Motivation +- **No CID is privileged.** Any known CID resolves to its record; all hash + types are symmetric. +- **Rids are UUIDv7**, 26-char Crockford Base32, **minted locally** with no + central allocator. +- **`canonical_cid` is derived on read** by rank (`internal/cid/rank.go` → + `cid.Better`), never stored. +- **`cids` is a key-set of bare CIDs** (`cids/` = `"true"`), no + `:` prefix — a CIDv1's multicodec already names the hash function. -Today, every metadata entry in Redis is keyed by a content hash, specifically -midhash256: +What changed in v2.0.0: + +> 1. **The rid is a hidden local name for a group of cids.** Metadata is no +> longer stored under `file:/`. Every value is a signed claim +> about a **cid** (`claim(uid, cid, key) → value`); `cid_group(cid → rid)` +> says which cids form one record. The rid is **derived from cid groups, +> never signed and never on the wire** (swarm, gateway); each box mints its +> own. +> 2. **The `cid:` reverse index is gone**, replaced by the `cid_group` +> table (still a primary-key lookup, so still O(1)-class). +> 3. **meta-core no longer writes `cids/*` members.** They are signed votes +> from the writers that learn a cid (e.g. the full-hash plugin); a non-null +> `cids/` vote links two groups. Only the watcher's midhash enters +> `cid_group` directly. +> 4. **`duplicates` / `dup:/` are gone.** Box-local paths live in +> the cache's `local_file(path → midhash, size, mtime)`; several paths with +> the same midhash are the duplicates. +> +> Earlier superseded points, kept for the record: the proposed +> `reconcileCanonicalCIDLocked` never shipped; `cids` never was a Redis SET +> of `:` tokens; Redis itself went in v1.1.0 (2026-10). Text +> below marked *(proposal)* or *(v1.x)* describes those shapes, not v2.0.0. + +## Motivation (history) + +Before UUID rids, every metadata entry in Redis was keyed by a content hash, +specifically midhash256: ``` file:midhash256:abc.../filePath file:midhash256:abc.../title -file:midhash256:abc.../fileinfo/duration file:midhash256:abc.../tmdb/poster -... ``` -Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields -*inside* that entry: +Other digests (sha256, IPFS CIDs, BitTorrent infohashes) lived as fields +*inside* that entry (`cid_sha256`, `cid_ipfs`). Two problems: -``` -file:midhash256:abc.../cid_sha256 = sha256:def... -file:midhash256:abc.../cid_ipfs = bafy... -``` +1. **midhash256 was privileged by accident.** It *was* the key, while every + other CID was a secondary field. Hash-algorithm changes, or new digest + types added by plugins, forced rewrites of every root. The privilege + itself was the issue, not which hash held it. +2. **CID lookup was O(n).** "Given any CID, return the metadata" scanned + every entry (`storage/client.go::LookupPathByCID`). Unacceptable for a + public API, especially when peers query by any CID variant. -Two problems with this: - -1. **midhash256 is privileged by accident.** It IS the key, while every other - CID is a secondary field. There's nothing intrinsically special about - midhash256 — it's just the first hash computed. Hash-algorithm changes, or - new digest types added by plugins, force rewrites of every root. The - "promotion" workarounds people reach for (re-keying entries when a stronger - hash arrives) solve the wrong problem; the privilege itself is the issue. - -2. **CID lookup is O(n).** Resolving "given any CID, return that file's - metadata" currently scans every indexed entry (see - `storage/client.go::LookupPathByCID`, which acknowledges this in a - comment). Acceptable for admin tooling, unacceptable for a public API - surface — especially when peers on meta-share may query by any CID variant. - -The product requirement is clear: **any CID a client knows about must resolve -to that file's metadata in O(1).** IPFS-compatible CIDs are the externally -meaningful identity for content retrieval; the internal storage layout should -not bias toward any one digest type. +The requirement: **any CID a client knows must resolve to that content's +metadata in O(1)**, and the storage layout must not bias toward any digest +type. The claims model adds a second one: **what is signed must name +something every peer agrees on** — a rid minted at random on one box does +not qualify, a cid does. ## Goal -A single externally-meaningful contract: - ``` -GET /api/meta/ → full metadata document for that file +GET /api/meta/ → full (merged) metadata document for that content ``` -Where `` may be any supported type (midhash256, sha256, IPFS, btih, -sha3, future additions). All CID types are treated symmetrically. No CID type -is "the key." +`` may be any supported type (midhash, sha256, IPFS, btih, sha3, future +additions), all treated symmetrically. -## Design +## Design (v2.0.0) -### Two-tier key structure - -The root key becomes an opaque internal UUID; CIDs are aliases that point to -it. +### Votes name cids; a rid names a group ``` -file:/ ← canonical storage (UUIDv7, never disclosed externally) -cid: → ← reverse index (one Redis key per known CID) +votes (signed, travel) derived (local, persistent: truth cid_group) +(A, M, title) = "The Matrix" M → R1 +(A, M, cids/Q) = "true" ← alias vote Q → R1 +(B, Q, title) = "Matrix, The" +(B, Q, sizeByte)= "734003200" record R1 = every vote on M or Q, merged per key ``` -The metadata keyspace under `file:/` remains open and flat, exactly as -today — anything that wrote `file:/fileinfo/duration` now writes -`file:/fileinfo/duration`. Only the root identifier changes. +- **First-seen cid, no link** → new group, fresh UUIDv7 rid. +- **`cids/` vote on X** → X and `` join one group. +- **Link between two existing groups** → the group with the **older rid + survives**; the other's cids move to it; the losing rid gets a `del` event + for each of its keys, the survivor `set` events. No `rid_alias`: consumers + already handle deletes, and cids re-resolve. +- **Reading record R** = every vote on every cid of R's group, merged per key + (one signer counts once per key), precomputed into the cache's + `file:/` rows (the v1 key layout, now derived). +- **Retracting a `cids/` link does not split the group** (phase 1): + the tombstone drops out of the merge, `cid_group` keeps both cids under one + rid. In the record view, membership is synthesised from `cid_group` + (`cids/ = "true"` for every cid of the group). -**Example layout:** +Why keep a rid at all, if votes never mention it: local consumers (meta-sort, +meta-fuse, meta-stremio) and the merged view need one stable name per record, +and a group's cid set grows over time. Why `cid_group` lives in **truth** +although it could be recomputed from `cids/*` votes: the rid names are minted +at random and consumers hold them, so a recomputation would rename every +record. -``` -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = /watch/Inception.mkv -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size = 4500000000 -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime = 1729872000000000000 -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec… = "true" ← midhash (record address) -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd… = "true" ← sha2-256 -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates = SET{/watch/dup1.mkv, /watch/dup2.mkv} -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception" -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5 -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = bafy... -file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng = ... +### Why UUIDv7 (still valid) -cid:bagacbabaec… → 01JKR8XW5T4QMV7AHNJ2DEFGHK -cid:bafybeigd… → 01JKR8XW5T4QMV7AHNJ2DEFGHK -``` +- **128 bits**: collision-free under distributed minting; no allocator. Each + box minting its own rids for the same content is now the normal case, not a + corner case — which is exactly why rids never travel. +- **Time-sortable prefix**: rids minted in a window sort together + (`k >= 'file:01JKR8' AND k < 'file:01JKR9'` on the cache's merged rows), + and "older rid + survives" on a group merge is a plain string comparison. +- **Monotonic-ish**: adjacent rids cluster in the cache's `file:/…` + key range. -*(As shipped — the proposal originally wrote `cids` as a Redis SET of -`:` tokens plus a `canonical_cid` scalar. See the Status note.)* +Encoded as 26-char Crockford Base32 (ULID-compatible) for compactness and +case-insensitive matching. Migrated records kept their v1.x UUIDs as rids. -### UUID format: UUIDv7 +### The `cids` key-set -UUIDv7 chosen over UUIDv4 or 64-bit random: +`cids/` = `"true"`, one claim per member, by whichever signer knows +the link. A key-set merges conflict-free: a member is present while at least +one vote supports it, and removal is a signed retraction. That is the reason +the proposal's Redis SET was reversed before shipping, and it is what lets +the claims model treat `cids` with no special merge code. -- **128 bits**: collision-free under distributed minting; no central allocator. -- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a - given window. Restores some of the debuggability lost by going opaque. -- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN - locality. +Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are **not** +claims — meta-core cannot sign and the disk can always be rescanned. The +watcher records them in `local_file` and puts the midhash into `cid_group`; +the record view joins them in unsigned. When several local paths hold the +same content, `filePath` is the lexicographically first one. -Encoded as a 26-character Crockford Base32 string (ULID-compatible) for -compactness and case-insensitive matching. +### Canonical CID selection (unchanged) -### New root fields owned by meta-core - -The metadata schema is open — any writer can put any key under -`file:/`. This proposal does not change that. It adds three keys whose -sole writer is meta-core itself: - -| Key | Type (as shipped) | Purpose | -|-----|------|---------| -| `cids/` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `:` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. | -| `duplicates` | Redis SET at the time; since 2026-10 one `dup:/` key per path | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. | -| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. | - -Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`) -keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`, -`subtitle/`, …) are unaffected. - -~~Redis SET is a departure from the all-strings convention in the current -storage doc, but `file:__index__` is already a SET — so the precedent exists -for using sets where they're the right structure.~~ **Reversed for `cids`.** -The all-strings convention won: `cids` ships as flat `cids/` STRING keys -so that two peers merging the same record can never disagree about set -membership. `duplicates` remained a Redis SET (it is leader-local and never -merged across peers) until the 2026-10 SQLite switch, which flattened it to -`dup:/` keys. - -### Canonical CID selection - -The canonical CID is a label, not a key — so nothing external breaks when it -changes. **As shipped it is not written at all**: it is computed on every read -from the `cids` members, which makes the "update" cost zero and removes the -reconciler entirely. - -Selection rule, highest-wins (this part shipped as designed — -`internal/cid/rank.go`): +The canonical CID is a label, not a key, so nothing external breaks when it +changes. It is computed on every read from the group's cids: 1. IPFS CID (any variant) 2. SHA-256 / SHA-3 family 3. BitTorrent infohash (v2 preferred over v1) 4. midhash256 -Ranked by **utility for external retrieval**, not by cryptographic strength. +Ranked by **utility for external retrieval**, not cryptographic strength: midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are -externally meaningful for content addressing. btih ranks above midhash because -it enables swarm interop even if it's weaker as a digest. +externally meaningful; btih ranks above midhash because it enables swarm +interop. The full ladder (including the custom locator codecs) lives in +`internal/cid/rank.go`, mirrored in meta-search / meta-share +(`scripts/check-cid-vectors.sh`). -~~Reconciler runs on the leader, recomputes `canonical_cid` whenever -`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no -reconciler. `GetMetadataDocument` folds the members through `cid.Better` on -each read. +## Lifecycles (v2.0.0) -## Internal operations - -The meta-core storage interface adds five operations. None are exposed -externally — they're internal to meta-core. External callers see only the -CID-based API. - -```go -// Mint creates a new root with no aliases. Called when the watcher -// computes a midhash for a previously-unseen file. Returns the new UUID. -Mint(filePath string, size int64, mtime int64) (uuid string, err error) - -// AddAlias registers a CID as resolving to this UUID. Updates both the -// cid: reverse index and the file:/cids/ key-set member. -// (As shipped: no reconciliation step — canonical is derived on read.) -AddAlias(uuid string, cid string) error - -// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown. -GetByCID(cid string) (uuid string, err error) - -// MergeIfAliasExists is called when the watcher mints a fresh UUID for a -// file path but, on hashing, discovers the resulting CID already belongs -// to a different UUID. Merges the new (mostly-empty) UUID into the -// existing one, transferring filePath into the duplicates set, and -// returns the canonical UUID. -MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error) - -// DeleteRoot removes the entire entry, including every cid:* alias and -// every file:/* key. Used for file deletion events. -DeleteRoot(uuid string) error -``` - -### Lifecycle: file arrival +### File arrival ``` -1. Watcher sees /watch/foo.mkv created. -2. Watcher computes midhash256 → "midhash256:abc...". -3. Watcher calls GetByCID("midhash256:abc..."). - → uuid_X (existing entry) or "" (new file). - -4a. If existing (uuid_X): - - Add "/watch/foo.mkv" to file:/duplicates. - - No new root created. - - meta-dup is notified via the existing keyspace event channel. - -4b. If new: - - Mint("/watch/foo.mkv", size, mtime) → uuid_Y. - - AddAlias(uuid_Y, "midhash256:abc..."). - - Plugins now process the file, referenced by midhash256:abc... - (UUID is never disclosed to them.) - -5. Later, the full-hash plugin computes sha256. - - Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...". - - meta-core resolves midhash256:abc → uuid_Y via GetByCID. - - meta-core calls AddAlias(uuid_Y, "sha256:def..."). - - Nothing else to do: the next read of this entry derives a canonical CID - of sha2-256, because it outranks the midhash. +1. Watcher sees /files/watch/foo.mkv, computes its midhash M. +2. local_file: /files/watch/foo.mkv → (M, size, mtime). +3. cid_group: M already grouped → nothing; otherwise M → fresh rid. +4. Plugins process the file by cid M (the rid is never disclosed to them) + and sign claims about M with their own keys. +5. The full-hash plugin computes sha256 S and signs (plugin, M, cids/S) = "true". + → S joins M's group (or the two groups merge, older rid surviving). + → The next read derives canonical_cid = S, because it outranks the midhash. ``` -### Lifecycle: file deletion +A second path with the same midhash is just another `local_file` row: same +cid, same group, same record; the record's `filePath` is the +lexicographically first of the paths. -``` -1. Watcher sees /watch/foo.mkv deleted. -2. Resolve which entry owns this path: - a. If /watch/foo.mkv is in some entry's duplicates set: - - Remove from set. Done. - b. If /watch/foo.mkv is the filePath of some entry: - - If duplicates is non-empty: promote one duplicate to filePath. - - If duplicates is empty: DeleteRoot(uuid). - - Iterates the cids set, deletes each cid: reverse-index key. - - Deletes every file:/* key. - - Removes uuid from file:__index__. -``` +### File deletion + +The watcher removes the path's `local_file` row; the record view loses that +`filePath`. Claims are signed truth from their writers and are not deleted +by meta-core (the store is permanent and additive, +`docs/project-architecture/metacore-as-permanent-store.md`). + +*(v1.x)* Deletion promoted a duplicate to `filePath`, or called `DeleteRoot`, +which removed every `cid:*` alias and every `file:/*` key. ## Public API ``` -GET /api/meta/ - → 200 { "filePath": "...", "canonical_cid": "...", - "cids": [...], "title": "...", "tmdb": {...}, ... } - → 404 if no such CID - -GET /api/file/ (already exists — implementation now O(1)) - → streams file content +GET /api/meta/ → 200 merged document { "filePath", "canonical_cid", "cids": [...], ... } + (optional ?as=: the viewer's own votes win) + → 404 if no such cid +GET /api/meta//claims → {rid, cids, claims}: every live signed claim of the cid's + group, all signers (?key=, ?uid=); 404 if no such cid +GET /api/metadata//claims → the same, addressed by rid (local consumers) ``` -The UUID is **not** included in the response. Clients address everything by -CID; meta-core's choice of internal identifier remains a private -implementation detail and may change. +The rid is not part of the cid-addressed contract and never appears in a +claim. It is visible only to local consumers that list or address records by +rid (`/api/metadata/{rid}`, `/api/metadata/{rid}/claims`, `hash-ids`, SSE event +keys; the claims routes also echo it as `rid`), and may change on a +group merge (delete event, then re-resolve by cid). -Auth: `/api/meta/` follows the same pattern as `/api/file/` and -bypasses the auth perimeter. It's the metadata twin of the content endpoint, -and meta-share calls it from federated search responses. +Auth: `/api/meta/` bypasses the perimeter like `/api/file/` — it is +the metadata twin of the content endpoint. -## Cross-service impact +## Cross-service impact (v2.0.0) -This is a meta-core schema change. Other services interact only through CIDs, -so the impact is contained but real. +- **Writers** (plugins, self-publishing feeders, gateway core, meta-share, + meta-search): address claims by cid only and sign them; they never resolve + a rid. +- **meta-fuse, meta-stremio**: unchanged — merged reads and SSE keep their + shape; a group merge reaches them as ordinary `del` + `set` events. +- **meta-search / meta-share**: announce the derived canonical cid (each + mirrors the Go rank) and relay raw claims; every cid of the group travels as + `cids/*` claims, so a peer querying by any digest gets a hit. -- **meta-sort**: writers must resolve plugin callbacks to a UUID before - writing. The existing `LeaderClient` gains a `getByCID` cache. -- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the - ID format changes. -- **meta-stremio**: queries by CID; needs no logic change beyond its storage - client. -- **meta-dup**: gains a cleaner data source — duplicates are now explicit in - the `duplicates` set, not implicit in colliding writes. -- **meta-search / meta-share**: announce the *derived* canonical CID rather - than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata` - rather than reading a stored field. Federated search responses include all - CIDs from the `cids` key-set so any peer querying by any digest gets a hit. - (Written before the 2026-06 discovery/transport split; discovery is now - meta-search's job.) -- **Plugins**: no change. They speak CIDs only. +## Migration (history) -## Migration +> **The v1.0 sweep below was never run.** The alpha shipped behind a +> clean-wipe gate. The v1.x → v2.0.0 conversion of watch.nsl.sh is a +> one-off tool outside the service image, `packages/meta-core/tools/claims-migrate` +> (canonical cid by rank per record; `cid:` aliases → `cid_group`, rids keep +> their UUIDs, and the `cid:` index wins when a record's `cids/*` member +> disagrees; fields re-signed by a throwaway legacy key); every other box +> starts empty. Implementation basis §11. -> **Historical — this sweep was never run.** The alpha shipped behind a -> clean-wipe gate (Redis was flushed and the library re-ingested), so no -> in-place migration code exists. Kept for the record. - -One-time sweep, idempotent, runs on first boot after upgrade: - -``` -For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …): - 1. Mint a fresh uuid_Y. - 2. AddAlias(uuid_Y, hashID). The existing primary becomes an alias. - 3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value). - 4. RENAME file:/ → file:/ for every - property (Lua script, one entry at a time; non-atomic across entries, - but each entry stays internally consistent). - 5. Reconcile canonical_cid. - 6. Remove hashID from file:__index__, add uuid_Y. -``` - -Crash-recoverable: re-running the sweep finds entries with both old and new -roots and resumes by detecting which step completed (presence of -`file:/cids` indicates step 3 finished; absence of any -`file:/*` keys indicates step 4 finished). +*(proposal)* One-time sweep over `file:__index__`: mint a uuid, `AddAlias` +the old hashID and every `cid_*` field, `RENAME` every property under the new +root, reconcile canonical, swap the index entry; crash-recoverable by +detecting which step completed. ## Decisions recorded | Question | Decision | Notes | |---|---|---| -| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. | -| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. | -| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. | -| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. | -| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. | -| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. | -| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. | -| Reverse-index format | Flat `cid:` STRING keys | Simple, scan-friendly, no hot-key contention. | +| When is the rid minted? | When a cid is first seen with no link to a known group | v1.x: at first hash of a watched file. Now any first-seen cid (watcher midhash or a claim about a new cid) mints one | +| Rid format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe | +| Rid visible externally? | No — never signed, never on the swarm/gateway wire | Local HTTP consumers may hold it | +| What do signed votes name? | A cid | Every peer agrees on a cid; rids differ per box | +| Record identity | A group of cids (`cid_group`, in truth) | Linked by `cids/` votes | +| Group merge | Older rid survives; losing rid → `del` events | No `rid_alias` (out of scope for phase 1) | +| Duplicate paths | `local_file` rows sharing a midhash (cache) | v1.x: `duplicates` SET, then `dup:/` | +| Canonical CID rank | IPFS > sha-family > btih > midhash, derived on read | Utility-weighted; never stored | +| `cids` shape | Key-set of bare CIDs, one claim per member | Merges with the generic rule; a SET would not | +| Reverse index | `cid_group(cid PK → rid)` in truth, copied to cache `cid:` rows for reads | v1.x: flat `cid:` keys were the only copy | +| Unlinking | A retracted `cids/*` link does not split a group | Phase 1 | ## Open questions -- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints - a UUID at discovery (before hashing), then `AddAlias` runs when the midhash - arrives. Requires an explicit "reconcile if alias already exists" step - because two paths with the same content would each get their own UUID until - hashing reveals the collision. Worth doing eventually; not required for - this iteration. - -- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every* - member of `cids` on the wire (`Record.cids`) and addresses itself by the - single derived canonical (`Record.cid`), so a peer querying by any digest - gets a hit without the DHT footprint of announcing each one separately. - -- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/` as a - content-streaming endpoint for external IPFS clients? Adjacent to this - design but not part of it. - -- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`) - handle `file::/...` with a special split rule for the colon - inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies - — but any external consumer subscribed by hashID pattern needs updating. - Inventory those before flipping the migration. +- **Discovery-time rid minting.** Minting at discovery (before hashing) + would give two same-content paths two rids until hashing links them. The + claims model already handles that case — it is an ordinary group merge — + so this is now a policy choice, not a missing mechanism. +- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/` for + external IPFS clients? Adjacent, not part of this design. +- ~~**meta-share announce strategy.**~~ Settled: the canonical cid addresses + the record, every group cid travels as a `cids/*` claim. +- ~~**Keyspace-notification consumers.**~~ Obsolete since Redis left. ## Non-goals -- **Not** a content-replication or IPFS-pinning system. Registering an IPFS - CID alias does not pin the file anywhere — it only states "if somebody - hashes this file with IPFS's chunker, this is the CID they'd get." -- **Not** a versioning scheme. Metadata is a snapshot of current state; - historical versions are out of scope. -- **Not** a strong-consistency system. Reverse-index writes happen under the - leader lock; reads from followers may briefly miss a freshly-added alias. +- **Not** a content-replication or pinning system. A `cids/` vote + pins nothing; it states "hashing these bytes with IPFS's chunker gives this + CID." +- **Not** a history. `ts` versions each `(uid, cid, key)` and the newest vote + replaces the older one; only tombstones are kept, to block replay. +- **Not** strongly consistent across the two databases: the cache is applied + after the truth commit. `POST /api/claims` waits for the apply before + answering, so the writer reads its own write; another reader may briefly + miss a just-accepted claim. It is behind, never wrong. diff --git a/editor/src/api/claimsApi.ts b/editor/src/api/claimsApi.ts new file mode 100644 index 0000000..f9e273a --- /dev/null +++ b/editor/src/api/claimsApi.ts @@ -0,0 +1,135 @@ +// Signed claims (meta-core v2.0.0). The editor no longer writes keys: an edit +// is a claim signed as the chosen identity account — through meta-core's +// account-key service (/api/identity/sign, compact low-S) — and posted to +// POST /api/claims. That vote wins in that account's view; the shared merged +// value only moves if enough signers agree. +// +// MMUR2 bytes (must match internal/claim and its golden vectors): +// "MMUR2\n" || canonical CBOR [cid, uid, key, value, ts] + +export interface Claim { + cid: string; + uid: string; + key: string; + value: string | { b64: string } | null; + ts: number; + sig: string; +} + +export interface ClaimsResponse { + rid: string; + cids: string[]; + claims: Claim[]; +} + +export interface Account { + uid: string; + curve: string; + createdAt: number; +} + +const SIGNER_LS_KEY = 'kv-editor.signer.v1'; + +async function jsonOrThrow(res: Response): Promise { + if (!res.ok) { + let msg = `HTTP ${res.status}`; + try { + const body = await res.json(); + if (body?.message) msg = body.message; + else if (body?.error) msg = body.error; + } catch {} + throw new Error(msg); + } + return res.json(); +} + +// --- canonical CBOR for the five MMUR2 shapes ------------------------------- + +function head(out: number[], major: number, n: number) { + const m = major << 5; + if (n < 24) out.push(m | n); + else if (n <= 0xff) out.push(m | 24, n); + else if (n <= 0xffff) out.push(m | 25, n >> 8, n & 0xff); + else if (n <= 0xffffffff) out.push(m | 26, (n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff); + else { + const hi = Math.floor(n / 2 ** 32); + const lo = n >>> 0; + out.push(m | 27, (hi >>> 24) & 0xff, (hi >>> 16) & 0xff, (hi >>> 8) & 0xff, hi & 0xff, + (lo >>> 24) & 0xff, (lo >>> 16) & 0xff, (lo >>> 8) & 0xff, lo & 0xff); + } +} + +function text(out: number[], s: string) { + const b = new TextEncoder().encode(s); + head(out, 3, b.length); + for (const x of b) out.push(x); +} + +export function signedBytes(cid: string, uid: string, key: string, value: string | null, ts: number): Uint8Array { + const out: number[] = [...new TextEncoder().encode('MMUR2\n')]; + head(out, 4, 5); + text(out, cid); + text(out, uid); + text(out, key); + if (value === null) out.push(0xf6); + else text(out, value); + head(out, 0, ts); + return new Uint8Array(out); +} + +function b64(bytes: Uint8Array): string { + let s = ''; + for (const x of bytes) s += String.fromCharCode(x); + return btoa(s); +} + +// --- signer ----------------------------------------------------------------- + +export function getSigner(): string { + try { return localStorage.getItem(SIGNER_LS_KEY) ?? ''; } catch { return ''; } +} + +export function setSigner(uid: string) { + try { localStorage.setItem(SIGNER_LS_KEY, uid); } catch {} +} + +export const ClaimsAPI = { + async accounts(): Promise { + const r = await jsonOrThrow<{ accounts: Account[] }>(await fetch('/api/identity/accounts')); + return r.accounts ?? []; + }, + + async forRecord(rid: string, key?: string): Promise { + const q = key ? `?key=${encodeURIComponent(key)}` : ''; + return jsonOrThrow(await fetch(`/api/metadata/${encodeURIComponent(rid)}/claims${q}`)); + }, + + // vote signs value (null = retraction) for key on the record's subject cid + // as the chosen account and posts it. + async vote(rid: string, key: string, value: string | null): Promise { + const uid = getSigner(); + if (!uid) throw new Error('choose a signing account first (top right)'); + const rec = await this.forRecord(rid); + const cid = rec.cids[0]; + if (!cid) throw new Error(`record ${rid} has no cid`); + // ts must beat this account's previous vote on the key. + const prev = rec.claims.filter((c) => c.uid === uid && c.key === key).map((c) => c.ts); + const ts = Math.max(Date.now(), ...prev.map((t) => t + 1)); + const bytes = signedBytes(cid, uid, key, value, ts); + const signed = await jsonOrThrow<{ sigB64: string }>(await fetch('/api/identity/sign', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ uid, bytesB64: b64(bytes), format: 'compact' }), + })); + const claim: Claim = { cid, uid, key, value, ts, sig: signed.sigB64 }; + const res = await jsonOrThrow<{ status: string; reason?: string }[]>(await fetch('/api/claims', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify([claim]), + })); + const st = res[0]?.status; + if (st !== 'accepted' && st !== 'unchanged') { + throw new Error(`claim ${st}${res[0]?.reason ? ': ' + res[0].reason : ''}`); + } + }, +}; diff --git a/editor/src/api/kvApi.ts b/editor/src/api/kvApi.ts index 82868cc..421dfdb 100644 --- a/editor/src/api/kvApi.ts +++ b/editor/src/api/kvApi.ts @@ -1,8 +1,17 @@ -// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of -// string-valued keys; this module wraps the S3-style listing, search, and -// per-key get/put/delete operations. +// The editor's key tree. Since meta-core v2.0.0 the records are no longer raw +// keys: `file:/` is the MERGED view, derived from signed claims. +// This module keeps the slash-tree the components were built on, as a view: +// +// - reads of `file:` keys come from the merged record API +// (/api/metadata/hash-ids, /api/metadata/{rid}); +// - writes of `file:` keys are signed claims (claimsApi.ts): put = a vote, +// del = a retraction of the chosen account's vote; +// - every other key is service scratch KV, through /api/kv/*. + +import { ClaimsAPI } from './claimsApi'; const API_BASE = '/api/kv'; +const RECORD_NS = 'file:'; export interface KVTreeResponse { prefix: string; @@ -61,6 +70,45 @@ async function jsonOrThrow(res: Response): Promise { return res.json(); } +// splitRecordKey decomposes file:/ (field may be empty). +export function splitRecordKey(key: string): { rid: string; field: string } | null { + if (!key.startsWith(RECORD_NS)) return null; + const rest = key.slice(RECORD_NS.length); + const slash = rest.indexOf('/'); + if (slash <= 0) return null; + return { rid: rest.slice(0, slash), field: rest.slice(slash + 1) }; +} + +let hashIdsCache: { at: number; ids: string[] } | null = null; + +async function hashIds(): Promise { + if (hashIdsCache && Date.now() - hashIdsCache.at < 5000) return hashIdsCache.ids; + const r = await jsonOrThrow<{ hashIds: string[] }>(await fetch('/api/metadata/hash-ids')); + hashIdsCache = { at: Date.now(), ids: (r.hashIds ?? []).slice().sort() }; + return hashIdsCache.ids; +} + +async function recordFields(rid: string): Promise> { + const res = await fetch(`/api/metadata/${encodeURIComponent(rid)}`); + if (res.status === 404) return {}; + const r = await jsonOrThrow<{ metadata: Record }>(res); + return r.metadata ?? {}; +} + +// listUnder lists the children of prefix among keys, S3-style. +function listUnder(prefix: string, delimiter: string, keys: string[]) { + const branches = new Set(); + const leaves: string[] = []; + for (const k of keys) { + if (!k.startsWith(prefix)) continue; + const rest = k.slice(prefix.length); + const i = delimiter ? rest.indexOf(delimiter) : -1; + if (i >= 0) branches.add(prefix + rest.slice(0, i + delimiter.length)); + else leaves.push(k); + } + return { branches: [...branches].sort(), leaves: leaves.sort() }; +} + export const KVAPI = { async info(): Promise { return jsonOrThrow(await fetch(`${API_BASE}/info`)); @@ -72,17 +120,34 @@ export const KVAPI = { cursor?: string; max?: number; }): Promise { + const prefix = params.prefix ?? ''; + const delimiter = params.delimiter ?? ''; + const done = (branches: string[], leaves: string[]): KVTreeResponse => + ({ prefix, delimiter, branches, leaves, cursor: '0', hasMore: false }); + + if (prefix.startsWith(RECORD_NS)) { + const parts = splitRecordKey(prefix); + if (!parts) { + const ids = await hashIds(); + return done(ids.map((id) => `${RECORD_NS}${id}/`), []); + } + const fields = await recordFields(parts.rid); + const keys = Object.keys(fields).map((f) => `${RECORD_NS}${parts.rid}/${f}`); + const l = listUnder(prefix, delimiter, keys); + return done(l.branches, l.leaves); + } const q = new URLSearchParams(); - if (params.prefix !== undefined) q.set('prefix', params.prefix); - if (params.delimiter !== undefined) q.set('delimiter', params.delimiter); + q.set('prefix', prefix); + q.set('delimiter', delimiter); if (params.cursor) q.set('cursor', params.cursor); if (params.max) q.set('max', String(params.max)); - return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`)); + const svc = await jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`)); + if (prefix === '' && delimiter) { + svc.branches = [RECORD_NS, ...svc.branches.filter((b) => b !== RECORD_NS)]; + } + return svc; }, - // Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap - // is reached. Useful for branches with up to a few thousand children where - // pagination UX would just be a worse spinner. async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise { const seenLeaves = new Set(); const seenBranches = new Set(); @@ -105,22 +170,73 @@ export const KVAPI = { }; }, + // search finds keys whose path contains the substring: records matched by + // free text (each matching field key), plus service keys. async search(contains: string, limit = 500): Promise { + const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record }[] }>( + await fetch('/api/metadata/search', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query: contains, limit: 100 }), + })); + const needle = contains.toLowerCase(); + const keys: string[] = []; + for (const hit of r.results ?? []) { + for (const [f, v] of Object.entries(hit.metadata)) { + const k = `${RECORD_NS}${hit.hashId}/${f}`; + if (k.toLowerCase().includes(needle) || v.toLowerCase().includes(needle)) keys.push(k); + } + } const q = new URLSearchParams({ contains, limit: String(limit) }); - return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`)); + const svc = await jsonOrThrow(await fetch(`${API_BASE}/search?${q}`)); + const all = [...keys, ...svc.keys].sort(); + return { contains, keys: all.slice(0, limit), truncated: all.length > limit || svc.truncated }; }, + // find returns record fields (one of `fields`) whose value contains the + // substring, via the merged-record search. async find(contains: string, fields: string[], limit = 200): Promise { - const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) }); - return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`)); + const matches: KVFindMatch[] = []; + let truncated = false; + for (const field of fields) { + const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record }[] }>( + await fetch('/api/metadata/search', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ filters: [{ property: field, values: [contains] }], limit }), + })); + for (const hit of r.results ?? []) { + const v = hit.metadata[field]; + if (v === undefined) continue; + const entryPath = `${RECORD_NS}${hit.hashId}/`; + matches.push({ key: entryPath + field, value: v, entryPath, field }); + if (matches.length >= limit) { truncated = true; break; } + } + if (truncated) break; + } + matches.sort((a, b) => a.key.localeCompare(b.key)); + return { contains, fields, matches, truncated }; }, async get(key: string): Promise { + const parts = splitRecordKey(key); + if (parts) { + const fields = await recordFields(parts.rid); + const value = fields[parts.field]; + return { key, type: 'string', value: value ?? '', exists: value !== undefined }; + } const q = new URLSearchParams({ key }); return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`)); }, + // put on a record key is the chosen account's signed vote. async put(key: string, value: string): Promise { + const parts = splitRecordKey(key); + if (parts) { + await ClaimsAPI.vote(parts.rid, parts.field, value); + hashIdsCache = null; + return { key, type: 'string', value, exists: true }; + } const res = await fetch(`${API_BASE}/value`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, @@ -129,7 +245,14 @@ export const KVAPI = { return jsonOrThrow(res); }, + // del on a record key retracts the chosen account's vote; the merged value + // stays while other signers still support it. async del(key: string): Promise<{ key: string; deleted: boolean }> { + const parts = splitRecordKey(key); + if (parts) { + await ClaimsAPI.vote(parts.rid, parts.field, null); + return { key, deleted: true }; + } const q = new URLSearchParams({ key }); return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' })); }, diff --git a/editor/src/api/snapshotApi.ts b/editor/src/api/snapshotApi.ts deleted file mode 100644 index fcc79a9..0000000 --- a/editor/src/api/snapshotApi.ts +++ /dev/null @@ -1,96 +0,0 @@ -const API_BASE = '/api/snapshot'; - -export type ImportMode = 'replace' | 'merge'; -export type ImportConflict = 'mine' | 'source'; - -export interface FileImportResult { - hashId: string; - added: number; - updated: number; - unchanged: number; - kept: number; - deleted: number; - error?: string; -} - -export interface PluginOutputResult { - written: number; - skipped: number; -} - -export interface ImportResult { - mode: ImportMode; - conflict?: ImportConflict; - dryRun: boolean; - totalFiles: number; - filesOk: number; - filesFailed: number; - files?: FileImportResult[]; - pluginOutput?: PluginOutputResult; -} - -export interface WipeResult { - metadataDeleted: number; -} - -export class SnapshotAPI { - /** - * Trigger a download of a metadata snapshot ZIP. - * Browser handles the download via a synthetic click. - */ - static downloadExport(includes: { pluginOutput?: boolean } = {}): void { - const parts: string[] = []; - if (includes.pluginOutput) parts.push('plugin-output'); - const qs = parts.length ? `?include=${parts.join(',')}` : ''; - const a = document.createElement('a'); - a.href = `${API_BASE}/export${qs}`; - a.rel = 'noopener'; - document.body.appendChild(a); - a.click(); - a.remove(); - } - - /** - * Upload a snapshot ZIP and apply it. - * Mode replace deletes and rewrites each CID's properties. - * Mode merge preserves untouched fields; conflict picks who wins on overlap. - */ - static async runImport( - file: File, - opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean } - ): Promise { - const params = new URLSearchParams(); - params.set('mode', opts.mode); - if (opts.mode === 'merge') { - params.set('conflict', opts.conflict ?? 'source'); - } - if (opts.dryRun) { - params.set('dry_run', 'true'); - } - const fd = new FormData(); - fd.append('snapshot', file); - const response = await fetch(`${API_BASE}/import?${params.toString()}`, { - method: 'POST', - body: fd, - }); - if (!response.ok) { - const err = await response.json().catch(() => ({})); - throw new Error(err.message || err.error || 'Import failed'); - } - return response.json(); - } - - /** - * Wipe all stored metadata. Pass scope=metadata. - */ - static async wipe(scope: 'metadata'): Promise { - const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, { - method: 'POST', - }); - if (!response.ok) { - const err = await response.json().catch(() => ({})); - throw new Error(err.message || err.error || 'Wipe failed'); - } - return response.json(); - } -} diff --git a/editor/src/components/SnapshotPanel.css b/editor/src/components/SnapshotPanel.css deleted file mode 100644 index dc27a96..0000000 --- a/editor/src/components/SnapshotPanel.css +++ /dev/null @@ -1,209 +0,0 @@ -.snapshot-panel { - max-width: 920px; - margin: 0 auto; - padding: 1.5rem; -} - -.snapshot-panel h2 { - margin: 0 0 0.5rem; -} - -.snapshot-intro { - color: var(--text-muted, #888); - margin-bottom: 1.5rem; -} - -.snapshot-section { - border: 1px solid var(--border, #e0e0e0); - border-radius: 6px; - padding: 1rem 1.25rem; - margin-bottom: 1.25rem; -} - -.snapshot-section.danger { - border-color: #d04a4a; - background: rgba(208, 74, 74, 0.04); -} - -.snapshot-section h3 { - margin: 0 0 0.5rem; -} - -.snapshot-row { - display: flex; - gap: 1rem; - align-items: flex-start; - margin: 0.75rem 0; - flex-wrap: wrap; -} - -.snapshot-file { - display: flex; - flex-direction: column; - gap: 0.25rem; -} - -.snapshot-file span { - font-size: 0.85rem; - color: var(--text-muted, #888); -} - -.snapshot-fieldset { - border: 1px solid var(--border, #e0e0e0); - border-radius: 4px; - padding: 0.5rem 0.75rem; - min-width: 280px; -} - -.snapshot-fieldset[disabled] { - opacity: 0.5; -} - -.snapshot-fieldset legend { - font-weight: 600; - font-size: 0.85rem; - padding: 0 0.25rem; -} - -.snapshot-fieldset label { - display: block; - margin: 0.25rem 0; - font-size: 0.9rem; - cursor: pointer; -} - -.snapshot-fieldset input[type='radio'] { - margin-right: 0.5rem; -} - -.snapshot-btn { - background: var(--bg-button, #f0f0f0); - border: 1px solid var(--border, #c0c0c0); - color: var(--text, #222); - padding: 0.5rem 1rem; - border-radius: 4px; - cursor: pointer; - font-size: 0.95rem; -} - -.snapshot-btn:hover:not(:disabled) { - background: var(--bg-button-hover, #e0e0e0); -} - -.snapshot-btn:disabled { - opacity: 0.5; - cursor: not-allowed; -} - -.snapshot-btn.primary { - background: #2a6df4; - border-color: #2a6df4; - color: white; -} - -.snapshot-btn.primary:hover:not(:disabled) { - background: #1a5de4; -} - -.snapshot-btn.danger { - background: #d04a4a; - border-color: #d04a4a; - color: white; -} - -.snapshot-btn.danger:hover:not(:disabled) { - background: #b83838; -} - -.snapshot-error { - background: rgba(208, 74, 74, 0.1); - color: #b83838; - padding: 0.5rem 0.75rem; - border-radius: 4px; - margin-top: 0.75rem; -} - -.snapshot-info { - background: rgba(42, 109, 244, 0.08); - color: #1a5de4; - padding: 0.5rem 0.75rem; - border-radius: 4px; - margin-top: 0.75rem; -} - -.snapshot-result { - margin-top: 0.75rem; - border: 1px solid var(--border, #e0e0e0); - border-radius: 4px; - padding: 0.75rem; -} - -.snapshot-result-summary { - display: flex; - gap: 1rem; - flex-wrap: wrap; - margin-bottom: 0.5rem; - font-size: 0.9rem; -} - -.snapshot-result-table { - width: 100%; - border-collapse: collapse; - font-size: 0.9rem; -} - -.snapshot-result-table th, -.snapshot-result-table td { - border: 1px solid var(--border, #e0e0e0); - padding: 0.35rem 0.5rem; - text-align: right; -} - -.snapshot-result-table th { - background: var(--bg-subtle, #f5f5f5); - font-weight: 600; -} - -.snapshot-result-errors { - margin-top: 0.5rem; - font-size: 0.85rem; -} - -.snapshot-result-errors summary { - cursor: pointer; - color: #b83838; -} - -.snapshot-result-errors ul { - margin: 0.5rem 0 0; - padding-left: 1.25rem; -} - -.snapshot-result-errors code { - font-size: 0.8rem; - word-break: break-all; -} - -.snapshot-checkbox { - display: flex; - align-items: flex-start; - gap: 0.5rem; - margin: 0.5rem 0 0.75rem; - cursor: pointer; - font-size: 0.95rem; -} - -.snapshot-checkbox input { - margin-top: 0.2rem; -} - -.snapshot-hint { - color: var(--text-muted, #888); - font-size: 0.9rem; -} - -.snapshot-result-aside { - margin-top: 0.5rem; - font-size: 0.85rem; - color: var(--text-muted, #555); -} diff --git a/editor/src/components/SnapshotPanel.tsx b/editor/src/components/SnapshotPanel.tsx deleted file mode 100644 index e984596..0000000 --- a/editor/src/components/SnapshotPanel.tsx +++ /dev/null @@ -1,271 +0,0 @@ -import { useState } from 'react'; -import { - SnapshotAPI, - ImportMode, - ImportConflict, - ImportResult, -} from '../api/snapshotApi'; -import './SnapshotPanel.css'; - -export function SnapshotPanel() { - const [exportPluginOutput, setExportPluginOutput] = useState(false); - const [importFile, setImportFile] = useState(null); - const [importMode, setImportMode] = useState('merge'); - const [importConflict, setImportConflict] = useState('source'); - const [importing, setImporting] = useState(false); - const [importResult, setImportResult] = useState(null); - const [importError, setImportError] = useState(null); - - const [wipeBusy, setWipeBusy] = useState(false); - const [wipeMessage, setWipeMessage] = useState(null); - - const runImport = async (dryRun: boolean) => { - if (!importFile) { - setImportError('Pick a snapshot ZIP first.'); - return; - } - setImporting(true); - setImportError(null); - setImportResult(null); - try { - const result = await SnapshotAPI.runImport(importFile, { - mode: importMode, - conflict: importMode === 'merge' ? importConflict : undefined, - dryRun, - }); - setImportResult(result); - } catch (e: any) { - setImportError(e.message || String(e)); - } finally { - setImporting(false); - } - }; - - const runWipe = async () => { - const ok = window.confirm( - 'This deletes ALL stored metadata. Files on disk are not touched, but every manual ' + - 'edit you made via this editor will be lost. Continue?' - ); - if (!ok) return; - setWipeBusy(true); - setWipeMessage(null); - try { - const r = await SnapshotAPI.wipe('metadata'); - setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`); - } catch (e: any) { - setWipeMessage(`Failed: ${e.message || e}`); - } finally { - setWipeBusy(false); - } - }; - - return ( -
-

Snapshot

-

- Export, import, or wipe the metadata KV store. Snapshots are portable ZIP - archives — one nested-JSON file per CID, plus a manifest. -

- -
-

Export

-

Downloads every CID's metadata as a ZIP. Optional cache files can be added below.

- - -
- -
-

Import

-
- -
- -
-
- Mode - - -
- -
- Conflict - - -
-
- -
- - -
- - {importError &&
{importError}
} - {importResult && } -
- -
-

Wipe metadata

-

- Deletes every file:* key plus file:__index__. - Files on disk are untouched. -

- - {wipeMessage &&
{wipeMessage}
} -
-
- ); -} - -function ImportResultView({ result }: { result: ImportResult }) { - const totals = (result.files ?? []).reduce( - (acc, f) => { - acc.added += f.added; - acc.updated += f.updated; - acc.unchanged += f.unchanged; - acc.kept += f.kept; - acc.deleted += f.deleted; - return acc; - }, - { added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 } - ); - return ( -
-
- {result.dryRun ? 'Dry run preview' : 'Import applied'} - - {result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files - - - mode={result.mode} - {result.conflict ? `, conflict=${result.conflict}` : ''} - -
- - - - - - - - - - - - - - - - - - - -
addedupdatedunchangedkept (mine)deleted (replace)
{totals.added}{totals.updated}{totals.unchanged}{totals.kept}{totals.deleted}
- {result.pluginOutput && ( -
- plugin-output: {result.pluginOutput.written} written - {result.pluginOutput.skipped > 0 - ? `, ${result.pluginOutput.skipped} skipped (existed)` - : ''} -
- )} - {result.files && result.files.some((f) => f.error) && ( -
- Per-file errors -
    - {result.files - .filter((f) => f.error) - .map((f) => ( -
  • - {f.hashId}: {f.error} -
    • - ))} -
-
- )} -
- ); -} diff --git a/editor/src/kv/ClaimsPanel.tsx b/editor/src/kv/ClaimsPanel.tsx new file mode 100644 index 0000000..7c864c9 --- /dev/null +++ b/editor/src/kv/ClaimsPanel.tsx @@ -0,0 +1,45 @@ +import { useEffect, useState } from 'react'; +import { Claim, ClaimsAPI, getSigner } from '../api/claimsApi'; +import { splitRecordKey } from '../api/kvApi'; + +// ClaimsPanel lists the signed claims behind one merged field: who said what, +// and when. The merged value is what most signers agree on. +export function ClaimsPanel({ recordKey, reloadToken }: { recordKey: string; reloadToken: number }) { + const [claims, setClaims] = useState(null); + const [error, setError] = useState(''); + const parts = splitRecordKey(recordKey); + + useEffect(() => { + if (!parts || !parts.field) return; + let cancelled = false; + ClaimsAPI.forRecord(parts.rid, parts.field) + .then((r) => { if (!cancelled) setClaims(r.claims); }) + .catch((e) => { if (!cancelled) setError(e.message); }); + return () => { cancelled = true; }; + }, [recordKey, reloadToken]); // eslint-disable-line react-hooks/exhaustive-deps + + if (!parts || !parts.field) return null; + if (error) return
claims: {error}
; + if (!claims) return
loading claims…
; + const me = getSigner(); + return ( +
+
Claims ({claims.length})
+ {claims.length === 0 ? ( +
No signed claim — a box-local fact (file path, size, mtime) or a group member.
+ ) : ( + + + {claims.map((c) => ( + + + + + + ))} + +
{c.uid.slice(0, 10)}…{c.uid === me ? ' (you)' : ''}{c.value === null ? retracted : typeof c.value === 'string' ? c.value : ciphertext}{new Date(c.ts).toLocaleString()}
+ )} +
+ ); +} diff --git a/editor/src/kv/KVApp.css b/editor/src/kv/KVApp.css index 7266c1a..2751745 100644 --- a/editor/src/kv/KVApp.css +++ b/editor/src/kv/KVApp.css @@ -894,3 +894,9 @@ .kv-modal-close:hover { opacity: 1; } + +.kv-signer { font-size: 12px; display: inline-flex; align-items: center; gap: 4px; } +.kv-signer select { font-size: 12px; } +.kv-claims { margin-top: 16px; } +.kv-claims-table { width: 100%; border-collapse: collapse; font-size: 12px; } +.kv-claims-table td { padding: 3px 6px; border-top: 1px solid rgba(127, 127, 127, 0.25); vertical-align: top; } diff --git a/editor/src/kv/KVApp.tsx b/editor/src/kv/KVApp.tsx index 8b60b2e..5fd747a 100644 --- a/editor/src/kv/KVApp.tsx +++ b/editor/src/kv/KVApp.tsx @@ -10,7 +10,7 @@ const FILE_BRANCH_RE = /^file:[^/]+\/$/; import { SearchFieldChips } from './SearchFieldChips'; import { ToastStack, makeToast } from './Toast'; import { ToastEntry } from './types'; -import { SnapshotPanel } from '../components/SnapshotPanel'; +import { SignerPicker } from './SignerPicker'; import './KVApp.css'; const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1'; @@ -32,7 +32,6 @@ function loadSearchFields(): string[] { // // A sticky header carries the search bar, the breadcrumb, and basic stats. export default function KVApp() { - const [snapshotOpen, setSnapshotOpen] = useState(false); // Selected node. branchPath is set when a branch is selected; selectedKey // when a leaf is. Mutually exclusive. const [selectedKey, setSelectedKey] = useState(null); @@ -176,14 +175,7 @@ export default function KVApp() { onClick={() => setReloadToken((n) => n + 1)} title="refresh tree" >Refresh - + pushToast(makeToast('error', msg))} /> {breadcrumb && breadcrumb.length > 0 && ( @@ -284,28 +276,6 @@ export default function KVApp() { - {snapshotOpen && ( -
setSnapshotOpen(false)} - > -
e.stopPropagation()} - role="dialog" - aria-modal="true" - > - - -
-
- )} - ); diff --git a/editor/src/kv/KVLeafEditor.tsx b/editor/src/kv/KVLeafEditor.tsx index 1240a55..6846406 100644 --- a/editor/src/kv/KVLeafEditor.tsx +++ b/editor/src/kv/KVLeafEditor.tsx @@ -5,9 +5,10 @@ import { ALL_TYPES, describeType, detectType } from './typeHeuristics'; import { lookupSchemaForKey } from './schemaLookup'; import { LeafValueEditor } from './leafRenderers'; import { FieldType } from './types'; +import { ClaimsPanel } from './ClaimsPanel'; interface Props { - // Full Redis key path of the selected leaf. + // Full key path of the selected leaf (file:/ = merged view). selectedKey: string; // Bumped after a delete or external change to force refetch. reloadToken: number; @@ -71,7 +72,7 @@ export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onE }; const del = async () => { - if (!confirm(`Delete ${selectedKey}?`)) return; + if (!confirm(`Retract your vote on ${selectedKey}? Other signers' claims keep the merged value.`)) return; setSaving(true); try { await KVAPI.del(selectedKey); @@ -136,9 +137,11 @@ export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onE className="kv-btn kv-btn-danger" onClick={del} disabled={saving} - >Delete + >Retract )} + + ); } diff --git a/editor/src/kv/SignerPicker.tsx b/editor/src/kv/SignerPicker.tsx new file mode 100644 index 0000000..1b56715 --- /dev/null +++ b/editor/src/kv/SignerPicker.tsx @@ -0,0 +1,36 @@ +import { useEffect, useState } from 'react'; +import { Account, ClaimsAPI, getSigner, setSigner } from '../api/claimsApi'; + +// SignerPicker chooses the identity account whose key signs the editor's +// edits. Every save is that account's claim — it wins in that account's view. +export function SignerPicker({ onError }: { onError: (msg: string) => void }) { + const [accounts, setAccounts] = useState([]); + const [uid, setUid] = useState(getSigner()); + + useEffect(() => { + ClaimsAPI.accounts() + .then((list) => { + setAccounts(list); + if (!uid && list.length === 1) { + setSigner(list[0].uid); + setUid(list[0].uid); + } + }) + .catch((e) => onError(`accounts: ${e.message}`)); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + return ( + + ); +} diff --git a/internal/api/cid_resolve.go b/internal/api/cid_resolve.go index 9bc445b..91279e0 100644 --- a/internal/api/cid_resolve.go +++ b/internal/api/cid_resolve.go @@ -56,6 +56,12 @@ func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) { return } + if as := r.URL.Query().Get("as"); as != "" { + if flat, err := s.storage.GetMetadataFlatAs(uuid, as); err == nil && flat != nil { + doc.Flat = flat + } + } + // Response envelope mirrors GetMetadataDocument JSON tags (metadata / // cids / canonical_cid / duplicates), with cid added so the caller can // correlate when they fan out N lookups. canonical_cid is derived by diff --git a/internal/api/claims.go b/internal/api/claims.go new file mode 100644 index 0000000..8b970b2 --- /dev/null +++ b/internal/api/claims.go @@ -0,0 +1,172 @@ +package api + +// Signed claims (meta-core v2.0.0, docs/implementation/metacore-claims-model.md). +// +// POST /api/claims the only metadata write path +// GET /api/meta/{cid}/claims every live claim of the cid's group +// GET /api/metadata/{rid}/claims the same, addressed by rid +// POST /api/merge stateless merge of claims the caller holds +// +// meta-core never signs a claim here: it verifies each one (MMUR2, low-S) and +// the signature is the write authentication. + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/storage" +) + +// GoneResponse is the body of every removed unsigned write route. +type GoneResponse struct { + Error string `json:"error"` + Use string `json:"use"` +} + +// handleGone answers the unsigned write routes removed in v2.0.0. +func (s *Server) handleGone(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusGone, GoneResponse{Error: "gone", Use: "POST /api/claims"}) +} + +// decodeClaims accepts a JSON array of claims or {"claims": [...]}. +func decodeClaims(r *http.Request) ([]claim.Claim, error) { + var raw json.RawMessage + if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&raw); err != nil { + return nil, err + } + var claims []claim.Claim + if len(raw) > 0 && raw[0] == '{' { + var env struct { + Claims []claim.Claim `json:"claims"` + } + if err := json.Unmarshal(raw, &env); err != nil { + return nil, err + } + return env.Claims, nil + } + if err := json.Unmarshal(raw, &claims); err != nil { + return nil, err + } + return claims, nil +} + +// handlePostClaims handles POST /api/claims: one status per claim, in input +// order. 200 even when some claims are refused — the statuses say which. +func (s *Server) handlePostClaims(w http.ResponseWriter, r *http.Request) { + if !s.storage.IsConnected() { + writeError(w, http.StatusServiceUnavailable, "storage not connected") + return + } + claims, err := decodeClaims(r) + if err != nil { + writeErrorSlug(w, http.StatusBadRequest, "invalid_json", "body must be a JSON array of claims: "+err.Error(), false) + return + } + if len(claims) > storage.MaxClaimsPerBatch { + writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per batch", false) + return + } + results, err := s.storage.WriteClaims(r.Context(), claims) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, results) +} + +// ClaimsResponse is the body of the claims read routes: the record's rid, the +// cids of its group, and every live signed claim on them. +type ClaimsResponse struct { + RID string `json:"rid"` + CIDs []string `json:"cids"` + Claims []claim.Claim `json:"claims"` +} + +// handleClaimsByCID handles GET /api/meta/{cid}/claims[?key=…]. +func (s *Server) handleClaimsByCID(w http.ResponseWriter, r *http.Request) { + cid := mux.Vars(r)["cid"] + if !s.storage.IsConnected() { + writeError(w, http.StatusServiceUnavailable, "storage not connected") + return + } + rid, err := s.storage.GetByCID(cid) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if rid == "" { + writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this CID", false) + return + } + s.writeClaims(w, r, rid) +} + +// handleClaimsByRID handles GET /api/metadata/{rid}/claims[?key=…]. The rid +// may also be a cid (resolved like every /api/metadata read). +func (s *Server) handleClaimsByRID(w http.ResponseWriter, r *http.Request) { + if !s.storage.IsConnected() { + writeError(w, http.StatusServiceUnavailable, "storage not connected") + return + } + rid := s.storage.ResolveRoot(mux.Vars(r)["hashId"]) + s.writeClaims(w, r, rid) +} + +// writeClaims answers both claims routes: ?key= keeps one exact key, ?uid= +// one signer; 404 when the record does not exist. +func (s *Server) writeClaims(w http.ResponseWriter, r *http.Request, rid string) { + cids, claims, err := s.storage.RecordClaims(rid, r.URL.Query().Get("key")) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if len(cids) == 0 { + writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this rid or cid", false) + return + } + out := make([]claim.Claim, 0, len(claims)) + uid := r.URL.Query().Get("uid") + for _, cl := range claims { + if uid == "" || cl.UID == uid { + out = append(out, cl) + } + } + writeJSON(w, http.StatusOK, ClaimsResponse{RID: rid, CIDs: cids, Claims: out}) +} + +// MergeRequest is the body of POST /api/merge. +type MergeRequest struct { + Claims []claim.Claim `json:"claims"` + As string `json:"as,omitempty"` +} + +// MergeResponse is the merged field map, the number of agreeing signers per +// key, and how many claims failed verification (dropped, never an error). +type MergeResponse struct { + Fields map[string]string `json:"fields"` + Nsrc map[string]int `json:"nsrc"` + Rejected int `json:"rejected"` +} + +// handleMerge handles POST /api/merge: verify, merge per key with the same +// rule the cache uses, overlay `as`'s own votes held by this core. No write. +func (s *Server) handleMerge(w http.ResponseWriter, r *http.Request) { + var req MergeRequest + if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&req); err != nil { + writeErrorSlug(w, http.StatusBadRequest, "invalid_json", err.Error(), false) + return + } + if len(req.Claims) > storage.MaxClaimsPerBatch { + writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per merge", false) + return + } + fields, nsrc, rejected, err := s.storage.MergeClaims(req.Claims, req.As) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, MergeResponse{Fields: fields, Nsrc: nsrc, Rejected: rejected}) +} diff --git a/internal/api/claims_test.go b/internal/api/claims_test.go new file mode 100644 index 0000000..d206ea3 --- /dev/null +++ b/internal/api/claims_test.go @@ -0,0 +1,182 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/storage" +) + +func withVars(r *http.Request, vars map[string]string) *http.Request { return mux.SetURLVars(r, vars) } + +// routedServer wires the real route table, so the test sees what clients see. +func routedServer(t *testing.T) *Server { + t.Helper() + s := &Server{storage: newTestStorage(t), router: mux.NewRouter()} + s.setupRoutes() + return s +} + +func call(t *testing.T, s *Server, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + rr := httptest.NewRecorder() + s.router.ServeHTTP(rr, req) + return rr +} + +// Every unsigned write route of v1.x answers 410 with a pointer to the claims. +func TestUnsignedWritesAreGone(t *testing.T) { + s := routedServer(t) + for _, c := range []struct{ method, path string }{ + {"PUT", "/meta/bafy"}, {"PATCH", "/meta/bafy"}, {"DELETE", "/meta/bafy"}, + {"POST", "/meta/bafy/_add/genres"}, {"PUT", "/meta/bafy/title"}, {"DELETE", "/meta/bafy/title"}, + {"PUT", "/api/metadata/bafy"}, {"DELETE", "/api/metadata/bafy"}, {"PUT", "/api/metadata/bafy/property"}, + {"POST", "/api/metadata/batch"}, + } { + rr := call(t, s, c.method, c.path, `{"title":"x"}`) + if rr.Code != http.StatusGone || !strings.Contains(rr.Body.String(), `"use":"POST /api/claims"`) { + t.Errorf("%s %s = %d %s", c.method, c.path, rr.Code, rr.Body) + } + } + if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"file:R/title","value":"x"}`); rr.Code != http.StatusConflict { + t.Errorf("raw record write via /api/kv/value = %d", rr.Code) + } + if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"btmagnet:abc","value":"magnet:"}`); rr.Code != http.StatusOK { + t.Errorf("service key write = %d %s", rr.Code, rr.Body) + } +} + +func TestPostClaimsAndReadBack(t *testing.T) { + s := routedServer(t) + a, b := testSigner(t, 1), testSigner(t, 2) + bad := signed(t, b, "bafyA", "year", "1999", 5) + bad.Value = claim.Text("2001") + body, _ := json.Marshal([]claim.Claim{ + signed(t, a, "bafyA", "title", "Matrix", 5), + signed(t, b, "bafyA", "title", "Matrix", 6), + bad, + }) + rr := call(t, s, "POST", "/api/claims", string(body)) + if rr.Code != http.StatusOK { + t.Fatalf("POST /api/claims = %d %s", rr.Code, rr.Body) + } + var res []storage.ClaimResult + _ = json.Unmarshal(rr.Body.Bytes(), &res) + if len(res) != 3 || res[0].Status != "accepted" || res[1].Status != "accepted" || res[2].Status != "bad_signature" { + t.Fatalf("statuses %+v", res) + } + + rr = call(t, s, "GET", "/api/meta/bafyA", "") + if !strings.Contains(rr.Body.String(), `"title":"Matrix"`) { + t.Fatalf("merged read = %s", rr.Body) + } + rr = call(t, s, "GET", "/api/meta/bafyA/claims", "") + var cr ClaimsResponse + _ = json.Unmarshal(rr.Body.Bytes(), &cr) + if rr.Code != http.StatusOK || len(cr.Claims) != 2 { + t.Fatalf("claims by cid = %d %s", rr.Code, rr.Body) + } + if cr.RID == "" || len(cr.CIDs) != 1 || cr.CIDs[0] != "bafyA" { + t.Fatalf("claims envelope rid/cids = %q %v", cr.RID, cr.CIDs) + } + rr = call(t, s, "GET", "/api/metadata/"+cr.RID+"/claims?key=title&uid="+a.UID, "") + var byRid ClaimsResponse + _ = json.Unmarshal(rr.Body.Bytes(), &byRid) + if rr.Code != http.StatusOK || len(byRid.Claims) != 1 || byRid.Claims[0].UID != a.UID || byRid.RID != cr.RID { + t.Fatalf("claims by rid with key+uid = %d %s", rr.Code, rr.Body) + } + if rr := call(t, s, "GET", "/api/meta/bafyA/claims?key=nope", ""); !strings.Contains(rr.Body.String(), `"claims":[]`) { + t.Fatalf("key filter = %s", rr.Body) + } + if rr := call(t, s, "GET", "/api/meta/bafyUnknown/claims", ""); rr.Code != http.StatusNotFound { + t.Fatalf("unknown cid claims = %d", rr.Code) + } + if rr := call(t, s, "GET", "/api/metadata/01NOSUCHRID/claims", ""); rr.Code != http.StatusNotFound { + t.Fatalf("unknown rid claims = %d", rr.Code) + } + + // ?as= overlays the viewer's own vote. + me := testSigner(t, 9) + storeClaims(t, s.storage, signed(t, me, "bafyA", "title", "My title", 7)) + if rr := call(t, s, "GET", "/api/meta/bafyA?as="+me.UID, ""); !strings.Contains(rr.Body.String(), `"title":"My title"`) { + t.Fatalf("?as overlay = %s", rr.Body) + } + if rr := call(t, s, "GET", "/api/meta/bafyA", ""); !strings.Contains(rr.Body.String(), `"title":"Matrix"`) { + t.Fatalf("shared view changed by a single viewer: %s", rr.Body) + } +} + +func TestMergeEndpoint(t *testing.T) { + s := routedServer(t) + a, b, c := testSigner(t, 1), testSigner(t, 2), testSigner(t, 3) + bad := signed(t, c, "bafyX", "title", "Forged", 1) + bad.UID = a.UID + body, _ := json.Marshal(MergeRequest{Claims: []claim.Claim{ + signed(t, a, "bafyX", "title", "Dune", 2), + signed(t, b, "bafyX", "title", "Dune", 3), + signed(t, c, "bafyX", "title", "Dune Part One", 1), + bad, + }}) + rr := call(t, s, "POST", "/api/merge", string(body)) + var mr MergeResponse + _ = json.Unmarshal(rr.Body.Bytes(), &mr) + if rr.Code != http.StatusOK || mr.Fields["title"] != "Dune" || mr.Nsrc["title"] != 2 || mr.Rejected != 1 { + t.Fatalf("merge = %d %s", rr.Code, rr.Body) + } + if !strings.Contains(rr.Body.String(), `"rejected":1`) || !strings.Contains(rr.Body.String(), `"nsrc":{"title":2}`) { + t.Fatalf("merge wire shape = %s", rr.Body) + } + + // `as`: the viewer's own vote in the request wins… + me := testSigner(t, 9) + body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{ + signed(t, a, "bafyX", "title", "Dune", 2), + signed(t, b, "bafyX", "title", "Dune", 3), + signed(t, me, "bafyX", "title", "Dune (my cut)", 4), + }}) + _ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr) + if mr.Fields["title"] != "Dune (my cut)" || mr.Nsrc["title"] != 1 { + t.Fatalf("request overlay = %+v", mr) + } + // …and so do its votes held by this core for the same cids. + storeClaims(t, s.storage, signed(t, me, "bafyX", "year", "2021", 5)) + body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{ + signed(t, a, "bafyX", "title", "Dune", 2), + signed(t, a, "bafyX", "year", "1984", 2), + }}) + _ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr) + if mr.Fields["year"] != "2021" || mr.Fields["title"] != "Dune" { + t.Fatalf("local overlay = %+v", mr) + } + // An all-invalid batch is still 200 with everything rejected. + bad2 := signed(t, a, "bafyX", "title", "x", 9) + bad2.Sig = bad2.Sig[:10] + body, _ = json.Marshal(MergeRequest{Claims: []claim.Claim{bad2}}) + rr = call(t, s, "POST", "/api/merge", string(body)) + if rr.Code != http.StatusOK || !strings.Contains(rr.Body.String(), `"rejected":1`) { + t.Fatalf("all-invalid merge = %d %s", rr.Code, rr.Body) + } +} + +func TestTooManyClaimsIs413(t *testing.T) { + s := routedServer(t) + var sb strings.Builder + sb.WriteString("[") + for i := 0; i <= storage.MaxClaimsPerBatch; i++ { + if i > 0 { + sb.WriteString(",") + } + sb.WriteString(`{"cid":"c","uid":"u","key":"k","value":null,"ts":1,"sig":""}`) + } + sb.WriteString("]") + if rr := call(t, s, "POST", "/api/claims", sb.String()); rr.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized batch = %d", rr.Code) + } +} diff --git a/internal/api/deprecated_field_test.go b/internal/api/deprecated_field_test.go deleted file mode 100644 index 6cea458..0000000 --- a/internal/api/deprecated_field_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" -) - -// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the -// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They -// must be rejected at the write boundary rather than stored. -// -// Why a hard 400 and not a silent rewrite: the reverse-index hook -// (maybeAddAliasFromFieldLocked) only registers an alias for a field named -// `cids/`. A `cid_*` field was therefore written verbatim and never -// indexed — the record silently became unresolvable by that CID. A loud -// failure is strictly better than a record that looks correct and cannot be -// found. -func TestRejectDeprecatedField(t *testing.T) { - rejected := []string{ - "cid_midhash256", - "cid_sha2-256", - "cid_btih_v2", - "cid_crc32", - "midhash256", - "canonical_cid", - } - for _, f := range rejected { - if !rejectDeprecatedField(f) { - t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f) - } - } - - accepted := []string{ - "cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4", - "filePath", - "title", - "fileinfo/duration", - "tmdb/poster", - // Near-misses that must NOT be caught: the prefix is `cid_`, with an - // underscore. A field merely starting with "cid" is fine. - "cids", - "cidrank", - } - for _, f := range accepted { - if rejectDeprecatedField(f) { - t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f) - } - } -} - -func TestValidateFields_Rejects400WithSlug(t *testing.T) { - rr := httptest.NewRecorder() - blocked := validateFields(rr, map[string]string{ - "title": "Inception", - "cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4", - }) - - if !blocked { - t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed") - } - if rr.Code != http.StatusBadRequest { - t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest) - } - - var body map[string]interface{} - if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { - t.Fatalf("response is not JSON: %v", err) - } - if got := body["error"]; got != ErrDeprecatedField { - t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField) - } -} - -func TestValidateFields_AllowsCidsKeyset(t *testing.T) { - rr := httptest.NewRecorder() - blocked := validateFields(rr, map[string]string{ - "filePath": "/watch/Inception.mkv", - "cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true", - "cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua": "true", - }) - if blocked { - t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String()) - } -} diff --git a/internal/api/files_tuples_test.go b/internal/api/files_tuples_test.go index caa6ee8..51ab6ce 100644 --- a/internal/api/files_tuples_test.go +++ b/internal/api/files_tuples_test.go @@ -28,25 +28,25 @@ func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesRespon func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string { t.Helper() - id, err := c.Mint(path, size, mtime) + id, _, err := c.RegisterFileTuple(path, size, mtime, "bafkmid"+path) if err != nil { - t.Fatalf("mint %s: %v", path, err) + t.Fatalf("register %s: %v", path, err) } return id } -// Only records that are actually files count: a gateway record and a record -// that merely carries a path + size (no mtimeNano) are both left out. +// Only local files count: a gateway record — even one whose claims carry a +// filePath and size — is left out. func TestFileTuples_OnlyFileBackedRoots(t *testing.T) { s, c := newFileTuplesTestServer(t) a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1) b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2) - if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil { - t.Fatal(err) - } - if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil { - t.Fatal(err) - } + gw := testSigner(t, 1) + storeClaims(t, c, + signed(t, gw, "bagcsgatewayrecord", "title", "Some Release", 1), + signed(t, gw, "bafkpartial", "filePath", "/files/cache/x.mkv", 1), + signed(t, gw, "bafkpartial", "sizeByte", "999", 1), + ) code, resp := getFileTuples(t, s, "") if code != http.StatusOK { diff --git a/internal/api/handlers.go b/internal/api/handlers.go index a602a10..cfde1c3 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -4,7 +4,6 @@ import ( "crypto/sha256" "encoding/base32" "encoding/json" - "fmt" "io" "net/http" "os" @@ -89,42 +88,6 @@ const ( ErrDeprecatedField = "deprecated_field" ) -// rejectDeprecatedField reports whether a field name is one of the removed -// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …). -// -// This is a tripwire, not a compatibility shim. Such a field used to be -// accepted and written verbatim — but the reverse-index hook -// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named -// `cids/`, so a `cid_*` field was stored and then NEVER INDEXED: the -// record silently became unresolvable by that CID. Failing the write loudly is -// strictly better than a record that looks fine and cannot be found. -// -// Writers must emit the bare-CID key-set instead: `cids/` = "true". -// See METADATA_KEYS.md §2/§14.13. -func rejectDeprecatedField(field string) bool { - return strings.HasPrefix(field, "cid_") || - field == "midhash256" || - field == "canonical_cid" -} - -// validateFields rejects a write carrying any deprecated CID field. Returns -// true if the request was rejected (the handler must return immediately). -func validateFields(w http.ResponseWriter, metadata map[string]string) bool { - for field := range metadata { - if rejectDeprecatedField(field) { - writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField, - fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+ - "write the bare-CID key-set instead: \"cids/\": \"true\" "+ - "(a cid_* field would be stored but never indexed, leaving the "+ - "record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.", - field), - false) - return true - } - } - return false -} - // defaultSlugFor maps an HTTP status code to the most likely error slug. // Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid" // instead of "unknown_root" on 404) call writeErrorSlug directly. @@ -250,7 +213,7 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) { // the watcher minted the root, regardless of which CID the caller knows. hashID = s.storage.ResolveRoot(hashID) - metadata, err := s.storage.GetMetadataFlat(hashID) + metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as")) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return @@ -269,76 +232,6 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, response) } -// handlePutMeta handles PUT /meta/{hash} -func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // CID → UUID resolution so writes land on the canonical root, not on - // a parallel midhash-rooted entry the watcher's UUID never sees. - hashID = s.storage.ResolveRoot(hashID) - - var metadata map[string]string - if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - - if validateFields(w, metadata) { - return - } - - if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - }) -} - -// handleDeleteMeta handles DELETE /meta/{hash} -func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - hashID = s.storage.ResolveRoot(hashID) - - deleted, err := s.storage.DeleteMetadata(hashID) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - "deleted": deleted, - }) -} - // handleListMeta handles GET /meta func (s *Server) handleListMeta(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { @@ -623,50 +516,6 @@ func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, ret }) } -// handlePatchMeta handles PATCH /meta/{hash} -// Merges the provided metadata into existing (does not delete missing keys) -func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // PATCH is the hot write path for meta-sort. Resolve so plugin output - // merges into the watcher's UUID root rather than minting a parallel - // midhash-rooted entry. - hashID = s.storage.ResolveRoot(hashID) - - var metadata map[string]string - if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - - if validateFields(w, metadata) { - return - } - - updated, err := s.storage.MergeMetadataFlat(hashID, metadata) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - "updated": updated, - }) -} - // handleGetProperty handles GET /meta/{hash}/{key} // Gets a single property value func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) { @@ -713,171 +562,6 @@ type PropertyBody struct { Value string `json:"value"` } -// handlePutProperty handles PUT /meta/{hash}/{key} -// Sets a single property value -// Accepts JSON body with {"value": "..."} or plain text -func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - key := vars["key"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if key == "" { - writeError(w, http.StatusBadRequest, "key is required") - return - } - - if validateFields(w, map[string]string{key: ""}) { - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - hashID = s.storage.ResolveRoot(hashID) - - // Read body - buf := make([]byte, 1024*1024) // 1MB max - n, err := r.Body.Read(buf) - if err != nil && err.Error() != "EOF" { - writeError(w, http.StatusBadRequest, "failed to read body") - return - } - body := buf[:n] - - // Try to parse as JSON {"value": "..."} first - var propBody PropertyBody - var value string - if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" { - // JSON body with value field - value = propBody.Value - } else { - // Plain text body - value = string(body) - } - - if err := s.storage.SetProperty(hashID, key, value); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - "property": key, - }) -} - -// handleDeleteProperty handles DELETE /meta/{hash}/{key} -// Deletes a single property -func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - key := vars["key"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if key == "" { - writeError(w, http.StatusBadRequest, "key is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - hashID = s.storage.ResolveRoot(hashID) - - if err := s.storage.DeleteProperty(hashID, key); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - "property": key, - }) -} - -// handleAddToSet handles POST /meta/{hash}/_add/{key} -// Adds a value to a set-type field -// Accepts JSON body with {"value": "..."} or plain text -func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hash"] - key := vars["key"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash is required") - return - } - - if key == "" { - writeError(w, http.StatusBadRequest, "key is required") - return - } - - if validateFields(w, map[string]string{key: ""}) { - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - hashID = s.storage.ResolveRoot(hashID) - - // Read body - buf := make([]byte, 1024*1024) // 1MB max - n, err := r.Body.Read(buf) - if err != nil && err.Error() != "EOF" { - writeError(w, http.StatusBadRequest, "failed to read body") - return - } - body := buf[:n] - - // Try to parse as JSON {"value": "..."} first - var propBody PropertyBody - var value string - if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" { - // JSON body with value field - value = propBody.Value - } else { - // Plain text body - value = string(body) - } - - if value == "" { - writeError(w, http.StatusBadRequest, "value is required") - return - } - - added, err := s.storage.AddToSet(hashID, key, value) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "hashId": hashID, - "property": key, - "added": added, - }) -} - // CIDRequest is the request body for POST /file/cid type CIDRequest struct { Path string `json:"path"` diff --git a/internal/api/handlers_identity.go b/internal/api/handlers_identity.go index c08fb77..ccff2ba 100644 --- a/internal/api/handlers_identity.go +++ b/internal/api/handlers_identity.go @@ -4,11 +4,14 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "log" "net/http" "strconv" "strings" + "time" + "github.com/metazla/meta-core/internal/claim" "github.com/metazla/meta-core/internal/identity" "github.com/metazla/meta-core/internal/storage" ) @@ -120,6 +123,9 @@ type identityDeleteResponse struct { type identitySignRequest struct { BytesB64 string `json:"bytesB64"` + // Format is "der" or "compact" (64-byte r||s, low-S). Default: compact for + // MMUR2 claim payloads ("MMUR2\n" prefix), DER otherwise (MMUR1 back-compat). + Format string `json:"format,omitempty"` // UID selects which account signs. Optional — when omitted and exactly one // account exists, that account is used (single-identity back-compat). UID string `json:"uid,omitempty"` @@ -143,6 +149,8 @@ type identitySignBatchRequest struct { // UID selects which account signs — same optional single-identity // back-compat as identitySignRequest. UID string `json:"uid,omitempty"` + // Format: see identitySignRequest.Format; applies to every payload. + Format string `json:"format,omitempty"` } type identitySignBatchResponse struct { @@ -445,10 +453,10 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) { "storage is not connected; refusing to delete the key while its data cannot be purged", true) return } - purged, err := s.storage.UDLPurgeUser(id.UID) + purged, err := s.retractAllClaims(r, id) if err != nil { writeErrorSlug(w, http.StatusInternalServerError, "purge_failed", - "could not purge this account's data; the account was NOT deleted: "+err.Error(), true) + "could not retract this account's claims; the account was NOT deleted: "+err.Error(), true) return } @@ -456,7 +464,7 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) { writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false) return } - log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids", + log.Printf("[API] Deleted account %s after retracting %d claims across %d cids", id.UID, purged.Records, purged.Cids) writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged}) } @@ -485,9 +493,9 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) { writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false) return } - sig, err := identity.Sign(id, raw) + sig, err := signPayload(id, raw, body.Format) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeError(w, http.StatusBadRequest, err.Error()) return } writeJSON(w, http.StatusOK, identitySignResponse{ @@ -537,9 +545,9 @@ func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) "payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false) return } - sig, err := identity.Sign(id, raw) + sig, err := signPayload(id, raw, body.Format) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"]: "+err.Error()) return } sigs = append(sigs, base64.StdEncoding.EncodeToString(sig)) @@ -562,3 +570,61 @@ func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) { AEADKeyB64: base64.StdEncoding.EncodeToString(key), }) } + +// signPayload signs raw with the account key in the requested encoding. +func signPayload(id *identity.Identity, raw []byte, format string) ([]byte, error) { + switch format { + case "compact": + return identity.SignCompact(id, raw) + case "der": + return identity.Sign(id, raw) + case "": + if strings.HasPrefix(string(raw), claim.Domain) { + return identity.SignCompact(id, raw) + } + return identity.Sign(id, raw) + } + return nil, fmt.Errorf("format must be \"compact\" or \"der\"") +} + +// retractAllClaims signs a retraction for every live claim the account holds +// and stores them (G11): the tombstones stay, so the votes cannot be replayed. +func (s *Server) retractAllClaims(r *http.Request, id *identity.Identity) (storage.UDLUserStats, error) { + refs, err := s.storage.LiveClaimsBySigner(id.UID) + if err != nil { + return storage.UDLUserStats{}, err + } + cids, keys := map[string]bool{}, map[string]bool{} + now := uint64(time.Now().UnixMilli()) + batch := make([]claim.Claim, 0, len(refs)) + for _, ref := range refs { + cur, ok, err := s.storage.UDLGetClaim(id.UID, ref[0], ref[1]) + if err != nil { + return storage.UDLUserStats{}, err + } + ts := now + if ok && cur.TS >= ts { + ts = cur.TS + 1 + } + tomb := claim.Claim{CID: ref[0], UID: id.UID, Key: ref[1], Value: claim.Null(), TS: ts} + if err := tomb.Sign(id); err != nil { + return storage.UDLUserStats{}, err + } + batch = append(batch, tomb) + cids[ref[0]], keys[ref[1]] = true, true + } + for start := 0; start < len(batch); start += storage.MaxClaimsPerBatch { + end := min(start+storage.MaxClaimsPerBatch, len(batch)) + results, err := s.storage.WriteClaims(r.Context(), batch[start:end]) + if err != nil { + return storage.UDLUserStats{}, err + } + for i, res := range results { + if res.Status != storage.StatusAccepted && res.Status != storage.StatusUnchanged { + return storage.UDLUserStats{}, fmt.Errorf("retraction of %s/%s refused: %s %s", + batch[start+i].CID, batch[start+i].Key, res.Status, res.Reason) + } + } + } + return storage.UDLUserStats{Records: len(batch), Cids: len(cids), Keys: len(keys)}, nil +} diff --git a/internal/api/identity_gate_test.go b/internal/api/identity_gate_test.go index 7bc1d56..4b8b23e 100644 --- a/internal/api/identity_gate_test.go +++ b/internal/api/identity_gate_test.go @@ -96,12 +96,16 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) { uid := gen["uid"].(string) priv := gen["privateKeyHex"].(string) - // Some data to lose. - for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} { - if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok { - t.Fatalf("seed: %v", err) - } + // Some data to lose: claims signed by the account. + acct, err := identity.Import(priv) + if err != nil { + t.Fatal(err) } + storeClaims(t, s.storage, + signed(t, acct, "cidA", "like", "true", 100), + signed(t, acct, "cidA", "userRating", "5", 100), + signed(t, acct, "cidB", "seek", []byte("enc"), 100), + ) // The count the UI shows in the confirmation. rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil) @@ -175,7 +179,7 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) { if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) { t.Fatal("the account file survived the delete") } - // Data gone — the orphaning this whole change exists to prevent. + // Every claim retracted (tombstones kept) — nothing left to orphan. after, err := s.storage.UDLAllUserStats() if err != nil { t.Fatalf("stats after: %v", err) diff --git a/internal/api/kv.go b/internal/api/kv.go index 7c4b3df..e5419c3 100644 --- a/internal/api/kv.go +++ b/internal/api/kv.go @@ -383,8 +383,9 @@ func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) { }) } -// handleKVValuePut handles PUT /api/kv/value with body {key, value}. -// Maintains the file index when the key matches file:/... +// handleKVValuePut handles PUT /api/kv/value with body {key, value}: service +// scratch KV only (btmagnet:*, searchcov:*), kept in the cache. Merged records +// and bookkeeping are refused — metadata is written as signed claims. func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { writeError(w, http.StatusServiceUnavailable, "storage not connected") @@ -401,19 +402,11 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) { return } - if err := s.storage.SetRawValue(req.Key, req.Value); err != nil { + if err := s.storage.SetServiceValue(req.Key, req.Value); err != nil { writeError(w, http.StatusConflict, err.Error()) return } - if hashID := extractFileHashID(req.Key); hashID != "" { - if err := s.storage.IndexAdd(hashID); err != nil { - // Non-fatal: the value was written; index entry is best-effort. - // Surface in the log path of writeJSON via response header. - w.Header().Set("X-KV-Index-Warning", err.Error()) - } - } - writeJSON(w, http.StatusOK, KVValueResponse{ Key: req.Key, Type: "string", @@ -422,9 +415,7 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) { }) } -// handleKVValueDelete handles DELETE /api/kv/value?key=... -// When the key matches file:/..., removes the file index entry if no -// sibling keys remain after the delete. +// handleKVValueDelete handles DELETE /api/kv/value?key=... (service keys only). func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { writeError(w, http.StatusServiceUnavailable, "storage not connected") @@ -437,36 +428,14 @@ func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) { return } - deleted, err := s.storage.DeleteRawKey(key) + deleted, err := s.storage.DeleteServiceValue(key) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeError(w, http.StatusConflict, err.Error()) return } - if deleted { - if hashID := extractFileHashID(key); hashID != "" { - if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil { - w.Header().Set("X-KV-Index-Warning", err.Error()) - } - } - } - writeJSON(w, http.StatusOK, KVValueDeleteResponse{ Key: key, Deleted: deleted, }) } - -// extractFileHashID returns the hashID for a key shaped like file:/... -// Returns "" when the key does not follow that convention. -func extractFileHashID(key string) string { - if !strings.HasPrefix(key, "file:") { - return "" - } - rest := strings.TrimPrefix(key, "file:") - slash := strings.Index(rest, "/") - if slash <= 0 { - return "" - } - return rest[:slash] -} diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 0884501..2bb5749 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -38,6 +38,9 @@ type MetadataSearchRequest struct { Property string `json:"property,omitempty"` PropertyValue string `json:"propertyValue,omitempty"` Limit int `json:"limit,omitempty"` + // As is a viewer uid: the search sees that signer's own claims overlaid + // (and their records as candidates). Also accepted as ?as=. + As string `json:"as,omitempty"` // Filters is the structured form: AND across entries, OR within an // entry's Values. Narrowed in SQL through the field index alongside the @@ -222,10 +225,10 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques return } - // Same resolution as the PUT below: callers hold a CID, not the root. + // Callers may hold a CID, not the root. hashID = s.storage.ResolveRoot(hashID) - metadata, err := s.storage.GetMetadataFlat(hashID) + metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as")) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return @@ -244,87 +247,6 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques writeJSON(w, http.StatusOK, response) } -// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId} -func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hashId"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash ID is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // The {hashId} in the URL is whatever CID the caller knows the file by — - // the gateway's auto-store keys this PUT by the hash it just computed - // (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is - // the dag-pb root, not the nzb-release cid the record was minted under. - // Without resolving, the write lands on file:/* and forks a - // second root for a file that already has one: a full duplicate record, - // cross-linked and separately indexed, which every downstream ingest then - // sees as two files. Unknown CIDs pass through unchanged, so a genuinely - // new gateway record still mints its root at the hash. - hashID = s.storage.ResolveRoot(hashID) - - var metadata map[string]string - if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - - // Exclude internal processing status field - delete(metadata, "processingStatus") - - if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "status": "ok", - "hashId": hashID, - "message": "Metadata updated successfully", - }) -} - -// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId} -func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hashId"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash ID is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // meta-gateway retracts a dropped hit by whichever CID it holds - // (meta_core.rs `delete`, which documents this resolution as a - // precondition). Without it, deleting by a non-root CID is a silent - // no-op and the retracted record lingers. - hashID = s.storage.ResolveRoot(hashID) - - deletedCount, err := s.storage.DeleteMetadata(hashID) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "status": "ok", - "hashId": hashID, - "deletedKeys": deletedCount, - }) -} - // handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -364,46 +286,6 @@ func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Reques }) } -// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property -func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - hashID := vars["hashId"] - - if hashID == "" { - writeError(w, http.StatusBadRequest, "hash ID is required") - return - } - - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - var req PropertyUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - - if req.Property == "" { - writeError(w, http.StatusBadRequest, "property is required") - return - } - - if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "status": "ok", - "hashId": hashID, - "property": req.Property, - "value": req.Value, - "message": "Property updated successfully", - }) -} - // handleSearchMetadata handles POST /api/metadata/search func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { @@ -420,6 +302,9 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) { if req.Limit == 0 { req.Limit = 100 } + if req.As == "" { + req.As = r.URL.Query().Get("as") + } // If hashId is provided, do exact match if req.HashID != "" { @@ -457,83 +342,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) { s.serveIndexedSearch(w, req) } -// handleBatchUpdate handles POST /api/metadata/batch -func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - var req MetadataBatchUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - - if len(req.Updates) == 0 { - writeError(w, http.StatusBadRequest, "updates array is required") - return - } - - var results []MetadataBatchResult - successCount := 0 - errorCount := 0 - - for _, update := range req.Updates { - if update.HashID == "" { - results = append(results, MetadataBatchResult{ - HashID: "unknown", - Status: "error", - Error: "Missing hash ID", - }) - errorCount++ - continue - } - - var updateErr error - - // Update complete metadata - if update.Metadata != nil { - // Exclude internal processing status - delete(update.Metadata, "processingStatus") - updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata) - } - - // Update individual properties - if updateErr == nil && update.Properties != nil { - for property, value := range update.Properties { - if err := s.storage.SetProperty(update.HashID, property, value); err != nil { - updateErr = err - break - } - } - } - - if updateErr != nil { - results = append(results, MetadataBatchResult{ - HashID: update.HashID, - Status: "error", - Error: updateErr.Error(), - }) - errorCount++ - } else { - results = append(results, MetadataBatchResult{ - HashID: update.HashID, - Status: "ok", - }) - successCount++ - } - } - - writeJSON(w, http.StatusOK, MetadataBatchResponse{ - Status: "completed", - Total: len(results), - Success: successCount, - Errors: errorCount, - Results: results, - }) -} - // handleClearMetadata handles POST /api/metadata/clear func (s *Server) handleClearMetadata(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { diff --git a/internal/api/search_index.go b/internal/api/search_index.go index 7dedde5..ee185c6 100644 --- a/internal/api/search_index.go +++ b/internal/api/search_index.go @@ -29,7 +29,7 @@ func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchReq for _, f := range filters { narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values}) } - hits, err := s.storage.Search(req.Query, narrow, req.Limit, func(hashID string, fields map[string]string) bool { + hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool { if req.Query != "" && !matchAllTokens(storage.Haystack(fields), tokens) && !strings.Contains(strings.ToLower(hashID), queryLower) { diff --git a/internal/api/server.go b/internal/api/server.go index 4088b63..2e152d4 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -176,7 +176,7 @@ func (s *Server) setupRoutes() { s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET") s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET") s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST") - s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST") + s.router.HandleFunc("/api/metadata/batch", s.handleGone).Methods("POST") s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST") // User Data Layer signing identity (see internal/identity). Auth-gated @@ -192,15 +192,18 @@ func (s *Server) setupRoutes() { s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST") s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET") - // Snapshot (export / import / wipe) - s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET") - s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST") - s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST") + // Signed claims (v2.0.0): the only metadata write path, the raw claims + // behind a record, and the stateless merge clients use for search hits. + s.router.HandleFunc("/api/claims", s.handlePostClaims).Methods("POST") + s.router.HandleFunc("/api/merge", s.handleMerge).Methods("POST") + s.router.HandleFunc("/api/meta/{cid}/claims", s.handleClaimsByCID).Methods("GET") + s.router.HandleFunc("/api/metadata/{hashId}/claims", s.handleClaimsByRID).Methods("GET") + s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET") - s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT") + s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleGone).Methods("PUT") s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET") - s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT") - s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE") + s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("PUT") + s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("DELETE") // Schema inference s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET") @@ -232,18 +235,15 @@ func (s *Server) setupRoutes() { // Metadata operations - base endpoints s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET") - s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT") - s.router.HandleFunc("/meta/{hash}", s.handlePatchMeta).Methods("PATCH") - s.router.HandleFunc("/meta/{hash}", s.handleDeleteMeta).Methods("DELETE") + s.router.HandleFunc("/meta/{hash}", s.handleGone).Methods("PUT", "PATCH", "DELETE") s.router.HandleFunc("/meta", s.handleListMeta).Methods("GET") // Metadata operations - set operations (must be before property routes) - s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleAddToSet).Methods("POST") + s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleGone).Methods("POST") // Metadata operations - property-level (key can contain slashes) s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGetProperty).Methods("GET") - s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handlePutProperty).Methods("PUT") - s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleDeleteProperty).Methods("DELETE") + s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGone).Methods("PUT", "DELETE") // Data operations s.router.HandleFunc("/data/{hash}/path", s.handleGetDataPath).Methods("GET") diff --git a/internal/api/snapshot.go b/internal/api/snapshot.go deleted file mode 100644 index a491b66..0000000 --- a/internal/api/snapshot.go +++ /dev/null @@ -1,167 +0,0 @@ -package api - -import ( - "bytes" - "fmt" - "io" - "net/http" - "path/filepath" - "strings" - "time" - - "github.com/metazla/meta-core/internal/snapshot" -) - -// Bound the upload size so a malicious client can't OOM the server. -// 2 GiB is well above any realistic export. -const maxImportBytes = 2 << 30 - -// pluginOutputDir is where plugin-produced artifacts live on disk -// (subtitles, posters, etc). Mounted from the host into the meta-core -// container at $FILES_PATH/plugin. -func (s *Server) pluginOutputDir() string { - return filepath.Join(s.config.FilesPath, "plugin") -} - -// handleSnapshotExport streams a ZIP snapshot of all metadata. -// GET /api/snapshot/export?include=hash-index,plugin-output -func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // The server has a 30s WriteTimeout (server.go:222). Long exports - // blow past that, leaving a truncated ZIP without a central directory. - // Clear the deadline for this handler so the export can run as long as - // needed. ResponseController is the standard way to do this. - rc := http.NewResponseController(w) - _ = rc.SetWriteDeadline(time.Time{}) - - include := splitCSVParam(r.URL.Query().Get("include")) - opts := snapshot.ExportOptions{ - IncludePluginOutput: include["plugin-output"], - PluginOutputDir: s.pluginOutputDir(), - } - - filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405")) - w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename)) - // We stream — no Content-Length. - - exporter := snapshot.NewExporter(s.storage) - if _, err := exporter.Export(w, opts); err != nil { - // Headers may already be sent; log and bail. - // Client will see a truncated zip, which is the best signal we can give. - fmt.Printf("[Snapshot] export failed: %v\n", err) - return - } -} - -// handleSnapshotImport applies an uploaded ZIP snapshot. -// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true -// Body: application/zip (raw) OR multipart/form-data with field "snapshot" -func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - // Long imports can blow past the server's 30s ReadTimeout/WriteTimeout - // when the snapshot is large. Clear both for this handler. - rc := http.NewResponseController(w) - _ = rc.SetReadDeadline(time.Time{}) - _ = rc.SetWriteDeadline(time.Time{}) - - q := r.URL.Query() - opts := snapshot.ImportOptions{ - Mode: snapshot.Mode(q.Get("mode")), - Conflict: snapshot.Conflict(q.Get("conflict")), - DryRun: strings.EqualFold(q.Get("dry_run"), "true"), - PluginOutputDir: s.pluginOutputDir(), - } - if opts.Mode == "" { - opts.Mode = snapshot.ModeMerge - } - if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" { - opts.Conflict = snapshot.ConflictSource - } - - body, err := readImportBody(r) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - - importer := snapshot.NewImporter(s.storage) - result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - - writeJSON(w, http.StatusOK, result) -} - -// handleSnapshotWipe clears all stored metadata. -// POST /api/snapshot/wipe?scope=metadata -func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - scopes := splitCSVParam(r.URL.Query().Get("scope")) - if len(scopes) == 0 { - writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)") - return - } - scope := snapshot.WipeScope{ - Metadata: scopes["metadata"], - } - if !scope.Metadata { - writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value") - return - } - - wiper := snapshot.NewWiper(s.storage) - res, err := wiper.Wipe(scope) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, http.StatusOK, res) -} - -// readImportBody accepts either a raw application/zip body or a multipart -// upload with a field named "snapshot". Buffers up to maxImportBytes. -func readImportBody(r *http.Request) ([]byte, error) { - r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes) - - ct := r.Header.Get("Content-Type") - if strings.HasPrefix(ct, "multipart/form-data") { - if err := r.ParseMultipartForm(32 << 20); err != nil { - return nil, fmt.Errorf("parse multipart: %w", err) - } - f, _, err := r.FormFile("snapshot") - if err != nil { - return nil, fmt.Errorf("missing 'snapshot' file field: %w", err) - } - defer f.Close() - return io.ReadAll(f) - } - - return io.ReadAll(r.Body) -} - -// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set. -func splitCSVParam(v string) map[string]bool { - out := map[string]bool{} - for _, part := range strings.Split(v, ",") { - p := strings.TrimSpace(part) - if p != "" { - out[p] = true - } - } - return out -} diff --git a/internal/api/sse_events.go b/internal/api/sse_events.go index 43b1dfb..b2d303b 100644 --- a/internal/api/sse_events.go +++ b/internal/api/sse_events.go @@ -112,7 +112,15 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string } cursor = head } else { - cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher) + if rebuilt := s.storage.RebuiltAt(); stream == storage.MetaEventsStream && lastID != "" && rebuilt != "" && + compareStreamIDs(lastID, rebuilt) < 0 { + // The cache was rebuilt after this cursor: the rebuild published no + // per-key events, so the consumer must take a snapshot. + writeGap(w, flusher, lastID, rebuilt, "rebuild") + cursor = "0" + } else { + cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher) + } } lastWrite := time.Now() @@ -256,14 +264,7 @@ func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID str } // Cursor was trimmed. Emit a gap event so consumers that care can react. - gap := map[string]string{ - "requested": lastID, - "resumeFrom": oldest, - "reason": "retention", - } - payload, _ := json.Marshal(gap) - fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload)) - flusher.Flush() + writeGap(w, flusher, lastID, oldest, "retention") // Return "0" so the next read picks up entries starting from the // stream's true minimum — including `oldest` itself. @@ -327,3 +328,15 @@ func parseStreamID(id string) (int64, int64) { } return ms, seq } + +// writeGap emits the synthetic `gap` event: the consumer's cursor cannot be +// resumed losslessly and it should take a snapshot (Last-Event-ID: 0). +func writeGap(w http.ResponseWriter, flusher http.Flusher, requested, resumeFrom, reason string) { + payload, _ := json.Marshal(map[string]string{ + "requested": requested, + "resumeFrom": resumeFrom, + "reason": reason, + }) + fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload)) + flusher.Flush() +} diff --git a/internal/api/sse_snapshot_test.go b/internal/api/sse_snapshot_test.go index b5eacee..fc79454 100644 --- a/internal/api/sse_snapshot_test.go +++ b/internal/api/sse_snapshot_test.go @@ -33,11 +33,11 @@ func (r *syncRecorder) body() string { func TestMetaEventsSnapshotThenTail(t *testing.T) { stor := newTestStorage(t) s := &Server{storage: stor} - for i, id := range []string{"R1", "R2"} { - if err := stor.SetMetadataFlat(id, map[string]string{"title": "T", "n": string(rune('a' + i))}); err != nil { - t.Fatal(err) - } + a := testSigner(t, 1) + for i, cid := range []string{"bafyR1", "bafyR2"} { + storeClaims(t, stor, signed(t, a, cid, "title", "T", 1), signed(t, a, cid, "n", string(rune('a'+i)), 1)) } + r1, r2 := stor.ResolveRoot("bafyR1"), stor.ResolveRoot("bafyR2") ctx, cancel := context.WithCancel(context.Background()) req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx) @@ -56,34 +56,63 @@ func TestMetaEventsSnapshotThenTail(t *testing.T) { time.Sleep(20 * time.Millisecond) } } - waitFor(`"key":"file:R2/title"`) - if err := stor.SetProperty("R1", "title", "changed"); err != nil { - t.Fatal(err) - } + waitFor(`"key":"file:` + r2 + `/title"`) + storeClaims(t, stor, signed(t, a, "bafyR1", "title", "changed", 2)) waitFor(`event: set`) time.Sleep(300 * time.Millisecond) cancel() <-done body := rec.body() - for _, k := range []string{"file:R1/title", "file:R1/n", "file:R2/title", "file:R2/n"} { + keys := []string{"file:" + r1 + "/title", "file:" + r1 + "/n", "file:" + r2 + "/title", "file:" + r2 + "/n", + "file:" + r1 + "/cids/bafyR1", "file:" + r2 + "/cids/bafyR2"} + for _, k := range keys { if !strings.Contains(body, `"key":"`+k+`"`) { t.Errorf("snapshot is missing %s", k) } } events := strings.Split(strings.TrimSpace(body), "\n\n") - if len(events) != 5 { - t.Fatalf("want 4 snapshot events + 1 live event, got %d:\n%s", len(events), body) + if len(events) != 7 { + t.Fatalf("want 6 snapshot events + 1 live event, got %d:\n%s", len(events), body) } - for i, e := range events[:3] { + for i, e := range events[:5] { if strings.Contains(e, "id: ") { t.Errorf("snapshot event %d carries an id: %q", i, e) } } - if !strings.HasPrefix(events[3], "id: ") || !strings.HasPrefix(events[4], "id: ") { + if !strings.HasPrefix(events[5], "id: ") || !strings.HasPrefix(events[6], "id: ") { t.Fatalf("the last snapshot event and the live event must carry ids:\n%s", body) } - if !strings.Contains(events[4], `"key":"file:R1/title"`) { - t.Fatalf("live tail did not follow the snapshot: %q", events[4]) + if !strings.Contains(events[6], `"key":"file:`+r1+`/title"`) { + t.Fatalf("live tail did not follow the snapshot: %q", events[6]) + } +} + +// A cursor older than the last cache rebuild gets `gap` (reason rebuild): the +// rebuild published no per-key events, so the consumer must re-snapshot. +func TestMetaEventsGapAfterRebuild(t *testing.T) { + stor := newTestStorage(t) + s := &Server{storage: stor} + storeClaims(t, stor, signed(t, testSigner(t, 1), "bafyA", "title", "T", 1)) + if err := stor.Rebuild(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx) + req.Header.Set("Last-Event-ID", "1000-0") + rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()} + done := make(chan struct{}) + go func() { s.handleEventsMeta(rec, req); close(done) }() + deadline := time.Now().Add(5 * time.Second) + for !strings.Contains(rec.body(), "event: gap") { + if time.Now().After(deadline) { + t.Fatalf("no gap after rebuild:\n%s", rec.body()) + } + time.Sleep(20 * time.Millisecond) + } + cancel() + <-done + if !strings.Contains(rec.body(), `"reason":"rebuild"`) { + t.Fatalf("gap reason: %s", rec.body()) } } diff --git a/internal/api/stats.go b/internal/api/stats.go index aab6fc1..6988233 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -30,7 +30,7 @@ type fileStatsJSON struct { // cheap (an index range count, a cached row count, a file stat) plus a // directory listing, recomputed on every call. The `redis*` JSON names // predate the move to SQLite and are kept for dashboard compatibility: -// redisKeys is the store's row count, redisMemory its on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file +// redisKeys is the number of claim rows (truth), redisMemory both databases' on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file // tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served // from a background-refreshed cache and are null until the first sweep lands. type StatsResponse struct { @@ -115,7 +115,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { } else { resp.Records = n } - if n, err := s.storage.CountKeys(); err == nil { + if n, err := s.storage.CountClaims(); err == nil { resp.RedisKeys = n } if mem, err := s.storage.GetMemoryInfo(); err == nil { diff --git a/internal/api/storage_helper_test.go b/internal/api/storage_helper_test.go index f318548..4b5761b 100644 --- a/internal/api/storage_helper_test.go +++ b/internal/api/storage_helper_test.go @@ -1,19 +1,68 @@ package api import ( + "context" + "fmt" "path/filepath" "testing" + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/identity" "github.com/metazla/meta-core/internal/storage" ) -// newTestStorage opens a fresh metadata store in the test's temp dir. +// newTestStorage opens (and starts) a fresh store in the test's temp dir. func newTestStorage(t *testing.T) *storage.Client { t.Helper() + dir := t.TempDir() c := storage.NewClient() - if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil { + if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil { t.Fatalf("open store: %v", err) } + if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil { + t.Fatalf("schema: %v", err) + } + if err := c.Start(); err != nil { + t.Fatalf("start: %v", err) + } t.Cleanup(func() { _ = c.Close() }) return c } + +func testSigner(t *testing.T, n byte) *identity.Identity { + t.Helper() + id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1)) + if err != nil { + t.Fatal(err) + } + return id +} + +// signed builds a claim signed by id; v is string, []byte or nil. +func signed(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim { + t.Helper() + cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts} + switch x := v.(type) { + case string: + cl.Value = claim.Text(x) + case []byte: + cl.Value = claim.Bytes(x) + } + if err := cl.Sign(id); err != nil { + t.Fatal(err) + } + return cl +} + +func storeClaims(t *testing.T, c *storage.Client, claims ...claim.Claim) { + t.Helper() + res, err := c.WriteClaims(context.Background(), claims) + if err != nil { + t.Fatal(err) + } + for i, r := range res { + if r.Status != storage.StatusAccepted { + t.Fatalf("claim %d: %s %s", i, r.Status, r.Reason) + } + } +} diff --git a/internal/api/udl.go b/internal/api/udl.go index d5a32d6..e452e77 100644 --- a/internal/api/udl.go +++ b/internal/api/udl.go @@ -1,13 +1,14 @@ package api -// User Data Layer (UDL) HTTP surface. +// User Data Layer facade (/api/udl/*). Since meta-core v2.0.0 a user's data are +// ordinary MMUR2 claims signed by the user's key (plaintext for public-social +// keys, ciphertext for private ones); these routes keep their paths for the +// clients (meta-watch, meta-read, meta-listen, the gateway mirror) and speak +// claims in the §3.4 JSON form: // -// Backs meta-watch's per-user store (My List, Continue Watching, ratings, -// profile) now that the authoritative store lives in meta-core instead of -// meta-watch's local op-log. Records arrive already signed (and, for -// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core -// version-gates and indexes them but never inspects the crypto. See -// internal/storage/udl.go for the key model. +// {"cid","uid","key","value": "…" | {"b64": "…"} | null,"ts","sig"} +// +// A write is the same as POST /api/claims; newest ts wins per (uid, cid, key). import ( "encoding/json" @@ -17,82 +18,46 @@ import ( "strings" "github.com/gorilla/mux" + + "github.com/metazla/meta-core/internal/claim" "github.com/metazla/meta-core/internal/storage" ) -// UDLPutRequest is the body of PUT /api/udl/record. -type UDLPutRequest struct { - Uid string `json:"uid"` - Cid string `json:"cid"` - Key string `json:"key"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` - Record string `json:"record"` // base64 CBOR, opaque + signed - Value *json.RawMessage `json:"value,omitempty"` - // Tombstone marks this write as a deletion of (uid, cid, key). The record - // itself is opaque here — meta-core never decodes the CBOR, and a private- - // tier value and a tombstone are equally "no plaintext value" — so the - // writer has to say. Without it the cid is never removed from - // udl:idx:user::key:, which then grows for the life of the - // account and is returned in full on every My List / Continue Watching - // read. Omitted by older writers, which keeps the previous behaviour. - Tombstone bool `json:"tombstone,omitempty"` -} - -// UDLPutResponse is the response of PUT /api/udl/record. +// UDLPutResponse answers PUT /api/udl/record. type UDLPutResponse struct { - Accepted bool `json:"accepted"` + Accepted bool `json:"accepted"` + Status storage.ClaimStatus `json:"status"` } -// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one -// request. Each entry is exactly a UDLPutRequest, so a client can build the -// same signed record for either endpoint. +// UDLPutBatchRequest is PUT /api/udl/records. type UDLPutBatchRequest struct { - Records []UDLPutRequest `json:"records"` + Records []claim.Claim `json:"records"` } -// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error -// means the write was stale (the single-record endpoint's 409), which in a -// batch is an ordinary outcome rather than a failure. +// UDLPutBatchResult is one entry of the batch response. type UDLPutBatchResult struct { - Key string `json:"key"` - Accepted bool `json:"accepted"` - Error string `json:"error,omitempty"` + Key string `json:"key"` + Accepted bool `json:"accepted"` + Status storage.ClaimStatus `json:"status"` + Error string `json:"error,omitempty"` } -// UDLPutBatchResponse mirrors the request order index-for-index. +// UDLPutBatchResponse answers PUT /api/udl/records. type UDLPutBatchResponse struct { Results []UDLPutBatchResult `json:"results"` } -// maxUDLBatch bounds one batch. Each record is a store write while the -// request is held open; sized to match the sign batch on the other side of the -// same operation (see maxSignBatch). const maxUDLBatch = 1000 -// UDLRecordResponse is the response of GET /api/udl/record. +// UDLRecordResponse answers GET /api/udl/record. Claim is the stored claim, +// tombstone included (its ts is what a newer write must exceed); Exists is +// false when it is absent or a tombstone. type UDLRecordResponse struct { - Record string `json:"record"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` - Exists bool `json:"exists"` - // Value is the plaintext JSON scalar stored beside the opaque record, and - // is present only for PUBLIC-tier keys (like, rating, profile:name, - // profile:avatar). json.RawMessage rather than string because the field is - // generic over keys — a bool for `like`, a number for `rating`, a string - // for a profile name (see aggregatePublicValues) — and because it is the - // exact type UDLPutRequest.Value uses, so a value written through PUT reads - // back byte-identical. - // - // Absent, never null, when there is no plaintext: private-tier cells, - // tombstones and never-written cells must not read as an empty public - // value. Readers key off presence, not `exists`. - Value *json.RawMessage `json:"value,omitempty"` + Exists bool `json:"exists"` + Claim *claim.Claim `json:"claim"` } -// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case -// field names; `avg` is null for boolean keys like `like`), so the meta-watch -// client deserializes it into the same struct it serializes to the UI. +// UDLAggregate is one key's public aggregate on a cid. type UDLAggregate struct { Key string `json:"key"` Count int `json:"count"` @@ -101,10 +66,17 @@ type UDLAggregate struct { SampleSize int `json:"sample_size"` } -// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key -func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) { +func (s *Server) udlReady(w http.ResponseWriter) bool { if !s.storage.IsConnected() { writeError(w, http.StatusServiceUnavailable, "storage not connected") + return false + } + return true +} + +// handleUDLRecordGet handles GET /api/udl/record?uid=&cid=&key=. +func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) { + if !s.udlReady(w) { return } q := r.URL.Query() @@ -113,85 +85,53 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "uid, cid and key are required") return } - - cell, exists, err := s.storage.UDLGetCell(uid, cid, key) + cl, ok, err := s.storage.UDLGetClaim(uid, cid, key) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } - resp := UDLRecordResponse{ - Record: cell.Record, - Version: cell.Version, - Ts: cell.Ts, - Exists: exists, - } - // json.Valid guards the encoder: a malformed `value` hash field (hand-edited - // the store, a future writer bug) would otherwise fail mid-encode and emit a - // truncated body instead of a readable error. - if exists && cell.HasValue && json.Valid([]byte(cell.Value)) { - raw := json.RawMessage(cell.Value) - resp.Value = &raw + resp := UDLRecordResponse{} + if ok { + resp.Claim = &cl + resp.Exists = !cl.Value.IsNull() } writeJSON(w, http.StatusOK, resp) } -// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write -// (version <= stored) returns 409 stale_version so the client can re-read and -// retry with a higher version. +// handleUDLRecordPut handles PUT /api/udl/record with one claim as the body. func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } - var req UDLPutRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") + var cl claim.Claim + if err := json.NewDecoder(r.Body).Decode(&cl); err != nil { + writeError(w, http.StatusBadRequest, "body must be one claim: "+err.Error()) return } - if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" { - writeError(w, http.StatusBadRequest, "uid, cid, key and record are required") - return - } - - publicValue := "" - hasValue := false - if req.Value != nil { - publicValue = string(*req.Value) - hasValue = true - } - - accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone) + results, err := s.storage.WriteClaims(r.Context(), []claim.Claim{cl}) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } - if !accepted { - writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true) - return + res := results[0] + switch res.Status { + case storage.StatusAccepted, storage.StatusUnchanged: + writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true, Status: res.Status}) + case storage.StatusStale: + writeErrorSlug(w, http.StatusConflict, "stale_version", "a claim with an equal or newer ts already exists", true) + default: + writeErrorSlug(w, http.StatusBadRequest, string(res.Status), res.Reason, false) } - writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true}) } -// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of -// handleUDLRecordPut. -// -// Exists for the bulk verbs above this layer ("mark this whole series seen"), -// where one user action produces one cell per episode. Each cell still goes -// through the same version-gated Lua upsert, so the CRDT semantics are -// identical — this batches the transport only. -// -// **Not transactional, and deliberately not 409.** A per-record `accepted` flag -// replaces the single-record conflict status: in a batch, one stale cell is a -// normal outcome (another device already marked that episode) and must not fail -// the 299 cells beside it. Callers read the array, not the status code. +// handleUDLRecordsPut handles PUT /api/udl/records {records: [claim, …]}. func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } var req UDLPutBatchRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") + writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error()) return } if len(req.Records) == 0 { @@ -202,91 +142,66 @@ func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "too many records in one batch") return } - results := make([]UDLPutBatchResult, 0, len(req.Records)) - for _, rec := range req.Records { - if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" { - results = append(results, UDLPutBatchResult{ - Key: rec.Key, - Error: "uid, cid, key and record are required", - }) - continue - } - publicValue := "" - hasValue := false - if rec.Value != nil { - publicValue = string(*rec.Value) - hasValue = true - } - accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone) - if err != nil { - // A storage error is per-record here rather than fatal for the whole - // batch: the cells already written stay written, and reporting which - // ones failed lets the caller retry exactly those. - results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()}) - continue - } - results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted}) + results, err := s.storage.WriteClaims(r.Context(), req.Records) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return } - writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results}) + out := make([]UDLPutBatchResult, len(results)) + for i, res := range results { + out[i] = UDLPutBatchResult{ + Key: req.Records[i].Key, + Status: res.Status, + Accepted: res.Status == storage.StatusAccepted || res.Status == storage.StatusUnchanged, + Error: res.Reason, + } + } + writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: out}) } -// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key} -// (active_for_user_key — My List, Continue Watching). +func (s *Server) writeUDLEntries(w http.ResponseWriter, uid, cid, key string) { + entries, err := s.storage.UDLClaimsForUser(uid, cid, key) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if entries == nil { + entries = []claim.Claim{} + } + writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries}) +} + +// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}: every live claim +// of uid on key (My List, Continue Watching). func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } vars := mux.Vars(r) - uid, key := vars["uid"], vars["key"] - if uid == "" || key == "" { + if vars["uid"] == "" || vars["key"] == "" { writeError(w, http.StatusBadRequest, "uid and key are required") return } - entries, err := s.storage.UDLListUserKey(uid, key) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries}) + s.writeUDLEntries(w, vars["uid"], "", vars["key"]) } -// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid} -// (list_for_user_cid). +// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}. func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } vars := mux.Vars(r) - uid, cid := vars["uid"], vars["cid"] - if uid == "" || cid == "" { + if vars["uid"] == "" || vars["cid"] == "" { writeError(w, http.StatusBadRequest, "uid and cid are required") return } - entries, err := s.storage.UDLListUserCid(uid, cid) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries}) + s.writeUDLEntries(w, vars["uid"], vars["cid"], "") } -// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= -// — every record a uid owns, for cross-instance profile recovery. -// -// The other user-scoped reads answer a question with one coordinate already -// known. Recovery knows only the uid: a box that has just imported a secret key -// has no idea which cids or keys the profile ever touched, so it needs the -// whole set or it gets nothing. -// -// Paged on a cid boundary — `nextCid` empty means the export is complete. The -// response is opaque signed records, exactly as stored; whoever consumes it is -// responsible for verifying each one before trusting it (meta-core has never -// held the account key and cannot verify on anyone's behalf). +// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= — the +// whole profile, paged on cid boundaries (recovery / export). func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } uid := mux.Vars(r)["uid"] @@ -301,31 +216,22 @@ func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "limit must be a positive integer") return } - if n < limit { - limit = n - } + limit = min(n, limit) } entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } - writeJSON(w, http.StatusOK, map[string]interface{}{ - "entries": entries, - "nextCid": nextCid, - }) + if entries == nil { + entries = []claim.Claim{} + } + writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries, "nextCid": nextCid}) } -// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a -// uid holds records for. -// -// Exists so a gateway mirror can enforce a per-profile quota without walking -// anything: 32 random bytes are a valid uid, so an unbounded mirror is a -// disk-fill DoS that needs no account. A quota check that itself cost a scan -// would just move the denial of service one layer up. +// handleUDLUserCount handles GET /api/udl/user/{uid}/count. func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } uid := mux.Vars(r)["uid"] @@ -341,10 +247,10 @@ func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n}) } -// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid). +// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users: signers holding live +// claims on the cid. func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } cid := mux.Vars(r)["cid"] @@ -360,12 +266,9 @@ func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]interface{}{"users": users}) } -// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating. -// Computes per-key population stats from the stored public value fields -// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds. +// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,userRating. func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") + if !s.udlReady(w) { return } cid := mux.Vars(r)["cid"] @@ -375,7 +278,7 @@ func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) { } keysParam := r.URL.Query().Get("keys") if keysParam == "" { - keysParam = "like,rating" + keysParam = "like,userRating" } aggregates := make([]UDLAggregate, 0) for _, key := range strings.Split(keysParam, ",") { diff --git a/internal/api/udl_test.go b/internal/api/udl_test.go index d9c3898..0ac900a 100644 --- a/internal/api/udl_test.go +++ b/internal/api/udl_test.go @@ -6,211 +6,107 @@ import ( "net/http/httptest" "strings" "testing" + + "github.com/metazla/meta-core/internal/claim" ) -// newUDLTestServer returns a Server wired to an in-memory Redis. -// -// The struct is built directly rather than through NewServer: NewServer boots -// the file watcher, mounts manager, watchers poller and WebDAV handler, none of -// which the UDL handlers touch — they use s.storage and nothing else. func newUDLTestServer(t *testing.T) *Server { t.Helper() return &Server{storage: newTestStorage(t)} } -// getRecord drives handleUDLRecordGet and decodes the body into raw JSON -// fields, so a test can assert on the *encoding* of `value` (is it a quoted -// string? is the key there at all?) rather than on what a typed struct would -// have coerced it into. -func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) { +func udlGet(t *testing.T, s *Server, uid, cid, key string) UDLRecordResponse { t.Helper() - req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil) rr := httptest.NewRecorder() s.handleUDLRecordGet(rr, req) - - var body map[string]json.RawMessage - if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { - t.Fatalf("decode response %q: %v", rr.Body.String(), err) + if rr.Code != http.StatusOK { + t.Fatalf("GET record = %d %s", rr.Code, rr.Body) } - return rr.Code, body + var resp UDLRecordResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + return resp } -// The plaintext value of a PUBLIC-tier cell must come back over the wire, and -// come back *as it was written* — a JSON string stays a quoted string, a bool -// stays a bool. -// -// Both halves matter. Without the value at all, the meta-core dashboard can -// only ever show 44-character base58 uids: display names live in the User Data -// Layer as the public cell (uid, "self", "profile:name"), and the plaintext -// sits in Redis where the storage layer has always read it — the handler just -// used to drop it. And the type has to stay json.RawMessage: this endpoint is -// generic over keys, so a `*string` would mangle `like` (bool) and `rating` -// (number) while looking correct for names. -func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) { - s := newUDLTestServer(t) - - const uid = "zAlice" - if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok { - t.Fatalf("seed name: ok=%v err=%v", ok, err) - } - if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok { - t.Fatalf("seed like: ok=%v err=%v", ok, err) - } - - code, body := getRecord(t, s, uid, "self", "profile:name") - if code != http.StatusOK { - t.Fatalf("status = %d, want 200", code) - } - if got := string(body["value"]); got != `"Ada Lovelace"` { - t.Fatalf("value = %s, want a quoted JSON string", got) - } - var name string - if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" { - t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err) - } - - // A boolean key survives unquoted — the assertion a *string field fails. - if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" { - t.Fatalf("like value = %s, want the bare JSON literal true", body["value"]) - } -} - -// A cell with no plaintext must omit `value` entirely rather than send null. -// -// That is the shape of every private-tier key (seek, watched, card, localpath — -// their real value rides encrypted inside the opaque record) and of every -// tombstone. A reader keys off presence: "there is no plaintext here" and "the -// plaintext is null" are different facts, and a name cleared by its owner -// leaves the cell existing with no value, which must not read as a name. -func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) { - s := newUDLTestServer(t) - - const uid = "zBob" - if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok { - t.Fatalf("seed private cell: ok=%v err=%v", ok, err) - } - - code, body := getRecord(t, s, uid, "bagcsaaa", "seek") - if code != http.StatusOK { - t.Fatalf("status = %d, want 200", code) - } - if _, ok := body["value"]; ok { - t.Fatalf("private cell must omit `value`, got %s", body["value"]) - } - if string(body["exists"]) != "true" { - t.Fatalf("exists = %s, want true", body["exists"]) - } -} - -// A cell that was never written answers 200 with exists:false and no value — -// not a 404, and not an empty-string value that would render as a blank name. -func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) { - s := newUDLTestServer(t) - - code, body := getRecord(t, s, "zNobody", "self", "profile:name") - if code != http.StatusOK { - t.Fatalf("status = %d, want 200", code) - } - if string(body["exists"]) != "false" { - t.Fatalf("exists = %s, want false", body["exists"]) - } - if _, ok := body["value"]; ok { - t.Fatalf("missing cell must omit `value`, got %s", body["value"]) - } -} - -// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the -// per-record results. -func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) { +func udlPutBatch(t *testing.T, s *Server, claims ...claim.Claim) UDLPutBatchResponse { t.Helper() - - req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body)) + body, _ := json.Marshal(UDLPutBatchRequest{Records: claims}) + req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(string(body))) rr := httptest.NewRecorder() s.handleUDLRecordsPut(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("PUT records = %d %s", rr.Code, rr.Body) + } + var resp UDLPutBatchResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + return resp +} - var out UDLPutBatchResponse - if rr.Code == http.StatusOK { - if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil { - t.Fatalf("decode response %q: %v", rr.Body.String(), err) +// The facade stores and returns MMUR2 claims: public values in clear, +// private ones as ciphertext, tombstones with their ts. +func TestUDLFacadeRoundTrip(t *testing.T) { + s := newUDLTestServer(t) + me := testSigner(t, 7) + res := udlPutBatch(t, s, + signed(t, me, "self", "profile:name", "Ada Lovelace", 100), + signed(t, me, "bagcsaaa", "like", "true", 100), + signed(t, me, "bagcsaaa", "seek", []byte("ciphertext"), 100), + ) + for _, r := range res.Results { + if !r.Accepted { + t.Fatalf("not accepted: %+v", r) } } - return rr.Code, out + got := udlGet(t, s, me.UID, "self", "profile:name") + if !got.Exists || got.Claim == nil || got.Claim.Value.Text != "Ada Lovelace" || got.Claim.Verify() != nil { + t.Fatalf("profile = %+v", got) + } + if seek := udlGet(t, s, me.UID, "bagcsaaa", "seek"); seek.Claim == nil || seek.Claim.Value.Kind != claim.KindBytes { + t.Fatalf("seek = %+v", seek) + } + if miss := udlGet(t, s, me.UID, "bagcsaaa", "nope"); miss.Exists || miss.Claim != nil { + t.Fatalf("missing cell = %+v", miss) + } + + // Retraction: stored, returned with its ts, no longer "exists". + udlPutBatch(t, s, signed(t, me, "bagcsaaa", "like", nil, 200)) + if tomb := udlGet(t, s, me.UID, "bagcsaaa", "like"); tomb.Exists || tomb.Claim == nil || tomb.Claim.TS != 200 { + t.Fatalf("tombstone = %+v", tomb) + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/udl/cid/self/aggregate?keys=profile:name", nil) + req = withVars(req, map[string]string{"cid": "self"}) + s.handleUDLAggregate(rr, req) + if !strings.Contains(rr.Body.String(), `"sample_size":1`) { + t.Fatalf("aggregate = %s", rr.Body) + } } -// **The load-bearing property of the batch endpoint: one stale cell does not -// fail the batch.** -// -// The single-record PUT answers a stale version with 409, which is right when -// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark -// this whole series seen") routinely include episodes another device already -// marked, and 409-ing there would abandon every cell after it. So staleness is -// reported per record as accepted:false, with 200 on the request itself. -func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) { +// A stale claim in a batch is reported in its slot without failing the rest. +func TestUDLRecordsPut_StaleSlotDoesNotFailTheBatch(t *testing.T) { s := newUDLTestServer(t) - - const uid = "zCarol" - const cid = "bagdsaaaseries" - // Seed s01e01 at version 4 — the batch below tries to write it at 2. - if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok { - t.Fatalf("seed: ok=%v err=%v", ok, err) - } - - code, out := putRecords(t, s, `{"records":[ - {"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"}, - {"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"} - ]}`) - - if code != http.StatusOK { - t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code) - } - if len(out.Results) != 2 { - t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results)) - } - if out.Results[0].Accepted || out.Results[0].Error != "" { - t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0]) - } - if !out.Results[1].Accepted { - t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1]) - } - - // The fresh cell really landed, and the stale one did not overwrite. - if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" { - t.Fatalf("s01e02 was not written") - } - if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" { - t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"]) + me := testSigner(t, 7) + udlPutBatch(t, s, signed(t, me, "bagcsaaa", "seen:s01e01", "true", 400)) + res := udlPutBatch(t, s, + signed(t, me, "bagcsaaa", "seen:s01e01", "false", 300), + signed(t, me, "bagcsaaa", "seen:s01e02", "true", 300), + ) + if res.Results[0].Accepted || res.Results[0].Status != "stale" || !res.Results[1].Accepted { + t.Fatalf("results = %+v", res.Results) } } -// A malformed record is reported in its own slot rather than rejecting the -// request, for the same reason: the caller retries that cell, not the batch. -func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) { - s := newUDLTestServer(t) - - code, out := putRecords(t, s, `{"records":[ - {"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"}, - {"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"} - ]}`) - - if code != http.StatusOK { - t.Fatalf("status = %d, want 200", code) - } - if out.Results[0].Error == "" { - t.Fatalf("record with no key must report an error, got %+v", out.Results[0]) - } - if !out.Results[1].Accepted { - t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1]) - } -} - -// An empty batch is a client bug, not a no-op: it almost always means the -// caller built its record list wrong, and answering 200 with an empty array -// would hide that. func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) { s := newUDLTestServer(t) - - if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400", code) + req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(`{"records":[]}`)) + rr := httptest.NewRecorder() + s.handleUDLRecordsPut(rr, req) + if rr.Code != http.StatusBadRequest { + t.Fatalf("empty batch = %d", rr.Code) } } diff --git a/internal/claim/claim.go b/internal/claim/claim.go new file mode 100644 index 0000000..0b94b4c --- /dev/null +++ b/internal/claim/claim.go @@ -0,0 +1,240 @@ +// Package claim is the MMUR2 signed claim: signer uid says key = value about +// content cid, at ts (milliseconds, also the version). +// +// Wire spec (load-bearing for every implementation — Go here, Rust in +// meta-feeder-sdk's `claims` feature and the clients' record/ mirror, TS in +// the plugins' meta-core-client.ts; all pinned by testdata/claim-vectors.json): +// +// domain = "MMUR2\n" +// payload = canonical CBOR array [cid, uid, key, value, ts] +// bytes_to_sign = domain || payload +// sig = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S +// +// value is CBOR text (public), bytes (private envelope) or null (retraction). +// Canonical = definite lengths, smallest integer heads — the encoder below is +// hand-rolled because those five shapes are all MMUR2 ever needs. +package claim + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + + "github.com/metazla/meta-core/internal/identity" +) + +// Domain separates MMUR2 bytes from every other signed payload (MMUR1 records, +// identity challenges). Changing it invalidates every claim everywhere. +const Domain = "MMUR2\n" + +// Kind of a claim value. The SQLite storage type mirrors it: TEXT, BLOB, NULL. +type Kind uint8 + +const ( + KindNull Kind = iota + KindText + KindBytes +) + +// Value is a claim's value. The zero Value is a retraction (null). +type Value struct { + Kind Kind + Text string + Bytes []byte +} + +func Text(s string) Value { return Value{Kind: KindText, Text: s} } +func Bytes(b []byte) Value { return Value{Kind: KindBytes, Bytes: b} } +func Null() Value { return Value{} } +func (v Value) IsNull() bool { return v.Kind == KindNull } + +// Equal reports byte-for-byte equality, kind included. +func (v Value) Equal(o Value) bool { + if v.Kind != o.Kind { + return false + } + switch v.Kind { + case KindText: + return v.Text == o.Text + case KindBytes: + return string(v.Bytes) == string(o.Bytes) + } + return true +} + +// Claim is one signed vote. +type Claim struct { + CID string + UID string + Key string + Value Value + TS uint64 + Sig []byte // 64-byte compact r||s +} + +// SignedBytes returns the exact bytes the signature covers. +func SignedBytes(cid, uid, key string, v Value, ts uint64) []byte { + b := make([]byte, 0, len(Domain)+len(cid)+len(uid)+len(key)+len(v.Text)+len(v.Bytes)+32) + b = append(b, Domain...) + b = appendHead(b, 4, 5) // array(5) + b = appendText(b, cid) + b = appendText(b, uid) + b = appendText(b, key) + switch v.Kind { + case KindText: + b = appendText(b, v.Text) + case KindBytes: + b = appendHead(b, 2, uint64(len(v.Bytes))) + b = append(b, v.Bytes...) + default: + b = append(b, 0xf6) // null + } + return appendHead(b, 0, ts) +} + +// SignedBytes of c. +func (c *Claim) SignedBytes() []byte { + return SignedBytes(c.CID, c.UID, c.Key, c.Value, c.TS) +} + +func appendText(b []byte, s string) []byte { + b = appendHead(b, 3, uint64(len(s))) + return append(b, s...) +} + +// appendHead writes a CBOR major type + argument in its smallest form. +func appendHead(b []byte, major byte, n uint64) []byte { + m := major << 5 + switch { + case n < 24: + return append(b, m|byte(n)) + case n <= 0xff: + return append(b, m|24, byte(n)) + case n <= 0xffff: + return append(b, m|25, byte(n>>8), byte(n)) + case n <= 0xffffffff: + return append(b, m|26, byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + default: + return append(b, m|27, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), + byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) + } +} + +// ErrBadSignature is returned for a signature that is malformed, high-S, or +// does not verify against the uid. +var ErrBadSignature = errors.New("bad signature") + +// IsLowS reports whether a compact signature's S is in the lower half. +func IsLowS(sig []byte) bool { + if len(sig) != 64 { + return false + } + var s secp256k1.ModNScalar + if overflow := s.SetByteSlice(sig[32:]); overflow { + return false + } + return !s.IsOverHalfOrder() +} + +// Verify checks the claim's shape and signature. +func (c *Claim) Verify() error { + if c.CID == "" || c.Key == "" || !identity.ValidUID(c.UID) { + return fmt.Errorf("%w: missing cid, key or valid uid", ErrBadSignature) + } + if len(c.Sig) != 64 || !IsLowS(c.Sig) { + return fmt.Errorf("%w: want a 64-byte low-S compact signature", ErrBadSignature) + } + if err := identity.Verify(c.UID, c.SignedBytes(), c.Sig); err != nil { + return fmt.Errorf("%w: %v", ErrBadSignature, err) + } + return nil +} + +// Sign fills c.Sig with id's compact low-S signature. Used by the account-key +// service (users who sign through meta-core until its separate ticket removes +// it), tests and the one-off migration tool — never by the claims engine. +func (c *Claim) Sign(id *identity.Identity) error { + if id.UID != c.UID { + return fmt.Errorf("claim uid %s is not the signing identity %s", c.UID, id.UID) + } + sig, err := identity.SignCompact(id, c.SignedBytes()) + if err != nil { + return err + } + c.Sig = sig + return nil +} + +// jsonClaim is the §3.4 HTTP form. +type jsonClaim struct { + CID string `json:"cid"` + UID string `json:"uid"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + TS uint64 `json:"ts"` + Sig string `json:"sig"` +} + +type jsonBytes struct { + B64 string `json:"b64"` +} + +// MarshalJSON emits {"cid","uid","key","value","ts","sig"}; bytes values as +// {"b64": "..."}, retractions as null. +func (c Claim) MarshalJSON() ([]byte, error) { + var v json.RawMessage + var err error + switch c.Value.Kind { + case KindText: + v, err = json.Marshal(c.Value.Text) + case KindBytes: + v, err = json.Marshal(jsonBytes{B64: base64.StdEncoding.EncodeToString(c.Value.Bytes)}) + default: + v = json.RawMessage("null") + } + if err != nil { + return nil, err + } + return json.Marshal(jsonClaim{ + CID: c.CID, UID: c.UID, Key: c.Key, Value: v, TS: c.TS, + Sig: base64.StdEncoding.EncodeToString(c.Sig), + }) +} + +// UnmarshalJSON parses the §3.4 form. +func (c *Claim) UnmarshalJSON(data []byte) error { + var j jsonClaim + if err := json.Unmarshal(data, &j); err != nil { + return err + } + sig, err := base64.StdEncoding.DecodeString(j.Sig) + if err != nil { + return fmt.Errorf("sig: %w", err) + } + *c = Claim{CID: j.CID, UID: j.UID, Key: j.Key, TS: j.TS, Sig: sig} + raw := j.Value + if len(raw) == 0 || string(raw) == "null" { + return nil + } + if raw[0] == '"' { + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return err + } + c.Value = Text(s) + return nil + } + var b jsonBytes + if err := json.Unmarshal(raw, &b); err != nil || b.B64 == "" && string(raw) != `{"b64":""}` { + return fmt.Errorf("value must be a string, {\"b64\": ...} or null") + } + bs, err := base64.StdEncoding.DecodeString(b.B64) + if err != nil { + return fmt.Errorf("value.b64: %w", err) + } + c.Value = Bytes(bs) + return nil +} diff --git a/internal/claim/claim_test.go b/internal/claim/claim_test.go new file mode 100644 index 0000000..25947f8 --- /dev/null +++ b/internal/claim/claim_test.go @@ -0,0 +1,207 @@ +package claim + +import ( + "encoding/hex" + "encoding/json" + "errors" + "os" + "strings" + "testing" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + + "github.com/metazla/meta-core/internal/identity" +) + +// The golden file every MMUR2 implementation is pinned to. Regenerate with +// CLAIM_VECTORS_UPDATE=1 go test ./internal/claim/ — then re-sync the vendored +// copies (scripts/check-claim-vectors.sh --fix). A change here is a wire break. +const vectorsPath = "testdata/claim-vectors.json" + +// Fixed test key: the scalar 0x0101…01. Never used outside tests. +const testKeyHex = "0101010101010101010101010101010101010101010101010101010101010101" + +type vector struct { + Name string `json:"name"` + CID string `json:"cid"` + UID string `json:"uid"` + Key string `json:"key"` + Value *string `json:"value"` // text value, or null + ValueB64 *string `json:"valueBytesHex,omitempty"` // bytes value as hex + TS uint64 `json:"ts"` + Signed string `json:"signedBytesHex"` + Sig string `json:"sigHex"` // deterministic RFC 6979, low-S +} + +type vectorFile struct { + Comment string `json:"_comment"` + PrivateKeyHex string `json:"privateKeyHex"` + UID string `json:"uid"` + Vectors []vector `json:"vectors"` + HighS struct { + Note string `json:"note"` + Vector string `json:"vector"` + SigHex string `json:"sigHex"` + } `json:"highS"` +} + +func (v vector) value(t *testing.T) Value { + t.Helper() + switch { + case v.ValueB64 != nil: + b, err := hex.DecodeString(*v.ValueB64) + if err != nil { + t.Fatal(err) + } + return Bytes(b) + case v.Value != nil: + return Text(*v.Value) + } + return Null() +} + +func strp(s string) *string { return &s } + +func testIdentity(t *testing.T) *identity.Identity { + t.Helper() + id, err := identity.Import(testKeyHex) + if err != nil { + t.Fatal(err) + } + return id +} + +func cases(uid string) []vector { + return []vector{ + {Name: "text", CID: "bafkreih5kapbjzq", UID: uid, Key: "title", Value: strp("The Matrix"), TS: 1793001234567}, + {Name: "empty-text", CID: "bafkreih5kapbjzq", UID: uid, Key: "title", Value: strp(""), TS: 1}, + {Name: "unicode", CID: "bafkreih5kapbjzq", UID: uid, Key: "titles/jpn/攻殻機動隊", Value: strp("true"), TS: 1793001234567}, + {Name: "keyset-member", CID: "bagacbabaec7v3", UID: uid, Key: "cids/bafkreih5kapbjzq", Value: strp("true"), TS: 23}, + {Name: "bytes", CID: "bafkreih5kapbjzq", UID: uid, Key: "seek", ValueB64: strp("00ff10ab"), TS: 24}, + {Name: "retraction", CID: "bafkreih5kapbjzq", UID: uid, Key: "genres/Drama", TS: 255}, + {Name: "long-value", CID: "bafkreih5kapbjzq", UID: uid, Key: "plot/eng", Value: strp(strings.Repeat("a", 300)), TS: 65536}, + {Name: "ts-64bit", CID: "bafkreih5kapbjzq", UID: uid, Key: "year", Value: strp("1999"), TS: 1 << 40}, + } +} + +func TestVectors(t *testing.T) { + id := testIdentity(t) + if os.Getenv("CLAIM_VECTORS_UPDATE") == "1" { + writeVectors(t, id) + } + raw, err := os.ReadFile(vectorsPath) + if err != nil { + t.Fatalf("read vectors: %v (generate with CLAIM_VECTORS_UPDATE=1)", err) + } + var f vectorFile + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + if f.UID != id.UID { + t.Fatalf("vector uid %s != test key uid %s", f.UID, id.UID) + } + for _, v := range f.Vectors { + c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS} + if got := hex.EncodeToString(c.SignedBytes()); got != v.Signed { + t.Errorf("%s: signed bytes\n got %s\nwant %s", v.Name, got, v.Signed) + } + if err := c.Sign(id); err != nil { + t.Fatal(err) + } + if got := hex.EncodeToString(c.Sig); got != v.Sig { + t.Errorf("%s: sig\n got %s\nwant %s", v.Name, got, v.Sig) + } + if err := c.Verify(); err != nil { + t.Errorf("%s: verify: %v", v.Name, err) + } + } + // The high-S twin of a valid signature must be refused. + for _, v := range f.Vectors { + if v.Name != f.HighS.Vector { + continue + } + sig, _ := hex.DecodeString(f.HighS.SigHex) + c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS, Sig: sig} + if err := c.Verify(); !errors.Is(err, ErrBadSignature) { + t.Errorf("high-S signature accepted: %v", err) + } + } +} + +func writeVectors(t *testing.T, id *identity.Identity) { + t.Helper() + f := vectorFile{ + Comment: "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.", + PrivateKeyHex: testKeyHex, + UID: id.UID, + } + for _, v := range cases(id.UID) { + c := Claim{CID: v.CID, UID: v.UID, Key: v.Key, Value: v.value(t), TS: v.TS} + if err := c.Sign(id); err != nil { + t.Fatal(err) + } + v.Signed = hex.EncodeToString(c.SignedBytes()) + v.Sig = hex.EncodeToString(c.Sig) + f.Vectors = append(f.Vectors, v) + } + first := f.Vectors[0] + sig, _ := hex.DecodeString(first.Sig) + f.HighS.Note = "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it" + f.HighS.Vector = first.Name + f.HighS.SigHex = hex.EncodeToString(highS(sig)) + out, err := json.MarshalIndent(f, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(vectorsPath, append(out, '\n'), 0o644); err != nil { + t.Fatal(err) + } +} + +func highS(sig []byte) []byte { + var s secp256k1.ModNScalar + s.SetByteSlice(sig[32:]) + s.Negate() + out := append([]byte{}, sig[:32]...) + b := s.Bytes() + return append(out, b[:]...) +} + +func TestJSONRoundTrip(t *testing.T) { + id := testIdentity(t) + for _, v := range []Value{Text("x"), Bytes([]byte{0, 1, 2}), Null()} { + c := Claim{CID: "bafy", UID: id.UID, Key: "k", Value: v, TS: 42} + if err := c.Sign(id); err != nil { + t.Fatal(err) + } + b, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + var back Claim + if err := json.Unmarshal(b, &back); err != nil { + t.Fatalf("%s: %v", b, err) + } + if !back.Value.Equal(v) || back.TS != 42 || string(back.Sig) != string(c.Sig) { + t.Errorf("round trip changed the claim: %s", b) + } + if err := back.Verify(); err != nil { + t.Errorf("%s: %v", b, err) + } + } +} + +func TestTamperRejected(t *testing.T) { + id := testIdentity(t) + c := Claim{CID: "bafy", UID: id.UID, Key: "title", Value: Text("A"), TS: 1} + if err := c.Sign(id); err != nil { + t.Fatal(err) + } + c.Value = Text("B") + if err := c.Verify(); !errors.Is(err, ErrBadSignature) { + t.Fatalf("tampered value verified: %v", err) + } +} diff --git a/internal/claim/testdata/claim-vectors.json b/internal/claim/testdata/claim-vectors.json new file mode 100644 index 0000000..bcb7656 --- /dev/null +++ b/internal/claim/testdata/claim-vectors.json @@ -0,0 +1,93 @@ +{ + "_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.", + "privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "vectors": [ + { + "name": "text", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "title", + "value": "The Matrix", + "ts": 1793001234567, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087", + "sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8" + }, + { + "name": "empty-text", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "title", + "value": "", + "ts": 1, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001", + "sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642" + }, + { + "name": "unicode", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "titles/jpn/攻殻機動隊", + "value": "true", + "ts": 1793001234567, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087", + "sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207" + }, + { + "name": "keyset-member", + "cid": "bagacbabaec7v3", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "cids/bafkreih5kapbjzq", + "value": "true", + "ts": 23, + "signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517", + "sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761" + }, + { + "name": "bytes", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "seek", + "value": null, + "valueBytesHex": "00ff10ab", + "ts": 24, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818", + "sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4" + }, + { + "name": "retraction", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "genres/Drama", + "value": null, + "ts": 255, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff", + "sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac" + }, + { + "name": "long-value", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "plot/eng", + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ts": 65536, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000", + "sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2" + }, + { + "name": "ts-64bit", + "cid": "bafkreih5kapbjzq", + "uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ", + "key": "year", + "value": "1999", + "ts": 1099511627776, + "signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000", + "sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d" + } + ], + "highS": { + "note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it", + "vector": "text", + "sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899" + } +} diff --git a/internal/config/config.go b/internal/config/config.go index ea37703..735d9b7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,7 +20,8 @@ type Config struct { PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front) // Storage configuration - DBPathOverride string // META_CORE_DB_PATH; empty = /db/meta-core.db + DBPathOverride string // META_CORE_DB_PATH; empty = /db/meta-core.db (truth) + CachePathOverride string // META_CORE_CACHE_PATH; empty = /db-cache/cache.db // HTTP API configuration HTTPPort int // HTTP API port (default: 9000) @@ -61,6 +62,7 @@ func Load() *Config { BaseURL: getEnv("BASE_URL", ""), PublicURL: getEnv("META_CORE_PUBLIC_URL", ""), DBPathOverride: getEnv("META_CORE_DB_PATH", ""), + CachePathOverride: getEnv("META_CORE_CACHE_PATH", ""), HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000), HTTPHost: getEnv("META_CORE_HTTP_HOST", "0.0.0.0"), HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000), @@ -91,7 +93,7 @@ func (c *Config) LockFilePath() string { return c.MetaCorePath + "/locks/kv-leader.lock" } -// DBPath returns the path of meta-core's SQLite database file. +// DBPath returns the path of meta-core's truth database (backed up). func (c *Config) DBPath() string { if c.DBPathOverride != "" { return c.DBPathOverride @@ -99,6 +101,15 @@ func (c *Config) DBPath() string { return c.MetaCorePath + "/db/meta-core.db" } +// CachePath returns the path of meta-core's cache database: everything +// derived from truth, never backed up, rebuilt when missing. +func (c *Config) CachePath() string { + if c.CachePathOverride != "" { + return c.CachePathOverride + } + return c.MetaCorePath + "/db-cache/cache.db" +} + // MountsFilePath returns the path to the mounts configuration file func (c *Config) MountsFilePath() string { return c.MountsDir + "/mounts.json" diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 7e4335e..961ee8d 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -307,6 +307,30 @@ func Sign(id *Identity, bytes []byte) ([]byte, error) { return sig.Serialize(), nil } +// SignCompact is Sign with the 64-byte compact r||s encoding MMUR2 claims +// use. decred's RFC 6979 signer always emits low-S, which MMUR2 requires. +func SignCompact(id *Identity, bytes []byte) ([]byte, error) { + if id == nil { + return nil, errors.New("no identity loaded") + } + raw, err := hex.DecodeString(id.PrivateKeyHex) + if err != nil { + return nil, fmt.Errorf("decode private key: %w", err) + } + if len(raw) != 32 { + return nil, fmt.Errorf("private key length %d", len(raw)) + } + priv := secp256k1.PrivKeyFromBytes(raw) + digest := sha256.Sum256(bytes) + sig := ecdsa.Sign(priv, digest[:]) + r, s := sig.R(), sig.S() + out := make([]byte, 64) + rb, sb := r.Bytes(), s.Bytes() + copy(out[:32], rb[:]) + copy(out[32:], sb[:]) + return out, nil +} + // DeriveAEADKey derives the symmetric key used to encrypt private-tier // User Data Layer records. The KDF parameters (salt, info, length) are // fixed for the v1 wire format. diff --git a/internal/kv/kv.go b/internal/kv/kv.go index 9544c42..22628b2 100644 --- a/internal/kv/kv.go +++ b/internal/kv/kv.go @@ -84,6 +84,14 @@ type Options struct { // values — the hook derived counters hang off (Tx.AddCounter). It must not // write kv rows itself. OnChange func(tx *Tx, key, old string, hadOld bool, new string, hasNew bool) + // NoKVTable opens the file without the kv table and its prepared + // statements: the caller owns every table through Schema and writes them + // with Tx.Exec / Tx.Query (meta-core's truth database). The kv methods + // (Get, Put, streams, counters) must not be used on such a store. + NoKVTable bool + // Synchronous is SQLite's synchronous pragma; default NORMAL. A store that + // is rebuildable from another (meta-core's cache) runs OFF. + Synchronous string } // Store is an open database. @@ -134,7 +142,10 @@ func Open(path string, opts Options) (*Store, error) { opts.EventMaxLen = 100_000 } - w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false)) + if opts.Synchronous == "" { + opts.Synchronous = "NORMAL" + } + w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false, opts.Synchronous)) if err != nil { return nil, fmt.Errorf("open writer: %w", err) } @@ -144,9 +155,11 @@ func Open(path string, opts Options) (*Store, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { - w.Close() - return nil, fmt.Errorf("create kv: %w", err) + if !opts.NoKVTable { + if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { + w.Close() + return nil, fmt.Errorf("create kv: %w", err) + } } if opts.Schema != "" { // No deadline: on an existing store, the first open after a new index @@ -162,7 +175,7 @@ func Open(path string, opts Options) (*Store, error) { } } - r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true)) + r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true, opts.Synchronous)) if err != nil { w.Close() return nil, fmt.Errorf("open readers: %w", err) @@ -174,7 +187,7 @@ func Open(path string, opts Options) (*Store, error) { // Fresh deadline: the schema step above may legitimately have taken minutes. ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - for _, p := range []struct { + stmts := []struct { dst **sql.Stmt query string }{ @@ -183,7 +196,11 @@ func Open(path string, opts Options) (*Store, error) { {&st.stDelete, `DELETE FROM kv WHERE k = ? RETURNING v`}, {&st.stGet, `SELECT v FROM kv WHERE k = ?`}, {&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`}, - } { + } + if opts.NoKVTable { + stmts = nil + } + for _, p := range stmts { if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil { r.Close() w.Close() @@ -199,11 +216,11 @@ func Open(path string, opts Options) (*Store, error) { // dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class // Redis' AOF-everysec gave us: a power loss may drop the last transactions, // never corrupt the file. -func dsn(path string, cacheMB int, readOnly bool) string { +func dsn(path string, cacheMB int, readOnly bool, synchronous string) string { q := url.Values{} q.Add("_pragma", "busy_timeout(30000)") q.Add("_pragma", "journal_mode(WAL)") - q.Add("_pragma", "synchronous(NORMAL)") + q.Add("_pragma", "synchronous("+synchronous+")") q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024)) q.Add("_pragma", "temp_store(MEMORY)") q.Add("_pragma", "journal_size_limit(268435456)") @@ -520,6 +537,7 @@ func (s *Store) runGroup(reqs []*writeReq) { return } g := &group{tx: sqlTx} + var commitHooks []func() for i, r := range reqs { if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil { _ = sqlTx.Rollback() @@ -534,6 +552,7 @@ func (s *Store) runGroup(reqs []*writeReq) { } if err == nil { g.mergeCounters(tx.counters) + commitHooks = append(commitHooks, tx.onCommit...) } if err != nil { results[i] = err @@ -565,6 +584,12 @@ func (s *Store) runGroup(reqs []*writeReq) { return } s.afterCommit(g.pending) + // Hooks run in the writer goroutine, in commit order, before any request + // returns: a caller that orders work by commit (meta-core's apply queue) + // sees it in exactly that order. + for _, h := range commitHooks { + h() + } for i, r := range reqs { r.done <- results[i] } @@ -578,6 +603,23 @@ type Tx struct { g *group events []map[string]string counters map[string]int64 + onCommit []func() + quiet bool +} + +// OnCommit registers fn to run after the transaction commits, only if this +// request succeeded. Hooks run in the writer goroutine in commit order and +// must not block or call Update. +func (t *Tx) OnCommit(fn func()) { t.onCommit = append(t.onCommit, fn) } + +// Quiet stops this request's changes from publishing events (a bulk rebuild +// whose consumers are told to take a snapshot instead). Counters and the +// OnChange hook still run. +func (t *Tx) Quiet() { t.quiet = true } + +// Query runs a raw multi-row query in the transaction. +func (t *Tx) Query(query string, args ...any) (*sql.Rows, error) { + return t.sqlTx().QueryContext(t.ctx, query, args...) } func (t *Tx) sqlTx() *sql.Tx { return t.g.tx } @@ -800,7 +842,7 @@ func (s *Store) WriteStats() (changed, unchanged int64) { } func (t *Tx) event(op, k string) { - if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) { + if t.quiet || t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) { return } t.events = append(t.events, map[string]string{ diff --git a/internal/snapshot/cache.go b/internal/snapshot/cache.go deleted file mode 100644 index 0018b53..0000000 --- a/internal/snapshot/cache.go +++ /dev/null @@ -1,143 +0,0 @@ -package snapshot - -import ( - "archive/zip" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "strings" -) - -// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP. -// Trailing slash because ZIP entry names are forward-slash relative paths. -const PluginOutputArchivePrefix = "cache/plugin-output/" - -// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every -// file under it into zw at PluginOutputArchivePrefix + relPath. -// -// Returns the number of files added. A missing srcDir is not an error: it -// just means no plugin has produced output yet, so the section stays empty. -func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) { - info, err := os.Stat(srcDir) - if errors.Is(err, os.ErrNotExist) { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("stat %s: %w", srcDir, err) - } - if !info.IsDir() { - return 0, fmt.Errorf("%s is not a directory", srcDir) - } - - count := 0 - err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if d.IsDir() { - return nil - } - rel, err := filepath.Rel(srcDir, path) - if err != nil { - return err - } - entry := PluginOutputArchivePrefix + filepath.ToSlash(rel) - - w, err := zw.Create(entry) - if err != nil { - return err - } - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - if _, err := io.Copy(w, f); err != nil { - return err - } - count++ - return nil - }) - if err != nil { - return count, fmt.Errorf("walk %s: %w", srcDir, err) - } - return count, nil -} - -// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from -// the ZIP and writes it under dstDir. Existing files are overwritten when -// overwrite=true; otherwise they're left alone (treats existing files as the -// "mine wins" choice — same toggle the metadata import uses). -// -// Path traversal is rejected: any entry whose relative path resolves outside -// dstDir is skipped with a warning. -func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) { - if dstDir == "" { - return 0, 0, fmt.Errorf("plugin-output destination dir is empty") - } - if err := os.MkdirAll(dstDir, 0o755); err != nil { - return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err) - } - - cleanRoot, err := filepath.Abs(dstDir) - if err != nil { - return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err) - } - - for _, f := range zr.File { - if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) { - continue - } - if strings.HasSuffix(f.Name, "/") { - continue - } - rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix) - dst := filepath.Join(cleanRoot, filepath.FromSlash(rel)) - - // Reject anything that escapes the dst root. - if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot { - log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name) - skipped++ - continue - } - - if !overwrite { - if _, err := os.Stat(dst); err == nil { - skipped++ - continue - } - } - - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err) - } - - if err := writeZipFile(f, dst); err != nil { - return written, skipped, err - } - written++ - } - return written, skipped, nil -} - -func writeZipFile(f *zip.File, dst string) error { - rc, err := f.Open() - if err != nil { - return fmt.Errorf("open zip entry %s: %w", f.Name, err) - } - defer rc.Close() - - out, err := os.Create(dst) - if err != nil { - return fmt.Errorf("create %s: %w", dst, err) - } - defer out.Close() - - if _, err := io.Copy(out, rc); err != nil { - return fmt.Errorf("write %s: %w", dst, err) - } - return nil -} diff --git a/internal/snapshot/export.go b/internal/snapshot/export.go deleted file mode 100644 index 9e16ea6..0000000 --- a/internal/snapshot/export.go +++ /dev/null @@ -1,132 +0,0 @@ -package snapshot - -import ( - "archive/zip" - "encoding/json" - "fmt" - "io" - "log" - "os" - "time" - - "github.com/metazla/meta-core/internal/storage" -) - -// ExportOptions controls what an export includes. -type ExportOptions struct { - IncludePluginOutput bool - // PluginOutputDir is the absolute path on disk that holds plugin-produced - // artifacts (typically $FILES_PATH/plugin). Required when - // IncludePluginOutput is true. - PluginOutputDir string - // SourceHost is recorded in the manifest so imports can attribute - // where the snapshot came from (defaults to OS hostname). - SourceHost string -} - -// Exporter writes snapshot ZIPs from the live metadata store. -type Exporter struct { - storage *storage.Client -} - -func NewExporter(stor *storage.Client) *Exporter { - return &Exporter{storage: stor} -} - -// Export streams a ZIP archive to w. The archive layout is: -// -// manifest.json -// index.json -// metadata/{cid}.json (one per known CID) -// -// Optional sections (hash-index, plugin-output) are gated by ExportOptions -// but are not yet implemented — they live on filesystems meta-core does not -// own, and need cross-service collection. The flag is plumbed for forward -// compatibility but does nothing today. -func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) { - if e.storage == nil || !e.storage.IsConnected() { - return Manifest{}, fmt.Errorf("storage not connected") - } - - hashIDs, err := e.storage.GetAllHashIDs() - if err != nil { - return Manifest{}, fmt.Errorf("list hashIDs: %w", err) - } - - zw := zip.NewWriter(w) - defer zw.Close() - - // 1. manifest.json — written last after we know FileCount, but we need - // the entry order to put it first in the archive for readers. Solve - // by buffering in memory: it's tiny. - host := opts.SourceHost - if host == "" { - if h, err := os.Hostname(); err == nil { - host = h - } - } - manifest := Manifest{ - SchemaVersion: ManifestSchemaVersion, - GeneratedAt: time.Now().UTC(), - SourceHost: host, - FileCount: len(hashIDs), - Includes: Includes{ - Metadata: true, - PluginOutput: opts.IncludePluginOutput, - }, - } - if err := writeJSON(zw, ManifestFile, manifest); err != nil { - return Manifest{}, fmt.Errorf("write manifest: %w", err) - } - - // 2. index.json — flat list of CIDs. Redundant with metadata/ but - // cheap and useful for readers that only need the keyset. - if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil { - return Manifest{}, fmt.Errorf("write index: %w", err) - } - - // 3. metadata/{cid}.json — one nested document per CID. - for _, hashID := range hashIDs { - flat, err := e.storage.GetMetadataFlat(hashID) - if err != nil { - log.Printf("[Snapshot] export: skipping %s: %v", hashID, err) - continue - } - if len(flat) == 0 { - continue - } - nested := Reconstruct(flat) - entry := MetadataDir + hashID + ".json" - if err := writeJSON(zw, entry, nested); err != nil { - return Manifest{}, fmt.Errorf("write %s: %w", entry, err) - } - } - - // 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and - // streams each file into the ZIP. Heavy (~120 MB on a populated dev - // stack), so it's gated by the ExportOptions toggle. - if opts.IncludePluginOutput { - if opts.PluginOutputDir == "" { - return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set") - } - n, err := addPluginOutput(zw, opts.PluginOutputDir) - if err != nil { - return Manifest{}, fmt.Errorf("add plugin-output: %w", err) - } - log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir) - } - - log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)", - len(hashIDs), host, opts.IncludePluginOutput) - return manifest, nil -} - -func writeJSON(zw *zip.Writer, name string, body interface{}) error { - w, err := zw.Create(name) - if err != nil { - return err - } - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(body) -} diff --git a/internal/snapshot/flatten.go b/internal/snapshot/flatten.go deleted file mode 100644 index 8758967..0000000 --- a/internal/snapshot/flatten.go +++ /dev/null @@ -1,195 +0,0 @@ -package snapshot - -import ( - "fmt" - "regexp" - "sort" - "strconv" - "strings" -) - -// Flat keys use "/" as the separator. Property paths inside the store are -// stored as e.g. "fileinfo/streamdetails/video/0/codec". -const flatSep = "/" - -// numericKey matches a path segment that is a non-negative decimal integer. -// Used to detect array slots (".../video/0/codec"). -var numericKey = regexp.MustCompile(`^\d+$`) - -// Reconstruct rebuilds a nested JSON-shaped value from a flat property map. -// Keys are property paths (no leading slash, no file:{cid}/ prefix). -// -// Sibling keys whose path segments are all sequential integers starting at 0 -// are emitted as JSON arrays. Other groups become JSON objects. Leaf values -// are coerced from string to int/float/bool/null when they parse cleanly so -// the exported JSON is human-readable; on import we round-trip back to strings. -func Reconstruct(flat map[string]string) interface{} { - root := map[string]interface{}{} - for path, value := range flat { - parts := strings.Split(path, flatSep) - setPath(root, parts, parseValue(value)) - } - return convertNumericMapsToArrays(root) -} - -// Flatten is the inverse of Reconstruct: walks a nested value and produces a -// flat property map whose values are all strings (store-ready). -// -// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing -// values are stored as "" to preserve the field's presence — this matches -// what flattenMetadata in the TS implementation does. -func Flatten(value interface{}) map[string]string { - out := map[string]string{} - flattenInto(value, "", out) - return out -} - -func setPath(root map[string]interface{}, parts []string, leaf interface{}) { - if len(parts) == 0 { - return - } - cur := root - for i, part := range parts { - if i == len(parts)-1 { - cur[part] = leaf - return - } - next, ok := cur[part].(map[string]interface{}) - if !ok { - next = map[string]interface{}{} - cur[part] = next - } - cur = next - } -} - -func convertNumericMapsToArrays(v interface{}) interface{} { - switch t := v.(type) { - case map[string]interface{}: - // Recurse first so child arrays are detected before parent. - for k, child := range t { - t[k] = convertNumericMapsToArrays(child) - } - if !looksLikeArray(t) { - return t - } - // Sort numerically and emit as []interface{}. - idx := make([]int, 0, len(t)) - for k := range t { - n, _ := strconv.Atoi(k) - idx = append(idx, n) - } - sort.Ints(idx) - arr := make([]interface{}, len(idx)) - for i, n := range idx { - arr[i] = t[strconv.Itoa(n)] - } - return arr - case []interface{}: - for i := range t { - t[i] = convertNumericMapsToArrays(t[i]) - } - return t - default: - return v - } -} - -// looksLikeArray returns true when every key in m is a non-negative integer -// and the keys form a contiguous range starting at 0. Empty maps return false -// — we can't tell whether the original was {} or [], so prefer {}. -func looksLikeArray(m map[string]interface{}) bool { - if len(m) == 0 { - return false - } - seen := make(map[int]bool, len(m)) - for k := range m { - if !numericKey.MatchString(k) { - return false - } - n, err := strconv.Atoi(k) - if err != nil || n < 0 { - return false - } - seen[n] = true - } - for i := 0; i < len(m); i++ { - if !seen[i] { - return false - } - } - return true -} - -func flattenInto(v interface{}, path string, out map[string]string) { - switch t := v.(type) { - case nil: - out[path] = "" - case map[string]interface{}: - if len(t) == 0 && path != "" { - // Empty objects don't have a flat-key representation; skip. - return - } - for k, child := range t { - next := k - if path != "" { - next = path + flatSep + k - } - flattenInto(child, next, out) - } - case []interface{}: - for i, child := range t { - next := strconv.Itoa(i) - if path != "" { - next = path + flatSep + next - } - flattenInto(child, next, out) - } - case string: - out[path] = t - case bool: - if t { - out[path] = "true" - } else { - out[path] = "false" - } - case float64: - // JSON numbers always decode to float64; emit ints without decimals. - if t == float64(int64(t)) { - out[path] = strconv.FormatInt(int64(t), 10) - } else { - out[path] = strconv.FormatFloat(t, 'f', -1, 64) - } - default: - out[path] = fmt.Sprintf("%v", t) - } -} - -// parseValue mirrors the TS reconstruct logic: convert obvious primitives -// back to their typed form for nicer exported JSON, leave anything ambiguous -// as a string so we don't lose data (e.g. "5.1" channel layouts). -func parseValue(s string) interface{} { - if s == "" { - return nil - } - switch s { - case "true": - return true - case "false": - return false - case "null": - return nil - } - // Only treat as number when the string is a clean integer or decimal, - // no leading zeros / trailing junk. Matches the TS regex. - if isCleanNumber(s) { - if n, err := strconv.ParseFloat(s, 64); err == nil { - return n - } - } - return s -} - -var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`) - -func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) } diff --git a/internal/snapshot/import.go b/internal/snapshot/import.go deleted file mode 100644 index 997d14e..0000000 --- a/internal/snapshot/import.go +++ /dev/null @@ -1,262 +0,0 @@ -package snapshot - -import ( - "archive/zip" - "encoding/json" - "fmt" - "io" - "log" - "strings" - - "github.com/metazla/meta-core/internal/storage" -) - -// Mode controls how an import reconciles existing CIDs. -type Mode string - -const ( - // ModeReplace deletes every existing property of an imported CID and - // writes the snapshot's values verbatim. Untouched CIDs are left alone. - ModeReplace Mode = "replace" - // ModeMerge keeps unmentioned existing properties and only touches the - // fields present in the snapshot. Conflict resolution within merge is - // controlled by Conflict. - ModeMerge Mode = "merge" -) - -// Conflict picks a winner for fields present on both sides during merge. -type Conflict string - -const ( - // ConflictMine keeps the existing (target) value when both sides have one. - ConflictMine Conflict = "mine" - // ConflictSource overwrites the existing value with the snapshot's value. - ConflictSource Conflict = "source" -) - -// ImportOptions controls how Import applies a snapshot. -type ImportOptions struct { - Mode Mode - Conflict Conflict - DryRun bool - // PluginOutputDir is the on-disk destination for cache/plugin-output/* - // entries. When empty, plugin-output is silently skipped on import. - PluginOutputDir string -} - -// FileResult summarizes what happened (or would happen, in dry-run) for one CID. -type FileResult struct { - HashID string `json:"hashId"` - Added int `json:"added"` // new property keys written - Updated int `json:"updated"` // existing property keys whose value changed - Unchanged int `json:"unchanged"` // existing property keys whose value matched - Kept int `json:"kept"` // existing property keys preserved by ConflictMine - Deleted int `json:"deleted"` // existing property keys removed (replace mode) - Error string `json:"error,omitempty"` -} - -// PluginOutputResult summarizes what happened to cache/plugin-output entries. -type PluginOutputResult struct { - Written int `json:"written"` - Skipped int `json:"skipped"` -} - -// ImportResult is the response body for the import endpoint. -type ImportResult struct { - Mode Mode `json:"mode"` - Conflict Conflict `json:"conflict,omitempty"` - DryRun bool `json:"dryRun"` - TotalFiles int `json:"totalFiles"` - FilesOK int `json:"filesOk"` - FilesFailed int `json:"filesFailed"` - Files []FileResult `json:"files,omitempty"` - PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"` -} - -// Importer applies snapshot ZIPs to the live metadata store. -type Importer struct { - storage *storage.Client -} - -func NewImporter(stor *storage.Client) *Importer { - return &Importer{storage: stor} -} - -// Import reads a ZIP from r (which must support io.ReaderAt; callers pass -// *bytes.Reader after buffering the upload) and applies it to the store. -func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) { - if i.storage == nil || !i.storage.IsConnected() { - return nil, fmt.Errorf("storage not connected") - } - if opts.Mode != ModeReplace && opts.Mode != ModeMerge { - return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode) - } - if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource { - return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict) - } - - zr, err := zip.NewReader(r, size) - if err != nil { - return nil, fmt.Errorf("open zip: %w", err) - } - - var manifest Manifest - if err := readJSON(zr, ManifestFile, &manifest); err != nil { - return nil, fmt.Errorf("read manifest: %w", err) - } - if manifest.SchemaVersion > ManifestSchemaVersion { - return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d", - manifest.SchemaVersion, ManifestSchemaVersion) - } - if manifest.SchemaVersion < ManifestSchemaVersion { - return nil, fmt.Errorf("snapshot schemaVersion %d is no longer supported (current: %d); v1 used content-hash root keys which are incompatible with the UUID-rooted schema", - manifest.SchemaVersion, ManifestSchemaVersion) - } - - result := &ImportResult{ - Mode: opts.Mode, - Conflict: opts.Conflict, - DryRun: opts.DryRun, - } - - for _, f := range zr.File { - if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") { - continue - } - hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json") - if hashID == "" { - continue - } - fr := i.applyOne(f, hashID, opts) - result.TotalFiles++ - if fr.Error != "" { - result.FilesFailed++ - } else { - result.FilesOK++ - } - result.Files = append(result.Files, fr) - } - - // Plugin-output extraction. Only applied on real imports (not dry-run). - // If the snapshot doesn't contain a cache/plugin-output/ section, this is - // a no-op. If the destination is unset, we skip silently — the manifest's - // pluginOutput flag tells the caller it was on the source side either way. - if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput { - // Existing files are kept under ConflictMine / replace-doesn't-apply, - // overwritten under ConflictSource. Replace mode behaves like - // "overwrite" for cache files (it's the destructive choice for - // metadata; mirror that intent here). - overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource - written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite) - if err != nil { - return result, fmt.Errorf("extract plugin-output: %w", err) - } - result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped} - log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped) - } - - log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d", - opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed) - return result, nil -} - -func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult { - fr := FileResult{HashID: hashID} - - rc, err := f.Open() - if err != nil { - fr.Error = fmt.Sprintf("open entry: %v", err) - return fr - } - defer rc.Close() - - var nested interface{} - if err := json.NewDecoder(rc).Decode(&nested); err != nil { - fr.Error = fmt.Sprintf("decode json: %v", err) - return fr - } - incoming := Flatten(nested) - - existing, err := i.storage.GetMetadataFlat(hashID) - if err != nil { - fr.Error = fmt.Sprintf("read existing: %v", err) - return fr - } - if existing == nil { - existing = map[string]string{} - } - - switch opts.Mode { - case ModeReplace: - // Diff for reporting. - toWrite := map[string]string{} - for k, v := range incoming { - toWrite[k] = v - if cur, ok := existing[k]; ok { - if cur == v { - fr.Unchanged++ - } else { - fr.Updated++ - } - } else { - fr.Added++ - } - } - for k := range existing { - if _, ok := incoming[k]; !ok { - fr.Deleted++ - } - } - if !opts.DryRun { - if _, err := i.storage.DeleteMetadata(hashID); err != nil { - fr.Error = fmt.Sprintf("delete existing: %v", err) - return fr - } - if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil { - fr.Error = fmt.Sprintf("write: %v", err) - return fr - } - } - - case ModeMerge: - toWrite := map[string]string{} - for k, v := range incoming { - cur, present := existing[k] - switch { - case !present: - toWrite[k] = v - fr.Added++ - case cur == v: - fr.Unchanged++ - case opts.Conflict == ConflictSource: - toWrite[k] = v - fr.Updated++ - case opts.Conflict == ConflictMine: - fr.Kept++ - } - } - if !opts.DryRun && len(toWrite) > 0 { - if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil { - fr.Error = fmt.Sprintf("merge: %v", err) - return fr - } - } - } - - return fr -} - -func readJSON(zr *zip.Reader, name string, out interface{}) error { - for _, f := range zr.File { - if f.Name != name { - continue - } - rc, err := f.Open() - if err != nil { - return err - } - defer rc.Close() - return json.NewDecoder(rc).Decode(out) - } - return fmt.Errorf("entry %q not found", name) -} diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go deleted file mode 100644 index b45f4a7..0000000 --- a/internal/snapshot/manifest.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package snapshot implements export/import/wipe of MetaMesh metadata. -// -// The on-disk format is a ZIP archive containing one nested-JSON file per -// indexed root. Schema is defined by ManifestSchemaVersion; readers must -// reject archives whose schemaVersion exceeds what they know. -// -// v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries -// under metadata/.json carry the flat string properties; the cid -// reverse-index and the duplicates SET are NOT serialized — they are rebuilt -// transparently on import because the cids/ key-set members ARE flat -// properties, and SetMetadataFlat / MergeMetadataFlat auto-register a -// reverse-index alias for every one they see (see -// internal/storage/cid_resolution.go: cidFromKeysetField). The legacy flat -// cid_*/midhash256 named fields are not recognised. -// -// v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc". -// v1 archives can no longer be imported — they would resurrect the -// privileged-hashID schema this layer was built to retire. -package snapshot - -import "time" - -// ManifestSchemaVersion is the on-disk format version. Bump when the layout -// of metadata/{root}.json or the manifest changes in a non-additive way. -const ManifestSchemaVersion = 2 - -// MetadataDir is the path inside the ZIP that holds per-CID JSON files. -const MetadataDir = "metadata/" - -// ManifestFile is the path inside the ZIP that holds the manifest JSON. -const ManifestFile = "manifest.json" - -// IndexFile is the path inside the ZIP that holds the file index (CID list). -const IndexFile = "index.json" - -// Manifest is the metadata header stored at the root of every snapshot ZIP. -type Manifest struct { - SchemaVersion int `json:"schemaVersion"` - GeneratedAt time.Time `json:"generatedAt"` - SourceHost string `json:"sourceHost,omitempty"` - MetaCoreVer string `json:"metaCoreVersion,omitempty"` - FileCount int `json:"fileCount"` - Includes Includes `json:"includes"` -} - -// Includes records which optional sections are present in a snapshot. -type Includes struct { - Metadata bool `json:"metadata"` - PluginOutput bool `json:"pluginOutput,omitempty"` -} - -// IndexFileBody is the structure of index.json inside the ZIP. -type IndexFileBody struct { - HashIDs []string `json:"hashIds"` -} diff --git a/internal/snapshot/wipe.go b/internal/snapshot/wipe.go deleted file mode 100644 index 8125c19..0000000 --- a/internal/snapshot/wipe.go +++ /dev/null @@ -1,42 +0,0 @@ -package snapshot - -import ( - "fmt" - - "github.com/metazla/meta-core/internal/storage" -) - -// WipeScope picks which classes of data to clear. -type WipeScope struct { - Metadata bool // file:* keys + file:__index__ -} - -// WipeResult reports what was cleared. -type WipeResult struct { - MetadataDeleted int64 `json:"metadataDeleted"` -} - -// Wiper clears metadata from the store. -type Wiper struct { - storage *storage.Client -} - -func NewWiper(stor *storage.Client) *Wiper { - return &Wiper{storage: stor} -} - -// Wipe applies the requested scope. Returns the count of files removed. -func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) { - if w.storage == nil || !w.storage.IsConnected() { - return nil, fmt.Errorf("storage not connected") - } - res := &WipeResult{} - if scope.Metadata { - n, err := w.storage.ClearAllMetadata() - if err != nil { - return nil, fmt.Errorf("clear metadata: %w", err) - } - res.MetadataDeleted = n - } - return res, nil -} diff --git a/internal/storage/apply.go b/internal/storage/apply.go new file mode 100644 index 0000000..fc40bf6 --- /dev/null +++ b/internal/storage/apply.go @@ -0,0 +1,612 @@ +package storage + +// The apply loop: folds committed truth changes into the cache. +// +// The cache's merged view keeps the v1 record layout — `file:/` +// rows of the cache kv table — so every reader (record reads, search, the +// field index, FTS, counters, SSE and its snapshot) is unchanged; those rows +// are now derived, never written by a client. Each committed change re-merges +// only the (rid, key) it touched, from the truth votes of every cid in the +// group. Group membership is synthesised: every cid of a group appears as +// `file:/cids/ = "true"` plus the `cid: → rid` lookup, whatever +// the cids/* claims say. +// +// Changes arrive in commit order (kv.Tx.OnCommit runs in the truth writer), so +// `meta:applied_seq` only ever covers a prefix of the truth log. A crash +// between the truth commit and the cache apply leaves the cache behind, never +// wrong; Connect catches up from truth. + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "strconv" + "strings" + "sync" + "time" + + "github.com/metazla/meta-core/internal/kv" +) + +const ( + appliedSeqKey = "meta:applied_seq" + // rebuiltAtKey holds the stream id at which the cache was last rebuilt. + // An SSE cursor older than it gets `gap`: the rebuild emitted no + // per-key events, so the consumer must take a snapshot. + rebuiltAtKey = "meta:rebuilt_at" +) + +var localFields = []string{"filePath", "sizeByte", "mtimeNano"} + +type applier struct { + c *Client + + mu sync.Mutex + queue []change + applied int64 + notify chan struct{} + + wake chan struct{} + stop chan struct{} + done chan struct{} +} + +func newApplier(c *Client, applied int64) *applier { + a := &applier{ + c: c, applied: applied, + notify: make(chan struct{}), + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } + go a.run() + return a +} + +func (a *applier) enqueue(ch []change) { + a.mu.Lock() + a.queue = append(a.queue, ch...) + a.mu.Unlock() + select { + case a.wake <- struct{}{}: + default: + } +} + +// wait blocks until the cache has applied every change up to seq. +func (a *applier) wait(ctx context.Context, seq int64) error { + for { + a.mu.Lock() + if a.applied >= seq { + a.mu.Unlock() + return nil + } + n := a.notify + a.mu.Unlock() + select { + case <-n: + case <-ctx.Done(): + return ctx.Err() + case <-a.stop: + return errors.New("storage closed before the cache applied the write") + } + } +} + +func (a *applier) close() { + close(a.stop) + <-a.done +} + +const applyBatchMax = 2000 + +func (a *applier) run() { + defer close(a.done) + for { + select { + case <-a.wake: + case <-a.stop: + return + } + for { + a.mu.Lock() + n := min(len(a.queue), applyBatchMax) + batch := append([]change(nil), a.queue[:n]...) + a.mu.Unlock() + if len(batch) == 0 { + break + } + if err := a.c.applyBatch(batch); err != nil { + log.Printf("[Storage] apply of %d changes failed, retrying: %v", len(batch), err) + select { + case <-time.After(time.Second): + continue + case <-a.stop: + return + } + } + a.mu.Lock() + a.queue = a.queue[n:] + if s := batch[len(batch)-1].seq; s > a.applied { + a.applied = s + } + close(a.notify) + a.notify = make(chan struct{}) + a.mu.Unlock() + } + } +} + +// applyBatch folds committed changes into the cache in one transaction. +func (c *Client) applyBatch(batch []change) error { + s, err := c.kv() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + return s.Update(ctx, func(tx *kv.Tx) error { + return c.applyTx(tx, batch) + }) +} + +func (c *Client) applyTx(tx *kv.Tx, batch []change) error { + ctx := tx.Context() + marks := map[string]map[string]bool{} + mark := func(rid, key string) { + if marks[rid] == nil { + marks[rid] = map[string]bool{} + } + marks[rid][key] = true + } + all := map[string]bool{} + dirty := map[string]bool{} + var maxSeq int64 + for _, it := range batch { + maxSeq = max(maxSeq, it.seq) + if it.regroup != nil { + loser, survivor := it.regroup[0], it.regroup[1] + if _, err := tx.DeletePrefix(recordPrefix(loser), 0); err != nil { + return err + } + if _, err := tx.Delete(rootKey(loser)); err != nil { + return err + } + if err := deleteSearchRowTx(tx, loser); err != nil { + return err + } + delete(marks, loser) + cids, err := c.truthMembers(ctx, survivor) + if err != nil { + return err + } + for _, cid := range cids { + if _, err := syncMember(tx, cid, survivor); err != nil { + return err + } + } + all[survivor], dirty[survivor] = true, true + continue + } + rid, err := c.truthRid(ctx, it.cid) + if err != nil { + return err + } + if rid == "" { + continue + } + newRoot, err := syncMember(tx, it.cid, rid) + if err != nil { + return err + } + if newRoot { + dirty[rid] = true + } + if it.key == "" { + continue + } + if it.live { + _, err = tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, it.pid, it.cid, it.key) + } else { + _, err = tx.Exec(`DELETE FROM claim_by_pid WHERE pid = ? AND cid = ? AND key = ?`, it.pid, it.cid, it.key) + } + if err != nil { + return err + } + mark(rid, it.key) + } + for rid := range all { + cids, err := cidsForRoot(ctx, tx, rid) + if err != nil { + return err + } + keys, err := c.claimKeys(ctx, cids) + if err != nil { + return err + } + for _, k := range append(keys, localFields...) { + mark(rid, k) + } + } + for rid, keys := range marks { + cids, err := cidsForRoot(ctx, tx, rid) + if err != nil { + return err + } + for key := range keys { + changed, err := c.remergeTx(tx, rid, cids, key) + if err != nil { + return err + } + if changed && affectsHaystack(key) { + dirty[rid] = true + } + } + } + for rid := range dirty { + if err := refreshSearchTx(tx, rid); err != nil { + return err + } + } + if maxSeq > 0 { + return tx.Put(appliedSeqKey, strconv.FormatInt(maxSeq, 10)) + } + return nil +} + +// syncMember makes cid a member of rid in the cache: the lookup row, the +// synthesised key-set member, and the root. Reports whether rid is new. +func syncMember(tx *kv.Tx, cid, rid string) (bool, error) { + cur, _, err := tx.Get(tx.Context(), CIDIndexPrefix+cid) + if err != nil { + return false, err + } + if cur != rid { + if cur != "" { + if _, err := tx.Delete(recordPrefix(cur) + CIDsKeyPrefix + cid); err != nil { + return false, err + } + } + if err := tx.Put(CIDIndexPrefix+cid, rid); err != nil { + return false, err + } + } + if err := tx.Put(recordPrefix(rid)+CIDsKeyPrefix+cid, "true"); err != nil { + return false, err + } + return tx.Set(rootKey(rid), "") +} + +// remergeTx recomputes file:/ from the group's votes (plus the +// box-local facts for filePath / sizeByte / mtimeNano) and reports whether +// the merged row changed. +func (c *Client) remergeTx(tx *kv.Tx, rid string, cids []string, key string) (bool, error) { + if strings.HasPrefix(key, CIDsKeyPrefix) { + return false, nil // membership is synthesised, never merged + } + votes, err := c.votesFor(tx.Context(), cids, key) + if err != nil { + return false, err + } + res := Merge(votes, "") + val, ok := res.Value, res.OK + if isLocalField(key) { + lv, lok, err := localFact(tx, cids, key) + if err != nil { + return false, err + } + // The disk is the truth for where and when a file is; a signed + // sizeByte (a content fact) wins over the on-disk size. + if lok && (key != "sizeByte" || !ok) { + val, ok = lv, true + } + } + k := recordPrefix(rid) + key + if ok { + return tx.Set(k, val) + } + return tx.Delete(k) +} + +// localFact reads a box-local field from local_file for the group's cids. +func localFact(tx *kv.Tx, cids []string, key string) (string, bool, error) { + if len(cids) == 0 { + return "", false, nil + } + args := make([]any, len(cids)) + for i, c := range cids { + args[i] = c + } + var path string + var size, mtime int64 + err := tx.QueryRow(`SELECT path, size, mtime FROM local_file WHERE midhash IN (?`+ + strings.Repeat(",?", len(cids)-1)+`) ORDER BY path LIMIT 1`, args...).Scan(&path, &size, &mtime) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + switch key { + case "filePath": + return path, true, nil + case "sizeByte": + return strconv.FormatInt(size, 10), true, nil + default: + return strconv.FormatInt(mtime, 10), true, nil + } +} + +// truthMembers lists every cid of rid's group (scans cid_group: truth has no +// rid index; only regroups and rebuilds call this). +func (c *Client) truthMembers(ctx context.Context, rid string) ([]string, error) { + rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid FROM cid_group WHERE rid = ?`, rid) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var cid string + if err := rows.Scan(&cid); err != nil { + return nil, err + } + out = append(out, cid) + } + return out, rows.Err() +} + +// truthLastSeq reads the newest committed seq. +func (c *Client) truthLastSeq(ctx context.Context) (int64, error) { + var v string + err := c.truth.DB().QueryRowContext(ctx, `SELECT value FROM meta WHERE name = ?`, lastSeqName).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + if err != nil { + return 0, err + } + return strconv.ParseInt(v, 10, 64) +} + +// catchUp brings the cache level with truth at boot: nothing to do, a replay +// of the changes after applied_seq (found by a full scan — there is no seq +// index, and this only runs after an unclean shutdown), or a full rebuild +// when the cache is missing or a group changed while it was behind. +func (c *Client) catchUp() (int64, error) { + s := c.store + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + last, err := c.truthLastSeq(ctx) + if err != nil { + return 0, err + } + appliedStr, has, err := s.Get(ctx, appliedSeqKey) + if err != nil { + return 0, err + } + applied, _ := strconv.ParseInt(appliedStr, 10, 64) + if has && applied >= last { + return applied, nil + } + if !has { + if last == 0 { + return 0, nil // fresh store + } + log.Printf("[Storage] cache is empty, rebuilding from truth (last_seq %d)", last) + return last, c.rebuild(ctx, last) + } + rows, err := c.truth.DB().QueryContext(ctx, + `SELECT cid, key, pid, value IS NOT NULL, seq FROM claim WHERE seq > ? ORDER BY seq`, applied) + if err != nil { + return 0, err + } + var missed []change + regroup := false + for rows.Next() { + var ch change + if err := rows.Scan(&ch.cid, &ch.key, &ch.pid, &ch.live, &ch.seq); err != nil { + rows.Close() + return 0, err + } + if strings.HasPrefix(ch.key, CIDsKeyPrefix) { + regroup = true + } + missed = append(missed, ch) + } + if err := rows.Close(); err != nil { + return 0, err + } + if regroup { + log.Printf("[Storage] cache behind by %d changes including a group link, rebuilding", len(missed)) + return last, c.rebuild(ctx, last) + } + log.Printf("[Storage] cache behind (applied %d < last %d), replaying %d changes", applied, last, len(missed)) + for start := 0; start < len(missed); start += applyBatchMax { + end := min(start+applyBatchMax, len(missed)) + if err := c.applyBatch(missed[start:end]); err != nil { + return 0, err + } + } + if err := s.Update(ctx, func(tx *kv.Tx) error { return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10)) }); err != nil { + return 0, err + } + return last, nil +} + +// Rebuild drops every derived row of the cache and recomputes it from truth. +// Exposed for tests and operators; boot calls it when the cache is missing. +func (c *Client) Rebuild() error { + if _, err := c.truthStore(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + last, err := c.truthLastSeq(ctx) + if err != nil { + return err + } + return c.rebuild(ctx, last) +} + +const rebuildBatch = 500 + +func (c *Client) rebuild(ctx context.Context, last int64) error { + s := c.store + started := time.Now() + for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} { + for { + var n int64 + if err := s.Update(ctx, func(tx *kv.Tx) error { + tx.Quiet() + var err error + n, err = tx.DeletePrefix(prefix, bulkBatch) + return err + }); err != nil { + return fmt.Errorf("clear %s: %w", prefix, err) + } + if n < bulkBatch { + break + } + } + } + if err := s.Update(ctx, func(tx *kv.Tx) error { + if _, err := tx.Exec(`DELETE FROM fts`); err != nil { + return err + } + _, err := tx.Exec(`DELETE FROM claim_by_pid`) + return err + }); err != nil { + return err + } + + groups := map[string][]string{} + var order []string + rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid, rid FROM cid_group`) + if err != nil { + return err + } + for rows.Next() { + var cid, rid string + if err := rows.Scan(&cid, &rid); err != nil { + rows.Close() + return err + } + if _, ok := groups[rid]; !ok { + order = append(order, rid) + } + groups[rid] = append(groups[rid], cid) + } + if err := rows.Close(); err != nil { + return err + } + for start := 0; start < len(order); start += rebuildBatch { + end := min(start+rebuildBatch, len(order)) + if err := s.Update(ctx, func(tx *kv.Tx) error { + tx.Quiet() + for _, rid := range order[start:end] { + cids := groups[rid] + for _, cid := range cids { + if _, err := syncMember(tx, cid, rid); err != nil { + return err + } + } + keys, err := c.claimKeys(ctx, cids) + if err != nil { + return err + } + for _, key := range append(keys, localFields...) { + if _, err := c.remergeTx(tx, rid, cids, key); err != nil { + return err + } + } + if err := refreshSearchTx(tx, rid); err != nil { + return err + } + } + return nil + }); err != nil { + return fmt.Errorf("rebuild records: %w", err) + } + } + if err := c.rebuildClaimIndex(ctx); err != nil { + return err + } + if err := c.RebuildCounters(); err != nil { + return err + } + rebuiltAt := strconv.FormatInt(time.Now().UnixMilli(), 10) + "-0" + if err := s.Update(ctx, func(tx *kv.Tx) error { + if err := tx.Put(rebuiltAtKey, rebuiltAt); err != nil { + return err + } + return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10)) + }); err != nil { + return err + } + log.Printf("[Storage] cache rebuilt: %d records in %s", len(order), time.Since(started).Round(time.Millisecond)) + return nil +} + +// rebuildClaimIndex refills claim_by_pid from truth, a page at a time. +func (c *Client) rebuildClaimIndex(ctx context.Context) error { + type row struct { + pid int64 + cid, key string + } + var after row + first := true + for { + q := `SELECT pid, cid, key FROM claim WHERE value IS NOT NULL` + var args []any + if !first { + q += ` AND (cid, key, pid) > (?, ?, ?)` + args = append(args, after.cid, after.key, after.pid) + } + q += ` ORDER BY cid, key, pid LIMIT 5000` + rs, err := c.truth.DB().QueryContext(ctx, q, args...) + if err != nil { + return err + } + var page []row + for rs.Next() { + var r row + if err := rs.Scan(&r.pid, &r.cid, &r.key); err != nil { + rs.Close() + return err + } + page = append(page, r) + } + if err := rs.Close(); err != nil { + return err + } + if len(page) == 0 { + return nil + } + if err := c.store.Update(ctx, func(tx *kv.Tx) error { + for _, r := range page { + if _, err := tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, r.pid, r.cid, r.key); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + after, first = page[len(page)-1], false + } +} + +// RebuiltAt is the stream id of the last cache rebuild ("" if never). +func (c *Client) RebuiltAt() string { + s, err := c.kv() + if err != nil { + return "" + } + ctx, cancel := opCtx() + defer cancel() + v, _, _ := s.Get(ctx, rebuiltAtKey) + return v +} diff --git a/internal/storage/cid_resolution.go b/internal/storage/cid_resolution.go index 5559776..59cd226 100644 --- a/internal/storage/cid_resolution.go +++ b/internal/storage/cid_resolution.go @@ -3,34 +3,22 @@ package storage import ( "context" "fmt" - "strconv" "strings" "github.com/metazla/meta-core/internal/cid" "github.com/metazla/meta-core/internal/kv" ) -// This file implements the CID-resolution layer that lives on top of the -// flat metadata storage in client.go. The short version: +// The cid → record resolution layer, read side. A record (rid) is a group of +// cids known to be the same content; the group lives in truth's cid_group and +// is mirrored into the cache as // -// - Roots are opaque UUIDv7s — file:/ for every property. -// The UUID is an internal implementation detail; the mental model is -// "N CIDs → one record of keys→metadata". -// - Every known CID for a file is registered as a reverse-index entry: -// cid: → -// and as a member of the per-root bare-CID key-set: -// file:/cids/ = "true" (flat key, sentinel value) -// - A CID is self-describing: its algorithm is recoverable from the CIDv1 -// multicodec (see internal/cid), so there is no per-algorithm field name -// and no : token prefix. The canonical externally-visible CID is -// *derived by rank* from the key-set on read (GetMetadataDocument), never -// stored. +// cid: → +// file:/cids/ = "true" (synthesised membership, one per cid) // -// The record write path (writeFieldsTx) catches any write of a cids/ -// field and registers the reverse-index entry in the same transaction, so -// writers that just emit the key-set members don't need to know about the -// reverse index at all. The helpers here take the caller's *kv.Tx: a write -// must never open a second transaction (there is one writer connection). +// by the apply loop (apply.go). A CID is self-describing (its algorithm is in +// the CIDv1 multicodec, see internal/cid); the canonical externally-visible +// CID is derived by rank on read (GetMetadataDocument), never stored. // CIDIndexPrefix is the key prefix for the reverse index. Combined // with a bare CID, it forms the full key: "cid:bafk…". @@ -43,114 +31,6 @@ const CIDIndexPrefix = "cid:" // reverse-index entries that need cleanup). const CIDsKeyPrefix = "cids/" -// Duplicate paths — other physical locations whose content hashes to the same -// CIDs as a root — live as dup:/ = "" (a key-set). The watcher -// writes one when it sees a new path for content that's already indexed. - -// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it -// in the root index. Returns the new UUID. The caller is expected to follow -// up with AddAlias for at least one CID — a root without any alias is -// reachable only via the index, which works but defeats the point. -func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) { - var uuid string - err := c.update(func(tx *kv.Tx) error { - var err error - uuid, err = mintTx(tx, filePath, size, mtimeNano) - return err - }) - return uuid, err -} - -func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) { - uuid, err := NewUUID() - if err != nil { - return "", fmt.Errorf("uuid: %w", err) - } - return uuid, writeFieldsTx(tx, uuid, map[string]string{ - "filePath": filePath, - "sizeByte": strconv.FormatInt(size, 10), - "mtimeNano": strconv.FormatInt(mtimeNano, 10), - }) -} - -// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse -// index cid: → uuid and the key-set member cids/ = "true". -// Idempotent: registering the same CID twice is a no-op. -func (c *Client) AddAlias(uuid, cidStr string) error { - if uuid == "" || cidStr == "" { - return fmt.Errorf("uuid and cid are required") - } - return c.update(func(tx *kv.Tx) error { - return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"}) - }) -} - -// addAliasTx registers the reverse-index entry for a cids/ member the -// caller is writing (writeFieldsTx writes the member itself). -func addAliasTx(tx *kv.Tx, uuid, cidStr string) error { - if uuid == "" || cidStr == "" { - return fmt.Errorf("uuid and cid are required") - } - // Self-pointing alias guard. If the caller is trying to register - // cid: → (i.e. the root IS the cid value), that's the - // dual-root failure mode in disguise — it means somebody wrote - // /meta/ without first resolving via ResolveRoot. Refuse the alias - // write; leave any existing alias alone. The key-set membership is still - // recorded by the caller, so the field-level write succeeds. - if uuid == cidStr { - return nil - } - // Guard against the dual-root pattern: if cid: already points at a - // DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the - // key-set membership keeps the CID discoverable via root → cids. - indexKey := CIDIndexPrefix + cidStr - existing, _, err := tx.Get(tx.Context(), indexKey) - if err != nil { - return fmt.Errorf("read existing alias: %w", err) - } - if existing != "" && existing != uuid { - return nil - } - if existing == uuid { - return nil - } - return tx.Put(indexKey, uuid) -} - -// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index -// entry cid: → . The caller deletes the cids/ key-set member -// itself (DeleteProperty does both). -// -// ⚠ Why this exists. A cids/ member write touches two keys — the flat -// file:/cids/ and this reverse alias — while the delete path once -// only removed the first. A "deleted" CID therefore still resolved: GET -// /meta/, ResolveRoot and GetByCID all kept answering with the record. -// The asymmetry is not theoretical. meta-share published dag-pb roots computed -// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing -// the member was expressible, removing the claim was not, so the false -// identity survived its own retraction. -// -// Guarded, not blind: the alias is removed only when it currently points at -// THIS uuid. Two records can legitimately race to claim a cid (see the -// dual-root guard in addAliasTx, which leaves the first writer's alias in -// place); deleting a member from the loser must not unpick the winner's index. -func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error { - if uuid == "" || cidStr == "" { - return nil - } - indexKey := CIDIndexPrefix + cidStr - existing, ok, err := tx.Get(tx.Context(), indexKey) - if err != nil { - return fmt.Errorf("read existing alias: %w", err) - } - if !ok || existing != uuid { - // Absent, or somebody else owns this alias. Leave it. - return nil - } - _, err = tx.Delete(indexKey) - return err -} - // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor // lookup, an SSE consumer) to the actual storage root key. // @@ -202,23 +82,6 @@ func (c *Client) GetByCID(cidStr string) (string, error) { return uuid, nil } -// AddDuplicatePath records that path is another physical location whose -// content matches uuid. Returns true if path was newly added. The watcher -// calls this when it sees a file whose midhash matches an existing root but -// at a different filePath. -func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) { - if uuid == "" || path == "" { - return false, nil - } - added := false - err := c.update(func(tx *kv.Tx) error { - var err error - added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "") - return err - }) - return added, err -} - // cidsForRoot returns a root's bare-CID key-set members (cids/ prefix // stripped). func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) { @@ -234,20 +97,6 @@ func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error return out, nil } -// DeleteRoot removes everything keyed off uuid: every file:/* property -// (including the cids/ key-set), every reverse-index entry for those -// CIDs, its duplicates, search row and index membership. Idempotent — a uuid -// with no remaining keys is a successful no-op. -func (c *Client) DeleteRoot(uuid string) error { - if uuid == "" { - return nil - } - return c.update(func(tx *kv.Tx) error { - _, err := deleteRecordTx(tx, uuid) - return err - }) -} - // cidFromKeysetField returns the bare CID encoded in a cids/ field name, // or "" if the field isn't a key-set member. func cidFromKeysetField(field string) string { @@ -311,13 +160,16 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { } } - dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0) + // Duplicates: every other local path holding the same content. + paths, err := localPaths(ctx, s.DB(), cids) if err != nil { return nil, fmt.Errorf("read duplicates: %w", err) } - dups := make([]string, 0, len(dupRows)) - for _, row := range dupRows { - dups = append(dups, row.K[len(dupPrefix(uuid)):]) + dups := make([]string, 0, len(paths)) + for _, p := range paths { + if p != flat["filePath"] { + dups = append(dups, p) + } } if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 { @@ -330,66 +182,3 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { Duplicates: dups, }, nil } - -// RegisterFileTuple records one physical file under its midhash256 CID, the -// same way the file watcher does for every file it scans: mint a UUID root -// (filePath/sizeByte/mtimeNano) and write `cids/`, whose hook -// registers the cid: reverse alias. Content already known under -// another path is recorded as a duplicate; the same path just refreshes -// size/mtime. -// -// Returns the root and whether it was newly minted. Shared by the watcher and -// POST /api/files/register, which is how a box running with the watcher off -// (a meta-watch client core) still gets files it writes under /files/plugin -// resolvable by CID. -func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) { - if midhash == "" { - return "", false, fmt.Errorf("midhash is required") - } - var ( - uuid string - minted bool - ) - // One transaction: the lookup and the mint cannot interleave with another - // registration of the same content, so two concurrent scans of one file - // can no longer mint two roots for it. - err := c.update(func(tx *kv.Tx) error { - ctx := tx.Context() - existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash) - if err != nil { - return fmt.Errorf("GetByCID(%s): %w", midhash, err) - } - uuid = existing - if uuid == "" { - if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil { - return fmt.Errorf("Mint(%s): %w", absPath, err) - } - minted = true - if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil { - return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err) - } - return nil - } - current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath") - if err != nil { - return err - } - if current != absPath { - if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil { - return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err) - } - return nil - } - if err := writeFieldsTx(tx, uuid, map[string]string{ - "sizeByte": strconv.FormatInt(size, 10), - "mtimeNano": strconv.FormatInt(mtimeNano, 10), - }); err != nil { - return fmt.Errorf("refresh tuple for %s: %w", uuid, err) - } - return nil - }) - if err != nil { - return "", false, err - } - return uuid, minted, nil -} diff --git a/internal/storage/cid_retraction_test.go b/internal/storage/cid_retraction_test.go deleted file mode 100644 index c3f1dd9..0000000 --- a/internal/storage/cid_retraction_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package storage - -import ( - "context" - "testing" -) - -// Tests for retracting a `cids/` key-set member (cid_resolution.go's -// removeAliasTx, reached through DeleteProperty). -// -// Why this is worth its own file. Adding a member writes two keys — the flat -// `file:/cids/` and the reverse alias `cid:` → ``. Until -// removeAliasLocked (now removeAliasTx) existed, deleting removed only the -// first, so a retracted CID kept resolving to the record: the member was gone -// from the document and the claim was not. -// -// That asymmetry had a live consequence. meta-share minted dag-pb roots and -// midhash256s over whatever `archive::pick_media` elected, which on an -// obfuscated posting was an arbitrary RAR volume or a par2 file. Six records on -// watch.nsl.sh carry such members. `METADATA_KEYS.md` §2 defines a `cids/` -// member as a digest of *this record's file* and notes that every member is a -// reverse index back to the record — so those entries assert that an archive -// volume is the episode. Retracting one has to remove the assertion, not just -// the row. - -// aliasTarget returns the uuid that cid: currently resolves to, or "". -func aliasTarget(t *testing.T, c *Client, cidStr string) string { - t.Helper() - got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr) - if err != nil { - t.Fatalf("get alias: %v", err) - } - return got -} - -func hasField(t *testing.T, c *Client, hashID, field string) bool { - t.Helper() - fields, err := c.GetMetadataFlat(hashID) - if err != nil { - t.Fatalf("read record: %v", err) - } - _, ok := fields[field] - return ok -} - -// The real shape: `…glnrqztt…` is the nzb-release locator whose record published -// a dag-pb root computed over volume 3 of a 35-volume RAR5 set. -const ( - poisonedRoot = "bafybeicicuzdvuztca2hp2g2k7scgr4nnibilzgoppljeprphybqot7v6i" - goodMidhash = "bagacbabaed6uiulnv7tscpxne22eztkjvnxg5almn43dpfecppixarfe7xhlu" -) - -func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) { - withClient(t, func(t *testing.T, c *Client) { - const uuid = "01KSBHMTESTRECORD" - - if err := c.SetMetadataFlat(uuid, map[string]string{ - "cids/" + poisonedRoot: "true", - "cids/" + goodMidhash: "true", - "fileType": "video", - }); err != nil { - t.Fatalf("seed: %v", err) - } - - // All three writes landed, and the CID resolves to the record. - if got := aliasTarget(t, c, poisonedRoot); got != uuid { - t.Fatalf("alias should point at the record before retraction, got %q", got) - } - if !hasField(t, c, uuid, "cids/"+poisonedRoot) { - t.Fatal("field index should list the member before retraction") - } - if got := c.ResolveRoot(poisonedRoot); got != uuid { - t.Fatalf("ResolveRoot should find the record before retraction, got %q", got) - } - - if err := c.DeleteProperty(uuid, "cids/"+poisonedRoot); err != nil { - t.Fatalf("DeleteProperty: %v", err) - } - - // 1. the flat key - if v, ok, _ := c.Store().Get(context.Background(), - recordPrefix(uuid)+"cids/"+poisonedRoot); ok { - t.Fatalf("flat key should be gone, still %q", v) - } - // 2. the record read - if hasField(t, c, uuid, "cids/"+poisonedRoot) { - t.Fatal("record read still lists the retracted member") - } - // 3. the reverse alias — the one that used to survive - if got := aliasTarget(t, c, poisonedRoot); got != "" { - t.Fatalf("reverse alias survived retraction, still points at %q", got) - } - if got := c.ResolveRoot(poisonedRoot); got == uuid { - t.Fatal("a retracted cid must not resolve to the record any more") - } - - // The siblings are untouched: retraction is surgical, not a purge. - if got := aliasTarget(t, c, goodMidhash); got != uuid { - t.Fatalf("an unrelated sibling lost its alias, got %q", got) - } - if !hasField(t, c, uuid, "cids/"+goodMidhash) { - t.Fatal("an unrelated sibling lost its field-index entry") - } - if v, err := c.GetProperty(uuid, "fileType"); err != nil || v != "video" { - t.Fatalf("an unrelated field changed: %q %v", v, err) - } - }) -} - -// Two records can race to claim one cid; addAliasLocked leaves the first -// writer's alias in place. Retracting the loser's member must not unpick the -// winner's index — which is why removeAliasLocked compares before deleting. -func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) { - withClient(t, func(t *testing.T, c *Client) { - const winner = "01KSBHMWINNER" - const loser = "01KSBHMLOSER" - const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m" - - if err := c.SetMetadataFlat(winner, map[string]string{"cids/" + shared: "true"}); err != nil { - t.Fatalf("seed winner: %v", err) - } - if err := c.SetMetadataFlat(loser, map[string]string{"cids/" + shared: "true"}); err != nil { - t.Fatalf("seed loser: %v", err) - } - if got := aliasTarget(t, c, shared); got != winner { - t.Fatalf("the first writer should own the alias, got %q", got) - } - - if err := c.DeleteProperty(loser, "cids/"+shared); err != nil { - t.Fatalf("DeleteProperty: %v", err) - } - - if got := aliasTarget(t, c, shared); got != winner { - t.Fatalf("retracting the loser's member stole the winner's alias, now %q", got) - } - if hasField(t, c, loser, "cids/"+shared) { - t.Fatal("the loser's own member should still be gone") - } - }) -} - -// A non-cids field has no alias to clean up, and the delete must not invent one. -func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) { - withClient(t, func(t *testing.T, c *Client) { - const uuid = "01KSBHMORDINARY" - if err := c.SetMetadataFlat(uuid, map[string]string{ - "cids/" + goodMidhash: "true", - "quality": "1080p", - }); err != nil { - t.Fatalf("seed: %v", err) - } - if err := c.DeleteProperty(uuid, "quality"); err != nil { - t.Fatalf("DeleteProperty: %v", err) - } - if got := aliasTarget(t, c, goodMidhash); got != uuid { - t.Fatalf("deleting an ordinary field disturbed a cid alias, got %q", got) - } - }) -} diff --git a/internal/storage/claims_test.go b/internal/storage/claims_test.go new file mode 100644 index 0000000..d1cc8a4 --- /dev/null +++ b/internal/storage/claims_test.go @@ -0,0 +1,419 @@ +package storage + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "testing" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/kv" +) + +func TestWriteClaimStatuses(t *testing.T) { + c := newTestClient(t) + a := signer(t, 1) + good := vote(t, a, "bafyA", "title", "The Matrix", 100) + tampered := vote(t, a, "bafyA", "year", "1999", 100) + tampered.Value = claim.Text("2000") + highS := vote(t, a, "bafyA", "studio", "WB", 100) + highS.Sig = flipS(highS.Sig) + + got := write(t, c, + good, + tampered, + highS, + claim.Claim{CID: "bafyA", UID: a.UID, TS: 1}, // no key + vote(t, a, "bafyA", "cid_sha256", "x", 100), // removed CID shape + ) + want := []ClaimStatus{StatusAccepted, StatusBadSignature, StatusBadSignature, StatusBadRequest, StatusBadRequest} + if !reflect.DeepEqual(got, want) { + t.Fatalf("statuses %v, want %v", got, want) + } + if st := write(t, c, good); st[0] != StatusUnchanged { + t.Fatalf("identical re-send: %s, want unchanged", st[0]) + } + if st := write(t, c, vote(t, a, "bafyA", "title", "Older", 50)); st[0] != StatusStale { + t.Fatalf("older ts: %s, want stale", st[0]) + } + if st := write(t, c, vote(t, a, "bafyA", "title", "Same ts other value", 100)); st[0] != StatusStale { + t.Fatalf("equal ts, different claim: %s, want stale", st[0]) + } + if got := record(t, c, "bafyA")["title"]; got != "The Matrix" { + t.Fatalf("title = %q", got) + } + // A re-persist of the same value with a newer ts keeps the stored row (and + // its older ts): it is "unchanged", not a new version. + if st := write(t, c, vote(t, a, "bafyA", "title", "The Matrix", 200)); st[0] != StatusUnchanged { + t.Fatalf("same value, newer ts: %s, want unchanged", st[0]) + } + // Proof the stored ts stayed 100: a new value at 150 (< 200) is accepted. + if st := write(t, c, vote(t, a, "bafyA", "title", "Updated", 150)); st[0] != StatusAccepted { + t.Fatalf("new value at ts 150 after a same-value re-send at 200: %s, want accepted", st[0]) + } +} + +func flipS(sig []byte) []byte { + // n - s for secp256k1, computed via the vectors' helper in internal/claim + // would be cleaner; here big-endian subtraction on bytes is enough. + n := []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, + 0xba, 0xae, 0xdc, 0xe6, 0xaf, 0x48, 0xa0, 0x3b, 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x41, + } + out := append([]byte{}, sig...) + borrow := 0 + for i := 31; i >= 0; i-- { + d := int(n[i]) - int(sig[32+i]) - borrow + borrow = 0 + if d < 0 { + d += 256 + borrow = 1 + } + out[32+i] = byte(d) + } + return out +} + +// A retraction is kept as a tombstone, so the vote it retracted cannot be +// replayed — by its signer's old copy or by a peer that still holds it. +func TestRetractionKeepsTombstoneAndBlocksReplay(t *testing.T) { + c := newTestClient(t) + a := signer(t, 1) + old := vote(t, a, "bafyA", "genres/Drama", "true", 10) + mustAccept(t, c, old, vote(t, a, "bafyA", "title", "X", 10)) + mustAccept(t, c, vote(t, a, "bafyA", "genres/Drama", nil, 20)) + if _, ok := record(t, c, "bafyA")["genres/Drama"]; ok { + t.Fatal("retracted member still merged") + } + if st := write(t, c, old); st[0] != StatusStale { + t.Fatalf("replay of the retracted vote: %s, want stale", st[0]) + } + if n, _ := c.CountClaims(); n != 2 { + t.Fatalf("claims = %d, want 2 (tombstone kept)", n) + } + live, _ := c.ClaimsForCIDs([]string{"bafyA"}, "") + if len(live) != 1 || live[0].Key != "title" { + t.Fatalf("live claims = %+v, want only title", live) + } +} + +func TestMergeRule(t *testing.T) { + tx := func(s string) claim.Value { return claim.Text(s) } + cases := []struct { + name string + votes []Vote + viewer string + want string + nsrc int + ok bool + }{ + {"support wins", []Vote{{Uid: "a", Value: tx("X"), TS: 9}, {Uid: "b", Value: tx("x "), TS: 9}, {Uid: "c", Value: tx("Y"), TS: 1}}, "", "X", 2, true}, + {"tie: oldest wins", []Vote{{Uid: "a", Value: tx("New"), TS: 9}, {Uid: "b", Value: tx("Old"), TS: 1}}, "", "Old", 1, true}, + {"tie: smallest wins", []Vote{{Uid: "a", Value: tx("b"), TS: 5}, {Uid: "b", Value: tx("a"), TS: 5}}, "", "a", 1, true}, + {"signer counts once, newest", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: tx("Y"), TS: 2}, {Uid: "b", Value: tx("Y"), TS: 3}}, "", "Y", 2, true}, + {"retraction drops", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: claim.Null(), TS: 2}}, "", "", 0, false}, + {"ciphertext never merges", []Vote{{Uid: "a", Value: claim.Bytes([]byte{1}), TS: 1}}, "", "", 0, false}, + {"viewer wins", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "b", Value: tx("X"), TS: 1}, {Uid: "v", Value: tx("Mine"), TS: 9}}, "v", "Mine", 1, true}, + {"viewer retraction does not hide", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "v", Value: claim.Null(), TS: 9}}, "v", "X", 1, true}, + } + for _, tc := range cases { + got := Merge(tc.votes, tc.viewer) + if got.OK != tc.ok || got.Value != tc.want || (tc.ok && got.Nsrc != tc.nsrc) { + t.Errorf("%s: got %+v, want %q nsrc %d ok %v", tc.name, got, tc.want, tc.nsrc, tc.ok) + } + } +} + +func TestMergedViewAndClaims(t *testing.T) { + c := newTestClient(t) + gwA, gwB, gwC := signer(t, 1), signer(t, 2), signer(t, 3) + mustAccept(t, c, + vote(t, gwA, "bafyA", "title", "Matrix", 10), + vote(t, gwB, "bafyA", "title", "Matrix", 20), + vote(t, gwC, "bafyA", "title", "The Matrix (1999)", 5), + ) + rec := record(t, c, "bafyA") + if rec["title"] != "Matrix" || rec["cids/bafyA"] != "true" { + t.Fatalf("merged %v", rec) + } + claims, err := c.ClaimsForRID(rid(t, c, "bafyA"), "title") + if err != nil || len(claims) != 3 { + t.Fatalf("claims %d (err %v), want 3", len(claims), err) + } + for _, cl := range claims { + if err := cl.Verify(); err != nil { + t.Fatalf("returned claim does not verify: %v", err) + } + } +} + +// A cids/* link merges two groups: the older rid survives, the loser's rows +// go with `del` events, and the survivor carries both groups' votes. +func TestGroupLinkMergesRecords(t *testing.T) { + c := newTestClient(t) + a, b := signer(t, 1), signer(t, 2) + mustAccept(t, c, vote(t, a, "bafyM", "title", "Local copy", 10)) + older := rid(t, c, "bafyM") + mustAccept(t, c, vote(t, b, "bafyQ", "year", "1999", 10)) + newer := rid(t, c, "bafyQ") + if older == newer || older > newer { + t.Fatalf("expected two groups, older first: %s %s", older, newer) + } + before, _ := c.Store().LastID(context.Background(), MetaEventsStream) + + mustAccept(t, c, vote(t, a, "bafyM", "cids/bafyQ", "true", 11)) + if got := rid(t, c, "bafyQ"); got != older { + t.Fatalf("bafyQ → %s, want the older rid %s", got, older) + } + rec := record(t, c, "bafyM") + if rec["title"] != "Local copy" || rec["year"] != "1999" || rec["cids/bafyQ"] != "true" || rec["cids/bafyM"] != "true" { + t.Fatalf("survivor %v", rec) + } + if f, _ := c.GetMetadataFlat(newer); f != nil { + t.Fatalf("loser still has rows: %v", f) + } + entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 1000) + sawDel := false + for _, e := range entries { + if e.Values["type"] == "del" && strings.HasPrefix(e.Values["key"], "file:"+newer+"/") { + sawDel = true + } + } + if !sawDel { + t.Fatal("no del event for the losing rid") + } + if n, _ := c.CountRoots(); n != 1 { + t.Fatalf("roots = %d, want 1", n) + } +} + +func TestViewerOverlay(t *testing.T) { + c := newTestClient(t) + a, b, me := signer(t, 1), signer(t, 2), signer(t, 9) + mustAccept(t, c, + vote(t, a, "bafyA", "title", "Shared", 1), + vote(t, b, "bafyA", "title", "Shared", 1), + vote(t, me, "bafyA", "title", "Mine", 2), + vote(t, me, "bafyA", "seek", []byte{1, 2, 3}, 2), // ciphertext: never shown + ) + r := rid(t, c, "bafyA") + shared, _ := c.GetMetadataFlat(r) + mine, _ := c.GetMetadataFlatAs(r, me.UID) + if shared["title"] != "Shared" || mine["title"] != "Mine" { + t.Fatalf("shared %q, mine %q", shared["title"], mine["title"]) + } + if _, ok := mine["seek"]; ok { + t.Fatal("ciphertext leaked into the view") + } + match := func(_ string, f map[string]string) bool { return strings.Contains(strings.ToLower(f["title"]), "mine") } + if hits, _ := c.Search("", nil, 10, match); len(hits) != 0 { + t.Fatalf("shared search found the viewer's value: %v", hits) + } + hits, err := c.SearchAs("", nil, 10, me.UID, match) + if err != nil || len(hits) != 1 { + t.Fatalf("viewer search hits %v (err %v), want 1", hits, err) + } + merged, _, dropped, err := c.MergeClaims([]claim.Claim{ + vote(t, a, "bafyA", "title", "Shared", 1), + vote(t, b, "bafyZ", "title", "Other", 1), + }, me.UID) + if err != nil || dropped != 0 || merged["title"] != "Mine" { + t.Fatalf("stateless merge as viewer = %v dropped %d err %v", merged, dropped, err) + } +} + +// A crash between the truth commit and the cache apply leaves the cache +// behind; the next boot replays the missed changes from truth. +func TestCrashBetweenCommitsIsCaughtUp(t *testing.T) { + dir := t.TempDir() + c := openClient(t, dir) + a := signer(t, 1) + mustAccept(t, c, vote(t, a, "bafyA", "title", "Before", 1)) + + // Commit straight to truth, as WriteClaims does, but never apply. + lost := vote(t, a, "bafyA", "title", "After", 2) + if err := c.truth.Update(context.Background(), func(tx *kv.Tx) error { + seq, err := readLastSeq(tx) + if err != nil { + return err + } + pid, err := ensurePrincipal(tx, lost.UID) + if err != nil { + return err + } + seq++ + if _, err := tx.Exec(`UPDATE claim SET value = ?, ts = ?, sig = ?, seq = ? WHERE cid = ? AND key = ? AND pid = ?`, + "After", int64(lost.TS), lost.Sig, seq, lost.CID, lost.Key, pid); err != nil { + return err + } + return writeLastSeq(tx, seq) + }); err != nil { + t.Fatal(err) + } + if got := record(t, c, "bafyA")["title"]; got != "Before" { + t.Fatalf("cache applied a change it was never given: %q", got) + } + c.Close() + + c = openClient(t, dir) + defer c.Close() + if got := record(t, c, "bafyA")["title"]; got != "After" { + t.Fatalf("after catch-up title = %q, want After", got) + } +} + +// The cache is a pure function of truth: deleting it and rebuilding yields +// exactly what incremental application produced. +func TestRebuildEqualsIncremental(t *testing.T) { + dir := t.TempDir() + c := openClient(t, dir) + a, b, u := signer(t, 1), signer(t, 2), signer(t, 3) + mustAccept(t, c, + vote(t, a, "bafy1", "title", "One", 1), vote(t, a, "bafy1", "genres/Action", "true", 1), + vote(t, b, "bafy1", "title", "Uno", 2), vote(t, b, "bafy2", "title", "Two", 2), + vote(t, a, "bafy2", "sizeByte", "123", 1), vote(t, u, "bafy2", "like", "true", 3), + vote(t, u, "bafy2", "seek", []byte{9}, 3), + ) + mustAccept(t, c, + vote(t, a, "bafy1", "genres/Action", nil, 4), // retraction + vote(t, b, "bafy1", "cids/bafy2", "true", 5), // group merge + vote(t, a, "bafy3", "title", "Three", 5), + ) + incremental := cacheState(t, c) + c.Close() + for _, f := range []string{"cache.db", "cache.db-wal", "cache.db-shm"} { + _ = os.Remove(filepath.Join(dir, f)) + } + c = openClient(t, dir) + defer c.Close() + if rebuilt := cacheState(t, c); !reflect.DeepEqual(rebuilt, incremental) { + t.Fatalf("rebuild differs from incremental:\n incremental %v\n rebuilt %v", incremental, rebuilt) + } + if c.RebuiltAt() == "" { + t.Fatal("rebuild did not record rebuilt_at (SSE consumers would not get a gap)") + } +} + +// cacheState is every derived row that must be a function of truth. +func cacheState(t *testing.T, c *Client) []string { + t.Helper() + var out []string + for _, k := range allKeys(t, c) { + if strings.HasPrefix(k, "ev:") || strings.HasPrefix(k, "evhwm:") || strings.HasPrefix(k, "meta:") || + strings.HasPrefix(k, ftsNS) || k == countersReady { + continue + } + v, _, _ := c.Store().Get(context.Background(), k) + out = append(out, k+"="+v) + } + rows, err := c.Store().DB().Query(`SELECT pid, cid, key FROM claim_by_pid`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var pid int64 + var cid, key string + _ = rows.Scan(&pid, &cid, &key) + out = append(out, "byp:"+strconv.FormatInt(pid, 10)+"/"+cid+"/"+key) + } + var hays []string + hr, _ := c.Store().DB().Query(`SELECT rid, hay FROM fts`) + for hr.Next() { + var r, h string + _ = hr.Scan(&r, &h) + hays = append(hays, "fts:"+r+"="+h) + } + hr.Close() + out = append(out, hays...) + sort.Strings(out) + return out +} + +func TestLocalFileFactsAndEvents(t *testing.T) { + c := newTestClient(t) + before, _ := c.Store().LastID(context.Background(), MetaEventsStream) + r, created, err := c.RegisterFileTuple("/files/watch/a.mkv", 1000, 42, "bafkmid") + if err != nil || !created { + t.Fatalf("register: %v created=%v", err, created) + } + rec, _ := c.GetMetadataFlat(r) + if rec["filePath"] != "/files/watch/a.mkv" || rec["sizeByte"] != "1000" || rec["mtimeNano"] != "42" || rec["cids/bafkmid"] != "true" { + t.Fatalf("record %v", rec) + } + entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 100) + sawPath := false + for _, e := range entries { + if e.Values["type"] == "set" && e.Values["key"] == "file:"+r+"/filePath" { + sawPath = true + } + } + if !sawPath { + t.Fatal("no set event for file:/filePath (meta-fuse builds its tree from it)") + } + // A signed sizeByte (content fact) wins over the on-disk size. + mustAccept(t, c, vote(t, signer(t, 1), "bafkmid", "sizeByte", "999", 1)) + if got, _ := c.GetProperty(r, "sizeByte"); got != "999" { + t.Fatalf("sizeByte = %q, want the signed 999", got) + } + // A second path of the same content is a duplicate. + if _, _, err := c.RegisterFileTuple("/files/watch/copy.mkv", 1000, 43, "bafkmid"); err != nil { + t.Fatal(err) + } + doc, _ := c.GetMetadataDocument(r) + if len(doc.Duplicates) != 1 || doc.Duplicates[0] != "/files/watch/copy.mkv" { + t.Fatalf("duplicates %v", doc.Duplicates) + } + tuples, _ := c.GetTuplesForAllFiles() + if len(tuples) != 2 || tuples[0].HashID != r { + t.Fatalf("tuples %+v", tuples) + } + // Deleting the files drops the local facts, keeps the claims. + mark, _ := c.Store().LastID(context.Background(), MetaEventsStream) + for _, p := range []string{"/files/watch/a.mkv", "/files/watch/copy.mkv"} { + if err := c.RemoveFileTuple(p); err != nil { + t.Fatal(err) + } + } + rec, _ = c.GetMetadataFlat(r) + if _, ok := rec["filePath"]; ok || rec["sizeByte"] != "999" { + t.Fatalf("after delete %v", rec) + } + entries, _ = c.Store().ReadAfter(context.Background(), MetaEventsStream, mark, 100) + sawDel := false + for _, e := range entries { + if e.Values["type"] == "del" && e.Values["key"] == "file:"+r+"/filePath" { + sawDel = true + } + } + if !sawDel { + t.Fatal("no del event for filePath") + } +} + +func TestSchemaGateRefusesV1Store(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meta-core.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID; + INSERT INTO kv VALUES('meta-core:schema-version','2')`); err != nil { + t.Fatal(err) + } + db.Close() + c := NewClient() + if err := c.Connect(path, filepath.Join(dir, "cache.db")); err != nil { + t.Fatal(err) + } + defer c.Close() + if err := c.EnsureSchemaVersion(SchemaVersion); err == nil || !strings.Contains(err.Error(), "v1.x") { + t.Fatalf("v1 store accepted: %v", err) + } +} diff --git a/internal/storage/clear_test.go b/internal/storage/clear_test.go index f00d5a0..57f23a9 100644 --- a/internal/storage/clear_test.go +++ b/internal/storage/clear_test.go @@ -1,79 +1,27 @@ package storage import ( - "context" - "path/filepath" "strings" "testing" - - "github.com/metazla/meta-core/internal/kv" ) -// newTestClient opens a fresh database in the test's temp dir. -func newTestClient(t *testing.T) *Client { - t.Helper() - c := NewClient() - if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil { - t.Fatalf("connect: %v", err) - } - t.Cleanup(func() { _ = c.Close() }) - return c -} - -// allKeys lists every row (test helper; full scan). -func allKeys(t *testing.T, c *Client) []string { - t.Helper() - var keys []string - if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error { - for _, r := range rows { - keys = append(keys, r.K) - } - return nil - }); err != nil { - t.Fatalf("scan: %v", err) - } - return keys -} - -// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the -// leak: ClearAllMetadata used to delete every record key and the root index, -// but left the cid: → root reverse-index entries behind. Those stale -// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry -// silently redirects a later write into a deleted root. -func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) { +// ClearAllMetadata wipes BOTH databases: votes, signers and groups in truth; +// merged records, the cid lookup, roots, search rows, the claim index and the +// local files in the cache. Service scratch keys are not metadata and survive. +func TestClearAllMetadataWipesTruthAndCache(t *testing.T) { c := newTestClient(t) - - // Two records, each with several CID aliases (a midhash plus siblings) — - // exactly what the watcher produces. - aliases := map[string][]string{} - for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} { - uuid, err := c.Mint(f, int64(100+i), int64(1000+i)) - if err != nil { - t.Fatalf("mint: %v", err) - } - cids := []string{"bafkmid" + uuid, "bafkfull" + uuid} - for _, cidStr := range cids { - if err := c.AddAlias(uuid, cidStr); err != nil { - t.Fatalf("add alias %s: %v", cidStr, err) - } - } - if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil { - t.Fatalf("add duplicate: %v", err) - } - aliases[uuid] = cids + a := signer(t, 1) + mustAccept(t, c, + vote(t, a, "bafkone", "title", "One", 1), + vote(t, a, "bafkone", "cids/bafkonefull", "true", 1), + vote(t, a, "bafktwo", "title", "Two", 1), + ) + if _, _, err := c.RegisterFileTuple("/files/watch/a.mkv", 10, 20, "bafkmid"); err != nil { + t.Fatal(err) } - - // An entry already orphaned by a previous (buggy) clear: a reverse-index - // key whose root no longer exists. The sweep is expected to collect it too. - if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil { - t.Fatalf("seed orphan: %v", err) - } - - for uuid, cids := range aliases { - for _, cidStr := range cids { - if got := c.ResolveRoot(cidStr); got != uuid { - t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid) - } + for k, v := range map[string]string{"btmagnet:abc": "magnet:?xt=urn:btih:abc", "searchcov:tmdb:1": "1"} { + if err := c.SetServiceValue(k, v); err != nil { + t.Fatalf("seed %s: %v", k, err) } } @@ -81,72 +29,47 @@ func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) { if err != nil { t.Fatalf("ClearAllMetadata: %v", err) } - if deleted != 2 { - t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted) + if deleted != 3 { + t.Errorf("deleted = %d, want 3 records", deleted) } - - // Nothing record-shaped may survive: not the records, not their aliases, - // duplicates or search rows, not the pre-existing orphan. (The event - // streams and the schema sentinel are not metadata and stay.) for _, k := range allKeys(t, c) { - for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} { + for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} { if strings.HasPrefix(k, ns) { t.Errorf("key survived the clear: %q", k) } } } - var ftsRows int - if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 { - t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err) - } - - for _, cids := range aliases { - for _, cidStr := range cids { - if got := c.ResolveRoot(cidStr); got != cidStr { - t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got) - } + for _, table := range []string{"fts", "claim_by_pid", "local_file"} { + var n int + if err := c.Store().DB().QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil || n != 0 { + t.Errorf("%s rows after clear = %d (err %v)", table, n, err) } } - if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" { - t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err) + if n, err := c.CountClaims(); err != nil || n != 0 { + t.Errorf("claims after clear = %d (err %v)", n, err) } -} - -// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius: -// keys outside the metadata namespaces (magnet cache, coverage markers, a key -// that merely CONTAINS "cid:") must survive a metadata wipe. -func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) { - c := newTestClient(t) - - uuid, err := c.Mint("watch/a.mkv", 100, 1000) - if err != nil { - t.Fatalf("mint: %v", err) + if got := c.ResolveRoot("bafkone"); got != "bafkone" { + t.Errorf("post-clear ResolveRoot = %q, want the hash unchanged", got) } - if err := c.AddAlias(uuid, "bafkmine"); err != nil { - t.Fatalf("add alias: %v", err) - } - for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} { - if err := c.SetRawValue(k, v); err != nil { - t.Fatalf("seed %s: %v", k, err) - } - } - - if _, err := c.ClearAllMetadata(); err != nil { - t.Fatalf("ClearAllMetadata: %v", err) - } - - for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} { + for _, k := range []string{"btmagnet:abc", "searchcov:tmdb:1"} { if _, _, ok, _ := c.GetKeyInfo(k); !ok { - t.Errorf("clear deleted foreign key %q", k) + t.Errorf("clear deleted service key %q", k) } } - if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok { - t.Errorf("own reverse-index entry survived the clear") + // The store keeps working after a wipe. + mustAccept(t, c, vote(t, a, "bafkone", "title", "Again", 1)) + if got := record(t, c, "bafkone")["title"]; got != "Again" { + t.Errorf("title after re-write = %q", got) } } -// withClient runs fn against a fresh client. -func withClient(t *testing.T, fn func(t *testing.T, c *Client)) { - t.Helper() - fn(t, newTestClient(t)) +// Raw writes are limited to service scratch keys: merged records and +// bookkeeping cannot be written around the claims. +func TestServiceValueRefusesRecordKeys(t *testing.T) { + c := newTestClient(t) + for _, k := range []string{"file:R1/title", "cid:bafk", "idx:roots/R1", "ev:meta:events/1", "meta:applied_seq"} { + if err := c.SetServiceValue(k, "x"); err == nil { + t.Errorf("SetServiceValue(%q) accepted", k) + } + } } diff --git a/internal/storage/client.go b/internal/storage/client.go index 7d411de..ce49662 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -7,7 +7,6 @@ import ( "fmt" "log" "sort" - "strconv" "strings" "sync" "time" @@ -16,21 +15,31 @@ import ( "github.com/metazla/meta-core/internal/schema" ) -// Key layout (everything is one string→string table, see internal/kv): +// meta-core keeps two SQLite files (docs/implementation/metacore-claims-model.md): // -// file:/ a record property — the ONLY thing under `file:` -// cid: reverse index → -// idx:roots/ the set of every root (was the Redis SET file:__index__) -// idx:fts/ rowid of the record's row in the FTS table -// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange -// dup:/ a duplicate path of the record (was the SET file:/duplicates) -// udl:… User Data Layer, see udl.go -// ev:/…, evhwm:… event streams, see internal/kv/stream.go +// - the TRUTH database (truth.go): principal, claim, cid_group, meta — the +// signed votes and the locally minted rid names, primary keys only, backed +// up; +// +// - the CACHE database: everything derived from truth, never backed up, +// rebuilt when missing (apply.go). Its kv table keeps the v1 layout: +// +// file:/ the MERGED record view — the ONLY thing under `file:` +// cid: cid → rid lookup (copy of truth's cid_group) +// idx:roots/ the set of every record +// idx:fts/ rowid of the record's row in the FTS table +// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange +// ev:/…, evhwm:… event streams, see internal/kv/stream.go +// meta:applied_seq, meta:rebuilt_at apply-loop bookkeeping +// anything else service scratch KV (btmagnet:*, searchcov:*), /api/kv/value +// +// plus the tables claim_by_pid and local_file (localfile.go), and the +// field index + FTS (search.go). // // Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is -// exactly the records, so a record is the key range [file:/, file:0), -// the field expression index covers only real fields, and every change under -// `file:` is a real field change worth publishing on meta:events. +// exactly the merged records, so a record is the key range [file:/, +// file:0), the field expression index covers only real fields, and every +// change under `file:` is a real field change worth publishing on meta:events. const ( recordNS = "file:" rootsNS = "idx:roots/" @@ -43,10 +52,9 @@ const ( readerPage = 5_000 ) -// MetaEventsStream receives a set/del entry for every record-key change. It is -// written by the storage engine in the same transaction as the change (the -// replacement for Redis keyspace notifications), and served as SSE by -// /api/events/meta. +// MetaEventsStream receives a set/del entry for every merged-record change. It +// is written by the cache engine in the same transaction as the change, and +// served as SSE by /api/events/meta. const MetaEventsStream = "meta:events" // MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is @@ -54,10 +62,13 @@ const MetaEventsStream = "meta:events" // "Backing-store retention and lifecycle". const MetaEventsMaxLen = 100_000 -// Client is meta-core's metadata store. +// Client is meta-core's metadata store: the truth database, the cache, and +// the apply loop between them. type Client struct { - store *kv.Store - mu sync.RWMutex + store *kv.Store // cache + truth *kv.Store + applier *applier + mu sync.RWMutex keysMu sync.Mutex keysCount int64 @@ -69,60 +80,98 @@ func NewClient() *Client { return &Client{} } -// Connect opens (creating if needed) the database file at path. -func (c *Client) Connect(path string) error { +// Connect opens (creating if needed) the truth database at truthPath and the +// cache at cachePath. Call EnsureSchemaVersion, then Start. +func (c *Client) Connect(truthPath, cachePath string) error { c.mu.Lock() defer c.mu.Unlock() - store, err := kv.Open(path, kv.Options{ + truth, err := kv.Open(truthPath, kv.Options{ + NoKVTable: true, + Schema: truthSchema, + WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64), + ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), + Readers: envInt("META_CORE_DB_READERS", 8), + }) + if err != nil { + return fmt.Errorf("open truth %s: %w", truthPath, err) + } + store, err := kv.Open(cachePath, kv.Options{ EventStream: MetaEventsStream, EventMaxLen: MetaEventsMaxLen, EventKey: func(k string) bool { return strings.HasPrefix(k, recordNS) }, WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64), ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), Readers: envInt("META_CORE_DB_READERS", 8), - Schema: searchSchema, + Schema: searchSchema + cacheTablesSchema, OnChange: onChange, + Synchronous: "OFF", }) if err != nil { - return fmt.Errorf("open %s: %w", path, err) + truth.Close() + return fmt.Errorf("open cache %s: %w", cachePath, err) } - c.store = store - log.Printf("[Storage] Opened %s", path) + c.truth, c.store = truth, store + log.Printf("[Storage] Opened truth %s, cache %s", truthPath, cachePath) return nil } -// Close closes the database. +// Start brings the cache level with truth (replay or rebuild) and starts the +// apply loop. Writes before Start fail. +func (c *Client) Start() error { + applied, err := c.catchUp() + if err != nil { + return fmt.Errorf("cache catch-up: %w", err) + } + c.mu.Lock() + c.applier = newApplier(c, applied) + c.mu.Unlock() + return nil +} + +// Close stops the apply loop and closes both databases. func (c *Client) Close() error { c.mu.Lock() defer c.mu.Unlock() - if c.store == nil { - return nil + if c.applier != nil { + c.applier.close() + c.applier = nil } - err := c.store.Close() - c.store = nil - return err + var errs []error + if c.store != nil { + errs = append(errs, c.store.Close()) + c.store = nil + } + if c.truth != nil { + errs = append(errs, c.truth.Close()) + c.truth = nil + } + return errors.Join(errs...) } -// IsConnected reports whether the database is open. +// IsConnected reports whether the databases are open. func (c *Client) IsConnected() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.store != nil + return c.store != nil && c.truth != nil } -// Health checks that the database answers. +// Health checks that both databases answer. func (c *Client) Health() bool { s, err := c.kv() if err != nil { return false } + t, err := c.truthStore() + if err != nil { + return false + } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - return s.Ping(ctx) == nil + return s.Ping(ctx) == nil && t.Ping(ctx) == nil } -// Store exposes the underlying engine (event streams for SSE and the schema +// Store exposes the cache engine (event streams for SSE and the schema // indexer). func (c *Client) Store() *kv.Store { c.mu.RLock() @@ -139,7 +188,16 @@ func (c *Client) kv() (*kv.Store, error) { return c.store, nil } -// update runs fn in one write transaction with the standard timeout. +func (c *Client) truthStore() (*kv.Store, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if c.truth == nil || c.applier == nil { + return nil, fmt.Errorf("not connected") + } + return c.truth, nil +} + +// update runs fn in one cache write transaction with the standard timeout. func (c *Client) update(fn func(tx *kv.Tx) error) error { s, err := c.kv() if err != nil { @@ -184,8 +242,92 @@ func splitRecordKey(key string) (hashID, field string, ok bool) { // isInternalKey reports keys that are derived bookkeeping, which the raw KV // editor must never write (it would desynchronise the derived state). func isInternalKey(key string) bool { - return strings.HasPrefix(key, "idx:") || strings.HasPrefix(key, dupNS) || - strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS) + for _, p := range []string{recordNS, CIDIndexPrefix, "idx:", dupNS, streamNS, hwmNS, "meta:", "meta-core:"} { + if strings.HasPrefix(key, p) { + return true + } + } + return false +} + +// SetServiceValue writes service scratch KV (btmagnet:*, searchcov:*) — the +// only raw key writes left. Merged records and bookkeeping are refused. +func (c *Client) SetServiceValue(key, value string) error { + if key == "" || isInternalKey(key) { + return fmt.Errorf("key %q is not a service key", key) + } + return c.update(func(tx *kv.Tx) error { return tx.Put(key, value) }) +} + +// DeleteServiceValue deletes a service scratch key. Returns true if it existed. +func (c *Client) DeleteServiceValue(key string) (bool, error) { + if key == "" || isInternalKey(key) { + return false, fmt.Errorf("key %q is not a service key", key) + } + existed := false + err := c.update(func(tx *kv.Tx) error { + var err error + existed, err = tx.Delete(key) + return err + }) + return existed, err +} + +// ClearAllMetadata wipes BOTH databases: every claim, signer and group in +// truth, and every derived row of the cache. Irreversible — truth holds votes +// nothing else can recover. Returns the number of records that existed. +func (c *Client) ClearAllMetadata() (int64, error) { + truth, err := c.truthStore() + if err != nil { + return 0, err + } + roots, err := c.CountRoots() + if err != nil { + return 0, err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + if err := truth.Update(ctx, func(tx *kv.Tx) error { + for _, t := range []string{"claim", "principal", "cid_group"} { + if _, err := tx.Exec(`DELETE FROM ` + t); err != nil { + return err + } + } + _, err := tx.Exec(`DELETE FROM meta WHERE name = ?`, lastSeqName) + return err + }); err != nil { + return 0, fmt.Errorf("clear truth: %w", err) + } + for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} { + for { + var n int64 + if err := c.update(func(tx *kv.Tx) error { + var err error + n, err = tx.DeletePrefix(prefix, bulkBatch) + return err + }); err != nil { + return 0, fmt.Errorf("clear %s: %w", prefix, err) + } + if n < bulkBatch { + break + } + } + } + if err := c.update(func(tx *kv.Tx) error { + for _, q := range []string{`DELETE FROM fts`, `DELETE FROM claim_by_pid`, `DELETE FROM local_file`} { + if _, err := tx.Exec(q); err != nil { + return err + } + } + return tx.Put(appliedSeqKey, "0") + }); err != nil { + return 0, fmt.Errorf("clear cache: %w", err) + } + c.applier.mu.Lock() + c.applier.applied = 0 + c.applier.mu.Unlock() + log.Printf("[Storage] Cleared both databases (%d records)", roots) + return roots, nil } // Derived counters, maintained by onChange in the same transaction as the @@ -300,44 +442,6 @@ func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]str return out, nil } -// writeFieldsTx is the one write path for record fields: the keys, the root -// index, the CID reverse aliases and the search row, all in the caller's -// transaction. -func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error { - if hashID == "" { - return fmt.Errorf("empty hash id") - } - prefix := recordPrefix(hashID) - searchChanged := false - for field, value := range metadata { - if isReservedFieldName(field) { - continue - } - changed, err := tx.Set(prefix+field, value) - if err != nil { - return err - } - searchChanged = searchChanged || (changed && affectsHaystack(field)) - } - newRoot, err := tx.Set(rootKey(hashID), "") - if err != nil { - return err - } - for field := range metadata { - if cidStr := cidFromKeysetField(field); cidStr != "" { - if err := addAliasTx(tx, hashID, cidStr); err != nil { - return err - } - } - } - // The FTS row is rewritten only when the haystack can have changed — a - // re-persisted record whose values are all in place costs reads only. - if searchChanged || newRoot { - return refreshSearchTx(tx, hashID) - } - return nil -} - // GetMetadataFlat retrieves all metadata for a file as a flat map. func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) { s, err := c.kv() @@ -349,28 +453,6 @@ func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) { return readRecord(ctx, s, hashID) } -// SetMetadataFlat stores metadata for a file using flat keys (merge: fields -// absent from metadata are kept — the Redis implementation never cleared -// either). -func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error { - if len(metadata) == 0 { - return nil - } - return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }) -} - -// MergeMetadataFlat merges new metadata into existing (PATCH semantics): new -// keys are added, existing keys are updated, missing keys are NOT deleted. -func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) { - if len(metadata) == 0 { - return 0, nil - } - if err := c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }); err != nil { - return 0, err - } - return len(metadata), nil -} - // GetAllHashIDs returns every root id, sorted. func (c *Client) GetAllHashIDs() ([]string, error) { s, err := c.kv() @@ -401,57 +483,6 @@ func (c *Client) GetProperty(hashID, property string) (string, error) { return v, err } -// SetProperty sets a single property value. -func (c *Client) SetProperty(hashID, property, value string) error { - if isReservedFieldName(property) { - return fmt.Errorf("field name %q is reserved", property) - } - return c.update(func(tx *kv.Tx) error { - return writeFieldsTx(tx, hashID, map[string]string{property: value}) - }) -} - -// deleteRecordTx removes everything keyed off hashID: its properties, the -// reverse-index entries its cids/ members own, its duplicates, its root-index -// entry and its search row. Returns the number of property keys removed. -func deleteRecordTx(tx *kv.Tx, hashID string) (int64, error) { - ctx := tx.Context() - cids, err := cidsForRoot(ctx, tx, hashID) - if err != nil { - return 0, err - } - for _, cidStr := range cids { - if _, err := tx.Delete(CIDIndexPrefix + cidStr); err != nil { - return 0, err - } - } - n, err := tx.DeletePrefix(recordPrefix(hashID), 0) - if err != nil { - return 0, err - } - if _, err := tx.DeletePrefix(dupPrefix(hashID), 0); err != nil { - return n, err - } - if _, err := tx.Delete(rootKey(hashID)); err != nil { - return n, err - } - return n, deleteSearchRowTx(tx, hashID) -} - -// DeleteMetadata deletes all metadata for a file, and unmaps every -// reverse-index entry registered for this root, so a subsequent -// GET /api/meta/ correctly returns 404 instead of pointing at a root -// with no keys. -func (c *Client) DeleteMetadata(hashID string) (int64, error) { - var n int64 - err := c.update(func(tx *kv.Tx) error { - var err error - n, err = deleteRecordTx(tx, hashID) - return err - }) - return n, err -} - // CountFiles returns the number of indexed roots. func (c *Client) CountFiles() (int, error) { n, err := c.CountRoots() @@ -496,161 +527,28 @@ func normalizeFilesRelativePath(p string) string { // DeleteProperty deletes a single property. // -// For a cids/ member this is the full inverse of the write: the flat key -// AND the cid: reverse alias both go. Anything less leaves the CID still -// resolving to this record — see removeAliasTx. -func (c *Client) DeleteProperty(hashID, property string) error { - return c.update(func(tx *kv.Tx) error { - if _, err := tx.Delete(recordPrefix(hashID) + property); err != nil { - return err - } - if cidStr := cidFromKeysetField(property); cidStr != "" { - if err := removeAliasTx(tx, hashID, cidStr); err != nil { - return fmt.Errorf("remove alias for %s: %w", cidStr, err) - } - } - if affectsHaystack(property) { - return refreshSearchTx(tx, hashID) - } - return nil - }) -} - -// AddToSet adds a value to a set-type field (stored as a pipe-delimited string -// in one flat key). Returns true if value was added, false if it already -// existed. -func (c *Client) AddToSet(hashID, property, value string) (bool, error) { - added := false - err := c.update(func(tx *kv.Tx) error { - current, _, err := tx.Get(tx.Context(), recordPrefix(hashID)+property) - if err != nil { - return err - } - var values []string - if current != "" { - values = strings.Split(current, "|") - } - for _, v := range values { - if v == value { - return nil - } - } - added = true - return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(append(values, value), "|")}) - }) - return added, err -} - -// RemoveFromSet removes a value from a set-type field (stored in one flat key). -// Returns true if value was removed, false if it didn't exist. -func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) { - removed := false - err := c.update(func(tx *kv.Tx) error { - key := recordPrefix(hashID) + property - current, ok, err := tx.Get(tx.Context(), key) - if err != nil || !ok { - return err - } - values := strings.Split(current, "|") - kept := make([]string, 0, len(values)) - for _, v := range values { - if v == value { - removed = true - } else { - kept = append(kept, v) - } - } - if !removed { - return nil - } - if len(kept) == 0 { - if _, err := tx.Delete(key); err != nil { - return err - } - if affectsHaystack(property) { - return refreshSearchTx(tx, hashID) - } - return nil - } - return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(kept, "|")}) - }) - return removed, err -} - -// FileTuple holds the (uuid, path, size, mtime) record for one file. Used by -// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple -// matches what's already stored. -type FileTuple struct { - HashID string - FilePath string - Size int64 - MtimeNano int64 -} - -// GetTuplesForAllFiles returns one FileTuple per indexed root, in batched -// multi-key reads. Skips files missing any of filePath, sizeByte, mtimeNano. -func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { - s, err := c.kv() - if err != nil { - return nil, err - } - ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout) - defer cancel() - - fields := [3]string{"filePath", "sizeByte", "mtimeNano"} - var out []FileTuple - err = s.ScanPrefix(ctx, rootsNS, 1000, func(rows []kv.KV) error { - keys := make([]string, 0, len(rows)*len(fields)) - for _, row := range rows { - prefix := recordPrefix(row.K[len(rootsNS):]) - for _, f := range fields { - keys = append(keys, prefix+f) - } - } - vals, err := s.MGet(ctx, keys) - if err != nil { - return err - } - for _, row := range rows { - h := row.K[len(rootsNS):] - prefix := recordPrefix(h) - filePath, sizeStr, mtimeStr := vals[prefix+fields[0]], vals[prefix+fields[1]], vals[prefix+fields[2]] - if filePath == "" || sizeStr == "" || mtimeStr == "" { - continue - } - size, err := strconv.ParseInt(sizeStr, 10, 64) - if err != nil { - continue - } - mtime, err := strconv.ParseInt(mtimeStr, 10, 64) - if err != nil { - continue - } - out = append(out, FileTuple{HashID: h, FilePath: filePath, Size: size, MtimeNano: mtime}) - } - return nil - }) - return out, err -} // GetMemoryInfo reports the store's footprint for dashboards: the database // file plus its WAL, human-readable. (Under Redis this was used_memory_human; // the data no longer lives in RAM, so the disk footprint is the honest figure.) func (c *Client) GetMemoryInfo() (string, error) { - s, err := c.kv() - if err != nil { + if _, err := c.kv(); err != nil { return "", err } - return humanBytes(s.FileSize()), nil + return humanBytes(c.DBSize()), nil } -// DBSize is the database file plus WAL, in bytes. +// DBSize is both database files plus their WALs, in bytes. func (c *Client) DBSize() int64 { - s, err := c.kv() - if err != nil { - return 0 + c.mu.RLock() + defer c.mu.RUnlock() + var n int64 + for _, st := range []*kv.Store{c.store, c.truth} { + if st != nil { + n += st.FileSize() + } } - return s.FileSize() + return n } func humanBytes(n int64) string { @@ -674,37 +572,6 @@ func humanBytes(n int64) string { // redirects the next /meta/ write back into that dead root, resurrecting // a zombie record under an old identity. // -// Runs as many bounded transactions rather than one: a single transaction over -// millions of rows would hold the writer for minutes and balloon the WAL. -func (c *Client) ClearAllMetadata() (int64, error) { - roots, err := c.CountRoots() - if err != nil { - return 0, err - } - for _, prefix := range []string{recordNS, CIDIndexPrefix, dupNS, rootsNS, ftsNS} { - for { - var n int64 - if err := c.update(func(tx *kv.Tx) error { - var err error - n, err = tx.DeletePrefix(prefix, bulkBatch) - return err - }); err != nil { - return 0, fmt.Errorf("clear %s: %w", prefix, err) - } - if n < bulkBatch { - break - } - } - } - if err := c.update(func(tx *kv.Tx) error { - _, err := tx.Exec(`DELETE FROM fts`) - return err - }); err != nil { - return 0, fmt.Errorf("clear search rows: %w", err) - } - log.Printf("[Storage] Cleared %d file metadata entries", roots) - return roots, nil -} // ScanKeys lists the immediate children at the given prefix, S3-style. // @@ -767,11 +634,24 @@ func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ( leaves = append(leaves, k) from, op = k, ">" } + leaves, branches = serviceOnly(leaves), serviceOnly(branches) sort.Strings(leaves) sort.Strings(branches) return leaves, branches, next, nil } +// serviceOnly drops merged-record and bookkeeping keys: the /api/kv browser +// shows service scratch KV only. +func serviceOnly(keys []string) []string { + out := keys[:0] + for _, k := range keys { + if !isInternalKey(k) { + out = append(out, k) + } + } + return out +} + // FindMatch is a single value-search hit. type FindMatch struct { Key string `json:"key"` @@ -876,10 +756,11 @@ func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) if err := rows.Err(); err != nil { return nil, false, err } - if len(results) > limit { - return results[:limit], true, nil + truncated := len(results) > limit + if truncated { + results = results[:limit] } - return results, false, nil + return serviceOnly(results), truncated, nil } // GetKeyInfo returns the key's type — "string" when it exists (every row is a @@ -889,6 +770,9 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, if err != nil { return "", "", false, err } + if isInternalKey(key) { + return "", "", false, nil + } ctx, cancel := opCtx() defer cancel() v, ok, err := s.Get(ctx, key) @@ -898,70 +782,6 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, return "string", v, true, nil } -// SetRawValue writes a string value at the exact key (the raw KV editor). -// Refuses derived bookkeeping keys. A record key keeps the search row honest -// like every other write path. -func (c *Client) SetRawValue(key, value string) error { - if isInternalKey(key) { - return fmt.Errorf("key %q is internal bookkeeping and not editable", key) - } - return c.update(func(tx *kv.Tx) error { - if err := tx.Put(key, value); err != nil { - return err - } - if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) { - return refreshSearchTx(tx, hashID) - } - return nil - }) -} - -// DeleteRawKey deletes the exact key. Returns true if the key existed. -func (c *Client) DeleteRawKey(key string) (bool, error) { - if isInternalKey(key) { - return false, fmt.Errorf("key %q is internal bookkeeping and not editable", key) - } - existed := false - err := c.update(func(tx *kv.Tx) error { - var err error - if existed, err = tx.Delete(key); err != nil || !existed { - return err - } - if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) { - return refreshSearchTx(tx, hashID) - } - return nil - }) - return existed, err -} - -// IndexAdd adds a hash ID to the root index. Idempotent. -func (c *Client) IndexAdd(hashID string) error { - return c.update(func(tx *kv.Tx) error { - if err := tx.Put(rootKey(hashID), ""); err != nil { - return err - } - return ensureSearchRowTx(tx, hashID) - }) -} - -// IndexRemoveIfEmpty removes hashID from the root index (and drops its search -// row) when no file:/* keys remain. Returns true if it was removed. -func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) { - removed := false - err := c.update(func(tx *kv.Tx) error { - n, err := tx.Count(tx.Context(), recordPrefix(hashID)) - if err != nil || n > 0 { - return err - } - if removed, err = tx.Delete(rootKey(hashID)); err != nil { - return err - } - return deleteSearchRowTx(tx, hashID) - }) - return removed, err -} - // GetPrefix returns the key namespace prefix. Always empty: meta-core owns its // database file outright, so there is nothing to share a keyspace with. func (c *Client) GetPrefix() string { return "" } diff --git a/internal/storage/counters_test.go b/internal/storage/counters_test.go index 1750b21..455876a 100644 --- a/internal/storage/counters_test.go +++ b/internal/storage/counters_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "testing" + "github.com/metazla/meta-core/internal/claim" "github.com/metazla/meta-core/internal/schema" ) @@ -19,48 +20,42 @@ func schemaJSON(t *testing.T, c *Client) string { return string(b) } -// The counters behind /api/schema and search's `total` are maintained by the -// write path. Whatever sequence of writes, modifications and deletions runs, -// they must equal what a full rebuild from the records computes. +// The counters behind /api/schema and search's `total` follow the merged view. +// Whatever sequence of claims, retractions and edits runs, they must equal what +// a full rebuild from the records computes. func TestCountersStayExactAcrossWrites(t *testing.T) { c := newTestClient(t) - seed(t, c, map[string]map[string]string{ - "R1": {"title": "One", "sizeByte": "100", "title/eng": "One"}, - "R2": {"title": "Two", "sizeByte": "x", "genres/Action": "true"}, - "R3": {"title": "Three", "sizeByte": "300"}, - }) - if err := c.SetProperty("R2", "sizeByte", "200"); err != nil { // label string → int - t.Fatal(err) - } - if err := c.SetProperty("R1", "title", "One"); err != nil { // unchanged: no delta - t.Fatal(err) - } - if err := c.DeleteProperty("R1", "title/eng"); err != nil { - t.Fatal(err) - } - if _, err := c.DeleteMetadata("R3"); err != nil { - t.Fatal(err) - } - - if n, err := c.CountRoots(); err != nil || n != 2 { - t.Fatalf("CountRoots = %d (err %v), want 2", n, err) + a, b := signer(t, 1), signer(t, 2) + mustAccept(t, c, + vote(t, a, "bafyR1", "title", "One", 10), vote(t, a, "bafyR1", "sizeByte", "100", 10), + vote(t, a, "bafyR1", "title/eng", "One", 10), + vote(t, a, "bafyR2", "title", "Two", 10), vote(t, a, "bafyR2", "sizeByte", "x", 10), + vote(t, a, "bafyR2", "genres/Action", "true", 10), + vote(t, a, "bafyR3", "title", "Three", 10), vote(t, a, "bafyR3", "sizeByte", "300", 10), + ) + mustAccept(t, c, + vote(t, a, "bafyR2", "sizeByte", "200", 20), // label string → int + vote(t, b, "bafyR1", "title", "One", 20), // agreeing vote: merged value unchanged + vote(t, a, "bafyR1", "title/eng", nil, 20), // retraction + ) + if n, err := c.CountRoots(); err != nil || n != 3 { + t.Fatalf("CountRoots = %d (err %v), want 3", n, err) } incremental := schemaJSON(t, c) snap, _ := schema.NewIndexer(c.Store()).Snapshot(context.Background()) - if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 2 { - t.Fatalf("sizeByte = %+v, want 2 ints", f) + if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 3 { + t.Fatalf("sizeByte = %+v, want 3 ints", f) } - if _, ok := snap.Fields["title"]; !ok { - t.Fatalf("title missing: %v", snap.Fields) + if _, ok := snap.Fields["title/eng"]; ok { + t.Fatalf("retracted title/eng still counted: %v", snap.Fields) } - if err := c.RebuildCounters(); err != nil { t.Fatal(err) } if rebuilt := schemaJSON(t, c); rebuilt != incremental { t.Fatalf("incremental counters drifted from a rebuild:\n incremental %s\n rebuilt %s", incremental, rebuilt) } - if n, _ := c.CountRoots(); n != 2 { + if n, _ := c.CountRoots(); n != 3 { t.Fatalf("CountRoots after rebuild = %d", n) } } @@ -77,16 +72,23 @@ func TestEnsureCountersBuildsOnce(t *testing.T) { } } -// A re-persisted record with nothing new writes nothing and publishes nothing. -func TestUnchangedRecordWriteIsANoOp(t *testing.T) { +// Re-persisting the same signed claims writes nothing and publishes nothing — +// the gateway's constant repeats. +func TestUnchangedClaimWriteIsANoOp(t *testing.T) { c := newTestClient(t) - rec := map[string]string{"title": "Naruto", "fileType": "video", "cids/bafkreiabc": "true"} - seed(t, c, map[string]map[string]string{"R1": rec}) + a := signer(t, 1) + batch := []claim.Claim{ + vote(t, a, "bafkreiabc", "title", "Naruto", 5), + vote(t, a, "bafkreiabc", "fileType", "video", 5), + } + mustAccept(t, c, batch...) before, _ := c.GetStreamLength(MetaEventsStream) - if err := c.SetMetadataFlat("R1", rec); err != nil { - t.Fatal(err) + for _, st := range write(t, c, batch...) { + if st != StatusUnchanged { + t.Fatalf("re-sent claim status %s, want unchanged", st) + } } if after, _ := c.GetStreamLength(MetaEventsStream); after != before { - t.Fatalf("re-persisting an unchanged record published %d events", after-before) + t.Fatalf("re-persisting unchanged claims published %d events", after-before) } } diff --git a/internal/storage/helpers_test.go b/internal/storage/helpers_test.go new file mode 100644 index 0000000..bb50b87 --- /dev/null +++ b/internal/storage/helpers_test.go @@ -0,0 +1,166 @@ +package storage + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/identity" + "github.com/metazla/meta-core/internal/kv" +) + +// openClient opens (and starts) a client on dir's truth + cache files. +func openClient(t *testing.T, dir string) *Client { + t.Helper() + c := NewClient() + if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil { + t.Fatalf("connect: %v", err) + } + if err := c.EnsureSchemaVersion(SchemaVersion); err != nil { + t.Fatalf("schema: %v", err) + } + if err := c.Start(); err != nil { + t.Fatalf("start: %v", err) + } + return c +} + +// newTestClient opens a fresh store in the test's temp dir. +func newTestClient(t *testing.T) *Client { + t.Helper() + c := openClient(t, t.TempDir()) + t.Cleanup(func() { _ = c.Close() }) + return c +} + +// signer returns a deterministic test identity (scalar n repeated). +func signer(t *testing.T, n byte) *identity.Identity { + t.Helper() + id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1)) + if err != nil { + t.Fatalf("identity: %v", err) + } + return id +} + +// vote builds a signed claim; v is a string (text), []byte (ciphertext) or nil +// (retraction). +func vote(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim { + t.Helper() + cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts} + switch x := v.(type) { + case string: + cl.Value = claim.Text(x) + case []byte: + cl.Value = claim.Bytes(x) + case nil: + default: + t.Fatalf("vote value %T", v) + } + if err := cl.Sign(id); err != nil { + t.Fatalf("sign: %v", err) + } + return cl +} + +// write posts claims and returns their statuses. +func write(t *testing.T, c *Client, claims ...claim.Claim) []ClaimStatus { + t.Helper() + res, err := c.WriteClaims(context.Background(), claims) + if err != nil { + t.Fatalf("write: %v", err) + } + out := make([]ClaimStatus, len(res)) + for i, r := range res { + out[i] = r.Status + } + return out +} + +// mustAccept posts claims and fails unless every one is accepted. +func mustAccept(t *testing.T, c *Client, claims ...claim.Claim) { + t.Helper() + for i, st := range write(t, c, claims...) { + if st != StatusAccepted { + t.Fatalf("claim %d (%s/%s): %s", i, claims[i].CID, claims[i].Key, st) + } + } +} + +// seed writes merged rows straight into the cache — the derived layout search +// and the counters work on — bypassing claims. Search tests pin SQL behaviour +// over that layout; how rows get there is the apply loop's business. +func seed(t *testing.T, c *Client, records map[string]map[string]string) { + t.Helper() + for id, fields := range records { + setMerged(t, c, id, fields) + } +} + +func setMerged(t *testing.T, c *Client, rid string, fields map[string]string) { + t.Helper() + if err := c.update(func(tx *kv.Tx) error { + for k, v := range fields { + if err := tx.Put(recordPrefix(rid)+k, v); err != nil { + return err + } + } + if err := tx.Put(rootKey(rid), ""); err != nil { + return err + } + return refreshSearchTx(tx, rid) + }); err != nil { + t.Fatalf("seed %s: %v", rid, err) + } +} + +func deleteMerged(t *testing.T, c *Client, rid string) { + t.Helper() + if err := c.update(func(tx *kv.Tx) error { + if _, err := tx.DeletePrefix(recordPrefix(rid), 0); err != nil { + return err + } + if _, err := tx.Delete(rootKey(rid)); err != nil { + return err + } + return deleteSearchRowTx(tx, rid) + }); err != nil { + t.Fatalf("delete %s: %v", rid, err) + } +} + +// allKeys lists every cache kv row (full scan). +func allKeys(t *testing.T, c *Client) []string { + t.Helper() + var keys []string + if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error { + for _, r := range rows { + keys = append(keys, r.K) + } + return nil + }); err != nil { + t.Fatalf("scan: %v", err) + } + return keys +} + +// rid resolves a cid to its record (fails when unknown). +func rid(t *testing.T, c *Client, cid string) string { + t.Helper() + r, err := c.GetByCID(cid) + if err != nil || r == "" { + t.Fatalf("cid %s unresolved (%v)", cid, err) + } + return r +} + +func record(t *testing.T, c *Client, cid string) map[string]string { + t.Helper() + f, err := c.GetMetadataFlat(rid(t, c, cid)) + if err != nil { + t.Fatal(err) + } + return f +} diff --git a/internal/storage/localfile.go b/internal/storage/localfile.go new file mode 100644 index 0000000..37ba3ef --- /dev/null +++ b/internal/storage/localfile.go @@ -0,0 +1,196 @@ +package storage + +// Box-local file facts: where the watcher saw each file on this disk. +// +// local_file(path) → midhash, size, mtime (cache table, unsigned) +// +// These are facts about this disk, not about the content, so they are never +// claims and never travel. meta-core cannot sign anyway, and a rescan of +// /files rebuilds them. The midhash gets a cid_group entry in truth (local +// naming, not a vote), and the record view joins filePath / sizeByte / +// mtimeNano in as `file:/` rows — which publishes the meta events +// meta-fuse builds its tree from. + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/metazla/meta-core/internal/kv" +) + +const cacheTablesSchema = ` +CREATE TABLE IF NOT EXISTS claim_by_pid( + pid INTEGER NOT NULL, + cid TEXT NOT NULL, + key TEXT NOT NULL, + PRIMARY KEY (pid, cid, key) +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS local_file( + path TEXT PRIMARY KEY, + midhash TEXT NOT NULL, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS local_file_midhash ON local_file(midhash, path); +` + +// RegisterFileTuple records one physical file under its midhash256 CID, the +// way the watcher does for every file it scans (and POST /api/files/register +// for a box with the watcher off). Returns the record's rid and whether its +// group was newly created. +func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) { + if midhash == "" { + return "", false, fmt.Errorf("midhash is required") + } + truth, err := c.truthStore() + if err != nil { + return "", false, err + } + var rid string + var created bool + ctx, cancel := opCtx() + defer cancel() + if err := truth.Update(ctx, func(tx *kv.Tx) error { + var err error + rid, created, err = ensureGroup(tx, midhash) + return err + }); err != nil { + return "", false, err + } + err = c.update(func(tx *kv.Tx) error { + var prev string + err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&prev) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if _, err := tx.Exec(`INSERT INTO local_file(path, midhash, size, mtime) VALUES(?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET midhash = excluded.midhash, size = excluded.size, mtime = excluded.mtime`, + absPath, midhash, size, mtimeNano); err != nil { + return err + } + if _, err := syncMember(tx, midhash, rid); err != nil { + return err + } + if err := c.remergeLocalTx(tx, rid); err != nil { + return err + } + if prev != "" && prev != midhash { + // The file at this path changed content: its old record loses it. + if old, _, err := tx.Get(tx.Context(), CIDIndexPrefix+prev); err == nil && old != "" && old != rid { + return c.remergeLocalTx(tx, old) + } + } + return nil + }) + if err != nil { + return "", false, err + } + return rid, created, nil +} + +// RemoveFileTuple forgets a path the watcher saw disappear. Only the local +// facts go — the record's claims stay (facts about content stay true). +func (c *Client) RemoveFileTuple(absPath string) error { + return c.update(func(tx *kv.Tx) error { + var midhash string + err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&midhash) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM local_file WHERE path = ?`, absPath); err != nil { + return err + } + rid, _, err := tx.Get(tx.Context(), CIDIndexPrefix+midhash) + if err != nil || rid == "" { + return err + } + return c.remergeLocalTx(tx, rid) + }) +} + +// remergeLocalTx recomputes the box-local fields of rid (and its search row +// when filePath changed). +func (c *Client) remergeLocalTx(tx *kv.Tx, rid string) error { + cids, err := cidsForRoot(tx.Context(), tx, rid) + if err != nil { + return err + } + searchChanged := false + for _, f := range localFields { + changed, err := c.remergeTx(tx, rid, cids, f) + if err != nil { + return err + } + searchChanged = searchChanged || (changed && affectsHaystack(f)) + } + if searchChanged { + return refreshSearchTx(tx, rid) + } + return nil +} + +// FileTuple holds the (rid, path, size, mtime) record for one file. Used by +// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple +// matches what's already stored. +type FileTuple struct { + HashID string + FilePath string + Size int64 + MtimeNano int64 +} + +// GetTuplesForAllFiles returns one FileTuple per local file. +func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { + s, err := c.kv() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout) + defer cancel() + rows, err := s.DB().QueryContext(ctx, + `SELECT l.path, l.size, l.mtime, COALESCE(k.v, '') FROM local_file l + LEFT JOIN kv k ON k.k = 'cid:' || l.midhash ORDER BY l.path`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []FileTuple + for rows.Next() { + var t FileTuple + if err := rows.Scan(&t.FilePath, &t.Size, &t.MtimeNano, &t.HashID); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +// localPaths lists every local path whose midhash is one of cids. +func localPaths(ctx context.Context, q interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +}, cids []string) ([]string, error) { + var out []string + for _, cid := range cids { + rows, err := q.QueryContext(ctx, `SELECT path FROM local_file WHERE midhash = ? ORDER BY path`, cid) + if err != nil { + return nil, err + } + for rows.Next() { + var p string + if err := rows.Scan(&p); err != nil { + rows.Close() + return nil, err + } + out = append(out, p) + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/internal/storage/merge.go b/internal/storage/merge.go new file mode 100644 index 0000000..8ec5973 --- /dev/null +++ b/internal/storage/merge.go @@ -0,0 +1,102 @@ +package storage + +import ( + "strconv" + "strings" + + "github.com/metazla/meta-core/internal/claim" +) + +// Vote is one signer's claim on one key, as the merge sees it. Pid identifies +// the signer (one vote per signer per key per group); Uid is carried for the +// stateless /api/merge, where claims arrive without a local pid. +type Vote struct { + Pid int64 + Uid string + Value claim.Value + TS uint64 +} + +// MergeResult is the merged value of one key. +type MergeResult struct { + Value string + Nsrc int // signers agreeing on the winning value + OK bool +} + +// Merge is the one merge rule (spec §7), shared by the apply loop, the +// ?as= overlay and POST /api/merge: +// +// 1. a signer counts once per key: its newest vote wins (it may have voted on +// several cids of the group); +// 2. the viewer's own live text vote wins outright (viewer == "" for the +// shared view); +// 3. retractions and ciphertext drop out — only text merges; +// 4. support: votes grouped by normalised value, most signers wins; +// 5. tie → the value first seen (oldest ts) — stable, and rewriting does not +// win; +// 6. tie → byte-wise smallest value, so every peer agrees. +// +// The stored value is the exact text of the oldest vote of the winning group. +func Merge(votes []Vote, viewer string) MergeResult { + newest := make(map[string]Vote, len(votes)) + for _, v := range votes { + id := signerID(v) + if cur, ok := newest[id]; !ok || v.TS > cur.TS { + newest[id] = v + } + } + type bucket struct { + n int + minTS uint64 + exemplar string + } + buckets := make(map[string]*bucket) + for _, v := range newest { + if v.Value.Kind != claim.KindText { + continue + } + if viewer != "" && v.Uid == viewer { + return MergeResult{Value: v.Value.Text, Nsrc: 1, OK: true} + } + norm := normalizeValue(v.Value.Text) + b := buckets[norm] + if b == nil { + buckets[norm] = &bucket{n: 1, minTS: v.TS, exemplar: v.Value.Text} + continue + } + b.n++ + if v.TS < b.minTS || (v.TS == b.minTS && v.Value.Text < b.exemplar) { + b.minTS, b.exemplar = v.TS, v.Value.Text + } + } + var best *bucket + for _, b := range buckets { + switch { + case best == nil, + b.n > best.n, + b.n == best.n && b.minTS < best.minTS, + b.n == best.n && b.minTS == best.minTS && b.exemplar < best.exemplar: + best = b + } + } + if best == nil { + return MergeResult{} + } + return MergeResult{Value: best.exemplar, Nsrc: best.n, OK: true} +} + +func signerID(v Vote) string { + if v.Uid != "" { + return v.Uid + } + return "#" + strconv.FormatInt(v.Pid, 10) +} + +// normalizeValue folds case and surrounding whitespace so "The Matrix" and +// "the matrix " count as the same value. (Unicode NFC folding is not applied: +// the module carries no normalisation table; values written by one plugin +// family are already consistent.) +func normalizeValue(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} diff --git a/internal/storage/schema_version.go b/internal/storage/schema_version.go index 6d4b8ad..b3f2ec8 100644 --- a/internal/storage/schema_version.go +++ b/internal/storage/schema_version.go @@ -1,70 +1,76 @@ package storage import ( + "context" + "database/sql" + "errors" "fmt" "strconv" "github.com/metazla/meta-core/internal/kv" ) -// SchemaVersion is the key-layout version this build of meta-core expects. -// Distinct from the snapshot manifest version (which is on-disk format); -// this number describes the record layout — primarily, "are roots UUIDs or -// content-hash tokens, and do reverse-index keys cid:* exist?" +// SchemaVersion is the truth-database layout this build of meta-core expects. // -// The Redis → SQLite move did not change the record layout (file:/, -// cid:), so it did not bump this: the one-time migration copied records -// verbatim. +// 2 — v1.x: one kv table holding records, the CID index and UDL cells +// 3 — v2.0.0: the claims model (truth: principal, claim, cid_group, meta) // -// Bump only when a layout change can't be detected from the data itself or -// healed by the write path. Each bump is a flag day in alpha: operators wipe -// and restart. -const SchemaVersion = 2 +// Bump only when a layout change can't be detected from the data itself. Each +// bump is a flag day: there is no migration code in meta-core (the one-off +// v2 → v3 conversion of watch.nsl.sh is a manual tool, tools/claims-migrate). +const SchemaVersion = 3 -// SchemaVersionKey is the sentinel under which we record SchemaVersion. -// Checked on every meta-core boot. -const SchemaVersionKey = "meta-core:schema-version" +// SchemaVersionKey is the truth `meta` row recording SchemaVersion. +const SchemaVersionKey = "schema-version" -// EnsureSchemaVersion verifies the store matches the expected schema version. -// Called once at startup, before any other reads/writes. +// EnsureSchemaVersion verifies the truth database matches the expected +// version. Called once at startup, before Start. // -// Behavior: -// - sentinel present and matches expected → OK -// - sentinel missing AND no roots → fresh store, stamp sentinel -// - sentinel missing AND roots exist → legacy data, refuse -// - sentinel present but mismatched → refuse, point operator at clean:all -// -// Refusal is hard-fail; the caller is expected to log.Fatalf and exit. -// The error message names the wipe remediation because in alpha there -// is no automated migration — operators bring up a clean store. +// - a v1.x store (the old kv table, which the default path still points at) +// → refuse +// - sentinel present and matching → OK +// - sentinel missing and no claims → fresh store, stamp it +// - sentinel missing with claims, or mismatched → refuse func (c *Client) EnsureSchemaVersion(expected int) error { - return c.update(func(tx *kv.Tx) error { - ctx := tx.Context() - have, ok, err := tx.Get(ctx, SchemaVersionKey) - if err != nil { - return fmt.Errorf("get schema version: %w", err) + c.mu.RLock() + truth := c.truth + c.mu.RUnlock() + if truth == nil { + return fmt.Errorf("not connected") + } + ctx, cancel := context.WithTimeout(context.Background(), opTimeout) + defer cancel() + return truth.Update(ctx, func(tx *kv.Tx) error { + var name string + err := tx.QueryRow(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'kv'`).Scan(&name) + if err == nil { + return fmt.Errorf( + "%s holds a meta-core v1.x store (schema version 2, the kv table). v2.0.0 "+ + "does not migrate it: convert it with tools/claims-migrate, or move it "+ + "away to start empty", truth.Path()) } - if !ok { - n, err := tx.Count(ctx, rootsNS) - if err != nil { - return fmt.Errorf("count roots: %w", err) + if !errors.Is(err, sql.ErrNoRows) { + return err + } + var have string + err = tx.QueryRow(`SELECT value FROM meta WHERE name = ?`, SchemaVersionKey).Scan(&have) + if errors.Is(err, sql.ErrNoRows) { + var n int64 + if err := tx.QueryRow(`SELECT count(*) FROM claim`).Scan(&n); err != nil { + return err } if n > 0 { - return fmt.Errorf( - "the store has %d indexed roots but no %q sentinel — this is alpha "+ - "data from before the UUID-rooted schema landed. Delete the "+ - "database file to wipe and restart, or downgrade meta-core to a compatible build", - n, SchemaVersionKey) + return fmt.Errorf("truth database has %d claims but no %q sentinel; refusing to guess its layout", n, SchemaVersionKey) } - return tx.Put(SchemaVersionKey, strconv.Itoa(expected)) + _, err := tx.Exec(`INSERT INTO meta(name, value) VALUES(?, ?)`, SchemaVersionKey, strconv.Itoa(expected)) + return err + } + if err != nil { + return fmt.Errorf("get schema version: %w", err) } if have == strconv.Itoa(expected) { return nil } - return fmt.Errorf( - "store schema version is %q, this build of meta-core expects %d. "+ - "Delete the database file to wipe and restart, or use a meta-core "+ - "build that matches the existing data", - have, expected) + return fmt.Errorf("truth schema version is %q, this build of meta-core expects %d", have, expected) }) } diff --git a/internal/storage/search.go b/internal/storage/search.go index 336c746..6b4b294 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -201,6 +201,14 @@ type SearchHit struct { // match is the authoritative test, run on each hydrated record. Pass the same // query/filters match evaluates, or results will be missing. func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) { + return c.SearchAs(query, filters, limit, "", match) +} + +// SearchAs is Search seen by viewer (a uid, "" for the shared view): every +// candidate is matched with the viewer's own claims overlaid, and the records +// the viewer holds claims on are candidates too, so their own corrections are +// searchable for them. +func (c *Client) SearchAs(query string, filters []SearchFilter, limit int, viewer string, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) { s, err := c.kv() if err != nil { return nil, err @@ -245,7 +253,7 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f } } for _, id := range survivors { - fields, err := readRecord(ctx, s, id) + fields, err := c.readRecordAs(ctx, s, id, viewer) if err != nil { return nil, err } @@ -261,6 +269,33 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f break } } + if viewer == "" { + return hits, nil + } + seen := make(map[string]bool, len(hits)) + for _, h := range hits { + seen[h.HashID] = true + } + own, err := c.viewerRIDs(ctx, viewer) + if err != nil { + return nil, err + } + for _, id := range own { + if seen[id] { + continue + } + fields, err := c.readRecordAs(ctx, s, id, viewer) + if err != nil { + return nil, err + } + if fields != nil && match(id, fields) { + hits = append(hits, SearchHit{HashID: id, Fields: fields}) + } + } + sort.Slice(hits, func(i, j int) bool { return hits[i].HashID < hits[j].HashID }) + if len(hits) > limit { + hits = hits[:limit] + } return hits, nil } diff --git a/internal/storage/search_test.go b/internal/storage/search_test.go index 21b0bb8..73ede72 100644 --- a/internal/storage/search_test.go +++ b/internal/storage/search_test.go @@ -19,15 +19,6 @@ import ( // The matcher here mirrors api.recordMatchesFilter closely enough to exercise // every candidate path; the API tests own the matcher itself. -func seed(t *testing.T, c *Client, records map[string]map[string]string) { - t.Helper() - for id, fields := range records { - if err := c.SetMetadataFlat(id, fields); err != nil { - t.Fatalf("seed %s: %v", id, err) - } - } -} - // matchFilter is a faithful copy of api.recordMatchesFilter's semantics. func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool { for _, v := range values { @@ -197,14 +188,10 @@ func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) { func TestSearchFollowsWritesAndDeletes(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}}) - if err := c.SetProperty("R1", "title", "After"); err != nil { - t.Fatal(err) - } + setMerged(t, c, "R1", map[string]string{"title": "After"}) equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1") equalIDs(t, search(t, c, "before", nil, textMatch("before"))) - if _, err := c.DeleteMetadata("R1"); err != nil { - t.Fatal(err) - } + deleteMerged(t, c, "R1") equalIDs(t, search(t, c, "after", nil, textMatch("after"))) equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true })) } diff --git a/internal/storage/truth.go b/internal/storage/truth.go new file mode 100644 index 0000000..4367c01 --- /dev/null +++ b/internal/storage/truth.go @@ -0,0 +1,701 @@ +package storage + +// The truth database: the only data meta-core cannot rebuild. +// +// principal(pid, uid) who signed (pid = local short form) +// claim(cid, key, pid) → value, ts, sig, seq every vote, signed (MMUR2) +// cid_group(cid) → rid which record a cid belongs to +// meta(name) → value schema version, last_seq +// +// Primary keys only — every other index lives in the cache, which is rebuilt +// from these four tables. A claim never mentions a rid: the rid is a local +// name for a group of cids, minted here (UUIDv7) and kept because consumers +// hold rids. + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/kv" +) + +const truthSchema = ` +CREATE TABLE IF NOT EXISTS principal( + pid INTEGER PRIMARY KEY, + uid TEXT NOT NULL UNIQUE +) STRICT; +CREATE TABLE IF NOT EXISTS claim( + cid TEXT NOT NULL, + key TEXT NOT NULL, + pid INTEGER NOT NULL, + value ANY, + ts INTEGER NOT NULL, + sig BLOB NOT NULL, + seq INTEGER NOT NULL, + PRIMARY KEY (cid, key, pid) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS cid_group(cid TEXT PRIMARY KEY, rid TEXT NOT NULL) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS meta(name TEXT PRIMARY KEY, value TEXT NOT NULL) STRICT, WITHOUT ROWID; +` + +const lastSeqName = "last_seq" + +// ClaimStatus is the per-claim outcome of POST /api/claims. +type ClaimStatus string + +const ( + StatusAccepted ClaimStatus = "accepted" + StatusUnchanged ClaimStatus = "unchanged" + StatusStale ClaimStatus = "stale" + StatusBadSignature ClaimStatus = "bad_signature" + StatusBadRequest ClaimStatus = "bad_request" +) + +// ClaimResult is one entry of the POST /api/claims response. +type ClaimResult struct { + Status ClaimStatus `json:"status"` + Reason string `json:"reason,omitempty"` +} + +// MaxClaimsPerBatch bounds one POST /api/claims. +const MaxClaimsPerBatch = 10_000 + +// change is one committed truth change, queued for the apply loop in commit +// order. key == "" means "membership only" (a cid joined a group). +type change struct { + seq int64 + cid string + key string + pid int64 + live bool // the claim now holds a value (false: tombstone) + regroup *[2]string // {loser, survivor}: two groups merged +} + +func valueArg(v claim.Value) any { + switch v.Kind { + case claim.KindText: + return v.Text + case claim.KindBytes: + if v.Bytes == nil { + return []byte{} + } + return v.Bytes + } + return nil +} + +// scanValue turns a claim.value column (TEXT / BLOB / NULL) back into a Value. +func scanValue(raw any) claim.Value { + switch x := raw.(type) { + case string: + return claim.Text(x) + case []byte: + return claim.Bytes(x) + } + return claim.Null() +} + +type storedClaim struct { + ts int64 + sig []byte + value claim.Value +} + +func readStored(ctx context.Context, q interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +}, cid, key, uid string) (storedClaim, bool, error) { + var sc storedClaim + var raw any + err := q.QueryRowContext(ctx, + `SELECT c.ts, c.sig, c.value FROM claim c JOIN principal p ON p.pid = c.pid + WHERE c.cid = ? AND c.key = ? AND p.uid = ?`, cid, key, uid).Scan(&sc.ts, &sc.sig, &raw) + sc.value = scanValue(raw) + if errors.Is(err, sql.ErrNoRows) { + return sc, false, nil + } + return sc, err == nil, err +} + +// precheck classifies a claim against the committed state without the writer: +// skip-identical happens BEFORE verification (most writes are re-persists of +// the same signed claim), stale is refused, everything else is verified. +func (c *Client) precheck(ctx context.Context, cl *claim.Claim) (ClaimStatus, string) { + if cl.CID == "" || cl.Key == "" || cl.UID == "" { + return StatusBadRequest, "cid, uid and key are required" + } + if !claimKeyAllowed(cl.Key) { + return StatusBadRequest, "key is reserved or malformed" + } + if len(cl.Key) > 4096 || len(cl.CID) > 512 { + return StatusBadRequest, "cid or key too long" + } + if cl.TS == 0 || cl.TS > 1<<62 { + return StatusBadRequest, "ts out of range" + } + sc, ok, err := readStored(ctx, c.truth.DB(), cl.CID, cl.Key, cl.UID) + if err != nil { + return StatusBadRequest, err.Error() + } + if ok { + if sc.ts == int64(cl.TS) && string(sc.sig) == string(cl.Sig) { + return StatusUnchanged, "" + } + // Same signer re-asserting the value it already holds (the gateway + // re-persists every search hit with a fresh ts): keep the stored row and + // its older ts, so oldest-ts tie-breaking stays anchored and truth takes + // no write. Nothing is stored, so the new signature needs no check. + if sc.value.Equal(cl.Value) { + return StatusUnchanged, "" + } + if sc.ts >= int64(cl.TS) { + return StatusStale, "a newer or equal ts is stored for this signer, cid and key" + } + } + if err := cl.Verify(); err != nil { + return StatusBadSignature, err.Error() + } + return StatusAccepted, "" +} + +// WriteClaims is POST /api/claims: verify, store, regroup, and wait until the +// cache has applied the batch (read-your-writes for the caller). +func (c *Client) WriteClaims(ctx context.Context, claims []claim.Claim) ([]ClaimResult, error) { + truth, err := c.truthStore() + if err != nil { + return nil, err + } + results := make([]ClaimResult, len(claims)) + var todo []int + for i := range claims { + st, reason := c.precheck(ctx, &claims[i]) + results[i] = ClaimResult{Status: st, Reason: reason} + if st == StatusAccepted { + todo = append(todo, i) + } + } + if len(todo) == 0 { + return results, nil + } + var maxSeq int64 + err = truth.Update(ctx, func(tx *kv.Tx) error { + seq, err := readLastSeq(tx) + if err != nil { + return err + } + var changes []change + pids := map[string]int64{} + for _, i := range todo { + cl := &claims[i] + // Re-check under the writer: a concurrent batch may have stored a + // newer vote since the precheck. + sc, ok, err := readStored(tx.Context(), txQuerier{tx}, cl.CID, cl.Key, cl.UID) + if err != nil { + return err + } + if ok && (sc.ts >= int64(cl.TS) || sc.value.Equal(cl.Value)) { + if sc.value.Equal(cl.Value) { + results[i] = ClaimResult{Status: StatusUnchanged} + } else { + results[i] = ClaimResult{Status: StatusStale, Reason: "a newer or equal ts is stored for this signer, cid and key"} + } + continue + } + pid, ok := pids[cl.UID] + if !ok { + if pid, err = ensurePrincipal(tx, cl.UID); err != nil { + return err + } + pids[cl.UID] = pid + } + seq++ + if _, err := tx.Exec( + `INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(cid, key, pid) DO UPDATE SET value = excluded.value, ts = excluded.ts, + sig = excluded.sig, seq = excluded.seq`, + cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, seq); err != nil { + return fmt.Errorf("store claim: %w", err) + } + if _, _, err := ensureGroup(tx, cl.CID); err != nil { + return err + } + changes = append(changes, change{seq: seq, cid: cl.CID, key: cl.Key, pid: pid, live: !cl.Value.IsNull()}) + if other := cidFromKeysetField(cl.Key); other != "" && cl.Value.Kind == claim.KindText && other != cl.CID { + more, err := linkGroups(tx, cl.CID, other, seq) + if err != nil { + return err + } + changes = append(changes, more...) + } + } + if len(changes) == 0 { + return nil + } + if err := writeLastSeq(tx, seq); err != nil { + return err + } + maxSeq = seq + tx.OnCommit(func() { c.applier.enqueue(changes) }) + return nil + }) + if err != nil { + return nil, err + } + if maxSeq > 0 { + if err := c.applier.wait(ctx, maxSeq); err != nil { + return results, err + } + } + return results, nil +} + +// txQuerier adapts a kv.Tx to the QueryRowContext shape readStored takes. +type txQuerier struct{ tx *kv.Tx } + +func (q txQuerier) QueryRowContext(_ context.Context, query string, args ...any) *sql.Row { + return q.tx.QueryRow(query, args...) +} + +func readLastSeq(tx *kv.Tx) (int64, error) { + var v string + err := tx.QueryRow(`SELECT value FROM meta WHERE name = ?`, lastSeqName).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + if err != nil { + return 0, err + } + return strconv.ParseInt(v, 10, 64) +} + +func writeLastSeq(tx *kv.Tx, seq int64) error { + _, err := tx.Exec(`INSERT INTO meta(name, value) VALUES(?, ?) + ON CONFLICT(name) DO UPDATE SET value = excluded.value`, lastSeqName, strconv.FormatInt(seq, 10)) + return err +} + +func ensurePrincipal(tx *kv.Tx, uid string) (int64, error) { + var pid int64 + err := tx.QueryRow(`SELECT pid FROM principal WHERE uid = ?`, uid).Scan(&pid) + if err == nil { + return pid, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + if err := tx.QueryRow(`INSERT INTO principal(uid) VALUES(?) RETURNING pid`, uid).Scan(&pid); err != nil { + return 0, fmt.Errorf("new principal: %w", err) + } + return pid, nil +} + +// ensureGroup returns cid's rid, minting a new group for an unknown cid. +func ensureGroup(tx *kv.Tx, cid string) (string, bool, error) { + var rid string + err := tx.QueryRow(`SELECT rid FROM cid_group WHERE cid = ?`, cid).Scan(&rid) + if err == nil { + return rid, false, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return "", false, err + } + if rid, err = NewUUID(); err != nil { + return "", false, err + } + if _, err := tx.Exec(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`, cid, rid); err != nil { + return "", false, fmt.Errorf("new group: %w", err) + } + return rid, true, nil +} + +// linkGroups records that a and b are the same content. An unknown b joins a's +// group; two known groups merge, the older rid surviving (UUIDv7 sorts by +// time). A retraction of the link does not split the group (phase 1). +func linkGroups(tx *kv.Tx, a, b string, seq int64) ([]change, error) { + ra, _, err := ensureGroup(tx, a) + if err != nil { + return nil, err + } + var rb string + err = tx.QueryRow(`SELECT rid FROM cid_group WHERE cid = ?`, b).Scan(&rb) + if errors.Is(err, sql.ErrNoRows) { + if _, err := tx.Exec(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`, b, ra); err != nil { + return nil, err + } + return []change{{seq: seq, cid: b}}, nil + } + if err != nil { + return nil, err + } + if ra == rb { + return nil, nil + } + survivor, loser := ra, rb + if rb < ra { + survivor, loser = rb, ra + } + // cid_group has no rid index (truth keeps primary keys only); a merge is + // rare, so the scan is acceptable. + if _, err := tx.Exec(`UPDATE cid_group SET rid = ? WHERE rid = ?`, survivor, loser); err != nil { + return nil, fmt.Errorf("merge groups: %w", err) + } + return []change{{seq: seq, regroup: &[2]string{loser, survivor}}}, nil +} + +// truthRid reads cid's committed group. +func (c *Client) truthRid(ctx context.Context, cid string) (string, error) { + var rid string + err := c.truth.DB().QueryRowContext(ctx, `SELECT rid FROM cid_group WHERE cid = ?`, cid).Scan(&rid) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return rid, err +} + +// votesFor reads every committed vote on key across cids, with signer uids. +func (c *Client) votesFor(ctx context.Context, cids []string, key string) ([]Vote, error) { + var out []Vote + for _, cid := range cids { + rows, err := c.truth.DB().QueryContext(ctx, + `SELECT c.pid, p.uid, c.value, c.ts FROM claim c JOIN principal p ON p.pid = c.pid + WHERE c.cid = ? AND c.key = ?`, cid, key) + if err != nil { + return nil, err + } + for rows.Next() { + var v Vote + var raw any + var ts int64 + if err := rows.Scan(&v.Pid, &v.Uid, &raw, &ts); err != nil { + rows.Close() + return nil, err + } + v.Value, v.TS = scanValue(raw), uint64(ts) + out = append(out, v) + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} + +// claimKeys lists the distinct keys claimed on any of cids. +func (c *Client) claimKeys(ctx context.Context, cids []string) ([]string, error) { + seen := map[string]bool{} + var out []string + for _, cid := range cids { + rows, err := c.truth.DB().QueryContext(ctx, `SELECT DISTINCT key FROM claim WHERE cid = ?`, cid) + if err != nil { + return nil, err + } + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, err + } + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} + +// ClaimsForCIDs returns every live signed claim on cids (tombstones excluded), +// optionally only for one key. +func (c *Client) ClaimsForCIDs(cids []string, key string) ([]claim.Claim, error) { + if _, err := c.truthStore(); err != nil { + return nil, err + } + ctx, cancel := opCtx() + defer cancel() + var out []claim.Claim + for _, cid := range cids { + q := `SELECT c.cid, p.uid, c.key, c.value, c.ts, c.sig FROM claim c JOIN principal p ON p.pid = c.pid + WHERE c.cid = ? AND c.value IS NOT NULL` + args := []any{cid} + if key != "" { + q += ` AND c.key = ?` + args = append(args, key) + } + q += ` ORDER BY c.key, p.uid` + rows, err := c.truth.DB().QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + for rows.Next() { + var cl claim.Claim + var raw any + var ts int64 + if err := rows.Scan(&cl.CID, &cl.UID, &cl.Key, &raw, &ts, &cl.Sig); err != nil { + rows.Close() + return nil, err + } + cl.Value, cl.TS = scanValue(raw), uint64(ts) + out = append(out, cl) + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} + +// ClaimsForRID returns the live claims of every cid in rid's group. +func (c *Client) ClaimsForRID(rid, key string) ([]claim.Claim, error) { + _, claims, err := c.RecordClaims(rid, key) + return claims, err +} + +// RecordClaims returns the cids of rid's group (empty: no such record) and +// their live claims, optionally for one key. +func (c *Client) RecordClaims(rid, key string) ([]string, []claim.Claim, error) { + s, err := c.kv() + if err != nil { + return nil, nil, err + } + ctx, cancel := opCtx() + defer cancel() + cids, err := cidsForRoot(ctx, s, rid) + if err != nil || len(cids) == 0 { + return cids, nil, err + } + claims, err := c.ClaimsForCIDs(cids, key) + return cids, claims, err +} + +// LiveClaimsBySigner lists (cid, key) of every live claim uid has signed — the +// set DELETE /api/identity retracts. +func (c *Client) LiveClaimsBySigner(uid string) ([][2]string, error) { + s, err := c.kv() + if err != nil { + return nil, err + } + pid, ok, err := c.pidOf(uid) + if err != nil || !ok { + return nil, err + } + ctx, cancel := opCtx() + defer cancel() + rows, err := s.DB().QueryContext(ctx, `SELECT cid, key FROM claim_by_pid WHERE pid = ? ORDER BY cid, key`, pid) + if err != nil { + return nil, err + } + defer rows.Close() + var out [][2]string + for rows.Next() { + var p [2]string + if err := rows.Scan(&p[0], &p[1]); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +func (c *Client) pidOf(uid string) (int64, bool, error) { + if _, err := c.truthStore(); err != nil { + return 0, false, err + } + ctx, cancel := opCtx() + defer cancel() + var pid int64 + err := c.truth.DB().QueryRowContext(ctx, `SELECT pid FROM principal WHERE uid = ?`, uid).Scan(&pid) + if errors.Is(err, sql.ErrNoRows) { + return 0, false, nil + } + return pid, err == nil, err +} + +// CountClaims counts claim rows (tombstones included) — /api/stats' redisKeys. +func (c *Client) CountClaims() (int64, error) { + if _, err := c.truthStore(); err != nil { + return 0, err + } + ctx, cancel := opCtx() + defer cancel() + var n int64 + err := c.truth.DB().QueryRowContext(ctx, `SELECT count(*) FROM claim`).Scan(&n) + return n, err +} + +func isLocalField(key string) bool { + return key == "filePath" || key == "mtimeNano" || key == "sizeByte" +} + +// claimKeyAllowed rejects keys that would corrupt the merged view's key +// layout, and the removed per-algorithm CID shapes (cid_*, midhash256, +// canonical_cid — METADATA_KEYS.md §14.13: write the cids/ key-set). +func claimKeyAllowed(key string) bool { + if strings.HasPrefix(key, "cid_") || key == "midhash256" || key == "canonical_cid" { + return false + } + return !strings.HasPrefix(key, "/") && !strings.Contains(key, "\x00") && !isReservedFieldName(key) +} + +// GetMetadataFlatAs is GetMetadataFlat seen by viewer: the merged record with +// the viewer's own live text claims overlaid (their value wins in their view). +func (c *Client) GetMetadataFlatAs(rid, viewer string) (map[string]string, error) { + s, err := c.kv() + if err != nil { + return nil, err + } + ctx, cancel := opCtx() + defer cancel() + return c.readRecordAs(ctx, s, rid, viewer) +} + +func (c *Client) readRecordAs(ctx context.Context, s *kv.Store, rid, viewer string) (map[string]string, error) { + fields, err := readRecord(ctx, s, rid) + if err != nil || viewer == "" { + return fields, err + } + over, err := c.viewerOverrides(ctx, s, rid, viewer) + if err != nil { + return nil, err + } + if len(over) == 0 { + return fields, nil + } + if fields == nil { + fields = make(map[string]string, len(over)) + } + for k, v := range over { + fields[k] = v + } + return fields, nil +} + +// viewerOverrides returns the viewer's newest live text vote per key on the +// cids of rid's group. +func (c *Client) viewerOverrides(ctx context.Context, s *kv.Store, rid, viewer string) (map[string]string, error) { + pid, ok, err := c.pidOf(viewer) + if err != nil || !ok { + return nil, err + } + cids, err := cidsForRoot(ctx, s, rid) + if err != nil || len(cids) == 0 { + return nil, err + } + out := map[string]string{} + newest := map[string]int64{} + for _, cid := range cids { + rows, err := s.DB().QueryContext(ctx, `SELECT key FROM claim_by_pid WHERE pid = ? AND cid = ?`, pid, cid) + if err != nil { + return nil, err + } + var keys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, err + } + keys = append(keys, k) + } + if err := rows.Close(); err != nil { + return nil, err + } + for _, k := range keys { + var raw any + var ts int64 + if err := c.truth.DB().QueryRowContext(ctx, + `SELECT value, ts FROM claim WHERE cid = ? AND key = ? AND pid = ?`, cid, k, pid).Scan(&raw, &ts); err != nil { + continue + } + if v := scanValue(raw); v.Kind == claim.KindText && ts > newest[k] { + out[k], newest[k] = v.Text, ts + } + } + } + return out, nil +} + +// viewerRIDs lists the records the viewer holds live claims on. +func (c *Client) viewerRIDs(ctx context.Context, viewer string) ([]string, error) { + pid, ok, err := c.pidOf(viewer) + if err != nil || !ok { + return nil, err + } + s := c.store + rows, err := s.DB().QueryContext(ctx, + `SELECT DISTINCT k.v FROM claim_by_pid b JOIN kv k ON k.k = 'cid:' || b.cid WHERE b.pid = ? LIMIT 10000`, pid) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var rid string + if err := rows.Scan(&rid); err != nil { + return nil, err + } + out = append(out, rid) + } + return out, rows.Err() +} + +// MergeClaims is the stateless POST /api/merge: verify each claim (invalid +// ones are dropped and counted, never an error for the batch), merge per key +// with the §7 rule, and overlay viewer's own votes — those in the request and +// those this core holds (claim_by_pid) for the same cids. Returns the merged +// fields, the agreeing-signer count per key, and how many claims were rejected. +func (c *Client) MergeClaims(claims []claim.Claim, viewer string) (map[string]string, map[string]int, int, error) { + byKey := map[string][]Vote{} + rejected := 0 + cids := map[string]bool{} + for i := range claims { + cl := &claims[i] + if err := cl.Verify(); err != nil { + rejected++ + continue + } + cids[cl.CID] = true + byKey[cl.Key] = append(byKey[cl.Key], Vote{Uid: cl.UID, Value: cl.Value, TS: cl.TS}) + } + fields := make(map[string]string, len(byKey)) + nsrc := make(map[string]int, len(byKey)) + for key, votes := range byKey { + if res := Merge(votes, viewer); res.OK { + fields[key], nsrc[key] = res.Value, res.Nsrc + } + } + if viewer == "" { + return fields, nsrc, rejected, nil + } + s, err := c.kv() + if err != nil { + return fields, nsrc, rejected, nil // no local core: request overlay only + } + if _, err := c.truthStore(); err != nil { + return fields, nsrc, rejected, nil + } + ctx, cancel := opCtx() + defer cancel() + seen := map[string]bool{} + for cid := range cids { + rid, _, err := s.Get(ctx, CIDIndexPrefix+cid) + if err != nil || rid == "" || seen[rid] { + continue + } + seen[rid] = true + over, err := c.viewerOverrides(ctx, s, rid, viewer) + if err != nil { + return nil, nil, rejected, err + } + for k, v := range over { + fields[k], nsrc[k] = v, 1 + } + } + return fields, nsrc, rejected, nil +} diff --git a/internal/storage/udl.go b/internal/storage/udl.go index 147852c..6dcaeb7 100644 --- a/internal/storage/udl.go +++ b/internal/storage/udl.go @@ -1,441 +1,185 @@ package storage -// User Data Layer (UDL) storage. -// -// Holds per-user, identity-signed records keyed by (uid, cid, key) — the -// subjective/per-user state that METADATA_KEYS.md rule #4 bars from the shared -// /file/{cid} hash (likes, ratings, watch progress, local paths). Each record -// is an opaque, client-signed (and, for private-tier keys, client-encrypted) -// CBOR blob; meta-core stores it verbatim and never needs the account key. -// -// Convergence is version-gated: a write is accepted only if its `version` -// strictly exceeds the stored one (a per-cell high-water-mark, replacing the -// op-log merge that used to live in meta-watch's store.rs). -// -// Key model (all under the literal "udl:" prefix, mirroring the "file:" one): -// -// udl:rec:// JSON { record, version, ts, value? } -// udl:idx:user::key: set of -> active_for_user_key (My List, Continue Watching) -// udl:idx:user::cid: set of -> list_for_user_cid -// udl:idx:user::cids set of -> whole-profile export (recovery) -// udl:idx:cid::key: set of -> aggregate -// udl:idx:cid::users set of -> uids_for_cid -// -// A "set" is a key-set: one key `\x1f` = "" per member, read as -// the key range under `\x1f`. The separator is a control character -// because UDL keys are not validated — with `/` the members of -// `…:key:a/b` would show up as members of `…:key:a`. -// -// The `value` field is the plaintext JSON scalar for PUBLIC keys only -// (like/rating), used for aggregation; it is absent for private-tier keys -// (whose real value rides encrypted inside `record`) and for tombstones. +// The User Data Layer facade (/api/udl/*). Since v2.0.0 a user's data are +// ordinary claims signed by the user's key — plaintext for public-social keys +// (like, userRating, profile:*), ciphertext for private ones (seek, watched, +// …). There is no separate store: these helpers read the claims of one signer +// through the cache's claim_by_pid index and the truth claim table. import ( - "context" - "encoding/json" - "fmt" + "database/sql" + "errors" "sort" - "github.com/metazla/meta-core/internal/kv" + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/identity" ) -// UDLCell is a decoded (uid, cid, key) record cell. -type UDLCell struct { - Record string // base64 CBOR, opaque + signed - Version int64 - Ts int64 - Value string // plaintext JSON scalar, public keys only - HasValue bool +// UDLUserStats summarises one signer's live claims. +type UDLUserStats struct { + Records int `json:"records"` + Cids int `json:"cids"` + Keys int `json:"keys"` } -// UDLUserKeyEntry is one row of active_for_user_key (all cids for a uid+key). -type UDLUserKeyEntry struct { - Cid string `json:"cid"` - Record string `json:"record"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` -} - -// UDLUserCidEntry is one row of list_for_user_cid (all keys for a uid+cid). -type UDLUserCidEntry struct { - Key string `json:"key"` - Record string `json:"record"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` -} - -func udlRecKey(uid, cid, key string) string { - return fmt.Sprintf("udl:rec:%s/%s/%s", uid, cid, key) -} -func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) } -func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) } -func udlIdxCidKey(cid, key string) string { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) } -func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) } - -// udlIdxUserCids is the set of every cid a uid has ANY record for — the one -// index that makes "export this whole profile" a set read instead of a -// keyspace scan. -// -// The other four indexes each answer a question with one coordinate already -// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and -// nothing else. Without this set the only ways to enumerate a profile are to -// iterate the hardcoded key list — which silently misses any key added later — -// or to SCAN `udl:rec:/*`, which is the full-keyspace-read pathology that -// already made `/api/udl/users/stats` admin-only. -func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) } - -// udlSetSep separates a set key from a member (see the key model above). -const udlSetSep = "\x1f" - -// udlCellJSON is a cell's stored form. Value is a pointer so an absent value -// (private-tier key, tombstone) stays distinguishable from an empty one. -type udlCellJSON struct { - Record string `json:"record"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` - Value *string `json:"value,omitempty"` -} - -func decodeUDLCell(raw string) (UDLCell, error) { - var j udlCellJSON - if err := json.Unmarshal([]byte(raw), &j); err != nil { - return UDLCell{}, err - } - cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts} - if j.Value != nil { - cell.Value, cell.HasValue = *j.Value, true - } - return cell, nil -} - -// udlMembers lists a key-set's members, sorted. -func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) { - prefix := set + udlSetSep - rows, err := r.Range(ctx, prefix, "", 0) - if err != nil { - return nil, err - } - out := make([]string, 0, len(rows)) - for _, row := range rows { - out = append(out, row.K[len(prefix):]) - } - return out, nil -} - -func udlAdd(tx *kv.Tx, set, member string) error { - _, err := tx.PutIfAbsent(set+udlSetSep+member, "") - return err -} - -func udlRem(tx *kv.Tx, set, member string) error { - _, err := tx.Delete(set + udlSetSep + member) - return err -} - -// readUDLCell reads one cell. exists is false when it was never written. -func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) { - raw, ok, err := r.Get(ctx, recKey) - if err != nil || !ok { - return UDLCell{}, false, err - } - cell, err := decodeUDLCell(raw) - if err != nil { - return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err) - } - return cell, true, nil -} - -// readUDLCells reads many cells in one multi-key read; absent cells are -// omitted from the map. -func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) { - raws, err := r.MGet(ctx, recKeys) - if err != nil { - return nil, err - } - out := make(map[string]UDLCell, len(raws)) - for k, raw := range raws { - cell, err := decodeUDLCell(raw) - if err != nil { - return nil, fmt.Errorf("decode %s: %w", k, err) - } - out[k] = cell - } - return out, nil -} - -// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored -// one, and maintains the indexes — one transaction, so the gate and the write -// cannot interleave with a concurrent upsert of the same cell. -// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by -// aggregation); pass hasValue=false for private-tier keys and tombstones. Set -// tombstone=true when the record deletes the cell. -// -// A tombstone still writes the cell — that is the CRDT state, and dropping it -// would let an older write resurrect the value — but it removes the cid from -// the user+key index. That index is what `active_for_user_key` returns, so -// leaving deleted cids in it means every My List / Continue Watching read -// carries every title ever un-liked or finished, forever. The other four -// indexes are left alone: they answer "who has an opinion about this cid", -// where a tombstone is still an answer. -// -// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even -// though it looks symmetric with the user+key index. It is the export index: a -// cid dropped from it is a cid whose tombstone never travels, and a tombstone -// that does not travel is exactly how an un-liked title comes back from the -// dead on the other box — the peer still holds the older `like` at a lower -// version, and with no tombstone to beat it, nothing ever supersedes it. -// -// Returns accepted=false when the write is stale. -func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) { - accepted := false - err := c.update(func(tx *kv.Tx) error { - recKey := udlRecKey(uid, cid, key) - stored, ok, err := readUDLCell(tx.Context(), tx, recKey) - if err != nil { - return err - } - if ok && stored.Version >= version { - return nil - } - cell := udlCellJSON{Record: record, Version: version, Ts: ts} - if hasValue { - v := publicValue - cell.Value = &v - } - body, err := json.Marshal(cell) - if err != nil { - return err - } - if err := tx.Put(recKey, string(body)); err != nil { - return err - } - if tombstone { - err = udlRem(tx, udlIdxUserKey(uid, key), cid) - } else { - err = udlAdd(tx, udlIdxUserKey(uid, key), cid) - } - if err != nil { - return err - } - for _, add := range [][2]string{ - {udlIdxUserCid(uid, cid), key}, - {udlIdxCidKey(cid, key), uid}, - {udlIdxCidUsers(cid), uid}, - {udlIdxUserCids(uid), cid}, - } { - if err := udlAdd(tx, add[0], add[1]); err != nil { - return err - } - } - accepted = true - return nil - }) - if err != nil { - return false, fmt.Errorf("udl upsert failed: %w", err) - } - return accepted, nil -} - -// UDLGetCell returns the record cell for (uid, cid, key). exists is false when -// no record has ever been written for the cell. -func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) { - s, err := c.kv() - if err != nil { - return UDLCell{}, false, err +// UDLGetClaim returns the stored claim of uid on (cid, key), tombstone +// included (a client needs its ts to write a newer one). ok is false when the +// signer never voted on it. +func (c *Client) UDLGetClaim(uid, cid, key string) (claim.Claim, bool, error) { + if _, err := c.truthStore(); err != nil { + return claim.Claim{}, false, err } ctx, cancel := opCtx() defer cancel() - return readUDLCell(ctx, s, udlRecKey(uid, cid, key)) + cl := claim.Claim{CID: cid, UID: uid, Key: key} + var raw any + var ts int64 + err := c.truth.DB().QueryRowContext(ctx, + `SELECT c.value, c.ts, c.sig FROM claim c JOIN principal p ON p.pid = c.pid + WHERE c.cid = ? AND c.key = ? AND p.uid = ?`, cid, key, uid).Scan(&raw, &ts, &cl.Sig) + if errors.Is(err, sql.ErrNoRows) { + return claim.Claim{}, false, nil + } + if err != nil { + return claim.Claim{}, false, err + } + cl.Value, cl.TS = scanValue(raw), uint64(ts) + return cl, true, nil } -// UDLListUserKey returns every cell for (uid, key) across all cids — the -// projection behind active_for_user_key (My List, Continue Watching). Includes -// tombstones; the caller decodes each record and filters. -func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) { - s, err := c.kv() +// UDLClaimsForUser returns uid's live claims, optionally narrowed to one cid +// and/or one key, ordered by cid then key. +func (c *Client) UDLClaimsForUser(uid, cid, key string) ([]claim.Claim, error) { + refs, err := c.LiveClaimsBySigner(uid) if err != nil { return nil, err } - ctx, cancel := opCtx() - defer cancel() - - cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key)) - if err != nil { - return nil, err - } - recKeys := make([]string, len(cids)) - for i, cid := range cids { - recKeys[i] = udlRecKey(uid, cid, key) - } - cells, err := readUDLCells(ctx, s, recKeys) - if err != nil { - return nil, err - } - out := make([]UDLUserKeyEntry, 0, len(cids)) - for i, cid := range cids { - cell, ok := cells[recKeys[i]] - if !ok { + var out []claim.Claim + for _, r := range refs { + if (cid != "" && r[0] != cid) || (key != "" && r[1] != key) { continue } - out = append(out, UDLUserKeyEntry{Cid: cid, Record: cell.Record, Version: cell.Version, Ts: cell.Ts}) - } - return out, nil -} - -// UDLListUserCid returns every cell for (uid, cid) across all keys — the -// projection behind list_for_user_cid. -func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) { - s, err := c.kv() - if err != nil { - return nil, err - } - ctx, cancel := opCtx() - defer cancel() - - keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid)) - if err != nil { - return nil, err - } - recKeys := make([]string, len(keys)) - for i, key := range keys { - recKeys[i] = udlRecKey(uid, cid, key) - } - cells, err := readUDLCells(ctx, s, recKeys) - if err != nil { - return nil, err - } - out := make([]UDLUserCidEntry, 0, len(keys)) - for i, key := range keys { - cell, ok := cells[recKeys[i]] - if !ok { - continue + cl, ok, err := c.UDLGetClaim(uid, r[0], r[1]) + if err != nil { + return nil, err + } + if ok && !cl.Value.IsNull() { + out = append(out, cl) } - out = append(out, UDLUserCidEntry{Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts}) } return out, nil } -// UDLUserAllEntry is one row of a whole-profile export. -type UDLUserAllEntry struct { - Cid string `json:"cid"` - Key string `json:"key"` - Record string `json:"record"` - Version int64 `json:"version"` - Ts int64 `json:"ts"` -} - -// UDLAllForUser returns every cell a uid owns, for cross-instance recovery. -// -// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never -// split across a page: the page ends on a cid boundary once `limit` rows have -// been emitted, and `nextCid` is the last cid included — pass it back as -// `afterCid` to continue. `nextCid` is empty when the export is complete. -// -// Tombstones are included. They are the CRDT state that stops a deleted value -// resurrecting on the box that receives this export, so omitting them would -// make recovery actively wrong rather than merely incomplete. -func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) { - s, err := c.kv() +// UDLAllForUser pages uid's live claims by cid: claims of cids strictly after +// `after`, stopping at a cid boundary once limit claims are collected. +// nextCid is "" when the walk is complete. +func (c *Client) UDLAllForUser(uid, after string, limit int) ([]claim.Claim, string, error) { + all, err := c.UDLClaimsForUser(uid, "", "") if err != nil { return nil, "", err } - if limit <= 0 { - limit = 1000 - } - ctx, cancel := opCtx() - defer cancel() - - cids, err := udlMembers(ctx, s, udlIdxUserCids(uid)) - if err != nil { - return nil, "", err - } - sort.Strings(cids) - - out := make([]UDLUserAllEntry, 0, limit) - nextCid := "" - for _, cid := range cids { - if afterCid != "" && cid <= afterCid { + var out []claim.Claim + for _, cl := range all { + if after != "" && cl.CID <= after { continue } - if len(out) >= limit { - nextCid = out[len(out)-1].Cid - break - } - keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid)) - if err != nil { - return nil, "", err - } - for _, key := range keys { - cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key)) - if err != nil { - return nil, "", err - } - if !ok { - continue - } - out = append(out, UDLUserAllEntry{ - Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts, - }) + if len(out) >= limit && cl.CID != out[len(out)-1].CID { + return out, out[len(out)-1].CID, nil } + out = append(out, cl) } - return out, nextCid, nil + return out, "", nil } -// UDLCountUserCids is the number of distinct cids a uid holds records for. -// -// One index range count, which is what makes it usable as a per-push quota -// check on the gateway mirror — a quota that cost a keyspace walk would be a -// denial-of-service in its own right. Counts cids rather than cells because -// that is what the index holds; a profile is bounded by the titles it has -// touched, not by how many keys it has per title. -func (c *Client) UDLCountUserCids(uid string) (int64, error) { - s, err := c.kv() +// UDLCountUserCids counts the distinct cids uid holds live claims on. +func (c *Client) UDLCountUserCids(uid string) (int, error) { + refs, err := c.LiveClaimsBySigner(uid) if err != nil { return 0, err } - ctx, cancel := opCtx() - defer cancel() - return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep) + seen := map[string]bool{} + for _, r := range refs { + seen[r[0]] = true + } + return len(seen), nil } -// UDLCidUsers returns every uid that has any record for cid — uids_for_cid. +// UDLCidUsers lists the signers holding live claims on cid. func (c *Client) UDLCidUsers(cid string) ([]string, error) { - s, err := c.kv() + claims, err := c.ClaimsForCIDs([]string{cid}, "") if err != nil { return nil, err } - ctx, cancel := opCtx() - defer cancel() - return udlMembers(ctx, s, udlIdxCidUsers(cid)) + seen := map[string]bool{} + out := []string{} + for _, cl := range claims { + if !seen[cl.UID] { + seen[cl.UID] = true + out = append(out, cl.UID) + } + } + sort.Strings(out) + return out, nil } -// UDLPublicValues returns the live plaintext public JSON scalar values for -// (cid, key) across all uids — the input to aggregation. Cells without a value -// (private-tier or tombstoned) are skipped. +// UDLPublicValues returns the plaintext values of every live claim on +// (cid, key) — the input of /api/udl/cid/{cid}/aggregate. func (c *Client) UDLPublicValues(cid, key string) ([]string, error) { - s, err := c.kv() + claims, err := c.ClaimsForCIDs([]string{cid}, key) if err != nil { return nil, err } + out := []string{} + for _, cl := range claims { + if cl.Value.Kind == claim.KindText { + out = append(out, cl.Value.Text) + } + } + return out, nil +} + +// UDLAllUserStats summarises every signer that holds live claims, keyed by +// uid. Callers that want only human accounts intersect it with the account +// roster. +func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) { + s, err := c.kv() + if err != nil { + return nil, err + } + if _, err := c.truthStore(); err != nil { + return nil, err + } ctx, cancel := opCtx() defer cancel() - - uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key)) + rows, err := s.DB().QueryContext(ctx, + `SELECT pid, count(*), count(DISTINCT cid), count(DISTINCT key) FROM claim_by_pid GROUP BY pid`) if err != nil { return nil, err } - recKeys := make([]string, len(uids)) - for i, uid := range uids { - recKeys[i] = udlRecKey(uid, cid, key) + type agg struct { + pid int64 + st UDLUserStats } - cells, err := readUDLCells(ctx, s, recKeys) - if err != nil { + var aggs []agg + for rows.Next() { + var a agg + if err := rows.Scan(&a.pid, &a.st.Records, &a.st.Cids, &a.st.Keys); err != nil { + rows.Close() + return nil, err + } + aggs = append(aggs, a) + } + if err := rows.Close(); err != nil { return nil, err } - out := make([]string, 0, len(uids)) - for _, k := range recKeys { - if cell, ok := cells[k]; ok && cell.HasValue { - out = append(out, cell.Value) + out := make(map[string]UDLUserStats, len(aggs)) + for _, a := range aggs { + var uid string + if err := c.truth.DB().QueryRowContext(ctx, `SELECT uid FROM principal WHERE pid = ?`, a.pid).Scan(&uid); err != nil { + continue + } + if identity.ValidUID(uid) { + out[uid] = a.st } } return out, nil diff --git a/internal/storage/udl_admin.go b/internal/storage/udl_admin.go deleted file mode 100644 index 9ec0125..0000000 --- a/internal/storage/udl_admin.go +++ /dev/null @@ -1,179 +0,0 @@ -package storage - -// Account-scoped UDL administration: what a uid holds, and removing all of it. -// -// Both operations enumerate the same thing — every `udl:rec://` -// cell. Keys are ordered, so one account's cells are the key range under -// `udl:rec:/`, and every account's are the range under `udl:rec:`. -// UDLAllUserStats stays a bulk call (one walk buckets every account) because -// the accounts page needs a number for each. - -import ( - "fmt" - "strings" - - "github.com/metazla/meta-core/internal/kv" -) - -// UDLUserStats counts what one account holds. Records is the number of cells, -// including tombstones — a retracted rating is still a row this account owns -// and still something a purge has to remove. -type UDLUserStats struct { - Records int `json:"records"` - Cids int `json:"cids"` - Keys int `json:"keys"` -} - -// safeUIDForPattern rejects anything that would reach across the key -// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric -// (identity.validUID), so this only ever fires on a hostile or malformed one — -// but it fires here, in the layer that builds the pattern, rather than relying -// on every caller having validated first. -func safeUIDForPattern(uid string) error { - if uid == "" { - return fmt.Errorf("empty uid") - } - if strings.ContainsAny(uid, "*?[]\\/:") { - return fmt.Errorf("uid contains characters that are not valid in a uid: %q", uid) - } - return nil -} - -// splitUDLRecSuffix splits the "//" tail of a udl:rec key. -// -// SplitN with a limit of 3 so a key containing a slash survives intact. cids -// never contain one; keys are a controlled vocabulary today, but a namespaced -// one arriving later must not silently truncate here — a truncated key would -// make the purge SREM the wrong index member and leave the real one behind. -func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) { - parts := strings.SplitN(suffix, "/", 3) - if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { - return "", "", "", false - } - return parts[0], parts[1], parts[2], true -} - -// UDLUserCells returns every (cid, key) pair this account holds a cell for, -// keyed by the "cid/key" tail with the cid as value (the purge needs them -// grouped). -func (c *Client) UDLUserCells(uid string) (map[string]string, error) { - if err := safeUIDForPattern(uid); err != nil { - return nil, err - } - s, err := c.kv() - if err != nil { - return nil, err - } - ctx, cancel := opCtx() - defer cancel() - - out := make(map[string]string) - err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error { - for _, row := range rows { - gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:")) - if !ok || gotUID != uid { - continue - } - out[cid+"/"+key] = cid - } - return nil - }) - return out, err -} - -// UDLAllUserStats counts every account's cells in one walk. -// -// Returns only accounts that actually hold records; a uid absent from the map -// holds none. Callers render a zero for those rather than treating it as an -// error — an account created a minute ago legitimately has nothing. -func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) { - s, err := c.kv() - if err != nil { - return nil, err - } - ctx, cancel := opCtx() - defer cancel() - - type acc struct { - cells int - cids map[string]struct{} - keys map[string]struct{} - } - byUID := make(map[string]*acc) - err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error { - for _, row := range rows { - uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:")) - if !ok { - continue - } - a := byUID[uid] - if a == nil { - a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})} - byUID[uid] = a - } - a.cells++ - a.cids[cid] = struct{}{} - a.keys[key] = struct{}{} - } - return nil - }) - if err != nil { - return nil, err - } - out := make(map[string]UDLUserStats, len(byUID)) - for uid, a := range byUID { - out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)} - } - return out, nil -} - -// UDLPurgeUser removes every trace of one account from the User Data Layer: -// its cells, its three user-scoped indexes, and its membership in the shared -// cid-scoped indexes. Returns what was removed. -// -// The shared indexes are the part that is easy to miss and expensive to leave. -// `udl:idx:cid::key:` and `udl:idx:cid::users` are keyed by cid, -// not by uid, so a purge that only deleted `udl:rec:/*` would leave this -// uid listed as an opinion-holder on every title it ever touched. -// -// Idempotent: purging an account with nothing stored is a no-op returning -// zeroes, so a retry is safe. -func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) { - cells, err := c.UDLUserCells(uid) - if err != nil { - return UDLUserStats{}, err - } - - stats := UDLUserStats{Records: len(cells)} - distinctCids := make(map[string]struct{}) - distinctKeys := make(map[string]struct{}) - err = c.update(func(tx *kv.Tx) error { - for cidKey, cid := range cells { - key := strings.TrimPrefix(cidKey, cid+"/") - distinctCids[cid] = struct{}{} - distinctKeys[key] = struct{}{} - if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil { - return err - } - if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil { - return err - } - } - for cid := range distinctCids { - if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil { - return err - } - } - // The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one - // key range, dropped whole — which also catches index entries whose - // cell is already gone (the state a half-finished purge leaves). - _, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0) - return err - }) - if err != nil { - return UDLUserStats{}, fmt.Errorf("purge: %w", err) - } - stats.Cids = len(distinctCids) - stats.Keys = len(distinctKeys) - return stats, nil -} diff --git a/internal/storage/udl_admin_test.go b/internal/storage/udl_admin_test.go deleted file mode 100644 index 26f9a95..0000000 --- a/internal/storage/udl_admin_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package storage - -import ( - "testing" -) - -// seedCell writes one live UDL cell the way a real client would. -func seedCell(t *testing.T, c *Client, uid, cid, key string) { - t.Helper() - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok { - t.Fatalf("seed %s/%s/%s: ok=%v err=%v", uid, cid, key, ok, err) - } -} - -// The purge has to reach the cid-scoped indexes, not just the user's own rows. -// -// This is the failure that motivated the whole endpoint: deleting an account -// while leaving `udl:idx:cid::users` intact means every aggregate read on -// every title that account ever touched keeps naming a uid nobody can resolve, -// forever — and the rows are unreachable through any API, so nothing will ever -// clean them up. -func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "cidOne", "like") - seedCell(t, c, "zAlice", "cidOne", "rating") - seedCell(t, c, "zAlice", "cidTwo", "seek") - - stats, err := c.UDLPurgeUser("zAlice") - if err != nil { - t.Fatalf("purge: %v", err) - } - if stats.Records != 3 || stats.Cids != 2 || stats.Keys != 3 { - t.Fatalf("stats = %+v, want 3 records / 2 cids / 3 keys", stats) - } - - // Cells gone. - if _, ok, err := c.UDLGetCell("zAlice", "cidOne", "like"); err != nil || ok { - t.Fatalf("cell survived the purge: ok=%v err=%v", ok, err) - } - // User-scoped indexes gone. - if entries, err := c.UDLListUserKey("zAlice", "like"); err != nil || len(entries) != 0 { - t.Fatalf("user+key index survived: %d entries, err=%v", len(entries), err) - } - if entries, err := c.UDLListUserCid("zAlice", "cidOne"); err != nil || len(entries) != 0 { - t.Fatalf("user+cid index survived: %d entries, err=%v", len(entries), err) - } - // Cid-scoped index gone — the one that is easy to miss. - for _, cid := range []string{"cidOne", "cidTwo"} { - uids, err := c.UDLCidUsers(cid) - if err != nil { - t.Fatalf("cid users %s: %v", cid, err) - } - for _, u := range uids { - if u == "zAlice" { - t.Fatalf("%s still lists the deleted account in uids_for_cid", cid) - } - } - } -} - -// Purging one account must not touch another's rows, including on a cid they -// both have an opinion about — where the two share an index set and a careless -// DEL of the whole key would take the survivor's membership with it. -func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "shared", "rating") - seedCell(t, c, "zBob", "shared", "rating") - - if _, err := c.UDLPurgeUser("zAlice"); err != nil { - t.Fatalf("purge: %v", err) - } - - if _, ok, err := c.UDLGetCell("zBob", "shared", "rating"); err != nil || !ok { - t.Fatalf("bob's cell must survive alice's purge: ok=%v err=%v", ok, err) - } - uids, err := c.UDLCidUsers("shared") - if err != nil { - t.Fatalf("cid users: %v", err) - } - if len(uids) != 1 || uids[0] != "zBob" { - t.Fatalf("uids_for_cid = %v, want exactly [zBob]", uids) - } - entries, err := c.UDLListUserKey("zBob", "rating") - if err != nil || len(entries) != 1 { - t.Fatalf("bob's user+key index must survive: %d entries, err=%v", len(entries), err) - } -} - -// A uid that is a prefix of another must not drag its neighbour's rows out. -// The key layout separates them with '/' and ':', so this passes only as long -// as the scan patterns keep those separators. -func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "cidOne", "like") - seedCell(t, c, "zAliceExtra", "cidOne", "like") - - stats, err := c.UDLPurgeUser("zAlice") - if err != nil { - t.Fatalf("purge: %v", err) - } - if stats.Records != 1 { - t.Fatalf("purged %d records, want exactly alice's 1", stats.Records) - } - if _, ok, err := c.UDLGetCell("zAliceExtra", "cidOne", "like"); err != nil || !ok { - t.Fatalf("zAliceExtra's cell must survive: ok=%v err=%v", ok, err) - } -} - -// Tombstoned cells still belong to the account. They are absent from the -// user+key index by design (see udl.go), so a purge that enumerated *that* -// index instead of the cells would leave every retracted rating behind — the -// rows most likely to have accumulated over an account's life. -func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "cidOne", "like") - if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok { - t.Fatalf("tombstone: ok=%v err=%v", ok, err) - } - // Precondition: the tombstone left the cell but dropped it from the index. - if entries, _ := c.UDLListUserKey("zAlice", "like"); len(entries) != 0 { - t.Fatalf("precondition: tombstone should be out of the user+key index") - } - - stats, err := c.UDLPurgeUser("zAlice") - if err != nil { - t.Fatalf("purge: %v", err) - } - if stats.Records != 1 { - t.Fatalf("purged %d records, want the 1 tombstoned cell", stats.Records) - } - if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); ok { - t.Fatal("tombstoned cell survived the purge") - } -} - -// Retry-safe: the delete handler purges before removing the key, so a caller -// that retries after a partial failure runs this a second time. -func TestUDLPurgeUser_IsIdempotent(t *testing.T) { - c := newTestClient(t) - seedCell(t, c, "zAlice", "cidOne", "like") - - if _, err := c.UDLPurgeUser("zAlice"); err != nil { - t.Fatalf("first purge: %v", err) - } - stats, err := c.UDLPurgeUser("zAlice") - if err != nil { - t.Fatalf("second purge must be a no-op, got: %v", err) - } - if stats.Records != 0 || stats.Cids != 0 { - t.Fatalf("second purge = %+v, want zeroes", stats) - } -} - -func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) { - c := newTestClient(t) - seedCell(t, c, "zAlice", "cidOne", "like") - - // "*" would otherwise match every account in the keyspace — a wildcard - // purge dressed up as a uid. - for _, uid := range []string{"*", "z*", "", "z:Alice", "z/Alice"} { - if _, err := c.UDLPurgeUser(uid); err == nil { - t.Errorf("uid %q must be refused before it reaches a SCAN pattern", uid) - } - } - if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); !ok { - t.Fatal("a refused purge must not have deleted anything") - } -} - -func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "cidOne", "like") - seedCell(t, c, "zAlice", "cidOne", "rating") - seedCell(t, c, "zAlice", "cidTwo", "like") - seedCell(t, c, "zBob", "cidOne", "seek") - - stats, err := c.UDLAllUserStats() - if err != nil { - t.Fatalf("stats: %v", err) - } - if got := stats["zAlice"]; got.Records != 3 || got.Cids != 2 || got.Keys != 2 { - t.Fatalf("alice = %+v, want 3 records / 2 cids / 2 keys", got) - } - if got := stats["zBob"]; got.Records != 1 || got.Cids != 1 || got.Keys != 1 { - t.Fatalf("bob = %+v, want 1/1/1", got) - } - // An account with nothing stored is simply absent — the caller renders 0. - if _, present := stats["zNobody"]; present { - t.Fatal("an account with no records must not appear in the map") - } -} - -// The stats and the purge must agree: the number shown in the delete -// confirmation is the number of rows the delete then removes. -func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) { - c := newTestClient(t) - - seedCell(t, c, "zAlice", "cidOne", "like") - seedCell(t, c, "zAlice", "cidTwo", "rating") - seedCell(t, c, "zAlice", "cidTwo", "seek") - - before, err := c.UDLAllUserStats() - if err != nil { - t.Fatalf("stats: %v", err) - } - purged, err := c.UDLPurgeUser("zAlice") - if err != nil { - t.Fatalf("purge: %v", err) - } - if before["zAlice"] != purged { - t.Fatalf("stats said %+v but purge removed %+v", before["zAlice"], purged) - } -} - -func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) { - // A namespaced key arriving later must not be truncated: the purge SREMs - // `udl:idx:cid::key:`, so a half-read key removes the wrong - // member and leaves the real one behind. - uid, cid, key, ok := splitUDLRecSuffix("zAlice/cidOne/ns/sub:key") - if !ok || uid != "zAlice" || cid != "cidOne" || key != "ns/sub:key" { - t.Fatalf("got (%q,%q,%q,%v)", uid, cid, key, ok) - } - for _, bad := range []string{"", "zAlice", "zAlice/cidOne", "zAlice//key", "/cid/key"} { - if _, _, _, ok := splitUDLRecSuffix(bad); ok { - t.Errorf("%q should not parse", bad) - } - } -} diff --git a/internal/storage/udl_export_test.go b/internal/storage/udl_export_test.go deleted file mode 100644 index 0f98726..0000000 --- a/internal/storage/udl_export_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package storage - -import ( - "testing" -) - -// The whole-profile export is what cross-instance recovery reads: a box that -// has just imported a secret key knows the uid and nothing else — not which -// cids the profile touched, not which keys. Every one of these tests is about -// that one caller. - -func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) { - c := newTestClient(t) - - const uid = "zAlice" - writes := []struct{ cid, key, rec string }{ - {"bagcsaaaaone", "like", "rec-one-like"}, - {"bagcsaaaaone", "seek", "rec-one-seek"}, - {"bagcsaaaatwo", "rating", "rec-two-rating"}, - {"bagcsaaaatwo", "profile:name", "rec-two-name"}, - {"bagcsaaathree", "watched", "rec-three-watched"}, - } - for _, w := range writes { - if ok, err := c.UDLUpsertIfNewer(uid, w.cid, w.key, 1, 100, w.rec, "", false, false); err != nil || !ok { - t.Fatalf("upsert %s/%s: ok=%v err=%v", w.cid, w.key, ok, err) - } - } - - entries, next, err := c.UDLAllForUser(uid, "", 0) - if err != nil { - t.Fatalf("export: %v", err) - } - if next != "" { - t.Fatalf("export of 5 rows should be complete, got nextCid=%q", next) - } - if len(entries) != len(writes) { - t.Fatalf("want %d rows, got %d: %+v", len(writes), len(entries), entries) - } - seen := map[string]string{} - for _, e := range entries { - seen[e.Cid+"/"+e.Key] = e.Record - } - for _, w := range writes { - if got := seen[w.cid+"/"+w.key]; got != w.rec { - t.Errorf("%s/%s: want record %q, got %q", w.cid, w.key, w.rec, got) - } - } -} - -// The failure this guards is silent and total: a uid whose cids were never -// indexed exports as empty, and recovery reports success having restored -// nothing. -func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) { - c := newTestClient(t) - - if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok { - t.Fatalf("upsert: ok=%v err=%v", ok, err) - } - entries, _, err := c.UDLAllForUser("zBob", "", 0) - if err != nil { - t.Fatalf("export: %v", err) - } - if len(entries) != 0 { - t.Fatalf("Bob owns nothing; got %+v", entries) - } -} - -// ⚠ The tombstone rule, and the reason the cids index is never pruned. -// -// A tombstone must EXPORT. It is the only thing that can beat the older live -// value the other box still holds at a lower version — drop it from the export -// and an un-liked title comes back from the dead on every peer that already -// saw the `like`. This is the one place the cids index deliberately diverges -// from the user+key index, which does prune. -func TestUDLAllForUser_ExportsTombstones(t *testing.T) { - c := newTestClient(t) - - const ( - uid = "zAlice" - cid = "bagcsaaaaone" - key = "like" - ) - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-liked", "", false, false); err != nil || !ok { - t.Fatalf("live upsert: ok=%v err=%v", ok, err) - } - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-unliked", "", false, true); err != nil || !ok { - t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err) - } - - // The user+key index prunes it — that is what keeps My List short. - live, err := c.UDLListUserKey(uid, key) - if err != nil { - t.Fatalf("active_for_user_key: %v", err) - } - if len(live) != 0 { - t.Fatalf("tombstoned cid should leave the user+key index, got %+v", live) - } - - // The export must still carry it, at the tombstone's version. - entries, _, err := c.UDLAllForUser(uid, "", 0) - if err != nil { - t.Fatalf("export: %v", err) - } - if len(entries) != 1 { - t.Fatalf("tombstone must export, got %+v", entries) - } - if entries[0].Record != "rec-unliked" || entries[0].Version != 2 { - t.Fatalf("export must carry the tombstone, not the superseded value: %+v", entries[0]) - } -} - -func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) { - c := newTestClient(t) - - const uid = "zAlice" - cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"} - for _, cid := range cids { - for _, key := range []string{"like", "seek"} { - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-"+cid+"-"+key, "", false, false); err != nil || !ok { - t.Fatalf("upsert: ok=%v err=%v", ok, err) - } - } - } - - // A cid's keys are never split across a page, so a limit of 1 still yields - // both of the first cid's rows. - page1, next1, err := c.UDLAllForUser(uid, "", 1) - if err != nil { - t.Fatalf("page 1: %v", err) - } - if len(page1) != 2 || page1[0].Cid != cids[0] { - t.Fatalf("page 1 should hold both rows of %s, got %+v", cids[0], page1) - } - if next1 != cids[0] { - t.Fatalf("cursor should name the last cid emitted, want %q got %q", cids[0], next1) - } - - // Walking the cursor must visit every row exactly once. - all := append([]UDLUserAllEntry{}, page1...) - cursor := next1 - for cursor != "" { - page, next, err := c.UDLAllForUser(uid, cursor, 1) - if err != nil { - t.Fatalf("page after %q: %v", cursor, err) - } - all = append(all, page...) - cursor = next - } - if len(all) != 6 { - t.Fatalf("want 6 rows across all pages, got %d: %+v", len(all), all) - } - seen := map[string]bool{} - for _, e := range all { - k := e.Cid + "/" + e.Key - if seen[k] { - t.Fatalf("%s emitted twice across pages", k) - } - seen[k] = true - } -} diff --git a/internal/storage/udl_test.go b/internal/storage/udl_test.go deleted file mode 100644 index c419c91..0000000 --- a/internal/storage/udl_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package storage - -import ( - "testing" -) - -// A tombstone must leave the cell in place but drop the cid from the user+key -// index. -// -// The cell has to stay: it *is* the CRDT state, and deleting it would let an -// older write resurrect the value the tombstone retracted. The index entry has -// to go: `active_for_user_key` returns every member of that set, so a set that -// only ever grows means each My List / Continue Watching read carries every -// title the user has ever un-liked or finished — with its full record blob — -// forever. The reader filters them out again, so the cost is invisible until -// the account is old. -func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) { - c := newTestClient(t) - - const ( - uid = "zAlice" - cid = "bagcsaaalocator" - key = "seek" - ) - - // A live position: indexed, readable. - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-v1", "", false, false); err != nil || !ok { - t.Fatalf("live upsert: ok=%v err=%v", ok, err) - } - entries, err := c.UDLListUserKey(uid, key) - if err != nil { - t.Fatalf("active_for_user_key: %v", err) - } - if len(entries) != 1 || entries[0].Cid != cid { - t.Fatalf("live position should be indexed, got %+v", entries) - } - - // Finishing it tombstones the seek. - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-v2-tombstone", "", false, true); err != nil || !ok { - t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err) - } - - entries, err = c.UDLListUserKey(uid, key) - if err != nil { - t.Fatalf("active_for_user_key after tombstone: %v", err) - } - if len(entries) != 0 { - t.Fatalf("tombstoned cid must leave the user+key index, got %+v", entries) - } - - // ...but the cell itself survives, at the tombstone's version. - cell, exists, err := c.UDLGetCell(uid, cid, key) - if err != nil { - t.Fatalf("get cell: %v", err) - } - if !exists { - t.Fatal("tombstone must not delete the cell — an older write could then resurrect the value") - } - if cell.Version != 2 || cell.Record != "rec-v2-tombstone" { - t.Fatalf("cell should hold the tombstone record at v2, got %+v", cell) - } - - // And the high-water mark still rejects the superseded write. - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 300, "rec-v1-replay", "", false, false); err != nil || ok { - t.Fatalf("stale replay must be rejected: ok=%v err=%v", ok, err) - } - - // Re-watching re-establishes the cell and puts it back in the index. - if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 3, 400, "rec-v3", "", false, false); err != nil || !ok { - t.Fatalf("re-establish upsert: ok=%v err=%v", ok, err) - } - entries, err = c.UDLListUserKey(uid, key) - if err != nil { - t.Fatalf("active_for_user_key after re-establish: %v", err) - } - if len(entries) != 1 || entries[0].Cid != cid { - t.Fatalf("re-established cid must return to the index, got %+v", entries) - } -} diff --git a/internal/storage/write_bench_test.go b/internal/storage/write_bench_test.go index d7845bc..1c96a86 100644 --- a/internal/storage/write_bench_test.go +++ b/internal/storage/write_bench_test.go @@ -1,109 +1,66 @@ package storage import ( + "context" "fmt" - "path/filepath" - "sync" "sync/atomic" "testing" + + "github.com/metazla/meta-core/internal/claim" ) -// BenchmarkConcurrentRecordWrites is the ingest shape the single-writer model -// has to absorb: many goroutines (feeder flushes, gateway persists, plugin -// callbacks) each writing a ~40-field record. SQLite serialises them; this -// measures what that costs. Reported as records/s. -func BenchmarkConcurrentRecordWrites(b *testing.B) { - c := NewClient() - if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil { - b.Fatal(err) +// benchRecord is a 40-field signed record, the shape gateway hits persist. +func benchRecord(b *testing.B, c *Client, i int) []claim.Claim { + t := &testing.T{} + id := signer(t, byte(1+i%7)) + cid := fmt.Sprintf("bafkbench%012d", i) + out := make([]claim.Claim, 0, 40) + for f := 0; f < 40; f++ { + out = append(out, vote(t, id, cid, fmt.Sprintf("field%02d", f), fmt.Sprintf("value-%d-%d", i, f), 1000)) } - defer c.Close() - - record := func(i int64) map[string]string { - m := map[string]string{ - "title": fmt.Sprintf("Some Show S01E%02d", i%40), - "fileName": fmt.Sprintf("some.show.s01e%02d.1080p.mkv", i%40), - "filePath": fmt.Sprintf("/files/watch/some.show.%d.mkv", i), - "fileType": "video", - "tmdbid": fmt.Sprintf("%d", 1000+i%500), - fmt.Sprintf("cids/bafkbench%020d", i): "true", - } - for f := 0; len(m) < 40; f++ { - m[fmt.Sprintf("categories/%d", f)] = "true" - } - return m - } - - const writers = 16 - var next int64 - b.ResetTimer() - var wg sync.WaitGroup - for w := 0; w < writers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - i := atomic.AddInt64(&next, 1) - if i > int64(b.N) { - return - } - if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), record(i)); err != nil { - b.Error(err) - return - } - } - }() - } - wg.Wait() - b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s") + return out } -// BenchmarkConcurrentRepersistUnchanged is the dominant write on a permanent -// store: a gateway re-persisting hits it already holds. The kv layer skips -// every unchanged field, so this should cost reads only. -func BenchmarkConcurrentRepersistUnchanged(b *testing.B) { - c := NewClient() - if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil { - b.Fatal(err) - } +// BenchmarkConcurrentClaimWrites measures new 40-claim records, 16 writers. +func BenchmarkConcurrentClaimWrites(b *testing.B) { + c := openClient(&testing.T{}, b.TempDir()) defer c.Close() - const distinct = 2000 - rec := func(i int64) map[string]string { - m := map[string]string{ - "title": fmt.Sprintf("Some Show S01E%02d", i%40), - "fileType": "video", - fmt.Sprintf("cids/bafkbench%020d", i): "true", - } - for f := 0; len(m) < 40; f++ { - m[fmt.Sprintf("categories/%d", f)] = "true" - } - return m + recs := make([][]claim.Claim, b.N) + for i := range recs { + recs[i] = benchRecord(b, c, i) } - for i := int64(0); i < distinct; i++ { - if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), rec(i)); err != nil { + b.ResetTimer() + b.SetParallelism(16) + var next int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i := int(atomicAdd(&next) - 1) + if _, err := c.WriteClaims(context.Background(), recs[i]); err != nil { + b.Error(err) + } + } + }) +} + +// BenchmarkRepersistUnchangedClaims measures the 94% case: the same signed +// claims again (skip-identical before verification). +func BenchmarkRepersistUnchangedClaims(b *testing.B) { + c := openClient(&testing.T{}, b.TempDir()) + defer c.Close() + const distinct = 200 + recs := make([][]claim.Claim, distinct) + for i := range recs { + recs[i] = benchRecord(b, c, i) + if _, err := c.WriteClaims(context.Background(), recs[i]); err != nil { b.Fatal(err) } } - const writers = 16 - var next int64 b.ResetTimer() - var wg sync.WaitGroup - for w := 0; w < writers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - i := atomic.AddInt64(&next, 1) - if i > int64(b.N) { - return - } - if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i%distinct), rec(i%distinct)); err != nil { - b.Error(err) - return - } - } - }() + for i := 0; i < b.N; i++ { + if _, err := c.WriteClaims(context.Background(), recs[i%distinct]); err != nil { + b.Fatal(err) + } } - wg.Wait() - b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s") } + +func atomicAdd(p *int64) int64 { return atomic.AddInt64(p, 1) } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 02bb39b..4a41398 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -261,6 +261,13 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul w.dispatcher.Dispatch(event) state.Delete(relPath) result.Deleted++ + // Only the box-local facts go; the record's claims stay. + if w.storage != nil { + abs := filepath.Join(w.filesPath, filepath.FromSlash(relPath)) + if err := w.storage.RemoveFileTuple(abs); err != nil { + log.Printf("[Watcher] forget %s: %v", abs, err) + } + } } } diff --git a/tools/claims-migrate/main.go b/tools/claims-migrate/main.go new file mode 100644 index 0000000..c465d34 --- /dev/null +++ b/tools/claims-migrate/main.go @@ -0,0 +1,71 @@ +// Command claims-migrate is the one-off conversion of a meta-core v1.2.0 store +// (one kv table, schema version 2) into a v2.0.0 truth database (the claims +// model, schema version 3) — docs/implementation/metacore-claims-model.md §11. +// It is run once by hand on watch.nsl.sh with meta-core stopped, and deleted +// afterwards; nothing in the build references it. +// +// claims-migrate -old /meta-core/db/meta-core.db -truth /out/meta-core.db \ +// -accounts /meta-core/identity/accounts [-dry-run] +// +// What it does: +// - generates a throwaway "legacy" secp256k1 key in memory (its uid is +// printed; the private key is never written anywhere); +// - every record (rid) with at least one cid: each file:/ becomes +// a claim on the record's canonical cid (internal/cid rank) signed by the +// legacy key at one conversion ts; cids/ members become alias claims +// (canonical, cids/) = "true"; filePath / mtimeNano (watcher facts, +// refilled into local_file) and keys meta-core refuses (cid_*, midhash256, +// canonical_cid, __fields__) are skipped; records with no cid are dropped; +// - cid_group from the cid: → rid index (rids keep their UUIDs); +// - UDL cells (MMUR1 records, signature checked) become MMUR2 claims signed +// by the account key found in -accounts; rating → userRating; private-tier +// envelopes are decrypted (AAD bound to the old version) and re-encrypted +// with a fresh nonce and the MMUR2 AAD (bound to ts); cells whose account is +// not on the box are dropped; +// - btmagnet:* / searchcov:* / other service scratch keys are dropped (they +// live in the cache now and are refetched); derived bookkeeping is skipped. +// +// Every claim is checked with claim.Verify before it is stored. The output is +// created with meta-core's own storage code, so the truth schema cannot drift; +// meta-core rebuilds the cache from it on first boot. +package main + +import ( + "flag" + "fmt" + "os" + "runtime" +) + +func main() { + var o options + flag.StringVar(&o.oldPath, "old", "", "path to the v1 meta-core.db (opened read-only; copy it with its -wal/-shm first)") + flag.StringVar(&o.truthPath, "truth", "", "output truth database path (must not exist)") + flag.StringVar(&o.accountsDir, "accounts", "", "identity accounts dir ({META_CORE_PATH}/identity/accounts); empty = drop every UDL cell") + flag.BoolVar(&o.dryRun, "dry-run", false, "convert, sign and verify everything but write nothing; print counts") + flag.IntVar(&o.workers, "workers", runtime.NumCPU(), "signing goroutines") + flag.Parse() + if o.oldPath == "" || (o.truthPath == "" && !o.dryRun) { + fmt.Fprintln(os.Stderr, "usage: claims-migrate -old -truth [-accounts ] [-dry-run]") + flag.PrintDefaults() + os.Exit(2) + } + if o.accountsDir != "" { + if fi, err := os.Stat(o.accountsDir); err != nil || !fi.IsDir() { + fmt.Fprintf(os.Stderr, "accounts dir %s is not a directory\n", o.accountsDir) + os.Exit(2) + } + } else { + fmt.Fprintln(os.Stderr, "warning: no -accounts: every UDL cell will be dropped") + } + o.log = os.Stderr + st, err := migrate(o) + if err != nil { + fmt.Fprintf(os.Stderr, "claims-migrate: %v\n", err) + os.Exit(1) + } + printSummary(os.Stdout, st, o.dryRun) + if !o.dryRun { + fmt.Printf("\ntruth database written: %s\n", o.truthPath) + } +} diff --git a/tools/claims-migrate/migrate.go b/tools/claims-migrate/migrate.go new file mode 100644 index 0000000..047ddd6 --- /dev/null +++ b/tools/claims-migrate/migrate.go @@ -0,0 +1,902 @@ +package main + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "time" + + _ "modernc.org/sqlite" + + "github.com/metazla/meta-core/internal/cid" + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/identity" + "github.com/metazla/meta-core/internal/storage" +) + +type options struct { + oldPath string + truthPath string + accountsDir string + dryRun bool + now time.Time // conversion timestamp; zero = time.Now() + workers int + log io.Writer +} + +// counts is a labelled tally, printed sorted. +type counts map[string]int + +func (c counts) total() int { + n := 0 + for _, v := range c { + n += v + } + return n +} + +type stats struct { + LegacyUID string + ConvTS uint64 + + Rids int // records migrated (have at least one cid) + RidsNoClaims int // migrated records whose only rows were cids / watcher facts + RecordsNoCID int // dropped: no cid at all + FieldClaims int + AliasClaims int + WatcherFacts counts // filePath / mtimeNano skipped + RefusedFields counts // by reason + Malformed int // file: keys without / + + CIDGroupRows int + CIDConflicts int // cids/ on a rid while cid: names another rid + OrphanCIDRows int // cid: → rid with no file: rows + + UDLMigrated counts + UDLDropped counts + UDLSigners int + GroupsMinted int // cid_group rows minted for UDL cids not in any record + + Scratch counts // service scratch keys dropped (btmagnet, searchcov, …) + Derived counts // derived bookkeeping skipped (idx, ev, dup, udl:idx, …) + + Principals int + ClaimsWritten int + Elapsed time.Duration +} + +func newStats() *stats { + return &stats{ + WatcherFacts: counts{}, RefusedFields: counts{}, + UDLMigrated: counts{}, UDLDropped: counts{}, + Scratch: counts{}, Derived: counts{}, + } +} + +// job is one claim to sign with id. +type job struct { + cl claim.Claim + id *identity.Identity +} + +const batchSize = 4096 + +func (o *options) logf(format string, args ...any) { + if o.log != nil { + fmt.Fprintf(o.log, format+"\n", args...) + } +} + +// migrate converts a meta-core v1.2.0 kv store into a v2.0.0 truth database. +func migrate(o options) (st *stats, err error) { + started := time.Now() + st = newStats() + if o.workers <= 0 { + o.workers = runtime.NumCPU() + } + if o.now.IsZero() { + o.now = time.Now() + } + st.ConvTS = uint64(o.now.UnixMilli()) + + old, err := openOld(o.oldPath) + if err != nil { + return nil, err + } + defer old.Close() + + legacy, err := identity.Generate() // in memory only, never saved + if err != nil { + return nil, err + } + st.LegacyUID = legacy.UID + o.logf("legacy signer uid: %s (private key kept in memory only)", legacy.UID) + + var out *sql.DB + if !o.dryRun { + if out, err = createTruth(o.truthPath); err != nil { + return nil, err + } + defer func() { + if out != nil { + out.Close() + } + if err != nil { + removeDB(o.truthPath) + } + }() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Pipeline: producer (this goroutine) → signers → one writer. + jobs := make(chan []job, o.workers*2) + signed := make(chan []claim.Claim, o.workers*2) + var firstErr error + var errMu sync.Mutex + fail := func(e error) { + errMu.Lock() + if firstErr == nil { + firstErr = e + } + errMu.Unlock() + cancel() + } + var wg sync.WaitGroup + for range o.workers { + wg.Add(1) + go func() { + defer wg.Done() + for batch := range jobs { + res := make([]claim.Claim, len(batch)) + for i := range batch { + cl := batch[i].cl + if e := cl.Sign(batch[i].id); e != nil { + fail(fmt.Errorf("sign %s/%s: %w", cl.CID, cl.Key, e)) + return + } + if e := cl.Verify(); e != nil { + fail(fmt.Errorf("verify %s/%s: %w", cl.CID, cl.Key, e)) + return + } + res[i] = cl + } + select { + case signed <- res: + case <-ctx.Done(): + return + } + } + }() + } + go func() { wg.Wait(); close(signed) }() + + w := &writer{db: out, pids: map[string]int64{}} + writerDone := make(chan error, 1) + go func() { + var e error + for batch := range signed { + if e == nil { + if e = w.write(batch); e != nil { + fail(e) + } + } + } + writerDone <- e + }() + + var pending []job + send := func(j job) error { + pending = append(pending, j) + if len(pending) < batchSize { + return nil + } + b := pending + pending = nil + select { + case jobs <- b: + return nil + case <-ctx.Done(): + return errors.New("aborted") + } + } + + groups, prodErr := produce(ctx, old, o, st, legacy, send) + if prodErr == nil && len(pending) > 0 { + select { + case jobs <- pending: + case <-ctx.Done(): + prodErr = errors.New("aborted") + } + } + close(jobs) + wErr := <-writerDone + errMu.Lock() + fe := firstErr + errMu.Unlock() + if fe != nil { + return nil, fe + } + if prodErr != nil { + return nil, prodErr + } + if wErr != nil { + return nil, wErr + } + + st.CIDGroupRows = len(groups) + st.Principals = len(w.pids) + st.ClaimsWritten = int(w.seq) + if !o.dryRun { + if err = w.finish(groups); err != nil { + return nil, err + } + if _, err = out.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + return nil, err + } + if err = out.Close(); err != nil { + out = nil + return nil, err + } + out = nil + if err = checkTruth(o.truthPath); err != nil { + return nil, fmt.Errorf("re-opening the result with meta-core's storage: %w", err) + } + } + st.Elapsed = time.Since(started) + return st, nil +} + +// produce reads the old store and emits every claim; it returns cid_group. +func produce(ctx context.Context, old *sql.DB, o options, st *stats, legacy *identity.Identity, send func(job) error) (map[string]string, error) { + // 1. cid index. + index := map[string]string{} + ridIdx := map[string][]string{} + rows, err := old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'cid:' AND k < 'cid;'`) + if err != nil { + return nil, err + } + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + rows.Close() + return nil, err + } + c := k[len("cid:"):] + if c == "" || v == "" { + continue + } + index[c] = v + ridIdx[v] = append(ridIdx[v], c) + } + if err := rows.Close(); err != nil { + return nil, err + } + o.logf("cid index: %d rows", len(index)) + + groups := map[string]string{} + + // 2. records, one rid at a time (the file: range is contiguous per rid). + rows, err = old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'file:' AND k < 'file;' ORDER BY k`) + if err != nil { + return nil, err + } + seen := map[string]bool{} + cur := "" + var fields [][2]string + flush := func() error { + if cur == "" { + return nil + } + if seen[cur] { + return fmt.Errorf("file: range for %s is not contiguous", cur) + } + seen[cur] = true + err := emitRecord(o, st, legacy, index, ridIdx, groups, cur, fields, send) + fields = fields[:0] + return err + } + n := 0 + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + rows.Close() + return nil, err + } + rid, field, ok := strings.Cut(k[len("file:"):], "/") + if !ok || rid == "" || field == "" { + st.Malformed++ + continue + } + if rid != cur { + if err := flush(); err != nil { + rows.Close() + return nil, err + } + cur = rid + } + fields = append(fields, [2]string{field, v}) + if n++; n%1_000_000 == 0 { + o.logf(" … %d file: rows, %d records", n, len(seen)) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + if err := flush(); err != nil { + return nil, err + } + for c := range index { + if _, ok := groups[c]; !ok { + st.OrphanCIDRows++ + } + } + o.logf("records: %d migrated, %d without a cid dropped", st.Rids, st.RecordsNoCID) + + // 3. UDL cells. + if err := emitUDL(ctx, old, o, st, groups, send); err != nil { + return nil, err + } + + // 4. everything else: count and drop. + rows, err = old.QueryContext(ctx, `SELECT k FROM kv + WHERE NOT (k >= 'file:' AND k < 'file;') AND NOT (k >= 'cid:' AND k < 'cid;') + AND NOT (k >= 'udl:rec:' AND k < 'udl:rec;')`) + if err != nil { + return nil, err + } + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, err + } + classifyOther(st, k) + } + if err := rows.Close(); err != nil { + return nil, err + } + return groups, nil +} + +func classifyOther(st *stats, k string) { + for _, p := range []string{"idx:", "ev:", "evhwm:", "dup:", "udl:idx:", "meta-core:", "meta:"} { + if strings.HasPrefix(k, p) { + st.Derived[p+"*"]++ + return + } + } + if p, _, ok := strings.Cut(k, ":"); ok && p != "" { + st.Scratch[p+":*"]++ + return + } + st.Scratch["(no prefix)"]++ +} + +// refusedReason mirrors storage.claimKeyAllowed + the write path's length +// checks: keys meta-core's POST /api/claims would refuse. +func refusedReason(key string) string { + switch { + case strings.HasPrefix(key, "cid_"): + return "cid_*" + case key == "midhash256", key == "canonical_cid", key == "__fields__": + return key + case strings.HasPrefix(key, "/"), strings.Contains(key, "\x00"): + return "malformed key" + case len(key) > 4096: + return "key too long" + } + return "" +} + +func emitRecord(o options, st *stats, legacy *identity.Identity, index map[string]string, + ridIdx map[string][]string, groups map[string]string, rid string, fields [][2]string, send func(job) error) error { + // Members: cids/* the cid index does not assign elsewhere, plus every cid + // the index assigns here. + member := map[string]bool{} + var members []string + for _, f := range fields { + c, ok := strings.CutPrefix(f[0], storage.CIDsKeyPrefix) + if !ok || c == "" { + continue + } + if r, ok := index[c]; ok && r != rid { + st.CIDConflicts++ + continue + } + if !member[c] { + member[c] = true + members = append(members, c) + } + } + for _, c := range ridIdx[rid] { + if !member[c] { + member[c] = true + members = append(members, c) + } + } + var usable []string + for _, c := range members { + if len(c) <= 512 { + usable = append(usable, c) + } + } + if len(usable) == 0 { + st.RecordsNoCID++ + return nil + } + canonical := usable[0] + for _, c := range usable[1:] { + canonical = cid.Better(canonical, c) + } + for _, c := range usable { + groups[c] = rid + } + st.Rids++ + emitted := 0 + mk := func(key, value string) error { + emitted++ + return send(job{id: legacy, cl: claim.Claim{ + CID: canonical, UID: legacy.UID, Key: key, Value: claim.Text(value), TS: st.ConvTS, + }}) + } + for _, f := range fields { + key, val := f[0], f[1] + if c, ok := strings.CutPrefix(key, storage.CIDsKeyPrefix); ok { + if c != canonical && member[c] && len(c) <= 512 && groups[c] == rid { + st.AliasClaims++ + if err := mk(key, "true"); err != nil { + return err + } + } + continue + } + if key == "filePath" || key == "mtimeNano" { + st.WatcherFacts[key]++ + continue + } + if r := refusedReason(key); r != "" { + st.RefusedFields[r]++ + continue + } + st.FieldClaims++ + if err := mk(key, val); err != nil { + return err + } + } + if emitted == 0 { + st.RidsNoClaims++ + } + return nil +} + +// publicUDLKeys are the plaintext tier (meta-watch record/tiers.rs); every +// other key is private and travels as an envelope. +var publicUDLKeys = map[string]bool{"like": true, "userRating": true, "profile:name": true, "profile:avatar": true} + +type udlCellJSON struct { + Record string `json:"record"` + Version int64 `json:"version"` + Ts int64 `json:"ts"` +} + +func decodeB64(s string) ([]byte, error) { + for _, enc := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding, base64.URLEncoding, base64.RawURLEncoding} { + if b, err := enc.DecodeString(s); err == nil { + return b, nil + } + } + return nil, errors.New("record is not base64") +} + +// udlTS converts an MMUR1 ts (seconds; tolerate ms) to MMUR2 milliseconds. +func udlTS(ts int64, fallback uint64) uint64 { + switch { + case ts <= 0: + return fallback + case ts < 100_000_000_000: // seconds (before year 5138) + return uint64(ts) * 1000 + default: + return uint64(ts) + } +} + +type account struct { + id *identity.Identity + aeadKey []byte +} + +func emitUDL(ctx context.Context, old *sql.DB, o options, st *stats, groups map[string]string, send func(job) error) error { + accounts := map[string]*account{} + loadAccount := func(uid string) (*account, error) { + if a, ok := accounts[uid]; ok { + return a, nil + } + var a *account + if o.accountsDir != "" && identity.ValidUID(uid) { + id, err := identity.LoadByUID(o.accountsDir, uid) + if err != nil { + return nil, fmt.Errorf("account %s: %w", uid, err) + } + if id != nil { + if id.UID != uid { + return nil, fmt.Errorf("account file %s.json holds uid %s", uid, id.UID) + } + k, err := identity.DeriveAEADKey(id) + if err != nil { + return nil, err + } + a = &account{id: id, aeadKey: k} + } + } + accounts[uid] = a + return a, nil + } + + type cellKey struct{ uid, cid, key string } + best := map[cellKey]job{} + var order []cellKey + drop := func(reason string) { st.UDLDropped[reason]++ } + kinds := map[cellKey]string{} + + rows, err := old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'udl:rec:' AND k < 'udl:rec;' ORDER BY k`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + return err + } + parts := strings.SplitN(k[len("udl:rec:"):], "/", 3) + if len(parts) != 3 { + drop("malformed cell key") + continue + } + var cell udlCellJSON + if err := json.Unmarshal([]byte(v), &cell); err != nil || cell.Record == "" { + drop("malformed cell JSON") + continue + } + raw, err := decodeB64(cell.Record) + if err != nil { + drop("record not base64") + continue + } + rec, err := decodeMMUR1(raw) + if err != nil { + drop("record not MMUR1 CBOR") + continue + } + if rec.UID != parts[0] || rec.CID != parts[1] || rec.Key != parts[2] { + drop("record does not match its cell") + continue + } + if !identity.ValidUID(rec.UID) || identity.Verify(rec.UID, rec.Signed, rec.Sig) != nil { + drop("bad MMUR1 signature") + continue + } + acct, err := loadAccount(rec.UID) + if err != nil { + return err + } + if acct == nil { + drop("no account on this box") + continue + } + key := rec.Key + if key == "rating" { + key = "userRating" // §14: a user's rating is userRating + } + if r := refusedReason(key); r != "" || rec.CID == "" || len(rec.CID) > 512 { + drop("key or cid refused by meta-core") + continue + } + ts := udlTS(rec.TS, st.ConvTS) + cl := claim.Claim{CID: rec.CID, UID: rec.UID, Key: key, TS: ts} + kind := "" + switch { + case rec.Value.kind == cNull: + kind = "tombstone" + case publicUDLKeys[key]: + t, ok := rec.Value.wireText() + if !ok { + drop("public key holds bytes") + continue + } + cl.Value, kind = claim.Text(t), "public" + default: + var plain []byte + if rec.Value.kind == cBytes { + plain, err = openEnvelope(acct.aeadKey, rec.Value.bytes, envelopeAAD(rec.CID, rec.UID, rec.Key, rec.Version)) + if err != nil { + drop("private envelope does not decrypt") + continue + } + kind = "private (re-encrypted)" + } else { + // A private-tier key stored in plaintext: encrypt it now, its + // plaintext being the CBOR of the typed value, as crypto.rs. + plain = append([]byte(nil), rec.Value.raw...) + kind = "private (plaintext, now encrypted)" + } + nonce := make([]byte, 24) + if _, err := rand.Read(nonce); err != nil { + return err + } + env, err := sealEnvelope(acct.aeadKey, plain, envelopeAAD(rec.CID, rec.UID, key, ts), nonce) + if err != nil { + return err + } + cl.Value = claim.Bytes(env) + } + ck := cellKey{rec.UID, rec.CID, key} + if prev, ok := best[ck]; ok { + drop("superseded (rating + userRating)") + if prev.cl.TS >= ts { + continue + } + } else { + order = append(order, ck) + } + best[ck], kinds[ck] = job{cl: cl, id: acct.id}, kind + } + if err := rows.Err(); err != nil { + return err + } + signers := map[string]bool{} + for _, ck := range order { + j := best[ck] + if _, ok := groups[j.cl.CID]; !ok { + rid, err := storage.NewUUID() + if err != nil { + return err + } + groups[j.cl.CID] = rid + st.GroupsMinted++ + } + st.UDLMigrated[kinds[ck]]++ + signers[j.cl.UID] = true + if err := send(j); err != nil { + return err + } + } + st.UDLSigners = len(signers) + return nil +} + +// writer stores signed claims in arrival order (seq = arrival order). +type writer struct { + db *sql.DB + pids map[string]int64 + seq int64 +} + +func (w *writer) write(batch []claim.Claim) error { + if w.db == nil { // dry run + for _, cl := range batch { + if _, ok := w.pids[cl.UID]; !ok { + w.pids[cl.UID] = int64(len(w.pids) + 1) + } + } + w.seq += int64(len(batch)) + return nil + } + tx, err := w.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + ins, err := tx.Prepare(`INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + defer ins.Close() + for _, cl := range batch { + pid, ok := w.pids[cl.UID] + if !ok { + if err := tx.QueryRow(`INSERT INTO principal(uid) VALUES(?) RETURNING pid`, cl.UID).Scan(&pid); err != nil { + return fmt.Errorf("principal %s: %w", cl.UID, err) + } + w.pids[cl.UID] = pid + } + w.seq++ + if _, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq); err != nil { + return fmt.Errorf("claim (%s, %s, %s): %w", cl.CID, cl.Key, cl.UID, err) + } + } + return tx.Commit() +} + +// finish writes cid_group and last_seq. +func (w *writer) finish(groups map[string]string) error { + tx, err := w.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + ins, err := tx.Prepare(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`) + if err != nil { + return err + } + defer ins.Close() + keys := make([]string, 0, len(groups)) + for c := range groups { + keys = append(keys, c) + } + sort.Strings(keys) + for _, c := range keys { + if _, err := ins.Exec(c, groups[c]); err != nil { + return fmt.Errorf("cid_group %s: %w", c, err) + } + } + if _, err := tx.Exec(`INSERT INTO meta(name, value) VALUES('last_seq', ?) + ON CONFLICT(name) DO UPDATE SET value = excluded.value`, fmt.Sprint(w.seq)); err != nil { + return err + } + return tx.Commit() +} + +// valueArg is storage's column encoding: TEXT, BLOB or NULL (STRICT ANY keeps it). +func valueArg(v claim.Value) any { + switch v.Kind { + case claim.KindText: + return v.Text + case claim.KindBytes: + if v.Bytes == nil { + return []byte{} + } + return v.Bytes + } + return nil +} + +func openOld(path string) (*sql.DB, error) { + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("old store: %w", err) + } + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=query_only(1)&_pragma=busy_timeout(10000)") + if err != nil { + return nil, err + } + var name string + if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'kv'`).Scan(&name); err != nil { + db.Close() + return nil, fmt.Errorf("%s has no kv table — not a meta-core v1.x store: %w", path, err) + } + var ver string + err = db.QueryRow(`SELECT v FROM kv WHERE k = 'meta-core:schema-version'`).Scan(&ver) + switch { + case errors.Is(err, sql.ErrNoRows): + // pre-sentinel store: accepted, the key layout is the same + case err != nil: + db.Close() + return nil, err + case ver != "2": + db.Close() + return nil, fmt.Errorf("%s has schema version %q, this tool converts version \"2\"", path, ver) + } + return db, nil +} + +// createTruth creates the truth file with meta-core's own schema code +// (storage.Client.Connect + EnsureSchemaVersion stamp schema-version 3), then +// reopens it for the bulk load. +func createTruth(path string) (*sql.DB, error) { + for _, p := range []string{path, path + "-wal", path + "-shm"} { + if _, err := os.Stat(p); err == nil { + return nil, fmt.Errorf("%s already exists; refusing to overwrite", p) + } + } + if fi, err := os.Stat(filepath.Dir(path)); err != nil || !fi.IsDir() { + return nil, fmt.Errorf("output directory %s does not exist", filepath.Dir(path)) + } + tmp, err := os.MkdirTemp("", "claims-migrate-cache-") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmp) + c := storage.NewClient() + if err := c.Connect(path, filepath.Join(tmp, "cache.db")); err != nil { + removeDB(path) + return nil, err + } + if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil { + c.Close() + removeDB(path) + return nil, err + } + if err := c.Close(); err != nil { + removeDB(path) + return nil, err + } + db, err := sql.Open("sqlite", "file:"+path+"?_pragma=journal_mode(WAL)&_pragma=synchronous(OFF)&_pragma=busy_timeout(30000)&_pragma=cache_size(-262144)") + if err != nil { + removeDB(path) + return nil, err + } + db.SetMaxOpenConns(1) + return db, nil +} + +// checkTruth reopens the result the way meta-core boots it (schema gate). +func checkTruth(path string) error { + tmp, err := os.MkdirTemp("", "claims-migrate-check-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + c := storage.NewClient() + if err := c.Connect(path, filepath.Join(tmp, "cache.db")); err != nil { + return err + } + defer c.Close() + return c.EnsureSchemaVersion(storage.SchemaVersion) +} + +func removeDB(path string) { + for _, p := range []string{path, path + "-wal", path + "-shm"} { + _ = os.Remove(p) + } +} + +// printSummary writes the human report. +func printSummary(w io.Writer, st *stats, dryRun bool) { + mode := "written" + if dryRun { + mode = "would be written (dry run: signed + verified, nothing stored)" + } + fmt.Fprintf(w, "\n=== claims-migrate summary ===\n") + fmt.Fprintf(w, "legacy signer uid %s\n", st.LegacyUID) + fmt.Fprintf(w, "conversion ts (ms) %d\n", st.ConvTS) + fmt.Fprintf(w, "records (rids) migrated %d (%d with no claims: only cids / watcher facts)\n", st.Rids, st.RidsNoClaims) + fmt.Fprintf(w, "records dropped (no cid) %d\n", st.RecordsNoCID) + fmt.Fprintf(w, "claims %-17s %d\n", "total", st.ClaimsWritten) + fmt.Fprintf(w, " legacy field claims %d\n", st.FieldClaims) + fmt.Fprintf(w, " legacy alias claims %d (cids/ = \"true\")\n", st.AliasClaims) + fmt.Fprintf(w, " UDL claims %d (%d signers)\n", st.UDLMigrated.total(), st.UDLSigners) + printCounts(w, " ", st.UDLMigrated) + fmt.Fprintf(w, " -> %s\n", mode) + fmt.Fprintf(w, "principals %d\n", st.Principals) + fmt.Fprintf(w, "cid_group rows %d (%d minted for UDL-only cids)\n", st.CIDGroupRows, st.GroupsMinted) + fmt.Fprintf(w, "cid conflicts %d (cids/ on a rid while cid: names another; index wins)\n", st.CIDConflicts) + fmt.Fprintf(w, "orphan cid: rows dropped %d (rid has no file: rows)\n", st.OrphanCIDRows) + fmt.Fprintf(w, "watcher facts skipped %d\n", st.WatcherFacts.total()) + printCounts(w, " ", st.WatcherFacts) + fmt.Fprintf(w, "fields refused %d\n", st.RefusedFields.total()) + printCounts(w, " ", st.RefusedFields) + fmt.Fprintf(w, "malformed file: keys %d\n", st.Malformed) + fmt.Fprintf(w, "UDL cells dropped %d\n", st.UDLDropped.total()) + printCounts(w, " ", st.UDLDropped) + fmt.Fprintf(w, "scratch keys dropped %d (refetched / rebuilt in the cache)\n", st.Scratch.total()) + printCounts(w, " ", st.Scratch) + fmt.Fprintf(w, "derived keys skipped %d\n", st.Derived.total()) + printCounts(w, " ", st.Derived) + fmt.Fprintf(w, "elapsed %s\n", st.Elapsed.Round(time.Millisecond)) +} + +func printCounts(w io.Writer, indent string, c counts) { + keys := make([]string, 0, len(c)) + for k := range c { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(w, "%s%-40s %d\n", indent, k, c[k]) + } +} diff --git a/tools/claims-migrate/migrate_test.go b/tools/claims-migrate/migrate_test.go new file mode 100644 index 0000000..62d503c --- /dev/null +++ b/tools/claims-migrate/migrate_test.go @@ -0,0 +1,375 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/metazla/meta-core/internal/claim" + "github.com/metazla/meta-core/internal/identity" + "github.com/metazla/meta-core/internal/storage" +) + +const ( + cidIPFS = "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 40 + cidSHA = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 30 + cidB = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua" + cidOrph = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq" + ridA = "01JKR8XW5T4QMV7AHNJ2DEFGHK" + ridB = "01JKR8XW5T4QMV7AHNJ2DEFGHM" + ridC = "01JKR8XW5T4QMV7AHNJ2DEFGHN" + ridD = "01JKR8XW5T4QMV7AHNJ2DEFGHP" + oldTSsec = 1_700_000_000 +) + +// --- minimal canonical CBOR encoder for building MMUR1 fixtures --- + +func head(major byte, n uint64) []byte { + m := major << 5 + switch { + case n < 24: + return []byte{m | byte(n)} + case n <= 0xff: + return []byte{m | 24, byte(n)} + case n <= 0xffff: + return []byte{m | 25, byte(n >> 8), byte(n)} + case n <= 0xffffffff: + return binary.BigEndian.AppendUint32([]byte{m | 26}, uint32(n)) + } + return binary.BigEndian.AppendUint64([]byte{m | 27}, n) +} + +func encText(s string) []byte { return append(head(3, uint64(len(s))), s...) } +func encBytes(b []byte) []byte { return append(head(2, uint64(len(b))), b...) } +func encUint(n uint64) []byte { return head(0, n) } +func encBool(b bool) []byte { + if b { + return []byte{0xf5} + } + return []byte{0xf4} +} +func encF64(f float64) []byte { + return binary.BigEndian.AppendUint64([]byte{0xfb}, math.Float64bits(f)) +} + +var encNull = []byte{0xf6} + +// mmur1 builds a signed MMUR1 record; badSig signs other bytes. +func mmur1(t *testing.T, id *identity.Identity, cid, key string, value []byte, version uint64, badSig bool) []byte { + t.Helper() + var elems []byte + elems = append(elems, encText(cid)...) + elems = append(elems, encText(id.UID)...) + elems = append(elems, encText(key)...) + elems = append(elems, value...) + elems = append(elems, encUint(version)...) + elems = append(elems, encUint(oldTSsec)...) + signed := append(append([]byte(mmur1Domain), 0x86), elems...) + if badSig { + signed = append(signed, 'x') + } + sig, err := identity.Sign(id, signed) + if err != nil { + t.Fatal(err) + } + return append(append([]byte{0x87}, elems...), encBytes(sig)...) +} + +func cell(t *testing.T, rec []byte, version uint64) string { + b, err := json.Marshal(map[string]any{ + "record": base64.StdEncoding.EncodeToString(rec), "version": version, "ts": oldTSsec, + }) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func importID(t *testing.T, n byte) *identity.Identity { + t.Helper() + id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 7)) + if err != nil { + t.Fatal(err) + } + return id +} + +type fixture struct { + dir, oldPath, accounts string + user, stranger *identity.Identity + seekPlain, durPlain []byte +} + +func buildV1(t *testing.T) fixture { + t.Helper() + dir := t.TempDir() + f := fixture{dir: dir, oldPath: filepath.Join(dir, "v1.db"), accounts: filepath.Join(dir, "accounts")} + f.user, f.stranger = importID(t, 0x11), importID(t, 0x22) + if err := os.MkdirAll(f.accounts, 0o700); err != nil { + t.Fatal(err) + } + if err := identity.SaveAccount(f.accounts, f.user); err != nil { + t.Fatal(err) + } + aead, err := identity.DeriveAEADKey(f.user) + if err != nil { + t.Fatal(err) + } + f.seekPlain = encF64(123.5) + f.durPlain = encF64(5400) + env, err := sealEnvelope(aead, f.seekPlain, envelopeAAD(cidIPFS, f.user.UID, "seek", 3), bytes.Repeat([]byte{9}, 24)) + if err != nil { + t.Fatal(err) + } + + db, err := sql.Open("sqlite", "file:"+f.oldPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { + t.Fatal(err) + } + u := f.user.UID + rows := map[string]string{ + "meta-core:schema-version": "2", + // record A: two cids, canonical = the dag-pb one + "file:" + ridA + "/cids/" + cidIPFS: "true", + "file:" + ridA + "/cids/" + cidSHA: "true", + "file:" + ridA + "/title": "Inception", + "file:" + ridA + "/year": "007", + "file:" + ridA + "/sizeByte": "15032385536", + "file:" + ridA + "/genres/Drama": "true", + "file:" + ridA + "/filePath": "/files/watch/Inception.mkv", + "file:" + ridA + "/mtimeNano": "1729872000000000000", + "file:" + ridA + "/cid_sha256": cidSHA, + "file:" + ridA + "/fileinfo/duration": "8878.5", + "cid:" + cidIPFS: ridA, + "cid:" + cidSHA: ridA, + // record B: its cid only in the index; claims cidSHA too (conflict) + "file:" + ridB + "/title": "Other", + "file:" + ridB + "/cids/" + cidSHA: "true", + "cid:" + cidB: ridB, + // record C: no cid at all → dropped + "file:" + ridC + "/title": "Nobody", + // orphan index row + "cid:" + cidOrph: ridD, + // scratch + derived + "btmagnet:abc": "magnet:?xt=urn:btih:abc", + "searchcov:q": "1", + "nzb:x": "y", + "idx:roots/" + ridA: "", + "dup:" + ridA + "/p": "", + "udl:idx:user:" + u + "\x1f": "", + // UDL + "udl:rec:" + u + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.user, cidIPFS, "like", encBool(true), 1, false), 1), + "udl:rec:" + u + "/" + cidIPFS + "/rating": cell(t, mmur1(t, f.user, cidIPFS, "rating", encUint(4), 1, false), 1), + "udl:rec:" + u + "/" + cidIPFS + "/seek": cell(t, mmur1(t, f.user, cidIPFS, "seek", encBytes(env), 3, false), 3), + "udl:rec:" + u + "/" + cidIPFS + "/watched": cell(t, mmur1(t, f.user, cidIPFS, "watched", encNull, 2, false), 2), + "udl:rec:" + u + "/" + cidIPFS + "/duration": cell(t, mmur1(t, f.user, cidIPFS, "duration", f.durPlain, 1, false), 1), + "udl:rec:" + u + "/self/profile:name": cell(t, mmur1(t, f.user, "self", "profile:name", encText("Alice"), 1, false), 1), + "udl:rec:" + u + "/" + cidSHA + "/like": cell(t, mmur1(t, f.user, cidSHA, "like", encBool(true), 1, true), 1), + "udl:rec:" + f.stranger.UID + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.stranger, cidIPFS, "like", encBool(false), 1, false), 1), + } + for k, v := range rows { + if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil { + t.Fatal(err) + } + } + return f +} + +func TestMigrate(t *testing.T) { + f := buildV1(t) + truthPath := filepath.Join(f.dir, "truth.db") + now := time.UnixMilli(1_790_000_000_000) + st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, now: now, workers: 3}) + if err != nil { + t.Fatalf("migrate: %v", err) + } + var sum bytes.Buffer + printSummary(&sum, st, false) + t.Log(sum.String()) + + check := func(name string, got, want int) { + t.Helper() + if got != want { + t.Errorf("%s = %d, want %d", name, got, want) + } + } + check("rids", st.Rids, 2) + check("records without cid", st.RecordsNoCID, 1) + check("orphan cid rows", st.OrphanCIDRows, 1) + check("cid conflicts", st.CIDConflicts, 1) + check("alias claims", st.AliasClaims, 1) + check("field claims", st.FieldClaims, 6) // A: title year sizeByte genres/Drama fileinfo/duration; B: title + check("refused cid_*", st.RefusedFields["cid_*"], 1) + check("watcher facts", st.WatcherFacts.total(), 2) + check("udl migrated", st.UDLMigrated.total(), 6) + check("udl no account", st.UDLDropped["no account on this box"], 1) + check("udl bad sig", st.UDLDropped["bad MMUR1 signature"], 1) + check("groups minted", st.GroupsMinted, 1) // "self" + check("cid_group rows", st.CIDGroupRows, 4) + check("scratch", st.Scratch.total(), 3) + check("principals", st.Principals, 2) + check("claims", st.ClaimsWritten, st.FieldClaims+st.AliasClaims+st.UDLMigrated.total()) + + // Reopen with meta-core's storage package: boot catch-up rebuilds the cache. + c := storage.NewClient() + if err := c.Connect(truthPath, filepath.Join(f.dir, "cache.db")); err != nil { + t.Fatal(err) + } + defer c.Close() + if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil { + t.Fatal(err) + } + if err := c.Start(); err != nil { + t.Fatal(err) + } + + for cid, want := range map[string]string{cidIPFS: ridA, cidSHA: ridA, cidB: ridB, cidOrph: ""} { + got, err := c.GetByCID(cid) + if err != nil || got != want { + t.Errorf("GetByCID(%s) = %q, %v; want %q", cid, got, err, want) + } + } + a, err := c.GetMetadataFlat(ridA) + if err != nil { + t.Fatal(err) + } + for k, want := range map[string]string{ + "title": "Inception", "year": "007", "sizeByte": "15032385536", "genres/Drama": "true", + "fileinfo/duration": "8878.5", "cids/" + cidIPFS: "true", "cids/" + cidSHA: "true", + } { + if a[k] != want { + t.Errorf("record A %s = %q, want %q", k, a[k], want) + } + } + for _, k := range []string{"filePath", "mtimeNano", "cid_sha256"} { + if _, ok := a[k]; ok { + t.Errorf("record A carries %s", k) + } + } + b, err := c.GetMetadataFlat(ridB) + if err != nil || b["title"] != "Other" { + t.Errorf("record B = %v, %v", b, err) + } + ids, err := c.GetAllHashIDs() + if err != nil { + t.Fatal(err) + } + if !slices.Contains(ids, ridA) || !slices.Contains(ids, ridB) || slices.Contains(ids, ridC) || slices.Contains(ids, ridD) { + t.Errorf("roots = %v", ids) + } + if len(ids) != 3 { + t.Errorf("want 3 roots (A, B, self), got %v", ids) + } + + // Legacy claims sit on the canonical cid, signed by the printed uid. + cls, err := c.ClaimsForCIDs([]string{cidIPFS}, "title") + if err != nil || len(cls) != 1 || cls[0].UID != st.LegacyUID || cls[0].TS != uint64(now.UnixMilli()) { + t.Fatalf("title claims = %+v, %v", cls, err) + } + if err := cls[0].Verify(); err != nil { + t.Error(err) + } + alias, err := c.ClaimsForCIDs([]string{cidIPFS}, "cids/"+cidSHA) + if err != nil || len(alias) != 1 || alias[0].Value.Text != "true" { + t.Errorf("alias claims = %+v, %v", alias, err) + } + + // UDL claims. + u := f.user.UID + wantTS := uint64(oldTSsec) * 1000 + get := func(cid, key string) claim.Claim { + t.Helper() + cl, ok, err := c.UDLGetClaim(u, cid, key) + if err != nil || !ok { + t.Fatalf("UDL %s/%s: ok=%v err=%v", cid, key, ok, err) + } + if cl.TS != wantTS { + t.Errorf("UDL %s ts = %d, want %d", key, cl.TS, wantTS) + } + if err := cl.Verify(); err != nil { + t.Errorf("UDL %s: %v", key, err) + } + return cl + } + if v := get(cidIPFS, "like").Value; v.Kind != claim.KindText || v.Text != "true" { + t.Errorf("like = %+v", v) + } + if v := get(cidIPFS, "userRating").Value; v.Kind != claim.KindText || v.Text != "4" { + t.Errorf("userRating = %+v", v) + } + if _, ok, _ := c.UDLGetClaim(u, cidIPFS, "rating"); ok { + t.Error("rating was not renamed") + } + if v := get(cidIPFS, "watched").Value; !v.IsNull() { + t.Errorf("watched tombstone = %+v", v) + } + if v := get("self", "profile:name").Value; v.Text != "Alice" { + t.Errorf("profile:name = %+v", v) + } + aead, _ := identity.DeriveAEADKey(f.user) + for key, plain := range map[string][]byte{"seek": f.seekPlain, "duration": f.durPlain} { + cl := get(cidIPFS, key) + if cl.Value.Kind != claim.KindBytes { + t.Fatalf("%s is not an envelope: %+v", key, cl.Value) + } + pt, err := openEnvelope(aead, cl.Value.Bytes, envelopeAAD(cidIPFS, u, key, cl.TS)) + if err != nil || !bytes.Equal(pt, plain) { + t.Errorf("%s decrypts to %x, %v; want %x", key, pt, err, plain) + } + } + if _, ok, _ := c.UDLGetClaim(f.stranger.UID, cidIPFS, "like"); ok { + t.Error("stranger's cell migrated without an account") + } + + // A second run refuses to overwrite. + if _, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts}); err == nil { + t.Error("second run overwrote the truth file") + } +} + +func TestDryRunWritesNothing(t *testing.T) { + f := buildV1(t) + truthPath := filepath.Join(f.dir, "truth.db") + st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, dryRun: true}) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(truthPath); !os.IsNotExist(err) { + t.Errorf("dry run created %s", truthPath) + } + if st.ClaimsWritten != st.FieldClaims+st.AliasClaims+st.UDLMigrated.total() || st.ClaimsWritten == 0 { + t.Errorf("dry-run counts: %+v", st) + } +} + +func TestNoAccountsDropsUDL(t *testing.T) { + f := buildV1(t) + st, err := migrate(options{oldPath: f.oldPath, dryRun: true}) + if err != nil { + t.Fatal(err) + } + if st.UDLMigrated.total() != 0 || st.UDLDropped["no account on this box"] != 7 { + t.Errorf("udl: migrated %v dropped %v", st.UDLMigrated, st.UDLDropped) + } +} + +func TestHalfFloat(t *testing.T) { + for h, want := range map[uint16]float64{0x3c00: 1, 0xc000: -2, 0x3e00: 1.5, 0x0000: 0, 0x7bff: 65504} { + if got := halfToFloat(h); got != want { + t.Errorf("half %04x = %v, want %v", h, got, want) + } + } +} diff --git a/tools/claims-migrate/mmur1.go b/tools/claims-migrate/mmur1.go new file mode 100644 index 0000000..172861d --- /dev/null +++ b/tools/claims-migrate/mmur1.go @@ -0,0 +1,267 @@ +package main + +// Decoding of the retired MMUR1 user-data record (meta-watch record/canonical.rs +// at meta-core v1.2.0) and the private-tier envelope (record/crypto.rs). +// +// record = canonical CBOR array(7) [cid, uid, key, value, version, ts, sig] +// bytes_to_sign = "MMUR1\n" || array(6) [cid, uid, key, value, version, ts] +// sig = DER ECDSA-secp256k1 over SHA-256(bytes_to_sign) +// +// value is CBOR null / bool / int / float / text / bytes (bytes = a private-tier +// XChaCha20-Poly1305 envelope whose plaintext is the CBOR of the typed value). +// The signed bytes are rebuilt from the record's own raw element bytes (array +// header 0x87 → 0x86, sig element cut off), so the old signature is checked +// against exactly what was signed, whatever float width the encoder chose. + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "strconv" + + "golang.org/x/crypto/chacha20poly1305" +) + +const mmur1Domain = "MMUR1\n" + +type cborKind uint8 + +const ( + cNull cborKind = iota + cBool + cInt + cFloat + cText + cBytes +) + +// cborItem is one decoded scalar plus its raw encoding. +type cborItem struct { + kind cborKind + b bool + i int64 + f float64 + s string + bytes []byte + raw []byte +} + +// wireText is the MMUR2 text form of a typed value — meta-watch's +// Value::to_wire_text: bools "true"/"false", ints in decimal, floats as Rust's +// f64 Display (shortest round-trip, never an exponent), text verbatim. +func (it cborItem) wireText() (string, bool) { + switch it.kind { + case cBool: + if it.b { + return "true", true + } + return "false", true + case cInt: + return strconv.FormatInt(it.i, 10), true + case cFloat: + return strconv.FormatFloat(it.f, 'f', -1, 64), true + case cText: + return it.s, true + } + return "", false +} + +type mmur1Record struct { + CID, UID, Key string + Value cborItem + Version uint64 + TS int64 + Sig []byte + Signed []byte // the exact MMUR1 bytes the signature covers +} + +var errCBOR = errors.New("malformed MMUR1 CBOR") + +// readHead parses a CBOR initial byte + argument (definite lengths only). +func readHead(b []byte, off int) (major byte, ai byte, n uint64, next int, err error) { + if off >= len(b) { + return 0, 0, 0, 0, errCBOR + } + ib := b[off] + major, ai = ib>>5, ib&31 + off++ + var size int + switch { + case ai < 24: + return major, ai, uint64(ai), off, nil + case ai == 24: + size = 1 + case ai == 25: + size = 2 + case ai == 26: + size = 4 + case ai == 27: + size = 8 + default: + return 0, 0, 0, 0, fmt.Errorf("%w: indefinite or reserved length", errCBOR) + } + if off+size > len(b) { + return 0, 0, 0, 0, errCBOR + } + for _, x := range b[off : off+size] { + n = n<<8 | uint64(x) + } + return major, ai, n, off + size, nil +} + +// readItem decodes one scalar item at off. +func readItem(b []byte, off int) (cborItem, int, error) { + start := off + major, ai, n, off, err := readHead(b, off) + if err != nil { + return cborItem{}, 0, err + } + var it cborItem + switch major { + case 0: + if n > math.MaxInt64 { + return it, 0, fmt.Errorf("%w: int out of range", errCBOR) + } + it = cborItem{kind: cInt, i: int64(n)} + case 1: + if n > math.MaxInt64 { + return it, 0, fmt.Errorf("%w: int out of range", errCBOR) + } + it = cborItem{kind: cInt, i: -1 - int64(n)} + case 2, 3: + if n > uint64(len(b)-off) { + return it, 0, errCBOR + } + end := off + int(n) + if major == 2 { + it = cborItem{kind: cBytes, bytes: append([]byte(nil), b[off:end]...)} + } else { + it = cborItem{kind: cText, s: string(b[off:end])} + } + off = end + case 7: + switch ai { + case 20, 21: + it = cborItem{kind: cBool, b: ai == 21} + case 22: + it = cborItem{kind: cNull} + case 25: + it = cborItem{kind: cFloat, f: halfToFloat(uint16(n))} + case 26: + it = cborItem{kind: cFloat, f: float64(math.Float32frombits(uint32(n)))} + case 27: + it = cborItem{kind: cFloat, f: math.Float64frombits(n)} + default: + return it, 0, fmt.Errorf("%w: unsupported simple value %d", errCBOR, ai) + } + default: + return it, 0, fmt.Errorf("%w: unsupported major type %d", errCBOR, major) + } + it.raw = b[start:off] + return it, off, nil +} + +func halfToFloat(h uint16) float64 { + sign := 1.0 + if h&0x8000 != 0 { + sign = -1 + } + exp := int(h>>10) & 0x1f + frac := float64(h & 0x3ff) + switch exp { + case 0: + return sign * math.Ldexp(frac, -24) + case 31: + if frac == 0 { + return math.Inf(int(sign)) + } + return math.NaN() + } + return sign * math.Ldexp(frac+1024, exp-25) +} + +// decodeMMUR1 parses a stored record and rebuilds its signed bytes. +func decodeMMUR1(b []byte) (*mmur1Record, error) { + if len(b) == 0 || b[0] != 0x87 { + return nil, fmt.Errorf("%w: want array(7)", errCBOR) + } + off := 1 + items := make([]cborItem, 7) + sigStart := 0 + for i := range items { + if i == 6 { + sigStart = off + } + it, next, err := readItem(b, off) + if err != nil { + return nil, err + } + items[i], off = it, next + } + if off != len(b) { + return nil, fmt.Errorf("%w: trailing bytes", errCBOR) + } + for i, want := range []cborKind{cText, cText, cText} { + if items[i].kind != want { + return nil, fmt.Errorf("%w: cid/uid/key must be text", errCBOR) + } + } + if items[4].kind != cInt || items[4].i < 0 { + return nil, fmt.Errorf("%w: version must be an unsigned int", errCBOR) + } + if items[5].kind != cInt { + return nil, fmt.Errorf("%w: ts must be an int", errCBOR) + } + if items[6].kind != cBytes { + return nil, fmt.Errorf("%w: sig must be bytes", errCBOR) + } + signed := make([]byte, 0, len(mmur1Domain)+sigStart) + signed = append(signed, mmur1Domain...) + signed = append(signed, 0x86) + signed = append(signed, b[1:sigStart]...) + return &mmur1Record{ + CID: items[0].s, UID: items[1].s, Key: items[2].s, + Value: items[3], + Version: uint64(items[4].i), + TS: items[5].i, + Sig: items[6].bytes, + Signed: signed, + }, nil +} + +// envelopeAAD is cid ‖ 0x1F ‖ uid ‖ 0x1F ‖ key ‖ 0x1F ‖ n_be(8) — n was the +// MMUR1 version and is the MMUR2 ts. +func envelopeAAD(cid, uid, key string, n uint64) []byte { + b := make([]byte, 0, len(cid)+len(uid)+len(key)+11) + b = append(b, cid...) + b = append(b, 0x1f) + b = append(b, uid...) + b = append(b, 0x1f) + b = append(b, key...) + b = append(b, 0x1f) + return binary.BigEndian.AppendUint64(b, n) +} + +// openEnvelope decrypts nonce(24) ‖ ciphertext+tag. +func openEnvelope(aeadKey, env, aad []byte) ([]byte, error) { + a, err := chacha20poly1305.NewX(aeadKey) + if err != nil { + return nil, err + } + if len(env) < chacha20poly1305.NonceSizeX+a.Overhead() { + return nil, errors.New("envelope too short") + } + return a.Open(nil, env[:chacha20poly1305.NonceSizeX], env[chacha20poly1305.NonceSizeX:], aad) +} + +// sealEnvelope encrypts plain under a fresh random nonce. +func sealEnvelope(aeadKey, plain, aad []byte, nonce []byte) ([]byte, error) { + a, err := chacha20poly1305.NewX(aeadKey) + if err != nil { + return nil, err + } + out := make([]byte, 0, len(nonce)+len(plain)+a.Overhead()) + out = append(out, nonce...) + return a.Seal(out, nonce, plain, aad), nil +} diff --git a/tools/search-parity/README.md b/tools/search-parity/README.md index 700023d..18e0c86 100644 --- a/tools/search-parity/README.md +++ b/tools/search-parity/README.md @@ -1,21 +1,23 @@ # search-parity Regression check for `POST /api/metadata/search` against a running meta-core. +Since v2.0.0 it reads the merged view from the cache database +(`/meta-core/db-cache/cache.db`), which keeps the v1 `file:/` layout. -- `parity.py ` builds ~230 queries from real records (free text, +- `parity.py ` builds ~230 queries from real records (free text, short tokens, key-set and flat filters, exact/contains, multi-filter). It computes each query's expected result by brute force: one streaming pass over every record, using a port of the Go matchers. Then it diffs that against the live API. Any `lost` result means the SQL candidate superset dropped a real match. -- `timing.py [meta-core.db]` times the same query set at limit 100 and 2000. +- `timing.py [cache.db]` times the same query set at limit 100 and 2000. Run them inside the container. It has python3 and reads the database read-only: ```bash docker cp tools/search-parity metacore-app:/tmp/sp -docker exec metacore-app python3 -u /tmp/sp/parity.py /meta-core/db/meta-core.db +docker exec metacore-app python3 -u /tmp/sp/parity.py /meta-core/db-cache/cache.db docker exec metacore-app python3 -u /tmp/sp/timing.py ``` diff --git a/tools/search-parity/parity.py b/tools/search-parity/parity.py index d57d97c..628424f 100644 --- a/tools/search-parity/parity.py +++ b/tools/search-parity/parity.py @@ -1,6 +1,10 @@ -"""Search parity check for the meta-core Redis -> SQLite cut-over. +"""Search parity check for meta-core search. - parity.py + parity.py + +Since v2.0.0 the searchable records are the MERGED view, kept in the cache +database's kv table under the v1 layout (file:/, idx:roots/): +point this at /meta-core/db-cache/cache.db, not the truth database. Builds a query set from real records, computes the EXPECTED result of each query by brute force — one streaming pass over every record in the database, diff --git a/tools/search-parity/timing.py b/tools/search-parity/timing.py index 2250e49..42fda2d 100644 --- a/tools/search-parity/timing.py +++ b/tools/search-parity/timing.py @@ -1,6 +1,6 @@ import importlib.util, json, sqlite3, sys, time spec = importlib.util.spec_from_file_location("parity", "/tmp/parity.py") -sys.argv = ["parity", "/meta-core/db/meta-core.db"] +sys.argv = ["parity", sys.argv[1] if len(sys.argv) > 1 else "/meta-core/db-cache/cache.db"] src = open("/tmp/parity.py").read().replace("\nmain()\n", "\n") ns = {} exec(compile(src, "parity", "exec"), ns)