From 5efdd2c07d1db20b37dd44c4dc774d27ee6a2f7b Mon Sep 17 00:00:00 2001 From: Worph Date: Tue, 25 Aug 2026 17:48:23 +0200 Subject: [PATCH] improved identity management --- dashboard/dist/index.html | 2 +- dashboard/package-lock.json | 10 + dashboard/package.json | 5 +- dashboard/src/lib/sign.ts | 127 +++++++++++ dashboard/src/pages/Identity.tsx | 268 +++++++++++++++++++---- internal/api/handlers_identity.go | 247 +++++++++++++++++---- internal/api/identity_gate_test.go | 257 ++++++++++++++++++++++ internal/api/server.go | 11 +- internal/api/udl.go | 33 +++ internal/identity/challenge.go | 126 +++++++++++ internal/identity/noble_interop_test.go | 43 ++++ internal/identity/verify.go | 160 ++++++++++++++ internal/identity/verify_test.go | 278 ++++++++++++++++++++++++ internal/storage/udl_admin.go | 245 +++++++++++++++++++++ internal/storage/udl_admin_test.go | 253 +++++++++++++++++++++ 15 files changed, 1976 insertions(+), 89 deletions(-) create mode 100644 dashboard/src/lib/sign.ts create mode 100644 internal/api/identity_gate_test.go create mode 100644 internal/identity/challenge.go create mode 100644 internal/identity/noble_interop_test.go create mode 100644 internal/identity/verify.go create mode 100644 internal/identity/verify_test.go create mode 100644 internal/storage/udl_admin.go create mode 100644 internal/storage/udl_admin_test.go diff --git a/dashboard/dist/index.html b/dashboard/dist/index.html index 7368629..78e8c47 100644 --- a/dashboard/dist/index.html +++ b/dashboard/dist/index.html @@ -21,7 +21,7 @@ min-height: 100vh; } - + diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index cb9b130..e69299c 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -8,6 +8,7 @@ "name": "@meta-core/dashboard", "version": "1.0.0", "dependencies": { + "@noble/secp256k1": "^3.1.0", "iso-639-3": "^3.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -744,6 +745,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@noble/secp256k1": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.1.0.tgz", + "integrity": "sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@remix-run/router": { "version": "1.23.2", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index 6b8580a..cbbaa5f 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -9,10 +9,11 @@ "preview": "vite preview" }, "dependencies": { + "@noble/secp256k1": "^3.1.0", + "iso-639-3": "^3.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "react-router-dom": "^6.20.0", - "iso-639-3": "^3.0.1" + "react-router-dom": "^6.20.0" }, "devDependencies": { "@types/react": "^18.2.38", diff --git a/dashboard/src/lib/sign.ts b/dashboard/src/lib/sign.ts new file mode 100644 index 0000000..bbcb6bd --- /dev/null +++ b/dashboard/src/lib/sign.ts @@ -0,0 +1,127 @@ +// Client-side proof of possession for account operations. +// +// meta-core stores every private key in plaintext, so it can sign as anybody — +// which means a signature it produced proves nothing about who asked. The only +// signature that carries authority is one made where the key actually is, and +// for a browser-driven admin console that is here. meta-core enforces the other +// half by refusing to sign anything in the challenge domain +// (identity.ChallengeDomain), so this path cannot be short-circuited. +// +// The secret key entered on this page is used for exactly one signature and is +// never sent anywhere: `/api/identity/challenge` receives only the uid, and the +// request that follows carries only the challenge and its signature. +// +// Same scheme, same curve and same encodings as meta-watch's sign-in +// (packages/meta-watch/ui/js/auth.js) — deliberately, so one implementation on +// the server verifies both. + +import * as secp from '@noble/secp256k1'; + +export type IdentityAction = 'reveal' | 'delete'; + +function hexToBytes(hex: string): Uint8Array | null { + let h = hex.trim(); + if (h.startsWith('0x') || h.startsWith('0X')) h = h.slice(2); + if (h.length !== 64 || /[^0-9a-fA-F]/.test(h)) return null; + const out = new Uint8Array(32); + for (let i = 0; i < 32; i++) out[i] = parseInt(h.slice(i * 2, i * 2 + 2), 16); + return out; +} + +function bytesToBase64(bytes: Uint8Array): string { + let s = ''; + for (const b of bytes) s += String.fromCharCode(b); + return btoa(s); +} + +const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; + +/// base58btc, the multibase "z" alphabet. Spelled out rather than pulled in: +/// the alternative is a second dependency for thirty lines of arithmetic. +function base58Encode(bytes: Uint8Array): string { + const digits: number[] = [0]; + for (const byte of bytes) { + let carry = byte; + for (let i = 0; i < digits.length; i++) { + carry += digits[i] << 8; + digits[i] = carry % 58; + carry = (carry / 58) | 0; + } + while (carry) { + digits.push(carry % 58); + carry = (carry / 58) | 0; + } + } + // Leading zero bytes encode as leading '1's rather than being dropped. + let out = ''; + for (let i = 0; i < bytes.length && bytes[i] === 0; i++) out += B58[0]; + for (let i = digits.length - 1; i >= 0; i--) out += B58[digits[i]]; + return out; +} + +/// The uid a secret key belongs to: "z" + base58btc(33-byte compressed pubkey). +/// Derived locally so a mistyped or wrong-account key is caught before any +/// request goes out. Mirrors `identityFromPrivKey` in meta-core's identity.go. +export function uidFromSecretKey(secretKeyHex: string): string | null { + const raw = hexToBytes(secretKeyHex); + if (!raw) return null; + try { + return 'z' + base58Encode(secp.getPublicKey(raw, true)); + } catch { + return null; // not a valid scalar + } +} + +/// Whether this page can sign at all. +/// +/// noble's signAsync takes its sha256 from crypto.subtle, which browsers expose +/// only in a secure context. The dashboard behind Caddy is https and fine; the +/// debug-direct port (18083) is plain http, where every signature would fail +/// with a cryptic error unless we say so first. +export function canSign(): boolean { + return Boolean(globalThis.crypto?.subtle); +} + +export const INSECURE_CONTEXT_MESSAGE = + 'This page cannot sign over a plain-http connection — browsers only expose the ' + + 'needed crypto on https (or localhost). Reach the dashboard through Caddy ' + + '(https://metacore-dev.localhost:8083) rather than the debug-direct port.'; + +/// Ask meta-core for a challenge, sign it locally, and hand back the proof. +/// +/// Throws with a message fit to show the user. The caller passes the proof +/// straight into the reveal/delete request body. +export async function proveOwnership( + uid: string, + action: IdentityAction, + secretKeyHex: string +): Promise<{ challenge: string; signature: string }> { + if (!canSign()) throw new Error(INSECURE_CONTEXT_MESSAGE); + + const raw = hexToBytes(secretKeyHex); + if (!raw) { + throw new Error('That is not a valid secret key: expected 64 hex characters.'); + } + const derived = uidFromSecretKey(secretKeyHex); + if (!derived) throw new Error('That is not a valid secp256k1 secret key.'); + if (derived !== uid) { + // Caught here rather than by the server, so the message can say which + // account the key actually is — the useful half of the answer. + throw new Error(`That key belongs to a different account (${derived}), not this one.`); + } + + const cRes = await fetch('/api/identity/challenge', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ uid, action }), + }); + const cBody = await cRes.json().catch(() => ({})); + if (!cRes.ok || !cBody.challenge) { + throw new Error(cBody.message ?? `Could not start verification (HTTP ${cRes.status}).`); + } + + // signAsync prehashes with sha256 and returns a 64-byte compact (r‖s) + // signature — noble v3 dropped DER. meta-core's identity.Verify accepts both. + const sig = await secp.signAsync(new TextEncoder().encode(cBody.challenge), raw); + return { challenge: cBody.challenge, signature: bytesToBase64(sig) }; +} diff --git a/dashboard/src/pages/Identity.tsx b/dashboard/src/pages/Identity.tsx index 57d48a4..14ee46a 100644 --- a/dashboard/src/pages/Identity.tsx +++ b/dashboard/src/pages/Identity.tsx @@ -1,5 +1,13 @@ import { useEffect, useRef, useState } from 'react'; +import { + INSECURE_CONTEXT_MESSAGE, + canSign, + proveOwnership, + uidFromSecretKey, + type IdentityAction, +} from '../lib/sign'; + // Admin console for meta-core's account keystore. // // meta-core stores one secp256k1 keypair per file under @@ -10,6 +18,13 @@ import { useEffect, useRef, useState } from 'react'; // The private key is returned by exactly two calls: POST /api/identity/generate // (once, at creation) and POST /api/identity/reveal (on demand, for an account // that already exists). Both land in the same exclusive SecretKeyPanel. +// +// Reveal and remove both require the account's own secret key, entered here and +// used to sign a one-shot challenge locally — the key itself is never sent. So +// this page can enumerate and create, but it holds no authority over an +// existing account that its operator does not already have the key for. There +// is deliberately no admin override: on this node the private key *is* the +// account. interface Account { uid: string; @@ -17,6 +32,14 @@ interface Account { createdAt: number; } +// What one account holds in the User Data Layer. Absent from +// /api/udl/users/stats means zero, not unknown. +interface Stats { + records: number; + cids: number; + keys: number; +} + // A secret on screen. Carries no createdAt: /api/identity/reveal answers with // {uid, curve, privateKeyHex} only. interface SecretKeyView { @@ -32,6 +55,14 @@ interface Pending { kind: 'remove' | 'reveal'; } +// Both confirmable operations map to a server-side action name, and the +// challenge is bound to it: a challenge minted to reveal a key cannot delete +// the account. Keeping the mapping in one place stops the two drifting. +const ACTION_FOR: Record = { + reveal: 'reveal', + remove: 'delete', +}; + // meta-watch's sentinel cell for profile records: display name and avatar are // about the *user*, not a content file, so they live at (uid, "self", ...). // meta-core stores them opaquely and knows nothing of the convention. @@ -173,6 +204,95 @@ async function fetchDisplayName(uid: string): Promise { } } +// How much each account holds. One call for the whole page, not one per row: +// meta-core answers it from a keyspace SCAN (there is no per-user cid index), +// so per-row calls would walk the keyspace once per account. +// +// Never throws, for the same reason fetchDisplayName does not: the counts are +// context on top of the uid, and the list must render identically when Redis is +// down — accounts are file-backed, their data is not. +// +// Returns null on failure rather than an empty map. The difference is +// load-bearing: an empty map renders as "no stored data" against every account, +// which inside a delete confirmation is a false statement about what is at +// stake. Unknown has to look like unknown. +async function fetchStats(): Promise | null> { + try { + const res = await fetch('/api/udl/users/stats'); + if (!res.ok) return null; + const data = (await res.json()) as { users?: Record }; + return data.users ?? {}; + } catch { + return null; + } +} + +// The secret-key box shared by the reveal and remove confirmations. +// +// Separate from the surrounding panel so both operations ask the same way, and +// so the key lives in this component's state rather than the page's — it is +// discarded with the panel instead of lingering in a parent that outlives it. +function SecretKeyConfirm({ + label, + actionLabel, + busy, + onConfirm, + onCancel, +}: { + label: string; + actionLabel: string; + busy: boolean; + onConfirm: (secretKeyHex: string) => void; + onCancel: () => void; +}) { + const [secret, setSecret] = useState(''); + const signable = canSign(); + // Derived locally, purely to tell the operator they have the wrong key before + // they hit the button. The server would refuse either way. + const derived = secret.trim() ? uidFromSecretKey(secret) : null; + const mismatch = Boolean(secret.trim() && derived && derived !== label); + + return ( +
{ + e.preventDefault(); + onConfirm(secret.trim()); + }} + > +

+ Paste this account's secret key to prove it is yours. It is used to sign a one-time + challenge in this browser and is never sent to the server. +

+ {!signable &&

{INSECURE_CONTEXT_MESSAGE}

} + setSecret(e.target.value)} + disabled={busy || !signable} + autoFocus + /> + {mismatch && ( +

+ That key belongs to a different account ({derived}). +

+ )} + + +
+ ); +} + // The one place a private key is ever displayed. Rendered exclusively (the // account list is hidden while it is open) so the "save it now" contract of a // freshly generated key is not competing with anything, and so the hardcoded @@ -351,6 +471,7 @@ function NewAccountPanel({ function AccountCard({ account, name, + stats, busy, disabled, pending, @@ -362,18 +483,24 @@ function AccountCard({ }: { account: Account; name?: string; + // undefined means "counted, and it holds nothing"; null means "could not be + // counted". The two must not read the same on a confirmation screen. + stats?: Stats | null; busy: boolean; disabled: boolean; pending: Pending | null; error: string | null; onAsk: (p: Pending) => void; onCancel: () => void; - onReveal: (uid: string) => void; - onRemove: (uid: string) => void; + onReveal: (uid: string, secretKeyHex: string) => void; + onRemove: (uid: string, secretKeyHex: string) => void; }) { const [copied, setCopied] = useState(false); const uidBoxId = `mm-uid-${account.uid}`; const label = name ?? account.uid; + const counted = stats !== null; + const records = stats?.records ?? 0; + const titles = stats?.cids ?? 0; const copyUid = async () => { await copyToClipboard(account.uid, uidBoxId); @@ -394,6 +521,16 @@ function AccountCard({

Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}

+

+ {/* An account that has never been used legitimately holds nothing, so + zero is a real answer — but only once the count actually ran. */} + {!counted + ? 'Stored data: could not be counted (storage unavailable)' + : records === 0 + ? 'No stored data' + : `${records.toLocaleString()} stored ${records === 1 ? 'record' : 'records'}` + + ` across ${titles.toLocaleString()} ${titles === 1 ? 'title' : 'titles'}`} +

- + onReveal(account.uid, secret)} + onCancel={onCancel} + />
)} @@ -454,21 +593,31 @@ function AccountCard({ verifying on every peer that replicated them. Deleting a key is not a recall.
  • - Its data stays, but goes read-only. The account's User Data Layer - cells — likes, ratings, watch progress — survive in Redis, and no newer version of - any of them can ever be written. + Its data is destroyed, not orphaned.{' '} + {!counted + ? 'This account\u2019s records could not be counted because storage is unavailable — ' + + 'so meta-core will refuse this delete rather than remove the key and strand them.' + : records === 0 + ? 'This account holds no User Data Layer records, so there is nothing to purge.' + : `All ${records.toLocaleString()} User Data Layer ${ + records === 1 ? 'record' : 'records' + } — likes, ratings, watch progress across ${titles.toLocaleString()} ${ + titles === 1 ? 'title' : 'titles' + } — are permanently deleted from Redis, along with every index that names this uid.`}{' '} + This cannot be undone and there is no backup.
  • This signs nobody out elsewhere. Any device that holds this key still holds it, and can import it back here at any time.
    • - - + onRemove(account.uid, secret)} + onCancel={onCancel} + /> )} @@ -478,6 +627,8 @@ function AccountCard({ export default function Identity() { const [accounts, setAccounts] = useState(null); const [names, setNames] = useState>({}); + const [stats, setStats] = useState | null>({}); + const [notice, setNotice] = useState(null); const [loading, setLoading] = useState(true); const [fetchError, setFetchError] = useState(null); const [secret, setSecret] = useState(null); @@ -508,30 +659,39 @@ export default function Identity() { setAccounts(list); setLoading(false); - const settled = await Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))); + // Names and counts both come after the list is on screen, and neither can + // fail the page: a down Redis costs decoration, not the account list. + const [settled, counts] = await Promise.all([ + Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))), + fetchStats(), + ]); if (gen !== reqGen.current) return; const next: Record = {}; settled.forEach((r, i) => { if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value; }); - // Replace rather than merge, so a removed account's name goes with it. + // Replace rather than merge, so a removed account's name and counts go with + // it instead of lingering against a uid that no longer exists. setNames(next); + setStats(counts); }; useEffect(() => { refresh(); }, []); - const reveal = async (uid: string) => { + const reveal = async (uid: string, secretKeyHex: string) => { setBusyUid(uid); setRowError(null); try { - // POST with a body rather than ?uid=, so the target stays out of the - // proxy's access log. The handler accepts either. + const proof = await proveOwnership(uid, ACTION_FOR.reveal, secretKeyHex); + // POST with a body rather than ?uid=, so neither the target nor the + // signature reaches the proxy's access log. The handler accepts the uid + // either way; the proof must be in the body. const res = await fetch('/api/identity/reveal', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ confirm: true, uid }), + body: JSON.stringify({ confirm: true, uid, ...proof }), }); if (!res.ok) { setRowError({ uid, message: await errorMessage(res, 'Reveal failed') }); @@ -547,19 +707,34 @@ export default function Identity() { } }; - const remove = async (uid: string) => { + const remove = async (uid: string, secretKeyHex: string) => { setBusyUid(uid); setRowError(null); + setNotice(null); try { - // The uid is mandatory: with more than one account the backend refuses an - // unqualified delete (400 uid_required) rather than guess. - const res = await fetch(`/api/identity?uid=${encodeURIComponent(uid)}&confirm=true`, { + const proof = await proveOwnership(uid, ACTION_FOR.remove, secretKeyHex); + // The uid is mandatory: the signature authorises one specific account, so + // the backend refuses an unqualified delete rather than guess a target. + const res = await fetch('/api/identity', { method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ confirm: true, uid, ...proof }), }); if (!res.ok) { setRowError({ uid, message: await errorMessage(res, 'Remove failed') }); return; } + // Report what the purge actually removed. "Account removed" on its own + // would not distinguish a clean delete from one that stranded the data. + const data = (await res.json().catch(() => ({}))) as { purged?: Stats }; + const purged = data.purged; + setNotice( + purged && purged.records > 0 + ? `Account removed. Purged ${purged.records.toLocaleString()} User Data Layer ` + + `${purged.records === 1 ? 'record' : 'records'} across ` + + `${purged.cids.toLocaleString()} ${purged.cids === 1 ? 'title' : 'titles'}.` + : 'Account removed. It held no User Data Layer records.' + ); setPending(null); await refresh(); } catch (e) { @@ -574,6 +749,12 @@ export default function Identity() { refresh(); }; + const ask = (p: Pending) => { + setRowError(null); + setNotice(null); + setPending(p); + }; + const showList = !loading && !fetchError && !secret; return ( @@ -594,15 +775,26 @@ export default function Identity() {

    - Admin page — no authorisation happens here. /api/identity/*{' '} - asks no questions: anything that can reach this API can list, create, reveal or delete{' '} - any account's private key. The only access control is the perimeter in front of - this port. Client-facing screens (meta-watch) authorise reveal and delete against the - caller's own session and expose no delete at all; this page has neither check. Treat - reaching it as equivalent to holding every key on this node. + The key is the only authority. Revealing or removing an account requires + that account's secret key, entered here and used to sign a one-time challenge in your + browser — there is no admin override, and holding this page is not one. Listing and + creating accounts are still open to anything that can reach this API, so the perimeter in + front of this port is what keeps the roster private. +

    +

    + What this does not defend against: anyone with filesystem access to{' '} + {'{META_CORE_PATH}'}/identity/accounts/ reads every private key directly. + The keys are stored in plaintext, matching the trust model of the rest of{' '} + /meta-core.

    + {notice && ( +
    +

    {notice}

    +
    + )} + {loading &&
    Loading…
    } {fetchError && ( @@ -639,14 +831,12 @@ export default function Identity() { key={a.uid} account={a} name={names[a.uid]} + stats={stats === null ? null : stats[a.uid]} busy={busyUid === a.uid} disabled={busyUid !== null} pending={pending} error={rowError?.uid === a.uid ? rowError.message : null} - onAsk={(p) => { - setRowError(null); - setPending(p); - }} + onAsk={ask} onCancel={() => setPending(null)} onReveal={reveal} onRemove={remove} diff --git a/internal/api/handlers_identity.go b/internal/api/handlers_identity.go index 381163c..c08fb77 100644 --- a/internal/api/handlers_identity.go +++ b/internal/api/handlers_identity.go @@ -3,11 +3,14 @@ package api import ( "encoding/base64" "encoding/json" + "errors" + "log" "net/http" "strconv" "strings" "github.com/metazla/meta-core/internal/identity" + "github.com/metazla/meta-core/internal/storage" ) // identityStatusResponse is the shape of GET /api/identity. The private key @@ -59,12 +62,40 @@ type identityImportResponse struct { Created bool `json:"created"` } -// identityRevealRequest asks for an existing account's private key back. The -// key was shown once at generate; without a way to see it again, a client with -// a real sign-out has no recovery path for a user who never saved it. +// identityChallengeRequest asks for a one-shot challenge to sign. +type identityChallengeRequest struct { + UID string `json:"uid"` + Action string `json:"action"` +} + +type identityChallengeResponse struct { + Challenge string `json:"challenge"` + ExpiresAt int64 `json:"expiresAt"` +} + +// identityProof is the proof-of-possession carried by reveal and delete: the +// challenge text as issued, and a signature over it by the account's key. +// +// The signature is produced by whoever holds the key — the browser, locally. +// It is NOT obtainable from POST /api/identity/sign, which refuses any payload +// in the challenge domain; that refusal is what makes this proof mean +// something on a node that stores every private key in plaintext. +type identityProof struct { + Challenge string `json:"challenge"` + Signature string `json:"signature"` +} + +// identityRevealRequest asks for an existing account's private key back. +// +// Gated on proof-of-possession, which makes it a "does this node hold the same +// key I do" check rather than the recovery path it once was. That is the +// deliberate consequence of the rule this file enforces: the private key, and +// nothing else, is authority over the account. An operator with root on the box +// can still read the file; an operator with only the API cannot. type identityRevealRequest struct { Confirm bool `json:"confirm"` UID string `json:"uid"` + identityProof } type identityRevealResponse struct { @@ -76,6 +107,15 @@ type identityRevealResponse struct { type identityDeleteRequest struct { Confirm bool `json:"confirm"` UID string `json:"uid"` + identityProof +} + +// identityDeleteResponse reports what the delete actually removed. The purge +// counts are the point: "account removed" alone leaves the caller unable to +// tell a clean removal from one that silently stranded a decade of ratings. +type identityDeleteResponse struct { + UID string `json:"uid"` + Purged storage.UDLUserStats `json:"purged"` } type identitySignRequest struct { @@ -114,6 +154,16 @@ type identitySignBatchResponse struct { // should page. Sized well above the longest real series (Supernatural, 327). const maxSignBatch = 1000 +// reservedPayloadMessage explains the one refusal the signing endpoints make. +// +// This is the load-bearing half of proof-of-possession on a node that holds +// every private key: without it, a caller wanting to delete someone else's +// account just asks meta-core to sign the challenge for them, and the gate on +// reveal/delete becomes a formality. See identity.ChallengeDomain. +const reservedPayloadMessage = "payload is in the reserved account-authorisation domain (" + + identity.ChallengeDomain + "…); meta-core will not sign one. " + + "Signing a challenge is what proves you hold the key, so it must be done where the key is." + type identityAEADKeyResponse struct { AEADKeyB64 string `json:"aeadKeyB64"` } @@ -243,23 +293,105 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) { }) } -// handleIdentityReveal returns an existing account's private key. Deliberately -// POST + explicit confirm (never a bare GET) so it cannot be triggered by a -// link or an image tag, and so it reads as the deliberate act it is. The -// endpoint is auth-gated at the perimeter like the rest of /api/identity/*. -func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) { - var body identityRevealRequest - _ = json.NewDecoder(r.Body).Decode(&body) - if !body.Confirm && r.URL.Query().Get("confirm") != "true" { - writeErrorSlug(w, http.StatusBadRequest, "confirm_required", - "revealing a private key requires {\"confirm\": true} or ?confirm=true", false) +// handleIdentityChallenge issues the one-shot text a client signs to prove it +// holds an account's private key. +// +// Unauthenticated by design: a challenge is useless without the key, and +// GET /api/identity/accounts already publishes every uid, so refusing to mint +// one for an unknown caller would protect nothing. Requiring the account to +// exist is for a clear error, not secrecy. +func (s *Server) handleIdentityChallenge(w http.ResponseWriter, r *http.Request) { + var body identityChallengeRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON body") return } uid := strings.TrimSpace(body.UID) + if uid == "" { + writeErrorSlug(w, http.StatusBadRequest, "uid_required", "uid is required", false) + return + } + action := identity.Action(strings.TrimSpace(body.Action)) + if !identity.ValidAction(action) { + writeErrorSlug(w, http.StatusBadRequest, "bad_action", + `action must be "reveal" or "delete"`, false) + return + } + if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) { + writeErrorSlug(w, http.StatusNotFound, "no_identity", "no identity for the given uid", false) + return + } + text, expires, err := s.challenges.Issue(uid, action) + if err != nil { + if errors.Is(err, identity.ErrChallengeFull) { + writeErrorSlug(w, http.StatusServiceUnavailable, "challenges_full", err.Error(), true) + return + } + writeErrorSlug(w, http.StatusBadRequest, "bad_uid", err.Error(), false) + return + } + writeJSON(w, http.StatusOK, identityChallengeResponse{ + Challenge: text, + ExpiresAt: expires.Unix(), + }) +} + +// requireProof resolves the target account and verifies the caller's +// proof-of-possession for `action`. status == 0 means the proof held. +// +// The uid is mandatory here, unlike elsewhere in this file: the old "sole +// account" fallback would have this pick a target the signature does not name, +// and a proof that authorises an unnamed account authorises the wrong one as +// soon as a second profile exists. +func (s *Server) requireProof(uid string, proof identityProof, action identity.Action) ( + id *identity.Identity, status int, slug, msg string, +) { + uid = strings.TrimSpace(uid) + if uid == "" { + return nil, http.StatusBadRequest, "uid_required", + "uid is required: the signature authorises one specific account" + } + if proof.Challenge == "" || proof.Signature == "" { + return nil, http.StatusUnauthorized, "signature_required", + "this operation requires proof you hold the account's private key: " + + "POST /api/identity/challenge, sign the returned text, and resend it as {challenge, signature}" + } + sig, err := base64.StdEncoding.DecodeString(proof.Signature) + if err != nil { + return nil, http.StatusBadRequest, "bad_signature", "signature is not valid base64" + } + // Resolve before redeeming so a request naming a missing account does not + // burn a challenge it could not have used anyway. + id, status, slug, msg = s.resolveAccount(uid) + if status != 0 { + return nil, status, slug, msg + } + // Redeem first, verify second: the challenge is single-use either way, so a + // wrong signature costs the attacker a fresh round trip per guess rather + // than letting them grind one challenge offline against the same nonce. + if err := s.challenges.Redeem(proof.Challenge, uid, action); err != nil { + return nil, http.StatusUnauthorized, "bad_challenge", err.Error() + } + if err := identity.Verify(uid, []byte(proof.Challenge), sig); err != nil { + return nil, http.StatusUnauthorized, "bad_signature", err.Error() + } + return id, 0, "", "" +} + +// handleIdentityReveal returns an existing account's private key to a caller +// that proves it already holds that key. +// +// Deliberately POST (never a bare GET) so it cannot be triggered by a link or +// an image tag. The proof is what actually guards it: before this gate existed, +// anything that could reach the port could read every private key on the node. +func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) { + var body identityRevealRequest + _ = json.NewDecoder(r.Body).Decode(&body) + uid := strings.TrimSpace(body.UID) if uid == "" { uid = strings.TrimSpace(r.URL.Query().Get("uid")) } - id, status, slug, msg := s.resolveAccount(uid) + id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionReveal) if status != 0 { writeErrorSlug(w, status, slug, msg, false) return @@ -271,46 +403,62 @@ func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) { }) } +// handleIdentityDelete removes one account and everything it wrote. +// +// Two things happen, in this order and no other: +// +// 1. the account's User Data Layer rows are purged (cells plus every index +// that names the uid, including the cid-scoped ones shared with other +// accounts); +// 2. the keypair file is removed. +// +// Purge-then-delete so a failure never strands data: if the purge errors the +// key is still on disk, the account still exists, and the caller can retry. +// The reverse order would leave rows nobody can ever attribute, reach or +// rewrite — the orphaning this endpoint exists to prevent. +// +// What deletion still does NOT do, and cannot: records already signed under +// this uid and replicated to other peers keep verifying there. Removing a key +// is not a recall. func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) { var body identityDeleteRequest - // Tolerate empty body — confirm flag in a query param is also acceptable - // for shells where sending a JSON body with DELETE is awkward. + // Tolerate an empty body so the uid may ride in the query — meta-watch's + // proxy puts it there. The proof itself must be in the body: a signature in + // a URL ends up in every access log between here and the browser. _ = json.NewDecoder(r.Body).Decode(&body) - if !body.Confirm && r.URL.Query().Get("confirm") != "true" { - writeErrorSlug(w, http.StatusBadRequest, "confirm_required", - "identity deletion requires {\"confirm\": true} or ?confirm=true", false) - return - } - dir := s.config.IdentityAccountsDir() uid := strings.TrimSpace(body.UID) if uid == "" { uid = strings.TrimSpace(r.URL.Query().Get("uid")) } - if uid == "" { - // Back-compat: with no uid, delete the sole account. Ambiguous (and so - // refused) once more than one account exists. - list, err := identity.List(dir) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - switch len(list) { - case 0: - w.WriteHeader(http.StatusNoContent) // idempotent - return - case 1: - uid = list[0].UID - default: - writeErrorSlug(w, http.StatusBadRequest, "uid_required", - "multiple accounts exist; specify uid to delete", false) - return - } - } - if err := identity.DeleteByUID(dir, uid); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionDelete) + if status != 0 { + writeErrorSlug(w, status, slug, msg, false) return } - w.WriteHeader(http.StatusNoContent) + + // A purge needs storage. Refusing here — rather than deleting the key and + // reporting the purge as "0 records" — is the difference between a retry + // and permanent orphaned data: once the key is gone the account cannot be + // named for a second attempt through this API. + if !s.storage.IsConnected() { + writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable", + "storage is not connected; refusing to delete the key while its data cannot be purged", true) + return + } + purged, err := s.storage.UDLPurgeUser(id.UID) + if err != nil { + writeErrorSlug(w, http.StatusInternalServerError, "purge_failed", + "could not purge this account's data; the account was NOT deleted: "+err.Error(), true) + return + } + + if err := identity.DeleteByUID(s.config.IdentityAccountsDir(), id.UID); err != nil { + writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false) + return + } + log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids", + id.UID, purged.Records, purged.Cids) + writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged}) } func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) { @@ -333,6 +481,10 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64") return } + if identity.IsChallengePayload(raw) { + writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false) + return + } sig, err := identity.Sign(id, raw) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) @@ -380,6 +532,11 @@ func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64") return } + if identity.IsChallengePayload(raw) { + writeErrorSlug(w, http.StatusForbidden, "reserved_payload", + "payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false) + return + } sig, err := identity.Sign(id, raw) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) diff --git a/internal/api/identity_gate_test.go b/internal/api/identity_gate_test.go new file mode 100644 index 0000000..94dd7de --- /dev/null +++ b/internal/api/identity_gate_test.go @@ -0,0 +1,257 @@ +package api + +// End-to-end exercise of the account gate against a live router + miniredis: +// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify +// both the key and every row are gone. Deliberately drives the real HTTP +// handlers rather than the packages beneath them, because the bugs that matter +// here (a challenge accepted for the wrong action, a purge that runs after the +// key is gone) live in the wiring. + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/gorilla/mux" + "github.com/metazla/meta-core/internal/config" + "github.com/metazla/meta-core/internal/identity" + "github.com/metazla/meta-core/internal/storage" +) + +func newTestServer(t *testing.T) *Server { + t.Helper() + mr := miniredis.RunT(t) + stor := storage.NewClient("") + if err := stor.Connect("redis://" + mr.Addr()); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = stor.Close() }) + + s := &Server{ + config: &config.Config{MetaCorePath: t.TempDir()}, + storage: stor, + challenges: identity.NewChallengeStore(), + router: mux.NewRouter(), + } + s.setupIdentityTestRoutes() + return s +} + +// Only the routes under test, so this does not depend on every other subsystem +// booting. +func (s *Server) setupIdentityTestRoutes() { + s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET") + s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST") + s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST") + s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST") + s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE") + s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST") + s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET") +} + +func do(t *testing.T, s *Server, method, path string, body any) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + var rdr *bytes.Reader + if body != nil { + b, _ := json.Marshal(body) + rdr = bytes.NewReader(b) + } else { + rdr = bytes.NewReader(nil) + } + req := httptest.NewRequest(method, path, rdr) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.router.ServeHTTP(rec, req) + out := map[string]any{} + _ = json.Unmarshal(rec.Body.Bytes(), &out) + return rec, out +} + +// signAs produces the proof a browser would, from the raw key. +func signAs(t *testing.T, privHex, challenge string) map[string]any { + t.Helper() + raw, err := hex.DecodeString(privHex) + if err != nil { + t.Fatalf("decode key: %v", err) + } + id, err := identity.Import(hex.EncodeToString(raw)) + if err != nil { + t.Fatalf("import: %v", err) + } + sig, err := identity.Sign(id, []byte(challenge)) + if err != nil { + t.Fatalf("sign: %v", err) + } + return map[string]any{ + "challenge": challenge, + "signature": base64.StdEncoding.EncodeToString(sig), + } +} + +func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) { + s := newTestServer(t) + + rec, gen := do(t, s, "POST", "/api/identity/generate", nil) + if rec.Code != http.StatusCreated { + t.Fatalf("generate: %d %s", rec.Code, rec.Body) + } + uid := gen["uid"].(string) + priv := gen["privateKeyHex"].(string) + + // Some data to lose. + for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} { + if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok { + t.Fatalf("seed: %v", err) + } + } + + // The count the UI shows in the confirmation. + rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil) + if rec.Code != http.StatusOK { + t.Fatalf("stats: %d %s", rec.Code, rec.Body) + } + users := stats["users"].(map[string]any) + mine := users[uid].(map[string]any) + if mine["records"].(float64) != 3 || mine["cids"].(float64) != 2 { + t.Fatalf("stats = %v, want 3 records / 2 cids", mine) + } + + // Unsigned delete must be refused, and must not have touched anything. + rec, body := do(t, s, "DELETE", "/api/identity", map[string]any{"uid": uid, "confirm": true}) + if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" { + t.Fatalf("unsigned delete = %d %v, want 401 signature_required", rec.Code, body) + } + if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) { + t.Fatal("a refused delete removed the key anyway") + } + + // meta-core must refuse to sign the caller's authorisation for them — + // without this the gate is decoration on a node holding every key. + ch, _, err := s.challenges.Issue(uid, identity.ActionDelete) + if err != nil { + t.Fatalf("issue: %v", err) + } + rec, body = do(t, s, "POST", "/api/identity/sign", map[string]any{ + "uid": uid, + "bytesB64": base64.StdEncoding.EncodeToString([]byte(ch)), + }) + if rec.Code != http.StatusForbidden || body["error"] != "reserved_payload" { + t.Fatalf("sign of a challenge = %d %v, want 403 reserved_payload", rec.Code, body) + } + + // A challenge minted for reveal must not authorise a delete. + rec, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"}) + if rec.Code != http.StatusOK { + t.Fatalf("challenge: %d %s", rec.Code, rec.Body) + } + proof := signAs(t, priv, chBody["challenge"].(string)) + req := map[string]any{"uid": uid, "confirm": true} + for k, v := range proof { + req[k] = v + } + rec, body = do(t, s, "DELETE", "/api/identity", req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("reveal challenge used for delete = %d %v, want 401", rec.Code, body) + } + + // The real thing. + rec, chBody = do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"}) + if rec.Code != http.StatusOK { + t.Fatalf("challenge: %d %s", rec.Code, rec.Body) + } + proof = signAs(t, priv, chBody["challenge"].(string)) + req = map[string]any{"uid": uid, "confirm": true} + for k, v := range proof { + req[k] = v + } + rec, body = do(t, s, "DELETE", "/api/identity", req) + if rec.Code != http.StatusOK { + t.Fatalf("signed delete = %d %s", rec.Code, rec.Body) + } + purged := body["purged"].(map[string]any) + if purged["records"].(float64) != 3 || purged["cids"].(float64) != 2 { + t.Fatalf("purged = %v, want 3 records / 2 cids", purged) + } + + // Key gone. + if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) { + t.Fatal("the account file survived the delete") + } + // Data gone — the orphaning this whole change exists to prevent. + after, err := s.storage.UDLAllUserStats() + if err != nil { + t.Fatalf("stats after: %v", err) + } + if got, present := after[uid]; present { + t.Fatalf("deleted account still holds %+v", got) + } + for _, cid := range []string{"cidA", "cidB"} { + uids, err := s.storage.UDLCidUsers(cid) + if err != nil { + t.Fatalf("cid users: %v", err) + } + if len(uids) != 0 { + t.Fatalf("%s still names %v after the purge", cid, uids) + } + } + + // The challenge is single-use, so replaying the same proof does nothing. + rec, _ = do(t, s, "DELETE", "/api/identity", req) + if rec.Code == http.StatusOK { + t.Fatal("a replayed proof must not succeed") + } +} + +func TestE2E_RevealRequiresProof(t *testing.T) { + s := newTestServer(t) + _, gen := do(t, s, "POST", "/api/identity/generate", nil) + uid := gen["uid"].(string) + priv := gen["privateKeyHex"].(string) + + rec, body := do(t, s, "POST", "/api/identity/reveal", map[string]any{"uid": uid, "confirm": true}) + if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" { + t.Fatalf("unsigned reveal = %d %v, want 401 signature_required", rec.Code, body) + } + + _, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"}) + proof := signAs(t, priv, chBody["challenge"].(string)) + req := map[string]any{"uid": uid, "confirm": true} + for k, v := range proof { + req[k] = v + } + rec, body = do(t, s, "POST", "/api/identity/reveal", req) + if rec.Code != http.StatusOK { + t.Fatalf("signed reveal = %d %s", rec.Code, rec.Body) + } + if body["privateKeyHex"] != priv { + t.Fatal("reveal returned a different key than generate did") + } +} + +// Another account's key must not authorise this one, even with a challenge +// minted for the right uid and action. +func TestE2E_AnotherAccountsKeyIsRefused(t *testing.T) { + s := newTestServer(t) + _, alice := do(t, s, "POST", "/api/identity/generate", nil) + _, mallory := do(t, s, "POST", "/api/identity/generate", nil) + + uid := alice["uid"].(string) + _, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"}) + proof := signAs(t, mallory["privateKeyHex"].(string), chBody["challenge"].(string)) + req := map[string]any{"uid": uid, "confirm": true} + for k, v := range proof { + req[k] = v + } + rec, body := do(t, s, "DELETE", "/api/identity", req) + if rec.Code != http.StatusUnauthorized || body["error"] != "bad_signature" { + t.Fatalf("mallory deleting alice = %d %v, want 401 bad_signature", rec.Code, body) + } + if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) { + t.Fatal("alice's account was deleted by mallory's signature") + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 6310815..fa7a351 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -42,8 +42,11 @@ type Server struct { schemaIndexer *schema.Indexer webdavHandler *webdav.Handler search *searchIndex - router *mux.Router - server *http.Server + // challenges backs proof-of-possession on reveal/delete. In-process, so a + // restart invalidates the ones in flight — see identity/challenge.go. + challenges *identity.ChallengeStore + router *mux.Router + server *http.Server } // NewServer creates a new API server @@ -139,6 +142,8 @@ func NewServer( // Initialize WebDAV handler (caching is handled by nginx proxy_cache) s.webdavHandler = webdav.NewHandler(cfg) + s.challenges = identity.NewChallengeStore() + // One-time migration: fold a pre-multi-account identity.json into the // per-account keystore so existing single-identity hosts keep their uid. if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil { @@ -170,6 +175,7 @@ func (s *Server) setupRoutes() { // at the perimeter — do NOT add to nginx-hash-lock unauth bypass. s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET") s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET") + s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST") s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST") s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST") s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST") @@ -212,6 +218,7 @@ func (s *Server) setupRoutes() { s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET") s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET") s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET") + s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET") // Metadata operations - base endpoints s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET") diff --git a/internal/api/udl.go b/internal/api/udl.go index c820d7b..56adc49 100644 --- a/internal/api/udl.go +++ b/internal/api/udl.go @@ -16,6 +16,7 @@ import ( "strings" "github.com/gorilla/mux" + "github.com/metazla/meta-core/internal/storage" ) // UDLPutRequest is the body of PUT /api/udl/record. @@ -349,3 +350,35 @@ func aggregatePublicValues(key string, values []string) UDLAggregate { } return agg } + +// udlUserStatsResponse is the shape of GET /api/udl/users/stats. +type udlUserStatsResponse struct { + // Users maps uid -> counts. Accounts holding nothing are absent rather than + // present with zeroes; a caller rendering a list treats "missing" as 0. + Users map[string]storage.UDLUserStats `json:"users"` +} + +// handleUDLUserStats handles GET /api/udl/users/stats — how much every account +// holds, in one keyspace walk. +// +// Bulk rather than per-uid because the only caller is the accounts page, which +// needs a number for every row: the counts come from a SCAN (there is no +// per-user cid index — see storage/udl_admin.go), and MATCH filters without +// indexing, so N per-uid calls would walk the whole keyspace N times. +// +// Deliberately not on the critical path of anything. It is an admin read whose +// cost grows with the keyspace, so callers should fetch it after rendering, +// never before. +func (s *Server) handleUDLUserStats(w http.ResponseWriter, r *http.Request) { + if !s.storage.IsConnected() { + writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable", + "storage not connected", true) + return + } + stats, err := s.storage.UDLAllUserStats() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, udlUserStatsResponse{Users: stats}) +} diff --git a/internal/identity/challenge.go b/internal/identity/challenge.go new file mode 100644 index 0000000..638718d --- /dev/null +++ b/internal/identity/challenge.go @@ -0,0 +1,126 @@ +package identity + +// In-memory, single-use challenge store backing proof-of-possession on the +// account operations that need it (reveal, delete). +// +// In memory on purpose: a challenge is worthless once redeemed and worthless +// once expired, so persisting it would only widen the window in which a leaked +// one is useful. A meta-core restart invalidates every challenge in flight, +// which costs a client one retry. +// +// Mirrors the sign-in challenge in meta-watch's src/auth.rs, down to the TTL — +// the browser signs immediately, so seconds is the right order and a narrow +// window bounds replay if one ever surfaces in a proxy log. + +import ( + "errors" + "sync" + "time" +) + +// ChallengeTTL is how long an issued challenge stays answerable. +const ChallengeTTL = 120 * time.Second + +// maxChallenges caps the store. An unauthenticated caller can mint challenges +// for any uid it can name, so the map is attacker-growable; without a ceiling +// that is a memory-exhaustion lever. At the cap, issuing fails rather than +// evicting: dropping someone else's live challenge on demand would let an +// attacker deny sign-outs, which is worse than making them retry. +const maxChallenges = 4096 + +var ( + // ErrChallengeUnknown covers absent, already-redeemed and expired alike. + // One error for all three deliberately: distinguishing them tells a prober + // whether a challenge string was ever real. + ErrChallengeUnknown = errors.New("challenge is unknown, already used, or expired") + ErrChallengeFull = errors.New("too many challenges in flight; try again shortly") +) + +type challengeEntry struct { + uid string + action Action + expires time.Time +} + +// ChallengeStore issues and redeems challenges. Safe for concurrent use. +type ChallengeStore struct { + mu sync.Mutex + entries map[string]challengeEntry + ttl time.Duration + // now is swappable so the tests can age entries without sleeping. + now func() time.Time +} + +func NewChallengeStore() *ChallengeStore { + return &ChallengeStore{ + entries: make(map[string]challengeEntry), + ttl: ChallengeTTL, + now: time.Now, + } +} + +// Issue mints a challenge for (uid, action) and remembers it until redeemed or +// expired. Returns the literal text the client signs and its expiry. +func (s *ChallengeStore) Issue(uid string, action Action) (string, time.Time, error) { + if !validUID(uid) { + return "", time.Time{}, errors.New("invalid uid") + } + if !ValidAction(action) { + return "", time.Time{}, errors.New("unknown action") + } + text, err := NewChallengeText(uid, action) + if err != nil { + return "", time.Time{}, err + } + + s.mu.Lock() + defer s.mu.Unlock() + s.sweepLocked() + if len(s.entries) >= maxChallenges { + return "", time.Time{}, ErrChallengeFull + } + expires := s.now().Add(s.ttl) + s.entries[text] = challengeEntry{uid: uid, action: action, expires: expires} + return text, expires, nil +} + +// Redeem consumes a challenge, asserting it was issued for this exact uid and +// action. Single-use: the entry is dropped on any lookup that finds it, so a +// mismatched redemption burns the challenge rather than leaving it to be +// retried against a different account. +func (s *ChallengeStore) Redeem(text, uid string, action Action) error { + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[text] + if !ok { + return ErrChallengeUnknown + } + delete(s.entries, text) + if s.now().After(entry.expires) { + return ErrChallengeUnknown + } + if entry.uid != uid || entry.action != action { + return ErrChallengeUnknown + } + // Re-read the signed bytes rather than trusting the map key alone: the + // signature covers `text`, so `text` is what has to name this uid+action. + if !ChallengeMatches(text, uid, action) { + return ErrChallengeUnknown + } + return nil +} + +// sweepLocked drops expired entries. Caller holds the lock. +// +// Called only from Issue: with no background goroutine, expiry is enforced +// lazily on the one path that grows the map, which is exactly where the bound +// matters. +func (s *ChallengeStore) sweepLocked() { + now := s.now() + for text, e := range s.entries { + if now.After(e.expires) { + delete(s.entries, text) + } + } +} diff --git a/internal/identity/noble_interop_test.go b/internal/identity/noble_interop_test.go new file mode 100644 index 0000000..6bb2c20 --- /dev/null +++ b/internal/identity/noble_interop_test.go @@ -0,0 +1,43 @@ +package identity + +import ( + "encoding/base64" + "testing" +) + +// A real signature produced by @noble/secp256k1 v3 — the library the dashboard +// and meta-watch both sign with — pinned as a fixture and verified here. +// +// This is the one seam neither side's own tests can cover: Go signs DER, the +// browser signs 64-byte compact, and every unit test above verifies signatures +// this package produced itself. If the encodings ever disagree, the gate fails +// closed in production (nobody can delete their own account) while every other +// test stays green. Regenerate with: +// +// node -e 'import("@noble/secp256k1").then(async s => { … })' +// +// against the same fixed scalar (32 zero bytes with 42 in the last). +func TestVerify_AcceptsARealNobleV3Signature(t *testing.T) { + const ( + uid = "ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY" + challenge = "metamesh-identity-v1:delete:ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY:deadbeef" + sigB64 = "lIiZGL0j9l/w2WXwzwe56cICnusuYdD3oaTuYfFTtNJBR9MolXFPk3014WQeAlvmbX4LfnW+TQz4asqoSwmSAQ==" + ) + + sig, err := base64.StdEncoding.DecodeString(sigB64) + if err != nil { + t.Fatalf("decode fixture: %v", err) + } + if len(sig) != 64 { + t.Fatalf("fixture is %d bytes, expected noble's 64-byte compact form", len(sig)) + } + if err := Verify(uid, []byte(challenge), sig); err != nil { + t.Fatalf("a genuine browser signature must verify: %v", err) + } + + // And the same signature must not verify over a different challenge — a + // verifier that ignored the payload would pass the line above too. + if err := Verify(uid, []byte(challenge+"x"), sig); err == nil { + t.Fatal("the fixture verified over the wrong payload") + } +} diff --git a/internal/identity/verify.go b/internal/identity/verify.go new file mode 100644 index 0000000..e0bba30 --- /dev/null +++ b/internal/identity/verify.go @@ -0,0 +1,160 @@ +package identity + +// Signature verification against a uid, and the challenge vocabulary that +// makes a signature mean "I hold this account's private key" rather than +// "meta-core signed something for me". +// +// The asymmetry that motivates this file: meta-core *holds* every private key, +// so it can sign as anybody. A signature is therefore only proof of possession +// if the payload is one meta-core refuses to sign on request — hence +// ChallengeDomain, which POST /api/identity/sign rejects outright. Without that +// refusal an attacker mints their own authorisation in one extra call and the +// whole gate is decoration. +// +// Verification itself needs no stored state: a uid *is* the public key +// ("z" + base58btc(33-byte compressed sec1)), so the pubkey is recovered from +// the uid the caller names. Mirrors `verify_local` / `verify_local_compact` in +// meta-watch's src/record/sign.rs — same curve, same digest, both signature +// encodings. + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "strings" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa" + "github.com/mr-tron/base58" +) + +// ChallengeDomain prefixes every byte string that authorises an account +// operation. POST /api/identity/sign and /sign-batch refuse payloads carrying +// it, so the only way to produce one is to hold the key yourself. +// +// Versioned because it is a wire commitment: changing it invalidates every +// challenge in flight, which is harmless, but changing it *silently* would let +// an old-domain payload keep working. Bump it deliberately. +const ChallengeDomain = "metamesh-identity-v1:" + +// Action is the operation a challenge authorises. A challenge minted to reveal +// a key must not also delete the account, so the action is inside the signed +// bytes and is re-checked at redemption. +type Action string + +const ( + ActionReveal Action = "reveal" + ActionDelete Action = "delete" +) + +// ValidAction reports whether a is an action a challenge may be issued for. +func ValidAction(a Action) bool { + return a == ActionReveal || a == ActionDelete +} + +// ValidUID exposes the uid shape check to callers outside this package (the +// API layer, which must reject a hostile uid before it reaches a Redis SCAN +// pattern or a file path). +func ValidUID(uid string) bool { return validUID(uid) } + +// IsChallengePayload reports whether b is (or claims to be) an account +// authorisation payload. Used by the signing endpoints to refuse minting one. +// +// Deliberately a prefix test on raw bytes rather than a parse: anything that +// merely *looks* like a challenge is refused, so a malformed or future-version +// payload fails closed instead of being signed. +func IsChallengePayload(b []byte) bool { + return strings.HasPrefix(string(b), ChallengeDomain) +} + +// PubkeyFromUID recovers the compressed public key a uid encodes. +func PubkeyFromUID(uid string) (*secp256k1.PublicKey, error) { + if !validUID(uid) { + return nil, fmt.Errorf("invalid uid: %q", uid) + } + if !strings.HasPrefix(uid, "z") { + return nil, errors.New("uid is not multibase base58btc (missing 'z' prefix)") + } + raw, err := base58.Decode(uid[1:]) + if err != nil { + return nil, fmt.Errorf("uid is not valid base58: %w", err) + } + if len(raw) != 33 { + return nil, fmt.Errorf("uid decodes to %d bytes, want a 33-byte compressed pubkey", len(raw)) + } + pub, err := secp256k1.ParsePubKey(raw) + if err != nil { + return nil, fmt.Errorf("uid is not a point on secp256k1: %w", err) + } + return pub, nil +} + +// Verify checks sig over SHA-256(payload) against the public key inside uid. +// +// Accepts both encodings in circulation: DER (what Sign emits, and what User +// Data Layer records carry) and 64-byte compact r‖s (what the browser emits — +// noble-secp256k1 v3 dropped DER, and meta-watch's sign-in already sends +// compact). Rejecting either would break one of the two callers. +func Verify(uid string, payload, sig []byte) error { + pub, err := PubkeyFromUID(uid) + if err != nil { + return err + } + parsed, err := parseSignature(sig) + if err != nil { + return err + } + digest := sha256.Sum256(payload) + if !parsed.Verify(digest[:], pub) { + return errors.New("signature does not verify for this uid") + } + return nil +} + +func parseSignature(sig []byte) (*ecdsa.Signature, error) { + if len(sig) == 64 { + var r, s secp256k1.ModNScalar + // SetByteSlice reports overflow (>= curve order), which is not a valid + // scalar. A zero r or s is likewise never produced by a real signer. + if r.SetByteSlice(sig[:32]) || r.IsZero() { + return nil, errors.New("compact signature has an invalid r") + } + if s.SetByteSlice(sig[32:]) || s.IsZero() { + return nil, errors.New("compact signature has an invalid s") + } + return ecdsa.NewSignature(&r, &s), nil + } + parsed, err := ecdsa.ParseDERSignature(sig) + if err != nil { + return nil, fmt.Errorf("signature is neither 64-byte compact nor valid DER: %w", err) + } + return parsed, nil +} + +// NewChallengeText mints the literal string a client signs. The uid and action +// are inside it, so a signature cannot be lifted from one account or operation +// onto another even if the nonce leaks. +func NewChallengeText(uid string, action Action) (string, error) { + nonce := make([]byte, 32) + if _, err := rand.Read(nonce); err != nil { + return "", fmt.Errorf("read nonce: %w", err) + } + return ChallengeDomain + string(action) + ":" + uid + ":" + hex.EncodeToString(nonce), nil +} + +// ChallengeMatches reports whether text is a well-formed challenge for exactly +// this uid and action. The store already keys on the full text, so this is a +// belt-and-braces re-read of the signed bytes themselves: it is the signed +// content, not the lookup key, that a verifier must agree with. +// +// Compared in constant time — the text is a bearer secret until redeemed. +func ChallengeMatches(text, uid string, action Action) bool { + want := ChallengeDomain + string(action) + ":" + uid + ":" + if len(text) <= len(want) { + return false + } + return subtle.ConstantTimeCompare([]byte(text[:len(want)]), []byte(want)) == 1 +} diff --git a/internal/identity/verify_test.go b/internal/identity/verify_test.go new file mode 100644 index 0000000..97581ba --- /dev/null +++ b/internal/identity/verify_test.go @@ -0,0 +1,278 @@ +package identity + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + "time" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa" +) + +// The whole gate rests on a uid being the public key: verification consults no +// stored state, so a signature is checked against the account the caller names +// rather than against whatever key meta-core happens to hold. +func TestVerify_AcceptsDERFromOurOwnSigner(t *testing.T) { + id, err := Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + payload := []byte(ChallengeDomain + "delete:" + id.UID + ":abcdef") + + sig, err := Sign(id, payload) + if err != nil { + t.Fatalf("sign: %v", err) + } + if err := Verify(id.UID, payload, sig); err != nil { + t.Fatalf("DER signature from Sign must verify: %v", err) + } +} + +// The browser is the signer that matters here, and noble-secp256k1 v3 emits +// 64-byte compact r‖s — it dropped DER entirely. A verifier that took only DER +// would reject every real proof while passing this package's own tests. +func TestVerify_AcceptsCompactAsTheBrowserEmitsIt(t *testing.T) { + id, err := Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + payload := []byte("some challenge text") + + raw, _ := hex.DecodeString(id.PrivateKeyHex) + priv := secp256k1.PrivKeyFromBytes(raw) + digest := sha256.Sum256(payload) + der := ecdsa.Sign(priv, digest[:]) + + // Re-encode the same signature compactly, the way the browser would. + parsed, err := ecdsa.ParseDERSignature(der.Serialize()) + if err != nil { + t.Fatalf("parse own DER: %v", err) + } + r := parsed.R() + sc := parsed.S() + compact := make([]byte, 0, 64) + rb := r.Bytes() + sb := sc.Bytes() + compact = append(compact, rb[:]...) + compact = append(compact, sb[:]...) + + if err := Verify(id.UID, payload, compact); err != nil { + t.Fatalf("compact signature must verify: %v", err) + } +} + +func TestVerify_RejectsAnotherAccountsSignature(t *testing.T) { + alice, _ := Generate() + mallory, _ := Generate() + payload := []byte("challenge for alice") + + sig, err := Sign(mallory, payload) + if err != nil { + t.Fatalf("sign: %v", err) + } + if err := Verify(alice.UID, payload, sig); err == nil { + t.Fatal("a signature by mallory must not verify as alice") + } +} + +func TestVerify_RejectsTamperedPayload(t *testing.T) { + id, _ := Generate() + sig, _ := Sign(id, []byte("delete account A")) + if err := Verify(id.UID, []byte("delete account B"), sig); err == nil { + t.Fatal("a signature must not verify over different bytes") + } +} + +func TestVerify_RejectsGarbageSignatureEncodings(t *testing.T) { + id, _ := Generate() + for name, sig := range map[string][]byte{ + "empty": {}, + "short": {1, 2, 3}, + "zero compact": make([]byte, 64), + "not der": []byte("this is not a signature at all......."), + "63 bytes": make([]byte, 63), + "65 bytes non-der": make([]byte, 65), + } { + if err := Verify(id.UID, []byte("x"), sig); err == nil { + t.Errorf("%s: expected rejection", name) + } + } +} + +func TestPubkeyFromUID_RejectsMalformedUIDs(t *testing.T) { + id, _ := Generate() + for name, uid := range map[string]string{ + "empty": "", + "no z prefix": id.UID[1:], + "truncated": id.UID[:20], + "path traversal": "../../etc/passwd", + "with slash": "z/" + id.UID, + "not on curve": "z" + strings.Repeat("1", 45), + } { + if _, err := PubkeyFromUID(uid); err == nil { + t.Errorf("%s (%q): expected rejection", name, uid) + } + } + if _, err := PubkeyFromUID(id.UID); err != nil { + t.Fatalf("a real uid must parse: %v", err) + } +} + +// The refusal that makes proof-of-possession mean anything on a node holding +// every private key. If this ever returns false for a challenge, an attacker +// asks /api/identity/sign to authorise their own delete. +func TestIsChallengePayload_CatchesAnythingInTheDomain(t *testing.T) { + yes := [][]byte{ + []byte(ChallengeDomain), + []byte(ChallengeDomain + "delete:zAlice:beef"), + []byte(ChallengeDomain + "some-future-action:whatever"), + []byte("metamesh-identity-v1:"), // literal, in case the const drifts + } + for _, b := range yes { + if !IsChallengePayload(b) { + t.Errorf("must refuse to sign %q", b) + } + } + no := [][]byte{ + {}, + []byte("metamesh-identity-v2:delete:zAlice"), // different domain, not ours to refuse + []byte("a record payload"), + []byte(" " + ChallengeDomain), // not a prefix + } + for _, b := range no { + if IsChallengePayload(b) { + t.Errorf("must still sign %q", b) + } + } +} + +func TestChallengeMatches_TiesTheTextToUIDAndAction(t *testing.T) { + text, err := NewChallengeText("zAlice", ActionDelete) + if err != nil { + t.Fatalf("mint: %v", err) + } + if !ChallengeMatches(text, "zAlice", ActionDelete) { + t.Fatal("its own uid+action must match") + } + if ChallengeMatches(text, "zBob", ActionDelete) { + t.Fatal("another uid must not match") + } + if ChallengeMatches(text, "zAlice", ActionReveal) { + t.Fatal("another action must not match") + } + // A prefix-only uid must not slip through: "zAlic" is a prefix of "zAlice", + // and a naive HasPrefix on the uid alone would accept it. + if ChallengeMatches(text, "zAlic", ActionDelete) { + t.Fatal("a uid that is a prefix of the real one must not match") + } +} + +func TestNewChallengeText_IsUnpredictable(t *testing.T) { + seen := make(map[string]struct{}, 64) + for i := 0; i < 64; i++ { + text, err := NewChallengeText("zAlice", ActionDelete) + if err != nil { + t.Fatalf("mint: %v", err) + } + if _, dup := seen[text]; dup { + t.Fatal("challenge repeated — the nonce is not random") + } + seen[text] = struct{}{} + } +} + +// --- challenge store -------------------------------------------------------- + +func TestChallengeStore_RedeemIsSingleUse(t *testing.T) { + s := NewChallengeStore() + text, _, err := s.Issue("zAlice", ActionDelete) + if err != nil { + t.Fatalf("issue: %v", err) + } + if err := s.Redeem(text, "zAlice", ActionDelete); err != nil { + t.Fatalf("first redeem: %v", err) + } + if err := s.Redeem(text, "zAlice", ActionDelete); err == nil { + t.Fatal("a challenge must not be redeemable twice — that is replay") + } +} + +// A challenge minted to *look at* a key must not delete the account, and one +// minted for Alice must not act on Bob. Both are inside the signed text, so a +// store that ignored them would authorise the wrong operation with a perfectly +// valid signature. +func TestChallengeStore_RedeemRefusesWrongUIDOrAction(t *testing.T) { + s := NewChallengeStore() + + text, _, _ := s.Issue("zAlice", ActionReveal) + if err := s.Redeem(text, "zAlice", ActionDelete); err == nil { + t.Fatal("a reveal challenge must not authorise a delete") + } + + text2, _, _ := s.Issue("zAlice", ActionDelete) + if err := s.Redeem(text2, "zBob", ActionDelete); err == nil { + t.Fatal("alice's challenge must not authorise deleting bob") + } +} + +// A failed redemption still consumes the challenge, so a wrong guess costs a +// round trip rather than leaving the nonce up for another attempt. +func TestChallengeStore_FailedRedeemStillBurnsIt(t *testing.T) { + s := NewChallengeStore() + text, _, _ := s.Issue("zAlice", ActionDelete) + _ = s.Redeem(text, "zBob", ActionDelete) + if err := s.Redeem(text, "zAlice", ActionDelete); err == nil { + t.Fatal("a burnt challenge must stay burnt for its rightful owner too") + } +} + +func TestChallengeStore_Expires(t *testing.T) { + s := NewChallengeStore() + now := time.Unix(1_700_000_000, 0) + s.now = func() time.Time { return now } + + text, expires, _ := s.Issue("zAlice", ActionDelete) + if !expires.Equal(now.Add(ChallengeTTL)) { + t.Fatalf("expiry %v, want %v", expires, now.Add(ChallengeTTL)) + } + now = now.Add(ChallengeTTL + time.Second) + if err := s.Redeem(text, "zAlice", ActionDelete); err == nil { + t.Fatal("an expired challenge must not redeem") + } +} + +// Issuing is unauthenticated, so the map is attacker-growable. The cap is what +// stops that being a memory-exhaustion lever; expiry sweeping is what stops the +// cap being reached by honest traffic. +func TestChallengeStore_CapsInFlightAndSweepsExpired(t *testing.T) { + s := NewChallengeStore() + now := time.Unix(1_700_000_000, 0) + s.now = func() time.Time { return now } + + for i := 0; i < maxChallenges; i++ { + if _, _, err := s.Issue("zAlice", ActionDelete); err != nil { + t.Fatalf("issue %d: %v", i, err) + } + } + if _, _, err := s.Issue("zAlice", ActionDelete); err == nil { + t.Fatal("expected the store to refuse once full") + } + + // Once they age out, the next issue sweeps and succeeds again. + now = now.Add(ChallengeTTL + time.Second) + if _, _, err := s.Issue("zAlice", ActionDelete); err != nil { + t.Fatalf("after expiry the store must accept again: %v", err) + } +} + +func TestChallengeStore_RefusesInvalidInputs(t *testing.T) { + s := NewChallengeStore() + if _, _, err := s.Issue("../etc", ActionDelete); err == nil { + t.Fatal("a uid that is not uid-shaped must be refused") + } + if _, _, err := s.Issue("zAlice", Action("sudo")); err == nil { + t.Fatal("an unknown action must be refused") + } +} diff --git a/internal/storage/udl_admin.go b/internal/storage/udl_admin.go new file mode 100644 index 0000000..5c7f521 --- /dev/null +++ b/internal/storage/udl_admin.go @@ -0,0 +1,245 @@ +package storage + +// Account-scoped UDL administration: what a uid holds, and removing all of it. +// +// Both operations enumerate the same thing — every `udl:rec://` +// cell — because there is no per-user cid index to consult. The Redis model +// (see udl.go) indexes user+key → cids and user+cid → keys, which answer +// "what is in My List", not "everything this account ever wrote". Adding a +// third index would put a write on every UDL upsert to serve two admin calls, +// so these SCAN instead and accept being O(keyspace). +// +// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid +// one: the accounts page needs a number for every account, and MATCH filters +// server-side without indexing, so N per-uid calls would walk the keyspace N +// times. One walk buckets them all. + +import ( + "context" + "fmt" + "strings" + "time" +) + +// UDLUserStats counts what one account holds. Records is the number of cells, +// including tombstones — a retracted rating is still a row this account owns +// and still something a purge has to remove. +type UDLUserStats struct { + Records int `json:"records"` + Cids int `json:"cids"` + Keys int `json:"keys"` +} + +// udlScanBatch is the SCAN COUNT hint, matching the rest of this package. +const udlScanBatch = 1000 + +// safeUIDForPattern rejects anything that would change the meaning of a SCAN +// glob or reach across the key structure. meta-core's uids are alphanumeric +// (identity.validUID), so this only ever fires on a hostile or malformed one — +// but it fires here, in the layer that builds the pattern, rather than relying +// on every caller having validated first. +func safeUIDForPattern(uid string) error { + if uid == "" { + return fmt.Errorf("empty uid") + } + if strings.ContainsAny(uid, "*?[]\\/:") { + return fmt.Errorf("uid contains characters that are not valid in a uid: %q", uid) + } + return nil +} + +// splitUDLRecSuffix splits the "//" tail of a udl:rec key. +// +// SplitN with a limit of 3 so a key containing a slash survives intact. cids +// never contain one; keys are a controlled vocabulary today, but a namespaced +// one arriving later must not silently truncate here — a truncated key would +// make the purge SREM the wrong index member and leave the real one behind. +func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) { + parts := strings.SplitN(suffix, "/", 3) + if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return "", "", "", false + } + return parts[0], parts[1], parts[2], true +} + +// scanKeys walks the keyspace for pattern and returns every match. Caller holds +// the lock. +// +// SCAN can return the same key more than once across cursor iterations (it +// guarantees no misses, not no duplicates), so callers that count must +// de-duplicate — both callers below collect into maps or sets for that reason. +func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) { + var ( + out []string + cursor uint64 + ) + for { + keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result() + if err != nil { + return nil, fmt.Errorf("scan %q: %w", pattern, err) + } + out = append(out, keys...) + cursor = next + if cursor == 0 { + return out, nil + } + } +} + +// UDLUserCells returns every (cid, key) pair this account holds a cell for. +// De-duplicated, since SCAN may repeat a key. +func (c *Client) UDLUserCells(uid string) (map[string]string, error) { + // Keyed by the full "cid/key" tail so the pair stays a pair; the value is + // the cid, which the purge needs grouped. + if err := safeUIDForPattern(uid); err != nil { + return nil, err + } + + c.mu.RLock() + defer c.mu.RUnlock() + if c.client == nil { + return nil, fmt.Errorf("not connected") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + prefix := c.buildKey("udl:rec:") + keys, err := c.scanKeys(ctx, prefix+uid+"/*") + if err != nil { + return nil, err + } + out := make(map[string]string, len(keys)) + for _, full := range keys { + gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix)) + // The glob can only match this uid, but a uid that is a prefix of + // another would still be caught by the '/' in the pattern; check anyway + // so a malformed key can never be attributed to the wrong account. + if !ok || gotUID != uid { + continue + } + out[cid+"/"+key] = cid + } + return out, nil +} + +// UDLAllUserStats counts every account's cells in one keyspace walk. +// +// Returns only accounts that actually hold records; a uid absent from the map +// holds none. Callers render a zero for those rather than treating it as an +// error — an account created a minute ago legitimately has nothing. +func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if c.client == nil { + return nil, fmt.Errorf("not connected") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + prefix := c.buildKey("udl:rec:") + keys, err := c.scanKeys(ctx, prefix+"*") + if err != nil { + return nil, err + } + + // Sets rather than counters: SCAN may hand back the same key twice, which + // would otherwise inflate every number on the page. + type acc struct { + cells map[string]struct{} + cids map[string]struct{} + keys map[string]struct{} + } + byUID := make(map[string]*acc) + for _, full := range keys { + uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix)) + if !ok { + continue + } + a := byUID[uid] + if a == nil { + a = &acc{ + cells: make(map[string]struct{}), + cids: make(map[string]struct{}), + keys: make(map[string]struct{}), + } + byUID[uid] = a + } + a.cells[cid+"/"+key] = struct{}{} + a.cids[cid] = struct{}{} + a.keys[key] = struct{}{} + } + + out := make(map[string]UDLUserStats, len(byUID)) + for uid, a := range byUID { + out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)} + } + return out, nil +} + +// UDLPurgeUser removes every trace of one account from the User Data Layer: +// its cells, its two user-scoped indexes, and its membership in the shared +// cid-scoped indexes. Returns what was removed. +// +// The shared indexes are the part that is easy to miss and expensive to leave. +// `udl:idx:cid::key:` and `udl:idx:cid::users` are keyed by cid, +// not by uid, so a purge that only deleted `udl:rec:/*` would leave this +// uid listed as an opinion-holder on every title it ever touched. Aggregation +// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the +// rating average stays right, but uids_for_cid keeps naming a deleted account +// forever and every aggregate read pays for the lookup. +// +// Idempotent: purging an account with nothing stored is a no-op returning +// zeroes, so a retry after a partial failure is safe. +func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) { + cells, err := c.UDLUserCells(uid) + if err != nil { + return UDLUserStats{}, err + } + + c.mu.Lock() + defer c.mu.Unlock() + if c.client == nil { + return UDLUserStats{}, fmt.Errorf("not connected") + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + stats := UDLUserStats{Records: len(cells)} + distinctCids := make(map[string]struct{}) + distinctKeys := make(map[string]struct{}) + + pipe := c.client.Pipeline() + for cidKey, cid := range cells { + key := strings.TrimPrefix(cidKey, cid+"/") + distinctCids[cid] = struct{}{} + distinctKeys[key] = struct{}{} + pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key))) + pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid) + } + for cid := range distinctCids { + pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid) + } + stats.Cids = len(distinctCids) + stats.Keys = len(distinctKeys) + + // The two user-scoped indexes are enumerable directly, and are dropped + // whole rather than member-by-member. Scanning them separately also catches + // index entries whose cell is already gone — exactly the state a previously + // half-finished purge leaves behind. + idxPrefix := c.buildKey("udl:idx:user:") + uid + ":" + idxKeys, err := c.scanKeys(ctx, idxPrefix+"*") + if err != nil { + return UDLUserStats{}, err + } + for _, k := range idxKeys { + pipe.Del(ctx, k) + } + + if _, err := pipe.Exec(ctx); err != nil { + return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err) + } + return stats, nil +} diff --git a/internal/storage/udl_admin_test.go b/internal/storage/udl_admin_test.go new file mode 100644 index 0000000..c736e90 --- /dev/null +++ b/internal/storage/udl_admin_test.go @@ -0,0 +1,253 @@ +package storage + +import ( + "testing" +) + +// seedCell writes one live UDL cell the way a real client would. +func seedCell(t *testing.T, c *Client, uid, cid, key string) { + t.Helper() + if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok { + t.Fatalf("seed %s/%s/%s: ok=%v err=%v", uid, cid, key, ok, err) + } +} + +// The purge has to reach the cid-scoped indexes, not just the user's own rows. +// +// This is the failure that motivated the whole endpoint: deleting an account +// while leaving `udl:idx:cid::users` intact means every aggregate read on +// every title that account ever touched keeps naming a uid nobody can resolve, +// forever — and the rows are unreachable through any API, so nothing will ever +// clean them up. +func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "cidOne", "like") + seedCell(t, c, "zAlice", "cidOne", "rating") + seedCell(t, c, "zAlice", "cidTwo", "seek") + + stats, err := c.UDLPurgeUser("zAlice") + if err != nil { + t.Fatalf("purge: %v", err) + } + if stats.Records != 3 || stats.Cids != 2 || stats.Keys != 3 { + t.Fatalf("stats = %+v, want 3 records / 2 cids / 3 keys", stats) + } + + // Cells gone. + if _, ok, err := c.UDLGetCell("zAlice", "cidOne", "like"); err != nil || ok { + t.Fatalf("cell survived the purge: ok=%v err=%v", ok, err) + } + // User-scoped indexes gone. + if entries, err := c.UDLListUserKey("zAlice", "like"); err != nil || len(entries) != 0 { + t.Fatalf("user+key index survived: %d entries, err=%v", len(entries), err) + } + if entries, err := c.UDLListUserCid("zAlice", "cidOne"); err != nil || len(entries) != 0 { + t.Fatalf("user+cid index survived: %d entries, err=%v", len(entries), err) + } + // Cid-scoped index gone — the one that is easy to miss. + for _, cid := range []string{"cidOne", "cidTwo"} { + uids, err := c.UDLCidUsers(cid) + if err != nil { + t.Fatalf("cid users %s: %v", cid, err) + } + for _, u := range uids { + if u == "zAlice" { + t.Fatalf("%s still lists the deleted account in uids_for_cid", cid) + } + } + } +} + +// Purging one account must not touch another's rows, including on a cid they +// both have an opinion about — where the two share an index set and a careless +// DEL of the whole key would take the survivor's membership with it. +func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "shared", "rating") + seedCell(t, c, "zBob", "shared", "rating") + + if _, err := c.UDLPurgeUser("zAlice"); err != nil { + t.Fatalf("purge: %v", err) + } + + if _, ok, err := c.UDLGetCell("zBob", "shared", "rating"); err != nil || !ok { + t.Fatalf("bob's cell must survive alice's purge: ok=%v err=%v", ok, err) + } + uids, err := c.UDLCidUsers("shared") + if err != nil { + t.Fatalf("cid users: %v", err) + } + if len(uids) != 1 || uids[0] != "zBob" { + t.Fatalf("uids_for_cid = %v, want exactly [zBob]", uids) + } + entries, err := c.UDLListUserKey("zBob", "rating") + if err != nil || len(entries) != 1 { + t.Fatalf("bob's user+key index must survive: %d entries, err=%v", len(entries), err) + } +} + +// A uid that is a prefix of another must not drag its neighbour's rows out. +// The key layout separates them with '/' and ':', so this passes only as long +// as the scan patterns keep those separators. +func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "cidOne", "like") + seedCell(t, c, "zAliceExtra", "cidOne", "like") + + stats, err := c.UDLPurgeUser("zAlice") + if err != nil { + t.Fatalf("purge: %v", err) + } + if stats.Records != 1 { + t.Fatalf("purged %d records, want exactly alice's 1", stats.Records) + } + if _, ok, err := c.UDLGetCell("zAliceExtra", "cidOne", "like"); err != nil || !ok { + t.Fatalf("zAliceExtra's cell must survive: ok=%v err=%v", ok, err) + } +} + +// Tombstoned cells still belong to the account. They are absent from the +// user+key index by design (see udl.go), so a purge that enumerated *that* +// index instead of the cells would leave every retracted rating behind — the +// rows most likely to have accumulated over an account's life. +func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "cidOne", "like") + if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok { + t.Fatalf("tombstone: ok=%v err=%v", ok, err) + } + // Precondition: the tombstone left the cell but dropped it from the index. + if entries, _ := c.UDLListUserKey("zAlice", "like"); len(entries) != 0 { + t.Fatalf("precondition: tombstone should be out of the user+key index") + } + + stats, err := c.UDLPurgeUser("zAlice") + if err != nil { + t.Fatalf("purge: %v", err) + } + if stats.Records != 1 { + t.Fatalf("purged %d records, want the 1 tombstoned cell", stats.Records) + } + if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); ok { + t.Fatal("tombstoned cell survived the purge") + } +} + +// Retry-safe: the delete handler purges before removing the key, so a caller +// that retries after a partial failure runs this a second time. +func TestUDLPurgeUser_IsIdempotent(t *testing.T) { + c, _ := newTestClient(t, "") + seedCell(t, c, "zAlice", "cidOne", "like") + + if _, err := c.UDLPurgeUser("zAlice"); err != nil { + t.Fatalf("first purge: %v", err) + } + stats, err := c.UDLPurgeUser("zAlice") + if err != nil { + t.Fatalf("second purge must be a no-op, got: %v", err) + } + if stats.Records != 0 || stats.Cids != 0 { + t.Fatalf("second purge = %+v, want zeroes", stats) + } +} + +func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) { + c, _ := newTestClient(t, "") + seedCell(t, c, "zAlice", "cidOne", "like") + + // "*" would otherwise match every account in the keyspace — a wildcard + // purge dressed up as a uid. + for _, uid := range []string{"*", "z*", "", "z:Alice", "z/Alice"} { + if _, err := c.UDLPurgeUser(uid); err == nil { + t.Errorf("uid %q must be refused before it reaches a SCAN pattern", uid) + } + } + if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); !ok { + t.Fatal("a refused purge must not have deleted anything") + } +} + +func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "cidOne", "like") + seedCell(t, c, "zAlice", "cidOne", "rating") + seedCell(t, c, "zAlice", "cidTwo", "like") + seedCell(t, c, "zBob", "cidOne", "seek") + + stats, err := c.UDLAllUserStats() + if err != nil { + t.Fatalf("stats: %v", err) + } + if got := stats["zAlice"]; got.Records != 3 || got.Cids != 2 || got.Keys != 2 { + t.Fatalf("alice = %+v, want 3 records / 2 cids / 2 keys", got) + } + if got := stats["zBob"]; got.Records != 1 || got.Cids != 1 || got.Keys != 1 { + t.Fatalf("bob = %+v, want 1/1/1", got) + } + // An account with nothing stored is simply absent — the caller renders 0. + if _, present := stats["zNobody"]; present { + t.Fatal("an account with no records must not appear in the map") + } +} + +// The stats and the purge must agree: the number shown in the delete +// confirmation is the number of rows the delete then removes. +func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) { + c, _ := newTestClient(t, "") + + seedCell(t, c, "zAlice", "cidOne", "like") + seedCell(t, c, "zAlice", "cidTwo", "rating") + seedCell(t, c, "zAlice", "cidTwo", "seek") + + before, err := c.UDLAllUserStats() + if err != nil { + t.Fatalf("stats: %v", err) + } + purged, err := c.UDLPurgeUser("zAlice") + if err != nil { + t.Fatalf("purge: %v", err) + } + if before["zAlice"] != purged { + t.Fatalf("stats said %+v but purge removed %+v", before["zAlice"], purged) + } +} + +// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or +// every account on a prefixed node would be reported under a mangled name and +// the purge would find nothing. +func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) { + c, _ := newTestClient(t, "tenantA:") + + seedCell(t, c, "zAlice", "cidOne", "like") + + stats, err := c.UDLAllUserStats() + if err != nil { + t.Fatalf("stats: %v", err) + } + if got := stats["zAlice"]; got.Records != 1 { + t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats) + } + if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 { + t.Fatalf("purge under a prefix: %+v err=%v", s, err) + } +} + +func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) { + // A namespaced key arriving later must not be truncated: the purge SREMs + // `udl:idx:cid::key:`, so a half-read key removes the wrong + // member and leaves the real one behind. + uid, cid, key, ok := splitUDLRecSuffix("zAlice/cidOne/ns/sub:key") + if !ok || uid != "zAlice" || cid != "cidOne" || key != "ns/sub:key" { + t.Fatalf("got (%q,%q,%q,%v)", uid, cid, key, ok) + } + for _, bad := range []string{"", "zAlice", "zAlice/cidOne", "zAlice//key", "/cid/key"} { + if _, _, _, ok := splitUDLRecSuffix(bad); ok { + t.Errorf("%q should not parse", bad) + } + } +}