add identity management

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

README.md

@@ -138,7 +138,7 @@ Environment variables consumed by `internal/config/config.go`:
 | `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
 | `REDIS_PORT` | `6379` | Local Redis port. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
-| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind. |
+| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
 | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
 | `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
 | `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |

dashboard/src/pages/Identity.tsx

@@ -1,21 +1,43 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 
-// Status returned by GET /api/identity. Private key is never present here —
-// it is shown exactly once at generation time (see GeneratedKeyPanel).
-interface IdentityStatus {
-  hasIdentity: boolean;
-  uid?: string;
-  curve?: string;
-  createdAt?: number;
-}
+// Admin console for meta-core's account keystore.
+//
+// meta-core stores one secp256k1 keypair per file under
+// {META_CORE_PATH}/identity/accounts/<uid>.json and has NO notion of a current
+// user — which account is "you" is decided by whichever client signs in
+// (meta-watch). So this page lists accounts; it never speaks of an active one.
+//
+// The private key is returned by exactly two calls: POST /api/identity/generate
+// (once, at creation) and POST /api/identity/reveal (on demand, for an account
+// that already exists). Both land in the same exclusive SecretKeyPanel.
 
-interface GeneratedKey {
+interface Account {
   uid: string;
   curve: string;
   createdAt: number;
-  privateKeyHex: string;
 }
 
+// A secret on screen. Carries no createdAt: /api/identity/reveal answers with
+// {uid, curve, privateKeyHex} only.
+interface SecretKeyView {
+  uid: string;
+  privateKeyHex: string;
+  kind: 'generated' | 'revealed';
+}
+
+// Which row has an open confirmation, and for what. One discriminated value
+// rather than a pair of ids, so at most one confirm panel can ever be open.
+interface Pending {
+  uid: string;
+  kind: 'remove' | 'reveal';
+}
+
+// meta-watch's sentinel cell for profile records: display name and avatar are
+// about the *user*, not a content file, so they live at (uid, "self", ...).
+// meta-core stores them opaquely and knows nothing of the convention.
+const PROFILE_CID = 'self';
+const PROFILE_NAME_KEY = 'profile:name';
+
 const cardStyle: React.CSSProperties = {
   background: '#16213e',
   borderRadius: '8px',
@@ -59,6 +81,21 @@ const monoStyle: React.CSSProperties = {
   borderRadius: '4px',
 };
 
+const dangerPanelStyle: React.CSSProperties = {
+  background: '#1f0a0a',
+  border: '1px solid #7f1d1d',
+  padding: '0.75rem 1rem',
+  borderRadius: '4px',
+  marginTop: '0.75rem',
+};
+
+const errorTextStyle: React.CSSProperties = {
+  color: '#f87171',
+  background: '#1f0a0a',
+  padding: '0.5rem 0.75rem',
+  borderRadius: '4px',
+};
+
 function formatCreatedAt(unixSec?: number): string {
   if (!unixSec) return '';
   try {
@@ -79,25 +116,76 @@ async function errorMessage(res: Response, fallback: string): Promise<string> {
   return `${fallback} (HTTP ${res.status})`;
 }
 
-function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss: () => void }) {
-  const [copied, setCopied] = useState(false);
-  const copy = async () => {
+// Copy text, falling back to selecting the element that displays it.
+//
+// The fallback is load-bearing, not politeness: the dashboard is served over
+// plain HTTP, where navigator.clipboard is undefined — without it the copy
+// buttons do nothing at all on a real deployment.
+async function copyToClipboard(text: string, fallbackElementId: string): Promise<void> {
   try {
-      await navigator.clipboard.writeText(data.privateKeyHex);
-      setCopied(true);
-      setTimeout(() => setCopied(false), 2000);
+    await navigator.clipboard.writeText(text);
   } catch {
-      // Clipboard API blocked (non-HTTPS context, etc.) — fall back to selection.
     const sel = window.getSelection();
-      const range = document.createRange();
-      const el = document.getElementById('mm-privkey-box');
+    const el = document.getElementById(fallbackElementId);
     if (sel && el) {
+      const range = document.createRange();
       range.selectNodeContents(el);
       sel.removeAllRanges();
       sel.addRange(range);
     }
   }
+}
+
+async function fetchAccounts(): Promise<Account[]> {
+  const res = await fetch('/api/identity/accounts');
+  if (!res.ok) throw new Error(await errorMessage(res, 'Failed to load accounts'));
+  const data = (await res.json()) as { accounts?: Account[] };
+  // Backend order is CreatedAt ascending then uid (identity.List) — stable, so
+  // render it as given rather than re-sorting here.
+  return data.accounts ?? [];
+}
+
+// Resolve an account's display name from the User Data Layer, or null.
+//
+// Never throws and never reports an error upward: a name is decoration on top
+// of the uid, and the account list must render identically whether Redis is up
+// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
+// — meta-core stores the plaintext beside the signed record and verifies
+// nothing — so the uid stays on screen as the real identity, and the length cap
+// bounds what a hostile writer can do to the layout.
+async function fetchDisplayName(uid: string): Promise<string | null> {
+  const url =
+    `/api/udl/record?uid=${encodeURIComponent(uid)}` +
+    `&cid=${encodeURIComponent(PROFILE_CID)}` +
+    `&key=${encodeURIComponent(PROFILE_NAME_KEY)}`;
+  try {
+    const res = await fetch(url);
+    if (!res.ok) return null; // 503 when storage is down, 500, …
+    const data = (await res.json()) as { exists?: boolean; value?: unknown };
+    // Key off `value`, not `exists`: clearing a name writes a tombstone, which
+    // leaves the cell existing with no plaintext. Private-tier cells look the
+    // same, which is exactly right — there is nothing to show for either.
+    if (typeof data.value !== 'string') return null;
+    const name = data.value.trim();
+    return name ? name.slice(0, 64) : null;
+  } catch {
+    return null;
+  }
+}
+
+// The one place a private key is ever displayed. Rendered exclusively (the
+// account list is hidden while it is open) so the "save it now" contract of a
+// freshly generated key is not competing with anything, and so the hardcoded
+// element id below stays unique.
+function SecretKeyPanel({ data, onDismiss }: { data: SecretKeyView; onDismiss: () => void }) {
+  const [copied, setCopied] = useState(false);
+
+  const copy = async () => {
+    await copyToClipboard(data.privateKeyHex, 'mm-privkey-box');
+    setCopied(true);
+    setTimeout(() => setCopied(false), 2000);
   };
+
   return (
     <div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
       <div
@@ -109,9 +197,19 @@ function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss:
           marginBottom: '1rem',
         }}
       >
+        {data.kind === 'generated' ? (
+          <>
             <strong>This private key is shown ONCE.</strong> Save it now in a password manager.
-        It will never be displayed again — there is no recovery path. Losing it means losing access
-        to every record signed under this identity.
+            You can ask for it again later with <em>Reveal private key</em>, but only for as long
+            as this account exists on this node — delete it and the key is gone for good.
+          </>
+        ) : (
+          <>
+            <strong>This is the live private key for an existing account.</strong> Anyone holding
+            it can sign as this uid forever, on any machine. There is no revocation and no
+            rotation. Do not paste it into chat, tickets or screenshots.
+          </>
+        )}
       </div>
       <p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
       <div style={monoStyle}>{data.uid}</div>
@@ -124,36 +222,40 @@ function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss:
           {copied ? 'Copied' : 'Copy private key'}
         </button>
         <button style={dangerButtonStyle} onClick={onDismiss}>
-          I have saved it
+          {data.kind === 'generated' ? 'I have saved it' : 'Done — hide it'}
         </button>
       </div>
     </div>
   );
 }
 
-function NoIdentityPanel({
+// Create / import. Always available, not just on an empty keystore: this is an
+// admin console, so adding an account to a populated node has to be possible.
+function NewAccountPanel({
   onGenerated,
   onImported,
 }: {
-  onGenerated: (g: GeneratedKey) => void;
+  onGenerated: (s: SecretKeyView) => void;
   onImported: () => void;
 }) {
   const [busy, setBusy] = useState(false);
   const [importing, setImporting] = useState(false);
   const [importHex, setImportHex] = useState('');
   const [error, setError] = useState<string | null>(null);
+  const [notice, setNotice] = useState<string | null>(null);
 
   const generate = async () => {
     setBusy(true);
     setError(null);
+    setNotice(null);
     try {
       const res = await fetch('/api/identity/generate', { method: 'POST' });
       if (!res.ok) {
         setError(await errorMessage(res, 'Generate failed'));
         return;
       }
-      const data = (await res.json()) as GeneratedKey;
-      onGenerated(data);
+      const data = (await res.json()) as { uid: string; privateKeyHex: string };
+      onGenerated({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'generated' });
     } catch (e) {
       setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
     } finally {
@@ -165,6 +267,7 @@ function NoIdentityPanel({
     e.preventDefault();
     setBusy(true);
     setError(null);
+    setNotice(null);
     try {
       const res = await fetch('/api/identity/import', {
         method: 'POST',
@@ -175,6 +278,15 @@ function NoIdentityPanel({
         setError(await errorMessage(res, 'Import failed'));
         return;
       }
+      // Import is idempotent — presenting the key IS the proof of ownership, so
+      // a key this node already holds is a success, not a conflict. `created`
+      // only decides the wording.
+      const data = (await res.json()) as { created?: boolean };
+      setNotice(
+        data.created
+          ? 'Imported — a new account was added to this keystore.'
+          : 'That key was already in this keystore; nothing changed.'
+      );
       setImportHex('');
       setImporting(false);
       onImported();
@@ -187,15 +299,16 @@ function NoIdentityPanel({
 
   return (
     <div style={cardStyle}>
-      <h3 style={{ marginTop: 0 }}>No identity yet</h3>
-      <p style={{ color: '#9ca3af' }}>
-        The User Data Layer needs a signing key for your ratings, watch progress, and "liked" flags
-        to be attributed to you across devices. Nothing happens automatically — generate one below,
-        or import a key you already have.
+      <h3 style={{ marginTop: 0 }}>Add an account</h3>
+      <p style={{ color: '#9ca3af', marginTop: 0 }}>
+        Generating mints a fresh secp256k1 keypair here. Importing adds a key created elsewhere —
+        it is also how an existing account is moved onto this node, so importing a key the
+        keystore already holds succeeds and changes nothing.
       </p>
-      {error && (
-        <p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
-          {error}
+      {error && <p style={errorTextStyle}>{error}</p>}
+      {notice && (
+        <p style={{ color: '#9ca3af', background: '#0a0a18', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
+          {notice}
         </p>
       )}
       <div>
@@ -235,71 +348,125 @@ function NoIdentityPanel({
   );
 }
 
-function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onReplace: () => void }) {
-  const [confirming, setConfirming] = useState(false);
-  const [busy, setBusy] = useState(false);
-  const [error, setError] = useState<string | null>(null);
+function AccountCard({
+  account,
+  name,
+  busy,
+  disabled,
+  pending,
+  error,
+  onAsk,
+  onCancel,
+  onReveal,
+  onRemove,
+}: {
+  account: Account;
+  name?: string;
+  busy: boolean;
+  disabled: boolean;
+  pending: Pending | null;
+  error: string | null;
+  onAsk: (p: Pending) => void;
+  onCancel: () => void;
+  onReveal: (uid: string) => void;
+  onRemove: (uid: string) => void;
+}) {
+  const [copied, setCopied] = useState(false);
+  const uidBoxId = `mm-uid-${account.uid}`;
+  const label = name ?? account.uid;
 
-  const remove = async () => {
-    setBusy(true);
-    setError(null);
-    try {
-      const res = await fetch('/api/identity?confirm=true', { method: 'DELETE' });
-      if (!res.ok && res.status !== 204) {
-        setError(await errorMessage(res, 'Delete failed'));
-        return;
-      }
-      onReplace();
-    } catch (e) {
-      setError(`Delete failed: ${e instanceof Error ? e.message : String(e)}`);
-    } finally {
-      setBusy(false);
-      setConfirming(false);
-    }
+  const copyUid = async () => {
+    await copyToClipboard(account.uid, uidBoxId);
+    setCopied(true);
+    setTimeout(() => setCopied(false), 2000);
   };
 
   return (
     <div style={cardStyle}>
-      <h3 style={{ marginTop: 0 }}>Active identity</h3>
-      <p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid:</p>
-      <div style={monoStyle}>{status.uid}</div>
-      <p style={{ color: '#888', marginTop: '1rem' }}>
-        Curve: {status.curve} · Created: {formatCreatedAt(status.createdAt)}
+      <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start', gap: '1rem' }}>
+        <div style={{ minWidth: 0, flex: 1 }}>
+          <h3 style={{ margin: '0 0 0.5rem', color: name ? '#fff' : '#888' }}>
+            {name ?? 'Unnamed account'}
+          </h3>
+          <div id={uidBoxId} style={monoStyle}>
+            {account.uid}
+          </div>
+          <p style={{ color: '#888', margin: '0.75rem 0 0', fontSize: '0.85rem' }}>
+            Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}
           </p>
-      <p style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
-        The private key lives at <code>/meta-core/identity/identity.json</code> on this volume and
-        is never returned through the API after generation. Other services on this stack (e.g.
-        meta-watch) sign records by calling <code>POST /api/identity/sign</code>.
-      </p>
-      {error && (
-        <p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
-          {error}
-        </p>
-      )}
-      {!confirming ? (
-        <button style={dangerButtonStyle} onClick={() => setConfirming(true)} disabled={busy}>
-          Replace identity
+        </div>
+        <div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem', flexShrink: 0 }}>
+          <button style={{ ...buttonStyle, marginRight: 0 }} onClick={copyUid} disabled={disabled}>
+            {copied ? 'Copied' : 'Copy uid'}
           </button>
-      ) : (
-        <div
-          style={{
-            background: '#1f0a0a',
-            border: '1px solid #7f1d1d',
-            padding: '0.75rem 1rem',
-            borderRadius: '4px',
-            marginTop: '0.5rem',
-          }}
+          <button
+            style={{ ...dangerButtonStyle, marginRight: 0 }}
+            onClick={() => onAsk({ uid: account.uid, kind: 'reveal' })}
+            disabled={disabled}
           >
-          <p style={{ color: '#fecaca', marginTop: 0 }}>
-            Replacing the identity deletes the current keypair from disk. Records signed by the
-            current uid will keep verifying for anyone who has them cached, but this node will no
-            longer be able to write under that uid. <strong>Make sure you have backed up the key
-            first.</strong>
-          </p>
-          <button style={dangerButtonStyle} onClick={remove} disabled={busy}>
-            {busy ? 'Deleting…' : 'Yes, delete identity'}
+            Reveal private key
           </button>
-          <button style={buttonStyle} onClick={() => setConfirming(false)} disabled={busy}>
+          <button
+            style={{ ...dangerButtonStyle, marginRight: 0 }}
+            onClick={() => onAsk({ uid: account.uid, kind: 'remove' })}
+            disabled={disabled}
+          >
+            Remove
+          </button>
+        </div>
+      </div>
+
+      {error && <p style={{ ...errorTextStyle, marginBottom: 0 }}>{error}</p>}
+
+      {pending?.uid === account.uid && pending.kind === 'reveal' && (
+        <div style={dangerPanelStyle}>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            Show the private key for <strong>{label}</strong>?
+          </p>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            The key <em>is</em> the account. Anyone who copies it can sign records as this uid on
+            any machine, forever — there is no way to revoke or rotate it afterwards. meta-core
+            will hand it over without asking who you are; see the note at the top of this page.
+          </p>
+          <button style={dangerButtonStyle} onClick={() => onReveal(account.uid)} disabled={busy}>
+            {busy ? 'Revealing…' : 'Yes, show the private key'}
+          </button>
+          <button style={buttonStyle} onClick={onCancel} disabled={busy}>
+            Cancel
+          </button>
+        </div>
+      )}
+
+      {pending?.uid === account.uid && pending.kind === 'remove' && (
+        <div style={dangerPanelStyle}>
+          <p style={{ color: '#fecaca', marginTop: 0 }}>
+            Remove <strong>{label}</strong> from this node?
+          </p>
+          <ul style={{ color: '#fecaca', paddingLeft: '1.2rem', margin: '0 0 0.75rem' }}>
+            <li>
+              <strong>The key is gone for good.</strong> The uid <em>is</em> the public key, and
+              the private half exists only in this node's keystore. If nobody saved it, nothing —
+              not you, not meta-core — can ever sign as this uid again. Reveal and copy it first
+              if there is any doubt.
+            </li>
+            <li>
+              <strong>Nothing is retracted.</strong> Records already signed under this uid keep
+              verifying on every peer that replicated them. Deleting a key is not a recall.
+            </li>
+            <li>
+              <strong>Its data stays, but goes read-only.</strong> The account's User Data Layer
+              cells — likes, ratings, watch progress — survive in Redis, and no newer version of
+              any of them can ever be written.
+            </li>
+            <li>
+              <strong>This signs nobody out elsewhere.</strong> Any device that holds this key
+              still holds it, and can import it back here at any time.
+            </li>
+          </ul>
+          <button style={dangerButtonStyle} onClick={() => onRemove(account.uid)} disabled={busy}>
+            {busy ? 'Removing…' : 'Yes, remove this account'}
+          </button>
+          <button style={buttonStyle} onClick={onCancel} disabled={busy}>
             Cancel
           </button>
         </div>
@@ -309,66 +476,183 @@ function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onRep
 }
 
 export default function Identity() {
-  const [status, setStatus] = useState<IdentityStatus | null>(null);
-  const [generated, setGenerated] = useState<GeneratedKey | null>(null);
+  const [accounts, setAccounts] = useState<Account[] | null>(null);
+  const [names, setNames] = useState<Record<string, string>>({});
   const [loading, setLoading] = useState(true);
   const [fetchError, setFetchError] = useState<string | null>(null);
+  const [secret, setSecret] = useState<SecretKeyView | null>(null);
+  const [pending, setPending] = useState<Pending | null>(null);
+  const [busyUid, setBusyUid] = useState<string | null>(null);
+  const [rowError, setRowError] = useState<{ uid: string; message: string } | null>(null);
+
+  // Bumped per refresh so a slow response from a superseded load cannot
+  // overwrite a newer one. The page does not poll (unlike the rest of the
+  // dashboard) — a key list has no business changing under the cursor.
+  const reqGen = useRef(0);
 
   const refresh = async () => {
+    const gen = ++reqGen.current;
     setLoading(true);
     setFetchError(null);
+    let list: Account[];
     try {
-      const res = await fetch('/api/identity');
-      if (!res.ok) {
-        setFetchError(await errorMessage(res, 'Failed to load identity status'));
+      list = await fetchAccounts();
+    } catch (e) {
+      if (gen !== reqGen.current) return;
+      setFetchError(e instanceof Error ? e.message : String(e));
+      setLoading(false);
       return;
     }
-      setStatus((await res.json()) as IdentityStatus);
-    } catch (e) {
-      setFetchError(`Failed to load identity status: ${e instanceof Error ? e.message : String(e)}`);
-    } finally {
+    if (gen !== reqGen.current) return;
+    // Render the list before resolving names, so a down Redis delays nothing.
+    setAccounts(list);
     setLoading(false);
-    }
+
+    const settled = await Promise.allSettled(list.map((a) => fetchDisplayName(a.uid)));
+    if (gen !== reqGen.current) return;
+    const next: Record<string, string> = {};
+    settled.forEach((r, i) => {
+      if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value;
+    });
+    // Replace rather than merge, so a removed account's name goes with it.
+    setNames(next);
   };
 
   useEffect(() => {
     refresh();
   }, []);
 
+  const reveal = async (uid: string) => {
+    setBusyUid(uid);
+    setRowError(null);
+    try {
+      // POST with a body rather than ?uid=, so the target stays out of the
+      // proxy's access log. The handler accepts either.
+      const res = await fetch('/api/identity/reveal', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ confirm: true, uid }),
+      });
+      if (!res.ok) {
+        setRowError({ uid, message: await errorMessage(res, 'Reveal failed') });
+        return;
+      }
+      const data = (await res.json()) as { uid: string; privateKeyHex: string };
+      setPending(null);
+      setSecret({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'revealed' });
+    } catch (e) {
+      setRowError({ uid, message: `Reveal failed: ${e instanceof Error ? e.message : String(e)}` });
+    } finally {
+      setBusyUid(null);
+    }
+  };
+
+  const remove = async (uid: string) => {
+    setBusyUid(uid);
+    setRowError(null);
+    try {
+      // The uid is mandatory: with more than one account the backend refuses an
+      // unqualified delete (400 uid_required) rather than guess.
+      const res = await fetch(`/api/identity?uid=${encodeURIComponent(uid)}&confirm=true`, {
+        method: 'DELETE',
+      });
+      if (!res.ok) {
+        setRowError({ uid, message: await errorMessage(res, 'Remove failed') });
+        return;
+      }
+      setPending(null);
+      await refresh();
+    } catch (e) {
+      setRowError({ uid, message: `Remove failed: ${e instanceof Error ? e.message : String(e)}` });
+    } finally {
+      setBusyUid(null);
+    }
+  };
+
+  const dismissSecret = () => {
+    setSecret(null);
+    refresh();
+  };
+
+  const showList = !loading && !fetchError && !secret;
+
   return (
     <div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
-      <h1>Identity</h1>
+      <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
+        <h1 style={{ marginBottom: 0 }}>Identity</h1>
+        <button style={{ ...buttonStyle, marginRight: 0 }} onClick={refresh} disabled={loading}>
+          {loading ? 'Loading…' : 'Refresh'}
+        </button>
+      </div>
       <p style={{ color: '#9ca3af' }}>
-        Signing key for the User Data Layer (per-user ratings, progress, "liked" flags). One
-        identity per node. The same key on another device makes that device count as "you".
+        Every signing key in this node's keystore. meta-core has no "current user" — it stores
+        accounts, and which one is <em>you</em> is decided by whichever client signs in (e.g.
+        meta-watch). Keys live one JSON file per account under{' '}
+        <code>{'{META_CORE_PATH}'}/identity/accounts/&lt;uid&gt;.json</code>, and are used to sign
+        User Data Layer records (ratings, watch progress, "liked" flags).
       </p>
 
+      <div style={{ ...dangerPanelStyle, marginTop: 0, marginBottom: '1rem' }}>
+        <p style={{ color: '#fecaca', margin: 0 }}>
+          <strong>Admin page — no authorisation happens here.</strong> <code>/api/identity/*</code>{' '}
+          asks no questions: anything that can reach this API can list, create, reveal or delete{' '}
+          <em>any</em> account's private key. The only access control is the perimeter in front of
+          this port. Client-facing screens (meta-watch) authorise reveal and delete against the
+          caller's own session and expose no delete at all; this page has neither check. Treat
+          reaching it as equivalent to holding every key on this node.
+        </p>
+      </div>
+
       {loading && <div style={cardStyle}>Loading…</div>}
+
       {fetchError && (
         <div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
-          <p style={{ color: '#f87171', margin: 0 }}>{fetchError}</p>
+          <p style={{ color: '#f87171', margin: '0 0 1rem' }}>{fetchError}</p>
           <button style={buttonStyle} onClick={refresh}>
             Retry
           </button>
         </div>
       )}
 
-      {!loading && !fetchError && generated && (
-        <GeneratedKeyPanel
-          data={generated}
-          onDismiss={() => {
-            setGenerated(null);
-            refresh();
+      {!loading && !fetchError && secret && <SecretKeyPanel data={secret} onDismiss={dismissSecret} />}
+
+      {showList && (
+        <>
+          <NewAccountPanel onGenerated={(s) => setSecret(s)} onImported={refresh} />
+
+          <h2 style={{ fontSize: '1.1rem', color: '#9ca3af', margin: '1.5rem 0 1rem' }}>
+            Accounts ({accounts?.length ?? 0})
+          </h2>
+
+          {accounts && accounts.length === 0 && (
+            <div style={cardStyle}>
+              <p style={{ color: '#9ca3af', margin: 0 }}>
+                No accounts in this keystore. Accounts are normally created by a client when
+                someone signs up; use <em>Add an account</em> above if you are bootstrapping this
+                node.
+              </p>
+            </div>
+          )}
+
+          {accounts?.map((a) => (
+            <AccountCard
+              key={a.uid}
+              account={a}
+              name={names[a.uid]}
+              busy={busyUid === a.uid}
+              disabled={busyUid !== null}
+              pending={pending}
+              error={rowError?.uid === a.uid ? rowError.message : null}
+              onAsk={(p) => {
+                setRowError(null);
+                setPending(p);
               }}
+              onCancel={() => setPending(null)}
+              onReveal={reveal}
+              onRemove={remove}
             />
-      )}
-
-      {!loading && !fetchError && !generated && status && !status.hasIdentity && (
-        <NoIdentityPanel onGenerated={(g) => setGenerated(g)} onImported={refresh} />
-      )}
-
-      {!loading && !fetchError && !generated && status && status.hasIdentity && (
-        <HasIdentityPanel status={status} onReplace={refresh} />
+          ))}
+        </>
       )}
     </div>
   );

docs/api-mediated-access.md

@@ -18,7 +18,7 @@ out to be load-bearing).
 
 meta-core was designed as the sidecar that mediates every metadata read and
 write — a single point that owns the Redis schema, enforces invariants
-(reverse index, canonical_cid reconciliation, schema-version sentinel),
+(reverse index, `cids/` key-set integrity, schema-version sentinel),
 and presents a stable HTTP contract. Other services were meant to talk to
 meta-core, not to Redis.
 
@@ -392,7 +392,7 @@ specifically with the Kubernetes flavour where streams are mediated too.
   coordinating releases across every other service. The UUID-root
   migration is the cautionary tale of what happens without this.
 - **Validation choke point.** Every write passes through one code path
-  that can enforce invariants (reverse index, canonical_cid, type
+  that can enforce invariants (reverse index, `cids/` key-set, type
   constraints, schema version).
 - **Observability.** Audit logs, request metrics, slow-query detection
   all live at one layer.
@@ -697,11 +697,11 @@ contract violation."
 **The fix.** Three layers in `internal/storage/cid_resolution.go`:
 
 1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
-   read/write handler, look up `cid:midhash256:<hash>` in the reverse
-   index. If found, use the UUID as the storage root. Falls through to
-   the bare hash for direct-UUID writes / legacy entries.
+   read/write handler, look up `cid:<hash>` in the reverse index (bare
+   CID — no algorithm prefix). If found, use the UUID as the storage root.
+   Falls through to the bare hash for direct-UUID writes / legacy entries.
 2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
-   register `cid:<algo>:<v>` → `<v>`. The dual-root bug's chicken-and-
+   register `cid:<v>` → `<v>`. The dual-root bug's chicken-and-
    egg is: meta-sort writes first, registers self-pointing alias,
    watcher then can't recover. Refusing the self-write keeps the
    watcher's legitimate alias intact.

docs/meta-core-architecture.md

@@ -221,12 +221,13 @@ operators wipe with `pnpm run clean:all` and restart.
 │  Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars)     │
 │  Storage: flat string keys                                         │
 │           file:<uuid>/<property>           → string value          │
-│           file:<uuid>/cids                 → SET of CID tokens     │
-│           file:<uuid>/canonical_cid        → preferred CID         │
+│           file:<uuid>/cids/<bareCid>       → "true" (key-set)      │
 │           file:<uuid>/duplicates           → SET of paths          │
 │                                                                    │
+│  No stored canonical_cid — derived by rank on read.                │
+│                                                                    │
 │  Aliases (reverse index, one Redis STRING each):                   │
-│           cid:<algorithm>:<value>          → <uuid>                │
+│           cid:<bareCid>                    → <uuid>                │
 │                                                                    │
 │  Index:   file:__index__                   → SET of all UUIDs      │
 │                                                                    │
@@ -515,7 +516,7 @@ via `/health`; metadata I/O and SSE event streams stop.
 | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). |
 | `REDIS_PORT` | `6379` | Local Redis port. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
-| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind address. |
+| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind address. |
 | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
 | `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
 | `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is stale. |

docs/metadata-storage-structure.md

@@ -19,10 +19,13 @@ For the external-access contract (HTTP-only, SSE for events), see
   `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
 - **Properties are flat Redis STRING keys**, one per leaf:
   `file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
-- **CIDs are reverse-index aliases**: `cid:<algorithm>:<value>` → `<uuid>`
-  (a Redis STRING per known CID).
-- **Per-root sets**: `file:<uuid>/cids` (every CID), `file:<uuid>/duplicates`
+- **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
+  (a Redis STRING per known CID; no algorithm prefix — the multicodec
+  already carries it).
+- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
+  `"true"` per known CID, midhash included), `file:<uuid>/duplicates`
   (alternate paths with the same content), plus `file:__index__` (every UUID).
+  There is **no stored `canonical_cid`** — it is derived by rank on read.
 - **A schema-version sentinel** (`meta-core:schema-version`) refuses to
   boot against legacy data.
 
@@ -59,37 +62,54 @@ file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster                       = "bafkreih...
 
 ### Reverse-index aliases (CIDs)
 
-Every CID known for a file is registered as its own Redis STRING:
+Every CID known for a file is registered as its own Redis STRING, keyed by
+the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
+function in the multicodec, so naming it again in the key would be
+redundant (and would let the same digest be indexed under two spellings):
 
 ```
-cid:<algorithm>:<value>   →   <uuid>
+cid:<bareCid>   →   <uuid>
 ```
 
 Examples:
 
 ```
-cid:midhash256:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:sha256:abc...                    →   01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:ipfs:bafy...                     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bagacbabaec7v3...     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
 ```
 
 The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
-registered CID, independent of which hash algorithm "primary" — there is
-no primary. The auto-alias hooks in `SetProperty` and `SetMetadataFlat`
-(see `internal/storage/cid_resolution.go`) catch any write of a `cid_*`
-or `midhash256` field and register the alias automatically, so plugins
-don't need to know about the reverse index.
+registered CID, independent of which hash algorithm — there is no primary.
+The auto-alias hook (`maybeAddAliasFromFieldLocked` in
+`internal/storage/cid_resolution.go`, called from `SetProperty`,
+`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
+field whose name starts with `cids/`, so writers emit key-set members
+without knowing the reverse index exists.
+
+> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
+> `midhash256` *named* field written to a root is stored verbatim and is
+> **never indexed** — the record becomes unresolvable by that CID. Writers
+> must collapse such fields to `cids/<cid>` members before the write. See
+> `METADATA_KEYS.md` §14.13.
 
 ### Per-root sets
 
 ```
-file:<uuid>/cids              →   Redis SET of "<algorithm>:<value>" tokens
-file:<uuid>/canonical_cid     →   STRING — the highest-ranked CID for advertising externally
+file:<uuid>/cids/<bareCid>    →   STRING "true" — one key per known CID (a key-set, not a Redis SET)
 file:<uuid>/duplicates        →   Redis SET of alternate paths with same content
 ```
 
-`canonical_cid` is reconciled by `reconcileCanonicalCIDLocked` whenever
-the `cids` set changes; ranking is done by `internal/cid/rank.go`.
+`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
+which keeps it conflict-free to merge across peers (a SET would not be).
+The record's own address (the midhash) is just the `0x1000` member — it has
+no separate field.
+
+There is **no stored `canonical_cid`**. The canonical CID is *derived on
+read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the
+key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns
+the winner as the `canonical_cid` field of the JSON response only. Nothing
+reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and
+was removed with the flat `cid_*` keys.
 
 ### Global index
 
@@ -141,14 +161,15 @@ The Go API in `internal/storage/client.go`:
 | Method | Behaviour |
 |---|---|
 | `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
-| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register any `cid_*` aliases. |
+| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
 | `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
 | `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
 | `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
 | `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
 | `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
-| `AddAlias(uuid, "<algo>:<value>")` | Set `cid:<algo>:<value>` → `<uuid>`, add to `file:<uuid>/cids`, reconcile `canonical_cid`. |
-| `ResolveCID("<algo>:<value>")` | `GET cid:<algo>:<value>` → uuid. |
+| `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
+| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
+| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
 
 ## Reconstruction (flat → nested)
 
@@ -237,10 +258,14 @@ docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo
 docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
 # string
 
-# Reverse-index lookups
-docker exec metacore-app redis-cli GET 'cid:midhash256:bafkreih...'      # → uuid
-docker exec metacore-app redis-cli SMEMBERS 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids'
-docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid'
+# Reverse-index lookup (bare CID — no algorithm prefix)
+docker exec metacore-app redis-cli GET 'cid:bafkreih...'      # → uuid
+
+# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
+docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
+
+# canonical_cid is NOT stored. Derive it via the API instead:
+curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
 
 # Schema-version sentinel
 docker exec metacore-app redis-cli GET 'meta-core:schema-version'

docs/uuid-rooted-metadata.md

@@ -4,6 +4,25 @@
 
 **Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
 
+> ⚠ **Two parts of this proposal were superseded before shipping.** This doc
+> is the design record; it is *not* the schema reference. For what is actually
+> stored, see `METADATA_KEYS.md` §2/§14.13 and
+> [metadata-storage-structure.md](./metadata-storage-structure.md).
+>
+> 1. **`canonical_cid` is not a stored key.** The proposed reconciler
+>    (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't
+>    conflict-free to merge across peers. The canonical CID is **derived on
+>    read** by ranking the `cids` members (`internal/cid/rank.go` →
+>    `cid.Better`), and appears only in the `GET /api/meta/{cid}` response.
+> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not
+>    `<algo>:<value>` tokens.** One flat STRING per member:
+>    `file:<uuid>/cids/<bareCid>` = `"true"`. The reverse index is likewise
+>    `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
+>    function, so the `<algo>:` prefix was redundant.
+>
+> Text below that references either shape describes the *proposal*, not the
+> code. The rank ordering and everything else did ship as written.
+
 ## Motivation
 
 Today, every metadata entry in Redis is keyed by a content hash, specifically
@@ -79,19 +98,21 @@ today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath          = /watch/Inception.mkv
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size              = 4500000000
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime             = 1729872000000000000
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids              = SET{midhash256:abc, sha256:def, ipfs:bafy...}
+file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec…  = "true"    ← midhash (record address)
+file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd…    = "true"    ← sha2-256
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates        = SET{/watch/dup1.mkv, /watch/dup2.mkv}
-file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid     = ipfs:bafy...
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title             = "Inception"
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster       = bafy...
 file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng      = ...
 
-cid:midhash256:abc  → 01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:sha256:def      → 01JKR8XW5T4QMV7AHNJ2DEFGHK
-cid:ipfs:bafy...    → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bagacbabaec…  → 01JKR8XW5T4QMV7AHNJ2DEFGHK
+cid:bafybeigd…    → 01JKR8XW5T4QMV7AHNJ2DEFGHK
 ```
 
+*(As shipped — the proposal originally wrote `cids` as a Redis SET of
+`<algo>:<value>` tokens plus a `canonical_cid` scalar. See the Status note.)*
+
 ### UUID format: UUIDv7
 
 UUIDv7 chosen over UUIDv4 or 64-bit random:
@@ -111,26 +132,33 @@ The metadata schema is open — any writer can put any key under
 `file:<uuid>/`. This proposal does not change that. It adds three keys whose
 sole writer is meta-core itself:
 
-| Key | Type | Purpose |
+| Key | Type (as shipped) | Purpose |
 |-----|------|---------|
-| `cids` | Redis SET | All CIDs known for this file. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. |
+| `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
 | `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
-| `canonical_cid` | STRING | The CID this entry should be presented as externally (meta-share announce, UI display). Selected by the rank function (below). |
+| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
 
 Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
 keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
 `subtitle/<lang>`, …) are unaffected.
 
-Redis SET is a departure from the all-strings convention in the current
+~~Redis SET is a departure from the all-strings convention in the current
 storage doc, but `file:__index__` is already a SET — so the precedent exists
-for using sets where they're the right structure.
+for using sets where they're the right structure.~~ **Reversed for `cids`.**
+The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
+so that two peers merging the same record can never disagree about set
+membership. `duplicates` remained a Redis SET (it is leader-local and never
+merged across peers).
 
 ### Canonical CID selection
 
-`canonical_cid` is a label, not a key. Updating it costs one write — no
-rename, no race, no broken external references.
+The canonical CID is a label, not a key — so nothing external breaks when it
+changes. **As shipped it is not written at all**: it is computed on every read
+from the `cids` members, which makes the "update" cost zero and removes the
+reconciler entirely.
 
-Selection rule, highest-wins:
+Selection rule, highest-wins (this part shipped as designed —
+`internal/cid/rank.go`):
 
 1. IPFS CID (any variant)
 2. SHA-256 / SHA-3 family
@@ -142,8 +170,10 @@ midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
 externally meaningful for content addressing. btih ranks above midhash because
 it enables swarm interop even if it's weaker as a digest.
 
-Reconciler runs on the leader, recomputes `canonical_cid` whenever
-`AddAlias` registers a new CID for an entry.
+~~Reconciler runs on the leader, recomputes `canonical_cid` whenever
+`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no
+reconciler. `GetMetadataDocument` folds the members through `cid.Better` on
+each read.
 
 ## Internal operations
 
@@ -157,8 +187,8 @@ CID-based API.
 Mint(filePath string, size int64, mtime int64) (uuid string, err error)
 
 // AddAlias registers a CID as resolving to this UUID. Updates both the
-// cid:<cid> reverse index and the file:<uuid>/cids set atomically.
-// Triggers canonical_cid reconciliation.
+// cid:<cid> reverse index and the file:<uuid>/cids/<cid> key-set member.
+// (As shipped: no reconciliation step — canonical is derived on read.)
 AddAlias(uuid string, cid string) error
 
 // GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
@@ -199,7 +229,8 @@ DeleteRoot(uuid string) error
    - Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
    - meta-core resolves midhash256:abc → uuid_Y via GetByCID.
    - meta-core calls AddAlias(uuid_Y, "sha256:def...").
-   - Reconciler updates canonical_cid if sha256 outranks midhash256.
+   - Nothing else to do: the next read of this entry derives a canonical CID
+     of sha2-256, because it outranks the midhash.
 ```
 
 ### Lifecycle: file deletion
@@ -250,13 +281,20 @@ so the impact is contained but real.
   client.
 - **meta-dup**: gains a cleaner data source — duplicates are now explicit in
   the `duplicates` set, not implicit in colliding writes.
-- **meta-share**: announces `canonical_cid` rather than midhash. Federated
-  search responses include all CIDs from the `cids` set so any peer querying
-  by any digest gets a hit.
+- **meta-search / meta-share**: announce the *derived* canonical CID rather
+  than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata`
+  rather than reading a stored field. Federated search responses include all
+  CIDs from the `cids` key-set so any peer querying by any digest gets a hit.
+  (Written before the 2026-06 discovery/transport split; discovery is now
+  meta-search's job.)
 - **Plugins**: no change. They speak CIDs only.
 
 ## Migration
 
+> **Historical — this sweep was never run.** The alpha shipped behind a
+> clean-wipe gate (Redis was flushed and the library re-ingested), so no
+> in-place migration code exists. Kept for the record.
+
 One-time sweep, idempotent, runs on first boot after upgrade:
 
 ```
@@ -285,7 +323,8 @@ roots and resumes by detecting which step completed (presence of
 | UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
 | Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
 | Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
-| `cids` and `duplicates` field type | Redis SET | Departure from "all-strings" convention but consistent with the existing `file:__index__` precedent. |
+| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/<cid>` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. |
+| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. |
 | Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
 
 ## Open questions
@@ -297,9 +336,10 @@ roots and resumes by detecting which step completed (presence of
   hashing reveals the collision. Worth doing eventually; not required for
   this iteration.
 
-- **meta-share announce strategy.** Announce only `canonical_cid`, or every
-  entry in `cids`? Maximum discoverability vs. DHT footprint. Decide as part
-  of the meta-share work, not here.
+- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every*
+  member of `cids` on the wire (`Record.cids`) and addresses itself by the
+  single derived canonical (`Record.cid`), so a peer querying by any digest
+  gets a hit without the DHT footprint of announcing each one separately.
 
 - **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
   content-streaming endpoint for external IPFS clients? Adjacent to this

go.mod

@@ -3,6 +3,7 @@ module github.com/metazla/meta-core
 go 1.21
 
 require (
+	github.com/alicebob/miniredis/v2 v2.31.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
@@ -13,6 +14,8 @@ require (
 )
 
 require (
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/yuin/gopher-lua v1.1.0 // indirect
 )

go.sum

@@ -1,15 +1,24 @@
+github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
+github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
 github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -18,7 +27,10 @@ github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
 github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
 github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
 github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/api/deprecated_field_test.go

@@ -0,0 +1,86 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the
+// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They
+// must be rejected at the write boundary rather than stored.
+//
+// Why a hard 400 and not a silent rewrite: the reverse-index hook
+// (maybeAddAliasFromFieldLocked) only registers an alias for a field named
+// `cids/<cid>`. A `cid_*` field was therefore written verbatim and never
+// indexed — the record silently became unresolvable by that CID. A loud
+// failure is strictly better than a record that looks correct and cannot be
+// found.
+func TestRejectDeprecatedField(t *testing.T) {
+	rejected := []string{
+		"cid_midhash256",
+		"cid_sha2-256",
+		"cid_btih_v2",
+		"cid_crc32",
+		"midhash256",
+		"canonical_cid",
+	}
+	for _, f := range rejected {
+		if !rejectDeprecatedField(f) {
+			t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f)
+		}
+	}
+
+	accepted := []string{
+		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
+		"filePath",
+		"title",
+		"fileinfo/duration",
+		"tmdb/poster",
+		// Near-misses that must NOT be caught: the prefix is `cid_`, with an
+		// underscore. A field merely starting with "cid" is fine.
+		"cids",
+		"cidrank",
+	}
+	for _, f := range accepted {
+		if rejectDeprecatedField(f) {
+			t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f)
+		}
+	}
+}
+
+func TestValidateFields_Rejects400WithSlug(t *testing.T) {
+	rr := httptest.NewRecorder()
+	blocked := validateFields(rr, map[string]string{
+		"title":          "Inception",
+		"cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
+	})
+
+	if !blocked {
+		t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed")
+	}
+	if rr.Code != http.StatusBadRequest {
+		t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest)
+	}
+
+	var body map[string]interface{}
+	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+		t.Fatalf("response is not JSON: %v", err)
+	}
+	if got := body["error"]; got != ErrDeprecatedField {
+		t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField)
+	}
+}
+
+func TestValidateFields_AllowsCidsKeyset(t *testing.T) {
+	rr := httptest.NewRecorder()
+	blocked := validateFields(rr, map[string]string{
+		"filePath": "/watch/Inception.mkv",
+		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true",
+		"cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua":     "true",
+	})
+	if blocked {
+		t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String())
+	}
+}

internal/api/handlers.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/base32"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -84,8 +85,45 @@ const (
 	ErrSchemaViolation   = "schema_violation"
 	ErrStorageUnavailable = "storage_unavailable"
 	ErrInternal          = "internal"
+	ErrDeprecatedField   = "deprecated_field"
 )
 
+// rejectDeprecatedField reports whether a field name is one of the removed
+// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …).
+//
+// This is a tripwire, not a compatibility shim. Such a field used to be
+// accepted and written verbatim — but the reverse-index hook
+// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named
+// `cids/<cid>`, so a `cid_*` field was stored and then NEVER INDEXED: the
+// record silently became unresolvable by that CID. Failing the write loudly is
+// strictly better than a record that looks fine and cannot be found.
+//
+// Writers must emit the bare-CID key-set instead: `cids/<bareCid>` = "true".
+// See METADATA_KEYS.md §2/§14.13.
+func rejectDeprecatedField(field string) bool {
+	return strings.HasPrefix(field, "cid_") ||
+		field == "midhash256" ||
+		field == "canonical_cid"
+}
+
+// validateFields rejects a write carrying any deprecated CID field. Returns
+// true if the request was rejected (the handler must return immediately).
+func validateFields(w http.ResponseWriter, metadata map[string]string) bool {
+	for field := range metadata {
+		if rejectDeprecatedField(field) {
+			writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField,
+				fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+
+					"write the bare-CID key-set instead: \"cids/<bareCid>\": \"true\" "+
+					"(a cid_* field would be stored but never indexed, leaving the "+
+					"record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.",
+					field),
+				false)
+			return true
+		}
+	}
+	return false
+}
+
 // defaultSlugFor maps an HTTP status code to the most likely error slug.
 // Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
 // instead of "unknown_root" on 404) call writeErrorSlug directly.
@@ -255,6 +293,10 @@ func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, metadata) {
+		return
+	}
+
 	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -620,6 +662,10 @@ func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, metadata) {
+		return
+	}
+
 	updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -697,6 +743,10 @@ func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, map[string]string{key: ""}) {
+		return
+	}
+
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return
@@ -790,6 +840,10 @@ func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if validateFields(w, map[string]string{key: ""}) {
+		return
+	}
+
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
 		return

internal/api/handlers_identity.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/metazla/meta-core/internal/identity"
@@ -50,6 +51,26 @@ type identityImportResponse struct {
 	UID       string `json:"uid"`
 	Curve     string `json:"curve"`
 	CreatedAt int64  `json:"createdAt"`
+	// Created distinguishes "this key was new here" (201) from "this key was
+	// already in the keystore" (200). Import is the sign-in primitive for
+	// clients: presenting the secret key is the proof of ownership, so a
+	// re-import must succeed rather than conflict. Clients use the flag only
+	// to word the confirmation ("new profile" vs "welcome back").
+	Created bool `json:"created"`
+}
+
+// identityRevealRequest asks for an existing account's private key back. The
+// key was shown once at generate; without a way to see it again, a client with
+// a real sign-out has no recovery path for a user who never saved it.
+type identityRevealRequest struct {
+	Confirm bool   `json:"confirm"`
+	UID     string `json:"uid"`
+}
+
+type identityRevealResponse struct {
+	UID           string `json:"uid"`
+	Curve         string `json:"curve"`
+	PrivateKeyHex string `json:"privateKeyHex"`
 }
 
 type identityDeleteRequest struct {
@@ -68,6 +89,31 @@ type identitySignResponse struct {
 	SigB64 string `json:"sigB64"`
 }
 
+// identitySignBatchRequest signs many payloads under one account in one call.
+//
+// Exists because a User Data Layer write signs exactly one record, and the bulk
+// verbs above it ("mark this whole series seen") produce one record per episode:
+// a 300-episode show is 300 sequential sign round trips through
+// meta-watch -> meta-search -> here, which dominates the operation. The private
+// key still never leaves this process; only the transport is batched.
+type identitySignBatchRequest struct {
+	// PayloadsB64 are the canonical byte strings to sign, in order. The
+	// response's signatures line up index-for-index.
+	PayloadsB64 []string `json:"payloadsB64"`
+	// UID selects which account signs — same optional single-identity
+	// back-compat as identitySignRequest.
+	UID string `json:"uid,omitempty"`
+}
+
+type identitySignBatchResponse struct {
+	SigsB64 []string `json:"sigsB64"`
+}
+
+// maxSignBatch bounds one batch. A signature is cheap but not free, and the
+// request holds the account lock for its duration; a caller with more than this
+// should page. Sized well above the longest real series (Supernatural, 327).
+const maxSignBatch = 1000
+
 type identityAEADKeyResponse struct {
 	AEADKeyB64 string `json:"aeadKeyB64"`
 }
@@ -172,9 +218,17 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	dir := s.config.IdentityAccountsDir()
-	if identity.ExistsByUID(dir, id.UID) {
-		writeErrorSlug(w, http.StatusConflict, "identity_exists",
-			"an account with this key already exists", false)
+	// Idempotent: importing a key that is already in the keystore is how a
+	// second device signs in to an existing account. Answer 200 with the
+	// stored account (preserving its original createdAt) instead of 409 —
+	// the caller proved ownership by presenting the key.
+	if existing, err := identity.LoadByUID(dir, id.UID); err == nil && existing != nil {
+		writeJSON(w, http.StatusOK, identityImportResponse{
+			UID:       existing.UID,
+			Curve:     existing.Curve,
+			CreatedAt: existing.CreatedAt,
+			Created:   false,
+		})
 		return
 	}
 	if err := identity.SaveAccount(dir, id); err != nil {
@@ -185,6 +239,35 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		UID:       id.UID,
 		Curve:     id.Curve,
 		CreatedAt: id.CreatedAt,
+		Created:   true,
+	})
+}
+
+// handleIdentityReveal returns an existing account's private key. Deliberately
+// POST + explicit confirm (never a bare GET) so it cannot be triggered by a
+// link or an image tag, and so it reads as the deliberate act it is. The
+// endpoint is auth-gated at the perimeter like the rest of /api/identity/*.
+func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
+	var body identityRevealRequest
+	_ = json.NewDecoder(r.Body).Decode(&body)
+	if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
+		writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
+			"revealing a private key requires {\"confirm\": true} or ?confirm=true", false)
+		return
+	}
+	uid := strings.TrimSpace(body.UID)
+	if uid == "" {
+		uid = strings.TrimSpace(r.URL.Query().Get("uid"))
+	}
+	id, status, slug, msg := s.resolveAccount(uid)
+	if status != 0 {
+		writeErrorSlug(w, status, slug, msg, false)
+		return
+	}
+	writeJSON(w, http.StatusOK, identityRevealResponse{
+		UID:           id.UID,
+		Curve:         id.Curve,
+		PrivateKeyHex: id.PrivateKeyHex,
 	})
 }
 
@@ -260,6 +343,53 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+// handleIdentitySignBatch signs every payload in the request under one account.
+//
+// All-or-nothing: a single bad base64 or signing failure fails the whole
+// request rather than returning a short array. The caller pairs signatures with
+// records by index, so a partial response would silently misalign them — and a
+// misaligned signature verifies against the wrong record, which is worse than
+// an error.
+func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) {
+	var body identitySignBatchRequest
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	uid := body.UID
+	if strings.TrimSpace(uid) == "" {
+		uid = r.URL.Query().Get("uid")
+	}
+	if len(body.PayloadsB64) == 0 {
+		writeError(w, http.StatusBadRequest, "payloadsB64 is required")
+		return
+	}
+	if len(body.PayloadsB64) > maxSignBatch {
+		writeError(w, http.StatusBadRequest, "too many payloads in one batch")
+		return
+	}
+	id, status, slug, msg := s.resolveAccount(uid)
+	if status != 0 {
+		writeErrorSlug(w, status, slug, msg, false)
+		return
+	}
+	sigs := make([]string, 0, len(body.PayloadsB64))
+	for i, p := range body.PayloadsB64 {
+		raw, err := base64.StdEncoding.DecodeString(p)
+		if err != nil {
+			writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64")
+			return
+		}
+		sig, err := identity.Sign(id, raw)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
+	}
+	writeJSON(w, http.StatusOK, identitySignBatchResponse{SigsB64: sigs})
+}
+
 func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
 	id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
 	if status != 0 {

internal/api/metadata.go

@@ -193,6 +193,9 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Same resolution as the PUT below: callers hold a CID, not the root.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	metadata, err := s.storage.GetMetadataFlat(hashID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -227,6 +230,17 @@ func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	// The {hashId} in the URL is whatever CID the caller knows the file by —
+	// the gateway's auto-store keys this PUT by the hash it just computed
+	// (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is
+	// the dag-pb root, not the nzb-release cid the record was minted under.
+	// Without resolving, the write lands on file:<thatCid>/* and forks a
+	// second root for a file that already has one: a full duplicate record,
+	// cross-linked and separately indexed, which every downstream ingest then
+	// sees as two files. Unknown CIDs pass through unchanged, so a genuinely
+	// new gateway record still mints its root at the hash.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	var metadata map[string]string
 	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
 		writeError(w, http.StatusBadRequest, "invalid JSON body")
@@ -263,6 +277,12 @@ func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	// meta-gateway retracts a dropped hit by whichever CID it holds
+	// (meta_core.rs `delete`, which documents this resolution as a
+	// precondition). Without it, deleting by a non-root CID is a silent
+	// no-op and the retracted record lingers.
+	hashID = s.storage.ResolveRoot(hashID)
+
 	deletedCount, err := s.storage.DeleteMetadata(hashID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())

internal/api/server.go

@@ -172,8 +172,10 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
 	s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
 	s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
+	s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
 	s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
 	s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
+	s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
 	s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
 
 	// Snapshot (export / import / wipe)
@@ -201,6 +203,16 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
 	s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
 
+	// User Data Layer (UDL) routes — per-user signed records (My List,
+	// Continue Watching, ratings, profile). See internal/api/udl.go.
+	s.router.HandleFunc("/api/udl/record", s.handleUDLRecordGet).Methods("GET")
+	s.router.HandleFunc("/api/udl/record", s.handleUDLRecordPut).Methods("PUT")
+	s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
+	s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
+	s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
+	s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
+
 	// Metadata operations - base endpoints
 	s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
 	s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
@@ -341,6 +353,21 @@ func (s *Server) Start() error {
 
 			// Republish events for all existing metadata
 			go func() {
+				// Build any missing per-record field indexes first, in ONE
+				// keyspace pass. The republish below reads every record, and
+				// an un-indexed record read costs a full-keyspace SCAN — on a
+				// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
+				// paid again on every restart). Warming first turns the whole
+				// republish into index-served reads. Idempotent, so a
+				// fully-indexed store just pays the single scan.
+				// See storage/field_index.go.
+				started := time.Now()
+				if n, err := s.storage.WarmFieldIndexes(); err != nil {
+					log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
+				} else if n > 0 {
+					log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
+				}
+
 				count, err := s.RepublishMetadata()
 				if err != nil {
 					log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)

internal/api/udl.go

@@ -0,0 +1,351 @@
+package api
+
+// User Data Layer (UDL) HTTP surface.
+//
+// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
+// profile) now that the authoritative store lives in meta-core instead of
+// meta-watch's local op-log. Records arrive already signed (and, for
+// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
+// version-gates and indexes them but never inspects the crypto. See
+// internal/storage/udl.go for the Redis model.
+
+import (
+	"encoding/json"
+	"math"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+)
+
+// UDLPutRequest is the body of PUT /api/udl/record.
+type UDLPutRequest struct {
+	Uid     string           `json:"uid"`
+	Cid     string           `json:"cid"`
+	Key     string           `json:"key"`
+	Version int64            `json:"version"`
+	Ts      int64            `json:"ts"`
+	Record  string           `json:"record"` // base64 CBOR, opaque + signed
+	Value   *json.RawMessage `json:"value,omitempty"`
+	// Tombstone marks this write as a deletion of (uid, cid, key). The record
+	// itself is opaque here — meta-core never decodes the CBOR, and a private-
+	// tier value and a tombstone are equally "no plaintext value" — so the
+	// writer has to say. Without it the cid is never removed from
+	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
+	// account and is returned in full on every My List / Continue Watching
+	// read. Omitted by older writers, which keeps the previous behaviour.
+	Tombstone bool `json:"tombstone,omitempty"`
+}
+
+// UDLPutResponse is the response of PUT /api/udl/record.
+type UDLPutResponse struct {
+	Accepted bool `json:"accepted"`
+}
+
+// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
+// request. Each entry is exactly a UDLPutRequest, so a client can build the
+// same signed record for either endpoint.
+type UDLPutBatchRequest struct {
+	Records []UDLPutRequest `json:"records"`
+}
+
+// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
+// means the write was stale (the single-record endpoint's 409), which in a
+// batch is an ordinary outcome rather than a failure.
+type UDLPutBatchResult struct {
+	Key      string `json:"key"`
+	Accepted bool   `json:"accepted"`
+	Error    string `json:"error,omitempty"`
+}
+
+// UDLPutBatchResponse mirrors the request order index-for-index.
+type UDLPutBatchResponse struct {
+	Results []UDLPutBatchResult `json:"results"`
+}
+
+// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
+// request is held open; sized to match the sign batch on the other side of the
+// same operation (see maxSignBatch).
+const maxUDLBatch = 1000
+
+// UDLRecordResponse is the response of GET /api/udl/record.
+type UDLRecordResponse struct {
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+	Exists  bool   `json:"exists"`
+	// Value is the plaintext JSON scalar stored beside the opaque record, and
+	// is present only for PUBLIC-tier keys (like, rating, profile:name,
+	// profile:avatar). json.RawMessage rather than string because the field is
+	// generic over keys — a bool for `like`, a number for `rating`, a string
+	// for a profile name (see aggregatePublicValues) — and because it is the
+	// exact type UDLPutRequest.Value uses, so a value written through PUT reads
+	// back byte-identical.
+	//
+	// Absent, never null, when there is no plaintext: private-tier cells,
+	// tombstones and never-written cells must not read as an empty public
+	// value. Readers key off presence, not `exists`.
+	Value *json.RawMessage `json:"value,omitempty"`
+}
+
+// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
+// field names; `avg` is null for boolean keys like `like`), so the meta-watch
+// client deserializes it into the same struct it serializes to the UI.
+type UDLAggregate struct {
+	Key        string   `json:"key"`
+	Count      int      `json:"count"`
+	Avg        *float64 `json:"avg"`
+	Trues      int      `json:"trues"`
+	SampleSize int      `json:"sample_size"`
+}
+
+// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
+func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	q := r.URL.Query()
+	uid, cid, key := q.Get("uid"), q.Get("cid"), q.Get("key")
+	if uid == "" || cid == "" || key == "" {
+		writeError(w, http.StatusBadRequest, "uid, cid and key are required")
+		return
+	}
+
+	cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	resp := UDLRecordResponse{
+		Record:  cell.Record,
+		Version: cell.Version,
+		Ts:      cell.Ts,
+		Exists:  exists,
+	}
+	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
+	// Redis, a future writer bug) would otherwise fail mid-encode and emit a
+	// truncated body instead of a readable error.
+	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
+		raw := json.RawMessage(cell.Value)
+		resp.Value = &raw
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
+// (version <= stored) returns 409 stale_version so the client can re-read and
+// retry with a higher version.
+func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	var req UDLPutRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
+		writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
+		return
+	}
+
+	publicValue := ""
+	hasValue := false
+	if req.Value != nil {
+		publicValue = string(*req.Value)
+		hasValue = true
+	}
+
+	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if !accepted {
+		writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
+		return
+	}
+	writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
+}
+
+// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
+// handleUDLRecordPut.
+//
+// Exists for the bulk verbs above this layer ("mark this whole series seen"),
+// where one user action produces one cell per episode. Each cell still goes
+// through the same version-gated Lua upsert, so the CRDT semantics are
+// identical — this batches the transport only.
+//
+// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
+// replaces the single-record conflict status: in a batch, one stale cell is a
+// normal outcome (another device already marked that episode) and must not fail
+// the 299 cells beside it. Callers read the array, not the status code.
+func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	var req UDLPutBatchRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		return
+	}
+	if len(req.Records) == 0 {
+		writeError(w, http.StatusBadRequest, "records is required")
+		return
+	}
+	if len(req.Records) > maxUDLBatch {
+		writeError(w, http.StatusBadRequest, "too many records in one batch")
+		return
+	}
+	results := make([]UDLPutBatchResult, 0, len(req.Records))
+	for _, rec := range req.Records {
+		if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
+			results = append(results, UDLPutBatchResult{
+				Key:   rec.Key,
+				Error: "uid, cid, key and record are required",
+			})
+			continue
+		}
+		publicValue := ""
+		hasValue := false
+		if rec.Value != nil {
+			publicValue = string(*rec.Value)
+			hasValue = true
+		}
+		accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
+		if err != nil {
+			// A storage error is per-record here rather than fatal for the whole
+			// batch: the cells already written stay written, and reporting which
+			// ones failed lets the caller retry exactly those.
+			results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
+			continue
+		}
+		results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
+	}
+	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
+}
+
+// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
+// (active_for_user_key — My List, Continue Watching).
+func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	vars := mux.Vars(r)
+	uid, key := vars["uid"], vars["key"]
+	if uid == "" || key == "" {
+		writeError(w, http.StatusBadRequest, "uid and key are required")
+		return
+	}
+	entries, err := s.storage.UDLListUserKey(uid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
+// (list_for_user_cid).
+func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	vars := mux.Vars(r)
+	uid, cid := vars["uid"], vars["cid"]
+	if uid == "" || cid == "" {
+		writeError(w, http.StatusBadRequest, "uid and cid are required")
+		return
+	}
+	entries, err := s.storage.UDLListUserCid(uid, cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
+func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	cid := mux.Vars(r)["cid"]
+	if cid == "" {
+		writeError(w, http.StatusBadRequest, "cid is required")
+		return
+	}
+	users, err := s.storage.UDLCidUsers(cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
+}
+
+// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
+// Computes per-key population stats from the stored public value fields
+// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
+func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	cid := mux.Vars(r)["cid"]
+	if cid == "" {
+		writeError(w, http.StatusBadRequest, "cid is required")
+		return
+	}
+	keysParam := r.URL.Query().Get("keys")
+	if keysParam == "" {
+		keysParam = "like,rating"
+	}
+	aggregates := make([]UDLAggregate, 0)
+	for _, key := range strings.Split(keysParam, ",") {
+		key = strings.TrimSpace(key)
+		if key == "" {
+			continue
+		}
+		values, err := s.storage.UDLPublicValues(cid, key)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		aggregates = append(aggregates, aggregatePublicValues(key, values))
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"aggregates": aggregates})
+}
+
+// aggregatePublicValues folds raw JSON scalar values into an UDLAggregate:
+// counts booleans as trues and averages numeric values.
+func aggregatePublicValues(key string, values []string) UDLAggregate {
+	agg := UDLAggregate{Key: key, SampleSize: len(values), Count: len(values)}
+	var sum float64
+	var numeric int
+	for _, raw := range values {
+		var v interface{}
+		if err := json.Unmarshal([]byte(raw), &v); err != nil {
+			continue
+		}
+		switch t := v.(type) {
+		case bool:
+			if t {
+				agg.Trues++
+			}
+		case float64:
+			sum += t
+			numeric++
+		}
+	}
+	if numeric > 0 {
+		avg := math.Round((sum/float64(numeric))*100) / 100
+		agg.Avg = &avg
+	}
+	return agg
+}

internal/api/udl_test.go

@@ -0,0 +1,226 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newUDLTestServer returns a Server wired to an in-memory Redis.
+//
+// The struct is built directly rather than through NewServer: NewServer boots
+// the file watcher, mounts manager, watchers poller and WebDAV handler, none of
+// which the UDL handlers touch — they use s.storage and nothing else.
+func newUDLTestServer(t *testing.T) *Server {
+	t.Helper()
+
+	mr := miniredis.RunT(t)
+	c := storage.NewClient("")
+	if err := c.Connect("redis://" + mr.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return &Server{storage: c}
+}
+
+// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
+// fields, so a test can assert on the *encoding* of `value` (is it a quoted
+// string? is the key there at all?) rather than on what a typed struct would
+// have coerced it into.
+func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) {
+	t.Helper()
+
+	req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
+	rr := httptest.NewRecorder()
+	s.handleUDLRecordGet(rr, req)
+
+	var body map[string]json.RawMessage
+	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+	}
+	return rr.Code, body
+}
+
+// The plaintext value of a PUBLIC-tier cell must come back over the wire, and
+// come back *as it was written* — a JSON string stays a quoted string, a bool
+// stays a bool.
+//
+// Both halves matter. Without the value at all, the meta-core dashboard can
+// only ever show 44-character base58 uids: display names live in the User Data
+// Layer as the public cell (uid, "self", "profile:name"), and the plaintext
+// sits in Redis where the storage layer has always read it — the handler just
+// used to drop it. And the type has to stay json.RawMessage: this endpoint is
+// generic over keys, so a `*string` would mangle `like` (bool) and `rating`
+// (number) while looking correct for names.
+func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zAlice"
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok {
+		t.Fatalf("seed name: ok=%v err=%v", ok, err)
+	}
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok {
+		t.Fatalf("seed like: ok=%v err=%v", ok, err)
+	}
+
+	code, body := getRecord(t, s, uid, "self", "profile:name")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if got := string(body["value"]); got != `"Ada Lovelace"` {
+		t.Fatalf("value = %s, want a quoted JSON string", got)
+	}
+	var name string
+	if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" {
+		t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err)
+	}
+
+	// A boolean key survives unquoted — the assertion a *string field fails.
+	if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" {
+		t.Fatalf("like value = %s, want the bare JSON literal true", body["value"])
+	}
+}
+
+// A cell with no plaintext must omit `value` entirely rather than send null.
+//
+// That is the shape of every private-tier key (seek, watched, card, localpath —
+// their real value rides encrypted inside the opaque record) and of every
+// tombstone. A reader keys off presence: "there is no plaintext here" and "the
+// plaintext is null" are different facts, and a name cleared by its owner
+// leaves the cell existing with no value, which must not read as a name.
+func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zBob"
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok {
+		t.Fatalf("seed private cell: ok=%v err=%v", ok, err)
+	}
+
+	code, body := getRecord(t, s, uid, "bagcsaaa", "seek")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if _, ok := body["value"]; ok {
+		t.Fatalf("private cell must omit `value`, got %s", body["value"])
+	}
+	if string(body["exists"]) != "true" {
+		t.Fatalf("exists = %s, want true", body["exists"])
+	}
+}
+
+// A cell that was never written answers 200 with exists:false and no value —
+// not a 404, and not an empty-string value that would render as a blank name.
+func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	code, body := getRecord(t, s, "zNobody", "self", "profile:name")
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if string(body["exists"]) != "false" {
+		t.Fatalf("exists = %s, want false", body["exists"])
+	}
+	if _, ok := body["value"]; ok {
+		t.Fatalf("missing cell must omit `value`, got %s", body["value"])
+	}
+}
+
+// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the
+// per-record results.
+func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) {
+	t.Helper()
+
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body))
+	rr := httptest.NewRecorder()
+	s.handleUDLRecordsPut(rr, req)
+
+	var out UDLPutBatchResponse
+	if rr.Code == http.StatusOK {
+		if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
+			t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+		}
+	}
+	return rr.Code, out
+}
+
+// **The load-bearing property of the batch endpoint: one stale cell does not
+// fail the batch.**
+//
+// The single-record PUT answers a stale version with 409, which is right when
+// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark
+// this whole series seen") routinely include episodes another device already
+// marked, and 409-ing there would abandon every cell after it. So staleness is
+// reported per record as accepted:false, with 200 on the request itself.
+func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	const uid = "zCarol"
+	const cid = "bagdsaaaseries"
+	// Seed s01e01 at version 4 — the batch below tries to write it at 2.
+	if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok {
+		t.Fatalf("seed: ok=%v err=%v", ok, err)
+	}
+
+	code, out := putRecords(t, s, `{"records":[
+		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"},
+		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"}
+	]}`)
+
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code)
+	}
+	if len(out.Results) != 2 {
+		t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results))
+	}
+	if out.Results[0].Accepted || out.Results[0].Error != "" {
+		t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0])
+	}
+	if !out.Results[1].Accepted {
+		t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1])
+	}
+
+	// The fresh cell really landed, and the stale one did not overwrite.
+	if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" {
+		t.Fatalf("s01e02 was not written")
+	}
+	if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" {
+		t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"])
+	}
+}
+
+// A malformed record is reported in its own slot rather than rejecting the
+// request, for the same reason: the caller retries that cell, not the batch.
+func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	code, out := putRecords(t, s, `{"records":[
+		{"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"},
+		{"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"}
+	]}`)
+
+	if code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", code)
+	}
+	if out.Results[0].Error == "" {
+		t.Fatalf("record with no key must report an error, got %+v", out.Results[0])
+	}
+	if !out.Results[1].Accepted {
+		t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1])
+	}
+}
+
+// An empty batch is a client bug, not a no-op: it almost always means the
+// caller built its record list wrong, and answering 200 with an empty array
+// would hide that.
+func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
+	s := newUDLTestServer(t)
+
+	if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400", code)
+	}
+}

internal/cid/cid-rank-vectors.json

@@ -0,0 +1,166 @@
+{
+  "$schema": "cid-rank-vectors/v1",
+  "$comment": [
+    "CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
+    "The rank decides which of a record's several CIDs is elected canonical - the cid a",
+    "record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
+    "they cannot share code (independent submodules, no cargo path-dep). These vectors are",
+    "the only thing holding them in sync: every implementation's test suite loads this file.",
+    "",
+    "This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
+    "own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
+    "",
+    "Ranking is by UTILITY for external retrieval, not cryptographic strength.",
+    "Ties break on the lexicographically smaller cid, so the election is deterministic",
+    "across peers and does not churn.",
+    "",
+    "See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
+  ],
+  "ladder": {
+    "40": "dag-pb (IPFS block)",
+    "30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
+    "25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
+    "20": "BitTorrent infohashes - swarm interop",
+    "10": "midhash256 - the record address",
+    "5": "opaque locators (nzb-release, url, card) - never outrank a real digest",
+    "0": "weak/unknown digests, unparseable input"
+  },
+  "tieBreak": "lexicographically smaller cid wins",
+  "vectors": [
+    {
+      "name": "ipfs-dagpb",
+      "cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 40,
+      "note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
+    },
+    {
+      "name": "sha2-256-raw-codec",
+      "cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 30,
+      "note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
+    },
+    {
+      "name": "sha2-256-codec-eq-mh",
+      "cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
+      "rank": 30,
+      "note": "REAL meta-hash fixture; meta-hash sets codec == mh"
+    },
+    {
+      "name": "sha3-256",
+      "cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "sha3-384",
+      "cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "nzb-posting",
+      "cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 25,
+      "note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
+    },
+    {
+      "name": "btih-v2-codec-eq-mh",
+      "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "meta-hash encoder convention"
+    },
+    {
+      "name": "btih-v2-raw-codec",
+      "cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "fullhash encoder convention - SAME tier, both conventions must match"
+    },
+    {
+      "name": "btih-v1-file",
+      "cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v1"
+    },
+    {
+      "name": "btih-v2-file",
+      "cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v2"
+    },
+    {
+      "name": "midhash256-codec-eq-mh",
+      "cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10,
+      "note": "the record address; internally meaningful, externally invisible"
+    },
+    {
+      "name": "midhash256-raw-codec",
+      "cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10
+    },
+    {
+      "name": "nzb-release-locator",
+      "cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
+      "rank": 5,
+      "note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
+    },
+    {
+      "name": "url-locator",
+      "cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
+      "rank": 5,
+      "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
+    },
+    {
+      "name": "card-locator",
+      "cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
+      "rank": 5,
+      "note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
+    },
+    {
+      "name": "sha1",
+      "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "md5",
+      "cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "crc32",
+      "cid": "bagzafmqcathsbnrk",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "bt-pieces-root",
+      "cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "torrent-internal merkle root, NOT a file-identity cid"
+    },
+    {
+      "name": "cidv0-rejected",
+      "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
+      "rank": 0,
+      "note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
+    },
+    {
+      "name": "token-form-rejected",
+      "cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
+    },
+    {
+      "name": "garbage",
+      "cid": "not-a-cid",
+      "rank": 0
+    },
+    {
+      "name": "empty",
+      "cid": "",
+      "rank": 0
+    }
+  ]
+}

internal/cid/rank.go

@@ -20,6 +20,14 @@ import (
 // Multicodec / multihash codes used for ranking. Source of truth for the
 // digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
 // METADATA_KEYS.md §2.
+//
+// The tier every code maps to is pinned by the golden fixture
+// `cid-rank-vectors.json` (vendored next to this package). This is one of
+// seven implementations of the same ladder across Go, Rust and TypeScript;
+// the fixture is the only thing keeping them in agreement. If you add a code
+// here, add a vector — an unranked code silently falls to tier 0, which is
+// exactly how the nzb-release/url locators came to be ranked 20 in meta-share
+// and 0 here and in meta-search.
 const (
 	CodecRaw   = 0x55   // raw — IPFS content-addressed block
 	CodecDagPB = 0x70   // dag-pb — IPFS content-addressed block
@@ -34,6 +42,44 @@ const (
 	CodeBtihV2      = 0x10B7  // custom — BitTorrent v2 info hash (BEP 52)
 	CodeBtihV1File  = 0x1001  // custom — per-file BitTorrent v1 CID
 	CodeBtihV2File  = 0x1002  // custom — per-file BitTorrent v2 CID
+
+	// Opaque locators. These are NOT digests: the payload rides in an identity
+	// multihash (0x00), so the code identifying them sits in the *content
+	// codec* slot, never the multihash slot. Rank must match on the codec for
+	// these — matching on mh would see 0x00 and rank them 0.
+	CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
+	CodeURL        = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
+	// CodeCard is the limit case of the family: the only CID with NO bytes
+	// behind it anywhere, ever. It addresses a *work* (a series, a film) via
+	// varint(len(source))‖source‖id — ("tmdb","tv:95479") — so `/api/file/{cid}`
+	// correctly 404s and bitswap never offers it. Ranking it here (rather than
+	// letting it fall to RankUnknown) is what makes the planned "one card, two
+	// CIDs" cross-source merge safe: a TMDB and a MyAnimeList locator can sit on
+	// one record, and a real digest added later still wins the election.
+	CodeCard = 0x1007 // custom — identity-multihash CID wrapping a card (work) id
+
+	// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
+	// REAL sha2-256 digest (over the normalised, sorted article Message-ID
+	// set of a Usenet posting we scanned ourselves — never a locator, no
+	// indexer host embedded), so it would silently fall into RankDigest via
+	// the mh switch unless matched on the codec FIRST. It is content-derived
+	// (so it outranks the opaque locators) but addresses no fetchable block —
+	// the digest is over an id set, not over any block's bytes, so it can
+	// never be bitswap-fetched under itself — so it must stay BELOW a real
+	// digest (once the media is materialised and re-seeded, that genuine
+	// sha2-256 cid has to win the canonical election, not this one).
+	CodeNzbPosting = 0x1003 // custom — self-scanned Usenet posting identity
+)
+
+// Rank tiers, named so the ladder reads as a ladder.
+const (
+	RankIPFS       = 40 // dag-pb — broadest network reach
+	RankDigest     = 30 // sha2-256 / sha3-* — real content digests
+	RankNzbPosting = 25 // content-derived, but the digest addresses no fetchable block
+	RankBtih       = 20 // BitTorrent infohashes — swarm interop
+	RankMidhash    = 10 // the record address — internally meaningful, externally invisible
+	RankLocator    = 5  // opaque locators — never outrank a real digest
+	RankUnknown    = 0  // weak/unknown digests, unparseable input
 )
 
 // Decode parses a multibase-base32 CIDv1 string ("b…") into its content
@@ -87,33 +133,55 @@ func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
 //   - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
 //   - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
 //     block of the file IS its sha2-256, so they share this tier).
+//   - nzb-posting is content-derived (a digest over a Usenet posting's article
+//     Message-ID set) so it outranks the opaque locators below, but the
+//     digest addresses no fetchable block, so it stays below a real content
+//     digest — once the media is materialised and re-seeded under a genuine
+//     sha2-256 cid, that cid must win the election, not this one.
 //   - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
 //     swarm interop, not strength.
-//   - midhash256 is the floor among real options: internally meaningful,
+//   - midhash256 is the floor among real *digests*: internally meaningful,
 //     externally invisible.
+//   - opaque locators (nzb-release, url, card) sit BELOW midhash. They are not
+//     content-derived — two URLs to the same bytes yield two different locator
+//     CIDs — so a locator must never displace a real digest as the identity a
+//     record is advertised under. Tier 5 means a locator wins the election only
+//     when it is the record's sole CID, which is exactly the guarantee
+//     METADATA_KEYS.md §2 states.
 //   - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
 func Rank(cidStr string) int {
 	codec, mh, err := Decode(cidStr)
 	if err != nil {
-		return 0
+		return RankUnknown
 	}
 	if codec == CodecDagPB {
-		return 40
+		return RankIPFS
 	}
 	// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
 	// multihash code, in either encoder convention.
 	if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
 		mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
-		return 20
+		return RankBtih
+	}
+	// Opaque locators. Codec-only: their multihash is identity (0x00), so
+	// there is nothing to match on the mh side.
+	if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard {
+		return RankLocator
+	}
+	// nzb-posting: codec-only, checked BEFORE the mh switch below — its
+	// multihash is a real sha2-256 and would otherwise match CodeSha256 there
+	// and rank 30, tying with genuine content digests.
+	if codec == CodeNzbPosting {
+		return RankNzbPosting
 	}
 	switch mh {
 	case CodeSha256, CodeSha3_256, CodeSha3_384:
-		return 30
+		return RankDigest
 	case CodeMidhash256:
-		return 10
+		return RankMidhash
 	default:
 		// md5, crc32, plain sha1, pieces-root, unknown.
-		return 0
+		return RankUnknown
 	}
 }
 

internal/cid/rank_test.go

@@ -3,10 +3,56 @@ package cid
 import (
 	"encoding/base32"
 	"encoding/binary"
+	"encoding/json"
+	"os"
 	"strings"
 	"testing"
 )
 
+// goldenVectors is the CANONICAL cross-language fixture: the same file is
+// vendored into meta-search, meta-share and meta-hash, and each of their test
+// suites asserts against it. It is the only mechanism keeping the seven
+// implementations of this ladder in agreement — they live in independent
+// submodules and cannot share code. `scripts/check-cid-vectors.sh` at the
+// meta-root fails if any vendored copy drifts from the canonical one.
+const goldenVectors = "cid-rank-vectors.json"
+
+type vector struct {
+	Name string `json:"name"`
+	CID  string `json:"cid"`
+	Rank int    `json:"rank"`
+	Note string `json:"note"`
+}
+
+func loadVectors(t *testing.T) []vector {
+	t.Helper()
+	raw, err := os.ReadFile(goldenVectors)
+	if err != nil {
+		t.Fatalf("read %s: %v (run ./scripts/check-cid-vectors.sh --fix from the meta-root)", goldenVectors, err)
+	}
+	var doc struct {
+		Vectors []vector `json:"vectors"`
+	}
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		t.Fatalf("parse %s: %v", goldenVectors, err)
+	}
+	if len(doc.Vectors) == 0 {
+		t.Fatalf("%s has no vectors", goldenVectors)
+	}
+	return doc.Vectors
+}
+
+// TestRank_GoldenVectors is the load-bearing test. Every other test in this
+// file is a convenience; this one is the contract with Rust and TypeScript.
+func TestRank_GoldenVectors(t *testing.T) {
+	for _, v := range loadVectors(t) {
+		if got := Rank(v.CID); got != v.Rank {
+			t.Errorf("Rank(%s) = %d, want %d\n  cid:  %s\n  note: %s",
+				v.Name, got, v.Rank, v.CID, v.Note)
+		}
+	}
+}
+
 // buildCID assembles a multibase-base32 CIDv1 from a content codec, a
 // multihash code, and a digest length (digest bytes are zero — only the
 // header is under test). Mirrors the encoder in watcher/midhash.go and the
@@ -104,12 +150,31 @@ func TestRank_Tiers(t *testing.T) {
 func TestRank_StrictOrdering(t *testing.T) {
 	ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
 	sha := Rank(cidSha256)
+	posting := Rank(buildCID(CodeNzbPosting, CodeSha256, 32))
 	btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
 	mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
 	unknown := Rank(cidMd5)
-	if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
-		t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
-			ipfs, sha, btih, mid, unknown)
+	if !(ipfs > sha && sha > posting && posting > btih && btih > mid && mid > unknown) {
+		t.Fatalf("rank ordering broken: ipfs=%d sha=%d posting=%d btih=%d mid=%d unknown=%d",
+			ipfs, sha, posting, btih, mid, unknown)
+	}
+}
+
+// TestRank_NzbPostingMatchesCodecNotMultihash pins the inverse trap:
+// nzb-posting's multihash is a REAL sha2-256 (unlike nzb-release/url, whose mh
+// is identity 0x00), so Rank must match on the codec BEFORE falling into the
+// mh switch — otherwise this ties with a genuine content digest at 30 instead
+// of sitting at 25.
+func TestRank_NzbPostingMatchesCodecNotMultihash(t *testing.T) {
+	posting := buildCID(CodeNzbPosting, CodeSha256, 32)
+	if got := Rank(posting); got != RankNzbPosting {
+		t.Errorf("Rank(nzb-posting) = %d, want %d (RankNzbPosting) — codec check must precede the mh switch", got, RankNzbPosting)
+	}
+	// A real sha2-256 with the SAME digest bytes must still rank 30 — the
+	// codec, not the digest, is what distinguishes the two.
+	realDigest := buildCID(CodecRaw, CodeSha256, 32)
+	if got := Rank(realDigest); got != RankDigest {
+		t.Errorf("Rank(sha2-256) regressed to %d, want %d", got, RankDigest)
 	}
 }
 

internal/config/config.go

@@ -24,7 +24,7 @@ type Config struct {
 
 	// HTTP API configuration
 	HTTPPort int    // HTTP API port (default: 9000)
-	HTTPHost string // HTTP API host (default: "127.0.0.1")
+	HTTPHost string // HTTP API host (default: "0.0.0.0" — meta-core always runs in a container and must answer siblings)
 
 	// Timing configuration
 	HealthCheckIntervalMS    int // Health check interval in ms (default: 5000)
@@ -54,7 +54,7 @@ func Load() *Config {
 		PublicURL:             getEnv("META_CORE_PUBLIC_URL", ""),
 		RedisPort:             getEnvInt("REDIS_PORT", 6379),
 		HTTPPort:              getEnvInt("META_CORE_HTTP_PORT", 9000),
-		HTTPHost:              getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
+		HTTPHost:              getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
 		HealthCheckIntervalMS:  getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
 		HeartbeatIntervalMS:   getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
 		StaleThresholdMS:      getEnvInt("STALE_THRESHOLD_MS", 60000),

internal/events/meta_publisher.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/redis/go-redis/v9"
+
+	"github.com/metazla/meta-core/internal/storage"
 )
 
 const (
@@ -126,6 +128,14 @@ func (p *MetaPublisher) publishEvent(channel, operation string) {
 	if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
 		return
 	}
+	// Skip the per-record field index. It is bookkeeping (see
+	// storage.FieldsField), not a metadata field — publishing it would make
+	// every downstream consumer that parses `file:<id>/<field>` off this
+	// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
+	// field change on a key they can never read.
+	if strings.HasSuffix(key, "/"+storage.FieldsField) {
+		return
+	}
 
 	// Build event fields
 	fields := map[string]interface{}{

internal/snapshot/manifest.go

@@ -5,11 +5,13 @@
 // reject archives whose schemaVersion exceeds what they know.
 //
 // v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries
-// under metadata/<root>.json carry the flat string properties; the cids
-// and duplicates Redis SETs are NOT serialized yet — they are rebuilt
-// transparently on import because SetMetadataFlat / MergeMetadataFlat
-// auto-register reverse-index aliases for any cid_*/midhash256 field they
-// see (see internal/storage/cid_resolution.go).
+// under metadata/<root>.json carry the flat string properties; the cid
+// reverse-index and the duplicates SET are NOT serialized — they are rebuilt
+// transparently on import because the cids/<cid> key-set members ARE flat
+// properties, and SetMetadataFlat / MergeMetadataFlat auto-register a
+// reverse-index alias for every one they see (see
+// internal/storage/cid_resolution.go: cidFromKeysetField). The legacy flat
+// cid_*/midhash256 named fields are not recognised.
 //
 // v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc".
 // v1 archives can no longer be imported — they would resurrect the

internal/storage/cid_resolution.go

@@ -83,6 +83,7 @@ func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
 	pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
 	pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
 	pipe.SAdd(ctx, c.buildIndexKey(), uuid)
+	c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
 	if _, err := pipe.Exec(ctx); err != nil {
 		return "", fmt.Errorf("mint pipeline: %w", err)
 	}
@@ -116,7 +117,13 @@ func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error
 		return fmt.Errorf("uuid and cid are required")
 	}
 
-	keysetKey := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + cidStr
+	keysetField := CIDsKeyPrefix + cidStr
+	keysetKey := c.buildKeyPrefix(uuid) + keysetField
+
+	// Every branch below writes the key-set member, so index the field name
+	// once here — including the two guard branches that skip the alias itself.
+	// Missing it would leave `cids/<cid>` invisible to a record read.
+	defer c.addFieldLocked(ctx, uuid, keysetField)
 
 	// Self-pointing alias guard. If the caller is trying to register
 	// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
@@ -233,26 +240,13 @@ func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
 	return added > 0, nil
 }
 
-// cidsForRootLocked scans the bare-CID key-set for a root and returns its
-// members (bare CIDs, with the cids/ prefix stripped). Caller must hold c.mu.
+// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
+// prefix stripped). Caller must hold c.mu.
+//
+// Reads the field index and filters by prefix; only an un-indexed root falls
+// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
 func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
-	prefix := c.buildKeyPrefix(uuid) + CIDsKeyPrefix
-	var out []string
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan cids: %w", err)
-		}
-		for _, k := range keys {
-			out = append(out, strings.TrimPrefix(k, prefix))
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-	return out, nil
+	return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
 }
 
 // DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property

internal/storage/clear_test.go

@@ -0,0 +1,138 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+)
+
+// newTestClient spins an in-memory Redis and returns a connected Client with
+// the given namespace prefix. The server is torn down with the test.
+func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
+	t.Helper()
+
+	s := miniredis.RunT(t)
+	c := NewClient(prefix)
+	if err := c.Connect("redis://" + s.Addr()); err != nil {
+		t.Fatalf("connect to miniredis: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return c, s
+}
+
+// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
+// leak: ClearAllMetadata used to delete every file:<root>/* key and the
+// file:__index__ set, but left the cid:<cid> → root reverse-index entries
+// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
+// a surviving entry silently redirects a later write into a deleted root.
+func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
+	for _, prefix := range []string{"", "mm:"} {
+		name := "no-prefix"
+		if prefix != "" {
+			name = "prefixed"
+		}
+		t.Run(name, func(t *testing.T) {
+			c, s := newTestClient(t, prefix)
+
+			// Two records, each with several CID aliases (a midhash plus
+			// siblings) — exactly what the watcher produces.
+			aliases := map[string][]string{}
+			for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
+				uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
+				if err != nil {
+					t.Fatalf("mint: %v", err)
+				}
+				cids := []string{
+					"bafkmid" + uuid,
+					"bafkfull" + uuid,
+				}
+				for _, cidStr := range cids {
+					if err := c.AddAlias(uuid, cidStr); err != nil {
+						t.Fatalf("add alias %s: %v", cidStr, err)
+					}
+				}
+				aliases[uuid] = cids
+			}
+
+			// An entry already orphaned by a previous (buggy) clear: a
+			// reverse-index key whose root no longer exists. The sweep is
+			// expected to collect it too.
+			if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
+				t.Fatalf("seed orphan: %v", err)
+			}
+
+			// Sanity: the aliases resolve before the clear.
+			for uuid, cids := range aliases {
+				for _, cidStr := range cids {
+					if got := c.ResolveRoot(cidStr); got != uuid {
+						t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
+					}
+				}
+			}
+
+			deleted, err := c.ClearAllMetadata()
+			if err != nil {
+				t.Fatalf("ClearAllMetadata: %v", err)
+			}
+			// deletedCount counts files, not keys.
+			if deleted != 2 {
+				t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
+			}
+
+			// No reverse-index key may survive — not for the cleared records,
+			// not the pre-existing orphan.
+			if leaked := s.Keys(); len(leaked) != 0 {
+				t.Errorf("keys survived the clear: %v", leaked)
+			}
+
+			// And the aliases must no longer resolve: ResolveRoot returns the
+			// hash unchanged when there is no index entry.
+			for _, cids := range aliases {
+				for _, cidStr := range cids {
+					if got := c.ResolveRoot(cidStr); got != cidStr {
+						t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
+					}
+				}
+			}
+			if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
+				t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
+			}
+		})
+	}
+}
+
+// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
+// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
+// so a clear on a prefixed client cannot reach another namespace's index.
+func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
+	c, s := newTestClient(t, "mm:")
+
+	uuid, err := c.Mint("watch/a.mkv", 100, 1000)
+	if err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	if err := c.AddAlias(uuid, "bafkmine"); err != nil {
+		t.Fatalf("add alias: %v", err)
+	}
+
+	// Another namespace's reverse-index entry, plus an unrelated key.
+	if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
+		t.Fatalf("seed foreign cid: %v", err)
+	}
+	if err := s.Set("unrelated", "keep-me"); err != nil {
+		t.Fatalf("seed unrelated: %v", err)
+	}
+
+	if _, err := c.ClearAllMetadata(); err != nil {
+		t.Fatalf("ClearAllMetadata: %v", err)
+	}
+
+	for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
+		if !s.Exists(k) {
+			t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
+		}
+	}
+	if s.Exists(c.buildCIDIndexKey("bafkmine")) {
+		t.Errorf("own reverse-index entry survived the clear")
+	}
+}

internal/storage/client.go

@@ -109,8 +109,12 @@ func (c *Client) buildIndexKey() string {
 	return c.buildKey("file:__index__")
 }
 
-// GetMetadataFlat retrieves all metadata for a file as a flat map
-// Uses flat keys: SCAN file:{hashId}/* then MGET
+// GetMetadataFlat retrieves all metadata for a file as a flat map.
+//
+// Reads the record's field-name index and MGETs those keys — two round-trips,
+// independent of keyspace size. Records written before the index existed fall
+// back to the old SCAN and are indexed on the way out. See field_index.go for
+// why the index exists and why it is safe to derive.
 func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
@@ -122,41 +126,7 @@ func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	prefix := c.buildKeyPrefix(hashID)
-	result := make(map[string]string)
-
-	// Scan for all keys with this prefix
-	var cursor uint64
-	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan failed: %w", err)
-		}
-
-		if len(keys) > 0 {
-			values, err := c.client.MGet(ctx, keys...).Result()
-			if err != nil {
-				return nil, fmt.Errorf("mget failed: %w", err)
-			}
-			for i, key := range keys {
-				field := strings.TrimPrefix(key, prefix)
-				if values[i] != nil {
-					result[field] = values[i].(string)
-				}
-			}
-		}
-
-		cursor = nextCursor
-		if cursor == 0 {
-			break
-		}
-	}
-
-	if len(result) == 0 {
-		return nil, nil
-	}
-
-	return result, nil
+	return c.readMetadataLocked(ctx, hashID)
 }
 
 // SetMetadataFlat stores metadata for a file using flat keys
@@ -179,9 +149,21 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 	prefix := c.buildKeyPrefix(hashID)
 	pipe := c.client.Pipeline()
 
+	fields := make([]string, 0, len(metadata))
 	for field, value := range metadata {
-		pipe.Set(ctx, prefix+field, value, 0)
+		// A payload carrying the reserved name (a hand-made snapshot, a buggy
+		// client) would SET a string over the index SET and break every
+		// subsequent read of this record. Drop it — it is derived state, so
+		// there is never a reason to accept it from outside.
+		if isReservedFieldName(field) {
+			continue
 		}
+		pipe.Set(ctx, prefix+field, value, 0)
+		fields = append(fields, field)
+	}
+	// Same pipeline as the SETs, so indexing the field names costs no extra
+	// round-trip. See field_index.go.
+	c.addFieldsToPipe(ctx, pipe, hashID, fields)
 
 	if _, err := pipe.Exec(ctx); err != nil {
 		return fmt.Errorf("pipeline exec failed: %w", err)
@@ -192,8 +174,9 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
 
-	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
-	// in the payload. See cid_resolution.go for the rules.
+	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
+	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
+	// midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField).
 	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
 
 	return nil
@@ -255,6 +238,12 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
+	// See SetMetadataFlat: the reserved name addresses the derived index SET,
+	// not a metadata field.
+	if isReservedFieldName(property) {
+		return fmt.Errorf("field name %q is reserved", property)
+	}
+
 	key := c.buildKeyPrefix(hashID) + property
 
 	// Use SET to set the key
@@ -262,6 +251,8 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 		return fmt.Errorf("set failed: %w", err)
 	}
 
+	c.addFieldLocked(ctx, hashID, property)
+
 	// Add to index set
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return fmt.Errorf("sadd index failed: %w", err)
@@ -386,32 +377,14 @@ func normalizeFilesRelativePath(p string) string {
 // getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock.
 // Callers must already hold c.mu.
 func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) {
-	prefix := c.buildKeyPrefix(hashID)
-	result := make(map[string]string)
-
-	var cursor uint64
-	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+	result, err := c.readMetadataLocked(ctx, hashID)
 	if err != nil {
-			return nil, fmt.Errorf("scan failed: %w", err)
-		}
-		if len(keys) > 0 {
-			values, err := c.client.MGet(ctx, keys...).Result()
-			if err != nil {
-				return nil, fmt.Errorf("mget failed: %w", err)
-			}
-			for i, key := range keys {
-				if values[i] == nil {
-					continue
-				}
-				field := strings.TrimPrefix(key, prefix)
-				result[field] = values[i].(string)
-			}
-		}
-		cursor = nextCursor
-		if cursor == 0 {
-			break
+		return nil, err
 	}
+	// Unlike GetMetadataFlat, this variant hands back an empty map rather than
+	// nil for a record with no fields — GetMetadataDocument writes into it.
+	if result == nil {
+		result = make(map[string]string)
 	}
 	return result, nil
 }
@@ -448,9 +421,17 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 	prefix := c.buildKeyPrefix(hashID)
 	pipe := c.client.Pipeline()
 
+	fields := make([]string, 0, len(metadata))
 	for field, value := range metadata {
-		pipe.Set(ctx, prefix+field, value, 0)
+		// See SetMetadataFlat: never let an inbound payload overwrite the
+		// derived index SET with a string.
+		if isReservedFieldName(field) {
+			continue
 		}
+		pipe.Set(ctx, prefix+field, value, 0)
+		fields = append(fields, field)
+	}
+	c.addFieldsToPipe(ctx, pipe, hashID, fields)
 
 	if _, err := pipe.Exec(ctx); err != nil {
 		return 0, fmt.Errorf("pipeline exec failed: %w", err)
@@ -461,8 +442,9 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 		return len(metadata), fmt.Errorf("sadd index failed: %w", err)
 	}
 
-	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
-	// in the payload. See cid_resolution.go for the rules.
+	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
+	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
+	// midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField).
 	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
 
 	return len(metadata), nil
@@ -482,7 +464,11 @@ func (c *Client) DeleteProperty(hashID, property string) error {
 	defer cancel()
 
 	key := c.buildKeyPrefix(hashID) + property
-	return c.client.Del(ctx, key).Err()
+	if err := c.client.Del(ctx, key).Err(); err != nil {
+		return err
+	}
+	c.removeFieldLocked(ctx, hashID, property)
+	return nil
 }
 
 // AddToSet adds a value to a set-type field (stored as pipe-delimited string in flat key)
@@ -528,6 +514,9 @@ func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
 		return false, err
 	}
 
+	// The SET above may have created the key, so the field name is new.
+	c.addFieldLocked(ctx, hashID, property)
+
 	// Add to index set
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return true, fmt.Errorf("sadd index failed: %w", err)
@@ -580,7 +569,12 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
 
 	// Save back (or delete key if empty)
 	if len(newValues) == 0 {
-		return true, c.client.Del(ctx, key).Err()
+		if err := c.client.Del(ctx, key).Err(); err != nil {
+			return true, err
+		}
+		// The key is gone, so the field name must leave the index too.
+		c.removeFieldLocked(ctx, hashID, property)
+		return true, nil
 	}
 
 	newValue := strings.Join(newValues, "|")
@@ -762,6 +756,12 @@ func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
 			if k == indexKey {
 				continue
 			}
+			// Per-record field indexes live under the same prefix and are SETs
+			// — MGET would error on them, and they are bookkeeping, not
+			// searchable metadata.
+			if strings.HasSuffix(k, "/"+FieldsField) {
+				continue
+			}
 			keyBatch = append(keyBatch, k)
 			if len(keyBatch) >= 1000 {
 				if err := flush(); err != nil {
@@ -819,8 +819,16 @@ func (c *Client) GetMemoryInfo() (string, error) {
 	return "N/A", nil
 }
 
-// ClearAllMetadata deletes all file metadata and index
-// Returns the number of files deleted
+// ClearAllMetadata deletes all file metadata, the file index, and the CID
+// reverse index. Returns the number of files deleted.
+//
+// The CID reverse index (cid:<cid> → root, see cid_resolution.go) MUST be
+// purged too: ResolveRoot consults it, so a surviving cid:<cid> entry pointing
+// at a root we just deleted silently redirects the next /meta/<cid> write back
+// into that dead root, resurrecting a zombie record under an old identity.
+// DeleteRoot already does this per-record; here we sweep the whole prefix
+// rather than deriving the CID list per root, which is both simpler and
+// self-healing — it also collects entries orphaned by earlier clears.
 func (c *Client) ClearAllMetadata() (int64, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -868,12 +876,40 @@ func (c *Client) ClearAllMetadata() (int64, error) {
 		deletedCount++
 	}
 
+	// Delete every CID reverse-index entry (cid:<cid> → root). Swept by
+	// prefix rather than per-root so entries orphaned by an earlier clear —
+	// which did not purge this index at all — are collected too. Built via
+	// buildCIDIndexKey so the configured namespace prefix is honoured.
+	cidIndexPattern := c.buildCIDIndexKey("*")
+	var cidIndexDeleted int64
+	var cidCursor uint64
+	for {
+		keys, nextCursor, err := c.client.Scan(ctx, cidCursor, cidIndexPattern, 1000).Result()
+		if err != nil {
+			log.Printf("[Storage] Warning: failed to scan %s: %v", cidIndexPattern, err)
+			break
+		}
+
+		if len(keys) > 0 {
+			if err := c.client.Del(ctx, keys...).Err(); err != nil {
+				log.Printf("[Storage] Warning: failed to delete cid index keys: %v", err)
+			} else {
+				cidIndexDeleted += int64(len(keys))
+			}
+		}
+
+		cidCursor = nextCursor
+		if cidCursor == 0 {
+			break
+		}
+	}
+
 	// Delete the index set
 	if err := c.client.Del(ctx, indexKey).Err(); err != nil {
 		log.Printf("[Storage] Warning: failed to delete index: %v", err)
 	}
 
-	log.Printf("[Storage] Cleared %d file metadata entries", deletedCount)
+	log.Printf("[Storage] Cleared %d file metadata entries (%d cid index entries)", deletedCount, cidIndexDeleted)
 	return deletedCount, nil
 }
 
@@ -1001,6 +1037,12 @@ func (c *Client) FindByValue(contains string, fields []string, limit int) ([]Fin
 		if field == "" {
 			continue
 		}
+		// The per-record field index is a SET, so an MGET over those keys
+		// would fail the whole query with WRONGTYPE. It holds field names,
+		// not values, so there is nothing here to search anyway.
+		if isReservedFieldName(field) {
+			continue
+		}
 		if len(matches) >= limit {
 			truncated = true
 			break
@@ -1169,7 +1211,17 @@ func (c *Client) SetRawValue(key, value string) error {
 	if t != "none" && t != "string" {
 		return fmt.Errorf("key has incompatible type %q (only string is editable)", t)
 	}
-	return c.client.Set(ctx, full, value, 0).Err()
+	if err := c.client.Set(ctx, full, value, 0).Err(); err != nil {
+		return err
+	}
+	// The raw editor can mint a record property out of band, so it has to keep
+	// the field index honest like every other write path. Non-record keys
+	// (`cid:*`, `udl:*`, `file:__index__`, the index set itself) split false
+	// and are left alone.
+	if hashID, field, ok := c.splitRecordKey(full); ok {
+		c.addFieldLocked(ctx, hashID, field)
+	}
+	return nil
 }
 
 // DeleteRawKey deletes the exact key. Returns true if the key existed.
@@ -1184,10 +1236,16 @@ func (c *Client) DeleteRawKey(key string) (bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	n, err := c.client.Del(ctx, c.buildKey(key)).Result()
+	full := c.buildKey(key)
+	n, err := c.client.Del(ctx, full).Result()
 	if err != nil {
 		return false, err
 	}
+	if n > 0 {
+		if hashID, field, ok := c.splitRecordKey(full); ok {
+			c.removeFieldLocked(ctx, hashID, field)
+		}
+	}
 	return n > 0, nil
 }
 
@@ -1219,14 +1277,20 @@ func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	prefix := c.buildKeyPrefix(hashID)
-	keys, _, err := c.client.Scan(ctx, 0, prefix+"*", 1).Result()
+	nonEmpty, err := c.recordHasFieldsLocked(ctx, hashID)
 	if err != nil {
-		return false, fmt.Errorf("scan failed: %w", err)
+		return false, err
 	}
-	if len(keys) > 0 {
+	if nonEmpty {
 		return false, nil
 	}
+
+	// Nothing left but (possibly) an empty field index — drop it too, so a
+	// de-indexed root leaves no orphan key behind.
+	if err := c.client.Del(ctx, c.buildFieldsKey(hashID)).Err(); err != nil {
+		return false, fmt.Errorf("del field index: %w", err)
+	}
+
 	removed, err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Result()
 	if err != nil {
 		return false, fmt.Errorf("srem failed: %w", err)

internal/storage/field_index.go

@@ -0,0 +1,486 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// The per-record field-name index.
+//
+// A record is stored as one flat Redis STRING per leaf
+// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
+// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
+// meta-search can react to one field changing instead of diffing a whole hash
+// (see docs/project-architecture/persistent-storage.md, and
+// dev/test/suites/flat-keys.bats which asserts records are never hashes).
+//
+// The cost of that shape was that reading one record meant asking Redis "which
+// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
+// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
+// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
+// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
+// the byte path for every poster meta-share served, which is how it surfaced.
+//
+// So we keep the flat keys and index their *names*:
+//
+//	file:<uuid>/__fields__   SET{ "filePath", "sizeByte", "cids/bafk…", … }
+//
+// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
+// This mirrors `file:__index__` (the set of every root) one level down, and the
+// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
+// from the index rather than scanning per root.
+//
+// **The set is derived, never authoritative.** The flat keys remain the source
+// of truth. A record whose set is missing (anything written before this landed)
+// falls back to the old SCAN and populates the set on the way out, so existing
+// deployments heal themselves on first touch with no migration and no
+// SchemaVersion bump — which matters, because a bump is a flag day that makes
+// operators wipe Redis (see schema_version.go).
+//
+// The consistency argument that makes a derived index safe here is meta-core's
+// single-writer rule: leader election publishes an empty `redisUrl` on purpose
+// (internal/leader/election.go), so nothing outside this process can write a
+// `file:` key behind the index's back.
+
+// FieldsField is the reserved field name holding a record's field-name index.
+//
+// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
+// prefix as the record's own fields. That keeps every existing sweep correct
+// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
+// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
+// The flip side is that every reader which enumerates that prefix has to skip
+// it, exactly as they already skip `file:__index__`.
+const FieldsField = "__fields__"
+
+// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
+// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
+const mgetBatch = 1000
+
+// buildFieldsKey constructs the key for a record's field-name index set.
+func (c *Client) buildFieldsKey(hashID string) string {
+	return c.buildKeyPrefix(hashID) + FieldsField
+}
+
+// isReservedFieldName reports whether a field name is bookkeeping rather than
+// metadata, and so must never be returned to an API caller, exported into a
+// snapshot, or folded into the search haystack.
+func isReservedFieldName(field string) bool {
+	return field == FieldsField
+}
+
+// readMetadataLocked returns a record's full flat metadata. Caller must hold
+// c.mu (either mode).
+//
+// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
+// legacy full-keyspace SCAN, whose result is used to populate the index so the
+// next read takes the fast path.
+//
+// Returns (nil, nil) for a record with no fields — callers treat that as 404.
+func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
+	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers fields: %w", err)
+	}
+
+	if len(fields) > 0 {
+		return c.mgetFieldsLocked(ctx, hashID, fields)
+	}
+
+	// No index yet: a record written before the index existed, or one that
+	// genuinely has no fields. Fall back to the scan, then backfill.
+	result, scanned, err := c.scanMetadataLocked(ctx, hashID)
+	if err != nil {
+		return nil, err
+	}
+	if len(scanned) > 0 {
+		// Best-effort: a failed backfill costs the next read a scan, it does
+		// not make the read wrong. Never fail the caller's read for it.
+		if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
+			logFieldIndexBackfillFailure(hashID, err)
+		}
+	}
+	if len(result) == 0 {
+		return nil, nil
+	}
+	return result, nil
+}
+
+// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
+//
+// A field named in the index whose key is gone reads back as nil and is simply
+// skipped: a *stale* index entry is harmless. (A *missing* one would silently
+// drop data, which is why every write path maintains the set.)
+func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
+	prefix := c.buildKeyPrefix(hashID)
+
+	// Keep the reserved name out of the key list AND out of the result — it is
+	// a SET, so MGET would error on it in real Redis rather than return nil.
+	wanted := make([]string, 0, len(fields))
+	for _, f := range fields {
+		if !isReservedFieldName(f) {
+			wanted = append(wanted, f)
+		}
+	}
+	if len(wanted) == 0 {
+		return nil, nil
+	}
+
+	result := make(map[string]string, len(wanted))
+	for start := 0; start < len(wanted); start += mgetBatch {
+		end := start + mgetBatch
+		if end > len(wanted) {
+			end = len(wanted)
+		}
+		chunk := wanted[start:end]
+		keys := make([]string, len(chunk))
+		for i, f := range chunk {
+			keys[i] = prefix + f
+		}
+		values, err := c.client.MGet(ctx, keys...).Result()
+		if err != nil {
+			return nil, fmt.Errorf("mget failed: %w", err)
+		}
+		for i, v := range values {
+			if s, ok := v.(string); ok {
+				result[chunk[i]] = s
+			}
+		}
+	}
+
+	if len(result) == 0 {
+		return nil, nil
+	}
+	return result, nil
+}
+
+// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
+// MGET. Retained as the fallback for un-indexed records and as the source for
+// the backfill. Returns the metadata plus the field names it saw (which
+// includes fields whose value came back nil, so the index stays complete).
+func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
+	prefix := c.buildKeyPrefix(hashID)
+	result := make(map[string]string)
+	var fields []string
+
+	var cursor uint64
+	for {
+		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return nil, nil, fmt.Errorf("scan failed: %w", err)
+		}
+
+		if len(keys) > 0 {
+			wanted := make([]string, 0, len(keys))
+			for _, key := range keys {
+				field := strings.TrimPrefix(key, prefix)
+				if isReservedFieldName(field) {
+					continue
+				}
+				fields = append(fields, field)
+				wanted = append(wanted, key)
+			}
+			if len(wanted) > 0 {
+				values, err := c.client.MGet(ctx, wanted...).Result()
+				if err != nil {
+					return nil, nil, fmt.Errorf("mget failed: %w", err)
+				}
+				for i, key := range wanted {
+					if s, ok := values[i].(string); ok {
+						result[strings.TrimPrefix(key, prefix)] = s
+					}
+				}
+			}
+		}
+
+		cursor = nextCursor
+		if cursor == 0 {
+			break
+		}
+	}
+
+	return result, fields, nil
+}
+
+// addFieldsToPipe records field names in the index as part of an existing
+// pipeline, so a multi-field write stays one round-trip.
+func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
+	names := filterReserved(fields)
+	if len(names) == 0 {
+		return
+	}
+	pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
+}
+
+// addFieldLocked records a single field name in the index. Caller must hold
+// c.mu. Best-effort by the same argument as the backfill: the flat key is the
+// source of truth, and a lost index entry is repaired by the next scan
+// fallback — so it must not fail the write it accompanies.
+func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
+	if isReservedFieldName(field) {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
+		logFieldIndexWriteFailure(hashID, field, err)
+	}
+}
+
+// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
+//
+// A stale entry left behind by a failure is harmless (the MGET reads nil and
+// skips it), so this is best-effort too.
+func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
+	if isReservedFieldName(field) {
+		return
+	}
+	if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
+		logFieldIndexWriteFailure(hashID, field, err)
+	}
+}
+
+// fieldsWithPrefixLocked returns the record's indexed field names under a
+// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
+// no index yet. Caller must hold c.mu.
+func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
+	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers fields: %w", err)
+	}
+	if len(fields) == 0 {
+		// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
+		// rather than the whole record, and do not backfill from it — a
+		// partial view must never be mistaken for a complete index.
+		return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
+	}
+
+	var out []string
+	for _, f := range fields {
+		if isReservedFieldName(f) {
+			continue
+		}
+		if strings.HasPrefix(f, fieldPrefix) {
+			out = append(out, strings.TrimPrefix(f, fieldPrefix))
+		}
+	}
+	// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
+	// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
+	// at these sizes and makes the response byte-stable, which matters for
+	// anything diffing or caching it.
+	sort.Strings(out)
+	return out, nil
+}
+
+func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
+	prefix := c.buildKeyPrefix(hashID) + fieldPrefix
+	var out []string
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return nil, fmt.Errorf("scan fields: %w", err)
+		}
+		for _, k := range keys {
+			out = append(out, strings.TrimPrefix(k, prefix))
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
+//
+// Without it the indexes still appear — each record's first read falls back to
+// a scan and backfills — but that transition is O(records x keyspace) and it
+// runs again from scratch after any wipe. Measured on the real store (12,345
+// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
+// during which meta-core stays pinned exactly as it was before the fix. Worse,
+// `RepublishMetadata` walks every record at startup, so that tail is paid on
+// every restart.
+//
+// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
+// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
+// for the same reason.
+//
+// Idempotent, and safe to run against a partially-indexed store: SADD of a
+// member that is already there is a no-op. Returns the number of records it
+// wrote an index for.
+func (c *Client) WarmFieldIndexes() (int, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return 0, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
+	indexKey := c.buildIndexKey()
+
+	byRecord := make(map[string][]string)
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
+		if err != nil {
+			return 0, fmt.Errorf("scan: %w", err)
+		}
+		for _, k := range keys {
+			if k == indexKey {
+				continue
+			}
+			// hashID has no '/'; field may contain them.
+			rest := strings.TrimPrefix(k, filePrefix)
+			slash := strings.IndexByte(rest, '/')
+			if slash < 0 {
+				continue
+			}
+			hashID, field := rest[:slash], rest[slash+1:]
+			if isReservedFieldName(field) {
+				// Already indexed — remember the record so we don't treat it
+				// as missing, but there is nothing to add for this key.
+				if _, seen := byRecord[hashID]; !seen {
+					byRecord[hashID] = nil
+				}
+				continue
+			}
+			byRecord[hashID] = append(byRecord[hashID], field)
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+
+	written := 0
+	pipe := c.client.Pipeline()
+	pending := 0
+	for hashID, fields := range byRecord {
+		if len(fields) == 0 {
+			continue
+		}
+		pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
+		written++
+		pending++
+		// Bound the pipeline so a large store doesn't buffer every command.
+		if pending >= 500 {
+			if _, err := pipe.Exec(ctx); err != nil {
+				return written, fmt.Errorf("sadd pipeline: %w", err)
+			}
+			pipe = c.client.Pipeline()
+			pending = 0
+		}
+	}
+	if pending > 0 {
+		if _, err := pipe.Exec(ctx); err != nil {
+			return written, fmt.Errorf("sadd pipeline: %w", err)
+		}
+	}
+	return written, nil
+}
+
+// recordHasFieldsLocked reports whether a record still has at least one
+// property key. Caller must hold c.mu.
+//
+// `__fields__` must not count — it lives under the record's own prefix, so a
+// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
+// whose every property has been deleted, and the root could never be
+// de-indexed.
+func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
+	n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
+	if err != nil && err != redis.Nil {
+		return false, fmt.Errorf("scard fields: %w", err)
+	}
+	if n > 0 {
+		return true, nil
+	}
+
+	// Either genuinely empty, or an un-indexed record from before the index
+	// existed — the scan is the only way to tell those apart.
+	prefix := c.buildKeyPrefix(hashID)
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return false, fmt.Errorf("scan failed: %w", err)
+		}
+		for _, k := range keys {
+			if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
+				return true, nil
+			}
+		}
+		cursor = next
+		if cursor == 0 {
+			return false, nil
+		}
+	}
+}
+
+// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
+// record property key. Reports ok=false for anything else — the index set
+// itself, `file:__index__`, and any key outside the `file:` namespace.
+//
+// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
+// the index honest without its callers knowing the layout.
+func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
+	rest := key
+	if c.prefix != "" {
+		var found bool
+		rest, found = strings.CutPrefix(rest, c.prefix)
+		if !found {
+			return "", "", false
+		}
+	}
+	rest, found := strings.CutPrefix(rest, "file:")
+	if !found {
+		return "", "", false
+	}
+	hashID, field, found = strings.Cut(rest, "/")
+	if !found || hashID == "" || field == "" {
+		// `file:__index__` has no slash and lands here.
+		return "", "", false
+	}
+	if isReservedFieldName(field) {
+		return "", "", false
+	}
+	return hashID, field, true
+}
+
+// The index is derived and self-healing, so a write failure degrades
+// performance (the next read falls back to a scan and re-populates) rather
+// than correctness. Log it rather than failing the caller's write — but log it
+// loudly, because a *persistent* failure means every read is scanning again.
+func logFieldIndexWriteFailure(hashID, field string, err error) {
+	log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
+}
+
+func logFieldIndexBackfillFailure(hashID string, err error) {
+	log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
+}
+
+func filterReserved(fields []string) []string {
+	out := make([]string, 0, len(fields))
+	for _, f := range fields {
+		if !isReservedFieldName(f) {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+func toAny(ss []string) []interface{} {
+	out := make([]interface{}, len(ss))
+	for i, s := range ss {
+		out[i] = s
+	}
+	return out
+}

internal/storage/field_index_test.go

@@ -0,0 +1,464 @@
+package storage
+
+import (
+	"context"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+// Tests for the per-record field-name index (field_index.go).
+//
+// The index is what makes a record read O(fields) instead of O(keyspace). It is
+// derived state, so the properties worth pinning are: (1) a read served from
+// the index returns exactly what a scan would have, (2) a record with no index
+// still reads correctly and gets one, and (3) every write path keeps it in
+// sync — because a *missing* entry silently drops a field from every
+// subsequent read, which is the one failure mode that would be invisible.
+//
+// Every case runs with and without a namespace prefix, matching clear_test.go.
+
+func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
+	t.Helper()
+	for _, prefix := range []string{"", "mm:"} {
+		name := "no-prefix"
+		if prefix != "" {
+			name = "prefix-" + prefix
+		}
+		t.Run(name, func(t *testing.T) {
+			c, _ := newTestClient(t, prefix)
+			fn(t, c)
+		})
+	}
+}
+
+func fieldNames(t *testing.T, c *Client, hashID string) []string {
+	t.Helper()
+	got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
+	if err != nil {
+		t.Fatalf("smembers fields: %v", err)
+	}
+	sort.Strings(got)
+	return got
+}
+
+func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
+	t.Helper()
+	got, err := c.GetMetadataFlat(hashID)
+	if err != nil {
+		t.Fatalf("GetMetadataFlat: %v", err)
+	}
+	return got
+}
+
+// SetMetadataFlat must index every field it writes, and the read must come back
+// through the index rather than a scan.
+func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		want := map[string]string{
+			"filePath":                      "/files/watch/a.mkv",
+			"sizeByte":                      "123",
+			"cids/bafkreiaaa":               "true",
+			"titles/jpn":                    "ナルト",
+			"categories/TV/UHD":             "true",
+			"info/files/0/path":             "a.mkv",
+			"provenance":                    `{"source":"gateway"}`,
+			"plot/eng":                      "a plot with / slashes",
+			"stream/3":                      "eng",
+			"genres/action":                 "true",
+			"originaltitle":                 "Naruto",
+			"deeply/nested/field/name/here": "x",
+		}
+		if err := c.SetMetadataFlat("root1", want); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
+			t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
+		}
+
+		// The index must name every field — including the ones containing '/',
+		// which is where a naive "split on the first slash" would go wrong.
+		wantNames := make([]string, 0, len(want))
+		for k := range want {
+			wantNames = append(wantNames, k)
+		}
+		sort.Strings(wantNames)
+		if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
+			t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
+		}
+	})
+}
+
+// A record written before the index existed has flat keys but no set. The read
+// must still be correct, and must leave an index behind so the next read takes
+// the fast path. This is the migration path for the ~12k records already on
+// disk: no flag day, no SchemaVersion bump.
+func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		prefix := c.buildKeyPrefix("legacy")
+		want := map[string]string{
+			"filePath":        "/files/watch/old.mkv",
+			"sizeByte":        "42",
+			"cids/bafkreiold": "true",
+		}
+		// Write the flat keys directly, bypassing every index-maintaining path
+		// — exactly the shape a pre-upgrade store is in.
+		for f, v := range want {
+			if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
+				t.Fatalf("seed %s: %v", f, err)
+			}
+		}
+		if names := fieldNames(t, c, "legacy"); len(names) != 0 {
+			t.Fatalf("precondition: expected no index, got %v", names)
+		}
+
+		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
+			t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
+		}
+
+		wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
+		if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
+			t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
+		}
+
+		// Second read, now served from the index, must agree with the first.
+		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
+			t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
+		}
+	})
+}
+
+// Each single-field write path must add to the index, and each delete path must
+// remove from it. A field written but not indexed vanishes from every later
+// read — silently — so this is the property that matters most.
+func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
+			t.Fatalf("SetProperty: %v", err)
+		}
+		if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
+			t.Fatalf("AddToSet: %v", err)
+		}
+		if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
+			t.Fatalf("MergeMetadataFlat: %v", err)
+		}
+
+		want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
+		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
+			t.Errorf("after writes\n got %v\nwant %v", got, want)
+		}
+
+		// DeleteProperty drops the key and the index entry.
+		if err := c.DeleteProperty("root2", "title"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		// RemoveFromSet deletes the key once its last member is gone.
+		if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
+			t.Fatalf("RemoveFromSet: %v", err)
+		}
+
+		want = map[string]string{"year": "2002"}
+		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
+			t.Errorf("after deletes\n got %v\nwant %v", got, want)
+		}
+		if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
+			t.Errorf("index after deletes = %v, want [year]", got)
+		}
+	})
+}
+
+// The raw KV editor can mint or drop a record property out of band. It is the
+// one write path whose caller does not know the record layout, so the storage
+// layer has to keep the index honest for it.
+func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
+			t.Fatalf("SetRawValue: %v", err)
+		}
+		if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
+			t.Errorf("index after SetRawValue = %v", got)
+		}
+		if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
+			t.Errorf("read after SetRawValue = %v", got)
+		}
+
+		if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
+			t.Fatalf("DeleteRawKey: %v", err)
+		}
+		if got := fieldNames(t, c, "root3"); len(got) != 0 {
+			t.Errorf("index after DeleteRawKey = %v, want empty", got)
+		}
+
+		// Keys outside the record namespace must be left alone entirely.
+		if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
+			t.Fatalf("SetRawValue non-record: %v", err)
+		}
+		if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
+			t.Error("cid: key was treated as a record property")
+		}
+		if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
+			t.Error("file:__index__ was treated as a record property")
+		}
+	})
+}
+
+// Mint and AddAlias write fields through their own pipelines, so they need the
+// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
+// it and a record's CID key-set becomes invisible to every read.
+func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
+		if err != nil {
+			t.Fatalf("Mint: %v", err)
+		}
+		if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
+			t.Errorf("index after Mint = %v", got)
+		}
+
+		if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
+			t.Fatalf("AddAlias: %v", err)
+		}
+		got := mustGetMetadata(t, c, uuid)
+		if got["cids/bafkreiaaa"] != "true" {
+			t.Errorf("alias key-set member missing from read: %v", got)
+		}
+
+		// And the CID enumeration, which now reads the index instead of
+		// scanning, must see it.
+		cids, err := c.cidsForRootLocked(context.Background(), uuid)
+		if err != nil {
+			t.Fatalf("cidsForRootLocked: %v", err)
+		}
+		if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
+			t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
+		}
+	})
+}
+
+// The index lives under the record's own prefix, so the "is this record empty?"
+// probe would otherwise always find it and the root could never be de-indexed.
+func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetProperty("root4", "title", "x"); err != nil {
+			t.Fatalf("SetProperty: %v", err)
+		}
+
+		removed, err := c.IndexRemoveIfEmpty("root4")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if removed {
+			t.Error("de-indexed a record that still has a field")
+		}
+
+		if err := c.DeleteProperty("root4", "title"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		removed, err = c.IndexRemoveIfEmpty("root4")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if !removed {
+			t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
+		}
+		// ...and the now-empty index key must not be left orphaned.
+		n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
+		if err != nil {
+			t.Fatalf("exists: %v", err)
+		}
+		if n != 0 {
+			t.Error("orphaned field index left behind after de-indexing")
+		}
+	})
+}
+
+// The same probe must still work for a record that predates the index.
+func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+		if err := c.IndexAdd("legacy2"); err != nil {
+			t.Fatalf("IndexAdd: %v", err)
+		}
+		removed, err := c.IndexRemoveIfEmpty("legacy2")
+		if err != nil {
+			t.Fatalf("IndexRemoveIfEmpty: %v", err)
+		}
+		if removed {
+			t.Error("de-indexed an un-indexed record that still has a field")
+		}
+	})
+}
+
+// The reserved name is bookkeeping. It must never reach an API caller as a
+// metadata field, and an inbound payload must never be able to overwrite the
+// SET with a string — that would break every subsequent read of the record.
+func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		err := c.SetMetadataFlat("root5", map[string]string{
+			"title":     "x",
+			FieldsField: "malicious",
+		})
+		if err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		got := mustGetMetadata(t, c, "root5")
+		if _, present := got[FieldsField]; present {
+			t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
+		}
+		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
+			t.Errorf("read = %v, want just title", got)
+		}
+
+		// The index must still be a SET — i.e. not clobbered by the payload.
+		if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
+			t.Errorf("index = %v, want [title]", got)
+		}
+
+		if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
+			t.Error("SetProperty accepted the reserved field name")
+		}
+	})
+}
+
+// Deleting a record must take its index with it, or a recreated root would
+// inherit phantom field names.
+func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		if _, err := c.DeleteMetadata("root6"); err != nil {
+			t.Fatalf("DeleteMetadata: %v", err)
+		}
+		if got := fieldNames(t, c, "root6"); len(got) != 0 {
+			t.Errorf("index survived DeleteMetadata: %v", got)
+		}
+		got, err := c.GetMetadataFlat("root6")
+		if err != nil {
+			t.Fatalf("GetMetadataFlat: %v", err)
+		}
+		if got != nil {
+			t.Errorf("deleted record still reads back: %v", got)
+		}
+	})
+}
+
+// A stale index entry (name present, key gone) is the benign direction: the
+// MGET reads nil and the field is skipped rather than erroring.
+func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+		if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		// Name a field that has no key behind it.
+		if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
+			t.Fatalf("sadd ghost: %v", err)
+		}
+
+		got := mustGetMetadata(t, c, "root7")
+		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
+			t.Errorf("read = %v, want just title", got)
+		}
+	})
+}
+
+// WarmFieldIndexes is what keeps an upgrade from spending hours reading
+// un-indexed records one full-keyspace scan at a time. It must index every
+// record in one pass, be idempotent, and leave already-indexed records alone.
+func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		ctx := context.Background()
+
+		// Two legacy records seeded straight into Redis (no index), plus one
+		// written through the normal path (already indexed).
+		legacy := map[string]map[string]string{
+			"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
+			"old2": {"filePath": "/b.mkv", "titles/jpn": "ナルト", "deep/a/b": "x"},
+		}
+		for root, fields := range legacy {
+			for f, v := range fields {
+				if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
+					t.Fatalf("seed: %v", err)
+				}
+			}
+			if err := c.IndexAdd(root); err != nil {
+				t.Fatalf("IndexAdd: %v", err)
+			}
+		}
+		if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		n, err := c.WarmFieldIndexes()
+		if err != nil {
+			t.Fatalf("WarmFieldIndexes: %v", err)
+		}
+		if n < 2 {
+			t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
+		}
+
+		if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
+			t.Errorf("old1 index = %v", got)
+		}
+		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn"}) {
+			t.Errorf("old2 index = %v", got)
+		}
+		if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
+			t.Errorf("fresh index disturbed = %v", got)
+		}
+
+		// Reads must now agree with what the records actually hold.
+		for root, want := range legacy {
+			if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
+				t.Errorf("%s read = %v, want %v", root, got, want)
+			}
+		}
+
+		// Idempotent: a second pass changes nothing.
+		if _, err := c.WarmFieldIndexes(); err != nil {
+			t.Fatalf("second WarmFieldIndexes: %v", err)
+		}
+		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn"}) {
+			t.Errorf("old2 index after second warm = %v", got)
+		}
+	})
+}
+
+// The bulk search read enumerates the whole file: keyspace and MGETs what it
+// finds. The index keys are SETs living in that same namespace, so it has to
+// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
+func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		if err := c.SetMetadataFlat("root8", map[string]string{
+			"title":    "Naruto Shippuden",
+			"fileName": "naruto.mkv",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		tuples, err := c.GetSearchIndexTuples()
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuples: %v", err)
+		}
+		var found bool
+		for _, tu := range tuples {
+			if tu.HashID != "root8" {
+				continue
+			}
+			found = true
+			if _, present := tu.Fields[FieldsField]; present {
+				t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
+			}
+		}
+		if !found {
+			t.Fatal("record missing from search tuples")
+		}
+	})
+}

internal/storage/udl.go

@@ -0,0 +1,302 @@
+package storage
+
+// User Data Layer (UDL) storage.
+//
+// Holds per-user, identity-signed records keyed by (uid, cid, key) — the
+// subjective/per-user state that METADATA_KEYS.md rule #4 bars from the shared
+// /file/{cid} hash (likes, ratings, watch progress, local paths). Each record
+// is an opaque, client-signed (and, for private-tier keys, client-encrypted)
+// CBOR blob; meta-core stores it verbatim and never needs the account key.
+//
+// Convergence is version-gated: a write is accepted only if its `version`
+// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
+// op-log merge that used to live in meta-watch's store.rs).
+//
+// Redis model (all under the literal "udl:" prefix, mirroring the "file:" one):
+//
+//	udl:rec:<uid>/<cid>/<key>       HASH { record, version, ts, value? }
+//	udl:idx:user:<uid>:key:<key>    SET of <cid>   -> active_for_user_key (My List, Continue Watching)
+//	udl:idx:user:<uid>:cid:<cid>    SET of <key>   -> list_for_user_cid
+//	udl:idx:cid:<cid>:key:<key>     SET of <uid>   -> aggregate
+//	udl:idx:cid:<cid>:users         SET of <uid>   -> uids_for_cid
+//
+// The `value` hash field is the plaintext JSON scalar for PUBLIC keys only
+// (like/rating), used for aggregation; it is absent for private-tier keys
+// (whose real value rides encrypted inside `record`) and for tombstones.
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// UDLCell is a decoded (uid, cid, key) record cell.
+type UDLCell struct {
+	Record   string // base64 CBOR, opaque + signed
+	Version  int64
+	Ts       int64
+	Value    string // plaintext JSON scalar, public keys only
+	HasValue bool
+}
+
+// UDLUserKeyEntry is one row of active_for_user_key (all cids for a uid+key).
+type UDLUserKeyEntry struct {
+	Cid     string `json:"cid"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+// UDLUserCidEntry is one row of list_for_user_cid (all keys for a uid+cid).
+type UDLUserCidEntry struct {
+	Key     string `json:"key"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+func udlRecKey(uid, cid, key string) string {
+	return fmt.Sprintf("udl:rec:%s/%s/%s", uid, cid, key)
+}
+func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
+func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
+func udlIdxCidKey(cid, key string) string  { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
+func udlIdxCidUsers(cid string) string      { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
+
+// udlUpsertScript performs the version-gated upsert + index maintenance
+// atomically. Accepts (returns 1) only when the incoming version strictly
+// exceeds the stored one; otherwise returns 0 (stale).
+//
+//	KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
+//	ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
+//	      6=cid 7=key 8=uid 9=tombstone("1"/"0")
+//
+// A tombstone still writes the cell — that is the CRDT state, and dropping it
+// would let an older write resurrect the value — but it SREMs the cid from the
+// user+key index. That index is what `active_for_user_key` returns, so leaving
+// deleted cids in it means every My List / Continue Watching read carries every
+// title ever un-liked or finished, forever. The other three indexes are left
+// alone: they answer "who has an opinion about this cid", where a tombstone is
+// still an answer.
+var udlUpsertScript = redis.NewScript(`
+local stored = redis.call('HGET', KEYS[1], 'version')
+if stored and tonumber(stored) >= tonumber(ARGV[1]) then
+  return 0
+end
+redis.call('HSET', KEYS[1], 'version', ARGV[1], 'ts', ARGV[2], 'record', ARGV[3])
+if ARGV[5] == '1' then
+  redis.call('HSET', KEYS[1], 'value', ARGV[4])
+else
+  redis.call('HDEL', KEYS[1], 'value')
+end
+if ARGV[9] == '1' then
+  redis.call('SREM', KEYS[2], ARGV[6])
+else
+  redis.call('SADD', KEYS[2], ARGV[6])
+end
+redis.call('SADD', KEYS[3], ARGV[7])
+redis.call('SADD', KEYS[4], ARGV[8])
+redis.call('SADD', KEYS[5], ARGV[8])
+return 1
+`)
+
+// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
+// one. publicValue/hasValue carry the plaintext JSON scalar for public keys
+// (used by aggregation); pass hasValue=false for private-tier keys and
+// tombstones. Set tombstone=true when the record deletes the cell, which also
+// prunes the cid from the user+key index. Returns accepted=false when the write
+// is stale.
+func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client == nil {
+		return false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	keys := []string{
+		c.buildKey(udlRecKey(uid, cid, key)),
+		c.buildKey(udlIdxUserKey(uid, key)),
+		c.buildKey(udlIdxUserCid(uid, cid)),
+		c.buildKey(udlIdxCidKey(cid, key)),
+		c.buildKey(udlIdxCidUsers(cid)),
+	}
+	hv := "0"
+	if hasValue {
+		hv = "1"
+	}
+	tomb := "0"
+	if tombstone {
+		tomb = "1"
+	}
+	res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
+	if err != nil {
+		return false, fmt.Errorf("udl upsert failed: %w", err)
+	}
+	n, _ := res.(int64)
+	return n == 1, nil
+}
+
+// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
+// no record has ever been written for the cell.
+func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return UDLCell{}, false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	return c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+}
+
+// udlReadCell HMGETs a cell hash. Caller holds the lock. exists is false when
+// the version field is absent (i.e. the hash does not exist).
+func (c *Client) udlReadCell(ctx context.Context, recKey string) (UDLCell, bool, error) {
+	vals, err := c.client.HMGet(ctx, c.buildKey(recKey), "record", "version", "ts", "value").Result()
+	if err != nil {
+		return UDLCell{}, false, fmt.Errorf("hmget failed: %w", err)
+	}
+	// vals: [record, version, ts, value] — nil for absent fields.
+	if len(vals) < 4 || vals[1] == nil {
+		return UDLCell{}, false, nil
+	}
+	cell := UDLCell{}
+	if s, ok := vals[0].(string); ok {
+		cell.Record = s
+	}
+	if s, ok := vals[1].(string); ok {
+		cell.Version, _ = strconv.ParseInt(s, 10, 64)
+	}
+	if s, ok := vals[2].(string); ok {
+		cell.Ts, _ = strconv.ParseInt(s, 10, 64)
+	}
+	if s, ok := vals[3].(string); ok {
+		cell.Value = s
+		cell.HasValue = true
+	}
+	return cell, true, nil
+}
+
+// UDLListUserKey returns every cell for (uid, key) across all cids — the
+// projection behind active_for_user_key (My List, Continue Watching). Includes
+// tombstones; the caller decodes each record and filters.
+func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserKey(uid, key))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]UDLUserKeyEntry, 0, len(cids))
+	for _, cid := range cids {
+		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		out = append(out, UDLUserKeyEntry{Cid: cid, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
+	}
+	return out, nil
+}
+
+// UDLListUserCid returns every cell for (uid, cid) across all keys — the
+// projection behind list_for_user_cid.
+func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]UDLUserCidEntry, 0, len(keys))
+	for _, key := range keys {
+		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		out = append(out, UDLUserCidEntry{Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
+	}
+	return out, nil
+}
+
+// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
+func (c *Client) UDLCidUsers(cid string) ([]string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidUsers(cid))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	return uids, nil
+}
+
+// UDLPublicValues returns the live plaintext public JSON scalar values for
+// (cid, key) across all uids — the input to aggregation. Cells without a value
+// field (private-tier or tombstoned) are skipped.
+func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidKey(cid, key))).Result()
+	if err != nil {
+		return nil, fmt.Errorf("smembers failed: %w", err)
+	}
+	out := make([]string, 0, len(uids))
+	for _, uid := range uids {
+		v, err := c.client.HGet(ctx, c.buildKey(udlRecKey(uid, cid, key)), "value").Result()
+		if err == redis.Nil {
+			continue // private-tier or tombstoned
+		}
+		if err != nil {
+			return nil, fmt.Errorf("hget failed: %w", err)
+		}
+		out = append(out, v)
+	}
+	return out, nil
+}

internal/storage/udl_test.go

@@ -0,0 +1,79 @@
+package storage
+
+import (
+	"testing"
+)
+
+// A tombstone must leave the cell in place but drop the cid from the user+key
+// index.
+//
+// The cell has to stay: it *is* the CRDT state, and deleting it would let an
+// older write resurrect the value the tombstone retracted. The index entry has
+// to go: `active_for_user_key` returns every member of that set, so a set that
+// only ever grows means each My List / Continue Watching read carries every
+// title the user has ever un-liked or finished — with its full record blob —
+// forever. The reader filters them out again, so the cost is invisible until
+// the account is old.
+func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaalocator"
+		key = "seek"
+	)
+
+	// A live position: indexed, readable.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-v1", "", false, false); err != nil || !ok {
+		t.Fatalf("live upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err := c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("live position should be indexed, got %+v", entries)
+	}
+
+	// Finishing it tombstones the seek.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-v2-tombstone", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
+	}
+
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after tombstone: %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("tombstoned cid must leave the user+key index, got %+v", entries)
+	}
+
+	// ...but the cell itself survives, at the tombstone's version.
+	cell, exists, err := c.UDLGetCell(uid, cid, key)
+	if err != nil {
+		t.Fatalf("get cell: %v", err)
+	}
+	if !exists {
+		t.Fatal("tombstone must not delete the cell — an older write could then resurrect the value")
+	}
+	if cell.Version != 2 || cell.Record != "rec-v2-tombstone" {
+		t.Fatalf("cell should hold the tombstone record at v2, got %+v", cell)
+	}
+
+	// And the high-water mark still rejects the superseded write.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 300, "rec-v1-replay", "", false, false); err != nil || ok {
+		t.Fatalf("stale replay must be rejected: ok=%v err=%v", ok, err)
+	}
+
+	// Re-watching re-establishes the cell and puts it back in the index.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 3, 400, "rec-v3", "", false, false); err != nil || !ok {
+		t.Fatalf("re-establish upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after re-establish: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("re-established cid must return to the index, got %+v", entries)
+	}
+}