diff --git a/dashboard/dist/index.html b/dashboard/dist/index.html index 5f090a6..7368629 100644 --- a/dashboard/dist/index.html +++ b/dashboard/dist/index.html @@ -21,7 +21,7 @@ min-height: 100vh; } - + diff --git a/dashboard/src/App.tsx b/dashboard/src/App.tsx index dce7294..ffcd2dd 100644 --- a/dashboard/src/App.tsx +++ b/dashboard/src/App.tsx @@ -3,6 +3,7 @@ import Dashboard from './pages/Dashboard'; import Mounts from './pages/Mounts'; import Watchers from './pages/Watchers'; import Files from './pages/Files'; +import Identity from './pages/Identity'; import ServiceNav from './components/ServiceNav'; const navStyle: React.CSSProperties = { @@ -74,6 +75,12 @@ export default function App() { > Files + (isActive ? activeLinkStyle : linkStyle)} + > + Identity + (isActive ? activeLinkStyle : linkStyle)} @@ -89,6 +96,7 @@ export default function App() { } /> } /> } /> + } /> } /> diff --git a/dashboard/src/pages/Identity.tsx b/dashboard/src/pages/Identity.tsx new file mode 100644 index 0000000..20b7723 --- /dev/null +++ b/dashboard/src/pages/Identity.tsx @@ -0,0 +1,375 @@ +import { useEffect, useState } from 'react'; + +// Status returned by GET /api/identity. Private key is never present here — +// it is shown exactly once at generation time (see GeneratedKeyPanel). +interface IdentityStatus { + hasIdentity: boolean; + uid?: string; + curve?: string; + createdAt?: number; +} + +interface GeneratedKey { + uid: string; + curve: string; + createdAt: number; + privateKeyHex: string; +} + +const cardStyle: React.CSSProperties = { + background: '#16213e', + borderRadius: '8px', + padding: '1.5rem', + marginBottom: '1rem', +}; + +const buttonStyle: React.CSSProperties = { + padding: '0.5rem 1rem', + background: '#0f3460', + color: '#fff', + border: 'none', + borderRadius: '4px', + cursor: 'pointer', + marginRight: '0.5rem', +}; + +const dangerButtonStyle: React.CSSProperties = { + ...buttonStyle, + background: '#7f1d1d', +}; + +const inputStyle: React.CSSProperties = { + width: '100%', + padding: '0.5rem', + marginBottom: '0.5rem', + background: '#1a1a2e', + border: '1px solid #0f3460', + borderRadius: '4px', + color: '#fff', + fontFamily: 'monospace', +}; + +const monoStyle: React.CSSProperties = { + fontFamily: 'monospace', + fontSize: '0.85rem', + wordBreak: 'break-all', + background: '#0a0a18', + border: '1px solid #0f3460', + padding: '0.75rem', + borderRadius: '4px', +}; + +function formatCreatedAt(unixSec?: number): string { + if (!unixSec) return ''; + try { + return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC'); + } catch { + return String(unixSec); + } +} + +// Read the JSON body of a non-2xx response if present, fall back to status text. +async function errorMessage(res: Response, fallback: string): Promise { + try { + const j = await res.json(); + if (j && (j.message || j.error)) return j.message ?? j.error; + } catch { + // not JSON + } + return `${fallback} (HTTP ${res.status})`; +} + +function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss: () => void }) { + const [copied, setCopied] = useState(false); + const copy = async () => { + try { + await navigator.clipboard.writeText(data.privateKeyHex); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + // Clipboard API blocked (non-HTTPS context, etc.) — fall back to selection. + const sel = window.getSelection(); + const range = document.createRange(); + const el = document.getElementById('mm-privkey-box'); + if (sel && el) { + range.selectNodeContents(el); + sel.removeAllRanges(); + sel.addRange(range); + } + } + }; + return ( +
+
+ This private key is shown ONCE. Save it now in a password manager. + It will never be displayed again — there is no recovery path. Losing it means losing access + to every record signed under this identity. +
+

uid (public):

+
{data.uid}
+

privateKeyHex (secret):

+
+ {data.privateKeyHex} +
+
+ + +
+
+ ); +} + +function NoIdentityPanel({ + onGenerated, + onImported, +}: { + onGenerated: (g: GeneratedKey) => void; + onImported: () => void; +}) { + const [busy, setBusy] = useState(false); + const [importing, setImporting] = useState(false); + const [importHex, setImportHex] = useState(''); + const [error, setError] = useState(null); + + const generate = async () => { + setBusy(true); + setError(null); + try { + const res = await fetch('/api/identity/generate', { method: 'POST' }); + if (!res.ok) { + setError(await errorMessage(res, 'Generate failed')); + return; + } + const data = (await res.json()) as GeneratedKey; + onGenerated(data); + } catch (e) { + setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setBusy(false); + } + }; + + const importKey = async (e: React.FormEvent) => { + e.preventDefault(); + setBusy(true); + setError(null); + try { + const res = await fetch('/api/identity/import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ privateKeyHex: importHex.trim() }), + }); + if (!res.ok) { + setError(await errorMessage(res, 'Import failed')); + return; + } + setImportHex(''); + setImporting(false); + onImported(); + } catch (e) { + setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setBusy(false); + } + }; + + return ( +
+

No identity yet

+

+ The User Data Layer needs a signing key for your ratings, watch progress, and "liked" flags + to be attributed to you across devices. Nothing happens automatically — generate one below, + or import a key you already have. +

+ {error && ( +

+ {error} +

+ )} +
+ + +
+ {importing && ( +
+

+ 64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine. +

+ setImportHex(e.target.value)} + required + autoFocus + /> + +
+ )} +
+ ); +} + +function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onReplace: () => void }) { + const [confirming, setConfirming] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + const remove = async () => { + setBusy(true); + setError(null); + try { + const res = await fetch('/api/identity?confirm=true', { method: 'DELETE' }); + if (!res.ok && res.status !== 204) { + setError(await errorMessage(res, 'Delete failed')); + return; + } + onReplace(); + } catch (e) { + setError(`Delete failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setBusy(false); + setConfirming(false); + } + }; + + return ( +
+

Active identity

+

uid:

+
{status.uid}
+

+ Curve: {status.curve} · Created: {formatCreatedAt(status.createdAt)} +

+

+ The private key lives at /meta-core/identity/identity.json on this volume and + is never returned through the API after generation. Other services on this stack (e.g. + meta-watch) sign records by calling POST /api/identity/sign. +

+ {error && ( +

+ {error} +

+ )} + {!confirming ? ( + + ) : ( +
+

+ Replacing the identity deletes the current keypair from disk. Records signed by the + current uid will keep verifying for anyone who has them cached, but this node will no + longer be able to write under that uid. Make sure you have backed up the key + first. +

+ + +
+ )} +
+ ); +} + +export default function Identity() { + const [status, setStatus] = useState(null); + const [generated, setGenerated] = useState(null); + const [loading, setLoading] = useState(true); + const [fetchError, setFetchError] = useState(null); + + const refresh = async () => { + setLoading(true); + setFetchError(null); + try { + const res = await fetch('/api/identity'); + if (!res.ok) { + setFetchError(await errorMessage(res, 'Failed to load identity status')); + return; + } + setStatus((await res.json()) as IdentityStatus); + } catch (e) { + setFetchError(`Failed to load identity status: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + refresh(); + }, []); + + return ( +
+

Identity

+

+ Signing key for the User Data Layer (per-user ratings, progress, "liked" flags). One + identity per node. The same key on another device makes that device count as "you". +

+ + {loading &&
Loading…
} + {fetchError && ( +
+

{fetchError}

+ +
+ )} + + {!loading && !fetchError && generated && ( + { + setGenerated(null); + refresh(); + }} + /> + )} + + {!loading && !fetchError && !generated && status && !status.hasIdentity && ( + setGenerated(g)} onImported={refresh} /> + )} + + {!loading && !fetchError && !generated && status && status.hasIdentity && ( + + )} +
+ ); +} diff --git a/go.mod b/go.mod index fb5930f..78a3765 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/metazla/meta-core go 1.21 require ( + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/mr-tron/base58 v1.3.0 github.com/redis/go-redis/v9 v9.4.0 - golang.org/x/net v0.20.0 + golang.org/x/crypto v0.31.0 + golang.org/x/net v0.21.0 ) require ( diff --git a/go.sum b/go.sum index 4f6f7dd..9f7d0e8 100644 --- a/go.sum +++ b/go.sum @@ -4,13 +4,21 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= +github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= diff --git a/internal/api/cid_resolve.go b/internal/api/cid_resolve.go index 1676423..9bc445b 100644 --- a/internal/api/cid_resolve.go +++ b/internal/api/cid_resolve.go @@ -15,10 +15,10 @@ import ( // - 404 if the CID isn't registered // - 503 if storage is down // -// CIDs in the URL must be in prefixed-token form ("midhash256:bafk…", -// "sha256:bafk…", "ipfs:bafy…"). Bare CIDs without an algorithm prefix -// won't resolve — callers should pass the token they got from a previous -// metadata document, /api/file/{cid}/info, or a meta-share federated query. +// CIDs in the URL are bare multibase CIDv1 strings ("bafk…") — the midhash, +// a sibling sha2-256, an IPFS CID, a btih, anything registered in the +// reverse index. A CID is self-describing (its algorithm is the multicodec), +// so there is no : prefix. // // This endpoint is the public read API for the CID-resolution layer and // is auth-bypassed alongside /api/file/{cid} so that meta-share peers and @@ -57,12 +57,14 @@ func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) { } // Response envelope mirrors GetMetadataDocument JSON tags (metadata / - // cids / duplicates), with cid added so the caller can correlate when - // they fan out N lookups. + // cids / canonical_cid / duplicates), with cid added so the caller can + // correlate when they fan out N lookups. canonical_cid is derived by + // rank from the cids key-set on read — it is not a stored field. writeJSON(w, http.StatusOK, map[string]interface{}{ - "cid": cid, - "metadata": doc.Flat, - "cids": doc.CIDs, - "duplicates": doc.Duplicates, + "cid": cid, + "metadata": doc.Flat, + "cids": doc.CIDs, + "canonical_cid": doc.Canonical, + "duplicates": doc.Duplicates, }) } diff --git a/internal/api/handlers_identity.go b/internal/api/handlers_identity.go new file mode 100644 index 0000000..e94dc31 --- /dev/null +++ b/internal/api/handlers_identity.go @@ -0,0 +1,194 @@ +package api + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + + "github.com/metazla/meta-core/internal/identity" +) + +// identityStatusResponse is the shape of GET /api/identity. The private key +// is intentionally absent — it is only ever returned once, by +// POST /api/identity/generate, immediately after creation. +type identityStatusResponse struct { + HasIdentity bool `json:"hasIdentity"` + UID string `json:"uid,omitempty"` + Curve string `json:"curve,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` +} + +type identityGenerateResponse struct { + UID string `json:"uid"` + Curve string `json:"curve"` + CreatedAt int64 `json:"createdAt"` + PrivateKeyHex string `json:"privateKeyHex"` +} + +type identityImportRequest struct { + PrivateKeyHex string `json:"privateKeyHex"` +} + +type identityImportResponse struct { + UID string `json:"uid"` + Curve string `json:"curve"` + CreatedAt int64 `json:"createdAt"` +} + +type identityDeleteRequest struct { + Confirm bool `json:"confirm"` +} + +type identitySignRequest struct { + BytesB64 string `json:"bytesB64"` +} + +type identitySignResponse struct { + SigB64 string `json:"sigB64"` +} + +type identityAEADKeyResponse struct { + AEADKeyB64 string `json:"aeadKeyB64"` +} + +func (s *Server) handleIdentityGet(w http.ResponseWriter, r *http.Request) { + id, err := identity.Load(s.config.IdentityFilePath()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if id == nil { + writeJSON(w, http.StatusOK, identityStatusResponse{HasIdentity: false}) + return + } + writeJSON(w, http.StatusOK, identityStatusResponse{ + HasIdentity: true, + UID: id.UID, + Curve: id.Curve, + CreatedAt: id.CreatedAt, + }) +} + +func (s *Server) handleIdentityGenerate(w http.ResponseWriter, r *http.Request) { + if identity.Exists(s.config.IdentityFilePath()) { + writeErrorSlug(w, http.StatusConflict, "identity_exists", + "identity already exists; delete it first to generate a new one", false) + return + } + id, err := identity.Generate() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if err := identity.Save(s.config.IdentityFilePath(), id); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusCreated, identityGenerateResponse{ + UID: id.UID, + Curve: id.Curve, + CreatedAt: id.CreatedAt, + PrivateKeyHex: id.PrivateKeyHex, + }) +} + +func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) { + if identity.Exists(s.config.IdentityFilePath()) { + writeErrorSlug(w, http.StatusConflict, "identity_exists", + "identity already exists; delete it first to import a new one", false) + return + } + var body identityImportRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON body") + return + } + if strings.TrimSpace(body.PrivateKeyHex) == "" { + writeError(w, http.StatusBadRequest, "privateKeyHex is required") + return + } + id, err := identity.Import(strings.TrimSpace(body.PrivateKeyHex)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if err := identity.Save(s.config.IdentityFilePath(), id); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusCreated, identityImportResponse{ + UID: id.UID, + Curve: id.Curve, + CreatedAt: id.CreatedAt, + }) +} + +func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) { + var body identityDeleteRequest + // Tolerate empty body — confirm flag in a query param is also acceptable + // for shells where sending a JSON body with DELETE is awkward. + _ = json.NewDecoder(r.Body).Decode(&body) + if !body.Confirm && r.URL.Query().Get("confirm") != "true" { + writeErrorSlug(w, http.StatusBadRequest, "confirm_required", + "identity deletion requires {\"confirm\": true} or ?confirm=true", false) + return + } + if err := identity.Delete(s.config.IdentityFilePath()); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) { + id, err := identity.Load(s.config.IdentityFilePath()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if id == nil { + writeErrorSlug(w, http.StatusNotFound, "no_identity", + "no identity is configured", false) + return + } + var body identitySignRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON body") + return + } + raw, err := base64.StdEncoding.DecodeString(body.BytesB64) + if err != nil { + writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64") + return + } + sig, err := identity.Sign(id, raw) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, identitySignResponse{ + SigB64: base64.StdEncoding.EncodeToString(sig), + }) +} + +func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) { + id, err := identity.Load(s.config.IdentityFilePath()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if id == nil { + writeErrorSlug(w, http.StatusNotFound, "no_identity", + "no identity is configured", false) + return + } + key, err := identity.DeriveAEADKey(id) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, identityAEADKeyResponse{ + AEADKeyB64: base64.StdEncoding.EncodeToString(key), + }) +} diff --git a/internal/api/server.go b/internal/api/server.go index d4b87c7..6acc9a1 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -155,6 +155,15 @@ func (s *Server) setupRoutes() { s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST") s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST") + // User Data Layer signing identity (see internal/identity). Auth-gated + // at the perimeter — do NOT add to nginx-hash-lock unauth bypass. + s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET") + s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST") + s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST") + s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE") + s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST") + s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET") + // Snapshot (export / import / wipe) s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET") s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST") diff --git a/internal/cid/rank.go b/internal/cid/rank.go index ec270a3..d2e2bf2 100644 --- a/internal/cid/rank.go +++ b/internal/cid/rank.go @@ -1,81 +1,126 @@ -// Package cid centralizes CID-format helpers: ranking by utility for -// external presentation, normalizing prefixed CID tokens, and the rules for -// picking which CID is the canonical externally-visible identifier of a -// file. +// Package cid centralizes CID-format helpers: decoding a multibase-base32 +// CIDv1 into its multicodec parts, ranking by utility for external +// presentation, and picking which CID is the canonical externally-visible +// identifier of a file. // -// A "CID token" in this codebase is the prefixed form ":", -// e.g. "midhash256:bafkr4ih...", "sha256:bafkrei...", "ipfs:bafy...". The -// prefix makes the type recoverable without parsing the multicodec, which is -// what the reverse index (cid: → uuid), the per-root cids set, and -// the canonical_cid field all rely on. +// Sibling CIDs are stored as a bare-CID key-set (`cids/ = "true"`); a +// CID is self-describing — its algorithm is recoverable from the CIDv1 +// codec / multihash code, so there is no `:` prefix and no +// per-algorithm field name. Ranking and selection parse the multicodec +// rather than a string prefix. package cid -import "strings" +import ( + "encoding/base32" + "encoding/binary" + "fmt" + "strings" +) -// Rank returns a score for a CID token. Higher = more preferred as the -// public-facing "canonical" identifier for a file. Unknown algorithms rank -// below all known ones (0) so a plugin that invents a new digest type -// can't accidentally outrank an established one until it's registered here. +// Multicodec / multihash codes used for ranking. Source of truth for the +// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and +// METADATA_KEYS.md §2. +const ( + CodecRaw = 0x55 // raw — IPFS content-addressed block + CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block + CodeSha1 = 0x11 // multihash sha1 + CodeSha256 = 0x12 // multihash sha2-256 + CodeSha3_384 = 0x15 // multihash sha3-384 + CodeSha3_256 = 0x16 // multihash sha3-256 + CodeMd5 = 0xd5 // multihash md5 + CodeCrc32 = 0x0132 // multihash crc32 + CodeMidhash256 = 0x1000 // custom — SHA-256(size‖middle 1MB) + CodeBtPiecesRoot = 0xb702 // bittorrent v2 pieces root + CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52) + CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID + CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID +) + +// Decode parses a multibase-base32 CIDv1 string ("b…") into its content +// codec and multihash function code. It is the inverse of the encoder in +// internal/watcher/midhash.go. Returns an error for anything that isn't a +// CIDv1 in base32lower (e.g. a CIDv0 "Qm…" or a non-CID string). +func Decode(cidStr string) (codec uint64, mhCode uint64, err error) { + if len(cidStr) < 2 || cidStr[0] != 'b' { + return 0, 0, fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr) + } + raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:])) + if err != nil { + return 0, 0, fmt.Errorf("cid %q: base32 decode: %w", cidStr, err) + } + version, n := binary.Uvarint(raw) + if n <= 0 { + return 0, 0, fmt.Errorf("cid %q: bad version varint", cidStr) + } + raw = raw[n:] + if version != 1 { + return 0, 0, fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version) + } + codec, n = binary.Uvarint(raw) + if n <= 0 { + return 0, 0, fmt.Errorf("cid %q: bad codec varint", cidStr) + } + raw = raw[n:] + mhCode, n = binary.Uvarint(raw) + if n <= 0 { + return 0, 0, fmt.Errorf("cid %q: bad multihash-code varint", cidStr) + } + return codec, mhCode, nil +} + +// Rank returns a score for a bare CID. Higher = more preferred as the +// public-facing "canonical" identifier for a file. Unknown / undecodable +// CIDs rank 0 so a new digest type can't accidentally outrank an +// established one until it's classified here. +// +// We rank by the **multihash code** (the algorithm), with the custom +// BitTorrent CID codecs checked alongside it. This is deliberate: MetaMesh's +// two CID encoders disagree on the *content codec* — meta-hash/the watcher +// set codec == algorithm code, while the fullhash plugin and gateway wrap +// every digest with the raw codec (0x55) and put the algorithm in the +// multihash code. The multihash code is the algorithm in all of them, so it +// is the only reliable discriminator. (The BT info hash is given a distinct +// multihash code — btih-v2 0x10B7 — so it can't be confused with a plain +// sha2-256, see METADATA_KEYS.md §14.13.) // // Ordering rationale (utility-weighted, NOT cryptographic-strength): -// - IPFS CIDs win because the network around them — gateways, swarm, -// content-addressing tooling — has the broadest reach. A cryptographically -// equivalent sha256 in a non-CID form has less downstream value. -// - sha256 / sha3 follow: standard, widely recognized, no platform lock-in. -// - BitTorrent infohashes (btih) outrank midhash256 not for strength -// (v1 is sha1, weaker than midhash256's sha256 inner) but for swarm -// interop — a btih makes a file discoverable on existing torrent -// infrastructure that knows nothing about MetaMesh. -// - midhash256 is the floor. Internally meaningful, externally invisible. -func Rank(token string) int { - switch AlgorithmOf(token) { - case "ipfs": +// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network. +// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS +// block of the file IS its sha2-256, so they share this tier). +// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for +// swarm interop, not strength. +// - midhash256 is the floor among real options: internally meaningful, +// externally invisible. +// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0. +func Rank(cidStr string) int { + codec, mh, err := Decode(cidStr) + if err != nil { + return 0 + } + if codec == CodecDagPB { return 40 - case "sha256", "sha3", "sha3-256", "sha3-384", "sha3-512": - return 30 - case "btih", "btih-v2": + } + // BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2 + // multihash code, in either encoder convention. + if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 || + mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 { return 20 - case "midhash256": + } + switch mh { + case CodeSha256, CodeSha3_256, CodeSha3_384: + return 30 + case CodeMidhash256: return 10 default: + // md5, crc32, plain sha1, pieces-root, unknown. return 0 } } -// AlgorithmOf returns the algorithm prefix of a CID token, or "" if the -// token is malformed (no colon, or empty prefix). -func AlgorithmOf(token string) string { - colon := strings.IndexByte(token, ':') - if colon <= 0 { - return "" - } - return token[:colon] -} - -// Token assembles a CID token from its algorithm and raw value. Mirror of -// AlgorithmOf — callers writing to the reverse index, cids set, or -// canonical_cid field route through this so the format stays consistent. -func Token(algorithm, raw string) string { - if algorithm == "" || raw == "" { - return "" - } - return algorithm + ":" + raw -} - -// ValueOf returns the raw value portion of a CID token (everything after -// the first colon). Returns "" for malformed tokens. -func ValueOf(token string) string { - colon := strings.IndexByte(token, ':') - if colon <= 0 || colon == len(token)-1 { - return "" - } - return token[colon+1:] -} - -// Better picks the higher-ranked of two tokens. On ties, returns the +// Better picks the higher-ranked of two CIDs. On ties, returns the // lexicographically smaller one — gives the reconciler a deterministic -// answer so an unchanged canonical_cid doesn't churn when peers add the -// same digest under a different ordering. +// answer so the chosen canonical doesn't churn when peers add the same +// digest under a different ordering. func Better(a, b string) string { ra, rb := Rank(a), Rank(b) if ra > rb { diff --git a/internal/cid/rank_test.go b/internal/cid/rank_test.go index ce2cf61..ac78e89 100644 --- a/internal/cid/rank_test.go +++ b/internal/cid/rank_test.go @@ -1,85 +1,130 @@ package cid -import "testing" +import ( + "encoding/base32" + "encoding/binary" + "strings" + "testing" +) -func TestRank_Ordering(t *testing.T) { +// buildCID assembles a multibase-base32 CIDv1 from a content codec, a +// multihash code, and a digest length (digest bytes are zero — only the +// header is under test). Mirrors the encoder in watcher/midhash.go and the +// TS CID.createV1(code, create(code, digest)) path. +func buildCID(codec, mhCode uint64, digestLen int) string { + var b []byte + b = binary.AppendUvarint(b, 1) // CIDv1 + b = binary.AppendUvarint(b, codec) + b = binary.AppendUvarint(b, mhCode) + b = binary.AppendUvarint(b, uint64(digestLen)) + b = append(b, make([]byte, digestLen)...) + enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b) + return "b" + strings.ToLower(enc) +} + +// Real CIDs produced by @metazla/meta-hash (from its main.test.ts). For +// digest CIDs meta-hash sets codec == multihash code == algorithm code. +const ( + cidCrc32 = "bagzafmqcathsbnrk" + cidMd5 = "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa" + cidSha1 = "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra" + cidSha256 = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua" + cidSha3256 = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq" + cidSha3384 = "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga" +) + +func TestDecode_Fixtures(t *testing.T) { cases := []struct { - token string - want int + name string + cid string + wantMh uint64 }{ - {"ipfs:bafy", 40}, - {"sha256:bafk", 30}, - {"sha3-256:abc", 30}, - {"btih:def", 20}, - {"btih-v2:def", 20}, - {"midhash256:bafk", 10}, - {"unknown:xyz", 0}, - {"", 0}, - {"bare-no-colon", 0}, + {"crc32", cidCrc32, CodeCrc32}, + {"md5", cidMd5, CodeMd5}, + {"sha1", cidSha1, CodeSha1}, + {"sha2-256", cidSha256, CodeSha256}, + {"sha3-256", cidSha3256, CodeSha3_256}, + {"sha3-384", cidSha3384, CodeSha3_384}, } for _, tc := range cases { - if got := Rank(tc.token); got != tc.want { - t.Errorf("Rank(%q) = %d, want %d", tc.token, got, tc.want) + codec, mh, err := Decode(tc.cid) + if err != nil { + t.Errorf("Decode(%s %q) error: %v", tc.name, tc.cid, err) + continue + } + // meta-hash digest CIDs use codec == multihash code. + if codec != tc.wantMh || mh != tc.wantMh { + t.Errorf("Decode(%s) = codec 0x%x mh 0x%x, want both 0x%x", tc.name, codec, mh, tc.wantMh) + } + } +} + +func TestDecode_Rejects(t *testing.T) { + for _, in := range []string{"", "Qmabc", "x", "bzzzz!!"} { + if _, _, err := Decode(in); err == nil { + t.Errorf("Decode(%q) expected error, got nil", in) + } + } +} + +func TestRank_Tiers(t *testing.T) { + ipfsDagPB := buildCID(CodecDagPB, CodeSha256, 32) // chunked IPFS file + rawSha256 := buildCID(CodecRaw, CodeSha256, 32) // fullhash-style sha2-256 + btihV2meta := buildCID(CodeBtihV2, CodeBtihV2, 32) // meta-hash btih-v2 (codec==mh) + btihV2raw := buildCID(CodecRaw, CodeBtihV2, 32) // fullhash btih-v2 (raw codec) + btV1File := buildCID(CodeBtihV1File, CodeBtihV1File, 20) // gateway per-file btih v1 + midhashRaw := buildCID(CodecRaw, CodeMidhash256, 32) // fullhash midhash (raw codec) + + cases := []struct { + name string + cid string + want int + }{ + {"ipfs-dagpb", ipfsDagPB, 40}, + {"sha2-256(fixture, codec==mh)", cidSha256, 30}, + {"sha2-256(raw codec)", rawSha256, 30}, + {"sha3-256", cidSha3256, 30}, + {"sha3-384", cidSha3384, 30}, + {"btih-v2(codec==mh)", btihV2meta, 20}, + {"btih-v2(raw codec)", btihV2raw, 20}, + {"btih-v1-file", btV1File, 20}, + {"midhash256(raw codec)", midhashRaw, 10}, + {"sha1", cidSha1, 0}, + {"md5", cidMd5, 0}, + {"crc32", cidCrc32, 0}, + {"garbage", "not-a-cid", 0}, + } + for _, tc := range cases { + if got := Rank(tc.cid); got != tc.want { + t.Errorf("Rank(%s) = %d, want %d", tc.name, got, tc.want) } } } func TestRank_StrictOrdering(t *testing.T) { - // The exact numbers don't matter, but the inequality chain does — this - // is what the canonical_cid reconciler relies on. - ipfs := Rank("ipfs:x") - sha := Rank("sha256:x") - btih := Rank("btih:x") - mid := Rank("midhash256:x") - unknown := Rank("unknown:x") - + ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32)) + sha := Rank(cidSha256) + btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32)) + mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32)) + unknown := Rank(cidMd5) if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) { t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d", ipfs, sha, btih, mid, unknown) } } -func TestAlgorithmOf(t *testing.T) { - cases := map[string]string{ - "ipfs:bafy": "ipfs", - "sha256:bafk": "sha256", - "midhash256:abc": "midhash256", - "": "", - "no-colon": "", - ":leading-colon": "", // empty algo not allowed - } - for in, want := range cases { - if got := AlgorithmOf(in); got != want { - t.Errorf("AlgorithmOf(%q) = %q, want %q", in, got, want) - } - } -} - -func TestToken(t *testing.T) { - if got := Token("sha256", "abc"); got != "sha256:abc" { - t.Errorf("Token(sha256, abc) = %q", got) - } - if got := Token("", "abc"); got != "" { - t.Errorf("Token with empty algo should return empty, got %q", got) - } - if got := Token("sha256", ""); got != "" { - t.Errorf("Token with empty value should return empty, got %q", got) - } -} - func TestBetter(t *testing.T) { - // IPFS beats sha256 - if got := Better("ipfs:a", "sha256:b"); got != "ipfs:a" { - t.Errorf("Better(ipfs, sha256) = %q, want ipfs:a", got) + dagpb := buildCID(CodecDagPB, CodeSha256, 32) + // dag-pb (IPFS) beats a sha2-256 digest, regardless of arg order. + if got := Better(dagpb, cidSha256); got != dagpb { + t.Errorf("Better(dagpb, sha256) = %q, want dagpb", got) } - if got := Better("sha256:b", "ipfs:a"); got != "ipfs:a" { - t.Errorf("Better(sha256, ipfs) = %q, want ipfs:a (order shouldn't matter)", got) + if got := Better(cidSha256, dagpb); got != dagpb { + t.Errorf("Better(sha256, dagpb) = %q, want dagpb (order shouldn't matter)", got) } - // Tie → lexicographically smaller wins, deterministic - if got := Better("ipfs:b", "ipfs:a"); got != "ipfs:a" { - t.Errorf("Better(ipfs:b, ipfs:a) = %q, want ipfs:a", got) - } - if got := Better("ipfs:a", "ipfs:b"); got != "ipfs:a" { - t.Errorf("Better(ipfs:a, ipfs:b) = %q, want ipfs:a", got) + // Tie → lexicographically smaller wins, deterministic. + a, b := "bafyaaa", "bafyaab" + if Better(a, b) != a || Better(b, a) != a { + t.Errorf("tie-break not deterministic for %q vs %q", a, b) } } diff --git a/internal/config/config.go b/internal/config/config.go index 8cf51d4..f6f3033 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -104,6 +104,18 @@ func (c *Config) WatchersFilePath() string { return c.MetaCorePath + "/watchers.json" } +// IdentityDir returns the directory holding the User Data Layer signing +// identity (created by the dashboard's Identity menu — never auto-generated +// at startup). +func (c *Config) IdentityDir() string { + return c.MetaCorePath + "/identity" +} + +// IdentityFilePath returns the path to the persisted identity JSON. +func (c *Config) IdentityFilePath() string { + return c.IdentityDir() + "/identity.json" +} + func getEnv(key, defaultValue string) string { if value := os.Getenv(key); value != "" { return value diff --git a/internal/identity/identity.go b/internal/identity/identity.go new file mode 100644 index 0000000..ffbbe07 --- /dev/null +++ b/internal/identity/identity.go @@ -0,0 +1,211 @@ +// Package identity manages the user-controlled signing identity used by the +// User Data Layer (see docs/project-architecture/user-data-layer.md). +// +// The keypair is secp256k1 (D1: wallet forward-compat). The uid is the +// multibase base58btc encoding of the 33-byte compressed pubkey, prefixed +// with "z" per multibase convention. +// +// Storage is plaintext JSON at {META_CORE_PATH}/identity/identity.json, +// matching the trust model of the rest of /meta-core. Signing happens +// in-process; meta-watch calls POST /api/identity/sign rather than holding +// the private key itself. +package identity + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa" + "github.com/mr-tron/base58" + "golang.org/x/crypto/hkdf" +) + +const ( + CurveSecp256k1 = "secp256k1" + + // HKDF parameters for deriving the AEAD key from the private key. + // Stable across versions — changing these is a breaking change for + // encrypted records on disk. + aeadKDFSalt = "metamesh-uds-v1" + aeadKDFInfo = "aead-key" + aeadKeyLen = 32 +) + +// Identity is the persisted form of a user's keypair. +// +// PrivateKeyHex is the 32-byte secp256k1 scalar, lower-case hex, no prefix. +// PublicKeyHex is the 33-byte compressed pubkey, lower-case hex, no prefix. +type Identity struct { + UID string `json:"uid"` + Curve string `json:"curve"` + PublicKeyHex string `json:"publicKeyHex"` + PrivateKeyHex string `json:"privateKeyHex"` + CreatedAt int64 `json:"createdAt"` +} + +// Generate creates a fresh secp256k1 identity. +func Generate() (*Identity, error) { + priv, err := secp256k1.GeneratePrivateKey() + if err != nil { + return nil, fmt.Errorf("generate keypair: %w", err) + } + return identityFromPrivKey(priv), nil +} + +// Import parses an externally-provided 32-byte secp256k1 scalar and derives +// the matching pubkey. Accepts hex with or without a "0x" prefix. +func Import(privateKeyHex string) (*Identity, error) { + if len(privateKeyHex) >= 2 && (privateKeyHex[:2] == "0x" || privateKeyHex[:2] == "0X") { + privateKeyHex = privateKeyHex[2:] + } + raw, err := hex.DecodeString(privateKeyHex) + if err != nil { + return nil, fmt.Errorf("invalid hex: %w", err) + } + if len(raw) != 32 { + return nil, fmt.Errorf("private key must be 32 bytes, got %d", len(raw)) + } + priv := secp256k1.PrivKeyFromBytes(raw) + // Reject the zero scalar (secp256k1 lib does not reject it on its own). + pubBytes := priv.PubKey().SerializeCompressed() + if isAllZero(raw) || isAllZero(pubBytes) { + return nil, errors.New("private key is the curve identity (zero); refusing") + } + return identityFromPrivKey(priv), nil +} + +func identityFromPrivKey(priv *secp256k1.PrivateKey) *Identity { + pubBytes := priv.PubKey().SerializeCompressed() + return &Identity{ + UID: uidFromCompressedPubkey(pubBytes), + Curve: CurveSecp256k1, + PublicKeyHex: hex.EncodeToString(pubBytes), + PrivateKeyHex: hex.EncodeToString(priv.Serialize()), + CreatedAt: time.Now().UTC().Unix(), + } +} + +// uidFromCompressedPubkey encodes the pubkey as a multibase base58btc +// string ("z" prefix). The encoding is deterministic and matches MetaMesh +// CID conventions. +func uidFromCompressedPubkey(pub []byte) string { + return "z" + base58.Encode(pub) +} + +// Load reads the identity file. Returns (nil, nil) if it does not exist — +// callers distinguish "no identity" from real errors. +func Load(path string) (*Identity, error) { + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read identity file: %w", err) + } + var id Identity + if err := json.Unmarshal(data, &id); err != nil { + return nil, fmt.Errorf("parse identity file: %w", err) + } + if id.Curve != CurveSecp256k1 { + return nil, fmt.Errorf("unsupported curve on disk: %q", id.Curve) + } + return &id, nil +} + +// Save writes the identity atomically (tmp + rename), matching the pattern +// used by discovery/service.go. +func Save(path string, id *Identity) error { + if id == nil { + return errors.New("nil identity") + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("mkdir identity dir: %w", err) + } + data, err := json.MarshalIndent(id, "", " ") + if err != nil { + return fmt.Errorf("marshal identity: %w", err) + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0600); err != nil { + return fmt.Errorf("write tmp identity: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("rename tmp identity: %w", err) + } + return nil +} + +// Delete removes the identity file. Idempotent. +func Delete(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +// Exists reports whether an identity file is present. +func Exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// Sign produces an ECDSA-secp256k1 signature over SHA-256(bytes). The output +// is DER-encoded — the wire format we commit to for User Data Layer records. +func Sign(id *Identity, bytes []byte) ([]byte, error) { + if id == nil { + return nil, errors.New("no identity loaded") + } + raw, err := hex.DecodeString(id.PrivateKeyHex) + if err != nil { + return nil, fmt.Errorf("decode private key: %w", err) + } + if len(raw) != 32 { + return nil, fmt.Errorf("private key length %d", len(raw)) + } + priv := secp256k1.PrivKeyFromBytes(raw) + digest := sha256.Sum256(bytes) + sig := ecdsa.Sign(priv, digest[:]) + return sig.Serialize(), nil +} + +// DeriveAEADKey derives the symmetric key used to encrypt private-tier +// User Data Layer records. The KDF parameters (salt, info, length) are +// fixed for the v1 wire format. +func DeriveAEADKey(id *Identity) ([]byte, error) { + if id == nil { + return nil, errors.New("no identity loaded") + } + raw, err := hex.DecodeString(id.PrivateKeyHex) + if err != nil { + return nil, fmt.Errorf("decode private key: %w", err) + } + r := hkdf.New(sha256.New, raw, []byte(aeadKDFSalt), []byte(aeadKDFInfo)) + out := make([]byte, aeadKeyLen) + if _, err := io.ReadFull(r, out); err != nil { + return nil, fmt.Errorf("hkdf: %w", err) + } + return out, nil +} + +func isAllZero(b []byte) bool { + for _, x := range b { + if x != 0 { + return false + } + } + return true +} + +// Ensure rand.Reader is referenced so dependency-stripped builds don't +// accidentally remove the crypto/rand import that secp256k1.GeneratePrivateKey +// relies on transitively. +var _ = rand.Reader diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go new file mode 100644 index 0000000..4877612 --- /dev/null +++ b/internal/identity/identity_test.go @@ -0,0 +1,203 @@ +package identity + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa" +) + +func TestGenerateUIDIsDeterministicFromPubkey(t *testing.T) { + id, err := Generate() + if err != nil { + t.Fatalf("generate: %v", err) + } + if !strings.HasPrefix(id.UID, "z") { + t.Fatalf("uid missing multibase 'z' prefix: %q", id.UID) + } + if id.Curve != CurveSecp256k1 { + t.Fatalf("curve = %q", id.Curve) + } + if len(id.PrivateKeyHex) != 64 { + t.Fatalf("privkey hex len = %d", len(id.PrivateKeyHex)) + } + if len(id.PublicKeyHex) != 66 { + t.Fatalf("compressed pubkey hex len = %d", len(id.PublicKeyHex)) + } + + // Reimporting yields the same uid. + imported, err := Import(id.PrivateKeyHex) + if err != nil { + t.Fatalf("import: %v", err) + } + if imported.UID != id.UID { + t.Fatalf("uid changed after import roundtrip: %q vs %q", id.UID, imported.UID) + } + if imported.PublicKeyHex != id.PublicKeyHex { + t.Fatalf("pubkey changed after import roundtrip") + } +} + +func TestImportRejectsBadInput(t *testing.T) { + if _, err := Import(""); err == nil { + t.Fatal("empty hex should error") + } + if _, err := Import("zz"); err == nil { + t.Fatal("non-hex should error") + } + if _, err := Import("00"); err == nil { + t.Fatal("wrong-length hex should error") + } + if _, err := Import(strings.Repeat("00", 32)); err == nil { + t.Fatal("zero scalar should error") + } + // Hex with 0x prefix should work. + id1, err := Generate() + if err != nil { + t.Fatal(err) + } + id2, err := Import("0x" + id1.PrivateKeyHex) + if err != nil { + t.Fatalf("0x-prefixed hex: %v", err) + } + if id1.UID != id2.UID { + t.Fatal("uid mismatch through 0x prefix") + } +} + +func TestSaveLoadRoundtrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "identity.json") + + if Exists(path) { + t.Fatal("file should not exist yet") + } + if got, _ := Load(path); got != nil { + t.Fatal("Load on missing file should return nil") + } + + id, err := Generate() + if err != nil { + t.Fatal(err) + } + if err := Save(path, id); err != nil { + t.Fatal(err) + } + if !Exists(path) { + t.Fatal("file should exist after Save") + } + + loaded, err := Load(path) + if err != nil { + t.Fatal(err) + } + if loaded == nil || loaded.UID != id.UID || loaded.PrivateKeyHex != id.PrivateKeyHex { + t.Fatalf("roundtrip mismatch: %+v vs %+v", id, loaded) + } + + // File should be 0600 (private key on disk). + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Errorf("identity file mode = %o, want 0600", info.Mode().Perm()) + } + + if err := Delete(path); err != nil { + t.Fatal(err) + } + if Exists(path) { + t.Fatal("file should not exist after Delete") + } + // Delete is idempotent. + if err := Delete(path); err != nil { + t.Fatalf("Delete on missing file should be nil: %v", err) + } +} + +func TestSignAndVerify(t *testing.T) { + id, err := Generate() + if err != nil { + t.Fatal(err) + } + msg := []byte("hello user data layer") + sig, err := Sign(id, msg) + if err != nil { + t.Fatal(err) + } + if len(sig) == 0 { + t.Fatal("empty sig") + } + + // Verify externally with the public key, matching what meta-watch will do. + pubBytes, err := hex.DecodeString(id.PublicKeyHex) + if err != nil { + t.Fatal(err) + } + pub, err := secp256k1.ParsePubKey(pubBytes) + if err != nil { + t.Fatal(err) + } + parsedSig, err := ecdsa.ParseDERSignature(sig) + if err != nil { + t.Fatalf("parse DER sig: %v", err) + } + digest := sha256.Sum256(msg) + if !parsedSig.Verify(digest[:], pub) { + t.Fatal("signature did not verify") + } + + // Flipping a byte must break verification. + tampered := append([]byte{}, msg...) + tampered[0] ^= 0x01 + digest2 := sha256.Sum256(tampered) + if parsedSig.Verify(digest2[:], pub) { + t.Fatal("signature verified on tampered message") + } +} + +func TestDeriveAEADKeyIsDeterministic(t *testing.T) { + id, err := Generate() + if err != nil { + t.Fatal(err) + } + k1, err := DeriveAEADKey(id) + if err != nil { + t.Fatal(err) + } + k2, err := DeriveAEADKey(id) + if err != nil { + t.Fatal(err) + } + if !bytesEqual(k1, k2) { + t.Fatal("AEAD key derivation is non-deterministic") + } + if len(k1) != aeadKeyLen { + t.Fatalf("AEAD key len = %d, want %d", len(k1), aeadKeyLen) + } + + // Different identity → different key. + other, _ := Generate() + k3, _ := DeriveAEADKey(other) + if bytesEqual(k1, k3) { + t.Fatal("two distinct identities derived the same AEAD key") + } +} + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/storage/cid_resolution.go b/internal/storage/cid_resolution.go index c50e93e..73314c6 100644 --- a/internal/storage/cid_resolution.go +++ b/internal/storage/cid_resolution.go @@ -12,37 +12,38 @@ import ( ) // This file implements the CID-resolution layer that lives on top of the -// flat metadata storage in client.go. The shape is described in detail in -// docs/uuid-rooted-metadata.md; the short version: +// flat metadata storage in client.go. The short version: // -// - Roots are opaque UUIDv7s — file:/ for every property +// - Roots are opaque UUIDv7s — file:/ for every property. +// The UUID is an internal implementation detail; the mental model is +// "N CIDs → one record of keys→metadata". // - Every known CID for a file is registered as a reverse-index entry: -// cid:: → (Redis STRING) -// and mirrored in a per-root set for fast enumeration: -// file:/cids (Redis SET) -// - A canonical_cid field tracks the best (rank-wise) CID to advertise -// externally; updated by reconcileCanonicalCIDLocked whenever the set -// changes +// cid: → (Redis STRING) +// and as a member of the per-root bare-CID key-set: +// file:/cids/ = "true" (flat key, sentinel value) +// - A CID is self-describing: its algorithm is recoverable from the CIDv1 +// multicodec (see internal/cid), so there is no per-algorithm field name +// and no : token prefix. The canonical externally-visible CID is +// *derived by rank* from the key-set on read (GetMetadataDocument), never +// stored. // // Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch -// any write of a cid_* or midhash256 field and register the alias, so -// plugins that just write their result fields don't need to know about -// the reverse index at all. The hook helpers are *Locked — they assume -// c.mu is already held by the caller, avoiding nested RLock (which can -// deadlock against a waiting writer). +// any write of a cids/ field and register the reverse-index entry, so +// writers that just emit the key-set members don't need to know about the +// reverse index at all. The hook helpers are *Locked — they assume c.mu is +// already held by the caller, avoiding nested RLock (which can deadlock +// against a waiting writer). // CIDIndexPrefix is the Redis key prefix for the reverse index. Combined -// with a CID token, it forms the full key: "cid:midhash256:bafk…". +// with a bare CID, it forms the full key: "cid:bafk…". const CIDIndexPrefix = "cid:" -// CIDsField is the per-root SET key that mirrors all CID aliases for a file. -// Stored at file:/cids — read by reconcileCanonicalCIDLocked and by -// DeleteRoot to find every reverse-index entry that needs to be cleaned up. -const CIDsField = "cids" - -// CanonicalCIDField is the per-root STRING key that stores the externally -// preferred CID for this file. Picked by cid.Better over the cids set. -const CanonicalCIDField = "canonical_cid" +// CIDsKeyPrefix is the field-name prefix for the bare-CID key-set. Sibling +// CIDs live as flat keys file:/cids/ = "true"; the set IS the +// keyset, the value is the sentinel. Read by GetMetadataDocument (hoisted +// into the typed CIDs array) and by DeleteRoot/DeleteMetadata (to find the +// reverse-index entries that need cleanup). +const CIDsKeyPrefix = "cids/" // DuplicatesField is the per-root SET key holding additional file paths // whose content hashes to the same CIDs as this root. The watcher writes @@ -51,9 +52,9 @@ const CanonicalCIDField = "canonical_cid" const DuplicatesField = "duplicates" // buildCIDIndexKey returns the absolute Redis key for a reverse-index entry. -// token is a CID in prefixed form ("midhash256:bafk…", "sha256:…", etc.). -func (c *Client) buildCIDIndexKey(token string) string { - return c.buildKey(CIDIndexPrefix + token) +// cidStr is a bare multibase CIDv1 ("bafk…"). +func (c *Client) buildCIDIndexKey(cidStr string) string { + return c.buildKey(CIDIndexPrefix + cidStr) } // Mint creates a new root with filePath/sizeByte/mtimeNano and registers it @@ -88,11 +89,10 @@ func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) { return uuid, nil } -// AddAlias registers a CID token as resolving to uuid. Idempotent: writing -// the same alias twice is a no-op. After the alias is registered, the -// canonical_cid field is reconciled — picks the highest-ranked token in -// the cids set and writes it to canonical_cid if it changed. -func (c *Client) AddAlias(uuid, token string) error { +// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse +// index cid: → uuid and the key-set member cids/ = "true". +// Idempotent: registering the same CID twice is a no-op. +func (c *Client) AddAlias(uuid, cidStr string) error { c.mu.RLock() defer c.mu.RUnlock() @@ -103,119 +103,102 @@ func (c *Client) AddAlias(uuid, token string) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return c.addAliasLocked(ctx, uuid, token) + return c.addAliasLocked(ctx, uuid, cidStr) } // addAliasLocked is the lock-free body of AddAlias. The caller must already -// hold c.mu (read or write). Splits out so the alias-maintenance hooks +// hold c.mu (read or write). Split out so the alias-maintenance hooks // invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse -// the work without re-entering the lock — nested RLock can deadlock -// against a waiting writer. -func (c *Client) addAliasLocked(ctx context.Context, uuid, token string) error { - if uuid == "" || token == "" { - return fmt.Errorf("uuid and token are required") - } - if cid.AlgorithmOf(token) == "" { - return fmt.Errorf("token %q is not in : form", token) +// the work without re-entering the lock — nested RLock can deadlock against +// a waiting writer. +func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error { + if uuid == "" || cidStr == "" { + return fmt.Errorf("uuid and cid are required") } + keysetKey := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + cidStr + // Self-pointing alias guard. If the caller is trying to register - // `cid::` → `` (i.e. the root IS the cid value), - // that's the dual-root failure mode in disguise — it means somebody - // wrote /meta/ without first resolving via ResolveRoot. Refuse - // the alias write; leave the existing alias (if any) alone. Still - // add to this root's cids set so the field-level write succeeds. - if uuid == cid.ValueOf(token) { - if _, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+CIDsField, token).Result(); err != nil { - return fmt.Errorf("sadd cids set: %w", err) - } - return c.reconcileCanonicalCIDLocked(ctx, uuid) + // cid: → (i.e. the root IS the cid value), that's the + // dual-root failure mode in disguise — it means somebody wrote + // /meta/ without first resolving via ResolveRoot. Refuse the alias + // write; leave any existing alias alone. Still record the key-set + // membership so the field-level write succeeds. + if uuid == cidStr { + return c.client.Set(ctx, keysetKey, "true", 0).Err() } - // Guard against the dual-root pattern: if `cid:` already points - // at a DIFFERENT uuid, don't silently overwrite. The historical bug: - // meta-sort writing cid_midhash256= to root "" caused the hook - // to register cid:midhash256: → , severing the watcher's prior - // cid:midhash256: → alias. With write-path resolution - // now in place callers shouldn't hit this anymore, but the guard is - // cheap and prevents regression if a new caller bypasses ResolveRoot. - indexKey := c.buildCIDIndexKey(token) + // Guard against the dual-root pattern: if cid: already points at a + // DIFFERENT uuid, don't silently overwrite. Leave the existing alias and + // just record the key-set membership so it's still discoverable via the + // root → cids enumeration. + indexKey := c.buildCIDIndexKey(cidStr) existing, err := c.client.Get(ctx, indexKey).Result() if err != nil && err != redis.Nil { return fmt.Errorf("read existing alias: %w", err) } if existing != "" && existing != uuid { - // Existing alias points elsewhere. Leave it alone and just add the - // token to this root's cids set so it's still discoverable via the - // root → cids enumeration. - if _, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+CIDsField, token).Result(); err != nil { - return fmt.Errorf("sadd cids set: %w", err) - } - return c.reconcileCanonicalCIDLocked(ctx, uuid) + return c.client.Set(ctx, keysetKey, "true", 0).Err() } - prefix := c.buildKeyPrefix(uuid) pipe := c.client.Pipeline() pipe.Set(ctx, indexKey, uuid, 0) - pipe.SAdd(ctx, prefix+CIDsField, token) + pipe.Set(ctx, keysetKey, "true", 0) if _, err := pipe.Exec(ctx); err != nil { return fmt.Errorf("add alias pipeline: %w", err) } - return c.reconcileCanonicalCIDLocked(ctx, uuid) + return nil } -// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an -// editor lookup, an SSE consumer) to the actual storage root key. +// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor +// lookup, an SSE consumer) to the actual storage root key. // -// Callers historically send a bare midhash256 value (e.g. -// "bagacba…") because that's what the watcher publishes on file:events. -// The UUID-rooted-metadata design (see docs/uuid-rooted-metadata.md) -// keeps roots opaque and aliases CIDs into the reverse index: +// Callers send a bare CID (e.g. the midhash "bafk…" the watcher publishes on +// file:events, or any sibling CID). Roots are kept opaque and every CID is +// aliased into the reverse index: // -// cid:midhash256:bagacba… → 01KSBHM* +// cid:bafk… → 01KSBHM* // -// Without resolution, every /meta/ write creates a parallel -// midhash-rooted entry that the watcher's UUID never sees. The editor -// then shows two entries per file: the (rich) midhash root and the -// (empty) UUID root containing only `duplicates`. +// Without resolution, every /meta/ write creates a parallel cid-rooted +// entry that the watcher's UUID never sees. // // Behaviour: // - empty hash → empty (caller's problem) -// - hash matches cid:midhash256: in the reverse index → return UUID +// - hash matches a reverse-index entry → return UUID // - otherwise return hash unchanged (legacy / explicit-UUID writes) // -// Errors are converted to "no resolution" — if Redis is down or the -// lookup fails we let the caller proceed with the bare hash and surface -// the Redis failure on the actual operation. +// Errors are converted to "no resolution" — if Redis is down or the lookup +// fails we let the caller proceed with the bare hash and surface the Redis +// failure on the actual operation. func (c *Client) ResolveRoot(hash string) string { if hash == "" { return hash } - uuid, err := c.GetByCID("midhash256:" + hash) + uuid, err := c.GetByCID(hash) if err != nil || uuid == "" { return hash } return uuid } -// GetByCID resolves a CID token to the UUID of its root. Returns ("", nil) -// when the token is unknown; the empty string is the "not found" signal -// rather than a typed error so callers can branch without errors.As. -func (c *Client) GetByCID(token string) (string, error) { +// GetByCID resolves a bare CID to the UUID of its root. Returns ("", nil) +// when the CID is unknown; the empty string is the "not found" signal rather +// than a typed error so callers can branch without errors.As. +func (c *Client) GetByCID(cidStr string) (string, error) { c.mu.RLock() defer c.mu.RUnlock() if c.client == nil { return "", fmt.Errorf("not connected") } - if token == "" { + if cidStr == "" { return "", nil } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(token)).Result() + uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result() if err == redis.Nil { return "", nil } @@ -250,9 +233,32 @@ func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) { return added > 0, nil } -// DeleteRoot removes everything keyed off uuid: every file:/* property, -// every reverse-index entry listed in the cids set, and the index membership. -// Idempotent — a uuid with no remaining keys is a successful no-op. +// cidsForRootLocked scans the bare-CID key-set for a root and returns its +// members (bare CIDs, with the cids/ prefix stripped). Caller must hold c.mu. +func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) { + prefix := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + var out []string + var cursor uint64 + for { + keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result() + if err != nil { + return nil, fmt.Errorf("scan cids: %w", err) + } + for _, k := range keys { + out = append(out, strings.TrimPrefix(k, prefix)) + } + cursor = next + if cursor == 0 { + break + } + } + return out, nil +} + +// DeleteRoot removes everything keyed off uuid: every file:/* property +// (including the cids/ key-set), every reverse-index entry for those +// CIDs, and the index membership. Idempotent — a uuid with no remaining keys +// is a successful no-op. func (c *Client) DeleteRoot(uuid string) error { c.mu.RLock() defer c.mu.RUnlock() @@ -269,27 +275,25 @@ func (c *Client) DeleteRoot(uuid string) error { prefix := c.buildKeyPrefix(uuid) - // 1. Read the cids set first — once we delete file:/* we lose the - // list of reverse-index entries to clean up. Tolerate a missing set - // (empty list = nothing to unmap). - tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result() - if err != nil && err != redis.Nil { - return fmt.Errorf("smembers cids: %w", err) + // 1. Read the cids key-set first — once we delete file:/* we lose + // the list of reverse-index entries to clean up. + cids, err := c.cidsForRootLocked(ctx, uuid) + if err != nil { + return err } - // 2. Delete every reverse-index entry. Build the keys upfront, then DEL - // in one shot so we don't pay per-RTT. - if len(tokens) > 0 { - keys := make([]string, 0, len(tokens)) - for _, t := range tokens { - keys = append(keys, c.buildCIDIndexKey(t)) + // 2. Delete every reverse-index entry in one shot. + if len(cids) > 0 { + keys := make([]string, 0, len(cids)) + for _, cidStr := range cids { + keys = append(keys, c.buildCIDIndexKey(cidStr)) } if err := c.client.Del(ctx, keys...).Err(); err != nil { return fmt.Errorf("del cid index: %w", err) } } - // 3. Scan and delete every file:/* key. + // 3. Scan and delete every file:/* key (cids/ keys included). var cursor uint64 for { keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result() @@ -314,115 +318,59 @@ func (c *Client) DeleteRoot(uuid string) error { return nil } -// reconcileCanonicalCIDLocked reads the cids set, picks the highest-ranked -// token, and updates the canonical_cid field if it changed. Called from -// addAliasLocked after a new alias is registered. Caller must hold c.mu. -func (c *Client) reconcileCanonicalCIDLocked(ctx context.Context, uuid string) error { - prefix := c.buildKeyPrefix(uuid) - - tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result() - if err != nil { - return fmt.Errorf("smembers cids: %w", err) - } - if len(tokens) == 0 { - return nil - } - - best := tokens[0] - for _, t := range tokens[1:] { - best = cid.Better(best, t) - } - - canonicalKey := prefix + CanonicalCIDField - current, err := c.client.Get(ctx, canonicalKey).Result() - if err != nil && err != redis.Nil { - return fmt.Errorf("get canonical_cid: %w", err) - } - if current == best { - return nil - } - if err := c.client.Set(ctx, canonicalKey, best, 0).Err(); err != nil { - return fmt.Errorf("set canonical_cid: %w", err) - } - return nil -} - -// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into -// a root. If it looks like a CID field — name is "midhash256" or starts -// with "cid_" — it registers the implied CID token as an alias. Plugins -// write CID fields without knowing about the reverse index; this hook -// keeps the index in sync transparently. +// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a +// root. If it's a key-set member — field name starts with "cids/" — it +// registers the implied CID in the reverse index. Writers emit cids/ +// members without knowing about the reverse index; this hook keeps it in +// sync transparently. // -// Caller must hold c.mu. Errors are intentionally swallowed at the call -// site (logged, not returned): an out-of-sync reverse index is recoverable -// via a sweep, while a failed property write is not. +// Caller must hold c.mu. Errors are intentionally swallowed at the call site +// (logged, not returned): an out-of-sync reverse index is recoverable via a +// sweep, while a failed property write is not. func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error { - token := cidTokenFromField(field, value) - if token == "" { + cidStr := cidFromKeysetField(field) + if cidStr == "" { return nil } - return c.addAliasLocked(ctx, uuid, token) + return c.addAliasLocked(ctx, uuid, cidStr) } // maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked // over every entry in a metadata map. Used by SetMetadataFlat / -// MergeMetadataFlat after the underlying pipeline write succeeds. Caller -// must hold c.mu. +// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must +// hold c.mu. func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) { - for k, v := range metadata { - _ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, v) + for k := range metadata { + _ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k]) } } -// cidTokenFromField turns a (field-name, raw-value) pair into a normalized -// CID token ":". Returns "" for non-CID fields. -// -// Acceptable inputs: -// - field="midhash256", value="bafk…" → "midhash256:bafk…" -// - field="cid_sha256", value="bafk…" → "sha256:bafk…" -// - field="cid_ipfs", value="ipfs:bafy…" → "ipfs:bafy…" (trusted) -// -// Plugins that already use the prefixed form get their value passed through; -// plugins that store the bare CID get prefixed from the field name. The -// detection heuristic is just "value contains a colon" — base32lower CIDs -// never contain colons, so this is unambiguous. -func cidTokenFromField(field, value string) string { - if value == "" { +// cidFromKeysetField returns the bare CID encoded in a cids/ field name, +// or "" if the field isn't a key-set member. +func cidFromKeysetField(field string) string { + if !strings.HasPrefix(field, CIDsKeyPrefix) { return "" } - var algo string - switch { - case field == "midhash256": - algo = "midhash256" - case strings.HasPrefix(field, "cid_"): - algo = strings.TrimPrefix(field, "cid_") - default: - return "" - } - if algo == "" { - return "" - } - if strings.IndexByte(value, ':') > 0 { - return value - } - return cid.Token(algo, value) + return strings.TrimPrefix(field, CIDsKeyPrefix) } // MetadataDocument is the complete state of one file as the public API -// returns it: every flat string field, plus the cids and duplicates sets -// hoisted into typed arrays. Distinct from GetMetadataFlat (which only -// returns string keys) — clients querying /api/meta/ want the SETs -// expanded too, since "what other CIDs identify this file" and "what other -// paths point at this content" are core to the response. +// returns it: every flat string field (the cids/ members included), +// plus the CIDs hoisted into a typed array, the canonical CID derived by +// rank, and the duplicates set. Distinct from GetMetadataFlat (which only +// returns the raw string keys). type MetadataDocument struct { Flat map[string]string `json:"metadata"` CIDs []string `json:"cids"` + Canonical string `json:"canonical_cid,omitempty"` Duplicates []string `json:"duplicates"` } // GetMetadataDocument returns the full document for a root: every string -// property under file:/* plus the cids and duplicates SETs. Returns -// nil if the root has no keys at all (deleted or never existed). +// property under file:/* (cids/ members included in Flat), the +// CIDs hoisted into a typed array, the canonical CID computed by rank, and +// the duplicates SET. Returns nil if the root has no keys at all (deleted or +// never existed). func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -443,10 +391,24 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { } prefix := c.buildKeyPrefix(uuid) - cids, err := c.client.SMembers(ctx, prefix+CIDsField).Result() - if err != nil && err != redis.Nil { - return nil, fmt.Errorf("smembers cids: %w", err) + // Hoist the cids/ key-set members into a typed array and derive the + // canonical CID by rank. The members remain in Flat too — that's how + // flat-map consumers (meta-share ingest) see them. + cids := make([]string, 0) + canonical := "" + for k := range flat { + if !strings.HasPrefix(k, CIDsKeyPrefix) { + continue + } + cidStr := strings.TrimPrefix(k, CIDsKeyPrefix) + cids = append(cids, cidStr) + if canonical == "" { + canonical = cidStr + } else { + canonical = cid.Better(canonical, cidStr) + } } + dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result() if err != nil && err != redis.Nil { return nil, fmt.Errorf("smembers duplicates: %w", err) @@ -458,19 +420,7 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { return &MetadataDocument{ Flat: flat, CIDs: cids, + Canonical: canonical, Duplicates: dups, }, nil } - -// lookupSidecarPathByCID — REMOVED. -// -// This function used to walk every metadata entry looking for poster/backdrop -// matches and return their `posterPath`/`backdropPath` companion fields. It -// was a workaround for plugin output (TMDB images, etc.) not being indexed -// by the watcher. -// -// Now that the plugin output directory is a first-class watcher root -// (see config.DefaultWatcherPaths), every plugin artefact has its own -// midhash256 alias and resolves through the standard reverse index. Storing -// paths in metadata broke the CID-as-identity contract; the watcher -// indexing fixes the root cause. diff --git a/internal/storage/client.go b/internal/storage/client.go index c0850e3..f8953fd 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -291,16 +291,16 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) { prefix := c.buildKeyPrefix(hashID) - // Read the cids set first — once file:/* is gone we've lost the + // Read the cids key-set first — once file:/* is gone we've lost the // list of reverse-index keys that need cleanup. - tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result() - if err != nil && err != redis.Nil { - return 0, fmt.Errorf("smembers cids: %w", err) + cids, err := c.cidsForRootLocked(ctx, hashID) + if err != nil { + return 0, err } - if len(tokens) > 0 { - idxKeys := make([]string, 0, len(tokens)) - for _, t := range tokens { - idxKeys = append(idxKeys, c.buildCIDIndexKey(t)) + if len(cids) > 0 { + idxKeys := make([]string, 0, len(cids)) + for _, cidStr := range cids { + idxKeys = append(idxKeys, c.buildCIDIndexKey(cidStr)) } if err := c.client.Del(ctx, idxKeys...).Err(); err != nil { return 0, fmt.Errorf("del cid index: %w", err) @@ -347,32 +347,24 @@ func (c *Client) CountFiles() (int, error) { return len(hashIDs), nil } -// LookupPathByCID resolves a CID to its file path on disk via the reverse -// index. O(1) lookup followed by a single GET of file:/filePath. +// LookupPathByCID resolves a bare CID to its file path on disk via the +// reverse index. O(1) lookup followed by a single GET of file:/filePath. // -// Accepts both prefixed tokens ("midhash256:bafk…", "sha256:…", -// "ipfs:bafy…") and bare midhash256 CIDs — the bare form is what the -// editor passes to /api/file/{cid}/info because that's how plugins -// write the value (e.g. `backdrop=`). For bare tokens we -// try the midhash256 prefix as a fallback. +// Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS +// CID, a btih, etc. (they all alias to the same root). The bare CID is what +// the editor passes to /api/file/{cid}/info and what plugins write (e.g. +// `backdrop=`). // // Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is // resolved through this same path: the plugin output directory is a -// watcher root (see config.DefaultWatcherPaths), so every plugin -// artefact gets a midhash256 alias and resolves through the reverse -// index like any other file. There is no path-companion sidecar field. -func (c *Client) LookupPathByCID(token string) (string, error) { - uuid, err := c.GetByCID(token) +// watcher root (see config.DefaultWatcherPaths), so every plugin artefact +// gets a midhash alias and resolves through the reverse index like any +// other file. There is no path-companion sidecar field. +func (c *Client) LookupPathByCID(cidStr string) (string, error) { + uuid, err := c.GetByCID(cidStr) if err != nil { return "", err } - if uuid == "" && !strings.Contains(token, ":") { - // Bare CID — most likely a midhash256 written by a plugin. - uuid, err = c.GetByCID("midhash256:" + token) - if err != nil { - return "", err - } - } if uuid == "" { return "", nil } @@ -599,23 +591,24 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) { // file. Used by the watcher hydrator at startup to avoid re-mid-hashing // files whose tuple matches what's already in Redis. // -// HashID is the root identifier (UUIDv7 in the current model); MidHash256 -// is the actual midhash CID for the file, present iff the watcher already -// computed and stored it. Hydration uses HashID for the state-registry -// key and MidHash256 for the in-memory "I already know this hash" flag. +// HashID is the root identifier (UUIDv7 in the current model). Hydration +// uses HashID + (FilePath, Size, MtimeNano) to seed the in-memory state so +// the next scan can skip re-mid-hashing files whose size+mtime are unchanged. type FileTuple struct { - HashID string - FilePath string - Size int64 - MtimeNano int64 - MidHash256 string + HashID string + FilePath string + Size int64 + MtimeNano int64 } // GetTuplesForAllFiles returns one FileTuple per indexed root, fetched in // batched MGETs (no SCAN per root). Skips files missing any of filePath, -// sizeByte, or mtimeNano. midhash256 is optional — files that haven't -// been hashed yet still appear in the result with an empty MidHash256. -// Sub-second on local Redis for ~10K files. +// sizeByte, or mtimeNano. Sub-second on local Redis for ~10K files. +// +// The midhash is intentionally NOT fetched here: it now lives only as a +// cids/ key-set member (no named field), and the skip-rehash decision +// is purely size+mtime — so reading it would cost a SCAN per file for no +// benefit. func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -638,8 +631,8 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { // Build the key list: 4 keys per root, in a fixed order so we can decode // the MGET response by position. fields[i] for hashIDs[j] sits at // keys[j*4+i]. - const fieldsPerFile = 4 - fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano", "midhash256"} + const fieldsPerFile = 3 + fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"} keys := make([]string, 0, len(hashIDs)*fieldsPerFile) for _, h := range hashIDs { prefix := c.buildKeyPrefix(h) @@ -669,7 +662,6 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { filePath, _ := values[base].(string) sizeStr, _ := values[base+1].(string) mtimeStr, _ := values[base+2].(string) - midhash, _ := values[base+3].(string) if filePath == "" || sizeStr == "" || mtimeStr == "" { continue } @@ -682,11 +674,10 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { continue } out = append(out, FileTuple{ - HashID: h, - FilePath: filePath, - Size: size, - MtimeNano: mtime, - MidHash256: midhash, + HashID: h, + FilePath: filePath, + Size: size, + MtimeNano: mtime, }) } return out, nil diff --git a/internal/storage/dual_root_migration.go b/internal/storage/dual_root_migration.go index 2a118d2..08da3fa 100644 --- a/internal/storage/dual_root_migration.go +++ b/internal/storage/dual_root_migration.go @@ -3,160 +3,39 @@ package storage import ( "context" "fmt" - "log" - "strings" - "time" ) -// MigrateDualRoots scans file:__index__ for hashIds that look like raw -// midhash256 CID values (i.e. NOT UUIDs/ULIDs) and have a matching alias -// `cid:midhash256:` pointing to a different UUID. For each such -// "stranded" root we: +// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the +// legacy storage model: /meta/{hash} write handlers that didn't resolve +// CID → UUID first created a parallel midhash-rooted entry alongside the +// watcher's UUID root. // -// 1. Copy every file:/ into file:/ -// using MergeMetadataFlat (preserves anything the UUID root already -// holds, e.g. duplicates and watcher-owned fields). -// 2. Delete the entire file:/* subtree. -// 3. Drop the midhash hashId from file:__index__ (kept on the UUID). +// That pattern is now structurally impossible: +// - writes resolve via ResolveRoot before landing (see handlers), and +// - CIDs are stored as a bare-CID key-set (cids/) keyed off opaque +// UUID roots, with the reverse index cid: → uuid as the single +// resolution path. // -// This is the one-shot fix for the dual-root pattern described in -// docs/api-mediated-access.md. The pattern was caused by the (now-fixed) -// /meta/{hash} write handlers not resolving CID → UUID before writing. -// -// Returns the number of stranded roots fixed. +// The legacy on-disk shape this migration targeted (midhash-rooted entries, +// : tokens, named midhash256 fields) no longer exists, and the +// cut-over wipes Redis, so there is nothing to migrate. The function is kept +// as a no-op so the admin endpoint and its wrappers continue to compile and +// respond. func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) { - c.mu.RLock() if c.client == nil { - c.mu.RUnlock() return 0, fmt.Errorf("not connected") } - hashIDs, err := c.getAllHashIDsInternal(ctx) - c.mu.RUnlock() - if err != nil { - return 0, fmt.Errorf("get hash ids: %w", err) - } - - // Build the orphaned-UUID index: for every UUID-rooted entry, record - // midhash256 → UUID. This lets us find the "real" target when an alias - // has been overwritten to self-point (the failure mode the in-flight - // fix prevents going forward, but a one-shot still needs to repair - // existing dual-rooted data). - midhashToUUID := make(map[string]string) - for _, hashID := range hashIDs { - if !looksLikeULID(hashID) { - continue - } - mh, err := c.GetProperty(hashID, "midhash256") - if err != nil || mh == "" { - continue - } - midhashToUUID[mh] = hashID - } - - fixed := 0 - for _, hashID := range hashIDs { - // Skip likely-UUID hashIds. ULIDs are 26 chars, uppercase Crockford - // Base32. We treat anything that's not all-uppercase-alphanumeric - // of length ~26 as a CID-shaped hashId worth checking. - if looksLikeULID(hashID) { - continue - } - - // Is there an alias pointing to a different uuid? - token := "midhash256:" + hashID - uuid, err := c.GetByCID(token) - if err != nil { - log.Printf("[MigrateDualRoots] GetByCID(%s): %v (skipping)", token, err) - continue - } - // Three cases: - // - uuid empty → no alias, this midhash root is standalone (skip) - // - uuid == hashID → self-pointing alias; check the orphan index - // for a UUID that claims this midhash via its midhash256 field - // - uuid != hashID and != "" → proper alias, migrate to that UUID - if uuid == "" || uuid == hashID { - if orphan, ok := midhashToUUID[hashID]; ok && orphan != "" { - uuid = orphan - } else { - continue - } - } - - // Pull every key under file:/* and merge into file:/. - flat, err := c.GetMetadataFlat(hashID) - if err != nil { - log.Printf("[MigrateDualRoots] GetMetadataFlat(%s): %v (skipping)", hashID, err) - continue - } - if len(flat) > 0 { - if _, err := c.MergeMetadataFlat(uuid, flat); err != nil { - log.Printf("[MigrateDualRoots] MergeMetadataFlat(%s ← %s): %v", uuid, hashID, err) - continue - } - } - - // Delete the stranded root entirely. - if _, err := c.DeleteMetadata(hashID); err != nil { - log.Printf("[MigrateDualRoots] DeleteMetadata(%s): %v", hashID, err) - continue - } - - // Re-point the alias to the surviving UUID. DeleteMetadata wipes - // the cid:* entries for the deleted root, so the alias for this - // midhash may have been removed. Restore it pointing at the - // canonical UUID so future ResolveRoot calls land correctly. - if err := c.AddAlias(uuid, "midhash256:"+hashID); err != nil { - log.Printf("[MigrateDualRoots] AddAlias(%s → midhash256:%s): %v", uuid, hashID, err) - // Not fatal — data is migrated; the alias mismatch can be - // re-fixed by another sweep. - } - - fixed++ - } - - if fixed > 0 { - log.Printf("[MigrateDualRoots] Reunited %d stranded midhash roots into their UUID counterparts", fixed) - } - return fixed, nil -} - -// looksLikeULID is a cheap heuristic for "this hashId is opaque internal -// UUID, not a CID we should resolve." ULIDs are 26-char Crockford Base32; -// CIDs in this codebase are longer and lowercase. Tolerates dashes for -// future-proofing. -func looksLikeULID(s string) bool { - if len(s) != 26 { - return false - } - for _, r := range s { - if r >= 'A' && r <= 'Z' { - continue - } - if r >= '0' && r <= '9' { - continue - } - return false - } - return true + return 0, nil } // MigrateDualRootsWithTimeout is a convenience wrapper used by the API -// handler and the startup hook. Default to a generous 5-minute deadline — -// the per-entry work is small but the index can be large. +// handler and the startup hook. func (c *Client) MigrateDualRootsWithTimeout() (int, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - return c.MigrateDualRoots(ctx) + return c.MigrateDualRoots(context.Background()) } -// MigrateDualRootsReport returns a human-readable summary of what -// MigrateDualRoots would do or did. Useful for the admin endpoint. +// MigrateDualRootsReport returns a human-readable summary. With the dual-root +// pattern retired (see MigrateDualRoots), this always reports nothing to do. func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) { - fixed, err := c.MigrateDualRoots(ctx) - if err != nil { - return "", fixed, err - } - parts := []string{} - parts = append(parts, fmt.Sprintf("merged %d stranded midhash roots into UUID roots", fixed)) - return strings.Join(parts, "; "), fixed, nil + return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil } diff --git a/internal/storage/uuid_test.go b/internal/storage/uuid_test.go index 1d364d5..66c8cea 100644 --- a/internal/storage/uuid_test.go +++ b/internal/storage/uuid_test.go @@ -52,26 +52,23 @@ func TestNewUUID_TimeSortable(t *testing.T) { } } -func TestCIDTokenFromField(t *testing.T) { +func TestCIDFromKeysetField(t *testing.T) { cases := []struct { - field, value string - want string + field string + want string }{ - {"midhash256", "bafk", "midhash256:bafk"}, - {"cid_sha256", "bafk", "sha256:bafk"}, - {"cid_ipfs", "bafy", "ipfs:bafy"}, - {"cid_sha256", "sha256:bafk", "sha256:bafk"}, // already prefixed - {"cid_ipfs", "ipfs:bafy", "ipfs:bafy"}, // already prefixed - {"title", "Inception", ""}, // non-CID field - {"tmdb/title", "Inception", ""}, // non-CID nested - {"cid_", "bafk", ""}, // empty algo - {"midhash256", "", ""}, // empty value - {"cid_sha256", "", ""}, // empty value + {"cids/bafkmidhash", "bafkmidhash"}, + {"cids/baejbeisha256", "baejbeisha256"}, + {"cids/bafyipfs", "bafyipfs"}, + {"title", ""}, // non-CID field + {"tmdb/title", ""}, // non-CID nested + {"cid_sha2-256", ""}, // legacy flat field — no longer recognized + {"midhash256", ""}, // legacy named field — no longer recognized + {"cids", ""}, // prefix without a member } for _, tc := range cases { - if got := cidTokenFromField(tc.field, tc.value); got != tc.want { - t.Errorf("cidTokenFromField(%q, %q) = %q, want %q", - tc.field, tc.value, got, tc.want) + if got := cidFromKeysetField(tc.field); got != tc.want { + t.Errorf("cidFromKeysetField(%q) = %q, want %q", tc.field, got, tc.want) } } } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 5ffa730..03dab3f 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -38,10 +38,11 @@ func (w *Watcher) SetStorageClient(s *storage.Client) { // Caller already has all four values from its existing os.Stat / FileInfo — // no extra IO is performed here beyond the Redis writes themselves. // -// Flow (see docs/uuid-rooted-metadata.md): -// - Look up the midhash in the reverse index (cid:midhash256:… → uuid) -// - If unknown, Mint a fresh UUID root and write the midhash256 field -// (the SetProperty hook auto-registers the alias) +// Flow: +// - Look up the midhash CID in the reverse index (cid: → uuid) +// - If unknown, Mint a fresh UUID root and write the midhash as a cids/ +// key-set member (the SetProperty hook auto-registers the reverse-index +// alias). The midhash stays the file's address; it has no named field. // - If known, this is a duplicate physical file for the same content — // record the new path in the duplicates set and refresh size/mtime // @@ -52,25 +53,24 @@ func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash if w.storage == nil || midhash == "" { return } - token := "midhash256:" + midhash - uuid, err := w.storage.GetByCID(token) + uuid, err := w.storage.GetByCID(midhash) if err != nil { - log.Printf("[Watcher] GetByCID(%s): %v", token, err) + log.Printf("[Watcher] GetByCID(%s): %v", midhash, err) return } if uuid == "" { // New content. Mint sets filePath/sizeByte/mtimeNano; SetProperty - // then writes midhash256 — and the cid_resolution.go hook on - // SetProperty registers the reverse-index alias for us. + // then writes the midhash as a cids/ key-set member — and the + // cid_resolution.go hook registers the reverse-index alias for us. uuid, err = w.storage.Mint(absPath, size, mtimeNano) if err != nil { log.Printf("[Watcher] Mint(%s): %v", absPath, err) return } - if err := w.storage.SetProperty(uuid, "midhash256", midhash); err != nil { - log.Printf("[Watcher] SetProperty midhash256 for %s: %v", uuid, err) + if err := w.storage.SetProperty(uuid, storage.CIDsKeyPrefix+midhash, "true"); err != nil { + log.Printf("[Watcher] SetProperty cids/%s for %s: %v", midhash, uuid, err) } return } @@ -359,7 +359,9 @@ func (w *Watcher) HydrateStateFromStorage(roots []WatcherRoot) (loaded, skipped ws.Set(rel, &FileState{ Size: t.Size, MtimeNano: t.MtimeNano, - MidHash: t.MidHash256, + // MidHash left empty: the skip-rehash decision is + // size+mtime only, and the midhash is no longer a named + // field (it lives as a cids/ member). }) loaded++ matched = true