improved identity management

This commit is contained in:
Worph
2026-08-25 17:48:23 +02:00
parent 6ae87a2cdb
commit 5efdd2c07d
15 changed files with 1976 additions and 89 deletions
+202 -45
View File
@@ -3,11 +3,14 @@ package api
import (
"encoding/base64"
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"strings"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
// identityStatusResponse is the shape of GET /api/identity. The private key
@@ -59,12 +62,40 @@ type identityImportResponse struct {
Created bool `json:"created"`
}
// identityRevealRequest asks for an existing account's private key back. The
// key was shown once at generate; without a way to see it again, a client with
// a real sign-out has no recovery path for a user who never saved it.
// identityChallengeRequest asks for a one-shot challenge to sign.
type identityChallengeRequest struct {
UID string `json:"uid"`
Action string `json:"action"`
}
type identityChallengeResponse struct {
Challenge string `json:"challenge"`
ExpiresAt int64 `json:"expiresAt"`
}
// identityProof is the proof-of-possession carried by reveal and delete: the
// challenge text as issued, and a signature over it by the account's key.
//
// The signature is produced by whoever holds the key — the browser, locally.
// It is NOT obtainable from POST /api/identity/sign, which refuses any payload
// in the challenge domain; that refusal is what makes this proof mean
// something on a node that stores every private key in plaintext.
type identityProof struct {
Challenge string `json:"challenge"`
Signature string `json:"signature"`
}
// identityRevealRequest asks for an existing account's private key back.
//
// Gated on proof-of-possession, which makes it a "does this node hold the same
// key I do" check rather than the recovery path it once was. That is the
// deliberate consequence of the rule this file enforces: the private key, and
// nothing else, is authority over the account. An operator with root on the box
// can still read the file; an operator with only the API cannot.
type identityRevealRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
type identityRevealResponse struct {
@@ -76,6 +107,15 @@ type identityRevealResponse struct {
type identityDeleteRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
// identityDeleteResponse reports what the delete actually removed. The purge
// counts are the point: "account removed" alone leaves the caller unable to
// tell a clean removal from one that silently stranded a decade of ratings.
type identityDeleteResponse struct {
UID string `json:"uid"`
Purged storage.UDLUserStats `json:"purged"`
}
type identitySignRequest struct {
@@ -114,6 +154,16 @@ type identitySignBatchResponse struct {
// should page. Sized well above the longest real series (Supernatural, 327).
const maxSignBatch = 1000
// reservedPayloadMessage explains the one refusal the signing endpoints make.
//
// This is the load-bearing half of proof-of-possession on a node that holds
// every private key: without it, a caller wanting to delete someone else's
// account just asks meta-core to sign the challenge for them, and the gate on
// reveal/delete becomes a formality. See identity.ChallengeDomain.
const reservedPayloadMessage = "payload is in the reserved account-authorisation domain (" +
identity.ChallengeDomain + "…); meta-core will not sign one. " +
"Signing a challenge is what proves you hold the key, so it must be done where the key is."
type identityAEADKeyResponse struct {
AEADKeyB64 string `json:"aeadKeyB64"`
}
@@ -243,23 +293,105 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
})
}
// handleIdentityReveal returns an existing account's private key. Deliberately
// POST + explicit confirm (never a bare GET) so it cannot be triggered by a
// link or an image tag, and so it reads as the deliberate act it is. The
// endpoint is auth-gated at the perimeter like the rest of /api/identity/*.
func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
var body identityRevealRequest
_ = json.NewDecoder(r.Body).Decode(&body)
if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
"revealing a private key requires {\"confirm\": true} or ?confirm=true", false)
// handleIdentityChallenge issues the one-shot text a client signs to prove it
// holds an account's private key.
//
// Unauthenticated by design: a challenge is useless without the key, and
// GET /api/identity/accounts already publishes every uid, so refusing to mint
// one for an unknown caller would protect nothing. Requiring the account to
// exist is for a clear error, not secrecy.
func (s *Server) handleIdentityChallenge(w http.ResponseWriter, r *http.Request) {
var body identityChallengeRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := strings.TrimSpace(body.UID)
if uid == "" {
writeErrorSlug(w, http.StatusBadRequest, "uid_required", "uid is required", false)
return
}
action := identity.Action(strings.TrimSpace(body.Action))
if !identity.ValidAction(action) {
writeErrorSlug(w, http.StatusBadRequest, "bad_action",
`action must be "reveal" or "delete"`, false)
return
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
writeErrorSlug(w, http.StatusNotFound, "no_identity", "no identity for the given uid", false)
return
}
text, expires, err := s.challenges.Issue(uid, action)
if err != nil {
if errors.Is(err, identity.ErrChallengeFull) {
writeErrorSlug(w, http.StatusServiceUnavailable, "challenges_full", err.Error(), true)
return
}
writeErrorSlug(w, http.StatusBadRequest, "bad_uid", err.Error(), false)
return
}
writeJSON(w, http.StatusOK, identityChallengeResponse{
Challenge: text,
ExpiresAt: expires.Unix(),
})
}
// requireProof resolves the target account and verifies the caller's
// proof-of-possession for `action`. status == 0 means the proof held.
//
// The uid is mandatory here, unlike elsewhere in this file: the old "sole
// account" fallback would have this pick a target the signature does not name,
// and a proof that authorises an unnamed account authorises the wrong one as
// soon as a second profile exists.
func (s *Server) requireProof(uid string, proof identityProof, action identity.Action) (
id *identity.Identity, status int, slug, msg string,
) {
uid = strings.TrimSpace(uid)
if uid == "" {
return nil, http.StatusBadRequest, "uid_required",
"uid is required: the signature authorises one specific account"
}
if proof.Challenge == "" || proof.Signature == "" {
return nil, http.StatusUnauthorized, "signature_required",
"this operation requires proof you hold the account's private key: " +
"POST /api/identity/challenge, sign the returned text, and resend it as {challenge, signature}"
}
sig, err := base64.StdEncoding.DecodeString(proof.Signature)
if err != nil {
return nil, http.StatusBadRequest, "bad_signature", "signature is not valid base64"
}
// Resolve before redeeming so a request naming a missing account does not
// burn a challenge it could not have used anyway.
id, status, slug, msg = s.resolveAccount(uid)
if status != 0 {
return nil, status, slug, msg
}
// Redeem first, verify second: the challenge is single-use either way, so a
// wrong signature costs the attacker a fresh round trip per guess rather
// than letting them grind one challenge offline against the same nonce.
if err := s.challenges.Redeem(proof.Challenge, uid, action); err != nil {
return nil, http.StatusUnauthorized, "bad_challenge", err.Error()
}
if err := identity.Verify(uid, []byte(proof.Challenge), sig); err != nil {
return nil, http.StatusUnauthorized, "bad_signature", err.Error()
}
return id, 0, "", ""
}
// handleIdentityReveal returns an existing account's private key to a caller
// that proves it already holds that key.
//
// Deliberately POST (never a bare GET) so it cannot be triggered by a link or
// an image tag. The proof is what actually guards it: before this gate existed,
// anything that could reach the port could read every private key on the node.
func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
var body identityRevealRequest
_ = json.NewDecoder(r.Body).Decode(&body)
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
id, status, slug, msg := s.resolveAccount(uid)
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionReveal)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
@@ -271,46 +403,62 @@ func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
})
}
// handleIdentityDelete removes one account and everything it wrote.
//
// Two things happen, in this order and no other:
//
// 1. the account's User Data Layer rows are purged (cells plus every index
// that names the uid, including the cid-scoped ones shared with other
// accounts);
// 2. the keypair file is removed.
//
// Purge-then-delete so a failure never strands data: if the purge errors the
// key is still on disk, the account still exists, and the caller can retry.
// The reverse order would leave rows nobody can ever attribute, reach or
// rewrite — the orphaning this endpoint exists to prevent.
//
// What deletion still does NOT do, and cannot: records already signed under
// this uid and replicated to other peers keep verifying there. Removing a key
// is not a recall.
func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
var body identityDeleteRequest
// Tolerate empty body — confirm flag in a query param is also acceptable
// for shells where sending a JSON body with DELETE is awkward.
// Tolerate an empty body so the uid may ride in the query — meta-watch's
// proxy puts it there. The proof itself must be in the body: a signature in
// a URL ends up in every access log between here and the browser.
_ = json.NewDecoder(r.Body).Decode(&body)
if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
"identity deletion requires {\"confirm\": true} or ?confirm=true", false)
return
}
dir := s.config.IdentityAccountsDir()
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
if uid == "" {
// Back-compat: with no uid, delete the sole account. Ambiguous (and so
// refused) once more than one account exists.
list, err := identity.List(dir)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
switch len(list) {
case 0:
w.WriteHeader(http.StatusNoContent) // idempotent
return
case 1:
uid = list[0].UID
default:
writeErrorSlug(w, http.StatusBadRequest, "uid_required",
"multiple accounts exist; specify uid to delete", false)
return
}
}
if err := identity.DeleteByUID(dir, uid); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionDelete)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
w.WriteHeader(http.StatusNoContent)
// A purge needs storage. Refusing here — rather than deleting the key and
// reporting the purge as "0 records" — is the difference between a retry
// and permanent orphaned data: once the key is gone the account cannot be
// named for a second attempt through this API.
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage is not connected; refusing to delete the key while its data cannot be purged", true)
return
}
purged, err := s.storage.UDLPurgeUser(id.UID)
if err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
"could not purge this account's data; the account was NOT deleted: "+err.Error(), true)
return
}
if err := identity.DeleteByUID(s.config.IdentityAccountsDir(), id.UID); err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
return
}
log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids",
id.UID, purged.Records, purged.Cids)
writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
}
func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
@@ -333,6 +481,10 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
return
}
sig, err := identity.Sign(id, raw)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -380,6 +532,11 @@ func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request)
writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload",
"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
return
}
sig, err := identity.Sign(id, raw)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
+257
View File
@@ -0,0 +1,257 @@
package api
// End-to-end exercise of the account gate against a live router + miniredis:
// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
// both the key and every row are gone. Deliberately drives the real HTTP
// handlers rather than the packages beneath them, because the bugs that matter
// here (a challenge accepted for the wrong action, a purge that runs after the
// key is gone) live in the wiring.
import (
"bytes"
"encoding/base64"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
func newTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
stor := storage.NewClient("")
if err := stor.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = stor.Close() })
s := &Server{
config: &config.Config{MetaCorePath: t.TempDir()},
storage: stor,
challenges: identity.NewChallengeStore(),
router: mux.NewRouter(),
}
s.setupIdentityTestRoutes()
return s
}
// Only the routes under test, so this does not depend on every other subsystem
// booting.
func (s *Server) setupIdentityTestRoutes() {
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
}
func do(t *testing.T, s *Server, method, path string, body any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
var rdr *bytes.Reader
if body != nil {
b, _ := json.Marshal(body)
rdr = bytes.NewReader(b)
} else {
rdr = bytes.NewReader(nil)
}
req := httptest.NewRequest(method, path, rdr)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, req)
out := map[string]any{}
_ = json.Unmarshal(rec.Body.Bytes(), &out)
return rec, out
}
// signAs produces the proof a browser would, from the raw key.
func signAs(t *testing.T, privHex, challenge string) map[string]any {
t.Helper()
raw, err := hex.DecodeString(privHex)
if err != nil {
t.Fatalf("decode key: %v", err)
}
id, err := identity.Import(hex.EncodeToString(raw))
if err != nil {
t.Fatalf("import: %v", err)
}
sig, err := identity.Sign(id, []byte(challenge))
if err != nil {
t.Fatalf("sign: %v", err)
}
return map[string]any{
"challenge": challenge,
"signature": base64.StdEncoding.EncodeToString(sig),
}
}
func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
s := newTestServer(t)
rec, gen := do(t, s, "POST", "/api/identity/generate", nil)
if rec.Code != http.StatusCreated {
t.Fatalf("generate: %d %s", rec.Code, rec.Body)
}
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
// Some data to lose.
for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} {
if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("seed: %v", err)
}
}
// The count the UI shows in the confirmation.
rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
if rec.Code != http.StatusOK {
t.Fatalf("stats: %d %s", rec.Code, rec.Body)
}
users := stats["users"].(map[string]any)
mine := users[uid].(map[string]any)
if mine["records"].(float64) != 3 || mine["cids"].(float64) != 2 {
t.Fatalf("stats = %v, want 3 records / 2 cids", mine)
}
// Unsigned delete must be refused, and must not have touched anything.
rec, body := do(t, s, "DELETE", "/api/identity", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned delete = %d %v, want 401 signature_required", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("a refused delete removed the key anyway")
}
// meta-core must refuse to sign the caller's authorisation for them —
// without this the gate is decoration on a node holding every key.
ch, _, err := s.challenges.Issue(uid, identity.ActionDelete)
if err != nil {
t.Fatalf("issue: %v", err)
}
rec, body = do(t, s, "POST", "/api/identity/sign", map[string]any{
"uid": uid,
"bytesB64": base64.StdEncoding.EncodeToString([]byte(ch)),
})
if rec.Code != http.StatusForbidden || body["error"] != "reserved_payload" {
t.Fatalf("sign of a challenge = %d %v, want 403 reserved_payload", rec.Code, body)
}
// A challenge minted for reveal must not authorise a delete.
rec, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("reveal challenge used for delete = %d %v, want 401", rec.Code, body)
}
// The real thing.
rec, chBody = do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof = signAs(t, priv, chBody["challenge"].(string))
req = map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed delete = %d %s", rec.Code, rec.Body)
}
purged := body["purged"].(map[string]any)
if purged["records"].(float64) != 3 || purged["cids"].(float64) != 2 {
t.Fatalf("purged = %v, want 3 records / 2 cids", purged)
}
// Key gone.
if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("the account file survived the delete")
}
// Data gone — the orphaning this whole change exists to prevent.
after, err := s.storage.UDLAllUserStats()
if err != nil {
t.Fatalf("stats after: %v", err)
}
if got, present := after[uid]; present {
t.Fatalf("deleted account still holds %+v", got)
}
for _, cid := range []string{"cidA", "cidB"} {
uids, err := s.storage.UDLCidUsers(cid)
if err != nil {
t.Fatalf("cid users: %v", err)
}
if len(uids) != 0 {
t.Fatalf("%s still names %v after the purge", cid, uids)
}
}
// The challenge is single-use, so replaying the same proof does nothing.
rec, _ = do(t, s, "DELETE", "/api/identity", req)
if rec.Code == http.StatusOK {
t.Fatal("a replayed proof must not succeed")
}
}
func TestE2E_RevealRequiresProof(t *testing.T) {
s := newTestServer(t)
_, gen := do(t, s, "POST", "/api/identity/generate", nil)
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
rec, body := do(t, s, "POST", "/api/identity/reveal", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned reveal = %d %v, want 401 signature_required", rec.Code, body)
}
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "POST", "/api/identity/reveal", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed reveal = %d %s", rec.Code, rec.Body)
}
if body["privateKeyHex"] != priv {
t.Fatal("reveal returned a different key than generate did")
}
}
// Another account's key must not authorise this one, even with a challenge
// minted for the right uid and action.
func TestE2E_AnotherAccountsKeyIsRefused(t *testing.T) {
s := newTestServer(t)
_, alice := do(t, s, "POST", "/api/identity/generate", nil)
_, mallory := do(t, s, "POST", "/api/identity/generate", nil)
uid := alice["uid"].(string)
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
proof := signAs(t, mallory["privateKeyHex"].(string), chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body := do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized || body["error"] != "bad_signature" {
t.Fatalf("mallory deleting alice = %d %v, want 401 bad_signature", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("alice's account was deleted by mallory's signature")
}
}
+9 -2
View File
@@ -42,8 +42,11 @@ type Server struct {
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler
search *searchIndex
router *mux.Router
server *http.Server
// challenges backs proof-of-possession on reveal/delete. In-process, so a
// restart invalidates the ones in flight — see identity/challenge.go.
challenges *identity.ChallengeStore
router *mux.Router
server *http.Server
}
// NewServer creates a new API server
@@ -139,6 +142,8 @@ func NewServer(
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
s.webdavHandler = webdav.NewHandler(cfg)
s.challenges = identity.NewChallengeStore()
// One-time migration: fold a pre-multi-account identity.json into the
// per-account keystore so existing single-identity hosts keep their uid.
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
@@ -170,6 +175,7 @@ func (s *Server) setupRoutes() {
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
@@ -212,6 +218,7 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
// Metadata operations - base endpoints
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
+33
View File
@@ -16,6 +16,7 @@ import (
"strings"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/storage"
)
// UDLPutRequest is the body of PUT /api/udl/record.
@@ -349,3 +350,35 @@ func aggregatePublicValues(key string, values []string) UDLAggregate {
}
return agg
}
// udlUserStatsResponse is the shape of GET /api/udl/users/stats.
type udlUserStatsResponse struct {
// Users maps uid -> counts. Accounts holding nothing are absent rather than
// present with zeroes; a caller rendering a list treats "missing" as 0.
Users map[string]storage.UDLUserStats `json:"users"`
}
// handleUDLUserStats handles GET /api/udl/users/stats — how much every account
// holds, in one keyspace walk.
//
// Bulk rather than per-uid because the only caller is the accounts page, which
// needs a number for every row: the counts come from a SCAN (there is no
// per-user cid index — see storage/udl_admin.go), and MATCH filters without
// indexing, so N per-uid calls would walk the whole keyspace N times.
//
// Deliberately not on the critical path of anything. It is an admin read whose
// cost grows with the keyspace, so callers should fetch it after rendering,
// never before.
func (s *Server) handleUDLUserStats(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage not connected", true)
return
}
stats, err := s.storage.UDLAllUserStats()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, udlUserStatsResponse{Users: stats})
}
+126
View File
@@ -0,0 +1,126 @@
package identity
// In-memory, single-use challenge store backing proof-of-possession on the
// account operations that need it (reveal, delete).
//
// In memory on purpose: a challenge is worthless once redeemed and worthless
// once expired, so persisting it would only widen the window in which a leaked
// one is useful. A meta-core restart invalidates every challenge in flight,
// which costs a client one retry.
//
// Mirrors the sign-in challenge in meta-watch's src/auth.rs, down to the TTL —
// the browser signs immediately, so seconds is the right order and a narrow
// window bounds replay if one ever surfaces in a proxy log.
import (
"errors"
"sync"
"time"
)
// ChallengeTTL is how long an issued challenge stays answerable.
const ChallengeTTL = 120 * time.Second
// maxChallenges caps the store. An unauthenticated caller can mint challenges
// for any uid it can name, so the map is attacker-growable; without a ceiling
// that is a memory-exhaustion lever. At the cap, issuing fails rather than
// evicting: dropping someone else's live challenge on demand would let an
// attacker deny sign-outs, which is worse than making them retry.
const maxChallenges = 4096
var (
// ErrChallengeUnknown covers absent, already-redeemed and expired alike.
// One error for all three deliberately: distinguishing them tells a prober
// whether a challenge string was ever real.
ErrChallengeUnknown = errors.New("challenge is unknown, already used, or expired")
ErrChallengeFull = errors.New("too many challenges in flight; try again shortly")
)
type challengeEntry struct {
uid string
action Action
expires time.Time
}
// ChallengeStore issues and redeems challenges. Safe for concurrent use.
type ChallengeStore struct {
mu sync.Mutex
entries map[string]challengeEntry
ttl time.Duration
// now is swappable so the tests can age entries without sleeping.
now func() time.Time
}
func NewChallengeStore() *ChallengeStore {
return &ChallengeStore{
entries: make(map[string]challengeEntry),
ttl: ChallengeTTL,
now: time.Now,
}
}
// Issue mints a challenge for (uid, action) and remembers it until redeemed or
// expired. Returns the literal text the client signs and its expiry.
func (s *ChallengeStore) Issue(uid string, action Action) (string, time.Time, error) {
if !validUID(uid) {
return "", time.Time{}, errors.New("invalid uid")
}
if !ValidAction(action) {
return "", time.Time{}, errors.New("unknown action")
}
text, err := NewChallengeText(uid, action)
if err != nil {
return "", time.Time{}, err
}
s.mu.Lock()
defer s.mu.Unlock()
s.sweepLocked()
if len(s.entries) >= maxChallenges {
return "", time.Time{}, ErrChallengeFull
}
expires := s.now().Add(s.ttl)
s.entries[text] = challengeEntry{uid: uid, action: action, expires: expires}
return text, expires, nil
}
// Redeem consumes a challenge, asserting it was issued for this exact uid and
// action. Single-use: the entry is dropped on any lookup that finds it, so a
// mismatched redemption burns the challenge rather than leaving it to be
// retried against a different account.
func (s *ChallengeStore) Redeem(text, uid string, action Action) error {
s.mu.Lock()
defer s.mu.Unlock()
entry, ok := s.entries[text]
if !ok {
return ErrChallengeUnknown
}
delete(s.entries, text)
if s.now().After(entry.expires) {
return ErrChallengeUnknown
}
if entry.uid != uid || entry.action != action {
return ErrChallengeUnknown
}
// Re-read the signed bytes rather than trusting the map key alone: the
// signature covers `text`, so `text` is what has to name this uid+action.
if !ChallengeMatches(text, uid, action) {
return ErrChallengeUnknown
}
return nil
}
// sweepLocked drops expired entries. Caller holds the lock.
//
// Called only from Issue: with no background goroutine, expiry is enforced
// lazily on the one path that grows the map, which is exactly where the bound
// matters.
func (s *ChallengeStore) sweepLocked() {
now := s.now()
for text, e := range s.entries {
if now.After(e.expires) {
delete(s.entries, text)
}
}
}
+43
View File
@@ -0,0 +1,43 @@
package identity
import (
"encoding/base64"
"testing"
)
// A real signature produced by @noble/secp256k1 v3 — the library the dashboard
// and meta-watch both sign with — pinned as a fixture and verified here.
//
// This is the one seam neither side's own tests can cover: Go signs DER, the
// browser signs 64-byte compact, and every unit test above verifies signatures
// this package produced itself. If the encodings ever disagree, the gate fails
// closed in production (nobody can delete their own account) while every other
// test stays green. Regenerate with:
//
// node -e 'import("@noble/secp256k1").then(async s => { … })'
//
// against the same fixed scalar (32 zero bytes with 42 in the last).
func TestVerify_AcceptsARealNobleV3Signature(t *testing.T) {
const (
uid = "ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY"
challenge = "metamesh-identity-v1:delete:ztbJ71eLvfmGXw2ydaSPHavfwjva5Bzadutw2tvYDVFVY:deadbeef"
sigB64 = "lIiZGL0j9l/w2WXwzwe56cICnusuYdD3oaTuYfFTtNJBR9MolXFPk3014WQeAlvmbX4LfnW+TQz4asqoSwmSAQ=="
)
sig, err := base64.StdEncoding.DecodeString(sigB64)
if err != nil {
t.Fatalf("decode fixture: %v", err)
}
if len(sig) != 64 {
t.Fatalf("fixture is %d bytes, expected noble's 64-byte compact form", len(sig))
}
if err := Verify(uid, []byte(challenge), sig); err != nil {
t.Fatalf("a genuine browser signature must verify: %v", err)
}
// And the same signature must not verify over a different challenge — a
// verifier that ignored the payload would pass the line above too.
if err := Verify(uid, []byte(challenge+"x"), sig); err == nil {
t.Fatal("the fixture verified over the wrong payload")
}
}
+160
View File
@@ -0,0 +1,160 @@
package identity
// Signature verification against a uid, and the challenge vocabulary that
// makes a signature mean "I hold this account's private key" rather than
// "meta-core signed something for me".
//
// The asymmetry that motivates this file: meta-core *holds* every private key,
// so it can sign as anybody. A signature is therefore only proof of possession
// if the payload is one meta-core refuses to sign on request — hence
// ChallengeDomain, which POST /api/identity/sign rejects outright. Without that
// refusal an attacker mints their own authorisation in one extra call and the
// whole gate is decoration.
//
// Verification itself needs no stored state: a uid *is* the public key
// ("z" + base58btc(33-byte compressed sec1)), so the pubkey is recovered from
// the uid the caller names. Mirrors `verify_local` / `verify_local_compact` in
// meta-watch's src/record/sign.rs — same curve, same digest, both signature
// encodings.
import (
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"fmt"
"strings"
"github.com/decred/dcrd/dcrec/secp256k1/v4"
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
"github.com/mr-tron/base58"
)
// ChallengeDomain prefixes every byte string that authorises an account
// operation. POST /api/identity/sign and /sign-batch refuse payloads carrying
// it, so the only way to produce one is to hold the key yourself.
//
// Versioned because it is a wire commitment: changing it invalidates every
// challenge in flight, which is harmless, but changing it *silently* would let
// an old-domain payload keep working. Bump it deliberately.
const ChallengeDomain = "metamesh-identity-v1:"
// Action is the operation a challenge authorises. A challenge minted to reveal
// a key must not also delete the account, so the action is inside the signed
// bytes and is re-checked at redemption.
type Action string
const (
ActionReveal Action = "reveal"
ActionDelete Action = "delete"
)
// ValidAction reports whether a is an action a challenge may be issued for.
func ValidAction(a Action) bool {
return a == ActionReveal || a == ActionDelete
}
// ValidUID exposes the uid shape check to callers outside this package (the
// API layer, which must reject a hostile uid before it reaches a Redis SCAN
// pattern or a file path).
func ValidUID(uid string) bool { return validUID(uid) }
// IsChallengePayload reports whether b is (or claims to be) an account
// authorisation payload. Used by the signing endpoints to refuse minting one.
//
// Deliberately a prefix test on raw bytes rather than a parse: anything that
// merely *looks* like a challenge is refused, so a malformed or future-version
// payload fails closed instead of being signed.
func IsChallengePayload(b []byte) bool {
return strings.HasPrefix(string(b), ChallengeDomain)
}
// PubkeyFromUID recovers the compressed public key a uid encodes.
func PubkeyFromUID(uid string) (*secp256k1.PublicKey, error) {
if !validUID(uid) {
return nil, fmt.Errorf("invalid uid: %q", uid)
}
if !strings.HasPrefix(uid, "z") {
return nil, errors.New("uid is not multibase base58btc (missing 'z' prefix)")
}
raw, err := base58.Decode(uid[1:])
if err != nil {
return nil, fmt.Errorf("uid is not valid base58: %w", err)
}
if len(raw) != 33 {
return nil, fmt.Errorf("uid decodes to %d bytes, want a 33-byte compressed pubkey", len(raw))
}
pub, err := secp256k1.ParsePubKey(raw)
if err != nil {
return nil, fmt.Errorf("uid is not a point on secp256k1: %w", err)
}
return pub, nil
}
// Verify checks sig over SHA-256(payload) against the public key inside uid.
//
// Accepts both encodings in circulation: DER (what Sign emits, and what User
// Data Layer records carry) and 64-byte compact r‖s (what the browser emits —
// noble-secp256k1 v3 dropped DER, and meta-watch's sign-in already sends
// compact). Rejecting either would break one of the two callers.
func Verify(uid string, payload, sig []byte) error {
pub, err := PubkeyFromUID(uid)
if err != nil {
return err
}
parsed, err := parseSignature(sig)
if err != nil {
return err
}
digest := sha256.Sum256(payload)
if !parsed.Verify(digest[:], pub) {
return errors.New("signature does not verify for this uid")
}
return nil
}
func parseSignature(sig []byte) (*ecdsa.Signature, error) {
if len(sig) == 64 {
var r, s secp256k1.ModNScalar
// SetByteSlice reports overflow (>= curve order), which is not a valid
// scalar. A zero r or s is likewise never produced by a real signer.
if r.SetByteSlice(sig[:32]) || r.IsZero() {
return nil, errors.New("compact signature has an invalid r")
}
if s.SetByteSlice(sig[32:]) || s.IsZero() {
return nil, errors.New("compact signature has an invalid s")
}
return ecdsa.NewSignature(&r, &s), nil
}
parsed, err := ecdsa.ParseDERSignature(sig)
if err != nil {
return nil, fmt.Errorf("signature is neither 64-byte compact nor valid DER: %w", err)
}
return parsed, nil
}
// NewChallengeText mints the literal string a client signs. The uid and action
// are inside it, so a signature cannot be lifted from one account or operation
// onto another even if the nonce leaks.
func NewChallengeText(uid string, action Action) (string, error) {
nonce := make([]byte, 32)
if _, err := rand.Read(nonce); err != nil {
return "", fmt.Errorf("read nonce: %w", err)
}
return ChallengeDomain + string(action) + ":" + uid + ":" + hex.EncodeToString(nonce), nil
}
// ChallengeMatches reports whether text is a well-formed challenge for exactly
// this uid and action. The store already keys on the full text, so this is a
// belt-and-braces re-read of the signed bytes themselves: it is the signed
// content, not the lookup key, that a verifier must agree with.
//
// Compared in constant time — the text is a bearer secret until redeemed.
func ChallengeMatches(text, uid string, action Action) bool {
want := ChallengeDomain + string(action) + ":" + uid + ":"
if len(text) <= len(want) {
return false
}
return subtle.ConstantTimeCompare([]byte(text[:len(want)]), []byte(want)) == 1
}
+278
View File
@@ -0,0 +1,278 @@
package identity
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
"github.com/decred/dcrd/dcrec/secp256k1/v4"
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
)
// The whole gate rests on a uid being the public key: verification consults no
// stored state, so a signature is checked against the account the caller names
// rather than against whatever key meta-core happens to hold.
func TestVerify_AcceptsDERFromOurOwnSigner(t *testing.T) {
id, err := Generate()
if err != nil {
t.Fatalf("generate: %v", err)
}
payload := []byte(ChallengeDomain + "delete:" + id.UID + ":abcdef")
sig, err := Sign(id, payload)
if err != nil {
t.Fatalf("sign: %v", err)
}
if err := Verify(id.UID, payload, sig); err != nil {
t.Fatalf("DER signature from Sign must verify: %v", err)
}
}
// The browser is the signer that matters here, and noble-secp256k1 v3 emits
// 64-byte compact r‖s — it dropped DER entirely. A verifier that took only DER
// would reject every real proof while passing this package's own tests.
func TestVerify_AcceptsCompactAsTheBrowserEmitsIt(t *testing.T) {
id, err := Generate()
if err != nil {
t.Fatalf("generate: %v", err)
}
payload := []byte("some challenge text")
raw, _ := hex.DecodeString(id.PrivateKeyHex)
priv := secp256k1.PrivKeyFromBytes(raw)
digest := sha256.Sum256(payload)
der := ecdsa.Sign(priv, digest[:])
// Re-encode the same signature compactly, the way the browser would.
parsed, err := ecdsa.ParseDERSignature(der.Serialize())
if err != nil {
t.Fatalf("parse own DER: %v", err)
}
r := parsed.R()
sc := parsed.S()
compact := make([]byte, 0, 64)
rb := r.Bytes()
sb := sc.Bytes()
compact = append(compact, rb[:]...)
compact = append(compact, sb[:]...)
if err := Verify(id.UID, payload, compact); err != nil {
t.Fatalf("compact signature must verify: %v", err)
}
}
func TestVerify_RejectsAnotherAccountsSignature(t *testing.T) {
alice, _ := Generate()
mallory, _ := Generate()
payload := []byte("challenge for alice")
sig, err := Sign(mallory, payload)
if err != nil {
t.Fatalf("sign: %v", err)
}
if err := Verify(alice.UID, payload, sig); err == nil {
t.Fatal("a signature by mallory must not verify as alice")
}
}
func TestVerify_RejectsTamperedPayload(t *testing.T) {
id, _ := Generate()
sig, _ := Sign(id, []byte("delete account A"))
if err := Verify(id.UID, []byte("delete account B"), sig); err == nil {
t.Fatal("a signature must not verify over different bytes")
}
}
func TestVerify_RejectsGarbageSignatureEncodings(t *testing.T) {
id, _ := Generate()
for name, sig := range map[string][]byte{
"empty": {},
"short": {1, 2, 3},
"zero compact": make([]byte, 64),
"not der": []byte("this is not a signature at all......."),
"63 bytes": make([]byte, 63),
"65 bytes non-der": make([]byte, 65),
} {
if err := Verify(id.UID, []byte("x"), sig); err == nil {
t.Errorf("%s: expected rejection", name)
}
}
}
func TestPubkeyFromUID_RejectsMalformedUIDs(t *testing.T) {
id, _ := Generate()
for name, uid := range map[string]string{
"empty": "",
"no z prefix": id.UID[1:],
"truncated": id.UID[:20],
"path traversal": "../../etc/passwd",
"with slash": "z/" + id.UID,
"not on curve": "z" + strings.Repeat("1", 45),
} {
if _, err := PubkeyFromUID(uid); err == nil {
t.Errorf("%s (%q): expected rejection", name, uid)
}
}
if _, err := PubkeyFromUID(id.UID); err != nil {
t.Fatalf("a real uid must parse: %v", err)
}
}
// The refusal that makes proof-of-possession mean anything on a node holding
// every private key. If this ever returns false for a challenge, an attacker
// asks /api/identity/sign to authorise their own delete.
func TestIsChallengePayload_CatchesAnythingInTheDomain(t *testing.T) {
yes := [][]byte{
[]byte(ChallengeDomain),
[]byte(ChallengeDomain + "delete:zAlice:beef"),
[]byte(ChallengeDomain + "some-future-action:whatever"),
[]byte("metamesh-identity-v1:"), // literal, in case the const drifts
}
for _, b := range yes {
if !IsChallengePayload(b) {
t.Errorf("must refuse to sign %q", b)
}
}
no := [][]byte{
{},
[]byte("metamesh-identity-v2:delete:zAlice"), // different domain, not ours to refuse
[]byte("a record payload"),
[]byte(" " + ChallengeDomain), // not a prefix
}
for _, b := range no {
if IsChallengePayload(b) {
t.Errorf("must still sign %q", b)
}
}
}
func TestChallengeMatches_TiesTheTextToUIDAndAction(t *testing.T) {
text, err := NewChallengeText("zAlice", ActionDelete)
if err != nil {
t.Fatalf("mint: %v", err)
}
if !ChallengeMatches(text, "zAlice", ActionDelete) {
t.Fatal("its own uid+action must match")
}
if ChallengeMatches(text, "zBob", ActionDelete) {
t.Fatal("another uid must not match")
}
if ChallengeMatches(text, "zAlice", ActionReveal) {
t.Fatal("another action must not match")
}
// A prefix-only uid must not slip through: "zAlic" is a prefix of "zAlice",
// and a naive HasPrefix on the uid alone would accept it.
if ChallengeMatches(text, "zAlic", ActionDelete) {
t.Fatal("a uid that is a prefix of the real one must not match")
}
}
func TestNewChallengeText_IsUnpredictable(t *testing.T) {
seen := make(map[string]struct{}, 64)
for i := 0; i < 64; i++ {
text, err := NewChallengeText("zAlice", ActionDelete)
if err != nil {
t.Fatalf("mint: %v", err)
}
if _, dup := seen[text]; dup {
t.Fatal("challenge repeated — the nonce is not random")
}
seen[text] = struct{}{}
}
}
// --- challenge store --------------------------------------------------------
func TestChallengeStore_RedeemIsSingleUse(t *testing.T) {
s := NewChallengeStore()
text, _, err := s.Issue("zAlice", ActionDelete)
if err != nil {
t.Fatalf("issue: %v", err)
}
if err := s.Redeem(text, "zAlice", ActionDelete); err != nil {
t.Fatalf("first redeem: %v", err)
}
if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
t.Fatal("a challenge must not be redeemable twice — that is replay")
}
}
// A challenge minted to *look at* a key must not delete the account, and one
// minted for Alice must not act on Bob. Both are inside the signed text, so a
// store that ignored them would authorise the wrong operation with a perfectly
// valid signature.
func TestChallengeStore_RedeemRefusesWrongUIDOrAction(t *testing.T) {
s := NewChallengeStore()
text, _, _ := s.Issue("zAlice", ActionReveal)
if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
t.Fatal("a reveal challenge must not authorise a delete")
}
text2, _, _ := s.Issue("zAlice", ActionDelete)
if err := s.Redeem(text2, "zBob", ActionDelete); err == nil {
t.Fatal("alice's challenge must not authorise deleting bob")
}
}
// A failed redemption still consumes the challenge, so a wrong guess costs a
// round trip rather than leaving the nonce up for another attempt.
func TestChallengeStore_FailedRedeemStillBurnsIt(t *testing.T) {
s := NewChallengeStore()
text, _, _ := s.Issue("zAlice", ActionDelete)
_ = s.Redeem(text, "zBob", ActionDelete)
if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
t.Fatal("a burnt challenge must stay burnt for its rightful owner too")
}
}
func TestChallengeStore_Expires(t *testing.T) {
s := NewChallengeStore()
now := time.Unix(1_700_000_000, 0)
s.now = func() time.Time { return now }
text, expires, _ := s.Issue("zAlice", ActionDelete)
if !expires.Equal(now.Add(ChallengeTTL)) {
t.Fatalf("expiry %v, want %v", expires, now.Add(ChallengeTTL))
}
now = now.Add(ChallengeTTL + time.Second)
if err := s.Redeem(text, "zAlice", ActionDelete); err == nil {
t.Fatal("an expired challenge must not redeem")
}
}
// Issuing is unauthenticated, so the map is attacker-growable. The cap is what
// stops that being a memory-exhaustion lever; expiry sweeping is what stops the
// cap being reached by honest traffic.
func TestChallengeStore_CapsInFlightAndSweepsExpired(t *testing.T) {
s := NewChallengeStore()
now := time.Unix(1_700_000_000, 0)
s.now = func() time.Time { return now }
for i := 0; i < maxChallenges; i++ {
if _, _, err := s.Issue("zAlice", ActionDelete); err != nil {
t.Fatalf("issue %d: %v", i, err)
}
}
if _, _, err := s.Issue("zAlice", ActionDelete); err == nil {
t.Fatal("expected the store to refuse once full")
}
// Once they age out, the next issue sweeps and succeeds again.
now = now.Add(ChallengeTTL + time.Second)
if _, _, err := s.Issue("zAlice", ActionDelete); err != nil {
t.Fatalf("after expiry the store must accept again: %v", err)
}
}
func TestChallengeStore_RefusesInvalidInputs(t *testing.T) {
s := NewChallengeStore()
if _, _, err := s.Issue("../etc", ActionDelete); err == nil {
t.Fatal("a uid that is not uid-shaped must be refused")
}
if _, _, err := s.Issue("zAlice", Action("sudo")); err == nil {
t.Fatal("an unknown action must be refused")
}
}
+245
View File
@@ -0,0 +1,245 @@
package storage
// Account-scoped UDL administration: what a uid holds, and removing all of it.
//
// Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
// cell — because there is no per-user cid index to consult. The Redis model
// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
// "what is in My List", not "everything this account ever wrote". Adding a
// third index would put a write on every UDL upsert to serve two admin calls,
// so these SCAN instead and accept being O(keyspace).
//
// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
// one: the accounts page needs a number for every account, and MATCH filters
// server-side without indexing, so N per-uid calls would walk the keyspace N
// times. One walk buckets them all.
import (
"context"
"fmt"
"strings"
"time"
)
// UDLUserStats counts what one account holds. Records is the number of cells,
// including tombstones — a retracted rating is still a row this account owns
// and still something a purge has to remove.
type UDLUserStats struct {
Records int `json:"records"`
Cids int `json:"cids"`
Keys int `json:"keys"`
}
// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
const udlScanBatch = 1000
// safeUIDForPattern rejects anything that would change the meaning of a SCAN
// glob or reach across the key structure. meta-core's uids are alphanumeric
// (identity.validUID), so this only ever fires on a hostile or malformed one —
// but it fires here, in the layer that builds the pattern, rather than relying
// on every caller having validated first.
func safeUIDForPattern(uid string) error {
if uid == "" {
return fmt.Errorf("empty uid")
}
if strings.ContainsAny(uid, "*?[]\\/:") {
return fmt.Errorf("uid contains characters that are not valid in a uid: %q", uid)
}
return nil
}
// splitUDLRecSuffix splits the "<uid>/<cid>/<key>" tail of a udl:rec key.
//
// SplitN with a limit of 3 so a key containing a slash survives intact. cids
// never contain one; keys are a controlled vocabulary today, but a namespaced
// one arriving later must not silently truncate here — a truncated key would
// make the purge SREM the wrong index member and leave the real one behind.
func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
parts := strings.SplitN(suffix, "/", 3)
if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
return "", "", "", false
}
return parts[0], parts[1], parts[2], true
}
// scanKeys walks the keyspace for pattern and returns every match. Caller holds
// the lock.
//
// SCAN can return the same key more than once across cursor iterations (it
// guarantees no misses, not no duplicates), so callers that count must
// de-duplicate — both callers below collect into maps or sets for that reason.
func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
var (
out []string
cursor uint64
)
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
if err != nil {
return nil, fmt.Errorf("scan %q: %w", pattern, err)
}
out = append(out, keys...)
cursor = next
if cursor == 0 {
return out, nil
}
}
}
// UDLUserCells returns every (cid, key) pair this account holds a cell for.
// De-duplicated, since SCAN may repeat a key.
func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
// the cid, which the purge needs grouped.
if err := safeUIDForPattern(uid); err != nil {
return nil, err
}
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+uid+"/*")
if err != nil {
return nil, err
}
out := make(map[string]string, len(keys))
for _, full := range keys {
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
// The glob can only match this uid, but a uid that is a prefix of
// another would still be caught by the '/' in the pattern; check anyway
// so a malformed key can never be attributed to the wrong account.
if !ok || gotUID != uid {
continue
}
out[cid+"/"+key] = cid
}
return out, nil
}
// UDLAllUserStats counts every account's cells in one keyspace walk.
//
// Returns only accounts that actually hold records; a uid absent from the map
// holds none. Callers render a zero for those rather than treating it as an
// error — an account created a minute ago legitimately has nothing.
func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+"*")
if err != nil {
return nil, err
}
// Sets rather than counters: SCAN may hand back the same key twice, which
// would otherwise inflate every number on the page.
type acc struct {
cells map[string]struct{}
cids map[string]struct{}
keys map[string]struct{}
}
byUID := make(map[string]*acc)
for _, full := range keys {
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
if !ok {
continue
}
a := byUID[uid]
if a == nil {
a = &acc{
cells: make(map[string]struct{}),
cids: make(map[string]struct{}),
keys: make(map[string]struct{}),
}
byUID[uid] = a
}
a.cells[cid+"/"+key] = struct{}{}
a.cids[cid] = struct{}{}
a.keys[key] = struct{}{}
}
out := make(map[string]UDLUserStats, len(byUID))
for uid, a := range byUID {
out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
}
return out, nil
}
// UDLPurgeUser removes every trace of one account from the User Data Layer:
// its cells, its two user-scoped indexes, and its membership in the shared
// cid-scoped indexes. Returns what was removed.
//
// The shared indexes are the part that is easy to miss and expensive to leave.
// `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
// not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
// uid listed as an opinion-holder on every title it ever touched. Aggregation
// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
// rating average stays right, but uids_for_cid keeps naming a deleted account
// forever and every aggregate read pays for the lookup.
//
// Idempotent: purging an account with nothing stored is a no-op returning
// zeroes, so a retry after a partial failure is safe.
func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
cells, err := c.UDLUserCells(uid)
if err != nil {
return UDLUserStats{}, err
}
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return UDLUserStats{}, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
stats := UDLUserStats{Records: len(cells)}
distinctCids := make(map[string]struct{})
distinctKeys := make(map[string]struct{})
pipe := c.client.Pipeline()
for cidKey, cid := range cells {
key := strings.TrimPrefix(cidKey, cid+"/")
distinctCids[cid] = struct{}{}
distinctKeys[key] = struct{}{}
pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
}
for cid := range distinctCids {
pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
}
stats.Cids = len(distinctCids)
stats.Keys = len(distinctKeys)
// The two user-scoped indexes are enumerable directly, and are dropped
// whole rather than member-by-member. Scanning them separately also catches
// index entries whose cell is already gone — exactly the state a previously
// half-finished purge leaves behind.
idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
if err != nil {
return UDLUserStats{}, err
}
for _, k := range idxKeys {
pipe.Del(ctx, k)
}
if _, err := pipe.Exec(ctx); err != nil {
return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
}
return stats, nil
}
+253
View File
@@ -0,0 +1,253 @@
package storage
import (
"testing"
)
// seedCell writes one live UDL cell the way a real client would.
func seedCell(t *testing.T, c *Client, uid, cid, key string) {
t.Helper()
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("seed %s/%s/%s: ok=%v err=%v", uid, cid, key, ok, err)
}
}
// The purge has to reach the cid-scoped indexes, not just the user's own rows.
//
// This is the failure that motivated the whole endpoint: deleting an account
// while leaving `udl:idx:cid:<cid>:users` intact means every aggregate read on
// every title that account ever touched keeps naming a uid nobody can resolve,
// forever — and the rows are unreachable through any API, so nothing will ever
// clean them up.
func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
seedCell(t, c, "zAlice", "cidTwo", "seek")
stats, err := c.UDLPurgeUser("zAlice")
if err != nil {
t.Fatalf("purge: %v", err)
}
if stats.Records != 3 || stats.Cids != 2 || stats.Keys != 3 {
t.Fatalf("stats = %+v, want 3 records / 2 cids / 3 keys", stats)
}
// Cells gone.
if _, ok, err := c.UDLGetCell("zAlice", "cidOne", "like"); err != nil || ok {
t.Fatalf("cell survived the purge: ok=%v err=%v", ok, err)
}
// User-scoped indexes gone.
if entries, err := c.UDLListUserKey("zAlice", "like"); err != nil || len(entries) != 0 {
t.Fatalf("user+key index survived: %d entries, err=%v", len(entries), err)
}
if entries, err := c.UDLListUserCid("zAlice", "cidOne"); err != nil || len(entries) != 0 {
t.Fatalf("user+cid index survived: %d entries, err=%v", len(entries), err)
}
// Cid-scoped index gone — the one that is easy to miss.
for _, cid := range []string{"cidOne", "cidTwo"} {
uids, err := c.UDLCidUsers(cid)
if err != nil {
t.Fatalf("cid users %s: %v", cid, err)
}
for _, u := range uids {
if u == "zAlice" {
t.Fatalf("%s still lists the deleted account in uids_for_cid", cid)
}
}
}
}
// Purging one account must not touch another's rows, including on a cid they
// both have an opinion about — where the two share an index set and a careless
// DEL of the whole key would take the survivor's membership with it.
func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "shared", "rating")
seedCell(t, c, "zBob", "shared", "rating")
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
t.Fatalf("purge: %v", err)
}
if _, ok, err := c.UDLGetCell("zBob", "shared", "rating"); err != nil || !ok {
t.Fatalf("bob's cell must survive alice's purge: ok=%v err=%v", ok, err)
}
uids, err := c.UDLCidUsers("shared")
if err != nil {
t.Fatalf("cid users: %v", err)
}
if len(uids) != 1 || uids[0] != "zBob" {
t.Fatalf("uids_for_cid = %v, want exactly [zBob]", uids)
}
entries, err := c.UDLListUserKey("zBob", "rating")
if err != nil || len(entries) != 1 {
t.Fatalf("bob's user+key index must survive: %d entries, err=%v", len(entries), err)
}
}
// A uid that is a prefix of another must not drag its neighbour's rows out.
// The key layout separates them with '/' and ':', so this passes only as long
// as the scan patterns keep those separators.
func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAliceExtra", "cidOne", "like")
stats, err := c.UDLPurgeUser("zAlice")
if err != nil {
t.Fatalf("purge: %v", err)
}
if stats.Records != 1 {
t.Fatalf("purged %d records, want exactly alice's 1", stats.Records)
}
if _, ok, err := c.UDLGetCell("zAliceExtra", "cidOne", "like"); err != nil || !ok {
t.Fatalf("zAliceExtra's cell must survive: ok=%v err=%v", ok, err)
}
}
// Tombstoned cells still belong to the account. They are absent from the
// user+key index by design (see udl.go), so a purge that enumerated *that*
// index instead of the cells would leave every retracted rating behind — the
// rows most likely to have accumulated over an account's life.
func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
t.Fatalf("tombstone: ok=%v err=%v", ok, err)
}
// Precondition: the tombstone left the cell but dropped it from the index.
if entries, _ := c.UDLListUserKey("zAlice", "like"); len(entries) != 0 {
t.Fatalf("precondition: tombstone should be out of the user+key index")
}
stats, err := c.UDLPurgeUser("zAlice")
if err != nil {
t.Fatalf("purge: %v", err)
}
if stats.Records != 1 {
t.Fatalf("purged %d records, want the 1 tombstoned cell", stats.Records)
}
if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); ok {
t.Fatal("tombstoned cell survived the purge")
}
}
// Retry-safe: the delete handler purges before removing the key, so a caller
// that retries after a partial failure runs this a second time.
func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
t.Fatalf("first purge: %v", err)
}
stats, err := c.UDLPurgeUser("zAlice")
if err != nil {
t.Fatalf("second purge must be a no-op, got: %v", err)
}
if stats.Records != 0 || stats.Cids != 0 {
t.Fatalf("second purge = %+v, want zeroes", stats)
}
}
func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
// "*" would otherwise match every account in the keyspace — a wildcard
// purge dressed up as a uid.
for _, uid := range []string{"*", "z*", "", "z:Alice", "z/Alice"} {
if _, err := c.UDLPurgeUser(uid); err == nil {
t.Errorf("uid %q must be refused before it reaches a SCAN pattern", uid)
}
}
if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); !ok {
t.Fatal("a refused purge must not have deleted anything")
}
}
func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
seedCell(t, c, "zAlice", "cidTwo", "like")
seedCell(t, c, "zBob", "cidOne", "seek")
stats, err := c.UDLAllUserStats()
if err != nil {
t.Fatalf("stats: %v", err)
}
if got := stats["zAlice"]; got.Records != 3 || got.Cids != 2 || got.Keys != 2 {
t.Fatalf("alice = %+v, want 3 records / 2 cids / 2 keys", got)
}
if got := stats["zBob"]; got.Records != 1 || got.Cids != 1 || got.Keys != 1 {
t.Fatalf("bob = %+v, want 1/1/1", got)
}
// An account with nothing stored is simply absent — the caller renders 0.
if _, present := stats["zNobody"]; present {
t.Fatal("an account with no records must not appear in the map")
}
}
// The stats and the purge must agree: the number shown in the delete
// confirmation is the number of rows the delete then removes.
func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
c, _ := newTestClient(t, "")
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidTwo", "rating")
seedCell(t, c, "zAlice", "cidTwo", "seek")
before, err := c.UDLAllUserStats()
if err != nil {
t.Fatalf("stats: %v", err)
}
purged, err := c.UDLPurgeUser("zAlice")
if err != nil {
t.Fatalf("purge: %v", err)
}
if before["zAlice"] != purged {
t.Fatalf("stats said %+v but purge removed %+v", before["zAlice"], purged)
}
}
// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or
// every account on a prefixed node would be reported under a mangled name and
// the purge would find nothing.
func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) {
c, _ := newTestClient(t, "tenantA:")
seedCell(t, c, "zAlice", "cidOne", "like")
stats, err := c.UDLAllUserStats()
if err != nil {
t.Fatalf("stats: %v", err)
}
if got := stats["zAlice"]; got.Records != 1 {
t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats)
}
if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 {
t.Fatalf("purge under a prefix: %+v err=%v", s, err)
}
}
func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
// A namespaced key arriving later must not be truncated: the purge SREMs
// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong
// member and leaves the real one behind.
uid, cid, key, ok := splitUDLRecSuffix("zAlice/cidOne/ns/sub:key")
if !ok || uid != "zAlice" || cid != "cidOne" || key != "ns/sub:key" {
t.Fatalf("got (%q,%q,%q,%v)", uid, cid, key, ok)
}
for _, bad := range []string{"", "zAlice", "zAlice/cidOne", "zAlice//key", "/cid/key"} {
if _, _, _, ok := splitUDLRecSuffix(bad); ok {
t.Errorf("%q should not parse", bad)
}
}
}