v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410

internal/api/cid_resolve.go

@@ -56,6 +56,12 @@ func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if as := r.URL.Query().Get("as"); as != "" {
+		if flat, err := s.storage.GetMetadataFlatAs(uuid, as); err == nil && flat != nil {
+			doc.Flat = flat
+		}
+	}
+
 	// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
 	// cids / canonical_cid / duplicates), with cid added so the caller can
 	// correlate when they fan out N lookups. canonical_cid is derived by

internal/api/claims.go

@@ -0,0 +1,172 @@
+package api
+
+// Signed claims (meta-core v2.0.0, docs/implementation/metacore-claims-model.md).
+//
+//	POST /api/claims                 the only metadata write path
+//	GET  /api/meta/{cid}/claims      every live claim of the cid's group
+//	GET  /api/metadata/{rid}/claims  the same, addressed by rid
+//	POST /api/merge                  stateless merge of claims the caller holds
+//
+// meta-core never signs a claim here: it verifies each one (MMUR2, low-S) and
+// the signature is the write authentication.
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// GoneResponse is the body of every removed unsigned write route.
+type GoneResponse struct {
+	Error string `json:"error"`
+	Use   string `json:"use"`
+}
+
+// handleGone answers the unsigned write routes removed in v2.0.0.
+func (s *Server) handleGone(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, http.StatusGone, GoneResponse{Error: "gone", Use: "POST /api/claims"})
+}
+
+// decodeClaims accepts a JSON array of claims or {"claims": [...]}.
+func decodeClaims(r *http.Request) ([]claim.Claim, error) {
+	var raw json.RawMessage
+	if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&raw); err != nil {
+		return nil, err
+	}
+	var claims []claim.Claim
+	if len(raw) > 0 && raw[0] == '{' {
+		var env struct {
+			Claims []claim.Claim `json:"claims"`
+		}
+		if err := json.Unmarshal(raw, &env); err != nil {
+			return nil, err
+		}
+		return env.Claims, nil
+	}
+	if err := json.Unmarshal(raw, &claims); err != nil {
+		return nil, err
+	}
+	return claims, nil
+}
+
+// handlePostClaims handles POST /api/claims: one status per claim, in input
+// order. 200 even when some claims are refused — the statuses say which.
+func (s *Server) handlePostClaims(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	claims, err := decodeClaims(r)
+	if err != nil {
+		writeErrorSlug(w, http.StatusBadRequest, "invalid_json", "body must be a JSON array of claims: "+err.Error(), false)
+		return
+	}
+	if len(claims) > storage.MaxClaimsPerBatch {
+		writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per batch", false)
+		return
+	}
+	results, err := s.storage.WriteClaims(r.Context(), claims)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, results)
+}
+
+// ClaimsResponse is the body of the claims read routes: the record's rid, the
+// cids of its group, and every live signed claim on them.
+type ClaimsResponse struct {
+	RID    string        `json:"rid"`
+	CIDs   []string      `json:"cids"`
+	Claims []claim.Claim `json:"claims"`
+}
+
+// handleClaimsByCID handles GET /api/meta/{cid}/claims[?key=…].
+func (s *Server) handleClaimsByCID(w http.ResponseWriter, r *http.Request) {
+	cid := mux.Vars(r)["cid"]
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	rid, err := s.storage.GetByCID(cid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if rid == "" {
+		writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this CID", false)
+		return
+	}
+	s.writeClaims(w, r, rid)
+}
+
+// handleClaimsByRID handles GET /api/metadata/{rid}/claims[?key=…]. The rid
+// may also be a cid (resolved like every /api/metadata read).
+func (s *Server) handleClaimsByRID(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	rid := s.storage.ResolveRoot(mux.Vars(r)["hashId"])
+	s.writeClaims(w, r, rid)
+}
+
+// writeClaims answers both claims routes: ?key= keeps one exact key, ?uid=
+// one signer; 404 when the record does not exist.
+func (s *Server) writeClaims(w http.ResponseWriter, r *http.Request, rid string) {
+	cids, claims, err := s.storage.RecordClaims(rid, r.URL.Query().Get("key"))
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if len(cids) == 0 {
+		writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this rid or cid", false)
+		return
+	}
+	out := make([]claim.Claim, 0, len(claims))
+	uid := r.URL.Query().Get("uid")
+	for _, cl := range claims {
+		if uid == "" || cl.UID == uid {
+			out = append(out, cl)
+		}
+	}
+	writeJSON(w, http.StatusOK, ClaimsResponse{RID: rid, CIDs: cids, Claims: out})
+}
+
+// MergeRequest is the body of POST /api/merge.
+type MergeRequest struct {
+	Claims []claim.Claim `json:"claims"`
+	As     string        `json:"as,omitempty"`
+}
+
+// MergeResponse is the merged field map, the number of agreeing signers per
+// key, and how many claims failed verification (dropped, never an error).
+type MergeResponse struct {
+	Fields   map[string]string `json:"fields"`
+	Nsrc     map[string]int    `json:"nsrc"`
+	Rejected int               `json:"rejected"`
+}
+
+// handleMerge handles POST /api/merge: verify, merge per key with the same
+// rule the cache uses, overlay `as`'s own votes held by this core. No write.
+func (s *Server) handleMerge(w http.ResponseWriter, r *http.Request) {
+	var req MergeRequest
+	if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&req); err != nil {
+		writeErrorSlug(w, http.StatusBadRequest, "invalid_json", err.Error(), false)
+		return
+	}
+	if len(req.Claims) > storage.MaxClaimsPerBatch {
+		writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per merge", false)
+		return
+	}
+	fields, nsrc, rejected, err := s.storage.MergeClaims(req.Claims, req.As)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, MergeResponse{Fields: fields, Nsrc: nsrc, Rejected: rejected})
+}

internal/api/claims_test.go

@@ -0,0 +1,182 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+func withVars(r *http.Request, vars map[string]string) *http.Request { return mux.SetURLVars(r, vars) }
+
+// routedServer wires the real route table, so the test sees what clients see.
+func routedServer(t *testing.T) *Server {
+	t.Helper()
+	s := &Server{storage: newTestStorage(t), router: mux.NewRouter()}
+	s.setupRoutes()
+	return s
+}
+
+func call(t *testing.T, s *Server, method, path, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(method, path, strings.NewReader(body))
+	rr := httptest.NewRecorder()
+	s.router.ServeHTTP(rr, req)
+	return rr
+}
+
+// Every unsigned write route of v1.x answers 410 with a pointer to the claims.
+func TestUnsignedWritesAreGone(t *testing.T) {
+	s := routedServer(t)
+	for _, c := range []struct{ method, path string }{
+		{"PUT", "/meta/bafy"}, {"PATCH", "/meta/bafy"}, {"DELETE", "/meta/bafy"},
+		{"POST", "/meta/bafy/_add/genres"}, {"PUT", "/meta/bafy/title"}, {"DELETE", "/meta/bafy/title"},
+		{"PUT", "/api/metadata/bafy"}, {"DELETE", "/api/metadata/bafy"}, {"PUT", "/api/metadata/bafy/property"},
+		{"POST", "/api/metadata/batch"},
+	} {
+		rr := call(t, s, c.method, c.path, `{"title":"x"}`)
+		if rr.Code != http.StatusGone || !strings.Contains(rr.Body.String(), `"use":"POST /api/claims"`) {
+			t.Errorf("%s %s = %d %s", c.method, c.path, rr.Code, rr.Body)
+		}
+	}
+	if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"file:R/title","value":"x"}`); rr.Code != http.StatusConflict {
+		t.Errorf("raw record write via /api/kv/value = %d", rr.Code)
+	}
+	if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"btmagnet:abc","value":"magnet:"}`); rr.Code != http.StatusOK {
+		t.Errorf("service key write = %d %s", rr.Code, rr.Body)
+	}
+}
+
+func TestPostClaimsAndReadBack(t *testing.T) {
+	s := routedServer(t)
+	a, b := testSigner(t, 1), testSigner(t, 2)
+	bad := signed(t, b, "bafyA", "year", "1999", 5)
+	bad.Value = claim.Text("2001")
+	body, _ := json.Marshal([]claim.Claim{
+		signed(t, a, "bafyA", "title", "Matrix", 5),
+		signed(t, b, "bafyA", "title", "Matrix", 6),
+		bad,
+	})
+	rr := call(t, s, "POST", "/api/claims", string(body))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("POST /api/claims = %d %s", rr.Code, rr.Body)
+	}
+	var res []storage.ClaimResult
+	_ = json.Unmarshal(rr.Body.Bytes(), &res)
+	if len(res) != 3 || res[0].Status != "accepted" || res[1].Status != "accepted" || res[2].Status != "bad_signature" {
+		t.Fatalf("statuses %+v", res)
+	}
+
+	rr = call(t, s, "GET", "/api/meta/bafyA", "")
+	if !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
+		t.Fatalf("merged read = %s", rr.Body)
+	}
+	rr = call(t, s, "GET", "/api/meta/bafyA/claims", "")
+	var cr ClaimsResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &cr)
+	if rr.Code != http.StatusOK || len(cr.Claims) != 2 {
+		t.Fatalf("claims by cid = %d %s", rr.Code, rr.Body)
+	}
+	if cr.RID == "" || len(cr.CIDs) != 1 || cr.CIDs[0] != "bafyA" {
+		t.Fatalf("claims envelope rid/cids = %q %v", cr.RID, cr.CIDs)
+	}
+	rr = call(t, s, "GET", "/api/metadata/"+cr.RID+"/claims?key=title&uid="+a.UID, "")
+	var byRid ClaimsResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &byRid)
+	if rr.Code != http.StatusOK || len(byRid.Claims) != 1 || byRid.Claims[0].UID != a.UID || byRid.RID != cr.RID {
+		t.Fatalf("claims by rid with key+uid = %d %s", rr.Code, rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyA/claims?key=nope", ""); !strings.Contains(rr.Body.String(), `"claims":[]`) {
+		t.Fatalf("key filter = %s", rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyUnknown/claims", ""); rr.Code != http.StatusNotFound {
+		t.Fatalf("unknown cid claims = %d", rr.Code)
+	}
+	if rr := call(t, s, "GET", "/api/metadata/01NOSUCHRID/claims", ""); rr.Code != http.StatusNotFound {
+		t.Fatalf("unknown rid claims = %d", rr.Code)
+	}
+
+	// ?as= overlays the viewer's own vote.
+	me := testSigner(t, 9)
+	storeClaims(t, s.storage, signed(t, me, "bafyA", "title", "My title", 7))
+	if rr := call(t, s, "GET", "/api/meta/bafyA?as="+me.UID, ""); !strings.Contains(rr.Body.String(), `"title":"My title"`) {
+		t.Fatalf("?as overlay = %s", rr.Body)
+	}
+	if rr := call(t, s, "GET", "/api/meta/bafyA", ""); !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
+		t.Fatalf("shared view changed by a single viewer: %s", rr.Body)
+	}
+}
+
+func TestMergeEndpoint(t *testing.T) {
+	s := routedServer(t)
+	a, b, c := testSigner(t, 1), testSigner(t, 2), testSigner(t, 3)
+	bad := signed(t, c, "bafyX", "title", "Forged", 1)
+	bad.UID = a.UID
+	body, _ := json.Marshal(MergeRequest{Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, b, "bafyX", "title", "Dune", 3),
+		signed(t, c, "bafyX", "title", "Dune Part One", 1),
+		bad,
+	}})
+	rr := call(t, s, "POST", "/api/merge", string(body))
+	var mr MergeResponse
+	_ = json.Unmarshal(rr.Body.Bytes(), &mr)
+	if rr.Code != http.StatusOK || mr.Fields["title"] != "Dune" || mr.Nsrc["title"] != 2 || mr.Rejected != 1 {
+		t.Fatalf("merge = %d %s", rr.Code, rr.Body)
+	}
+	if !strings.Contains(rr.Body.String(), `"rejected":1`) || !strings.Contains(rr.Body.String(), `"nsrc":{"title":2}`) {
+		t.Fatalf("merge wire shape = %s", rr.Body)
+	}
+
+	// `as`: the viewer's own vote in the request wins…
+	me := testSigner(t, 9)
+	body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, b, "bafyX", "title", "Dune", 3),
+		signed(t, me, "bafyX", "title", "Dune (my cut)", 4),
+	}})
+	_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
+	if mr.Fields["title"] != "Dune (my cut)" || mr.Nsrc["title"] != 1 {
+		t.Fatalf("request overlay = %+v", mr)
+	}
+	// …and so do its votes held by this core for the same cids.
+	storeClaims(t, s.storage, signed(t, me, "bafyX", "year", "2021", 5))
+	body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
+		signed(t, a, "bafyX", "title", "Dune", 2),
+		signed(t, a, "bafyX", "year", "1984", 2),
+	}})
+	_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
+	if mr.Fields["year"] != "2021" || mr.Fields["title"] != "Dune" {
+		t.Fatalf("local overlay = %+v", mr)
+	}
+	// An all-invalid batch is still 200 with everything rejected.
+	bad2 := signed(t, a, "bafyX", "title", "x", 9)
+	bad2.Sig = bad2.Sig[:10]
+	body, _ = json.Marshal(MergeRequest{Claims: []claim.Claim{bad2}})
+	rr = call(t, s, "POST", "/api/merge", string(body))
+	if rr.Code != http.StatusOK || !strings.Contains(rr.Body.String(), `"rejected":1`) {
+		t.Fatalf("all-invalid merge = %d %s", rr.Code, rr.Body)
+	}
+}
+
+func TestTooManyClaimsIs413(t *testing.T) {
+	s := routedServer(t)
+	var sb strings.Builder
+	sb.WriteString("[")
+	for i := 0; i <= storage.MaxClaimsPerBatch; i++ {
+		if i > 0 {
+			sb.WriteString(",")
+		}
+		sb.WriteString(`{"cid":"c","uid":"u","key":"k","value":null,"ts":1,"sig":""}`)
+	}
+	sb.WriteString("]")
+	if rr := call(t, s, "POST", "/api/claims", sb.String()); rr.Code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized batch = %d", rr.Code)
+	}
+}

internal/api/deprecated_field_test.go

@@ -1,86 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the
-// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They
-// must be rejected at the write boundary rather than stored.
-//
-// Why a hard 400 and not a silent rewrite: the reverse-index hook
-// (maybeAddAliasFromFieldLocked) only registers an alias for a field named
-// `cids/<cid>`. A `cid_*` field was therefore written verbatim and never
-// indexed — the record silently became unresolvable by that CID. A loud
-// failure is strictly better than a record that looks correct and cannot be
-// found.
-func TestRejectDeprecatedField(t *testing.T) {
-	rejected := []string{
-		"cid_midhash256",
-		"cid_sha2-256",
-		"cid_btih_v2",
-		"cid_crc32",
-		"midhash256",
-		"canonical_cid",
-	}
-	for _, f := range rejected {
-		if !rejectDeprecatedField(f) {
-			t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f)
-		}
-	}
-
-	accepted := []string{
-		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
-		"filePath",
-		"title",
-		"fileinfo/duration",
-		"tmdb/poster",
-		// Near-misses that must NOT be caught: the prefix is `cid_`, with an
-		// underscore. A field merely starting with "cid" is fine.
-		"cids",
-		"cidrank",
-	}
-	for _, f := range accepted {
-		if rejectDeprecatedField(f) {
-			t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f)
-		}
-	}
-}
-
-func TestValidateFields_Rejects400WithSlug(t *testing.T) {
-	rr := httptest.NewRecorder()
-	blocked := validateFields(rr, map[string]string{
-		"title":          "Inception",
-		"cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
-	})
-
-	if !blocked {
-		t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed")
-	}
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest)
-	}
-
-	var body map[string]interface{}
-	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-		t.Fatalf("response is not JSON: %v", err)
-	}
-	if got := body["error"]; got != ErrDeprecatedField {
-		t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField)
-	}
-}
-
-func TestValidateFields_AllowsCidsKeyset(t *testing.T) {
-	rr := httptest.NewRecorder()
-	blocked := validateFields(rr, map[string]string{
-		"filePath": "/watch/Inception.mkv",
-		"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true",
-		"cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua":     "true",
-	})
-	if blocked {
-		t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String())
-	}
-}

internal/api/files_tuples_test.go

@@ -28,25 +28,25 @@ func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesRespon
 
 func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
 	t.Helper()
-	id, err := c.Mint(path, size, mtime)
+	id, _, err := c.RegisterFileTuple(path, size, mtime, "bafkmid"+path)
 	if err != nil {
-		t.Fatalf("mint %s: %v", path, err)
+		t.Fatalf("register %s: %v", path, err)
 	}
 	return id
 }
 
-// Only records that are actually files count: a gateway record and a record
-// that merely carries a path + size (no mtimeNano) are both left out.
+// Only local files count: a gateway record — even one whose claims carry a
+// filePath and size — is left out.
 func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
 	s, c := newFileTuplesTestServer(t)
 	a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
 	b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
-	if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil {
-		t.Fatal(err)
-	}
+	gw := testSigner(t, 1)
+	storeClaims(t, c,
+		signed(t, gw, "bagcsgatewayrecord", "title", "Some Release", 1),
+		signed(t, gw, "bafkpartial", "filePath", "/files/cache/x.mkv", 1),
+		signed(t, gw, "bafkpartial", "sizeByte", "999", 1),
+	)
 
 	code, resp := getFileTuples(t, s, "")
 	if code != http.StatusOK {

internal/api/handlers.go

@@ -4,7 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/base32"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -89,42 +88,6 @@ const (
 	ErrDeprecatedField    = "deprecated_field"
 )
 
-// rejectDeprecatedField reports whether a field name is one of the removed
-// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …).
-//
-// This is a tripwire, not a compatibility shim. Such a field used to be
-// accepted and written verbatim — but the reverse-index hook
-// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named
-// `cids/<cid>`, so a `cid_*` field was stored and then NEVER INDEXED: the
-// record silently became unresolvable by that CID. Failing the write loudly is
-// strictly better than a record that looks fine and cannot be found.
-//
-// Writers must emit the bare-CID key-set instead: `cids/<bareCid>` = "true".
-// See METADATA_KEYS.md §2/§14.13.
-func rejectDeprecatedField(field string) bool {
-	return strings.HasPrefix(field, "cid_") ||
-		field == "midhash256" ||
-		field == "canonical_cid"
-}
-
-// validateFields rejects a write carrying any deprecated CID field. Returns
-// true if the request was rejected (the handler must return immediately).
-func validateFields(w http.ResponseWriter, metadata map[string]string) bool {
-	for field := range metadata {
-		if rejectDeprecatedField(field) {
-			writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField,
-				fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+
-					"write the bare-CID key-set instead: \"cids/<bareCid>\": \"true\" "+
-					"(a cid_* field would be stored but never indexed, leaving the "+
-					"record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.",
-					field),
-				false)
-			return true
-		}
-	}
-	return false
-}
-
 // defaultSlugFor maps an HTTP status code to the most likely error slug.
 // Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
 // instead of "unknown_root" on 404) call writeErrorSlug directly.
@@ -250,7 +213,7 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
 	// the watcher minted the root, regardless of which CID the caller knows.
 	hashID = s.storage.ResolveRoot(hashID)
 
-	metadata, err := s.storage.GetMetadataFlat(hashID)
+	metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -269,76 +232,6 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handlePutMeta handles PUT /meta/{hash}
-func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// CID → UUID resolution so writes land on the canonical root, not on
-	// a parallel midhash-rooted entry the watcher's UUID never sees.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if validateFields(w, metadata) {
-		return
-	}
-
-	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-	})
-}
-
-// handleDeleteMeta handles DELETE /meta/{hash}
-func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	deleted, err := s.storage.DeleteMetadata(hashID)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-		"deleted": deleted,
-	})
-}
-
 // handleListMeta handles GET /meta
 func (s *Server) handleListMeta(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
@@ -623,50 +516,6 @@ func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, ret
 	})
 }
 
-// handlePatchMeta handles PATCH /meta/{hash}
-// Merges the provided metadata into existing (does not delete missing keys)
-func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// PATCH is the hot write path for meta-sort. Resolve so plugin output
-	// merges into the watcher's UUID root rather than minting a parallel
-	// midhash-rooted entry.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if validateFields(w, metadata) {
-		return
-	}
-
-	updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"hashId":  hashID,
-		"updated": updated,
-	})
-}
-
 // handleGetProperty handles GET /meta/{hash}/{key}
 // Gets a single property value
 func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
@@ -713,171 +562,6 @@ type PropertyBody struct {
 	Value string `json:"value"`
 }
 
-// handlePutProperty handles PUT /meta/{hash}/{key}
-// Sets a single property value
-// Accepts JSON body with {"value": "..."} or plain text
-func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if validateFields(w, map[string]string{key: ""}) {
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	// Read body
-	buf := make([]byte, 1024*1024) // 1MB max
-	n, err := r.Body.Read(buf)
-	if err != nil && err.Error() != "EOF" {
-		writeError(w, http.StatusBadRequest, "failed to read body")
-		return
-	}
-	body := buf[:n]
-
-	// Try to parse as JSON {"value": "..."} first
-	var propBody PropertyBody
-	var value string
-	if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
-		// JSON body with value field
-		value = propBody.Value
-	} else {
-		// Plain text body
-		value = string(body)
-	}
-
-	if err := s.storage.SetProperty(hashID, key, value); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-	})
-}
-
-// handleDeleteProperty handles DELETE /meta/{hash}/{key}
-// Deletes a single property
-func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	if err := s.storage.DeleteProperty(hashID, key); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-	})
-}
-
-// handleAddToSet handles POST /meta/{hash}/_add/{key}
-// Adds a value to a set-type field
-// Accepts JSON body with {"value": "..."} or plain text
-func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hash"]
-	key := vars["key"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash is required")
-		return
-	}
-
-	if key == "" {
-		writeError(w, http.StatusBadRequest, "key is required")
-		return
-	}
-
-	if validateFields(w, map[string]string{key: ""}) {
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	hashID = s.storage.ResolveRoot(hashID)
-
-	// Read body
-	buf := make([]byte, 1024*1024) // 1MB max
-	n, err := r.Body.Read(buf)
-	if err != nil && err.Error() != "EOF" {
-		writeError(w, http.StatusBadRequest, "failed to read body")
-		return
-	}
-	body := buf[:n]
-
-	// Try to parse as JSON {"value": "..."} first
-	var propBody PropertyBody
-	var value string
-	if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
-		// JSON body with value field
-		value = propBody.Value
-	} else {
-		// Plain text body
-		value = string(body)
-	}
-
-	if value == "" {
-		writeError(w, http.StatusBadRequest, "value is required")
-		return
-	}
-
-	added, err := s.storage.AddToSet(hashID, key, value)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success":  true,
-		"hashId":   hashID,
-		"property": key,
-		"added":    added,
-	})
-}
-
 // CIDRequest is the request body for POST /file/cid
 type CIDRequest struct {
 	Path string `json:"path"`

internal/api/handlers_identity.go

@@ -4,11 +4,14 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/metazla/meta-core/internal/claim"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/storage"
 )
@@ -120,6 +123,9 @@ type identityDeleteResponse struct {
 
 type identitySignRequest struct {
 	BytesB64 string `json:"bytesB64"`
+	// Format is "der" or "compact" (64-byte r||s, low-S). Default: compact for
+	// MMUR2 claim payloads ("MMUR2\n" prefix), DER otherwise (MMUR1 back-compat).
+	Format string `json:"format,omitempty"`
 	// UID selects which account signs. Optional — when omitted and exactly one
 	// account exists, that account is used (single-identity back-compat).
 	UID string `json:"uid,omitempty"`
@@ -143,6 +149,8 @@ type identitySignBatchRequest struct {
 	// UID selects which account signs — same optional single-identity
 	// back-compat as identitySignRequest.
 	UID string `json:"uid,omitempty"`
+	// Format: see identitySignRequest.Format; applies to every payload.
+	Format string `json:"format,omitempty"`
 }
 
 type identitySignBatchResponse struct {
@@ -445,10 +453,10 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
 			"storage is not connected; refusing to delete the key while its data cannot be purged", true)
 		return
 	}
-	purged, err := s.storage.UDLPurgeUser(id.UID)
+	purged, err := s.retractAllClaims(r, id)
 	if err != nil {
 		writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
-			"could not purge this account's data; the account was NOT deleted: "+err.Error(), true)
+			"could not retract this account's claims; the account was NOT deleted: "+err.Error(), true)
 		return
 	}
 
@@ -456,7 +464,7 @@ func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
 		writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
 		return
 	}
-	log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids",
+	log.Printf("[API] Deleted account %s after retracting %d claims across %d cids",
 		id.UID, purged.Records, purged.Cids)
 	writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
 }
@@ -485,9 +493,9 @@ func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
 		writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
 		return
 	}
-	sig, err := identity.Sign(id, raw)
+	sig, err := signPayload(id, raw, body.Format)
 	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
+		writeError(w, http.StatusBadRequest, err.Error())
 		return
 	}
 	writeJSON(w, http.StatusOK, identitySignResponse{
@@ -537,9 +545,9 @@ func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request)
 				"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
 			return
 		}
-		sig, err := identity.Sign(id, raw)
+		sig, err := signPayload(id, raw, body.Format)
 		if err != nil {
-			writeError(w, http.StatusInternalServerError, err.Error())
+			writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"]: "+err.Error())
 			return
 		}
 		sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
@@ -562,3 +570,61 @@ func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
 		AEADKeyB64: base64.StdEncoding.EncodeToString(key),
 	})
 }
+
+// signPayload signs raw with the account key in the requested encoding.
+func signPayload(id *identity.Identity, raw []byte, format string) ([]byte, error) {
+	switch format {
+	case "compact":
+		return identity.SignCompact(id, raw)
+	case "der":
+		return identity.Sign(id, raw)
+	case "":
+		if strings.HasPrefix(string(raw), claim.Domain) {
+			return identity.SignCompact(id, raw)
+		}
+		return identity.Sign(id, raw)
+	}
+	return nil, fmt.Errorf("format must be \"compact\" or \"der\"")
+}
+
+// retractAllClaims signs a retraction for every live claim the account holds
+// and stores them (G11): the tombstones stay, so the votes cannot be replayed.
+func (s *Server) retractAllClaims(r *http.Request, id *identity.Identity) (storage.UDLUserStats, error) {
+	refs, err := s.storage.LiveClaimsBySigner(id.UID)
+	if err != nil {
+		return storage.UDLUserStats{}, err
+	}
+	cids, keys := map[string]bool{}, map[string]bool{}
+	now := uint64(time.Now().UnixMilli())
+	batch := make([]claim.Claim, 0, len(refs))
+	for _, ref := range refs {
+		cur, ok, err := s.storage.UDLGetClaim(id.UID, ref[0], ref[1])
+		if err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		ts := now
+		if ok && cur.TS >= ts {
+			ts = cur.TS + 1
+		}
+		tomb := claim.Claim{CID: ref[0], UID: id.UID, Key: ref[1], Value: claim.Null(), TS: ts}
+		if err := tomb.Sign(id); err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		batch = append(batch, tomb)
+		cids[ref[0]], keys[ref[1]] = true, true
+	}
+	for start := 0; start < len(batch); start += storage.MaxClaimsPerBatch {
+		end := min(start+storage.MaxClaimsPerBatch, len(batch))
+		results, err := s.storage.WriteClaims(r.Context(), batch[start:end])
+		if err != nil {
+			return storage.UDLUserStats{}, err
+		}
+		for i, res := range results {
+			if res.Status != storage.StatusAccepted && res.Status != storage.StatusUnchanged {
+				return storage.UDLUserStats{}, fmt.Errorf("retraction of %s/%s refused: %s %s",
+					batch[start+i].CID, batch[start+i].Key, res.Status, res.Reason)
+			}
+		}
+	}
+	return storage.UDLUserStats{Records: len(batch), Cids: len(cids), Keys: len(keys)}, nil
+}

internal/api/identity_gate_test.go

@@ -96,12 +96,16 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
 	uid := gen["uid"].(string)
 	priv := gen["privateKeyHex"].(string)
 
-	// Some data to lose.
-	for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} {
-		if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok {
-			t.Fatalf("seed: %v", err)
-		}
+	// Some data to lose: claims signed by the account.
+	acct, err := identity.Import(priv)
+	if err != nil {
+		t.Fatal(err)
 	}
+	storeClaims(t, s.storage,
+		signed(t, acct, "cidA", "like", "true", 100),
+		signed(t, acct, "cidA", "userRating", "5", 100),
+		signed(t, acct, "cidB", "seek", []byte("enc"), 100),
+	)
 
 	// The count the UI shows in the confirmation.
 	rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
@@ -175,7 +179,7 @@ func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
 	if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
 		t.Fatal("the account file survived the delete")
 	}
-	// Data gone — the orphaning this whole change exists to prevent.
+	// Every claim retracted (tombstones kept) — nothing left to orphan.
 	after, err := s.storage.UDLAllUserStats()
 	if err != nil {
 		t.Fatalf("stats after: %v", err)

internal/api/kv.go

@@ -383,8 +383,9 @@ func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// handleKVValuePut handles PUT /api/kv/value with body {key, value}.
-// Maintains the file index when the key matches file:<hashId>/...
+// handleKVValuePut handles PUT /api/kv/value with body {key, value}: service
+// scratch KV only (btmagnet:*, searchcov:*), kept in the cache. Merged records
+// and bookkeeping are refused — metadata is written as signed claims.
 func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
@@ -401,19 +402,11 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := s.storage.SetRawValue(req.Key, req.Value); err != nil {
+	if err := s.storage.SetServiceValue(req.Key, req.Value); err != nil {
 		writeError(w, http.StatusConflict, err.Error())
 		return
 	}
 
-	if hashID := extractFileHashID(req.Key); hashID != "" {
-		if err := s.storage.IndexAdd(hashID); err != nil {
-			// Non-fatal: the value was written; index entry is best-effort.
-			// Surface in the log path of writeJSON via response header.
-			w.Header().Set("X-KV-Index-Warning", err.Error())
-		}
-	}
-
 	writeJSON(w, http.StatusOK, KVValueResponse{
 		Key:    req.Key,
 		Type:   "string",
@@ -422,9 +415,7 @@ func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// handleKVValueDelete handles DELETE /api/kv/value?key=...
-// When the key matches file:<hashId>/..., removes the file index entry if no
-// sibling keys remain after the delete.
+// handleKVValueDelete handles DELETE /api/kv/value?key=... (service keys only).
 func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
@@ -437,36 +428,14 @@ func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	deleted, err := s.storage.DeleteRawKey(key)
+	deleted, err := s.storage.DeleteServiceValue(key)
 	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
+		writeError(w, http.StatusConflict, err.Error())
 		return
 	}
 
-	if deleted {
-		if hashID := extractFileHashID(key); hashID != "" {
-			if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil {
-				w.Header().Set("X-KV-Index-Warning", err.Error())
-			}
-		}
-	}
-
 	writeJSON(w, http.StatusOK, KVValueDeleteResponse{
 		Key:     key,
 		Deleted: deleted,
 	})
 }
-
-// extractFileHashID returns the hashID for a key shaped like file:<hashId>/...
-// Returns "" when the key does not follow that convention.
-func extractFileHashID(key string) string {
-	if !strings.HasPrefix(key, "file:") {
-		return ""
-	}
-	rest := strings.TrimPrefix(key, "file:")
-	slash := strings.Index(rest, "/")
-	if slash <= 0 {
-		return ""
-	}
-	return rest[:slash]
-}

internal/api/metadata.go

@@ -38,6 +38,9 @@ type MetadataSearchRequest struct {
 	Property      string `json:"property,omitempty"`
 	PropertyValue string `json:"propertyValue,omitempty"`
 	Limit         int    `json:"limit,omitempty"`
+	// As is a viewer uid: the search sees that signer's own claims overlaid
+	// (and their records as candidates). Also accepted as ?as=.
+	As string `json:"as,omitempty"`
 
 	// Filters is the structured form: AND across entries, OR within an
 	// entry's Values. Narrowed in SQL through the field index alongside the
@@ -222,10 +225,10 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	// Same resolution as the PUT below: callers hold a CID, not the root.
+	// Callers may hold a CID, not the root.
 	hashID = s.storage.ResolveRoot(hashID)
 
-	metadata, err := s.storage.GetMetadataFlat(hashID)
+	metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -244,87 +247,6 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
 	writeJSON(w, http.StatusOK, response)
 }
 
-// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId}
-func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// The {hashId} in the URL is whatever CID the caller knows the file by —
-	// the gateway's auto-store keys this PUT by the hash it just computed
-	// (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is
-	// the dag-pb root, not the nzb-release cid the record was minted under.
-	// Without resolving, the write lands on file:<thatCid>/* and forks a
-	// second root for a file that already has one: a full duplicate record,
-	// cross-linked and separately indexed, which every downstream ingest then
-	// sees as two files. Unknown CIDs pass through unchanged, so a genuinely
-	// new gateway record still mints its root at the hash.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	var metadata map[string]string
-	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	// Exclude internal processing status field
-	delete(metadata, "processingStatus")
-
-	if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":  "ok",
-		"hashId":  hashID,
-		"message": "Metadata updated successfully",
-	})
-}
-
-// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId}
-func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// meta-gateway retracts a dropped hit by whichever CID it holds
-	// (meta_core.rs `delete`, which documents this resolution as a
-	// precondition). Without it, deleting by a non-root CID is a silent
-	// no-op and the retracted record lingers.
-	hashID = s.storage.ResolveRoot(hashID)
-
-	deletedCount, err := s.storage.DeleteMetadata(hashID)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":      "ok",
-		"hashId":      hashID,
-		"deletedKeys": deletedCount,
-	})
-}
-
 // handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X
 func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
@@ -364,46 +286,6 @@ func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Reques
 	})
 }
 
-// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property
-func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	hashID := vars["hashId"]
-
-	if hashID == "" {
-		writeError(w, http.StatusBadRequest, "hash ID is required")
-		return
-	}
-
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	var req PropertyUpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if req.Property == "" {
-		writeError(w, http.StatusBadRequest, "property is required")
-		return
-	}
-
-	if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"status":   "ok",
-		"hashId":   hashID,
-		"property": req.Property,
-		"value":    req.Value,
-		"message":  "Property updated successfully",
-	})
-}
-
 // handleSearchMetadata handles POST /api/metadata/search
 func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {
@@ -420,6 +302,9 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	if req.Limit == 0 {
 		req.Limit = 100
 	}
+	if req.As == "" {
+		req.As = r.URL.Query().Get("as")
+	}
 
 	// If hashId is provided, do exact match
 	if req.HashID != "" {
@@ -457,83 +342,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	s.serveIndexedSearch(w, req)
 }
 
-// handleBatchUpdate handles POST /api/metadata/batch
-func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	var req MetadataBatchUpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-
-	if len(req.Updates) == 0 {
-		writeError(w, http.StatusBadRequest, "updates array is required")
-		return
-	}
-
-	var results []MetadataBatchResult
-	successCount := 0
-	errorCount := 0
-
-	for _, update := range req.Updates {
-		if update.HashID == "" {
-			results = append(results, MetadataBatchResult{
-				HashID: "unknown",
-				Status: "error",
-				Error:  "Missing hash ID",
-			})
-			errorCount++
-			continue
-		}
-
-		var updateErr error
-
-		// Update complete metadata
-		if update.Metadata != nil {
-			// Exclude internal processing status
-			delete(update.Metadata, "processingStatus")
-			updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata)
-		}
-
-		// Update individual properties
-		if updateErr == nil && update.Properties != nil {
-			for property, value := range update.Properties {
-				if err := s.storage.SetProperty(update.HashID, property, value); err != nil {
-					updateErr = err
-					break
-				}
-			}
-		}
-
-		if updateErr != nil {
-			results = append(results, MetadataBatchResult{
-				HashID: update.HashID,
-				Status: "error",
-				Error:  updateErr.Error(),
-			})
-			errorCount++
-		} else {
-			results = append(results, MetadataBatchResult{
-				HashID: update.HashID,
-				Status: "ok",
-			})
-			successCount++
-		}
-	}
-
-	writeJSON(w, http.StatusOK, MetadataBatchResponse{
-		Status:  "completed",
-		Total:   len(results),
-		Success: successCount,
-		Errors:  errorCount,
-		Results: results,
-	})
-}
-
 // handleClearMetadata handles POST /api/metadata/clear
 func (s *Server) handleClearMetadata(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {

internal/api/search_index.go

@@ -29,7 +29,7 @@ func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchReq
 	for _, f := range filters {
 		narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
 	}
-	hits, err := s.storage.Search(req.Query, narrow, req.Limit, func(hashID string, fields map[string]string) bool {
+	hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool {
 		if req.Query != "" &&
 			!matchAllTokens(storage.Haystack(fields), tokens) &&
 			!strings.Contains(strings.ToLower(hashID), queryLower) {

internal/api/server.go

@@ -176,7 +176,7 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
 	s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
 	s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
-	s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
+	s.router.HandleFunc("/api/metadata/batch", s.handleGone).Methods("POST")
 	s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
 
 	// User Data Layer signing identity (see internal/identity). Auth-gated
@@ -192,15 +192,18 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
 	s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
 
-	// Snapshot (export / import / wipe)
-	s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
-	s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
-	s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
+	// Signed claims (v2.0.0): the only metadata write path, the raw claims
+	// behind a record, and the stateless merge clients use for search hits.
+	s.router.HandleFunc("/api/claims", s.handlePostClaims).Methods("POST")
+	s.router.HandleFunc("/api/merge", s.handleMerge).Methods("POST")
+	s.router.HandleFunc("/api/meta/{cid}/claims", s.handleClaimsByCID).Methods("GET")
+	s.router.HandleFunc("/api/metadata/{hashId}/claims", s.handleClaimsByRID).Methods("GET")
+
 	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
-	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
+	s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleGone).Methods("PUT")
 	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
-	s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
-	s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
+	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("PUT")
+	s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("DELETE")
 
 	// Schema inference
 	s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
@@ -232,18 +235,15 @@ func (s *Server) setupRoutes() {
 
 	// Metadata operations - base endpoints
 	s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
-	s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
-	s.router.HandleFunc("/meta/{hash}", s.handlePatchMeta).Methods("PATCH")
-	s.router.HandleFunc("/meta/{hash}", s.handleDeleteMeta).Methods("DELETE")
+	s.router.HandleFunc("/meta/{hash}", s.handleGone).Methods("PUT", "PATCH", "DELETE")
 	s.router.HandleFunc("/meta", s.handleListMeta).Methods("GET")
 
 	// Metadata operations - set operations (must be before property routes)
-	s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleAddToSet).Methods("POST")
+	s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleGone).Methods("POST")
 
 	// Metadata operations - property-level (key can contain slashes)
 	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGetProperty).Methods("GET")
-	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handlePutProperty).Methods("PUT")
-	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleDeleteProperty).Methods("DELETE")
+	s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGone).Methods("PUT", "DELETE")
 
 	// Data operations
 	s.router.HandleFunc("/data/{hash}/path", s.handleGetDataPath).Methods("GET")

internal/api/snapshot.go

@@ -1,167 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/metazla/meta-core/internal/snapshot"
-)
-
-// Bound the upload size so a malicious client can't OOM the server.
-// 2 GiB is well above any realistic export.
-const maxImportBytes = 2 << 30
-
-// pluginOutputDir is where plugin-produced artifacts live on disk
-// (subtitles, posters, etc). Mounted from the host into the meta-core
-// container at $FILES_PATH/plugin.
-func (s *Server) pluginOutputDir() string {
-	return filepath.Join(s.config.FilesPath, "plugin")
-}
-
-// handleSnapshotExport streams a ZIP snapshot of all metadata.
-// GET /api/snapshot/export?include=hash-index,plugin-output
-func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// The server has a 30s WriteTimeout (server.go:222). Long exports
-	// blow past that, leaving a truncated ZIP without a central directory.
-	// Clear the deadline for this handler so the export can run as long as
-	// needed. ResponseController is the standard way to do this.
-	rc := http.NewResponseController(w)
-	_ = rc.SetWriteDeadline(time.Time{})
-
-	include := splitCSVParam(r.URL.Query().Get("include"))
-	opts := snapshot.ExportOptions{
-		IncludePluginOutput: include["plugin-output"],
-		PluginOutputDir:     s.pluginOutputDir(),
-	}
-
-	filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
-	w.Header().Set("Content-Type", "application/zip")
-	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
-	// We stream — no Content-Length.
-
-	exporter := snapshot.NewExporter(s.storage)
-	if _, err := exporter.Export(w, opts); err != nil {
-		// Headers may already be sent; log and bail.
-		// Client will see a truncated zip, which is the best signal we can give.
-		fmt.Printf("[Snapshot] export failed: %v\n", err)
-		return
-	}
-}
-
-// handleSnapshotImport applies an uploaded ZIP snapshot.
-// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
-// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
-func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
-	// when the snapshot is large. Clear both for this handler.
-	rc := http.NewResponseController(w)
-	_ = rc.SetReadDeadline(time.Time{})
-	_ = rc.SetWriteDeadline(time.Time{})
-
-	q := r.URL.Query()
-	opts := snapshot.ImportOptions{
-		Mode:            snapshot.Mode(q.Get("mode")),
-		Conflict:        snapshot.Conflict(q.Get("conflict")),
-		DryRun:          strings.EqualFold(q.Get("dry_run"), "true"),
-		PluginOutputDir: s.pluginOutputDir(),
-	}
-	if opts.Mode == "" {
-		opts.Mode = snapshot.ModeMerge
-	}
-	if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
-		opts.Conflict = snapshot.ConflictSource
-	}
-
-	body, err := readImportBody(r)
-	if err != nil {
-		writeError(w, http.StatusBadRequest, err.Error())
-		return
-	}
-
-	importer := snapshot.NewImporter(s.storage)
-	result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
-	if err != nil {
-		writeError(w, http.StatusBadRequest, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, result)
-}
-
-// handleSnapshotWipe clears all stored metadata.
-// POST /api/snapshot/wipe?scope=metadata
-func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	scopes := splitCSVParam(r.URL.Query().Get("scope"))
-	if len(scopes) == 0 {
-		writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
-		return
-	}
-	scope := snapshot.WipeScope{
-		Metadata: scopes["metadata"],
-	}
-	if !scope.Metadata {
-		writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
-		return
-	}
-
-	wiper := snapshot.NewWiper(s.storage)
-	res, err := wiper.Wipe(scope)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, res)
-}
-
-// readImportBody accepts either a raw application/zip body or a multipart
-// upload with a field named "snapshot". Buffers up to maxImportBytes.
-func readImportBody(r *http.Request) ([]byte, error) {
-	r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
-
-	ct := r.Header.Get("Content-Type")
-	if strings.HasPrefix(ct, "multipart/form-data") {
-		if err := r.ParseMultipartForm(32 << 20); err != nil {
-			return nil, fmt.Errorf("parse multipart: %w", err)
-		}
-		f, _, err := r.FormFile("snapshot")
-		if err != nil {
-			return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
-		}
-		defer f.Close()
-		return io.ReadAll(f)
-	}
-
-	return io.ReadAll(r.Body)
-}
-
-// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
-func splitCSVParam(v string) map[string]bool {
-	out := map[string]bool{}
-	for _, part := range strings.Split(v, ",") {
-		p := strings.TrimSpace(part)
-		if p != "" {
-			out[p] = true
-		}
-	}
-	return out
-}

internal/api/sse_events.go

@@ -112,7 +112,15 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 		}
 		cursor = head
 	} else {
-		cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher)
+		if rebuilt := s.storage.RebuiltAt(); stream == storage.MetaEventsStream && lastID != "" && rebuilt != "" &&
+			compareStreamIDs(lastID, rebuilt) < 0 {
+			// The cache was rebuilt after this cursor: the rebuild published no
+			// per-key events, so the consumer must take a snapshot.
+			writeGap(w, flusher, lastID, rebuilt, "rebuild")
+			cursor = "0"
+		} else {
+			cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher)
+		}
 	}
 
 	lastWrite := time.Now()
@@ -256,14 +264,7 @@ func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID str
 	}
 
 	// Cursor was trimmed. Emit a gap event so consumers that care can react.
-	gap := map[string]string{
-		"requested":  lastID,
-		"resumeFrom": oldest,
-		"reason":     "retention",
-	}
-	payload, _ := json.Marshal(gap)
-	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
-	flusher.Flush()
+	writeGap(w, flusher, lastID, oldest, "retention")
 
 	// Return "0" so the next read picks up entries starting from the
 	// stream's true minimum — including `oldest` itself.
@@ -327,3 +328,15 @@ func parseStreamID(id string) (int64, int64) {
 	}
 	return ms, seq
 }
+
+// writeGap emits the synthetic `gap` event: the consumer's cursor cannot be
+// resumed losslessly and it should take a snapshot (Last-Event-ID: 0).
+func writeGap(w http.ResponseWriter, flusher http.Flusher, requested, resumeFrom, reason string) {
+	payload, _ := json.Marshal(map[string]string{
+		"requested":  requested,
+		"resumeFrom": resumeFrom,
+		"reason":     reason,
+	})
+	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
+	flusher.Flush()
+}

internal/api/sse_snapshot_test.go

@@ -33,11 +33,11 @@ func (r *syncRecorder) body() string {
 func TestMetaEventsSnapshotThenTail(t *testing.T) {
 	stor := newTestStorage(t)
 	s := &Server{storage: stor}
-	for i, id := range []string{"R1", "R2"} {
-		if err := stor.SetMetadataFlat(id, map[string]string{"title": "T", "n": string(rune('a' + i))}); err != nil {
-			t.Fatal(err)
-		}
+	a := testSigner(t, 1)
+	for i, cid := range []string{"bafyR1", "bafyR2"} {
+		storeClaims(t, stor, signed(t, a, cid, "title", "T", 1), signed(t, a, cid, "n", string(rune('a'+i)), 1))
 	}
+	r1, r2 := stor.ResolveRoot("bafyR1"), stor.ResolveRoot("bafyR2")
 
 	ctx, cancel := context.WithCancel(context.Background())
 	req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
@@ -56,34 +56,63 @@ func TestMetaEventsSnapshotThenTail(t *testing.T) {
 			time.Sleep(20 * time.Millisecond)
 		}
 	}
-	waitFor(`"key":"file:R2/title"`)
-	if err := stor.SetProperty("R1", "title", "changed"); err != nil {
-		t.Fatal(err)
-	}
+	waitFor(`"key":"file:` + r2 + `/title"`)
+	storeClaims(t, stor, signed(t, a, "bafyR1", "title", "changed", 2))
 	waitFor(`event: set`)
 	time.Sleep(300 * time.Millisecond)
 	cancel()
 	<-done
 
 	body := rec.body()
-	for _, k := range []string{"file:R1/title", "file:R1/n", "file:R2/title", "file:R2/n"} {
+	keys := []string{"file:" + r1 + "/title", "file:" + r1 + "/n", "file:" + r2 + "/title", "file:" + r2 + "/n",
+		"file:" + r1 + "/cids/bafyR1", "file:" + r2 + "/cids/bafyR2"}
+	for _, k := range keys {
 		if !strings.Contains(body, `"key":"`+k+`"`) {
 			t.Errorf("snapshot is missing %s", k)
 		}
 	}
 	events := strings.Split(strings.TrimSpace(body), "\n\n")
-	if len(events) != 5 {
-		t.Fatalf("want 4 snapshot events + 1 live event, got %d:\n%s", len(events), body)
+	if len(events) != 7 {
+		t.Fatalf("want 6 snapshot events + 1 live event, got %d:\n%s", len(events), body)
 	}
-	for i, e := range events[:3] {
+	for i, e := range events[:5] {
 		if strings.Contains(e, "id: ") {
 			t.Errorf("snapshot event %d carries an id: %q", i, e)
 		}
 	}
-	if !strings.HasPrefix(events[3], "id: ") || !strings.HasPrefix(events[4], "id: ") {
+	if !strings.HasPrefix(events[5], "id: ") || !strings.HasPrefix(events[6], "id: ") {
 		t.Fatalf("the last snapshot event and the live event must carry ids:\n%s", body)
 	}
-	if !strings.Contains(events[4], `"key":"file:R1/title"`) {
-		t.Fatalf("live tail did not follow the snapshot: %q", events[4])
+	if !strings.Contains(events[6], `"key":"file:`+r1+`/title"`) {
+		t.Fatalf("live tail did not follow the snapshot: %q", events[6])
+	}
+}
+
+// A cursor older than the last cache rebuild gets `gap` (reason rebuild): the
+// rebuild published no per-key events, so the consumer must re-snapshot.
+func TestMetaEventsGapAfterRebuild(t *testing.T) {
+	stor := newTestStorage(t)
+	s := &Server{storage: stor}
+	storeClaims(t, stor, signed(t, testSigner(t, 1), "bafyA", "title", "T", 1))
+	if err := stor.Rebuild(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
+	req.Header.Set("Last-Event-ID", "1000-0")
+	rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()}
+	done := make(chan struct{})
+	go func() { s.handleEventsMeta(rec, req); close(done) }()
+	deadline := time.Now().Add(5 * time.Second)
+	for !strings.Contains(rec.body(), "event: gap") {
+		if time.Now().After(deadline) {
+			t.Fatalf("no gap after rebuild:\n%s", rec.body())
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	cancel()
+	<-done
+	if !strings.Contains(rec.body(), `"reason":"rebuild"`) {
+		t.Fatalf("gap reason: %s", rec.body())
 	}
 }

internal/api/stats.go

@@ -30,7 +30,7 @@ type fileStatsJSON struct {
 // cheap (an index range count, a cached row count, a file stat) plus a
 // directory listing, recomputed on every call. The `redis*` JSON names
 // predate the move to SQLite and are kept for dashboard compatibility:
-// redisKeys is the store's row count, redisMemory its on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
+// redisKeys is the number of claim rows (truth), redisMemory both databases' on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
 // tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
 // from a background-refreshed cache and are null until the first sweep lands.
 type StatsResponse struct {
@@ -115,7 +115,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
 	} else {
 		resp.Records = n
 	}
-	if n, err := s.storage.CountKeys(); err == nil {
+	if n, err := s.storage.CountClaims(); err == nil {
 		resp.RedisKeys = n
 	}
 	if mem, err := s.storage.GetMemoryInfo(); err == nil {

internal/api/storage_helper_test.go

@@ -1,19 +1,68 @@
 package api
 
 import (
+	"context"
+	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// newTestStorage opens a fresh metadata store in the test's temp dir.
+// newTestStorage opens (and starts) a fresh store in the test's temp dir.
 func newTestStorage(t *testing.T) *storage.Client {
 	t.Helper()
+	dir := t.TempDir()
 	c := storage.NewClient()
-	if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
+	if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil {
 		t.Fatalf("open store: %v", err)
 	}
+	if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
+		t.Fatalf("schema: %v", err)
+	}
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
 	t.Cleanup(func() { _ = c.Close() })
 	return c
 }
+
+func testSigner(t *testing.T, n byte) *identity.Identity {
+	t.Helper()
+	id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+// signed builds a claim signed by id; v is string, []byte or nil.
+func signed(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim {
+	t.Helper()
+	cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts}
+	switch x := v.(type) {
+	case string:
+		cl.Value = claim.Text(x)
+	case []byte:
+		cl.Value = claim.Bytes(x)
+	}
+	if err := cl.Sign(id); err != nil {
+		t.Fatal(err)
+	}
+	return cl
+}
+
+func storeClaims(t *testing.T, c *storage.Client, claims ...claim.Claim) {
+	t.Helper()
+	res, err := c.WriteClaims(context.Background(), claims)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, r := range res {
+		if r.Status != storage.StatusAccepted {
+			t.Fatalf("claim %d: %s %s", i, r.Status, r.Reason)
+		}
+	}
+}

internal/api/udl.go

@@ -1,13 +1,14 @@
 package api
 
-// User Data Layer (UDL) HTTP surface.
+// User Data Layer facade (/api/udl/*). Since meta-core v2.0.0 a user's data are
+// ordinary MMUR2 claims signed by the user's key (plaintext for public-social
+// keys, ciphertext for private ones); these routes keep their paths for the
+// clients (meta-watch, meta-read, meta-listen, the gateway mirror) and speak
+// claims in the §3.4 JSON form:
 //
-// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
-// profile) now that the authoritative store lives in meta-core instead of
-// meta-watch's local op-log. Records arrive already signed (and, for
-// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
-// version-gates and indexes them but never inspects the crypto. See
-// internal/storage/udl.go for the key model.
+//	{"cid","uid","key","value": "…" | {"b64": "…"} | null,"ts","sig"}
+//
+// A write is the same as POST /api/claims; newest ts wins per (uid, cid, key).
 
 import (
 	"encoding/json"
@@ -17,82 +18,46 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// UDLPutRequest is the body of PUT /api/udl/record.
-type UDLPutRequest struct {
-	Uid     string           `json:"uid"`
-	Cid     string           `json:"cid"`
-	Key     string           `json:"key"`
-	Version int64            `json:"version"`
-	Ts      int64            `json:"ts"`
-	Record  string           `json:"record"` // base64 CBOR, opaque + signed
-	Value   *json.RawMessage `json:"value,omitempty"`
-	// Tombstone marks this write as a deletion of (uid, cid, key). The record
-	// itself is opaque here — meta-core never decodes the CBOR, and a private-
-	// tier value and a tombstone are equally "no plaintext value" — so the
-	// writer has to say. Without it the cid is never removed from
-	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
-	// account and is returned in full on every My List / Continue Watching
-	// read. Omitted by older writers, which keeps the previous behaviour.
-	Tombstone bool `json:"tombstone,omitempty"`
-}
-
-// UDLPutResponse is the response of PUT /api/udl/record.
+// UDLPutResponse answers PUT /api/udl/record.
 type UDLPutResponse struct {
-	Accepted bool `json:"accepted"`
+	Accepted bool                `json:"accepted"`
+	Status   storage.ClaimStatus `json:"status"`
 }
 
-// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
-// request. Each entry is exactly a UDLPutRequest, so a client can build the
-// same signed record for either endpoint.
+// UDLPutBatchRequest is PUT /api/udl/records.
 type UDLPutBatchRequest struct {
-	Records []UDLPutRequest `json:"records"`
+	Records []claim.Claim `json:"records"`
 }
 
-// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
-// means the write was stale (the single-record endpoint's 409), which in a
-// batch is an ordinary outcome rather than a failure.
+// UDLPutBatchResult is one entry of the batch response.
 type UDLPutBatchResult struct {
-	Key      string `json:"key"`
-	Accepted bool   `json:"accepted"`
-	Error    string `json:"error,omitempty"`
+	Key      string              `json:"key"`
+	Accepted bool                `json:"accepted"`
+	Status   storage.ClaimStatus `json:"status"`
+	Error    string              `json:"error,omitempty"`
 }
 
-// UDLPutBatchResponse mirrors the request order index-for-index.
+// UDLPutBatchResponse answers PUT /api/udl/records.
 type UDLPutBatchResponse struct {
 	Results []UDLPutBatchResult `json:"results"`
 }
 
-// maxUDLBatch bounds one batch. Each record is a store write while the
-// request is held open; sized to match the sign batch on the other side of the
-// same operation (see maxSignBatch).
 const maxUDLBatch = 1000
 
-// UDLRecordResponse is the response of GET /api/udl/record.
+// UDLRecordResponse answers GET /api/udl/record. Claim is the stored claim,
+// tombstone included (its ts is what a newer write must exceed); Exists is
+// false when it is absent or a tombstone.
 type UDLRecordResponse struct {
-	Record  string `json:"record"`
-	Version int64  `json:"version"`
-	Ts      int64  `json:"ts"`
-	Exists  bool   `json:"exists"`
-	// Value is the plaintext JSON scalar stored beside the opaque record, and
-	// is present only for PUBLIC-tier keys (like, rating, profile:name,
-	// profile:avatar). json.RawMessage rather than string because the field is
-	// generic over keys — a bool for `like`, a number for `rating`, a string
-	// for a profile name (see aggregatePublicValues) — and because it is the
-	// exact type UDLPutRequest.Value uses, so a value written through PUT reads
-	// back byte-identical.
-	//
-	// Absent, never null, when there is no plaintext: private-tier cells,
-	// tombstones and never-written cells must not read as an empty public
-	// value. Readers key off presence, not `exists`.
-	Value *json.RawMessage `json:"value,omitempty"`
+	Exists bool         `json:"exists"`
+	Claim  *claim.Claim `json:"claim"`
 }
 
-// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
-// field names; `avg` is null for boolean keys like `like`), so the meta-watch
-// client deserializes it into the same struct it serializes to the UI.
+// UDLAggregate is one key's public aggregate on a cid.
 type UDLAggregate struct {
 	Key        string   `json:"key"`
 	Count      int      `json:"count"`
@@ -101,10 +66,17 @@ type UDLAggregate struct {
 	SampleSize int      `json:"sample_size"`
 }
 
-// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
-func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+func (s *Server) udlReady(w http.ResponseWriter) bool {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return false
+	}
+	return true
+}
+
+// handleUDLRecordGet handles GET /api/udl/record?uid=&cid=&key=.
+func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+	if !s.udlReady(w) {
 		return
 	}
 	q := r.URL.Query()
@@ -113,85 +85,53 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "uid, cid and key are required")
 		return
 	}
-
-	cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
+	cl, ok, err := s.storage.UDLGetClaim(uid, cid, key)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	resp := UDLRecordResponse{
-		Record:  cell.Record,
-		Version: cell.Version,
-		Ts:      cell.Ts,
-		Exists:  exists,
-	}
-	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
-	// the store, a future writer bug) would otherwise fail mid-encode and emit a
-	// truncated body instead of a readable error.
-	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
-		raw := json.RawMessage(cell.Value)
-		resp.Value = &raw
+	resp := UDLRecordResponse{}
+	if ok {
+		resp.Claim = &cl
+		resp.Exists = !cl.Value.IsNull()
 	}
 	writeJSON(w, http.StatusOK, resp)
 }
 
-// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
-// (version <= stored) returns 409 stale_version so the client can re-read and
-// retry with a higher version.
+// handleUDLRecordPut handles PUT /api/udl/record with one claim as the body.
 func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
-	var req UDLPutRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+	var cl claim.Claim
+	if err := json.NewDecoder(r.Body).Decode(&cl); err != nil {
+		writeError(w, http.StatusBadRequest, "body must be one claim: "+err.Error())
 		return
 	}
-	if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
-		writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
-		return
-	}
-
-	publicValue := ""
-	hasValue := false
-	if req.Value != nil {
-		publicValue = string(*req.Value)
-		hasValue = true
-	}
-
-	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
+	results, err := s.storage.WriteClaims(r.Context(), []claim.Claim{cl})
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	if !accepted {
-		writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
-		return
+	res := results[0]
+	switch res.Status {
+	case storage.StatusAccepted, storage.StatusUnchanged:
+		writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true, Status: res.Status})
+	case storage.StatusStale:
+		writeErrorSlug(w, http.StatusConflict, "stale_version", "a claim with an equal or newer ts already exists", true)
+	default:
+		writeErrorSlug(w, http.StatusBadRequest, string(res.Status), res.Reason, false)
 	}
-	writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
 }
 
-// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
-// handleUDLRecordPut.
-//
-// Exists for the bulk verbs above this layer ("mark this whole series seen"),
-// where one user action produces one cell per episode. Each cell still goes
-// through the same version-gated Lua upsert, so the CRDT semantics are
-// identical — this batches the transport only.
-//
-// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
-// replaces the single-record conflict status: in a batch, one stale cell is a
-// normal outcome (another device already marked that episode) and must not fail
-// the 299 cells beside it. Callers read the array, not the status code.
+// handleUDLRecordsPut handles PUT /api/udl/records {records: [claim, …]}.
 func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	var req UDLPutBatchRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
 		return
 	}
 	if len(req.Records) == 0 {
@@ -202,91 +142,66 @@ func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "too many records in one batch")
 		return
 	}
-	results := make([]UDLPutBatchResult, 0, len(req.Records))
-	for _, rec := range req.Records {
-		if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
-			results = append(results, UDLPutBatchResult{
-				Key:   rec.Key,
-				Error: "uid, cid, key and record are required",
-			})
-			continue
-		}
-		publicValue := ""
-		hasValue := false
-		if rec.Value != nil {
-			publicValue = string(*rec.Value)
-			hasValue = true
-		}
-		accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
-		if err != nil {
-			// A storage error is per-record here rather than fatal for the whole
-			// batch: the cells already written stay written, and reporting which
-			// ones failed lets the caller retry exactly those.
-			results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
-			continue
-		}
-		results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
+	results, err := s.storage.WriteClaims(r.Context(), req.Records)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
 	}
-	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
+	out := make([]UDLPutBatchResult, len(results))
+	for i, res := range results {
+		out[i] = UDLPutBatchResult{
+			Key:      req.Records[i].Key,
+			Status:   res.Status,
+			Accepted: res.Status == storage.StatusAccepted || res.Status == storage.StatusUnchanged,
+			Error:    res.Reason,
+		}
+	}
+	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: out})
 }
 
-// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
-// (active_for_user_key — My List, Continue Watching).
+func (s *Server) writeUDLEntries(w http.ResponseWriter, uid, cid, key string) {
+	entries, err := s.storage.UDLClaimsForUser(uid, cid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}: every live claim
+// of uid on key (My List, Continue Watching).
 func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	vars := mux.Vars(r)
-	uid, key := vars["uid"], vars["key"]
-	if uid == "" || key == "" {
+	if vars["uid"] == "" || vars["key"] == "" {
 		writeError(w, http.StatusBadRequest, "uid and key are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserKey(uid, key)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+	s.writeUDLEntries(w, vars["uid"], "", vars["key"])
 }
 
-// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
-// (list_for_user_cid).
+// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}.
 func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	vars := mux.Vars(r)
-	uid, cid := vars["uid"], vars["cid"]
-	if uid == "" || cid == "" {
+	if vars["uid"] == "" || vars["cid"] == "" {
 		writeError(w, http.StatusBadRequest, "uid and cid are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserCid(uid, cid)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+	s.writeUDLEntries(w, vars["uid"], vars["cid"], "")
 }
 
-// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
-// — every record a uid owns, for cross-instance profile recovery.
-//
-// The other user-scoped reads answer a question with one coordinate already
-// known. Recovery knows only the uid: a box that has just imported a secret key
-// has no idea which cids or keys the profile ever touched, so it needs the
-// whole set or it gets nothing.
-//
-// Paged on a cid boundary — `nextCid` empty means the export is complete. The
-// response is opaque signed records, exactly as stored; whoever consumes it is
-// responsible for verifying each one before trusting it (meta-core has never
-// held the account key and cannot verify on anyone's behalf).
+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= — the
+// whole profile, paged on cid boundaries (recovery / export).
 func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	uid := mux.Vars(r)["uid"]
@@ -301,31 +216,22 @@ func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
 			return
 		}
-		if n < limit {
-			limit = n
-		}
+		limit = min(n, limit)
 	}
 	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"entries": entries,
-		"nextCid": nextCid,
-	})
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries, "nextCid": nextCid})
 }
 
-// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
-// uid holds records for.
-//
-// Exists so a gateway mirror can enforce a per-profile quota without walking
-// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
-// disk-fill DoS that needs no account. A quota check that itself cost a scan
-// would just move the denial of service one layer up.
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count.
 func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	uid := mux.Vars(r)["uid"]
@@ -341,10 +247,10 @@ func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
 }
 
-// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
+// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users: signers holding live
+// claims on the cid.
 func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -360,12 +266,9 @@ func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
 }
 
-// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
-// Computes per-key population stats from the stored public value fields
-// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
+// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,userRating.
 func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -375,7 +278,7 @@ func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
 	}
 	keysParam := r.URL.Query().Get("keys")
 	if keysParam == "" {
-		keysParam = "like,rating"
+		keysParam = "like,userRating"
 	}
 	aggregates := make([]UDLAggregate, 0)
 	for _, key := range strings.Split(keysParam, ",") {

internal/api/udl_test.go

@@ -6,211 +6,107 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/metazla/meta-core/internal/claim"
 )
 
-// newUDLTestServer returns a Server wired to an in-memory Redis.
-//
-// The struct is built directly rather than through NewServer: NewServer boots
-// the file watcher, mounts manager, watchers poller and WebDAV handler, none of
-// which the UDL handlers touch — they use s.storage and nothing else.
 func newUDLTestServer(t *testing.T) *Server {
 	t.Helper()
 	return &Server{storage: newTestStorage(t)}
 }
 
-// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
-// fields, so a test can assert on the *encoding* of `value` (is it a quoted
-// string? is the key there at all?) rather than on what a typed struct would
-// have coerced it into.
-func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) {
+func udlGet(t *testing.T, s *Server, uid, cid, key string) UDLRecordResponse {
 	t.Helper()
-
 	req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
 	rr := httptest.NewRecorder()
 	s.handleUDLRecordGet(rr, req)
-
-	var body map[string]json.RawMessage
-	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-		t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("GET record = %d %s", rr.Code, rr.Body)
 	}
-	return rr.Code, body
+	var resp UDLRecordResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	return resp
 }
 
-// The plaintext value of a PUBLIC-tier cell must come back over the wire, and
-// come back *as it was written* — a JSON string stays a quoted string, a bool
-// stays a bool.
-//
-// Both halves matter. Without the value at all, the meta-core dashboard can
-// only ever show 44-character base58 uids: display names live in the User Data
-// Layer as the public cell (uid, "self", "profile:name"), and the plaintext
-// sits in Redis where the storage layer has always read it — the handler just
-// used to drop it. And the type has to stay json.RawMessage: this endpoint is
-// generic over keys, so a `*string` would mangle `like` (bool) and `rating`
-// (number) while looking correct for names.
-func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	const uid = "zAlice"
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok {
-		t.Fatalf("seed name: ok=%v err=%v", ok, err)
-	}
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok {
-		t.Fatalf("seed like: ok=%v err=%v", ok, err)
-	}
-
-	code, body := getRecord(t, s, uid, "self", "profile:name")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if got := string(body["value"]); got != `"Ada Lovelace"` {
-		t.Fatalf("value = %s, want a quoted JSON string", got)
-	}
-	var name string
-	if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" {
-		t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err)
-	}
-
-	// A boolean key survives unquoted — the assertion a *string field fails.
-	if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" {
-		t.Fatalf("like value = %s, want the bare JSON literal true", body["value"])
-	}
-}
-
-// A cell with no plaintext must omit `value` entirely rather than send null.
-//
-// That is the shape of every private-tier key (seek, watched, card, localpath —
-// their real value rides encrypted inside the opaque record) and of every
-// tombstone. A reader keys off presence: "there is no plaintext here" and "the
-// plaintext is null" are different facts, and a name cleared by its owner
-// leaves the cell existing with no value, which must not read as a name.
-func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	const uid = "zBob"
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok {
-		t.Fatalf("seed private cell: ok=%v err=%v", ok, err)
-	}
-
-	code, body := getRecord(t, s, uid, "bagcsaaa", "seek")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if _, ok := body["value"]; ok {
-		t.Fatalf("private cell must omit `value`, got %s", body["value"])
-	}
-	if string(body["exists"]) != "true" {
-		t.Fatalf("exists = %s, want true", body["exists"])
-	}
-}
-
-// A cell that was never written answers 200 with exists:false and no value —
-// not a 404, and not an empty-string value that would render as a blank name.
-func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	code, body := getRecord(t, s, "zNobody", "self", "profile:name")
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if string(body["exists"]) != "false" {
-		t.Fatalf("exists = %s, want false", body["exists"])
-	}
-	if _, ok := body["value"]; ok {
-		t.Fatalf("missing cell must omit `value`, got %s", body["value"])
-	}
-}
-
-// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the
-// per-record results.
-func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) {
+func udlPutBatch(t *testing.T, s *Server, claims ...claim.Claim) UDLPutBatchResponse {
 	t.Helper()
-
-	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body))
+	body, _ := json.Marshal(UDLPutBatchRequest{Records: claims})
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(string(body)))
 	rr := httptest.NewRecorder()
 	s.handleUDLRecordsPut(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("PUT records = %d %s", rr.Code, rr.Body)
+	}
+	var resp UDLPutBatchResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	return resp
+}
 
-	var out UDLPutBatchResponse
-	if rr.Code == http.StatusOK {
-		if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
-			t.Fatalf("decode response %q: %v", rr.Body.String(), err)
+// The facade stores and returns MMUR2 claims: public values in clear,
+// private ones as ciphertext, tombstones with their ts.
+func TestUDLFacadeRoundTrip(t *testing.T) {
+	s := newUDLTestServer(t)
+	me := testSigner(t, 7)
+	res := udlPutBatch(t, s,
+		signed(t, me, "self", "profile:name", "Ada Lovelace", 100),
+		signed(t, me, "bagcsaaa", "like", "true", 100),
+		signed(t, me, "bagcsaaa", "seek", []byte("ciphertext"), 100),
+	)
+	for _, r := range res.Results {
+		if !r.Accepted {
+			t.Fatalf("not accepted: %+v", r)
 		}
 	}
-	return rr.Code, out
+	got := udlGet(t, s, me.UID, "self", "profile:name")
+	if !got.Exists || got.Claim == nil || got.Claim.Value.Text != "Ada Lovelace" || got.Claim.Verify() != nil {
+		t.Fatalf("profile = %+v", got)
+	}
+	if seek := udlGet(t, s, me.UID, "bagcsaaa", "seek"); seek.Claim == nil || seek.Claim.Value.Kind != claim.KindBytes {
+		t.Fatalf("seek = %+v", seek)
+	}
+	if miss := udlGet(t, s, me.UID, "bagcsaaa", "nope"); miss.Exists || miss.Claim != nil {
+		t.Fatalf("missing cell = %+v", miss)
+	}
+
+	// Retraction: stored, returned with its ts, no longer "exists".
+	udlPutBatch(t, s, signed(t, me, "bagcsaaa", "like", nil, 200))
+	if tomb := udlGet(t, s, me.UID, "bagcsaaa", "like"); tomb.Exists || tomb.Claim == nil || tomb.Claim.TS != 200 {
+		t.Fatalf("tombstone = %+v", tomb)
+	}
+
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/api/udl/cid/self/aggregate?keys=profile:name", nil)
+	req = withVars(req, map[string]string{"cid": "self"})
+	s.handleUDLAggregate(rr, req)
+	if !strings.Contains(rr.Body.String(), `"sample_size":1`) {
+		t.Fatalf("aggregate = %s", rr.Body)
+	}
 }
 
-// **The load-bearing property of the batch endpoint: one stale cell does not
-// fail the batch.**
-//
-// The single-record PUT answers a stale version with 409, which is right when
-// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark
-// this whole series seen") routinely include episodes another device already
-// marked, and 409-ing there would abandon every cell after it. So staleness is
-// reported per record as accepted:false, with 200 on the request itself.
-func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) {
+// A stale claim in a batch is reported in its slot without failing the rest.
+func TestUDLRecordsPut_StaleSlotDoesNotFailTheBatch(t *testing.T) {
 	s := newUDLTestServer(t)
-
-	const uid = "zCarol"
-	const cid = "bagdsaaaseries"
-	// Seed s01e01 at version 4 — the batch below tries to write it at 2.
-	if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok {
-		t.Fatalf("seed: ok=%v err=%v", ok, err)
-	}
-
-	code, out := putRecords(t, s, `{"records":[
-		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"},
-		{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"}
-	]}`)
-
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code)
-	}
-	if len(out.Results) != 2 {
-		t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results))
-	}
-	if out.Results[0].Accepted || out.Results[0].Error != "" {
-		t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0])
-	}
-	if !out.Results[1].Accepted {
-		t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1])
-	}
-
-	// The fresh cell really landed, and the stale one did not overwrite.
-	if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" {
-		t.Fatalf("s01e02 was not written")
-	}
-	if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" {
-		t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"])
+	me := testSigner(t, 7)
+	udlPutBatch(t, s, signed(t, me, "bagcsaaa", "seen:s01e01", "true", 400))
+	res := udlPutBatch(t, s,
+		signed(t, me, "bagcsaaa", "seen:s01e01", "false", 300),
+		signed(t, me, "bagcsaaa", "seen:s01e02", "true", 300),
+	)
+	if res.Results[0].Accepted || res.Results[0].Status != "stale" || !res.Results[1].Accepted {
+		t.Fatalf("results = %+v", res.Results)
 	}
 }
 
-// A malformed record is reported in its own slot rather than rejecting the
-// request, for the same reason: the caller retries that cell, not the batch.
-func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) {
-	s := newUDLTestServer(t)
-
-	code, out := putRecords(t, s, `{"records":[
-		{"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"},
-		{"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"}
-	]}`)
-
-	if code != http.StatusOK {
-		t.Fatalf("status = %d, want 200", code)
-	}
-	if out.Results[0].Error == "" {
-		t.Fatalf("record with no key must report an error, got %+v", out.Results[0])
-	}
-	if !out.Results[1].Accepted {
-		t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1])
-	}
-}
-
-// An empty batch is a client bug, not a no-op: it almost always means the
-// caller built its record list wrong, and answering 200 with an empty array
-// would hide that.
 func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
 	s := newUDLTestServer(t)
-
-	if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest {
-		t.Fatalf("status = %d, want 400", code)
+	req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(`{"records":[]}`))
+	rr := httptest.NewRecorder()
+	s.handleUDLRecordsPut(rr, req)
+	if rr.Code != http.StatusBadRequest {
+		t.Fatalf("empty batch = %d", rr.Code)
 	}
 }