v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410

internal/api/udl.go

@@ -1,13 +1,14 @@
 package api
 
-// User Data Layer (UDL) HTTP surface.
+// User Data Layer facade (/api/udl/*). Since meta-core v2.0.0 a user's data are
+// ordinary MMUR2 claims signed by the user's key (plaintext for public-social
+// keys, ciphertext for private ones); these routes keep their paths for the
+// clients (meta-watch, meta-read, meta-listen, the gateway mirror) and speak
+// claims in the §3.4 JSON form:
 //
-// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
-// profile) now that the authoritative store lives in meta-core instead of
-// meta-watch's local op-log. Records arrive already signed (and, for
-// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
-// version-gates and indexes them but never inspects the crypto. See
-// internal/storage/udl.go for the key model.
+//	{"cid","uid","key","value": "…" | {"b64": "…"} | null,"ts","sig"}
+//
+// A write is the same as POST /api/claims; newest ts wins per (uid, cid, key).
 
 import (
 	"encoding/json"
@@ -17,82 +18,46 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+
+	"github.com/metazla/meta-core/internal/claim"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// UDLPutRequest is the body of PUT /api/udl/record.
-type UDLPutRequest struct {
-	Uid     string           `json:"uid"`
-	Cid     string           `json:"cid"`
-	Key     string           `json:"key"`
-	Version int64            `json:"version"`
-	Ts      int64            `json:"ts"`
-	Record  string           `json:"record"` // base64 CBOR, opaque + signed
-	Value   *json.RawMessage `json:"value,omitempty"`
-	// Tombstone marks this write as a deletion of (uid, cid, key). The record
-	// itself is opaque here — meta-core never decodes the CBOR, and a private-
-	// tier value and a tombstone are equally "no plaintext value" — so the
-	// writer has to say. Without it the cid is never removed from
-	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
-	// account and is returned in full on every My List / Continue Watching
-	// read. Omitted by older writers, which keeps the previous behaviour.
-	Tombstone bool `json:"tombstone,omitempty"`
-}
-
-// UDLPutResponse is the response of PUT /api/udl/record.
+// UDLPutResponse answers PUT /api/udl/record.
 type UDLPutResponse struct {
-	Accepted bool `json:"accepted"`
+	Accepted bool                `json:"accepted"`
+	Status   storage.ClaimStatus `json:"status"`
 }
 
-// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
-// request. Each entry is exactly a UDLPutRequest, so a client can build the
-// same signed record for either endpoint.
+// UDLPutBatchRequest is PUT /api/udl/records.
 type UDLPutBatchRequest struct {
-	Records []UDLPutRequest `json:"records"`
+	Records []claim.Claim `json:"records"`
 }
 
-// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
-// means the write was stale (the single-record endpoint's 409), which in a
-// batch is an ordinary outcome rather than a failure.
+// UDLPutBatchResult is one entry of the batch response.
 type UDLPutBatchResult struct {
-	Key      string `json:"key"`
-	Accepted bool   `json:"accepted"`
-	Error    string `json:"error,omitempty"`
+	Key      string              `json:"key"`
+	Accepted bool                `json:"accepted"`
+	Status   storage.ClaimStatus `json:"status"`
+	Error    string              `json:"error,omitempty"`
 }
 
-// UDLPutBatchResponse mirrors the request order index-for-index.
+// UDLPutBatchResponse answers PUT /api/udl/records.
 type UDLPutBatchResponse struct {
 	Results []UDLPutBatchResult `json:"results"`
 }
 
-// maxUDLBatch bounds one batch. Each record is a store write while the
-// request is held open; sized to match the sign batch on the other side of the
-// same operation (see maxSignBatch).
 const maxUDLBatch = 1000
 
-// UDLRecordResponse is the response of GET /api/udl/record.
+// UDLRecordResponse answers GET /api/udl/record. Claim is the stored claim,
+// tombstone included (its ts is what a newer write must exceed); Exists is
+// false when it is absent or a tombstone.
 type UDLRecordResponse struct {
-	Record  string `json:"record"`
-	Version int64  `json:"version"`
-	Ts      int64  `json:"ts"`
-	Exists  bool   `json:"exists"`
-	// Value is the plaintext JSON scalar stored beside the opaque record, and
-	// is present only for PUBLIC-tier keys (like, rating, profile:name,
-	// profile:avatar). json.RawMessage rather than string because the field is
-	// generic over keys — a bool for `like`, a number for `rating`, a string
-	// for a profile name (see aggregatePublicValues) — and because it is the
-	// exact type UDLPutRequest.Value uses, so a value written through PUT reads
-	// back byte-identical.
-	//
-	// Absent, never null, when there is no plaintext: private-tier cells,
-	// tombstones and never-written cells must not read as an empty public
-	// value. Readers key off presence, not `exists`.
-	Value *json.RawMessage `json:"value,omitempty"`
+	Exists bool         `json:"exists"`
+	Claim  *claim.Claim `json:"claim"`
 }
 
-// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
-// field names; `avg` is null for boolean keys like `like`), so the meta-watch
-// client deserializes it into the same struct it serializes to the UI.
+// UDLAggregate is one key's public aggregate on a cid.
 type UDLAggregate struct {
 	Key        string   `json:"key"`
 	Count      int      `json:"count"`
@@ -101,10 +66,17 @@ type UDLAggregate struct {
 	SampleSize int      `json:"sample_size"`
 }
 
-// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
-func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+func (s *Server) udlReady(w http.ResponseWriter) bool {
 	if !s.storage.IsConnected() {
 		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return false
+	}
+	return true
+}
+
+// handleUDLRecordGet handles GET /api/udl/record?uid=&cid=&key=.
+func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
+	if !s.udlReady(w) {
 		return
 	}
 	q := r.URL.Query()
@@ -113,85 +85,53 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "uid, cid and key are required")
 		return
 	}
-
-	cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
+	cl, ok, err := s.storage.UDLGetClaim(uid, cid, key)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	resp := UDLRecordResponse{
-		Record:  cell.Record,
-		Version: cell.Version,
-		Ts:      cell.Ts,
-		Exists:  exists,
-	}
-	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
-	// the store, a future writer bug) would otherwise fail mid-encode and emit a
-	// truncated body instead of a readable error.
-	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
-		raw := json.RawMessage(cell.Value)
-		resp.Value = &raw
+	resp := UDLRecordResponse{}
+	if ok {
+		resp.Claim = &cl
+		resp.Exists = !cl.Value.IsNull()
 	}
 	writeJSON(w, http.StatusOK, resp)
 }
 
-// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
-// (version <= stored) returns 409 stale_version so the client can re-read and
-// retry with a higher version.
+// handleUDLRecordPut handles PUT /api/udl/record with one claim as the body.
 func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
-	var req UDLPutRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+	var cl claim.Claim
+	if err := json.NewDecoder(r.Body).Decode(&cl); err != nil {
+		writeError(w, http.StatusBadRequest, "body must be one claim: "+err.Error())
 		return
 	}
-	if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
-		writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
-		return
-	}
-
-	publicValue := ""
-	hasValue := false
-	if req.Value != nil {
-		publicValue = string(*req.Value)
-		hasValue = true
-	}
-
-	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
+	results, err := s.storage.WriteClaims(r.Context(), []claim.Claim{cl})
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	if !accepted {
-		writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
-		return
+	res := results[0]
+	switch res.Status {
+	case storage.StatusAccepted, storage.StatusUnchanged:
+		writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true, Status: res.Status})
+	case storage.StatusStale:
+		writeErrorSlug(w, http.StatusConflict, "stale_version", "a claim with an equal or newer ts already exists", true)
+	default:
+		writeErrorSlug(w, http.StatusBadRequest, string(res.Status), res.Reason, false)
 	}
-	writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
 }
 
-// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
-// handleUDLRecordPut.
-//
-// Exists for the bulk verbs above this layer ("mark this whole series seen"),
-// where one user action produces one cell per episode. Each cell still goes
-// through the same version-gated Lua upsert, so the CRDT semantics are
-// identical — this batches the transport only.
-//
-// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
-// replaces the single-record conflict status: in a batch, one stale cell is a
-// normal outcome (another device already marked that episode) and must not fail
-// the 299 cells beside it. Callers read the array, not the status code.
+// handleUDLRecordsPut handles PUT /api/udl/records {records: [claim, …]}.
 func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	var req UDLPutBatchRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeError(w, http.StatusBadRequest, "invalid JSON body")
+		writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
 		return
 	}
 	if len(req.Records) == 0 {
@@ -202,91 +142,66 @@ func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "too many records in one batch")
 		return
 	}
-	results := make([]UDLPutBatchResult, 0, len(req.Records))
-	for _, rec := range req.Records {
-		if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
-			results = append(results, UDLPutBatchResult{
-				Key:   rec.Key,
-				Error: "uid, cid, key and record are required",
-			})
-			continue
-		}
-		publicValue := ""
-		hasValue := false
-		if rec.Value != nil {
-			publicValue = string(*rec.Value)
-			hasValue = true
-		}
-		accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
-		if err != nil {
-			// A storage error is per-record here rather than fatal for the whole
-			// batch: the cells already written stay written, and reporting which
-			// ones failed lets the caller retry exactly those.
-			results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
-			continue
-		}
-		results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
+	results, err := s.storage.WriteClaims(r.Context(), req.Records)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
 	}
-	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
+	out := make([]UDLPutBatchResult, len(results))
+	for i, res := range results {
+		out[i] = UDLPutBatchResult{
+			Key:      req.Records[i].Key,
+			Status:   res.Status,
+			Accepted: res.Status == storage.StatusAccepted || res.Status == storage.StatusUnchanged,
+			Error:    res.Reason,
+		}
+	}
+	writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: out})
 }
 
-// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
-// (active_for_user_key — My List, Continue Watching).
+func (s *Server) writeUDLEntries(w http.ResponseWriter, uid, cid, key string) {
+	entries, err := s.storage.UDLClaimsForUser(uid, cid, key)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+}
+
+// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}: every live claim
+// of uid on key (My List, Continue Watching).
 func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	vars := mux.Vars(r)
-	uid, key := vars["uid"], vars["key"]
-	if uid == "" || key == "" {
+	if vars["uid"] == "" || vars["key"] == "" {
 		writeError(w, http.StatusBadRequest, "uid and key are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserKey(uid, key)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+	s.writeUDLEntries(w, vars["uid"], "", vars["key"])
 }
 
-// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
-// (list_for_user_cid).
+// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}.
 func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	vars := mux.Vars(r)
-	uid, cid := vars["uid"], vars["cid"]
-	if uid == "" || cid == "" {
+	if vars["uid"] == "" || vars["cid"] == "" {
 		writeError(w, http.StatusBadRequest, "uid and cid are required")
 		return
 	}
-	entries, err := s.storage.UDLListUserCid(uid, cid)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
+	s.writeUDLEntries(w, vars["uid"], vars["cid"], "")
 }
 
-// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
-// — every record a uid owns, for cross-instance profile recovery.
-//
-// The other user-scoped reads answer a question with one coordinate already
-// known. Recovery knows only the uid: a box that has just imported a secret key
-// has no idea which cids or keys the profile ever touched, so it needs the
-// whole set or it gets nothing.
-//
-// Paged on a cid boundary — `nextCid` empty means the export is complete. The
-// response is opaque signed records, exactly as stored; whoever consumes it is
-// responsible for verifying each one before trusting it (meta-core has never
-// held the account key and cannot verify on anyone's behalf).
+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= — the
+// whole profile, paged on cid boundaries (recovery / export).
 func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	uid := mux.Vars(r)["uid"]
@@ -301,31 +216,22 @@ func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
 			return
 		}
-		if n < limit {
-			limit = n
-		}
+		limit = min(n, limit)
 	}
 	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"entries": entries,
-		"nextCid": nextCid,
-	})
+	if entries == nil {
+		entries = []claim.Claim{}
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries, "nextCid": nextCid})
 }
 
-// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
-// uid holds records for.
-//
-// Exists so a gateway mirror can enforce a per-profile quota without walking
-// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
-// disk-fill DoS that needs no account. A quota check that itself cost a scan
-// would just move the denial of service one layer up.
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count.
 func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	uid := mux.Vars(r)["uid"]
@@ -341,10 +247,10 @@ func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
 }
 
-// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
+// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users: signers holding live
+// claims on the cid.
 func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -360,12 +266,9 @@ func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
 }
 
-// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
-// Computes per-key population stats from the stored public value fields
-// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
+// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,userRating.
 func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+	if !s.udlReady(w) {
 		return
 	}
 	cid := mux.Vars(r)["cid"]
@@ -375,7 +278,7 @@ func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
 	}
 	keysParam := r.URL.Query().Get("keys")
 	if keysParam == "" {
-		keysParam = "like,rating"
+		keysParam = "like,userRating"
 	}
 	aggregates := make([]UDLAggregate, 0)
 	for _, key := range strings.Split(keysParam, ",") {