chnage service discovery and identity

2026-09-13 19:48:52 +02:00
parent 0cccceccd9
commit 355c666183
15 changed files with 1571 additions and 45 deletions
@@ -99,3 +99,33 @@ func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Requ
 		"byKey":   res.ByKey,
 	})
 }
+
+// handleMigrateUserCids triggers the one-shot sweep that builds
+// `udl:idx:user:<uid>:cids` for User Data Layer records written before that
+// index existed. See storage.UDLBackfillUserCids.
+//
+// The index is what makes whole-profile export a set read instead of a
+// keyspace scan, and it is maintained by the upsert script from now on — so
+// this only ever matters once, for data already on disk. Until it has run, a
+// profile that predates the index exports as **empty**, which means a
+// cross-instance recovery reports success having restored nothing.
+//
+// Returns {fixed: N} — cids added, not cids examined. Safe to call repeatedly:
+// a second pass adds nothing.
+func (s *Server) handleMigrateUserCids(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	fixed, err := s.storage.UDLBackfillUserCids()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"success": true,
+		"fixed":   fixed,
+	})
+}
@@ -14,6 +14,7 @@ import (

 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 )

 // contentTypeByExt maps file extensions to MIME types
@@ -79,13 +80,13 @@ type ErrorResponse struct {
 // Stable error-slug strings. Pinned in the doc; consumers may match on
 // these. Do not rename without a deprecation cycle.
 const (
-	ErrAliasCollision    = "alias_collision"
-	ErrUnknownRoot       = "unknown_root"
-	ErrUnknownCID        = "unknown_cid"
-	ErrSchemaViolation   = "schema_violation"
+	ErrAliasCollision     = "alias_collision"
+	ErrUnknownRoot        = "unknown_root"
+	ErrUnknownCID         = "unknown_cid"
+	ErrSchemaViolation    = "schema_violation"
 	ErrStorageUnavailable = "storage_unavailable"
-	ErrInternal          = "internal"
-	ErrDeprecatedField   = "deprecated_field"
+	ErrInternal           = "internal"
+	ErrDeprecatedField    = "deprecated_field"
 )

 // rejectDeprecatedField reports whether a field name is one of the removed
@@ -594,6 +595,44 @@ func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, service)
 }

+// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
+//
+// This is the meta-discovery v1 replacement for /api/services: the list is
+// built from UDP announces held in memory, not from files under
+// /meta-core/services, so it works across stacks that share a network but no
+// volume. Instances of the same service are collapsed to the most recent, which
+// preserves the one-row-per-service contract the old endpoint had.
+//
+// A `?all=1` query returns every instance uncollapsed, for debugging.
+func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
+	response := map[string]interface{}{
+		"current":   s.config.ServiceName,
+		"neighbors": []interface{}{},
+		"count":     0,
+	}
+
+	if s.mesh == nil {
+		// Discovery off or failed to bind. Report empty rather than 503 —
+		// the nav is decoration and callers already treat [] as "hide".
+		response["enabled"] = false
+		writeJSON(w, http.StatusOK, response)
+		return
+	}
+
+	var list []meshdisco.Neighbor
+	if r.URL.Query().Get("all") != "" {
+		list = s.mesh.Neighbors()
+	} else {
+		list = s.mesh.NeighborsByName()
+	}
+
+	response["enabled"] = true
+	response["neighbors"] = list
+	response["count"] = len(list)
+	response["self"] = s.mesh.Self()
+	writeJSON(w, http.StatusOK, response)
+}
+
 // handleCleanupStats handles GET /services/cleanup/stats
 func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
 	if s.cleaner == nil {
@@ -959,10 +998,10 @@ func (s *Server) handleComputeFileCID(w http.ResponseWriter, r *http.Request) {
 	// CIDv1 format: version (0x01) + codec (0x55) + multihash
 	// Multihash format: hash-code (0x12) + length (0x20) + hash
 	cidBytes := make([]byte, 0, 2+2+32)
-	cidBytes = append(cidBytes, 0x01)       // CIDv1
-	cidBytes = append(cidBytes, 0x55)       // raw codec
-	cidBytes = append(cidBytes, 0x12)       // sha256 code
-	cidBytes = append(cidBytes, 0x20)       // 32 bytes
+	cidBytes = append(cidBytes, 0x01) // CIDv1
+	cidBytes = append(cidBytes, 0x55) // raw codec
+	cidBytes = append(cidBytes, 0x12) // sha256 code
+	cidBytes = append(cidBytes, 0x20) // 32 bytes
 	cidBytes = append(cidBytes, hashBytes...)

 	// Encode as base32lower with 'b' prefix (multibase)
@@ -13,6 +13,7 @@ import (
 	"github.com/metazla/meta-core/internal/events"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/mounts"
 	"github.com/metazla/meta-core/internal/schema"
 	"github.com/metazla/meta-core/internal/storage"
@@ -45,8 +46,19 @@ type Server struct {
 	// challenges backs proof-of-possession on reveal/delete. In-process, so a
 	// restart invalidates the ones in flight — see identity/challenge.go.
 	challenges *identity.ChallengeStore
-	router     *mux.Router
-	server     *http.Server
+	// mesh is the UDP discovery node (meta-discovery v1). nil when discovery
+	// is disabled or failed to bind, in which case /api/neighbors reports an
+	// empty list rather than erroring — a dashboard nav is decoration and
+	// must not take the API down with it.
+	mesh   *meshdisco.Node
+	router *mux.Router
+	server *http.Server
+}
+
+// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
+// server's constructor signature (and its tests) stay untouched.
+func (s *Server) SetMeshNode(n *meshdisco.Node) {
+	s.mesh = n
 }

 // NewServer creates a new API server
@@ -216,6 +228,8 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
 	s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
 	s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
 	s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
@@ -264,6 +278,7 @@ func (s *Server) setupRoutes() {
 	// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
 	// (METADATA_KEYS.md §14.17). Idempotent.
 	s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
+	s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST")
 	// One-shot sweep of query echoes on literature cards (meta-read
 	// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
 	s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
@@ -282,6 +297,11 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
 	s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")

+	// meta-discovery v1: neighbours heard over UDP. Supersedes /api/services,
+	// which is still served above while consumers migrate.
+	s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
+	s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
+
 	// Mount management routes (if manager initialized)
 	if s.mountsHandlers != nil {
 		s.mountsHandlers.RegisterRoutes(s.router)
@@ -13,6 +13,7 @@ import (
 	"encoding/json"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"

 	"github.com/gorilla/mux"
@@ -271,6 +272,75 @@ func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
 }

+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
+// — every record a uid owns, for cross-instance profile recovery.
+//
+// The other user-scoped reads answer a question with one coordinate already
+// known. Recovery knows only the uid: a box that has just imported a secret key
+// has no idea which cids or keys the profile ever touched, so it needs the
+// whole set or it gets nothing.
+//
+// Paged on a cid boundary — `nextCid` empty means the export is complete. The
+// response is opaque signed records, exactly as stored; whoever consumes it is
+// responsible for verifying each one before trusting it (meta-core has never
+// held the account key and cannot verify on anyone's behalf).
+func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	limit := maxUDLBatch
+	if v := r.URL.Query().Get("limit"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n <= 0 {
+			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
+			return
+		}
+		if n < limit {
+			limit = n
+		}
+	}
+	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"entries": entries,
+		"nextCid": nextCid,
+	})
+}
+
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
+// uid holds records for.
+//
+// Exists so a gateway mirror can enforce a per-profile quota without walking
+// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
+// disk-fill DoS that needs no account. A quota check that itself cost a scan
+// would just move the denial of service one layer up.
+func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	n, err := s.storage.UDLCountUserCids(uid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
+}
+
 // handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
 func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {