improvement on user management

internal/api/handlers_identity.go

@@ -50,6 +50,26 @@ type identityImportResponse struct {
 	UID       string `json:"uid"`
 	Curve     string `json:"curve"`
 	CreatedAt int64  `json:"createdAt"`
+	// Created distinguishes "this key was new here" (201) from "this key was
+	// already in the keystore" (200). Import is the sign-in primitive for
+	// clients: presenting the secret key is the proof of ownership, so a
+	// re-import must succeed rather than conflict. Clients use the flag only
+	// to word the confirmation ("new profile" vs "welcome back").
+	Created bool `json:"created"`
+}
+
+// identityRevealRequest asks for an existing account's private key back. The
+// key was shown once at generate; without a way to see it again, a client with
+// a real sign-out has no recovery path for a user who never saved it.
+type identityRevealRequest struct {
+	Confirm bool   `json:"confirm"`
+	UID     string `json:"uid"`
+}
+
+type identityRevealResponse struct {
+	UID           string `json:"uid"`
+	Curve         string `json:"curve"`
+	PrivateKeyHex string `json:"privateKeyHex"`
 }
 
 type identityDeleteRequest struct {
@@ -172,9 +192,17 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	dir := s.config.IdentityAccountsDir()
-	if identity.ExistsByUID(dir, id.UID) {
-		writeErrorSlug(w, http.StatusConflict, "identity_exists",
-			"an account with this key already exists", false)
+	// Idempotent: importing a key that is already in the keystore is how a
+	// second device signs in to an existing account. Answer 200 with the
+	// stored account (preserving its original createdAt) instead of 409 —
+	// the caller proved ownership by presenting the key.
+	if existing, err := identity.LoadByUID(dir, id.UID); err == nil && existing != nil {
+		writeJSON(w, http.StatusOK, identityImportResponse{
+			UID:       existing.UID,
+			Curve:     existing.Curve,
+			CreatedAt: existing.CreatedAt,
+			Created:   false,
+		})
 		return
 	}
 	if err := identity.SaveAccount(dir, id); err != nil {
@@ -185,6 +213,35 @@ func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
 		UID:       id.UID,
 		Curve:     id.Curve,
 		CreatedAt: id.CreatedAt,
+		Created:   true,
+	})
+}
+
+// handleIdentityReveal returns an existing account's private key. Deliberately
+// POST + explicit confirm (never a bare GET) so it cannot be triggered by a
+// link or an image tag, and so it reads as the deliberate act it is. The
+// endpoint is auth-gated at the perimeter like the rest of /api/identity/*.
+func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
+	var body identityRevealRequest
+	_ = json.NewDecoder(r.Body).Decode(&body)
+	if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
+		writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
+			"revealing a private key requires {\"confirm\": true} or ?confirm=true", false)
+		return
+	}
+	uid := strings.TrimSpace(body.UID)
+	if uid == "" {
+		uid = strings.TrimSpace(r.URL.Query().Get("uid"))
+	}
+	id, status, slug, msg := s.resolveAccount(uid)
+	if status != 0 {
+		writeErrorSlug(w, status, slug, msg, false)
+		return
+	}
+	writeJSON(w, http.StatusOK, identityRevealResponse{
+		UID:           id.UID,
+		Curve:         id.Curve,
+		PrivateKeyHex: id.PrivateKeyHex,
 	})
 }
 

internal/api/server.go

@@ -172,6 +172,7 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
 	s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
 	s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
+	s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
 	s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
 	s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
 	s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")

internal/api/udl.go

@@ -27,6 +27,14 @@ type UDLPutRequest struct {
 	Ts      int64            `json:"ts"`
 	Record  string           `json:"record"` // base64 CBOR, opaque + signed
 	Value   *json.RawMessage `json:"value,omitempty"`
+	// Tombstone marks this write as a deletion of (uid, cid, key). The record
+	// itself is opaque here — meta-core never decodes the CBOR, and a private-
+	// tier value and a tombstone are equally "no plaintext value" — so the
+	// writer has to say. Without it the cid is never removed from
+	// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
+	// account and is returned in full on every My List / Continue Watching
+	// read. Omitted by older writers, which keeps the previous behaviour.
+	Tombstone bool `json:"tombstone,omitempty"`
 }
 
 // UDLPutResponse is the response of PUT /api/udl/record.
@@ -104,7 +112,7 @@ func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
 		hasValue = true
 	}
 
-	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue)
+	accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return

internal/cid/cid-rank-vectors.json

@@ -19,9 +19,10 @@
   "ladder": {
     "40": "dag-pb (IPFS block)",
     "30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
+    "25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
     "20": "BitTorrent infohashes - swarm interop",
     "10": "midhash256 - the record address",
-    "5": "opaque locators (nzb-release, url) - never outrank a real digest",
+    "5": "opaque locators (nzb-release, url, card) - never outrank a real digest",
     "0": "weak/unknown digests, unparseable input"
   },
   "tieBreak": "lexicographically smaller cid wins",
@@ -56,6 +57,12 @@
       "rank": 30,
       "note": "REAL meta-hash fixture"
     },
+    {
+      "name": "nzb-posting",
+      "cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 25,
+      "note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
+    },
     {
       "name": "btih-v2-codec-eq-mh",
       "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
@@ -103,6 +110,12 @@
       "rank": 5,
       "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
     },
+    {
+      "name": "card-locator",
+      "cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
+      "rank": 5,
+      "note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
+    },
     {
       "name": "sha1",
       "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",

internal/cid/rank.go

@@ -49,12 +49,33 @@ const (
 	// these — matching on mh would see 0x00 and rank them 0.
 	CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
 	CodeURL        = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
+	// CodeCard is the limit case of the family: the only CID with NO bytes
+	// behind it anywhere, ever. It addresses a *work* (a series, a film) via
+	// varint(len(source))‖source‖id — ("tmdb","tv:95479") — so `/api/file/{cid}`
+	// correctly 404s and bitswap never offers it. Ranking it here (rather than
+	// letting it fall to RankUnknown) is what makes the planned "one card, two
+	// CIDs" cross-source merge safe: a TMDB and a MyAnimeList locator can sit on
+	// one record, and a real digest added later still wins the election.
+	CodeCard = 0x1007 // custom — identity-multihash CID wrapping a card (work) id
+
+	// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
+	// REAL sha2-256 digest (over the normalised, sorted article Message-ID
+	// set of a Usenet posting we scanned ourselves — never a locator, no
+	// indexer host embedded), so it would silently fall into RankDigest via
+	// the mh switch unless matched on the codec FIRST. It is content-derived
+	// (so it outranks the opaque locators) but addresses no fetchable block —
+	// the digest is over an id set, not over any block's bytes, so it can
+	// never be bitswap-fetched under itself — so it must stay BELOW a real
+	// digest (once the media is materialised and re-seeded, that genuine
+	// sha2-256 cid has to win the canonical election, not this one).
+	CodeNzbPosting = 0x1003 // custom — self-scanned Usenet posting identity
 )
 
 // Rank tiers, named so the ladder reads as a ladder.
 const (
 	RankIPFS       = 40 // dag-pb — broadest network reach
 	RankDigest     = 30 // sha2-256 / sha3-* — real content digests
+	RankNzbPosting = 25 // content-derived, but the digest addresses no fetchable block
 	RankBtih       = 20 // BitTorrent infohashes — swarm interop
 	RankMidhash    = 10 // the record address — internally meaningful, externally invisible
 	RankLocator    = 5  // opaque locators — never outrank a real digest
@@ -112,11 +133,16 @@ func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
 //   - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
 //   - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
 //     block of the file IS its sha2-256, so they share this tier).
+//   - nzb-posting is content-derived (a digest over a Usenet posting's article
+//     Message-ID set) so it outranks the opaque locators below, but the
+//     digest addresses no fetchable block, so it stays below a real content
+//     digest — once the media is materialised and re-seeded under a genuine
+//     sha2-256 cid, that cid must win the election, not this one.
 //   - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
 //     swarm interop, not strength.
 //   - midhash256 is the floor among real *digests*: internally meaningful,
 //     externally invisible.
-//   - opaque locators (nzb-release, url) sit BELOW midhash. They are not
+//   - opaque locators (nzb-release, url, card) sit BELOW midhash. They are not
 //     content-derived — two URLs to the same bytes yield two different locator
 //     CIDs — so a locator must never displace a real digest as the identity a
 //     record is advertised under. Tier 5 means a locator wins the election only
@@ -139,9 +165,15 @@ func Rank(cidStr string) int {
 	}
 	// Opaque locators. Codec-only: their multihash is identity (0x00), so
 	// there is nothing to match on the mh side.
-	if codec == CodeNzbRelease || codec == CodeURL {
+	if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard {
 		return RankLocator
 	}
+	// nzb-posting: codec-only, checked BEFORE the mh switch below — its
+	// multihash is a real sha2-256 and would otherwise match CodeSha256 there
+	// and rank 30, tying with genuine content digests.
+	if codec == CodeNzbPosting {
+		return RankNzbPosting
+	}
 	switch mh {
 	case CodeSha256, CodeSha3_256, CodeSha3_384:
 		return RankDigest

internal/cid/rank_test.go

@@ -150,12 +150,31 @@ func TestRank_Tiers(t *testing.T) {
 func TestRank_StrictOrdering(t *testing.T) {
 	ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
 	sha := Rank(cidSha256)
+	posting := Rank(buildCID(CodeNzbPosting, CodeSha256, 32))
 	btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
 	mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
 	unknown := Rank(cidMd5)
-	if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
-		t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
-			ipfs, sha, btih, mid, unknown)
+	if !(ipfs > sha && sha > posting && posting > btih && btih > mid && mid > unknown) {
+		t.Fatalf("rank ordering broken: ipfs=%d sha=%d posting=%d btih=%d mid=%d unknown=%d",
+			ipfs, sha, posting, btih, mid, unknown)
+	}
+}
+
+// TestRank_NzbPostingMatchesCodecNotMultihash pins the inverse trap:
+// nzb-posting's multihash is a REAL sha2-256 (unlike nzb-release/url, whose mh
+// is identity 0x00), so Rank must match on the codec BEFORE falling into the
+// mh switch — otherwise this ties with a genuine content digest at 30 instead
+// of sitting at 25.
+func TestRank_NzbPostingMatchesCodecNotMultihash(t *testing.T) {
+	posting := buildCID(CodeNzbPosting, CodeSha256, 32)
+	if got := Rank(posting); got != RankNzbPosting {
+		t.Errorf("Rank(nzb-posting) = %d, want %d (RankNzbPosting) — codec check must precede the mh switch", got, RankNzbPosting)
+	}
+	// A real sha2-256 with the SAME digest bytes must still rank 30 — the
+	// codec, not the digest, is what distinguishes the two.
+	realDigest := buildCID(CodecRaw, CodeSha256, 32)
+	if got := Rank(realDigest); got != RankDigest {
+		t.Errorf("Rank(sha2-256) regressed to %d, want %d", got, RankDigest)
 	}
 }
 

internal/storage/udl.go

@@ -72,7 +72,15 @@ func udlIdxCidUsers(cid string) string      { return fmt.Sprintf("udl:idx:cid:%s
 //
 //	KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
 //	ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
-//	      6=cid 7=key 8=uid
+//	      6=cid 7=key 8=uid 9=tombstone("1"/"0")
+//
+// A tombstone still writes the cell — that is the CRDT state, and dropping it
+// would let an older write resurrect the value — but it SREMs the cid from the
+// user+key index. That index is what `active_for_user_key` returns, so leaving
+// deleted cids in it means every My List / Continue Watching read carries every
+// title ever un-liked or finished, forever. The other three indexes are left
+// alone: they answer "who has an opinion about this cid", where a tombstone is
+// still an answer.
 var udlUpsertScript = redis.NewScript(`
 local stored = redis.call('HGET', KEYS[1], 'version')
 if stored and tonumber(stored) >= tonumber(ARGV[1]) then
@@ -84,7 +92,11 @@ if ARGV[5] == '1' then
 else
   redis.call('HDEL', KEYS[1], 'value')
 end
+if ARGV[9] == '1' then
+  redis.call('SREM', KEYS[2], ARGV[6])
+else
   redis.call('SADD', KEYS[2], ARGV[6])
+end
 redis.call('SADD', KEYS[3], ARGV[7])
 redis.call('SADD', KEYS[4], ARGV[8])
 redis.call('SADD', KEYS[5], ARGV[8])
@@ -94,8 +106,10 @@ return 1
 // UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
 // one. publicValue/hasValue carry the plaintext JSON scalar for public keys
 // (used by aggregation); pass hasValue=false for private-tier keys and
-// tombstones. Returns accepted=false when the write is stale.
-func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue bool) (bool, error) {
+// tombstones. Set tombstone=true when the record deletes the cell, which also
+// prunes the cid from the user+key index. Returns accepted=false when the write
+// is stale.
+func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -117,7 +131,11 @@ func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, recor
 	if hasValue {
 		hv = "1"
 	}
-	res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid).Result()
+	tomb := "0"
+	if tombstone {
+		tomb = "1"
+	}
+	res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
 	if err != nil {
 		return false, fmt.Errorf("udl upsert failed: %w", err)
 	}

internal/storage/udl_test.go

@@ -0,0 +1,79 @@
+package storage
+
+import (
+	"testing"
+)
+
+// A tombstone must leave the cell in place but drop the cid from the user+key
+// index.
+//
+// The cell has to stay: it *is* the CRDT state, and deleting it would let an
+// older write resurrect the value the tombstone retracted. The index entry has
+// to go: `active_for_user_key` returns every member of that set, so a set that
+// only ever grows means each My List / Continue Watching read carries every
+// title the user has ever un-liked or finished — with its full record blob —
+// forever. The reader filters them out again, so the cost is invisible until
+// the account is old.
+func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
+	c, _ := newTestClient(t, "")
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaalocator"
+		key = "seek"
+	)
+
+	// A live position: indexed, readable.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-v1", "", false, false); err != nil || !ok {
+		t.Fatalf("live upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err := c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("live position should be indexed, got %+v", entries)
+	}
+
+	// Finishing it tombstones the seek.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-v2-tombstone", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
+	}
+
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after tombstone: %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("tombstoned cid must leave the user+key index, got %+v", entries)
+	}
+
+	// ...but the cell itself survives, at the tombstone's version.
+	cell, exists, err := c.UDLGetCell(uid, cid, key)
+	if err != nil {
+		t.Fatalf("get cell: %v", err)
+	}
+	if !exists {
+		t.Fatal("tombstone must not delete the cell — an older write could then resurrect the value")
+	}
+	if cell.Version != 2 || cell.Record != "rec-v2-tombstone" {
+		t.Fatalf("cell should hold the tombstone record at v2, got %+v", cell)
+	}
+
+	// And the high-water mark still rejects the superseded write.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 300, "rec-v1-replay", "", false, false); err != nil || ok {
+		t.Fatalf("stale replay must be rejected: ok=%v err=%v", ok, err)
+	}
+
+	// Re-watching re-establishes the cell and puts it back in the index.
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 3, 400, "rec-v3", "", false, false); err != nil || !ok {
+		t.Fatalf("re-establish upsert: ok=%v err=%v", ok, err)
+	}
+	entries, err = c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key after re-establish: %v", err)
+	}
+	if len(entries) != 1 || entries[0].Cid != cid {
+		t.Fatalf("re-established cid must return to the index, got %+v", entries)
+	}
+}