chnage service discovery and identity

This commit is contained in:
Worph
2026-09-13 19:48:52 +02:00
parent 0cccceccd9
commit 355c666183
15 changed files with 1571 additions and 45 deletions
+180 -1
View File
@@ -17,6 +17,7 @@ package storage
// udl:rec:<uid>/<cid>/<key> HASH { record, version, ts, value? }
// udl:idx:user:<uid>:key:<key> SET of <cid> -> active_for_user_key (My List, Continue Watching)
// udl:idx:user:<uid>:cid:<cid> SET of <key> -> list_for_user_cid
// udl:idx:user:<uid>:cids SET of <cid> -> whole-profile export (recovery)
// udl:idx:cid:<cid>:key:<key> SET of <uid> -> aggregate
// udl:idx:cid:<cid>:users SET of <uid> -> uids_for_cid
//
@@ -27,7 +28,9 @@ package storage
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
@@ -66,11 +69,24 @@ func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s
func udlIdxCidKey(cid, key string) string { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
// index that makes "export this whole profile" a set read instead of a
// keyspace scan.
//
// The other four indexes each answer a question with one coordinate already
// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and
// nothing else. Without this set the only ways to enumerate a profile are to
// iterate the hardcoded key list — which silently misses any key added later —
// or to SCAN `udl:rec:<uid>/*`, which is the full-keyspace-read pathology that
// already made `/api/udl/users/stats` admin-only.
func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
// udlUpsertScript performs the version-gated upsert + index maintenance
// atomically. Accepts (returns 1) only when the incoming version strictly
// exceeds the stored one; otherwise returns 0 (stale).
//
// KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
// 6=idx:user:cids
// ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
// 6=cid 7=key 8=uid 9=tombstone("1"/"0")
//
@@ -78,9 +94,16 @@ func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s
// would let an older write resurrect the value — but it SREMs the cid from the
// user+key index. That index is what `active_for_user_key` returns, so leaving
// deleted cids in it means every My List / Continue Watching read carries every
// title ever un-liked or finished, forever. The other three indexes are left
// title ever un-liked or finished, forever. The other four indexes are left
// alone: they answer "who has an opinion about this cid", where a tombstone is
// still an answer.
//
// ⚠ `idx:user:cids` (KEYS[6]) in particular must NEVER be pruned on a
// tombstone, even though it looks symmetric with KEYS[2]. It is the export
// index: a cid dropped from it is a cid whose tombstone never travels, and a
// tombstone that does not travel is exactly how an un-liked title comes back
// from the dead on the other box — the peer still holds the older `like` at a
// lower version, and with no tombstone to beat it, nothing ever supersedes it.
var udlUpsertScript = redis.NewScript(`
local stored = redis.call('HGET', KEYS[1], 'version')
if stored and tonumber(stored) >= tonumber(ARGV[1]) then
@@ -100,6 +123,7 @@ end
redis.call('SADD', KEYS[3], ARGV[7])
redis.call('SADD', KEYS[4], ARGV[8])
redis.call('SADD', KEYS[5], ARGV[8])
redis.call('SADD', KEYS[6], ARGV[6])
return 1
`)
@@ -126,6 +150,7 @@ func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, recor
c.buildKey(udlIdxUserCid(uid, cid)),
c.buildKey(udlIdxCidKey(cid, key)),
c.buildKey(udlIdxCidUsers(cid)),
c.buildKey(udlIdxUserCids(uid)),
}
hv := "0"
if hasValue {
@@ -250,6 +275,160 @@ func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
return out, nil
}
// UDLUserAllEntry is one row of a whole-profile export.
type UDLUserAllEntry struct {
Cid string `json:"cid"`
Key string `json:"key"`
Record string `json:"record"`
Version int64 `json:"version"`
Ts int64 `json:"ts"`
}
// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
//
// Ordered by cid (sorted, so paging is stable across calls even though the
// underlying index is a Redis SET), starting strictly after `afterCid`. A cid's
// keys are never split across a page: the page ends on a cid boundary once
// `limit` rows have been emitted, and `nextCid` is the last cid included — pass
// it back as `afterCid` to continue. `nextCid` is empty when the export is
// complete.
//
// Tombstones are included. They are the CRDT state that stops a deleted value
// resurrecting on the box that receives this export, so omitting them would
// make recovery actively wrong rather than merely incomplete.
func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, "", fmt.Errorf("not connected")
}
if limit <= 0 {
limit = 1000
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
}
sort.Strings(cids)
out := make([]UDLUserAllEntry, 0, limit)
nextCid := ""
for _, cid := range cids {
if afterCid != "" && cid <= afterCid {
continue
}
if len(out) >= limit {
nextCid = out[len(out)-1].Cid
break
}
keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
}
sort.Strings(keys)
for _, key := range keys {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
if err != nil {
return nil, "", err
}
if !ok {
continue
}
out = append(out, UDLUserAllEntry{
Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts,
})
}
}
return out, nextCid, nil
}
// UDLCountUserCids is the number of distinct cids a uid holds records for.
//
// O(1) (a Redis SCARD), which is what makes it usable as a per-push quota
// check on the gateway mirror — a quota that cost a keyspace walk would be a
// denial-of-service in its own right. Counts cids rather than cells because
// that is what the index holds; a profile is bounded by the titles it has
// touched, not by how many keys it has per title.
func (c *Client) UDLCountUserCids(uid string) (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.SCard(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
if err != nil {
return 0, fmt.Errorf("scard failed: %w", err)
}
return n, nil
}
// UDLBackfillUserCids rebuilds the `idx:user:<uid>:cids` sets for data written
// before that index existed, and reports how many cids it added.
//
// One-shot: the upsert script maintains the set from now on. It is a SCAN, but
// a targeted one (`udl:idx:user:*:cid:*`, which is one key per (uid,cid), not
// per record) run once at boot rather than per read — the distinction that
// makes the difference between a migration and the pathology this index exists
// to avoid.
func (c *Client) UDLBackfillUserCids() (int, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
pattern := c.buildKey("udl:idx:user:*:cid:*")
prefix := c.buildKey("udl:idx:user:")
added := 0
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 500).Result()
if err != nil {
return added, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
// <prefix><uid>:cid:<cid> — split on the FIRST separator, not the
// last. A uid is multibase base58btc, whose alphabet contains no
// colon at all, so the first ":cid:" always ends the uid. A cid
// that happens to contain ":cid:" in its own name would make the
// last occurrence split in the wrong place.
rest := strings.TrimPrefix(k, prefix)
i := strings.Index(rest, ":cid:")
if i < 0 {
continue
}
uid, cid := rest[:i], rest[i+len(":cid:"):]
if uid == "" || cid == "" {
continue
}
n, err := c.client.SAdd(ctx, c.buildKey(udlIdxUserCids(uid)), cid).Result()
if err != nil {
return added, fmt.Errorf("sadd failed: %w", err)
}
added += int(n)
}
cursor = next
if cursor == 0 {
break
}
}
return added, nil
}
// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
func (c *Client) UDLCidUsers(cid string) ([]string, error) {
c.mu.RLock()
+5 -3
View File
@@ -179,7 +179,7 @@ func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
}
// UDLPurgeUser removes every trace of one account from the User Data Layer:
// its cells, its two user-scoped indexes, and its membership in the shared
// its cells, its three user-scoped indexes, and its membership in the shared
// cid-scoped indexes. Returns what was removed.
//
// The shared indexes are the part that is easy to miss and expensive to leave.
@@ -225,8 +225,10 @@ func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
stats.Cids = len(distinctCids)
stats.Keys = len(distinctKeys)
// The two user-scoped indexes are enumerable directly, and are dropped
// whole rather than member-by-member. Scanning them separately also catches
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are enumerable
// directly, and are dropped whole rather than member-by-member. The glob is
// what keeps this correct as indexes are added — `:cids` needed no change
// here when it landed. Scanning them separately also catches
// index entries whose cell is already gone — exactly the state a previously
// half-finished purge leaves behind.
idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
+234
View File
@@ -0,0 +1,234 @@
package storage
import (
"context"
"testing"
)
// The whole-profile export is what cross-instance recovery reads: a box that
// has just imported a secret key knows the uid and nothing else — not which
// cids the profile touched, not which keys. Every one of these tests is about
// that one caller.
func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
c, _ := newTestClient(t, "")
const uid = "zAlice"
writes := []struct{ cid, key, rec string }{
{"bagcsaaaaone", "like", "rec-one-like"},
{"bagcsaaaaone", "seek", "rec-one-seek"},
{"bagcsaaaatwo", "rating", "rec-two-rating"},
{"bagcsaaaatwo", "profile:name", "rec-two-name"},
{"bagcsaaathree", "watched", "rec-three-watched"},
}
for _, w := range writes {
if ok, err := c.UDLUpsertIfNewer(uid, w.cid, w.key, 1, 100, w.rec, "", false, false); err != nil || !ok {
t.Fatalf("upsert %s/%s: ok=%v err=%v", w.cid, w.key, ok, err)
}
}
entries, next, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if next != "" {
t.Fatalf("export of 5 rows should be complete, got nextCid=%q", next)
}
if len(entries) != len(writes) {
t.Fatalf("want %d rows, got %d: %+v", len(writes), len(entries), entries)
}
seen := map[string]string{}
for _, e := range entries {
seen[e.Cid+"/"+e.Key] = e.Record
}
for _, w := range writes {
if got := seen[w.cid+"/"+w.key]; got != w.rec {
t.Errorf("%s/%s: want record %q, got %q", w.cid, w.key, w.rec, got)
}
}
}
// The failure this guards is silent and total: a uid whose cids were never
// indexed exports as empty, and recovery reports success having restored
// nothing.
func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
c, _ := newTestClient(t, "")
if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
entries, _, err := c.UDLAllForUser("zBob", "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if len(entries) != 0 {
t.Fatalf("Bob owns nothing; got %+v", entries)
}
}
// ⚠ The tombstone rule, and the reason the cids index is never pruned.
//
// A tombstone must EXPORT. It is the only thing that can beat the older live
// value the other box still holds at a lower version — drop it from the export
// and an un-liked title comes back from the dead on every peer that already
// saw the `like`. This is the one place the cids index deliberately diverges
// from the user+key index, which does prune.
func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
c, _ := newTestClient(t, "")
const (
uid = "zAlice"
cid = "bagcsaaaaone"
key = "like"
)
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-liked", "", false, false); err != nil || !ok {
t.Fatalf("live upsert: ok=%v err=%v", ok, err)
}
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-unliked", "", false, true); err != nil || !ok {
t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
}
// The user+key index prunes it — that is what keeps My List short.
live, err := c.UDLListUserKey(uid, key)
if err != nil {
t.Fatalf("active_for_user_key: %v", err)
}
if len(live) != 0 {
t.Fatalf("tombstoned cid should leave the user+key index, got %+v", live)
}
// The export must still carry it, at the tombstone's version.
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if len(entries) != 1 {
t.Fatalf("tombstone must export, got %+v", entries)
}
if entries[0].Record != "rec-unliked" || entries[0].Version != 2 {
t.Fatalf("export must carry the tombstone, not the superseded value: %+v", entries[0])
}
}
func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
c, _ := newTestClient(t, "")
const uid = "zAlice"
cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
for _, cid := range cids {
for _, key := range []string{"like", "seek"} {
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-"+cid+"-"+key, "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
}
}
// A cid's keys are never split across a page, so a limit of 1 still yields
// both of the first cid's rows.
page1, next1, err := c.UDLAllForUser(uid, "", 1)
if err != nil {
t.Fatalf("page 1: %v", err)
}
if len(page1) != 2 || page1[0].Cid != cids[0] {
t.Fatalf("page 1 should hold both rows of %s, got %+v", cids[0], page1)
}
if next1 != cids[0] {
t.Fatalf("cursor should name the last cid emitted, want %q got %q", cids[0], next1)
}
// Walking the cursor must visit every row exactly once.
all := append([]UDLUserAllEntry{}, page1...)
cursor := next1
for cursor != "" {
page, next, err := c.UDLAllForUser(uid, cursor, 1)
if err != nil {
t.Fatalf("page after %q: %v", cursor, err)
}
all = append(all, page...)
cursor = next
}
if len(all) != 6 {
t.Fatalf("want 6 rows across all pages, got %d: %+v", len(all), all)
}
seen := map[string]bool{}
for _, e := range all {
k := e.Cid + "/" + e.Key
if seen[k] {
t.Fatalf("%s emitted twice across pages", k)
}
seen[k] = true
}
}
// Data written before the cids index existed must still export, or every
// profile that predates this feature silently recovers as empty.
func TestUDLBackfillUserCids_MakesLegacyDataExportable(t *testing.T) {
c, _ := newTestClient(t, "")
const (
uid = "zAlice"
cid = "bagcsaaaaone"
key = "like"
)
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
// Simulate the pre-index state: the cell and the other indexes exist, the
// cids set does not.
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if entries, _, err := c.UDLAllForUser(uid, "", 0); err != nil || len(entries) != 0 {
t.Fatalf("precondition: legacy data should export as empty, got %+v err=%v", entries, err)
}
added, err := c.UDLBackfillUserCids()
if err != nil {
t.Fatalf("backfill: %v", err)
}
if added != 1 {
t.Fatalf("want 1 cid backfilled, got %d", added)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export after backfill: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid || entries[0].Key != key {
t.Fatalf("backfilled data should export, got %+v", entries)
}
// Idempotent: a second run adds nothing.
if added, err := c.UDLBackfillUserCids(); err != nil || added != 0 {
t.Fatalf("second backfill should be a no-op, added=%d err=%v", added, err)
}
}
// The backfill splits `<uid>:cid:<cid>` on the FIRST separator, because a uid
// is multibase base58btc and its alphabet has no colon. Splitting on the last
// occurrence — the obvious-looking choice — mis-parses a cid that contains
// ":cid:" in its own name into uid "zAlice:cid:weird", cid "name".
func TestUDLBackfillUserCids_SplitsOnTheFirstSeparator(t *testing.T) {
c, _ := newTestClient(t, "")
const uid = "zAlice"
cid := "weird:cid:name"
if ok, err := c.UDLUpsertIfNewer(uid, cid, "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if _, err := c.UDLBackfillUserCids(); err != nil {
t.Fatalf("backfill: %v", err)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid {
t.Fatalf("want the cid parsed whole, got %+v", entries)
}
}