meta-core/internal/storage/udl.go

package storage

// User Data Layer (UDL) storage.
//
// Holds per-user, identity-signed records keyed by (uid, cid, key) — the
// subjective/per-user state that METADATA_KEYS.md rule #4 bars from the shared
// /file/{cid} hash (likes, ratings, watch progress, local paths). Each record
// is an opaque, client-signed (and, for private-tier keys, client-encrypted)
// CBOR blob; meta-core stores it verbatim and never needs the account key.
//
// Convergence is version-gated: a write is accepted only if its `version`
// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
// op-log merge that used to live in meta-watch's store.rs).
//
// Key model (all under the literal "udl:" prefix, mirroring the "file:" one):
//
//	udl:rec:<uid>/<cid>/<key>       JSON { record, version, ts, value? }
//	udl:idx:user:<uid>:key:<key>    set of <cid>   -> active_for_user_key (My List, Continue Watching)
//	udl:idx:user:<uid>:cid:<cid>    set of <key>   -> list_for_user_cid
//	udl:idx:user:<uid>:cids         set of <cid>   -> whole-profile export (recovery)
//	udl:idx:cid:<cid>:key:<key>     set of <uid>   -> aggregate
//	udl:idx:cid:<cid>:users         set of <uid>   -> uids_for_cid
//
// A "set" is a key-set: one key `<set>\x1f<member>` = "" per member, read as
// the key range under `<set>\x1f`. The separator is a control character
// because UDL keys are not validated — with `/` the members of
// `…:key:a/b` would show up as members of `…:key:a`.
//
// The `value` field is the plaintext JSON scalar for PUBLIC keys only
// (like/rating), used for aggregation; it is absent for private-tier keys
// (whose real value rides encrypted inside `record`) and for tombstones.

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"

	"github.com/metazla/meta-core/internal/kv"
)

// UDLCell is a decoded (uid, cid, key) record cell.
type UDLCell struct {
	Record   string // base64 CBOR, opaque + signed
	Version  int64
	Ts       int64
	Value    string // plaintext JSON scalar, public keys only
	HasValue bool
}

// UDLUserKeyEntry is one row of active_for_user_key (all cids for a uid+key).
type UDLUserKeyEntry struct {
	Cid     string `json:"cid"`
	Record  string `json:"record"`
	Version int64  `json:"version"`
	Ts      int64  `json:"ts"`
}

// UDLUserCidEntry is one row of list_for_user_cid (all keys for a uid+cid).
type UDLUserCidEntry struct {
	Key     string `json:"key"`
	Record  string `json:"record"`
	Version int64  `json:"version"`
	Ts      int64  `json:"ts"`
}

func udlRecKey(uid, cid, key string) string {
	return fmt.Sprintf("udl:rec:%s/%s/%s", uid, cid, key)
}
func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
func udlIdxCidKey(cid, key string) string  { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
func udlIdxCidUsers(cid string) string     { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }

// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
// index that makes "export this whole profile" a set read instead of a
// keyspace scan.
//
// The other four indexes each answer a question with one coordinate already
// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and
// nothing else. Without this set the only ways to enumerate a profile are to
// iterate the hardcoded key list — which silently misses any key added later —
// or to SCAN `udl:rec:<uid>/*`, which is the full-keyspace-read pathology that
// already made `/api/udl/users/stats` admin-only.
func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }

// udlSetSep separates a set key from a member (see the key model above).
const udlSetSep = "\x1f"

// udlCellJSON is a cell's stored form. Value is a pointer so an absent value
// (private-tier key, tombstone) stays distinguishable from an empty one.
type udlCellJSON struct {
	Record  string  `json:"record"`
	Version int64   `json:"version"`
	Ts      int64   `json:"ts"`
	Value   *string `json:"value,omitempty"`
}

func decodeUDLCell(raw string) (UDLCell, error) {
	var j udlCellJSON
	if err := json.Unmarshal([]byte(raw), &j); err != nil {
		return UDLCell{}, err
	}
	cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts}
	if j.Value != nil {
		cell.Value, cell.HasValue = *j.Value, true
	}
	return cell, nil
}

// udlMembers lists a key-set's members, sorted.
func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) {
	prefix := set + udlSetSep
	rows, err := r.Range(ctx, prefix, "", 0)
	if err != nil {
		return nil, err
	}
	out := make([]string, 0, len(rows))
	for _, row := range rows {
		out = append(out, row.K[len(prefix):])
	}
	return out, nil
}

func udlAdd(tx *kv.Tx, set, member string) error {
	_, err := tx.PutIfAbsent(set+udlSetSep+member, "")
	return err
}

func udlRem(tx *kv.Tx, set, member string) error {
	_, err := tx.Delete(set + udlSetSep + member)
	return err
}

// readUDLCell reads one cell. exists is false when it was never written.
func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) {
	raw, ok, err := r.Get(ctx, recKey)
	if err != nil || !ok {
		return UDLCell{}, false, err
	}
	cell, err := decodeUDLCell(raw)
	if err != nil {
		return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err)
	}
	return cell, true, nil
}

// readUDLCells reads many cells in one multi-key read; absent cells are
// omitted from the map.
func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) {
	raws, err := r.MGet(ctx, recKeys)
	if err != nil {
		return nil, err
	}
	out := make(map[string]UDLCell, len(raws))
	for k, raw := range raws {
		cell, err := decodeUDLCell(raw)
		if err != nil {
			return nil, fmt.Errorf("decode %s: %w", k, err)
		}
		out[k] = cell
	}
	return out, nil
}

// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
// one, and maintains the indexes — one transaction, so the gate and the write
// cannot interleave with a concurrent upsert of the same cell.
// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by
// aggregation); pass hasValue=false for private-tier keys and tombstones. Set
// tombstone=true when the record deletes the cell.
//
// A tombstone still writes the cell — that is the CRDT state, and dropping it
// would let an older write resurrect the value — but it removes the cid from
// the user+key index. That index is what `active_for_user_key` returns, so
// leaving deleted cids in it means every My List / Continue Watching read
// carries every title ever un-liked or finished, forever. The other four
// indexes are left alone: they answer "who has an opinion about this cid",
// where a tombstone is still an answer.
//
// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even
// though it looks symmetric with the user+key index. It is the export index: a
// cid dropped from it is a cid whose tombstone never travels, and a tombstone
// that does not travel is exactly how an un-liked title comes back from the
// dead on the other box — the peer still holds the older `like` at a lower
// version, and with no tombstone to beat it, nothing ever supersedes it.
//
// Returns accepted=false when the write is stale.
func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
	accepted := false
	err := c.update(func(tx *kv.Tx) error {
		recKey := udlRecKey(uid, cid, key)
		stored, ok, err := readUDLCell(tx.Context(), tx, recKey)
		if err != nil {
			return err
		}
		if ok && stored.Version >= version {
			return nil
		}
		cell := udlCellJSON{Record: record, Version: version, Ts: ts}
		if hasValue {
			v := publicValue
			cell.Value = &v
		}
		body, err := json.Marshal(cell)
		if err != nil {
			return err
		}
		if err := tx.Put(recKey, string(body)); err != nil {
			return err
		}
		if tombstone {
			err = udlRem(tx, udlIdxUserKey(uid, key), cid)
		} else {
			err = udlAdd(tx, udlIdxUserKey(uid, key), cid)
		}
		if err != nil {
			return err
		}
		for _, add := range [][2]string{
			{udlIdxUserCid(uid, cid), key},
			{udlIdxCidKey(cid, key), uid},
			{udlIdxCidUsers(cid), uid},
			{udlIdxUserCids(uid), cid},
		} {
			if err := udlAdd(tx, add[0], add[1]); err != nil {
				return err
			}
		}
		accepted = true
		return nil
	})
	if err != nil {
		return false, fmt.Errorf("udl upsert failed: %w", err)
	}
	return accepted, nil
}

// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
// no record has ever been written for the cell.
func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
	s, err := c.kv()
	if err != nil {
		return UDLCell{}, false, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	return readUDLCell(ctx, s, udlRecKey(uid, cid, key))
}

// UDLListUserKey returns every cell for (uid, key) across all cids — the
// projection behind active_for_user_key (My List, Continue Watching). Includes
// tombstones; the caller decodes each record and filters.
func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := opCtx()
	defer cancel()

	cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key))
	if err != nil {
		return nil, err
	}
	recKeys := make([]string, len(cids))
	for i, cid := range cids {
		recKeys[i] = udlRecKey(uid, cid, key)
	}
	cells, err := readUDLCells(ctx, s, recKeys)
	if err != nil {
		return nil, err
	}
	out := make([]UDLUserKeyEntry, 0, len(cids))
	for i, cid := range cids {
		cell, ok := cells[recKeys[i]]
		if !ok {
			continue
		}
		out = append(out, UDLUserKeyEntry{Cid: cid, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
	}
	return out, nil
}

// UDLListUserCid returns every cell for (uid, cid) across all keys — the
// projection behind list_for_user_cid.
func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := opCtx()
	defer cancel()

	keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
	if err != nil {
		return nil, err
	}
	recKeys := make([]string, len(keys))
	for i, key := range keys {
		recKeys[i] = udlRecKey(uid, cid, key)
	}
	cells, err := readUDLCells(ctx, s, recKeys)
	if err != nil {
		return nil, err
	}
	out := make([]UDLUserCidEntry, 0, len(keys))
	for i, key := range keys {
		cell, ok := cells[recKeys[i]]
		if !ok {
			continue
		}
		out = append(out, UDLUserCidEntry{Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
	}
	return out, nil
}

// UDLUserAllEntry is one row of a whole-profile export.
type UDLUserAllEntry struct {
	Cid     string `json:"cid"`
	Key     string `json:"key"`
	Record  string `json:"record"`
	Version int64  `json:"version"`
	Ts      int64  `json:"ts"`
}

// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
//
// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never
// split across a page: the page ends on a cid boundary once `limit` rows have
// been emitted, and `nextCid` is the last cid included — pass it back as
// `afterCid` to continue. `nextCid` is empty when the export is complete.
//
// Tombstones are included. They are the CRDT state that stops a deleted value
// resurrecting on the box that receives this export, so omitting them would
// make recovery actively wrong rather than merely incomplete.
func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, "", err
	}
	if limit <= 0 {
		limit = 1000
	}
	ctx, cancel := opCtx()
	defer cancel()

	cids, err := udlMembers(ctx, s, udlIdxUserCids(uid))
	if err != nil {
		return nil, "", err
	}
	sort.Strings(cids)

	out := make([]UDLUserAllEntry, 0, limit)
	nextCid := ""
	for _, cid := range cids {
		if afterCid != "" && cid <= afterCid {
			continue
		}
		if len(out) >= limit {
			nextCid = out[len(out)-1].Cid
			break
		}
		keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
		if err != nil {
			return nil, "", err
		}
		for _, key := range keys {
			cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key))
			if err != nil {
				return nil, "", err
			}
			if !ok {
				continue
			}
			out = append(out, UDLUserAllEntry{
				Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts,
			})
		}
	}
	return out, nextCid, nil
}

// UDLCountUserCids is the number of distinct cids a uid holds records for.
//
// One index range count, which is what makes it usable as a per-push quota
// check on the gateway mirror — a quota that cost a keyspace walk would be a
// denial-of-service in its own right. Counts cids rather than cells because
// that is what the index holds; a profile is bounded by the titles it has
// touched, not by how many keys it has per title.
func (c *Client) UDLCountUserCids(uid string) (int64, error) {
	s, err := c.kv()
	if err != nil {
		return 0, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep)
}

// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
func (c *Client) UDLCidUsers(cid string) ([]string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	return udlMembers(ctx, s, udlIdxCidUsers(cid))
}

// UDLPublicValues returns the live plaintext public JSON scalar values for
// (cid, key) across all uids — the input to aggregation. Cells without a value
// (private-tier or tombstoned) are skipped.
func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := opCtx()
	defer cancel()

	uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key))
	if err != nil {
		return nil, err
	}
	recKeys := make([]string, len(uids))
	for i, uid := range uids {
		recKeys[i] = udlRecKey(uid, cid, key)
	}
	cells, err := readUDLCells(ctx, s, recKeys)
	if err != nil {
		return nil, err
	}
	out := make([]string, 0, len(uids))
	for _, k := range recKeys {
		if cell, ok := cells[k]; ok && cell.HasValue {
			out = append(out, cell.Value)
		}
	}
	return out, nil
}