v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410
Build and Push Docker Image / build (push) Successful in 5m1s
Build and Push Docker Image / build (push) Successful in 5m1s
This commit is contained in:
Worph
@@ -0,0 +1,612 @@
|
||||
package storage
|
||||
|
||||
// The apply loop: folds committed truth changes into the cache.
|
||||
//
|
||||
// The cache's merged view keeps the v1 record layout — `file:<rid>/<field>`
|
||||
// rows of the cache kv table — so every reader (record reads, search, the
|
||||
// field index, FTS, counters, SSE and its snapshot) is unchanged; those rows
|
||||
// are now derived, never written by a client. Each committed change re-merges
|
||||
// only the (rid, key) it touched, from the truth votes of every cid in the
|
||||
// group. Group membership is synthesised: every cid of a group appears as
|
||||
// `file:<rid>/cids/<cid> = "true"` plus the `cid:<cid> → rid` lookup, whatever
|
||||
// the cids/* claims say.
|
||||
//
|
||||
// Changes arrive in commit order (kv.Tx.OnCommit runs in the truth writer), so
|
||||
// `meta:applied_seq` only ever covers a prefix of the truth log. A crash
|
||||
// between the truth commit and the cache apply leaves the cache behind, never
|
||||
// wrong; Connect catches up from truth.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
const (
|
||||
appliedSeqKey = "meta:applied_seq"
|
||||
// rebuiltAtKey holds the stream id at which the cache was last rebuilt.
|
||||
// An SSE cursor older than it gets `gap`: the rebuild emitted no
|
||||
// per-key events, so the consumer must take a snapshot.
|
||||
rebuiltAtKey = "meta:rebuilt_at"
|
||||
)
|
||||
|
||||
var localFields = []string{"filePath", "sizeByte", "mtimeNano"}
|
||||
|
||||
type applier struct {
|
||||
c *Client
|
||||
|
||||
mu sync.Mutex
|
||||
queue []change
|
||||
applied int64
|
||||
notify chan struct{}
|
||||
|
||||
wake chan struct{}
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newApplier(c *Client, applied int64) *applier {
|
||||
a := &applier{
|
||||
c: c, applied: applied,
|
||||
notify: make(chan struct{}),
|
||||
wake: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
go a.run()
|
||||
return a
|
||||
}
|
||||
|
||||
func (a *applier) enqueue(ch []change) {
|
||||
a.mu.Lock()
|
||||
a.queue = append(a.queue, ch...)
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case a.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// wait blocks until the cache has applied every change up to seq.
|
||||
func (a *applier) wait(ctx context.Context, seq int64) error {
|
||||
for {
|
||||
a.mu.Lock()
|
||||
if a.applied >= seq {
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
n := a.notify
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case <-n:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-a.stop:
|
||||
return errors.New("storage closed before the cache applied the write")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *applier) close() {
|
||||
close(a.stop)
|
||||
<-a.done
|
||||
}
|
||||
|
||||
const applyBatchMax = 2000
|
||||
|
||||
func (a *applier) run() {
|
||||
defer close(a.done)
|
||||
for {
|
||||
select {
|
||||
case <-a.wake:
|
||||
case <-a.stop:
|
||||
return
|
||||
}
|
||||
for {
|
||||
a.mu.Lock()
|
||||
n := min(len(a.queue), applyBatchMax)
|
||||
batch := append([]change(nil), a.queue[:n]...)
|
||||
a.mu.Unlock()
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
if err := a.c.applyBatch(batch); err != nil {
|
||||
log.Printf("[Storage] apply of %d changes failed, retrying: %v", len(batch), err)
|
||||
select {
|
||||
case <-time.After(time.Second):
|
||||
continue
|
||||
case <-a.stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.queue = a.queue[n:]
|
||||
if s := batch[len(batch)-1].seq; s > a.applied {
|
||||
a.applied = s
|
||||
}
|
||||
close(a.notify)
|
||||
a.notify = make(chan struct{})
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// applyBatch folds committed changes into the cache in one transaction.
|
||||
func (c *Client) applyBatch(batch []change) error {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
return s.Update(ctx, func(tx *kv.Tx) error {
|
||||
return c.applyTx(tx, batch)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) applyTx(tx *kv.Tx, batch []change) error {
|
||||
ctx := tx.Context()
|
||||
marks := map[string]map[string]bool{}
|
||||
mark := func(rid, key string) {
|
||||
if marks[rid] == nil {
|
||||
marks[rid] = map[string]bool{}
|
||||
}
|
||||
marks[rid][key] = true
|
||||
}
|
||||
all := map[string]bool{}
|
||||
dirty := map[string]bool{}
|
||||
var maxSeq int64
|
||||
for _, it := range batch {
|
||||
maxSeq = max(maxSeq, it.seq)
|
||||
if it.regroup != nil {
|
||||
loser, survivor := it.regroup[0], it.regroup[1]
|
||||
if _, err := tx.DeletePrefix(recordPrefix(loser), 0); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Delete(rootKey(loser)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := deleteSearchRowTx(tx, loser); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(marks, loser)
|
||||
cids, err := c.truthMembers(ctx, survivor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, cid := range cids {
|
||||
if _, err := syncMember(tx, cid, survivor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
all[survivor], dirty[survivor] = true, true
|
||||
continue
|
||||
}
|
||||
rid, err := c.truthRid(ctx, it.cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rid == "" {
|
||||
continue
|
||||
}
|
||||
newRoot, err := syncMember(tx, it.cid, rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if newRoot {
|
||||
dirty[rid] = true
|
||||
}
|
||||
if it.key == "" {
|
||||
continue
|
||||
}
|
||||
if it.live {
|
||||
_, err = tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, it.pid, it.cid, it.key)
|
||||
} else {
|
||||
_, err = tx.Exec(`DELETE FROM claim_by_pid WHERE pid = ? AND cid = ? AND key = ?`, it.pid, it.cid, it.key)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mark(rid, it.key)
|
||||
}
|
||||
for rid := range all {
|
||||
cids, err := cidsForRoot(ctx, tx, rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys, err := c.claimKeys(ctx, cids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range append(keys, localFields...) {
|
||||
mark(rid, k)
|
||||
}
|
||||
}
|
||||
for rid, keys := range marks {
|
||||
cids, err := cidsForRoot(ctx, tx, rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for key := range keys {
|
||||
changed, err := c.remergeTx(tx, rid, cids, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed && affectsHaystack(key) {
|
||||
dirty[rid] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for rid := range dirty {
|
||||
if err := refreshSearchTx(tx, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if maxSeq > 0 {
|
||||
return tx.Put(appliedSeqKey, strconv.FormatInt(maxSeq, 10))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncMember makes cid a member of rid in the cache: the lookup row, the
|
||||
// synthesised key-set member, and the root. Reports whether rid is new.
|
||||
func syncMember(tx *kv.Tx, cid, rid string) (bool, error) {
|
||||
cur, _, err := tx.Get(tx.Context(), CIDIndexPrefix+cid)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if cur != rid {
|
||||
if cur != "" {
|
||||
if _, err := tx.Delete(recordPrefix(cur) + CIDsKeyPrefix + cid); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if err := tx.Put(CIDIndexPrefix+cid, rid); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if err := tx.Put(recordPrefix(rid)+CIDsKeyPrefix+cid, "true"); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tx.Set(rootKey(rid), "")
|
||||
}
|
||||
|
||||
// remergeTx recomputes file:<rid>/<key> from the group's votes (plus the
|
||||
// box-local facts for filePath / sizeByte / mtimeNano) and reports whether
|
||||
// the merged row changed.
|
||||
func (c *Client) remergeTx(tx *kv.Tx, rid string, cids []string, key string) (bool, error) {
|
||||
if strings.HasPrefix(key, CIDsKeyPrefix) {
|
||||
return false, nil // membership is synthesised, never merged
|
||||
}
|
||||
votes, err := c.votesFor(tx.Context(), cids, key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
res := Merge(votes, "")
|
||||
val, ok := res.Value, res.OK
|
||||
if isLocalField(key) {
|
||||
lv, lok, err := localFact(tx, cids, key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// The disk is the truth for where and when a file is; a signed
|
||||
// sizeByte (a content fact) wins over the on-disk size.
|
||||
if lok && (key != "sizeByte" || !ok) {
|
||||
val, ok = lv, true
|
||||
}
|
||||
}
|
||||
k := recordPrefix(rid) + key
|
||||
if ok {
|
||||
return tx.Set(k, val)
|
||||
}
|
||||
return tx.Delete(k)
|
||||
}
|
||||
|
||||
// localFact reads a box-local field from local_file for the group's cids.
|
||||
func localFact(tx *kv.Tx, cids []string, key string) (string, bool, error) {
|
||||
if len(cids) == 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
args := make([]any, len(cids))
|
||||
for i, c := range cids {
|
||||
args[i] = c
|
||||
}
|
||||
var path string
|
||||
var size, mtime int64
|
||||
err := tx.QueryRow(`SELECT path, size, mtime FROM local_file WHERE midhash IN (?`+
|
||||
strings.Repeat(",?", len(cids)-1)+`) ORDER BY path LIMIT 1`, args...).Scan(&path, &size, &mtime)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
switch key {
|
||||
case "filePath":
|
||||
return path, true, nil
|
||||
case "sizeByte":
|
||||
return strconv.FormatInt(size, 10), true, nil
|
||||
default:
|
||||
return strconv.FormatInt(mtime, 10), true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// truthMembers lists every cid of rid's group (scans cid_group: truth has no
|
||||
// rid index; only regroups and rebuilds call this).
|
||||
func (c *Client) truthMembers(ctx context.Context, rid string) ([]string, error) {
|
||||
rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid FROM cid_group WHERE rid = ?`, rid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var cid string
|
||||
if err := rows.Scan(&cid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, cid)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// truthLastSeq reads the newest committed seq.
|
||||
func (c *Client) truthLastSeq(ctx context.Context) (int64, error) {
|
||||
var v string
|
||||
err := c.truth.DB().QueryRowContext(ctx, `SELECT value FROM meta WHERE name = ?`, lastSeqName).Scan(&v)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseInt(v, 10, 64)
|
||||
}
|
||||
|
||||
// catchUp brings the cache level with truth at boot: nothing to do, a replay
|
||||
// of the changes after applied_seq (found by a full scan — there is no seq
|
||||
// index, and this only runs after an unclean shutdown), or a full rebuild
|
||||
// when the cache is missing or a group changed while it was behind.
|
||||
func (c *Client) catchUp() (int64, error) {
|
||||
s := c.store
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
|
||||
defer cancel()
|
||||
last, err := c.truthLastSeq(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
appliedStr, has, err := s.Get(ctx, appliedSeqKey)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
applied, _ := strconv.ParseInt(appliedStr, 10, 64)
|
||||
if has && applied >= last {
|
||||
return applied, nil
|
||||
}
|
||||
if !has {
|
||||
if last == 0 {
|
||||
return 0, nil // fresh store
|
||||
}
|
||||
log.Printf("[Storage] cache is empty, rebuilding from truth (last_seq %d)", last)
|
||||
return last, c.rebuild(ctx, last)
|
||||
}
|
||||
rows, err := c.truth.DB().QueryContext(ctx,
|
||||
`SELECT cid, key, pid, value IS NOT NULL, seq FROM claim WHERE seq > ? ORDER BY seq`, applied)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var missed []change
|
||||
regroup := false
|
||||
for rows.Next() {
|
||||
var ch change
|
||||
if err := rows.Scan(&ch.cid, &ch.key, &ch.pid, &ch.live, &ch.seq); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
if strings.HasPrefix(ch.key, CIDsKeyPrefix) {
|
||||
regroup = true
|
||||
}
|
||||
missed = append(missed, ch)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if regroup {
|
||||
log.Printf("[Storage] cache behind by %d changes including a group link, rebuilding", len(missed))
|
||||
return last, c.rebuild(ctx, last)
|
||||
}
|
||||
log.Printf("[Storage] cache behind (applied %d < last %d), replaying %d changes", applied, last, len(missed))
|
||||
for start := 0; start < len(missed); start += applyBatchMax {
|
||||
end := min(start+applyBatchMax, len(missed))
|
||||
if err := c.applyBatch(missed[start:end]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := s.Update(ctx, func(tx *kv.Tx) error { return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10)) }); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
// Rebuild drops every derived row of the cache and recomputes it from truth.
|
||||
// Exposed for tests and operators; boot calls it when the cache is missing.
|
||||
func (c *Client) Rebuild() error {
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
|
||||
defer cancel()
|
||||
last, err := c.truthLastSeq(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.rebuild(ctx, last)
|
||||
}
|
||||
|
||||
const rebuildBatch = 500
|
||||
|
||||
func (c *Client) rebuild(ctx context.Context, last int64) error {
|
||||
s := c.store
|
||||
started := time.Now()
|
||||
for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
|
||||
for {
|
||||
var n int64
|
||||
if err := s.Update(ctx, func(tx *kv.Tx) error {
|
||||
tx.Quiet()
|
||||
var err error
|
||||
n, err = tx.DeletePrefix(prefix, bulkBatch)
|
||||
return err
|
||||
}); err != nil {
|
||||
return fmt.Errorf("clear %s: %w", prefix, err)
|
||||
}
|
||||
if n < bulkBatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.Update(ctx, func(tx *kv.Tx) error {
|
||||
if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(`DELETE FROM claim_by_pid`)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
groups := map[string][]string{}
|
||||
var order []string
|
||||
rows, err := c.truth.DB().QueryContext(ctx, `SELECT cid, rid FROM cid_group`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var cid, rid string
|
||||
if err := rows.Scan(&cid, &rid); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if _, ok := groups[rid]; !ok {
|
||||
order = append(order, rid)
|
||||
}
|
||||
groups[rid] = append(groups[rid], cid)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for start := 0; start < len(order); start += rebuildBatch {
|
||||
end := min(start+rebuildBatch, len(order))
|
||||
if err := s.Update(ctx, func(tx *kv.Tx) error {
|
||||
tx.Quiet()
|
||||
for _, rid := range order[start:end] {
|
||||
cids := groups[rid]
|
||||
for _, cid := range cids {
|
||||
if _, err := syncMember(tx, cid, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
keys, err := c.claimKeys(ctx, cids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range append(keys, localFields...) {
|
||||
if _, err := c.remergeTx(tx, rid, cids, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := refreshSearchTx(tx, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("rebuild records: %w", err)
|
||||
}
|
||||
}
|
||||
if err := c.rebuildClaimIndex(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.RebuildCounters(); err != nil {
|
||||
return err
|
||||
}
|
||||
rebuiltAt := strconv.FormatInt(time.Now().UnixMilli(), 10) + "-0"
|
||||
if err := s.Update(ctx, func(tx *kv.Tx) error {
|
||||
if err := tx.Put(rebuiltAtKey, rebuiltAt); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Put(appliedSeqKey, strconv.FormatInt(last, 10))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[Storage] cache rebuilt: %d records in %s", len(order), time.Since(started).Round(time.Millisecond))
|
||||
return nil
|
||||
}
|
||||
|
||||
// rebuildClaimIndex refills claim_by_pid from truth, a page at a time.
|
||||
func (c *Client) rebuildClaimIndex(ctx context.Context) error {
|
||||
type row struct {
|
||||
pid int64
|
||||
cid, key string
|
||||
}
|
||||
var after row
|
||||
first := true
|
||||
for {
|
||||
q := `SELECT pid, cid, key FROM claim WHERE value IS NOT NULL`
|
||||
var args []any
|
||||
if !first {
|
||||
q += ` AND (cid, key, pid) > (?, ?, ?)`
|
||||
args = append(args, after.cid, after.key, after.pid)
|
||||
}
|
||||
q += ` ORDER BY cid, key, pid LIMIT 5000`
|
||||
rs, err := c.truth.DB().QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var page []row
|
||||
for rs.Next() {
|
||||
var r row
|
||||
if err := rs.Scan(&r.pid, &r.cid, &r.key); err != nil {
|
||||
rs.Close()
|
||||
return err
|
||||
}
|
||||
page = append(page, r)
|
||||
}
|
||||
if err := rs.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(page) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := c.store.Update(ctx, func(tx *kv.Tx) error {
|
||||
for _, r := range page {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO claim_by_pid(pid, cid, key) VALUES(?, ?, ?)`, r.pid, r.cid, r.key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
after, first = page[len(page)-1], false
|
||||
}
|
||||
}
|
||||
|
||||
// RebuiltAt is the stream id of the last cache rebuild ("" if never).
|
||||
func (c *Client) RebuiltAt() string {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
v, _, _ := s.Get(ctx, rebuiltAtKey)
|
||||
return v
|
||||
}
|
||||
@@ -3,34 +3,22 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/cid"
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// This file implements the CID-resolution layer that lives on top of the
|
||||
// flat metadata storage in client.go. The short version:
|
||||
// The cid → record resolution layer, read side. A record (rid) is a group of
|
||||
// cids known to be the same content; the group lives in truth's cid_group and
|
||||
// is mirrored into the cache as
|
||||
//
|
||||
// - Roots are opaque UUIDv7s — file:<uuid>/<property> for every property.
|
||||
// The UUID is an internal implementation detail; the mental model is
|
||||
// "N CIDs → one record of keys→metadata".
|
||||
// - Every known CID for a file is registered as a reverse-index entry:
|
||||
// cid:<bareCid> → <uuid>
|
||||
// and as a member of the per-root bare-CID key-set:
|
||||
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
|
||||
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
|
||||
// multicodec (see internal/cid), so there is no per-algorithm field name
|
||||
// and no <algo>: token prefix. The canonical externally-visible CID is
|
||||
// *derived by rank* from the key-set on read (GetMetadataDocument), never
|
||||
// stored.
|
||||
// cid:<bareCid> → <rid>
|
||||
// file:<rid>/cids/<bareCid> = "true" (synthesised membership, one per cid)
|
||||
//
|
||||
// The record write path (writeFieldsTx) catches any write of a cids/<cid>
|
||||
// field and registers the reverse-index entry in the same transaction, so
|
||||
// writers that just emit the key-set members don't need to know about the
|
||||
// reverse index at all. The helpers here take the caller's *kv.Tx: a write
|
||||
// must never open a second transaction (there is one writer connection).
|
||||
// by the apply loop (apply.go). A CID is self-describing (its algorithm is in
|
||||
// the CIDv1 multicodec, see internal/cid); the canonical externally-visible
|
||||
// CID is derived by rank on read (GetMetadataDocument), never stored.
|
||||
|
||||
// CIDIndexPrefix is the key prefix for the reverse index. Combined
|
||||
// with a bare CID, it forms the full key: "cid:bafk…".
|
||||
@@ -43,114 +31,6 @@ const CIDIndexPrefix = "cid:"
|
||||
// reverse-index entries that need cleanup).
|
||||
const CIDsKeyPrefix = "cids/"
|
||||
|
||||
// Duplicate paths — other physical locations whose content hashes to the same
|
||||
// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
|
||||
// writes one when it sees a new path for content that's already indexed.
|
||||
|
||||
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
|
||||
// in the root index. Returns the new UUID. The caller is expected to follow
|
||||
// up with AddAlias for at least one CID — a root without any alias is
|
||||
// reachable only via the index, which works but defeats the point.
|
||||
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
|
||||
var uuid string
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
uuid, err = mintTx(tx, filePath, size, mtimeNano)
|
||||
return err
|
||||
})
|
||||
return uuid, err
|
||||
}
|
||||
|
||||
func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
|
||||
uuid, err := NewUUID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uuid: %w", err)
|
||||
}
|
||||
return uuid, writeFieldsTx(tx, uuid, map[string]string{
|
||||
"filePath": filePath,
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
})
|
||||
}
|
||||
|
||||
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
|
||||
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
|
||||
// Idempotent: registering the same CID twice is a no-op.
|
||||
func (c *Client) AddAlias(uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return fmt.Errorf("uuid and cid are required")
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
|
||||
})
|
||||
}
|
||||
|
||||
// addAliasTx registers the reverse-index entry for a cids/<cid> member the
|
||||
// caller is writing (writeFieldsTx writes the member itself).
|
||||
func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return fmt.Errorf("uuid and cid are required")
|
||||
}
|
||||
// Self-pointing alias guard. If the caller is trying to register
|
||||
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
|
||||
// dual-root failure mode in disguise — it means somebody wrote
|
||||
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
|
||||
// write; leave any existing alias alone. The key-set membership is still
|
||||
// recorded by the caller, so the field-level write succeeds.
|
||||
if uuid == cidStr {
|
||||
return nil
|
||||
}
|
||||
// Guard against the dual-root pattern: if cid:<cid> already points at a
|
||||
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
|
||||
// key-set membership keeps the CID discoverable via root → cids.
|
||||
indexKey := CIDIndexPrefix + cidStr
|
||||
existing, _, err := tx.Get(tx.Context(), indexKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read existing alias: %w", err)
|
||||
}
|
||||
if existing != "" && existing != uuid {
|
||||
return nil
|
||||
}
|
||||
if existing == uuid {
|
||||
return nil
|
||||
}
|
||||
return tx.Put(indexKey, uuid)
|
||||
}
|
||||
|
||||
// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
|
||||
// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
|
||||
// itself (DeleteProperty does both).
|
||||
//
|
||||
// ⚠ Why this exists. A cids/ member write touches two keys — the flat
|
||||
// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
|
||||
// only removed the first. A "deleted" CID therefore still resolved: GET
|
||||
// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
|
||||
// The asymmetry is not theoretical. meta-share published dag-pb roots computed
|
||||
// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
|
||||
// the member was expressible, removing the claim was not, so the false
|
||||
// identity survived its own retraction.
|
||||
//
|
||||
// Guarded, not blind: the alias is removed only when it currently points at
|
||||
// THIS uuid. Two records can legitimately race to claim a cid (see the
|
||||
// dual-root guard in addAliasTx, which leaves the first writer's alias in
|
||||
// place); deleting a member from the loser must not unpick the winner's index.
|
||||
func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return nil
|
||||
}
|
||||
indexKey := CIDIndexPrefix + cidStr
|
||||
existing, ok, err := tx.Get(tx.Context(), indexKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read existing alias: %w", err)
|
||||
}
|
||||
if !ok || existing != uuid {
|
||||
// Absent, or somebody else owns this alias. Leave it.
|
||||
return nil
|
||||
}
|
||||
_, err = tx.Delete(indexKey)
|
||||
return err
|
||||
}
|
||||
|
||||
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
|
||||
// lookup, an SSE consumer) to the actual storage root key.
|
||||
//
|
||||
@@ -202,23 +82,6 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
|
||||
return uuid, nil
|
||||
}
|
||||
|
||||
// AddDuplicatePath records that path is another physical location whose
|
||||
// content matches uuid. Returns true if path was newly added. The watcher
|
||||
// calls this when it sees a file whose midhash matches an existing root but
|
||||
// at a different filePath.
|
||||
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
|
||||
if uuid == "" || path == "" {
|
||||
return false, nil
|
||||
}
|
||||
added := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
|
||||
return err
|
||||
})
|
||||
return added, err
|
||||
}
|
||||
|
||||
// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
|
||||
// stripped).
|
||||
func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
|
||||
@@ -234,20 +97,6 @@ func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
|
||||
// (including the cids/<cid> key-set), every reverse-index entry for those
|
||||
// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
|
||||
// with no remaining keys is a successful no-op.
|
||||
func (c *Client) DeleteRoot(uuid string) error {
|
||||
if uuid == "" {
|
||||
return nil
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
_, err := deleteRecordTx(tx, uuid)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
|
||||
// or "" if the field isn't a key-set member.
|
||||
func cidFromKeysetField(field string) string {
|
||||
@@ -311,13 +160,16 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
|
||||
}
|
||||
}
|
||||
|
||||
dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
|
||||
// Duplicates: every other local path holding the same content.
|
||||
paths, err := localPaths(ctx, s.DB(), cids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read duplicates: %w", err)
|
||||
}
|
||||
dups := make([]string, 0, len(dupRows))
|
||||
for _, row := range dupRows {
|
||||
dups = append(dups, row.K[len(dupPrefix(uuid)):])
|
||||
dups := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
if p != flat["filePath"] {
|
||||
dups = append(dups, p)
|
||||
}
|
||||
}
|
||||
|
||||
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
|
||||
@@ -330,66 +182,3 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
|
||||
Duplicates: dups,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RegisterFileTuple records one physical file under its midhash256 CID, the
|
||||
// same way the file watcher does for every file it scans: mint a UUID root
|
||||
// (filePath/sizeByte/mtimeNano) and write `cids/<midhash>`, whose hook
|
||||
// registers the cid:<midhash> reverse alias. Content already known under
|
||||
// another path is recorded as a duplicate; the same path just refreshes
|
||||
// size/mtime.
|
||||
//
|
||||
// Returns the root and whether it was newly minted. Shared by the watcher and
|
||||
// POST /api/files/register, which is how a box running with the watcher off
|
||||
// (a meta-watch client core) still gets files it writes under /files/plugin
|
||||
// resolvable by CID.
|
||||
func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
|
||||
if midhash == "" {
|
||||
return "", false, fmt.Errorf("midhash is required")
|
||||
}
|
||||
var (
|
||||
uuid string
|
||||
minted bool
|
||||
)
|
||||
// One transaction: the lookup and the mint cannot interleave with another
|
||||
// registration of the same content, so two concurrent scans of one file
|
||||
// can no longer mint two roots for it.
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
ctx := tx.Context()
|
||||
existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GetByCID(%s): %w", midhash, err)
|
||||
}
|
||||
uuid = existing
|
||||
if uuid == "" {
|
||||
if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
|
||||
return fmt.Errorf("Mint(%s): %w", absPath, err)
|
||||
}
|
||||
minted = true
|
||||
if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
|
||||
return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != absPath {
|
||||
if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
|
||||
return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := writeFieldsTx(tx, uuid, map[string]string{
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return uuid, minted, nil
|
||||
}
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests for retracting a `cids/<cid>` key-set member (cid_resolution.go's
|
||||
// removeAliasTx, reached through DeleteProperty).
|
||||
//
|
||||
// Why this is worth its own file. Adding a member writes two keys — the flat
|
||||
// `file:<uuid>/cids/<cid>` and the reverse alias `cid:<cid>` → `<uuid>`. Until
|
||||
// removeAliasLocked (now removeAliasTx) existed, deleting removed only the
|
||||
// first, so a retracted CID kept resolving to the record: the member was gone
|
||||
// from the document and the claim was not.
|
||||
//
|
||||
// That asymmetry had a live consequence. meta-share minted dag-pb roots and
|
||||
// midhash256s over whatever `archive::pick_media` elected, which on an
|
||||
// obfuscated posting was an arbitrary RAR volume or a par2 file. Six records on
|
||||
// watch.nsl.sh carry such members. `METADATA_KEYS.md` §2 defines a `cids/`
|
||||
// member as a digest of *this record's file* and notes that every member is a
|
||||
// reverse index back to the record — so those entries assert that an archive
|
||||
// volume is the episode. Retracting one has to remove the assertion, not just
|
||||
// the row.
|
||||
|
||||
// aliasTarget returns the uuid that cid:<cidStr> currently resolves to, or "".
|
||||
func aliasTarget(t *testing.T, c *Client, cidStr string) string {
|
||||
t.Helper()
|
||||
got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr)
|
||||
if err != nil {
|
||||
t.Fatalf("get alias: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func hasField(t *testing.T, c *Client, hashID, field string) bool {
|
||||
t.Helper()
|
||||
fields, err := c.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
t.Fatalf("read record: %v", err)
|
||||
}
|
||||
_, ok := fields[field]
|
||||
return ok
|
||||
}
|
||||
|
||||
// The real shape: `…glnrqztt…` is the nzb-release locator whose record published
|
||||
// a dag-pb root computed over volume 3 of a 35-volume RAR5 set.
|
||||
const (
|
||||
poisonedRoot = "bafybeicicuzdvuztca2hp2g2k7scgr4nnibilzgoppljeprphybqot7v6i"
|
||||
goodMidhash = "bagacbabaed6uiulnv7tscpxne22eztkjvnxg5almn43dpfecppixarfe7xhlu"
|
||||
)
|
||||
|
||||
func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
|
||||
withClient(t, func(t *testing.T, c *Client) {
|
||||
const uuid = "01KSBHMTESTRECORD"
|
||||
|
||||
if err := c.SetMetadataFlat(uuid, map[string]string{
|
||||
"cids/" + poisonedRoot: "true",
|
||||
"cids/" + goodMidhash: "true",
|
||||
"fileType": "video",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
// All three writes landed, and the CID resolves to the record.
|
||||
if got := aliasTarget(t, c, poisonedRoot); got != uuid {
|
||||
t.Fatalf("alias should point at the record before retraction, got %q", got)
|
||||
}
|
||||
if !hasField(t, c, uuid, "cids/"+poisonedRoot) {
|
||||
t.Fatal("field index should list the member before retraction")
|
||||
}
|
||||
if got := c.ResolveRoot(poisonedRoot); got != uuid {
|
||||
t.Fatalf("ResolveRoot should find the record before retraction, got %q", got)
|
||||
}
|
||||
|
||||
if err := c.DeleteProperty(uuid, "cids/"+poisonedRoot); err != nil {
|
||||
t.Fatalf("DeleteProperty: %v", err)
|
||||
}
|
||||
|
||||
// 1. the flat key
|
||||
if v, ok, _ := c.Store().Get(context.Background(),
|
||||
recordPrefix(uuid)+"cids/"+poisonedRoot); ok {
|
||||
t.Fatalf("flat key should be gone, still %q", v)
|
||||
}
|
||||
// 2. the record read
|
||||
if hasField(t, c, uuid, "cids/"+poisonedRoot) {
|
||||
t.Fatal("record read still lists the retracted member")
|
||||
}
|
||||
// 3. the reverse alias — the one that used to survive
|
||||
if got := aliasTarget(t, c, poisonedRoot); got != "" {
|
||||
t.Fatalf("reverse alias survived retraction, still points at %q", got)
|
||||
}
|
||||
if got := c.ResolveRoot(poisonedRoot); got == uuid {
|
||||
t.Fatal("a retracted cid must not resolve to the record any more")
|
||||
}
|
||||
|
||||
// The siblings are untouched: retraction is surgical, not a purge.
|
||||
if got := aliasTarget(t, c, goodMidhash); got != uuid {
|
||||
t.Fatalf("an unrelated sibling lost its alias, got %q", got)
|
||||
}
|
||||
if !hasField(t, c, uuid, "cids/"+goodMidhash) {
|
||||
t.Fatal("an unrelated sibling lost its field-index entry")
|
||||
}
|
||||
if v, err := c.GetProperty(uuid, "fileType"); err != nil || v != "video" {
|
||||
t.Fatalf("an unrelated field changed: %q %v", v, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Two records can race to claim one cid; addAliasLocked leaves the first
|
||||
// writer's alias in place. Retracting the loser's member must not unpick the
|
||||
// winner's index — which is why removeAliasLocked compares before deleting.
|
||||
func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
|
||||
withClient(t, func(t *testing.T, c *Client) {
|
||||
const winner = "01KSBHMWINNER"
|
||||
const loser = "01KSBHMLOSER"
|
||||
const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m"
|
||||
|
||||
if err := c.SetMetadataFlat(winner, map[string]string{"cids/" + shared: "true"}); err != nil {
|
||||
t.Fatalf("seed winner: %v", err)
|
||||
}
|
||||
if err := c.SetMetadataFlat(loser, map[string]string{"cids/" + shared: "true"}); err != nil {
|
||||
t.Fatalf("seed loser: %v", err)
|
||||
}
|
||||
if got := aliasTarget(t, c, shared); got != winner {
|
||||
t.Fatalf("the first writer should own the alias, got %q", got)
|
||||
}
|
||||
|
||||
if err := c.DeleteProperty(loser, "cids/"+shared); err != nil {
|
||||
t.Fatalf("DeleteProperty: %v", err)
|
||||
}
|
||||
|
||||
if got := aliasTarget(t, c, shared); got != winner {
|
||||
t.Fatalf("retracting the loser's member stole the winner's alias, now %q", got)
|
||||
}
|
||||
if hasField(t, c, loser, "cids/"+shared) {
|
||||
t.Fatal("the loser's own member should still be gone")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A non-cids field has no alias to clean up, and the delete must not invent one.
|
||||
func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) {
|
||||
withClient(t, func(t *testing.T, c *Client) {
|
||||
const uuid = "01KSBHMORDINARY"
|
||||
if err := c.SetMetadataFlat(uuid, map[string]string{
|
||||
"cids/" + goodMidhash: "true",
|
||||
"quality": "1080p",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := c.DeleteProperty(uuid, "quality"); err != nil {
|
||||
t.Fatalf("DeleteProperty: %v", err)
|
||||
}
|
||||
if got := aliasTarget(t, c, goodMidhash); got != uuid {
|
||||
t.Fatalf("deleting an ordinary field disturbed a cid alias, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
func TestWriteClaimStatuses(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
a := signer(t, 1)
|
||||
good := vote(t, a, "bafyA", "title", "The Matrix", 100)
|
||||
tampered := vote(t, a, "bafyA", "year", "1999", 100)
|
||||
tampered.Value = claim.Text("2000")
|
||||
highS := vote(t, a, "bafyA", "studio", "WB", 100)
|
||||
highS.Sig = flipS(highS.Sig)
|
||||
|
||||
got := write(t, c,
|
||||
good,
|
||||
tampered,
|
||||
highS,
|
||||
claim.Claim{CID: "bafyA", UID: a.UID, TS: 1}, // no key
|
||||
vote(t, a, "bafyA", "cid_sha256", "x", 100), // removed CID shape
|
||||
)
|
||||
want := []ClaimStatus{StatusAccepted, StatusBadSignature, StatusBadSignature, StatusBadRequest, StatusBadRequest}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("statuses %v, want %v", got, want)
|
||||
}
|
||||
if st := write(t, c, good); st[0] != StatusUnchanged {
|
||||
t.Fatalf("identical re-send: %s, want unchanged", st[0])
|
||||
}
|
||||
if st := write(t, c, vote(t, a, "bafyA", "title", "Older", 50)); st[0] != StatusStale {
|
||||
t.Fatalf("older ts: %s, want stale", st[0])
|
||||
}
|
||||
if st := write(t, c, vote(t, a, "bafyA", "title", "Same ts other value", 100)); st[0] != StatusStale {
|
||||
t.Fatalf("equal ts, different claim: %s, want stale", st[0])
|
||||
}
|
||||
if got := record(t, c, "bafyA")["title"]; got != "The Matrix" {
|
||||
t.Fatalf("title = %q", got)
|
||||
}
|
||||
// A re-persist of the same value with a newer ts keeps the stored row (and
|
||||
// its older ts): it is "unchanged", not a new version.
|
||||
if st := write(t, c, vote(t, a, "bafyA", "title", "The Matrix", 200)); st[0] != StatusUnchanged {
|
||||
t.Fatalf("same value, newer ts: %s, want unchanged", st[0])
|
||||
}
|
||||
// Proof the stored ts stayed 100: a new value at 150 (< 200) is accepted.
|
||||
if st := write(t, c, vote(t, a, "bafyA", "title", "Updated", 150)); st[0] != StatusAccepted {
|
||||
t.Fatalf("new value at ts 150 after a same-value re-send at 200: %s, want accepted", st[0])
|
||||
}
|
||||
}
|
||||
|
||||
func flipS(sig []byte) []byte {
|
||||
// n - s for secp256k1, computed via the vectors' helper in internal/claim
|
||||
// would be cleaner; here big-endian subtraction on bytes is enough.
|
||||
n := []byte{
|
||||
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
|
||||
0xba, 0xae, 0xdc, 0xe6, 0xaf, 0x48, 0xa0, 0x3b, 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x41,
|
||||
}
|
||||
out := append([]byte{}, sig...)
|
||||
borrow := 0
|
||||
for i := 31; i >= 0; i-- {
|
||||
d := int(n[i]) - int(sig[32+i]) - borrow
|
||||
borrow = 0
|
||||
if d < 0 {
|
||||
d += 256
|
||||
borrow = 1
|
||||
}
|
||||
out[32+i] = byte(d)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A retraction is kept as a tombstone, so the vote it retracted cannot be
|
||||
// replayed — by its signer's old copy or by a peer that still holds it.
|
||||
func TestRetractionKeepsTombstoneAndBlocksReplay(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
a := signer(t, 1)
|
||||
old := vote(t, a, "bafyA", "genres/Drama", "true", 10)
|
||||
mustAccept(t, c, old, vote(t, a, "bafyA", "title", "X", 10))
|
||||
mustAccept(t, c, vote(t, a, "bafyA", "genres/Drama", nil, 20))
|
||||
if _, ok := record(t, c, "bafyA")["genres/Drama"]; ok {
|
||||
t.Fatal("retracted member still merged")
|
||||
}
|
||||
if st := write(t, c, old); st[0] != StatusStale {
|
||||
t.Fatalf("replay of the retracted vote: %s, want stale", st[0])
|
||||
}
|
||||
if n, _ := c.CountClaims(); n != 2 {
|
||||
t.Fatalf("claims = %d, want 2 (tombstone kept)", n)
|
||||
}
|
||||
live, _ := c.ClaimsForCIDs([]string{"bafyA"}, "")
|
||||
if len(live) != 1 || live[0].Key != "title" {
|
||||
t.Fatalf("live claims = %+v, want only title", live)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeRule(t *testing.T) {
|
||||
tx := func(s string) claim.Value { return claim.Text(s) }
|
||||
cases := []struct {
|
||||
name string
|
||||
votes []Vote
|
||||
viewer string
|
||||
want string
|
||||
nsrc int
|
||||
ok bool
|
||||
}{
|
||||
{"support wins", []Vote{{Uid: "a", Value: tx("X"), TS: 9}, {Uid: "b", Value: tx("x "), TS: 9}, {Uid: "c", Value: tx("Y"), TS: 1}}, "", "X", 2, true},
|
||||
{"tie: oldest wins", []Vote{{Uid: "a", Value: tx("New"), TS: 9}, {Uid: "b", Value: tx("Old"), TS: 1}}, "", "Old", 1, true},
|
||||
{"tie: smallest wins", []Vote{{Uid: "a", Value: tx("b"), TS: 5}, {Uid: "b", Value: tx("a"), TS: 5}}, "", "a", 1, true},
|
||||
{"signer counts once, newest", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: tx("Y"), TS: 2}, {Uid: "b", Value: tx("Y"), TS: 3}}, "", "Y", 2, true},
|
||||
{"retraction drops", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "a", Value: claim.Null(), TS: 2}}, "", "", 0, false},
|
||||
{"ciphertext never merges", []Vote{{Uid: "a", Value: claim.Bytes([]byte{1}), TS: 1}}, "", "", 0, false},
|
||||
{"viewer wins", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "b", Value: tx("X"), TS: 1}, {Uid: "v", Value: tx("Mine"), TS: 9}}, "v", "Mine", 1, true},
|
||||
{"viewer retraction does not hide", []Vote{{Uid: "a", Value: tx("X"), TS: 1}, {Uid: "v", Value: claim.Null(), TS: 9}}, "v", "X", 1, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := Merge(tc.votes, tc.viewer)
|
||||
if got.OK != tc.ok || got.Value != tc.want || (tc.ok && got.Nsrc != tc.nsrc) {
|
||||
t.Errorf("%s: got %+v, want %q nsrc %d ok %v", tc.name, got, tc.want, tc.nsrc, tc.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedViewAndClaims(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
gwA, gwB, gwC := signer(t, 1), signer(t, 2), signer(t, 3)
|
||||
mustAccept(t, c,
|
||||
vote(t, gwA, "bafyA", "title", "Matrix", 10),
|
||||
vote(t, gwB, "bafyA", "title", "Matrix", 20),
|
||||
vote(t, gwC, "bafyA", "title", "The Matrix (1999)", 5),
|
||||
)
|
||||
rec := record(t, c, "bafyA")
|
||||
if rec["title"] != "Matrix" || rec["cids/bafyA"] != "true" {
|
||||
t.Fatalf("merged %v", rec)
|
||||
}
|
||||
claims, err := c.ClaimsForRID(rid(t, c, "bafyA"), "title")
|
||||
if err != nil || len(claims) != 3 {
|
||||
t.Fatalf("claims %d (err %v), want 3", len(claims), err)
|
||||
}
|
||||
for _, cl := range claims {
|
||||
if err := cl.Verify(); err != nil {
|
||||
t.Fatalf("returned claim does not verify: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cids/* link merges two groups: the older rid survives, the loser's rows
|
||||
// go with `del` events, and the survivor carries both groups' votes.
|
||||
func TestGroupLinkMergesRecords(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
a, b := signer(t, 1), signer(t, 2)
|
||||
mustAccept(t, c, vote(t, a, "bafyM", "title", "Local copy", 10))
|
||||
older := rid(t, c, "bafyM")
|
||||
mustAccept(t, c, vote(t, b, "bafyQ", "year", "1999", 10))
|
||||
newer := rid(t, c, "bafyQ")
|
||||
if older == newer || older > newer {
|
||||
t.Fatalf("expected two groups, older first: %s %s", older, newer)
|
||||
}
|
||||
before, _ := c.Store().LastID(context.Background(), MetaEventsStream)
|
||||
|
||||
mustAccept(t, c, vote(t, a, "bafyM", "cids/bafyQ", "true", 11))
|
||||
if got := rid(t, c, "bafyQ"); got != older {
|
||||
t.Fatalf("bafyQ → %s, want the older rid %s", got, older)
|
||||
}
|
||||
rec := record(t, c, "bafyM")
|
||||
if rec["title"] != "Local copy" || rec["year"] != "1999" || rec["cids/bafyQ"] != "true" || rec["cids/bafyM"] != "true" {
|
||||
t.Fatalf("survivor %v", rec)
|
||||
}
|
||||
if f, _ := c.GetMetadataFlat(newer); f != nil {
|
||||
t.Fatalf("loser still has rows: %v", f)
|
||||
}
|
||||
entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 1000)
|
||||
sawDel := false
|
||||
for _, e := range entries {
|
||||
if e.Values["type"] == "del" && strings.HasPrefix(e.Values["key"], "file:"+newer+"/") {
|
||||
sawDel = true
|
||||
}
|
||||
}
|
||||
if !sawDel {
|
||||
t.Fatal("no del event for the losing rid")
|
||||
}
|
||||
if n, _ := c.CountRoots(); n != 1 {
|
||||
t.Fatalf("roots = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewerOverlay(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
a, b, me := signer(t, 1), signer(t, 2), signer(t, 9)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafyA", "title", "Shared", 1),
|
||||
vote(t, b, "bafyA", "title", "Shared", 1),
|
||||
vote(t, me, "bafyA", "title", "Mine", 2),
|
||||
vote(t, me, "bafyA", "seek", []byte{1, 2, 3}, 2), // ciphertext: never shown
|
||||
)
|
||||
r := rid(t, c, "bafyA")
|
||||
shared, _ := c.GetMetadataFlat(r)
|
||||
mine, _ := c.GetMetadataFlatAs(r, me.UID)
|
||||
if shared["title"] != "Shared" || mine["title"] != "Mine" {
|
||||
t.Fatalf("shared %q, mine %q", shared["title"], mine["title"])
|
||||
}
|
||||
if _, ok := mine["seek"]; ok {
|
||||
t.Fatal("ciphertext leaked into the view")
|
||||
}
|
||||
match := func(_ string, f map[string]string) bool { return strings.Contains(strings.ToLower(f["title"]), "mine") }
|
||||
if hits, _ := c.Search("", nil, 10, match); len(hits) != 0 {
|
||||
t.Fatalf("shared search found the viewer's value: %v", hits)
|
||||
}
|
||||
hits, err := c.SearchAs("", nil, 10, me.UID, match)
|
||||
if err != nil || len(hits) != 1 {
|
||||
t.Fatalf("viewer search hits %v (err %v), want 1", hits, err)
|
||||
}
|
||||
merged, _, dropped, err := c.MergeClaims([]claim.Claim{
|
||||
vote(t, a, "bafyA", "title", "Shared", 1),
|
||||
vote(t, b, "bafyZ", "title", "Other", 1),
|
||||
}, me.UID)
|
||||
if err != nil || dropped != 0 || merged["title"] != "Mine" {
|
||||
t.Fatalf("stateless merge as viewer = %v dropped %d err %v", merged, dropped, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A crash between the truth commit and the cache apply leaves the cache
|
||||
// behind; the next boot replays the missed changes from truth.
|
||||
func TestCrashBetweenCommitsIsCaughtUp(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c := openClient(t, dir)
|
||||
a := signer(t, 1)
|
||||
mustAccept(t, c, vote(t, a, "bafyA", "title", "Before", 1))
|
||||
|
||||
// Commit straight to truth, as WriteClaims does, but never apply.
|
||||
lost := vote(t, a, "bafyA", "title", "After", 2)
|
||||
if err := c.truth.Update(context.Background(), func(tx *kv.Tx) error {
|
||||
seq, err := readLastSeq(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pid, err := ensurePrincipal(tx, lost.UID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seq++
|
||||
if _, err := tx.Exec(`UPDATE claim SET value = ?, ts = ?, sig = ?, seq = ? WHERE cid = ? AND key = ? AND pid = ?`,
|
||||
"After", int64(lost.TS), lost.Sig, seq, lost.CID, lost.Key, pid); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeLastSeq(tx, seq)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := record(t, c, "bafyA")["title"]; got != "Before" {
|
||||
t.Fatalf("cache applied a change it was never given: %q", got)
|
||||
}
|
||||
c.Close()
|
||||
|
||||
c = openClient(t, dir)
|
||||
defer c.Close()
|
||||
if got := record(t, c, "bafyA")["title"]; got != "After" {
|
||||
t.Fatalf("after catch-up title = %q, want After", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The cache is a pure function of truth: deleting it and rebuilding yields
|
||||
// exactly what incremental application produced.
|
||||
func TestRebuildEqualsIncremental(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c := openClient(t, dir)
|
||||
a, b, u := signer(t, 1), signer(t, 2), signer(t, 3)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafy1", "title", "One", 1), vote(t, a, "bafy1", "genres/Action", "true", 1),
|
||||
vote(t, b, "bafy1", "title", "Uno", 2), vote(t, b, "bafy2", "title", "Two", 2),
|
||||
vote(t, a, "bafy2", "sizeByte", "123", 1), vote(t, u, "bafy2", "like", "true", 3),
|
||||
vote(t, u, "bafy2", "seek", []byte{9}, 3),
|
||||
)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafy1", "genres/Action", nil, 4), // retraction
|
||||
vote(t, b, "bafy1", "cids/bafy2", "true", 5), // group merge
|
||||
vote(t, a, "bafy3", "title", "Three", 5),
|
||||
)
|
||||
incremental := cacheState(t, c)
|
||||
c.Close()
|
||||
for _, f := range []string{"cache.db", "cache.db-wal", "cache.db-shm"} {
|
||||
_ = os.Remove(filepath.Join(dir, f))
|
||||
}
|
||||
c = openClient(t, dir)
|
||||
defer c.Close()
|
||||
if rebuilt := cacheState(t, c); !reflect.DeepEqual(rebuilt, incremental) {
|
||||
t.Fatalf("rebuild differs from incremental:\n incremental %v\n rebuilt %v", incremental, rebuilt)
|
||||
}
|
||||
if c.RebuiltAt() == "" {
|
||||
t.Fatal("rebuild did not record rebuilt_at (SSE consumers would not get a gap)")
|
||||
}
|
||||
}
|
||||
|
||||
// cacheState is every derived row that must be a function of truth.
|
||||
func cacheState(t *testing.T, c *Client) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
for _, k := range allKeys(t, c) {
|
||||
if strings.HasPrefix(k, "ev:") || strings.HasPrefix(k, "evhwm:") || strings.HasPrefix(k, "meta:") ||
|
||||
strings.HasPrefix(k, ftsNS) || k == countersReady {
|
||||
continue
|
||||
}
|
||||
v, _, _ := c.Store().Get(context.Background(), k)
|
||||
out = append(out, k+"="+v)
|
||||
}
|
||||
rows, err := c.Store().DB().Query(`SELECT pid, cid, key FROM claim_by_pid`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var pid int64
|
||||
var cid, key string
|
||||
_ = rows.Scan(&pid, &cid, &key)
|
||||
out = append(out, "byp:"+strconv.FormatInt(pid, 10)+"/"+cid+"/"+key)
|
||||
}
|
||||
var hays []string
|
||||
hr, _ := c.Store().DB().Query(`SELECT rid, hay FROM fts`)
|
||||
for hr.Next() {
|
||||
var r, h string
|
||||
_ = hr.Scan(&r, &h)
|
||||
hays = append(hays, "fts:"+r+"="+h)
|
||||
}
|
||||
hr.Close()
|
||||
out = append(out, hays...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func TestLocalFileFactsAndEvents(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
before, _ := c.Store().LastID(context.Background(), MetaEventsStream)
|
||||
r, created, err := c.RegisterFileTuple("/files/watch/a.mkv", 1000, 42, "bafkmid")
|
||||
if err != nil || !created {
|
||||
t.Fatalf("register: %v created=%v", err, created)
|
||||
}
|
||||
rec, _ := c.GetMetadataFlat(r)
|
||||
if rec["filePath"] != "/files/watch/a.mkv" || rec["sizeByte"] != "1000" || rec["mtimeNano"] != "42" || rec["cids/bafkmid"] != "true" {
|
||||
t.Fatalf("record %v", rec)
|
||||
}
|
||||
entries, _ := c.Store().ReadAfter(context.Background(), MetaEventsStream, before, 100)
|
||||
sawPath := false
|
||||
for _, e := range entries {
|
||||
if e.Values["type"] == "set" && e.Values["key"] == "file:"+r+"/filePath" {
|
||||
sawPath = true
|
||||
}
|
||||
}
|
||||
if !sawPath {
|
||||
t.Fatal("no set event for file:<rid>/filePath (meta-fuse builds its tree from it)")
|
||||
}
|
||||
// A signed sizeByte (content fact) wins over the on-disk size.
|
||||
mustAccept(t, c, vote(t, signer(t, 1), "bafkmid", "sizeByte", "999", 1))
|
||||
if got, _ := c.GetProperty(r, "sizeByte"); got != "999" {
|
||||
t.Fatalf("sizeByte = %q, want the signed 999", got)
|
||||
}
|
||||
// A second path of the same content is a duplicate.
|
||||
if _, _, err := c.RegisterFileTuple("/files/watch/copy.mkv", 1000, 43, "bafkmid"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc, _ := c.GetMetadataDocument(r)
|
||||
if len(doc.Duplicates) != 1 || doc.Duplicates[0] != "/files/watch/copy.mkv" {
|
||||
t.Fatalf("duplicates %v", doc.Duplicates)
|
||||
}
|
||||
tuples, _ := c.GetTuplesForAllFiles()
|
||||
if len(tuples) != 2 || tuples[0].HashID != r {
|
||||
t.Fatalf("tuples %+v", tuples)
|
||||
}
|
||||
// Deleting the files drops the local facts, keeps the claims.
|
||||
mark, _ := c.Store().LastID(context.Background(), MetaEventsStream)
|
||||
for _, p := range []string{"/files/watch/a.mkv", "/files/watch/copy.mkv"} {
|
||||
if err := c.RemoveFileTuple(p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rec, _ = c.GetMetadataFlat(r)
|
||||
if _, ok := rec["filePath"]; ok || rec["sizeByte"] != "999" {
|
||||
t.Fatalf("after delete %v", rec)
|
||||
}
|
||||
entries, _ = c.Store().ReadAfter(context.Background(), MetaEventsStream, mark, 100)
|
||||
sawDel := false
|
||||
for _, e := range entries {
|
||||
if e.Values["type"] == "del" && e.Values["key"] == "file:"+r+"/filePath" {
|
||||
sawDel = true
|
||||
}
|
||||
}
|
||||
if !sawDel {
|
||||
t.Fatal("no del event for filePath")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaGateRefusesV1Store(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "meta-core.db")
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID;
|
||||
INSERT INTO kv VALUES('meta-core:schema-version','2')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
c := NewClient()
|
||||
if err := c.Connect(path, filepath.Join(dir, "cache.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
if err := c.EnsureSchemaVersion(SchemaVersion); err == nil || !strings.Contains(err.Error(), "v1.x") {
|
||||
t.Fatalf("v1 store accepted: %v", err)
|
||||
}
|
||||
}
|
||||
+41
-118
@@ -1,79 +1,27 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// newTestClient opens a fresh database in the test's temp dir.
|
||||
func newTestClient(t *testing.T) *Client {
|
||||
t.Helper()
|
||||
c := NewClient()
|
||||
if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = c.Close() })
|
||||
return c
|
||||
}
|
||||
|
||||
// allKeys lists every row (test helper; full scan).
|
||||
func allKeys(t *testing.T, c *Client) []string {
|
||||
t.Helper()
|
||||
var keys []string
|
||||
if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
|
||||
for _, r := range rows {
|
||||
keys = append(keys, r.K)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
|
||||
// leak: ClearAllMetadata used to delete every record key and the root index,
|
||||
// but left the cid:<cid> → root reverse-index entries behind. Those stale
|
||||
// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry
|
||||
// silently redirects a later write into a deleted root.
|
||||
func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
|
||||
// ClearAllMetadata wipes BOTH databases: votes, signers and groups in truth;
|
||||
// merged records, the cid lookup, roots, search rows, the claim index and the
|
||||
// local files in the cache. Service scratch keys are not metadata and survive.
|
||||
func TestClearAllMetadataWipesTruthAndCache(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
// Two records, each with several CID aliases (a midhash plus siblings) —
|
||||
// exactly what the watcher produces.
|
||||
aliases := map[string][]string{}
|
||||
for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
|
||||
uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
|
||||
if err != nil {
|
||||
t.Fatalf("mint: %v", err)
|
||||
}
|
||||
cids := []string{"bafkmid" + uuid, "bafkfull" + uuid}
|
||||
for _, cidStr := range cids {
|
||||
if err := c.AddAlias(uuid, cidStr); err != nil {
|
||||
t.Fatalf("add alias %s: %v", cidStr, err)
|
||||
}
|
||||
}
|
||||
if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil {
|
||||
t.Fatalf("add duplicate: %v", err)
|
||||
}
|
||||
aliases[uuid] = cids
|
||||
a := signer(t, 1)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafkone", "title", "One", 1),
|
||||
vote(t, a, "bafkone", "cids/bafkonefull", "true", 1),
|
||||
vote(t, a, "bafktwo", "title", "Two", 1),
|
||||
)
|
||||
if _, _, err := c.RegisterFileTuple("/files/watch/a.mkv", 10, 20, "bafkmid"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// An entry already orphaned by a previous (buggy) clear: a reverse-index
|
||||
// key whose root no longer exists. The sweep is expected to collect it too.
|
||||
if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil {
|
||||
t.Fatalf("seed orphan: %v", err)
|
||||
}
|
||||
|
||||
for uuid, cids := range aliases {
|
||||
for _, cidStr := range cids {
|
||||
if got := c.ResolveRoot(cidStr); got != uuid {
|
||||
t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
|
||||
}
|
||||
for k, v := range map[string]string{"btmagnet:abc": "magnet:?xt=urn:btih:abc", "searchcov:tmdb:1": "1"} {
|
||||
if err := c.SetServiceValue(k, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,72 +29,47 @@ func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ClearAllMetadata: %v", err)
|
||||
}
|
||||
if deleted != 2 {
|
||||
t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
|
||||
if deleted != 3 {
|
||||
t.Errorf("deleted = %d, want 3 records", deleted)
|
||||
}
|
||||
|
||||
// Nothing record-shaped may survive: not the records, not their aliases,
|
||||
// duplicates or search rows, not the pre-existing orphan. (The event
|
||||
// streams and the schema sentinel are not metadata and stay.)
|
||||
for _, k := range allKeys(t, c) {
|
||||
for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} {
|
||||
for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
|
||||
if strings.HasPrefix(k, ns) {
|
||||
t.Errorf("key survived the clear: %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
var ftsRows int
|
||||
if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 {
|
||||
t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err)
|
||||
}
|
||||
|
||||
for _, cids := range aliases {
|
||||
for _, cidStr := range cids {
|
||||
if got := c.ResolveRoot(cidStr); got != cidStr {
|
||||
t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
|
||||
}
|
||||
for _, table := range []string{"fts", "claim_by_pid", "local_file"} {
|
||||
var n int
|
||||
if err := c.Store().DB().QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil || n != 0 {
|
||||
t.Errorf("%s rows after clear = %d (err %v)", table, n, err)
|
||||
}
|
||||
}
|
||||
if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
|
||||
t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
|
||||
if n, err := c.CountClaims(); err != nil || n != 0 {
|
||||
t.Errorf("claims after clear = %d (err %v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
|
||||
// keys outside the metadata namespaces (magnet cache, coverage markers, a key
|
||||
// that merely CONTAINS "cid:") must survive a metadata wipe.
|
||||
func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
uuid, err := c.Mint("watch/a.mkv", 100, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("mint: %v", err)
|
||||
if got := c.ResolveRoot("bafkone"); got != "bafkone" {
|
||||
t.Errorf("post-clear ResolveRoot = %q, want the hash unchanged", got)
|
||||
}
|
||||
if err := c.AddAlias(uuid, "bafkmine"); err != nil {
|
||||
t.Fatalf("add alias: %v", err)
|
||||
}
|
||||
for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} {
|
||||
if err := c.SetRawValue(k, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := c.ClearAllMetadata(); err != nil {
|
||||
t.Fatalf("ClearAllMetadata: %v", err)
|
||||
}
|
||||
|
||||
for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} {
|
||||
for _, k := range []string{"btmagnet:abc", "searchcov:tmdb:1"} {
|
||||
if _, _, ok, _ := c.GetKeyInfo(k); !ok {
|
||||
t.Errorf("clear deleted foreign key %q", k)
|
||||
t.Errorf("clear deleted service key %q", k)
|
||||
}
|
||||
}
|
||||
if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok {
|
||||
t.Errorf("own reverse-index entry survived the clear")
|
||||
// The store keeps working after a wipe.
|
||||
mustAccept(t, c, vote(t, a, "bafkone", "title", "Again", 1))
|
||||
if got := record(t, c, "bafkone")["title"]; got != "Again" {
|
||||
t.Errorf("title after re-write = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// withClient runs fn against a fresh client.
|
||||
func withClient(t *testing.T, fn func(t *testing.T, c *Client)) {
|
||||
t.Helper()
|
||||
fn(t, newTestClient(t))
|
||||
// Raw writes are limited to service scratch keys: merged records and
|
||||
// bookkeeping cannot be written around the claims.
|
||||
func TestServiceValueRefusesRecordKeys(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
for _, k := range []string{"file:R1/title", "cid:bafk", "idx:roots/R1", "ev:meta:events/1", "meta:applied_seq"} {
|
||||
if err := c.SetServiceValue(k, "x"); err == nil {
|
||||
t.Errorf("SetServiceValue(%q) accepted", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+214
-394
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,21 +15,31 @@ import (
|
||||
"github.com/metazla/meta-core/internal/schema"
|
||||
)
|
||||
|
||||
// Key layout (everything is one string→string table, see internal/kv):
|
||||
// meta-core keeps two SQLite files (docs/implementation/metacore-claims-model.md):
|
||||
//
|
||||
// file:<rid>/<field> a record property — the ONLY thing under `file:`
|
||||
// cid:<cid> reverse index → <rid>
|
||||
// idx:roots/<rid> the set of every root (was the Redis SET file:__index__)
|
||||
// idx:fts/<rid> rowid of the record's row in the FTS table
|
||||
// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
|
||||
// dup:<rid>/<path> a duplicate path of the record (was the SET file:<rid>/duplicates)
|
||||
// udl:… User Data Layer, see udl.go
|
||||
// ev:<stream>/…, evhwm:… event streams, see internal/kv/stream.go
|
||||
// - the TRUTH database (truth.go): principal, claim, cid_group, meta — the
|
||||
// signed votes and the locally minted rid names, primary keys only, backed
|
||||
// up;
|
||||
//
|
||||
// - the CACHE database: everything derived from truth, never backed up,
|
||||
// rebuilt when missing (apply.go). Its kv table keeps the v1 layout:
|
||||
//
|
||||
// file:<rid>/<field> the MERGED record view — the ONLY thing under `file:`
|
||||
// cid:<cid> cid → rid lookup (copy of truth's cid_group)
|
||||
// idx:roots/<rid> the set of every record
|
||||
// idx:fts/<rid> rowid of the record's row in the FTS table
|
||||
// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
|
||||
// ev:<stream>/…, evhwm:… event streams, see internal/kv/stream.go
|
||||
// meta:applied_seq, meta:rebuilt_at apply-loop bookkeeping
|
||||
// anything else service scratch KV (btmagnet:*, searchcov:*), /api/kv/value
|
||||
//
|
||||
// plus the tables claim_by_pid and local_file (localfile.go), and the
|
||||
// field index + FTS (search.go).
|
||||
//
|
||||
// Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is
|
||||
// exactly the records, so a record is the key range [file:<rid>/, file:<rid>0),
|
||||
// the field expression index covers only real fields, and every change under
|
||||
// `file:` is a real field change worth publishing on meta:events.
|
||||
// exactly the merged records, so a record is the key range [file:<rid>/,
|
||||
// file:<rid>0), the field expression index covers only real fields, and every
|
||||
// change under `file:` is a real field change worth publishing on meta:events.
|
||||
const (
|
||||
recordNS = "file:"
|
||||
rootsNS = "idx:roots/"
|
||||
@@ -43,10 +52,9 @@ const (
|
||||
readerPage = 5_000
|
||||
)
|
||||
|
||||
// MetaEventsStream receives a set/del entry for every record-key change. It is
|
||||
// written by the storage engine in the same transaction as the change (the
|
||||
// replacement for Redis keyspace notifications), and served as SSE by
|
||||
// /api/events/meta.
|
||||
// MetaEventsStream receives a set/del entry for every merged-record change. It
|
||||
// is written by the cache engine in the same transaction as the change, and
|
||||
// served as SSE by /api/events/meta.
|
||||
const MetaEventsStream = "meta:events"
|
||||
|
||||
// MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is
|
||||
@@ -54,10 +62,13 @@ const MetaEventsStream = "meta:events"
|
||||
// "Backing-store retention and lifecycle".
|
||||
const MetaEventsMaxLen = 100_000
|
||||
|
||||
// Client is meta-core's metadata store.
|
||||
// Client is meta-core's metadata store: the truth database, the cache, and
|
||||
// the apply loop between them.
|
||||
type Client struct {
|
||||
store *kv.Store
|
||||
mu sync.RWMutex
|
||||
store *kv.Store // cache
|
||||
truth *kv.Store
|
||||
applier *applier
|
||||
mu sync.RWMutex
|
||||
|
||||
keysMu sync.Mutex
|
||||
keysCount int64
|
||||
@@ -69,60 +80,98 @@ func NewClient() *Client {
|
||||
return &Client{}
|
||||
}
|
||||
|
||||
// Connect opens (creating if needed) the database file at path.
|
||||
func (c *Client) Connect(path string) error {
|
||||
// Connect opens (creating if needed) the truth database at truthPath and the
|
||||
// cache at cachePath. Call EnsureSchemaVersion, then Start.
|
||||
func (c *Client) Connect(truthPath, cachePath string) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
store, err := kv.Open(path, kv.Options{
|
||||
truth, err := kv.Open(truthPath, kv.Options{
|
||||
NoKVTable: true,
|
||||
Schema: truthSchema,
|
||||
WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
|
||||
ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
|
||||
Readers: envInt("META_CORE_DB_READERS", 8),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("open truth %s: %w", truthPath, err)
|
||||
}
|
||||
store, err := kv.Open(cachePath, kv.Options{
|
||||
EventStream: MetaEventsStream,
|
||||
EventMaxLen: MetaEventsMaxLen,
|
||||
EventKey: func(k string) bool { return strings.HasPrefix(k, recordNS) },
|
||||
WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
|
||||
ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
|
||||
Readers: envInt("META_CORE_DB_READERS", 8),
|
||||
Schema: searchSchema,
|
||||
Schema: searchSchema + cacheTablesSchema,
|
||||
OnChange: onChange,
|
||||
Synchronous: "OFF",
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("open %s: %w", path, err)
|
||||
truth.Close()
|
||||
return fmt.Errorf("open cache %s: %w", cachePath, err)
|
||||
}
|
||||
c.store = store
|
||||
log.Printf("[Storage] Opened %s", path)
|
||||
c.truth, c.store = truth, store
|
||||
log.Printf("[Storage] Opened truth %s, cache %s", truthPath, cachePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
// Start brings the cache level with truth (replay or rebuild) and starts the
|
||||
// apply loop. Writes before Start fail.
|
||||
func (c *Client) Start() error {
|
||||
applied, err := c.catchUp()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cache catch-up: %w", err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.applier = newApplier(c, applied)
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close stops the apply loop and closes both databases.
|
||||
func (c *Client) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.store == nil {
|
||||
return nil
|
||||
if c.applier != nil {
|
||||
c.applier.close()
|
||||
c.applier = nil
|
||||
}
|
||||
err := c.store.Close()
|
||||
c.store = nil
|
||||
return err
|
||||
var errs []error
|
||||
if c.store != nil {
|
||||
errs = append(errs, c.store.Close())
|
||||
c.store = nil
|
||||
}
|
||||
if c.truth != nil {
|
||||
errs = append(errs, c.truth.Close())
|
||||
c.truth = nil
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// IsConnected reports whether the database is open.
|
||||
// IsConnected reports whether the databases are open.
|
||||
func (c *Client) IsConnected() bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.store != nil
|
||||
return c.store != nil && c.truth != nil
|
||||
}
|
||||
|
||||
// Health checks that the database answers.
|
||||
// Health checks that both databases answer.
|
||||
func (c *Client) Health() bool {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t, err := c.truthStore()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
return s.Ping(ctx) == nil
|
||||
return s.Ping(ctx) == nil && t.Ping(ctx) == nil
|
||||
}
|
||||
|
||||
// Store exposes the underlying engine (event streams for SSE and the schema
|
||||
// Store exposes the cache engine (event streams for SSE and the schema
|
||||
// indexer).
|
||||
func (c *Client) Store() *kv.Store {
|
||||
c.mu.RLock()
|
||||
@@ -139,7 +188,16 @@ func (c *Client) kv() (*kv.Store, error) {
|
||||
return c.store, nil
|
||||
}
|
||||
|
||||
// update runs fn in one write transaction with the standard timeout.
|
||||
func (c *Client) truthStore() (*kv.Store, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.truth == nil || c.applier == nil {
|
||||
return nil, fmt.Errorf("not connected")
|
||||
}
|
||||
return c.truth, nil
|
||||
}
|
||||
|
||||
// update runs fn in one cache write transaction with the standard timeout.
|
||||
func (c *Client) update(fn func(tx *kv.Tx) error) error {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
@@ -184,8 +242,92 @@ func splitRecordKey(key string) (hashID, field string, ok bool) {
|
||||
// isInternalKey reports keys that are derived bookkeeping, which the raw KV
|
||||
// editor must never write (it would desynchronise the derived state).
|
||||
func isInternalKey(key string) bool {
|
||||
return strings.HasPrefix(key, "idx:") || strings.HasPrefix(key, dupNS) ||
|
||||
strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS)
|
||||
for _, p := range []string{recordNS, CIDIndexPrefix, "idx:", dupNS, streamNS, hwmNS, "meta:", "meta-core:"} {
|
||||
if strings.HasPrefix(key, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetServiceValue writes service scratch KV (btmagnet:*, searchcov:*) — the
|
||||
// only raw key writes left. Merged records and bookkeeping are refused.
|
||||
func (c *Client) SetServiceValue(key, value string) error {
|
||||
if key == "" || isInternalKey(key) {
|
||||
return fmt.Errorf("key %q is not a service key", key)
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error { return tx.Put(key, value) })
|
||||
}
|
||||
|
||||
// DeleteServiceValue deletes a service scratch key. Returns true if it existed.
|
||||
func (c *Client) DeleteServiceValue(key string) (bool, error) {
|
||||
if key == "" || isInternalKey(key) {
|
||||
return false, fmt.Errorf("key %q is not a service key", key)
|
||||
}
|
||||
existed := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
existed, err = tx.Delete(key)
|
||||
return err
|
||||
})
|
||||
return existed, err
|
||||
}
|
||||
|
||||
// ClearAllMetadata wipes BOTH databases: every claim, signer and group in
|
||||
// truth, and every derived row of the cache. Irreversible — truth holds votes
|
||||
// nothing else can recover. Returns the number of records that existed.
|
||||
func (c *Client) ClearAllMetadata() (int64, error) {
|
||||
truth, err := c.truthStore()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
roots, err := c.CountRoots()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
if err := truth.Update(ctx, func(tx *kv.Tx) error {
|
||||
for _, t := range []string{"claim", "principal", "cid_group"} {
|
||||
if _, err := tx.Exec(`DELETE FROM ` + t); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := tx.Exec(`DELETE FROM meta WHERE name = ?`, lastSeqName)
|
||||
return err
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("clear truth: %w", err)
|
||||
}
|
||||
for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
|
||||
for {
|
||||
var n int64
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
n, err = tx.DeletePrefix(prefix, bulkBatch)
|
||||
return err
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("clear %s: %w", prefix, err)
|
||||
}
|
||||
if n < bulkBatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
for _, q := range []string{`DELETE FROM fts`, `DELETE FROM claim_by_pid`, `DELETE FROM local_file`} {
|
||||
if _, err := tx.Exec(q); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Put(appliedSeqKey, "0")
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("clear cache: %w", err)
|
||||
}
|
||||
c.applier.mu.Lock()
|
||||
c.applier.applied = 0
|
||||
c.applier.mu.Unlock()
|
||||
log.Printf("[Storage] Cleared both databases (%d records)", roots)
|
||||
return roots, nil
|
||||
}
|
||||
|
||||
// Derived counters, maintained by onChange in the same transaction as the
|
||||
@@ -300,44 +442,6 @@ func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]str
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// writeFieldsTx is the one write path for record fields: the keys, the root
|
||||
// index, the CID reverse aliases and the search row, all in the caller's
|
||||
// transaction.
|
||||
func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
|
||||
if hashID == "" {
|
||||
return fmt.Errorf("empty hash id")
|
||||
}
|
||||
prefix := recordPrefix(hashID)
|
||||
searchChanged := false
|
||||
for field, value := range metadata {
|
||||
if isReservedFieldName(field) {
|
||||
continue
|
||||
}
|
||||
changed, err := tx.Set(prefix+field, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
searchChanged = searchChanged || (changed && affectsHaystack(field))
|
||||
}
|
||||
newRoot, err := tx.Set(rootKey(hashID), "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for field := range metadata {
|
||||
if cidStr := cidFromKeysetField(field); cidStr != "" {
|
||||
if err := addAliasTx(tx, hashID, cidStr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// The FTS row is rewritten only when the haystack can have changed — a
|
||||
// re-persisted record whose values are all in place costs reads only.
|
||||
if searchChanged || newRoot {
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetMetadataFlat retrieves all metadata for a file as a flat map.
|
||||
func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
|
||||
s, err := c.kv()
|
||||
@@ -349,28 +453,6 @@ func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
|
||||
return readRecord(ctx, s, hashID)
|
||||
}
|
||||
|
||||
// SetMetadataFlat stores metadata for a file using flat keys (merge: fields
|
||||
// absent from metadata are kept — the Redis implementation never cleared
|
||||
// either).
|
||||
func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) })
|
||||
}
|
||||
|
||||
// MergeMetadataFlat merges new metadata into existing (PATCH semantics): new
|
||||
// keys are added, existing keys are updated, missing keys are NOT deleted.
|
||||
func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) {
|
||||
if len(metadata) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(metadata), nil
|
||||
}
|
||||
|
||||
// GetAllHashIDs returns every root id, sorted.
|
||||
func (c *Client) GetAllHashIDs() ([]string, error) {
|
||||
s, err := c.kv()
|
||||
@@ -401,57 +483,6 @@ func (c *Client) GetProperty(hashID, property string) (string, error) {
|
||||
return v, err
|
||||
}
|
||||
|
||||
// SetProperty sets a single property value.
|
||||
func (c *Client) SetProperty(hashID, property, value string) error {
|
||||
if isReservedFieldName(property) {
|
||||
return fmt.Errorf("field name %q is reserved", property)
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
return writeFieldsTx(tx, hashID, map[string]string{property: value})
|
||||
})
|
||||
}
|
||||
|
||||
// deleteRecordTx removes everything keyed off hashID: its properties, the
|
||||
// reverse-index entries its cids/ members own, its duplicates, its root-index
|
||||
// entry and its search row. Returns the number of property keys removed.
|
||||
func deleteRecordTx(tx *kv.Tx, hashID string) (int64, error) {
|
||||
ctx := tx.Context()
|
||||
cids, err := cidsForRoot(ctx, tx, hashID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, cidStr := range cids {
|
||||
if _, err := tx.Delete(CIDIndexPrefix + cidStr); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n, err := tx.DeletePrefix(recordPrefix(hashID), 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.DeletePrefix(dupPrefix(hashID), 0); err != nil {
|
||||
return n, err
|
||||
}
|
||||
if _, err := tx.Delete(rootKey(hashID)); err != nil {
|
||||
return n, err
|
||||
}
|
||||
return n, deleteSearchRowTx(tx, hashID)
|
||||
}
|
||||
|
||||
// DeleteMetadata deletes all metadata for a file, and unmaps every
|
||||
// reverse-index entry registered for this root, so a subsequent
|
||||
// GET /api/meta/<old-cid> correctly returns 404 instead of pointing at a root
|
||||
// with no keys.
|
||||
func (c *Client) DeleteMetadata(hashID string) (int64, error) {
|
||||
var n int64
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
n, err = deleteRecordTx(tx, hashID)
|
||||
return err
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CountFiles returns the number of indexed roots.
|
||||
func (c *Client) CountFiles() (int, error) {
|
||||
n, err := c.CountRoots()
|
||||
@@ -496,161 +527,28 @@ func normalizeFilesRelativePath(p string) string {
|
||||
|
||||
// DeleteProperty deletes a single property.
|
||||
//
|
||||
// For a cids/<cid> member this is the full inverse of the write: the flat key
|
||||
// AND the cid:<cid> reverse alias both go. Anything less leaves the CID still
|
||||
// resolving to this record — see removeAliasTx.
|
||||
func (c *Client) DeleteProperty(hashID, property string) error {
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
if _, err := tx.Delete(recordPrefix(hashID) + property); err != nil {
|
||||
return err
|
||||
}
|
||||
if cidStr := cidFromKeysetField(property); cidStr != "" {
|
||||
if err := removeAliasTx(tx, hashID, cidStr); err != nil {
|
||||
return fmt.Errorf("remove alias for %s: %w", cidStr, err)
|
||||
}
|
||||
}
|
||||
if affectsHaystack(property) {
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// AddToSet adds a value to a set-type field (stored as a pipe-delimited string
|
||||
// in one flat key). Returns true if value was added, false if it already
|
||||
// existed.
|
||||
func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
|
||||
added := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
current, _, err := tx.Get(tx.Context(), recordPrefix(hashID)+property)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var values []string
|
||||
if current != "" {
|
||||
values = strings.Split(current, "|")
|
||||
}
|
||||
for _, v := range values {
|
||||
if v == value {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
added = true
|
||||
return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(append(values, value), "|")})
|
||||
})
|
||||
return added, err
|
||||
}
|
||||
|
||||
// RemoveFromSet removes a value from a set-type field (stored in one flat key).
|
||||
// Returns true if value was removed, false if it didn't exist.
|
||||
func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
|
||||
removed := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
key := recordPrefix(hashID) + property
|
||||
current, ok, err := tx.Get(tx.Context(), key)
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
values := strings.Split(current, "|")
|
||||
kept := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
if v == value {
|
||||
removed = true
|
||||
} else {
|
||||
kept = append(kept, v)
|
||||
}
|
||||
}
|
||||
if !removed {
|
||||
return nil
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
if _, err := tx.Delete(key); err != nil {
|
||||
return err
|
||||
}
|
||||
if affectsHaystack(property) {
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(kept, "|")})
|
||||
})
|
||||
return removed, err
|
||||
}
|
||||
|
||||
// FileTuple holds the (uuid, path, size, mtime) record for one file. Used by
|
||||
// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple
|
||||
// matches what's already stored.
|
||||
type FileTuple struct {
|
||||
HashID string
|
||||
FilePath string
|
||||
Size int64
|
||||
MtimeNano int64
|
||||
}
|
||||
|
||||
// GetTuplesForAllFiles returns one FileTuple per indexed root, in batched
|
||||
// multi-key reads. Skips files missing any of filePath, sizeByte, mtimeNano.
|
||||
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
|
||||
defer cancel()
|
||||
|
||||
fields := [3]string{"filePath", "sizeByte", "mtimeNano"}
|
||||
var out []FileTuple
|
||||
err = s.ScanPrefix(ctx, rootsNS, 1000, func(rows []kv.KV) error {
|
||||
keys := make([]string, 0, len(rows)*len(fields))
|
||||
for _, row := range rows {
|
||||
prefix := recordPrefix(row.K[len(rootsNS):])
|
||||
for _, f := range fields {
|
||||
keys = append(keys, prefix+f)
|
||||
}
|
||||
}
|
||||
vals, err := s.MGet(ctx, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, row := range rows {
|
||||
h := row.K[len(rootsNS):]
|
||||
prefix := recordPrefix(h)
|
||||
filePath, sizeStr, mtimeStr := vals[prefix+fields[0]], vals[prefix+fields[1]], vals[prefix+fields[2]]
|
||||
if filePath == "" || sizeStr == "" || mtimeStr == "" {
|
||||
continue
|
||||
}
|
||||
size, err := strconv.ParseInt(sizeStr, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, FileTuple{HashID: h, FilePath: filePath, Size: size, MtimeNano: mtime})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// GetMemoryInfo reports the store's footprint for dashboards: the database
|
||||
// file plus its WAL, human-readable. (Under Redis this was used_memory_human;
|
||||
// the data no longer lives in RAM, so the disk footprint is the honest figure.)
|
||||
func (c *Client) GetMemoryInfo() (string, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
if _, err := c.kv(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return humanBytes(s.FileSize()), nil
|
||||
return humanBytes(c.DBSize()), nil
|
||||
}
|
||||
|
||||
// DBSize is the database file plus WAL, in bytes.
|
||||
// DBSize is both database files plus their WALs, in bytes.
|
||||
func (c *Client) DBSize() int64 {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return 0
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
var n int64
|
||||
for _, st := range []*kv.Store{c.store, c.truth} {
|
||||
if st != nil {
|
||||
n += st.FileSize()
|
||||
}
|
||||
}
|
||||
return s.FileSize()
|
||||
return n
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
@@ -674,37 +572,6 @@ func humanBytes(n int64) string {
|
||||
// redirects the next /meta/<cid> write back into that dead root, resurrecting
|
||||
// a zombie record under an old identity.
|
||||
//
|
||||
// Runs as many bounded transactions rather than one: a single transaction over
|
||||
// millions of rows would hold the writer for minutes and balloon the WAL.
|
||||
func (c *Client) ClearAllMetadata() (int64, error) {
|
||||
roots, err := c.CountRoots()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, prefix := range []string{recordNS, CIDIndexPrefix, dupNS, rootsNS, ftsNS} {
|
||||
for {
|
||||
var n int64
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
n, err = tx.DeletePrefix(prefix, bulkBatch)
|
||||
return err
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("clear %s: %w", prefix, err)
|
||||
}
|
||||
if n < bulkBatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
_, err := tx.Exec(`DELETE FROM fts`)
|
||||
return err
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("clear search rows: %w", err)
|
||||
}
|
||||
log.Printf("[Storage] Cleared %d file metadata entries", roots)
|
||||
return roots, nil
|
||||
}
|
||||
|
||||
// ScanKeys lists the immediate children at the given prefix, S3-style.
|
||||
//
|
||||
@@ -767,11 +634,24 @@ func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) (
|
||||
leaves = append(leaves, k)
|
||||
from, op = k, ">"
|
||||
}
|
||||
leaves, branches = serviceOnly(leaves), serviceOnly(branches)
|
||||
sort.Strings(leaves)
|
||||
sort.Strings(branches)
|
||||
return leaves, branches, next, nil
|
||||
}
|
||||
|
||||
// serviceOnly drops merged-record and bookkeeping keys: the /api/kv browser
|
||||
// shows service scratch KV only.
|
||||
func serviceOnly(keys []string) []string {
|
||||
out := keys[:0]
|
||||
for _, k := range keys {
|
||||
if !isInternalKey(k) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindMatch is a single value-search hit.
|
||||
type FindMatch struct {
|
||||
Key string `json:"key"`
|
||||
@@ -876,10 +756,11 @@ func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error)
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(results) > limit {
|
||||
return results[:limit], true, nil
|
||||
truncated := len(results) > limit
|
||||
if truncated {
|
||||
results = results[:limit]
|
||||
}
|
||||
return results, false, nil
|
||||
return serviceOnly(results), truncated, nil
|
||||
}
|
||||
|
||||
// GetKeyInfo returns the key's type — "string" when it exists (every row is a
|
||||
@@ -889,6 +770,9 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool,
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
if isInternalKey(key) {
|
||||
return "", "", false, nil
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
v, ok, err := s.Get(ctx, key)
|
||||
@@ -898,70 +782,6 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool,
|
||||
return "string", v, true, nil
|
||||
}
|
||||
|
||||
// SetRawValue writes a string value at the exact key (the raw KV editor).
|
||||
// Refuses derived bookkeeping keys. A record key keeps the search row honest
|
||||
// like every other write path.
|
||||
func (c *Client) SetRawValue(key, value string) error {
|
||||
if isInternalKey(key) {
|
||||
return fmt.Errorf("key %q is internal bookkeeping and not editable", key)
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
if err := tx.Put(key, value); err != nil {
|
||||
return err
|
||||
}
|
||||
if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteRawKey deletes the exact key. Returns true if the key existed.
|
||||
func (c *Client) DeleteRawKey(key string) (bool, error) {
|
||||
if isInternalKey(key) {
|
||||
return false, fmt.Errorf("key %q is internal bookkeeping and not editable", key)
|
||||
}
|
||||
existed := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
if existed, err = tx.Delete(key); err != nil || !existed {
|
||||
return err
|
||||
}
|
||||
if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return existed, err
|
||||
}
|
||||
|
||||
// IndexAdd adds a hash ID to the root index. Idempotent.
|
||||
func (c *Client) IndexAdd(hashID string) error {
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
if err := tx.Put(rootKey(hashID), ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureSearchRowTx(tx, hashID)
|
||||
})
|
||||
}
|
||||
|
||||
// IndexRemoveIfEmpty removes hashID from the root index (and drops its search
|
||||
// row) when no file:<hashID>/* keys remain. Returns true if it was removed.
|
||||
func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
|
||||
removed := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
n, err := tx.Count(tx.Context(), recordPrefix(hashID))
|
||||
if err != nil || n > 0 {
|
||||
return err
|
||||
}
|
||||
if removed, err = tx.Delete(rootKey(hashID)); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteSearchRowTx(tx, hashID)
|
||||
})
|
||||
return removed, err
|
||||
}
|
||||
|
||||
// GetPrefix returns the key namespace prefix. Always empty: meta-core owns its
|
||||
// database file outright, so there is nothing to share a keyspace with.
|
||||
func (c *Client) GetPrefix() string { return "" }
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/schema"
|
||||
)
|
||||
|
||||
@@ -19,48 +20,42 @@ func schemaJSON(t *testing.T, c *Client) string {
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// The counters behind /api/schema and search's `total` are maintained by the
|
||||
// write path. Whatever sequence of writes, modifications and deletions runs,
|
||||
// they must equal what a full rebuild from the records computes.
|
||||
// The counters behind /api/schema and search's `total` follow the merged view.
|
||||
// Whatever sequence of claims, retractions and edits runs, they must equal what
|
||||
// a full rebuild from the records computes.
|
||||
func TestCountersStayExactAcrossWrites(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
seed(t, c, map[string]map[string]string{
|
||||
"R1": {"title": "One", "sizeByte": "100", "title/eng": "One"},
|
||||
"R2": {"title": "Two", "sizeByte": "x", "genres/Action": "true"},
|
||||
"R3": {"title": "Three", "sizeByte": "300"},
|
||||
})
|
||||
if err := c.SetProperty("R2", "sizeByte", "200"); err != nil { // label string → int
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.SetProperty("R1", "title", "One"); err != nil { // unchanged: no delta
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.DeleteProperty("R1", "title/eng"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.DeleteMetadata("R3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if n, err := c.CountRoots(); err != nil || n != 2 {
|
||||
t.Fatalf("CountRoots = %d (err %v), want 2", n, err)
|
||||
a, b := signer(t, 1), signer(t, 2)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafyR1", "title", "One", 10), vote(t, a, "bafyR1", "sizeByte", "100", 10),
|
||||
vote(t, a, "bafyR1", "title/eng", "One", 10),
|
||||
vote(t, a, "bafyR2", "title", "Two", 10), vote(t, a, "bafyR2", "sizeByte", "x", 10),
|
||||
vote(t, a, "bafyR2", "genres/Action", "true", 10),
|
||||
vote(t, a, "bafyR3", "title", "Three", 10), vote(t, a, "bafyR3", "sizeByte", "300", 10),
|
||||
)
|
||||
mustAccept(t, c,
|
||||
vote(t, a, "bafyR2", "sizeByte", "200", 20), // label string → int
|
||||
vote(t, b, "bafyR1", "title", "One", 20), // agreeing vote: merged value unchanged
|
||||
vote(t, a, "bafyR1", "title/eng", nil, 20), // retraction
|
||||
)
|
||||
if n, err := c.CountRoots(); err != nil || n != 3 {
|
||||
t.Fatalf("CountRoots = %d (err %v), want 3", n, err)
|
||||
}
|
||||
incremental := schemaJSON(t, c)
|
||||
snap, _ := schema.NewIndexer(c.Store()).Snapshot(context.Background())
|
||||
if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 2 {
|
||||
t.Fatalf("sizeByte = %+v, want 2 ints", f)
|
||||
if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 3 {
|
||||
t.Fatalf("sizeByte = %+v, want 3 ints", f)
|
||||
}
|
||||
if _, ok := snap.Fields["title"]; !ok {
|
||||
t.Fatalf("title missing: %v", snap.Fields)
|
||||
if _, ok := snap.Fields["title/eng"]; ok {
|
||||
t.Fatalf("retracted title/eng still counted: %v", snap.Fields)
|
||||
}
|
||||
|
||||
if err := c.RebuildCounters(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rebuilt := schemaJSON(t, c); rebuilt != incremental {
|
||||
t.Fatalf("incremental counters drifted from a rebuild:\n incremental %s\n rebuilt %s", incremental, rebuilt)
|
||||
}
|
||||
if n, _ := c.CountRoots(); n != 2 {
|
||||
if n, _ := c.CountRoots(); n != 3 {
|
||||
t.Fatalf("CountRoots after rebuild = %d", n)
|
||||
}
|
||||
}
|
||||
@@ -77,16 +72,23 @@ func TestEnsureCountersBuildsOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A re-persisted record with nothing new writes nothing and publishes nothing.
|
||||
func TestUnchangedRecordWriteIsANoOp(t *testing.T) {
|
||||
// Re-persisting the same signed claims writes nothing and publishes nothing —
|
||||
// the gateway's constant repeats.
|
||||
func TestUnchangedClaimWriteIsANoOp(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
rec := map[string]string{"title": "Naruto", "fileType": "video", "cids/bafkreiabc": "true"}
|
||||
seed(t, c, map[string]map[string]string{"R1": rec})
|
||||
a := signer(t, 1)
|
||||
batch := []claim.Claim{
|
||||
vote(t, a, "bafkreiabc", "title", "Naruto", 5),
|
||||
vote(t, a, "bafkreiabc", "fileType", "video", 5),
|
||||
}
|
||||
mustAccept(t, c, batch...)
|
||||
before, _ := c.GetStreamLength(MetaEventsStream)
|
||||
if err := c.SetMetadataFlat("R1", rec); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, st := range write(t, c, batch...) {
|
||||
if st != StatusUnchanged {
|
||||
t.Fatalf("re-sent claim status %s, want unchanged", st)
|
||||
}
|
||||
}
|
||||
if after, _ := c.GetStreamLength(MetaEventsStream); after != before {
|
||||
t.Fatalf("re-persisting an unchanged record published %d events", after-before)
|
||||
t.Fatalf("re-persisting unchanged claims published %d events", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/identity"
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// openClient opens (and starts) a client on dir's truth + cache files.
|
||||
func openClient(t *testing.T, dir string) *Client {
|
||||
t.Helper()
|
||||
c := NewClient()
|
||||
if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
if err := c.EnsureSchemaVersion(SchemaVersion); err != nil {
|
||||
t.Fatalf("schema: %v", err)
|
||||
}
|
||||
if err := c.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// newTestClient opens a fresh store in the test's temp dir.
|
||||
func newTestClient(t *testing.T) *Client {
|
||||
t.Helper()
|
||||
c := openClient(t, t.TempDir())
|
||||
t.Cleanup(func() { _ = c.Close() })
|
||||
return c
|
||||
}
|
||||
|
||||
// signer returns a deterministic test identity (scalar n repeated).
|
||||
func signer(t *testing.T, n byte) *identity.Identity {
|
||||
t.Helper()
|
||||
id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1))
|
||||
if err != nil {
|
||||
t.Fatalf("identity: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// vote builds a signed claim; v is a string (text), []byte (ciphertext) or nil
|
||||
// (retraction).
|
||||
func vote(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim {
|
||||
t.Helper()
|
||||
cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts}
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
cl.Value = claim.Text(x)
|
||||
case []byte:
|
||||
cl.Value = claim.Bytes(x)
|
||||
case nil:
|
||||
default:
|
||||
t.Fatalf("vote value %T", v)
|
||||
}
|
||||
if err := cl.Sign(id); err != nil {
|
||||
t.Fatalf("sign: %v", err)
|
||||
}
|
||||
return cl
|
||||
}
|
||||
|
||||
// write posts claims and returns their statuses.
|
||||
func write(t *testing.T, c *Client, claims ...claim.Claim) []ClaimStatus {
|
||||
t.Helper()
|
||||
res, err := c.WriteClaims(context.Background(), claims)
|
||||
if err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
out := make([]ClaimStatus, len(res))
|
||||
for i, r := range res {
|
||||
out[i] = r.Status
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mustAccept posts claims and fails unless every one is accepted.
|
||||
func mustAccept(t *testing.T, c *Client, claims ...claim.Claim) {
|
||||
t.Helper()
|
||||
for i, st := range write(t, c, claims...) {
|
||||
if st != StatusAccepted {
|
||||
t.Fatalf("claim %d (%s/%s): %s", i, claims[i].CID, claims[i].Key, st)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seed writes merged rows straight into the cache — the derived layout search
|
||||
// and the counters work on — bypassing claims. Search tests pin SQL behaviour
|
||||
// over that layout; how rows get there is the apply loop's business.
|
||||
func seed(t *testing.T, c *Client, records map[string]map[string]string) {
|
||||
t.Helper()
|
||||
for id, fields := range records {
|
||||
setMerged(t, c, id, fields)
|
||||
}
|
||||
}
|
||||
|
||||
func setMerged(t *testing.T, c *Client, rid string, fields map[string]string) {
|
||||
t.Helper()
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
for k, v := range fields {
|
||||
if err := tx.Put(recordPrefix(rid)+k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Put(rootKey(rid), ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return refreshSearchTx(tx, rid)
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", rid, err)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteMerged(t *testing.T, c *Client, rid string) {
|
||||
t.Helper()
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
if _, err := tx.DeletePrefix(recordPrefix(rid), 0); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Delete(rootKey(rid)); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteSearchRowTx(tx, rid)
|
||||
}); err != nil {
|
||||
t.Fatalf("delete %s: %v", rid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// allKeys lists every cache kv row (full scan).
|
||||
func allKeys(t *testing.T, c *Client) []string {
|
||||
t.Helper()
|
||||
var keys []string
|
||||
if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
|
||||
for _, r := range rows {
|
||||
keys = append(keys, r.K)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// rid resolves a cid to its record (fails when unknown).
|
||||
func rid(t *testing.T, c *Client, cid string) string {
|
||||
t.Helper()
|
||||
r, err := c.GetByCID(cid)
|
||||
if err != nil || r == "" {
|
||||
t.Fatalf("cid %s unresolved (%v)", cid, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func record(t *testing.T, c *Client, cid string) map[string]string {
|
||||
t.Helper()
|
||||
f, err := c.GetMetadataFlat(rid(t, c, cid))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package storage
|
||||
|
||||
// Box-local file facts: where the watcher saw each file on this disk.
|
||||
//
|
||||
// local_file(path) → midhash, size, mtime (cache table, unsigned)
|
||||
//
|
||||
// These are facts about this disk, not about the content, so they are never
|
||||
// claims and never travel. meta-core cannot sign anyway, and a rescan of
|
||||
// /files rebuilds them. The midhash gets a cid_group entry in truth (local
|
||||
// naming, not a vote), and the record view joins filePath / sizeByte /
|
||||
// mtimeNano in as `file:<rid>/<field>` rows — which publishes the meta events
|
||||
// meta-fuse builds its tree from.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
const cacheTablesSchema = `
|
||||
CREATE TABLE IF NOT EXISTS claim_by_pid(
|
||||
pid INTEGER NOT NULL,
|
||||
cid TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
PRIMARY KEY (pid, cid, key)
|
||||
) WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS local_file(
|
||||
path TEXT PRIMARY KEY,
|
||||
midhash TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
mtime INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS local_file_midhash ON local_file(midhash, path);
|
||||
`
|
||||
|
||||
// RegisterFileTuple records one physical file under its midhash256 CID, the
|
||||
// way the watcher does for every file it scans (and POST /api/files/register
|
||||
// for a box with the watcher off). Returns the record's rid and whether its
|
||||
// group was newly created.
|
||||
func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
|
||||
if midhash == "" {
|
||||
return "", false, fmt.Errorf("midhash is required")
|
||||
}
|
||||
truth, err := c.truthStore()
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var rid string
|
||||
var created bool
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
if err := truth.Update(ctx, func(tx *kv.Tx) error {
|
||||
var err error
|
||||
rid, created, err = ensureGroup(tx, midhash)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
err = c.update(func(tx *kv.Tx) error {
|
||||
var prev string
|
||||
err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&prev)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO local_file(path, midhash, size, mtime) VALUES(?, ?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET midhash = excluded.midhash, size = excluded.size, mtime = excluded.mtime`,
|
||||
absPath, midhash, size, mtimeNano); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := syncMember(tx, midhash, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.remergeLocalTx(tx, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
if prev != "" && prev != midhash {
|
||||
// The file at this path changed content: its old record loses it.
|
||||
if old, _, err := tx.Get(tx.Context(), CIDIndexPrefix+prev); err == nil && old != "" && old != rid {
|
||||
return c.remergeLocalTx(tx, old)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return rid, created, nil
|
||||
}
|
||||
|
||||
// RemoveFileTuple forgets a path the watcher saw disappear. Only the local
|
||||
// facts go — the record's claims stay (facts about content stay true).
|
||||
func (c *Client) RemoveFileTuple(absPath string) error {
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
var midhash string
|
||||
err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&midhash)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM local_file WHERE path = ?`, absPath); err != nil {
|
||||
return err
|
||||
}
|
||||
rid, _, err := tx.Get(tx.Context(), CIDIndexPrefix+midhash)
|
||||
if err != nil || rid == "" {
|
||||
return err
|
||||
}
|
||||
return c.remergeLocalTx(tx, rid)
|
||||
})
|
||||
}
|
||||
|
||||
// remergeLocalTx recomputes the box-local fields of rid (and its search row
|
||||
// when filePath changed).
|
||||
func (c *Client) remergeLocalTx(tx *kv.Tx, rid string) error {
|
||||
cids, err := cidsForRoot(tx.Context(), tx, rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
searchChanged := false
|
||||
for _, f := range localFields {
|
||||
changed, err := c.remergeTx(tx, rid, cids, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
searchChanged = searchChanged || (changed && affectsHaystack(f))
|
||||
}
|
||||
if searchChanged {
|
||||
return refreshSearchTx(tx, rid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileTuple holds the (rid, path, size, mtime) record for one file. Used by
|
||||
// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple
|
||||
// matches what's already stored.
|
||||
type FileTuple struct {
|
||||
HashID string
|
||||
FilePath string
|
||||
Size int64
|
||||
MtimeNano int64
|
||||
}
|
||||
|
||||
// GetTuplesForAllFiles returns one FileTuple per local file.
|
||||
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
|
||||
defer cancel()
|
||||
rows, err := s.DB().QueryContext(ctx,
|
||||
`SELECT l.path, l.size, l.mtime, COALESCE(k.v, '') FROM local_file l
|
||||
LEFT JOIN kv k ON k.k = 'cid:' || l.midhash ORDER BY l.path`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []FileTuple
|
||||
for rows.Next() {
|
||||
var t FileTuple
|
||||
if err := rows.Scan(&t.FilePath, &t.Size, &t.MtimeNano, &t.HashID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// localPaths lists every local path whose midhash is one of cids.
|
||||
func localPaths(ctx context.Context, q interface {
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}, cids []string) ([]string, error) {
|
||||
var out []string
|
||||
for _, cid := range cids {
|
||||
rows, err := q.QueryContext(ctx, `SELECT path FROM local_file WHERE midhash = ? ORDER BY path`, cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var p string
|
||||
if err := rows.Scan(&p); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
)
|
||||
|
||||
// Vote is one signer's claim on one key, as the merge sees it. Pid identifies
|
||||
// the signer (one vote per signer per key per group); Uid is carried for the
|
||||
// stateless /api/merge, where claims arrive without a local pid.
|
||||
type Vote struct {
|
||||
Pid int64
|
||||
Uid string
|
||||
Value claim.Value
|
||||
TS uint64
|
||||
}
|
||||
|
||||
// MergeResult is the merged value of one key.
|
||||
type MergeResult struct {
|
||||
Value string
|
||||
Nsrc int // signers agreeing on the winning value
|
||||
OK bool
|
||||
}
|
||||
|
||||
// Merge is the one merge rule (spec §7), shared by the apply loop, the
|
||||
// ?as=<uid> overlay and POST /api/merge:
|
||||
//
|
||||
// 1. a signer counts once per key: its newest vote wins (it may have voted on
|
||||
// several cids of the group);
|
||||
// 2. the viewer's own live text vote wins outright (viewer == "" for the
|
||||
// shared view);
|
||||
// 3. retractions and ciphertext drop out — only text merges;
|
||||
// 4. support: votes grouped by normalised value, most signers wins;
|
||||
// 5. tie → the value first seen (oldest ts) — stable, and rewriting does not
|
||||
// win;
|
||||
// 6. tie → byte-wise smallest value, so every peer agrees.
|
||||
//
|
||||
// The stored value is the exact text of the oldest vote of the winning group.
|
||||
func Merge(votes []Vote, viewer string) MergeResult {
|
||||
newest := make(map[string]Vote, len(votes))
|
||||
for _, v := range votes {
|
||||
id := signerID(v)
|
||||
if cur, ok := newest[id]; !ok || v.TS > cur.TS {
|
||||
newest[id] = v
|
||||
}
|
||||
}
|
||||
type bucket struct {
|
||||
n int
|
||||
minTS uint64
|
||||
exemplar string
|
||||
}
|
||||
buckets := make(map[string]*bucket)
|
||||
for _, v := range newest {
|
||||
if v.Value.Kind != claim.KindText {
|
||||
continue
|
||||
}
|
||||
if viewer != "" && v.Uid == viewer {
|
||||
return MergeResult{Value: v.Value.Text, Nsrc: 1, OK: true}
|
||||
}
|
||||
norm := normalizeValue(v.Value.Text)
|
||||
b := buckets[norm]
|
||||
if b == nil {
|
||||
buckets[norm] = &bucket{n: 1, minTS: v.TS, exemplar: v.Value.Text}
|
||||
continue
|
||||
}
|
||||
b.n++
|
||||
if v.TS < b.minTS || (v.TS == b.minTS && v.Value.Text < b.exemplar) {
|
||||
b.minTS, b.exemplar = v.TS, v.Value.Text
|
||||
}
|
||||
}
|
||||
var best *bucket
|
||||
for _, b := range buckets {
|
||||
switch {
|
||||
case best == nil,
|
||||
b.n > best.n,
|
||||
b.n == best.n && b.minTS < best.minTS,
|
||||
b.n == best.n && b.minTS == best.minTS && b.exemplar < best.exemplar:
|
||||
best = b
|
||||
}
|
||||
}
|
||||
if best == nil {
|
||||
return MergeResult{}
|
||||
}
|
||||
return MergeResult{Value: best.exemplar, Nsrc: best.n, OK: true}
|
||||
}
|
||||
|
||||
func signerID(v Vote) string {
|
||||
if v.Uid != "" {
|
||||
return v.Uid
|
||||
}
|
||||
return "#" + strconv.FormatInt(v.Pid, 10)
|
||||
}
|
||||
|
||||
// normalizeValue folds case and surrounding whitespace so "The Matrix" and
|
||||
// "the matrix " count as the same value. (Unicode NFC folding is not applied:
|
||||
// the module carries no normalisation table; values written by one plugin
|
||||
// family are already consistent.)
|
||||
func normalizeValue(s string) string {
|
||||
return strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
@@ -1,70 +1,76 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// SchemaVersion is the key-layout version this build of meta-core expects.
|
||||
// Distinct from the snapshot manifest version (which is on-disk format);
|
||||
// this number describes the record layout — primarily, "are roots UUIDs or
|
||||
// content-hash tokens, and do reverse-index keys cid:* exist?"
|
||||
// SchemaVersion is the truth-database layout this build of meta-core expects.
|
||||
//
|
||||
// The Redis → SQLite move did not change the record layout (file:<rid>/<field>,
|
||||
// cid:<cid>), so it did not bump this: the one-time migration copied records
|
||||
// verbatim.
|
||||
// 2 — v1.x: one kv table holding records, the CID index and UDL cells
|
||||
// 3 — v2.0.0: the claims model (truth: principal, claim, cid_group, meta)
|
||||
//
|
||||
// Bump only when a layout change can't be detected from the data itself or
|
||||
// healed by the write path. Each bump is a flag day in alpha: operators wipe
|
||||
// and restart.
|
||||
const SchemaVersion = 2
|
||||
// Bump only when a layout change can't be detected from the data itself. Each
|
||||
// bump is a flag day: there is no migration code in meta-core (the one-off
|
||||
// v2 → v3 conversion of watch.nsl.sh is a manual tool, tools/claims-migrate).
|
||||
const SchemaVersion = 3
|
||||
|
||||
// SchemaVersionKey is the sentinel under which we record SchemaVersion.
|
||||
// Checked on every meta-core boot.
|
||||
const SchemaVersionKey = "meta-core:schema-version"
|
||||
// SchemaVersionKey is the truth `meta` row recording SchemaVersion.
|
||||
const SchemaVersionKey = "schema-version"
|
||||
|
||||
// EnsureSchemaVersion verifies the store matches the expected schema version.
|
||||
// Called once at startup, before any other reads/writes.
|
||||
// EnsureSchemaVersion verifies the truth database matches the expected
|
||||
// version. Called once at startup, before Start.
|
||||
//
|
||||
// Behavior:
|
||||
// - sentinel present and matches expected → OK
|
||||
// - sentinel missing AND no roots → fresh store, stamp sentinel
|
||||
// - sentinel missing AND roots exist → legacy data, refuse
|
||||
// - sentinel present but mismatched → refuse, point operator at clean:all
|
||||
//
|
||||
// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
|
||||
// The error message names the wipe remediation because in alpha there
|
||||
// is no automated migration — operators bring up a clean store.
|
||||
// - a v1.x store (the old kv table, which the default path still points at)
|
||||
// → refuse
|
||||
// - sentinel present and matching → OK
|
||||
// - sentinel missing and no claims → fresh store, stamp it
|
||||
// - sentinel missing with claims, or mismatched → refuse
|
||||
func (c *Client) EnsureSchemaVersion(expected int) error {
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
ctx := tx.Context()
|
||||
have, ok, err := tx.Get(ctx, SchemaVersionKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get schema version: %w", err)
|
||||
c.mu.RLock()
|
||||
truth := c.truth
|
||||
c.mu.RUnlock()
|
||||
if truth == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), opTimeout)
|
||||
defer cancel()
|
||||
return truth.Update(ctx, func(tx *kv.Tx) error {
|
||||
var name string
|
||||
err := tx.QueryRow(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'kv'`).Scan(&name)
|
||||
if err == nil {
|
||||
return fmt.Errorf(
|
||||
"%s holds a meta-core v1.x store (schema version 2, the kv table). v2.0.0 "+
|
||||
"does not migrate it: convert it with tools/claims-migrate, or move it "+
|
||||
"away to start empty", truth.Path())
|
||||
}
|
||||
if !ok {
|
||||
n, err := tx.Count(ctx, rootsNS)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count roots: %w", err)
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
var have string
|
||||
err = tx.QueryRow(`SELECT value FROM meta WHERE name = ?`, SchemaVersionKey).Scan(&have)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
var n int64
|
||||
if err := tx.QueryRow(`SELECT count(*) FROM claim`).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
return fmt.Errorf(
|
||||
"the store has %d indexed roots but no %q sentinel — this is alpha "+
|
||||
"data from before the UUID-rooted schema landed. Delete the "+
|
||||
"database file to wipe and restart, or downgrade meta-core to a compatible build",
|
||||
n, SchemaVersionKey)
|
||||
return fmt.Errorf("truth database has %d claims but no %q sentinel; refusing to guess its layout", n, SchemaVersionKey)
|
||||
}
|
||||
return tx.Put(SchemaVersionKey, strconv.Itoa(expected))
|
||||
_, err := tx.Exec(`INSERT INTO meta(name, value) VALUES(?, ?)`, SchemaVersionKey, strconv.Itoa(expected))
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("get schema version: %w", err)
|
||||
}
|
||||
if have == strconv.Itoa(expected) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"store schema version is %q, this build of meta-core expects %d. "+
|
||||
"Delete the database file to wipe and restart, or use a meta-core "+
|
||||
"build that matches the existing data",
|
||||
have, expected)
|
||||
return fmt.Errorf("truth schema version is %q, this build of meta-core expects %d", have, expected)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -201,6 +201,14 @@ type SearchHit struct {
|
||||
// match is the authoritative test, run on each hydrated record. Pass the same
|
||||
// query/filters match evaluates, or results will be missing.
|
||||
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
|
||||
return c.SearchAs(query, filters, limit, "", match)
|
||||
}
|
||||
|
||||
// SearchAs is Search seen by viewer (a uid, "" for the shared view): every
|
||||
// candidate is matched with the viewer's own claims overlaid, and the records
|
||||
// the viewer holds claims on are candidates too, so their own corrections are
|
||||
// searchable for them.
|
||||
func (c *Client) SearchAs(query string, filters []SearchFilter, limit int, viewer string, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -245,7 +253,7 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
|
||||
}
|
||||
}
|
||||
for _, id := range survivors {
|
||||
fields, err := readRecord(ctx, s, id)
|
||||
fields, err := c.readRecordAs(ctx, s, id, viewer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -261,6 +269,33 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
|
||||
break
|
||||
}
|
||||
}
|
||||
if viewer == "" {
|
||||
return hits, nil
|
||||
}
|
||||
seen := make(map[string]bool, len(hits))
|
||||
for _, h := range hits {
|
||||
seen[h.HashID] = true
|
||||
}
|
||||
own, err := c.viewerRIDs(ctx, viewer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range own {
|
||||
if seen[id] {
|
||||
continue
|
||||
}
|
||||
fields, err := c.readRecordAs(ctx, s, id, viewer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fields != nil && match(id, fields) {
|
||||
hits = append(hits, SearchHit{HashID: id, Fields: fields})
|
||||
}
|
||||
}
|
||||
sort.Slice(hits, func(i, j int) bool { return hits[i].HashID < hits[j].HashID })
|
||||
if len(hits) > limit {
|
||||
hits = hits[:limit]
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -19,15 +19,6 @@ import (
|
||||
// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
|
||||
// every candidate path; the API tests own the matcher itself.
|
||||
|
||||
func seed(t *testing.T, c *Client, records map[string]map[string]string) {
|
||||
t.Helper()
|
||||
for id, fields := range records {
|
||||
if err := c.SetMetadataFlat(id, fields); err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
|
||||
func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
|
||||
for _, v := range values {
|
||||
@@ -197,14 +188,10 @@ func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
|
||||
func TestSearchFollowsWritesAndDeletes(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
|
||||
if err := c.SetProperty("R1", "title", "After"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setMerged(t, c, "R1", map[string]string{"title": "After"})
|
||||
equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
|
||||
equalIDs(t, search(t, c, "before", nil, textMatch("before")))
|
||||
if _, err := c.DeleteMetadata("R1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleteMerged(t, c, "R1")
|
||||
equalIDs(t, search(t, c, "after", nil, textMatch("after")))
|
||||
equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,701 @@
|
||||
package storage
|
||||
|
||||
// The truth database: the only data meta-core cannot rebuild.
|
||||
//
|
||||
// principal(pid, uid) who signed (pid = local short form)
|
||||
// claim(cid, key, pid) → value, ts, sig, seq every vote, signed (MMUR2)
|
||||
// cid_group(cid) → rid which record a cid belongs to
|
||||
// meta(name) → value schema version, last_seq
|
||||
//
|
||||
// Primary keys only — every other index lives in the cache, which is rebuilt
|
||||
// from these four tables. A claim never mentions a rid: the rid is a local
|
||||
// name for a group of cids, minted here (UUIDv7) and kept because consumers
|
||||
// hold rids.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
const truthSchema = `
|
||||
CREATE TABLE IF NOT EXISTS principal(
|
||||
pid INTEGER PRIMARY KEY,
|
||||
uid TEXT NOT NULL UNIQUE
|
||||
) STRICT;
|
||||
CREATE TABLE IF NOT EXISTS claim(
|
||||
cid TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
value ANY,
|
||||
ts INTEGER NOT NULL,
|
||||
sig BLOB NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
PRIMARY KEY (cid, key, pid)
|
||||
) STRICT, WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS cid_group(cid TEXT PRIMARY KEY, rid TEXT NOT NULL) STRICT, WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS meta(name TEXT PRIMARY KEY, value TEXT NOT NULL) STRICT, WITHOUT ROWID;
|
||||
`
|
||||
|
||||
const lastSeqName = "last_seq"
|
||||
|
||||
// ClaimStatus is the per-claim outcome of POST /api/claims.
|
||||
type ClaimStatus string
|
||||
|
||||
const (
|
||||
StatusAccepted ClaimStatus = "accepted"
|
||||
StatusUnchanged ClaimStatus = "unchanged"
|
||||
StatusStale ClaimStatus = "stale"
|
||||
StatusBadSignature ClaimStatus = "bad_signature"
|
||||
StatusBadRequest ClaimStatus = "bad_request"
|
||||
)
|
||||
|
||||
// ClaimResult is one entry of the POST /api/claims response.
|
||||
type ClaimResult struct {
|
||||
Status ClaimStatus `json:"status"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// MaxClaimsPerBatch bounds one POST /api/claims.
|
||||
const MaxClaimsPerBatch = 10_000
|
||||
|
||||
// change is one committed truth change, queued for the apply loop in commit
|
||||
// order. key == "" means "membership only" (a cid joined a group).
|
||||
type change struct {
|
||||
seq int64
|
||||
cid string
|
||||
key string
|
||||
pid int64
|
||||
live bool // the claim now holds a value (false: tombstone)
|
||||
regroup *[2]string // {loser, survivor}: two groups merged
|
||||
}
|
||||
|
||||
func valueArg(v claim.Value) any {
|
||||
switch v.Kind {
|
||||
case claim.KindText:
|
||||
return v.Text
|
||||
case claim.KindBytes:
|
||||
if v.Bytes == nil {
|
||||
return []byte{}
|
||||
}
|
||||
return v.Bytes
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanValue turns a claim.value column (TEXT / BLOB / NULL) back into a Value.
|
||||
func scanValue(raw any) claim.Value {
|
||||
switch x := raw.(type) {
|
||||
case string:
|
||||
return claim.Text(x)
|
||||
case []byte:
|
||||
return claim.Bytes(x)
|
||||
}
|
||||
return claim.Null()
|
||||
}
|
||||
|
||||
type storedClaim struct {
|
||||
ts int64
|
||||
sig []byte
|
||||
value claim.Value
|
||||
}
|
||||
|
||||
func readStored(ctx context.Context, q interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}, cid, key, uid string) (storedClaim, bool, error) {
|
||||
var sc storedClaim
|
||||
var raw any
|
||||
err := q.QueryRowContext(ctx,
|
||||
`SELECT c.ts, c.sig, c.value FROM claim c JOIN principal p ON p.pid = c.pid
|
||||
WHERE c.cid = ? AND c.key = ? AND p.uid = ?`, cid, key, uid).Scan(&sc.ts, &sc.sig, &raw)
|
||||
sc.value = scanValue(raw)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return sc, false, nil
|
||||
}
|
||||
return sc, err == nil, err
|
||||
}
|
||||
|
||||
// precheck classifies a claim against the committed state without the writer:
|
||||
// skip-identical happens BEFORE verification (most writes are re-persists of
|
||||
// the same signed claim), stale is refused, everything else is verified.
|
||||
func (c *Client) precheck(ctx context.Context, cl *claim.Claim) (ClaimStatus, string) {
|
||||
if cl.CID == "" || cl.Key == "" || cl.UID == "" {
|
||||
return StatusBadRequest, "cid, uid and key are required"
|
||||
}
|
||||
if !claimKeyAllowed(cl.Key) {
|
||||
return StatusBadRequest, "key is reserved or malformed"
|
||||
}
|
||||
if len(cl.Key) > 4096 || len(cl.CID) > 512 {
|
||||
return StatusBadRequest, "cid or key too long"
|
||||
}
|
||||
if cl.TS == 0 || cl.TS > 1<<62 {
|
||||
return StatusBadRequest, "ts out of range"
|
||||
}
|
||||
sc, ok, err := readStored(ctx, c.truth.DB(), cl.CID, cl.Key, cl.UID)
|
||||
if err != nil {
|
||||
return StatusBadRequest, err.Error()
|
||||
}
|
||||
if ok {
|
||||
if sc.ts == int64(cl.TS) && string(sc.sig) == string(cl.Sig) {
|
||||
return StatusUnchanged, ""
|
||||
}
|
||||
// Same signer re-asserting the value it already holds (the gateway
|
||||
// re-persists every search hit with a fresh ts): keep the stored row and
|
||||
// its older ts, so oldest-ts tie-breaking stays anchored and truth takes
|
||||
// no write. Nothing is stored, so the new signature needs no check.
|
||||
if sc.value.Equal(cl.Value) {
|
||||
return StatusUnchanged, ""
|
||||
}
|
||||
if sc.ts >= int64(cl.TS) {
|
||||
return StatusStale, "a newer or equal ts is stored for this signer, cid and key"
|
||||
}
|
||||
}
|
||||
if err := cl.Verify(); err != nil {
|
||||
return StatusBadSignature, err.Error()
|
||||
}
|
||||
return StatusAccepted, ""
|
||||
}
|
||||
|
||||
// WriteClaims is POST /api/claims: verify, store, regroup, and wait until the
|
||||
// cache has applied the batch (read-your-writes for the caller).
|
||||
func (c *Client) WriteClaims(ctx context.Context, claims []claim.Claim) ([]ClaimResult, error) {
|
||||
truth, err := c.truthStore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results := make([]ClaimResult, len(claims))
|
||||
var todo []int
|
||||
for i := range claims {
|
||||
st, reason := c.precheck(ctx, &claims[i])
|
||||
results[i] = ClaimResult{Status: st, Reason: reason}
|
||||
if st == StatusAccepted {
|
||||
todo = append(todo, i)
|
||||
}
|
||||
}
|
||||
if len(todo) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
var maxSeq int64
|
||||
err = truth.Update(ctx, func(tx *kv.Tx) error {
|
||||
seq, err := readLastSeq(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var changes []change
|
||||
pids := map[string]int64{}
|
||||
for _, i := range todo {
|
||||
cl := &claims[i]
|
||||
// Re-check under the writer: a concurrent batch may have stored a
|
||||
// newer vote since the precheck.
|
||||
sc, ok, err := readStored(tx.Context(), txQuerier{tx}, cl.CID, cl.Key, cl.UID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok && (sc.ts >= int64(cl.TS) || sc.value.Equal(cl.Value)) {
|
||||
if sc.value.Equal(cl.Value) {
|
||||
results[i] = ClaimResult{Status: StatusUnchanged}
|
||||
} else {
|
||||
results[i] = ClaimResult{Status: StatusStale, Reason: "a newer or equal ts is stored for this signer, cid and key"}
|
||||
}
|
||||
continue
|
||||
}
|
||||
pid, ok := pids[cl.UID]
|
||||
if !ok {
|
||||
if pid, err = ensurePrincipal(tx, cl.UID); err != nil {
|
||||
return err
|
||||
}
|
||||
pids[cl.UID] = pid
|
||||
}
|
||||
seq++
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(cid, key, pid) DO UPDATE SET value = excluded.value, ts = excluded.ts,
|
||||
sig = excluded.sig, seq = excluded.seq`,
|
||||
cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, seq); err != nil {
|
||||
return fmt.Errorf("store claim: %w", err)
|
||||
}
|
||||
if _, _, err := ensureGroup(tx, cl.CID); err != nil {
|
||||
return err
|
||||
}
|
||||
changes = append(changes, change{seq: seq, cid: cl.CID, key: cl.Key, pid: pid, live: !cl.Value.IsNull()})
|
||||
if other := cidFromKeysetField(cl.Key); other != "" && cl.Value.Kind == claim.KindText && other != cl.CID {
|
||||
more, err := linkGroups(tx, cl.CID, other, seq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changes = append(changes, more...)
|
||||
}
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := writeLastSeq(tx, seq); err != nil {
|
||||
return err
|
||||
}
|
||||
maxSeq = seq
|
||||
tx.OnCommit(func() { c.applier.enqueue(changes) })
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if maxSeq > 0 {
|
||||
if err := c.applier.wait(ctx, maxSeq); err != nil {
|
||||
return results, err
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// txQuerier adapts a kv.Tx to the QueryRowContext shape readStored takes.
|
||||
type txQuerier struct{ tx *kv.Tx }
|
||||
|
||||
func (q txQuerier) QueryRowContext(_ context.Context, query string, args ...any) *sql.Row {
|
||||
return q.tx.QueryRow(query, args...)
|
||||
}
|
||||
|
||||
func readLastSeq(tx *kv.Tx) (int64, error) {
|
||||
var v string
|
||||
err := tx.QueryRow(`SELECT value FROM meta WHERE name = ?`, lastSeqName).Scan(&v)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseInt(v, 10, 64)
|
||||
}
|
||||
|
||||
func writeLastSeq(tx *kv.Tx, seq int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO meta(name, value) VALUES(?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET value = excluded.value`, lastSeqName, strconv.FormatInt(seq, 10))
|
||||
return err
|
||||
}
|
||||
|
||||
func ensurePrincipal(tx *kv.Tx, uid string) (int64, error) {
|
||||
var pid int64
|
||||
err := tx.QueryRow(`SELECT pid FROM principal WHERE uid = ?`, uid).Scan(&pid)
|
||||
if err == nil {
|
||||
return pid, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.QueryRow(`INSERT INTO principal(uid) VALUES(?) RETURNING pid`, uid).Scan(&pid); err != nil {
|
||||
return 0, fmt.Errorf("new principal: %w", err)
|
||||
}
|
||||
return pid, nil
|
||||
}
|
||||
|
||||
// ensureGroup returns cid's rid, minting a new group for an unknown cid.
|
||||
func ensureGroup(tx *kv.Tx, cid string) (string, bool, error) {
|
||||
var rid string
|
||||
err := tx.QueryRow(`SELECT rid FROM cid_group WHERE cid = ?`, cid).Scan(&rid)
|
||||
if err == nil {
|
||||
return rid, false, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, err
|
||||
}
|
||||
if rid, err = NewUUID(); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`, cid, rid); err != nil {
|
||||
return "", false, fmt.Errorf("new group: %w", err)
|
||||
}
|
||||
return rid, true, nil
|
||||
}
|
||||
|
||||
// linkGroups records that a and b are the same content. An unknown b joins a's
|
||||
// group; two known groups merge, the older rid surviving (UUIDv7 sorts by
|
||||
// time). A retraction of the link does not split the group (phase 1).
|
||||
func linkGroups(tx *kv.Tx, a, b string, seq int64) ([]change, error) {
|
||||
ra, _, err := ensureGroup(tx, a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rb string
|
||||
err = tx.QueryRow(`SELECT rid FROM cid_group WHERE cid = ?`, b).Scan(&rb)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
if _, err := tx.Exec(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`, b, ra); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []change{{seq: seq, cid: b}}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ra == rb {
|
||||
return nil, nil
|
||||
}
|
||||
survivor, loser := ra, rb
|
||||
if rb < ra {
|
||||
survivor, loser = rb, ra
|
||||
}
|
||||
// cid_group has no rid index (truth keeps primary keys only); a merge is
|
||||
// rare, so the scan is acceptable.
|
||||
if _, err := tx.Exec(`UPDATE cid_group SET rid = ? WHERE rid = ?`, survivor, loser); err != nil {
|
||||
return nil, fmt.Errorf("merge groups: %w", err)
|
||||
}
|
||||
return []change{{seq: seq, regroup: &[2]string{loser, survivor}}}, nil
|
||||
}
|
||||
|
||||
// truthRid reads cid's committed group.
|
||||
func (c *Client) truthRid(ctx context.Context, cid string) (string, error) {
|
||||
var rid string
|
||||
err := c.truth.DB().QueryRowContext(ctx, `SELECT rid FROM cid_group WHERE cid = ?`, cid).Scan(&rid)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
return rid, err
|
||||
}
|
||||
|
||||
// votesFor reads every committed vote on key across cids, with signer uids.
|
||||
func (c *Client) votesFor(ctx context.Context, cids []string, key string) ([]Vote, error) {
|
||||
var out []Vote
|
||||
for _, cid := range cids {
|
||||
rows, err := c.truth.DB().QueryContext(ctx,
|
||||
`SELECT c.pid, p.uid, c.value, c.ts FROM claim c JOIN principal p ON p.pid = c.pid
|
||||
WHERE c.cid = ? AND c.key = ?`, cid, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var v Vote
|
||||
var raw any
|
||||
var ts int64
|
||||
if err := rows.Scan(&v.Pid, &v.Uid, &raw, &ts); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
v.Value, v.TS = scanValue(raw), uint64(ts)
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// claimKeys lists the distinct keys claimed on any of cids.
|
||||
func (c *Client) claimKeys(ctx context.Context, cids []string) ([]string, error) {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, cid := range cids {
|
||||
rows, err := c.truth.DB().QueryContext(ctx, `SELECT DISTINCT key FROM claim WHERE cid = ?`, cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if !seen[k] {
|
||||
seen[k] = true
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClaimsForCIDs returns every live signed claim on cids (tombstones excluded),
|
||||
// optionally only for one key.
|
||||
func (c *Client) ClaimsForCIDs(cids []string, key string) ([]claim.Claim, error) {
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
var out []claim.Claim
|
||||
for _, cid := range cids {
|
||||
q := `SELECT c.cid, p.uid, c.key, c.value, c.ts, c.sig FROM claim c JOIN principal p ON p.pid = c.pid
|
||||
WHERE c.cid = ? AND c.value IS NOT NULL`
|
||||
args := []any{cid}
|
||||
if key != "" {
|
||||
q += ` AND c.key = ?`
|
||||
args = append(args, key)
|
||||
}
|
||||
q += ` ORDER BY c.key, p.uid`
|
||||
rows, err := c.truth.DB().QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var cl claim.Claim
|
||||
var raw any
|
||||
var ts int64
|
||||
if err := rows.Scan(&cl.CID, &cl.UID, &cl.Key, &raw, &ts, &cl.Sig); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
cl.Value, cl.TS = scanValue(raw), uint64(ts)
|
||||
out = append(out, cl)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClaimsForRID returns the live claims of every cid in rid's group.
|
||||
func (c *Client) ClaimsForRID(rid, key string) ([]claim.Claim, error) {
|
||||
_, claims, err := c.RecordClaims(rid, key)
|
||||
return claims, err
|
||||
}
|
||||
|
||||
// RecordClaims returns the cids of rid's group (empty: no such record) and
|
||||
// their live claims, optionally for one key.
|
||||
func (c *Client) RecordClaims(rid, key string) ([]string, []claim.Claim, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
cids, err := cidsForRoot(ctx, s, rid)
|
||||
if err != nil || len(cids) == 0 {
|
||||
return cids, nil, err
|
||||
}
|
||||
claims, err := c.ClaimsForCIDs(cids, key)
|
||||
return cids, claims, err
|
||||
}
|
||||
|
||||
// LiveClaimsBySigner lists (cid, key) of every live claim uid has signed — the
|
||||
// set DELETE /api/identity retracts.
|
||||
func (c *Client) LiveClaimsBySigner(uid string) ([][2]string, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pid, ok, err := c.pidOf(uid)
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
rows, err := s.DB().QueryContext(ctx, `SELECT cid, key FROM claim_by_pid WHERE pid = ? ORDER BY cid, key`, pid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out [][2]string
|
||||
for rows.Next() {
|
||||
var p [2]string
|
||||
if err := rows.Scan(&p[0], &p[1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *Client) pidOf(uid string) (int64, bool, error) {
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
var pid int64
|
||||
err := c.truth.DB().QueryRowContext(ctx, `SELECT pid FROM principal WHERE uid = ?`, uid).Scan(&pid)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, false, nil
|
||||
}
|
||||
return pid, err == nil, err
|
||||
}
|
||||
|
||||
// CountClaims counts claim rows (tombstones included) — /api/stats' redisKeys.
|
||||
func (c *Client) CountClaims() (int64, error) {
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
var n int64
|
||||
err := c.truth.DB().QueryRowContext(ctx, `SELECT count(*) FROM claim`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func isLocalField(key string) bool {
|
||||
return key == "filePath" || key == "mtimeNano" || key == "sizeByte"
|
||||
}
|
||||
|
||||
// claimKeyAllowed rejects keys that would corrupt the merged view's key
|
||||
// layout, and the removed per-algorithm CID shapes (cid_*, midhash256,
|
||||
// canonical_cid — METADATA_KEYS.md §14.13: write the cids/<bareCid> key-set).
|
||||
func claimKeyAllowed(key string) bool {
|
||||
if strings.HasPrefix(key, "cid_") || key == "midhash256" || key == "canonical_cid" {
|
||||
return false
|
||||
}
|
||||
return !strings.HasPrefix(key, "/") && !strings.Contains(key, "\x00") && !isReservedFieldName(key)
|
||||
}
|
||||
|
||||
// GetMetadataFlatAs is GetMetadataFlat seen by viewer: the merged record with
|
||||
// the viewer's own live text claims overlaid (their value wins in their view).
|
||||
func (c *Client) GetMetadataFlatAs(rid, viewer string) (map[string]string, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
return c.readRecordAs(ctx, s, rid, viewer)
|
||||
}
|
||||
|
||||
func (c *Client) readRecordAs(ctx context.Context, s *kv.Store, rid, viewer string) (map[string]string, error) {
|
||||
fields, err := readRecord(ctx, s, rid)
|
||||
if err != nil || viewer == "" {
|
||||
return fields, err
|
||||
}
|
||||
over, err := c.viewerOverrides(ctx, s, rid, viewer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(over) == 0 {
|
||||
return fields, nil
|
||||
}
|
||||
if fields == nil {
|
||||
fields = make(map[string]string, len(over))
|
||||
}
|
||||
for k, v := range over {
|
||||
fields[k] = v
|
||||
}
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
// viewerOverrides returns the viewer's newest live text vote per key on the
|
||||
// cids of rid's group.
|
||||
func (c *Client) viewerOverrides(ctx context.Context, s *kv.Store, rid, viewer string) (map[string]string, error) {
|
||||
pid, ok, err := c.pidOf(viewer)
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
cids, err := cidsForRoot(ctx, s, rid)
|
||||
if err != nil || len(cids) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
newest := map[string]int64{}
|
||||
for _, cid := range cids {
|
||||
rows, err := s.DB().QueryContext(ctx, `SELECT key FROM claim_by_pid WHERE pid = ? AND cid = ?`, pid, cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range keys {
|
||||
var raw any
|
||||
var ts int64
|
||||
if err := c.truth.DB().QueryRowContext(ctx,
|
||||
`SELECT value, ts FROM claim WHERE cid = ? AND key = ? AND pid = ?`, cid, k, pid).Scan(&raw, &ts); err != nil {
|
||||
continue
|
||||
}
|
||||
if v := scanValue(raw); v.Kind == claim.KindText && ts > newest[k] {
|
||||
out[k], newest[k] = v.Text, ts
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// viewerRIDs lists the records the viewer holds live claims on.
|
||||
func (c *Client) viewerRIDs(ctx context.Context, viewer string) ([]string, error) {
|
||||
pid, ok, err := c.pidOf(viewer)
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
s := c.store
|
||||
rows, err := s.DB().QueryContext(ctx,
|
||||
`SELECT DISTINCT k.v FROM claim_by_pid b JOIN kv k ON k.k = 'cid:' || b.cid WHERE b.pid = ? LIMIT 10000`, pid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var rid string
|
||||
if err := rows.Scan(&rid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, rid)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MergeClaims is the stateless POST /api/merge: verify each claim (invalid
|
||||
// ones are dropped and counted, never an error for the batch), merge per key
|
||||
// with the §7 rule, and overlay viewer's own votes — those in the request and
|
||||
// those this core holds (claim_by_pid) for the same cids. Returns the merged
|
||||
// fields, the agreeing-signer count per key, and how many claims were rejected.
|
||||
func (c *Client) MergeClaims(claims []claim.Claim, viewer string) (map[string]string, map[string]int, int, error) {
|
||||
byKey := map[string][]Vote{}
|
||||
rejected := 0
|
||||
cids := map[string]bool{}
|
||||
for i := range claims {
|
||||
cl := &claims[i]
|
||||
if err := cl.Verify(); err != nil {
|
||||
rejected++
|
||||
continue
|
||||
}
|
||||
cids[cl.CID] = true
|
||||
byKey[cl.Key] = append(byKey[cl.Key], Vote{Uid: cl.UID, Value: cl.Value, TS: cl.TS})
|
||||
}
|
||||
fields := make(map[string]string, len(byKey))
|
||||
nsrc := make(map[string]int, len(byKey))
|
||||
for key, votes := range byKey {
|
||||
if res := Merge(votes, viewer); res.OK {
|
||||
fields[key], nsrc[key] = res.Value, res.Nsrc
|
||||
}
|
||||
}
|
||||
if viewer == "" {
|
||||
return fields, nsrc, rejected, nil
|
||||
}
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return fields, nsrc, rejected, nil // no local core: request overlay only
|
||||
}
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return fields, nsrc, rejected, nil
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
seen := map[string]bool{}
|
||||
for cid := range cids {
|
||||
rid, _, err := s.Get(ctx, CIDIndexPrefix+cid)
|
||||
if err != nil || rid == "" || seen[rid] {
|
||||
continue
|
||||
}
|
||||
seen[rid] = true
|
||||
over, err := c.viewerOverrides(ctx, s, rid, viewer)
|
||||
if err != nil {
|
||||
return nil, nil, rejected, err
|
||||
}
|
||||
for k, v := range over {
|
||||
fields[k], nsrc[k] = v, 1
|
||||
}
|
||||
}
|
||||
return fields, nsrc, rejected, nil
|
||||
}
|
||||
+125
-381
@@ -1,441 +1,185 @@
|
||||
package storage
|
||||
|
||||
// User Data Layer (UDL) storage.
|
||||
//
|
||||
// Holds per-user, identity-signed records keyed by (uid, cid, key) — the
|
||||
// subjective/per-user state that METADATA_KEYS.md rule #4 bars from the shared
|
||||
// /file/{cid} hash (likes, ratings, watch progress, local paths). Each record
|
||||
// is an opaque, client-signed (and, for private-tier keys, client-encrypted)
|
||||
// CBOR blob; meta-core stores it verbatim and never needs the account key.
|
||||
//
|
||||
// Convergence is version-gated: a write is accepted only if its `version`
|
||||
// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
|
||||
// op-log merge that used to live in meta-watch's store.rs).
|
||||
//
|
||||
// Key model (all under the literal "udl:" prefix, mirroring the "file:" one):
|
||||
//
|
||||
// udl:rec:<uid>/<cid>/<key> JSON { record, version, ts, value? }
|
||||
// udl:idx:user:<uid>:key:<key> set of <cid> -> active_for_user_key (My List, Continue Watching)
|
||||
// udl:idx:user:<uid>:cid:<cid> set of <key> -> list_for_user_cid
|
||||
// udl:idx:user:<uid>:cids set of <cid> -> whole-profile export (recovery)
|
||||
// udl:idx:cid:<cid>:key:<key> set of <uid> -> aggregate
|
||||
// udl:idx:cid:<cid>:users set of <uid> -> uids_for_cid
|
||||
//
|
||||
// A "set" is a key-set: one key `<set>\x1f<member>` = "" per member, read as
|
||||
// the key range under `<set>\x1f`. The separator is a control character
|
||||
// because UDL keys are not validated — with `/` the members of
|
||||
// `…:key:a/b` would show up as members of `…:key:a`.
|
||||
//
|
||||
// The `value` field is the plaintext JSON scalar for PUBLIC keys only
|
||||
// (like/rating), used for aggregation; it is absent for private-tier keys
|
||||
// (whose real value rides encrypted inside `record`) and for tombstones.
|
||||
// The User Data Layer facade (/api/udl/*). Since v2.0.0 a user's data are
|
||||
// ordinary claims signed by the user's key — plaintext for public-social keys
|
||||
// (like, userRating, profile:*), ciphertext for private ones (seek, watched,
|
||||
// …). There is no separate store: these helpers read the claims of one signer
|
||||
// through the cache's claim_by_pid index and the truth claim table.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/identity"
|
||||
)
|
||||
|
||||
// UDLCell is a decoded (uid, cid, key) record cell.
|
||||
type UDLCell struct {
|
||||
Record string // base64 CBOR, opaque + signed
|
||||
Version int64
|
||||
Ts int64
|
||||
Value string // plaintext JSON scalar, public keys only
|
||||
HasValue bool
|
||||
// UDLUserStats summarises one signer's live claims.
|
||||
type UDLUserStats struct {
|
||||
Records int `json:"records"`
|
||||
Cids int `json:"cids"`
|
||||
Keys int `json:"keys"`
|
||||
}
|
||||
|
||||
// UDLUserKeyEntry is one row of active_for_user_key (all cids for a uid+key).
|
||||
type UDLUserKeyEntry struct {
|
||||
Cid string `json:"cid"`
|
||||
Record string `json:"record"`
|
||||
Version int64 `json:"version"`
|
||||
Ts int64 `json:"ts"`
|
||||
}
|
||||
|
||||
// UDLUserCidEntry is one row of list_for_user_cid (all keys for a uid+cid).
|
||||
type UDLUserCidEntry struct {
|
||||
Key string `json:"key"`
|
||||
Record string `json:"record"`
|
||||
Version int64 `json:"version"`
|
||||
Ts int64 `json:"ts"`
|
||||
}
|
||||
|
||||
func udlRecKey(uid, cid, key string) string {
|
||||
return fmt.Sprintf("udl:rec:%s/%s/%s", uid, cid, key)
|
||||
}
|
||||
func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
|
||||
func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
|
||||
func udlIdxCidKey(cid, key string) string { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
|
||||
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
|
||||
|
||||
// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
|
||||
// index that makes "export this whole profile" a set read instead of a
|
||||
// keyspace scan.
|
||||
//
|
||||
// The other four indexes each answer a question with one coordinate already
|
||||
// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and
|
||||
// nothing else. Without this set the only ways to enumerate a profile are to
|
||||
// iterate the hardcoded key list — which silently misses any key added later —
|
||||
// or to SCAN `udl:rec:<uid>/*`, which is the full-keyspace-read pathology that
|
||||
// already made `/api/udl/users/stats` admin-only.
|
||||
func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
|
||||
|
||||
// udlSetSep separates a set key from a member (see the key model above).
|
||||
const udlSetSep = "\x1f"
|
||||
|
||||
// udlCellJSON is a cell's stored form. Value is a pointer so an absent value
|
||||
// (private-tier key, tombstone) stays distinguishable from an empty one.
|
||||
type udlCellJSON struct {
|
||||
Record string `json:"record"`
|
||||
Version int64 `json:"version"`
|
||||
Ts int64 `json:"ts"`
|
||||
Value *string `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
func decodeUDLCell(raw string) (UDLCell, error) {
|
||||
var j udlCellJSON
|
||||
if err := json.Unmarshal([]byte(raw), &j); err != nil {
|
||||
return UDLCell{}, err
|
||||
}
|
||||
cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts}
|
||||
if j.Value != nil {
|
||||
cell.Value, cell.HasValue = *j.Value, true
|
||||
}
|
||||
return cell, nil
|
||||
}
|
||||
|
||||
// udlMembers lists a key-set's members, sorted.
|
||||
func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) {
|
||||
prefix := set + udlSetSep
|
||||
rows, err := r.Range(ctx, prefix, "", 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, row.K[len(prefix):])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func udlAdd(tx *kv.Tx, set, member string) error {
|
||||
_, err := tx.PutIfAbsent(set+udlSetSep+member, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func udlRem(tx *kv.Tx, set, member string) error {
|
||||
_, err := tx.Delete(set + udlSetSep + member)
|
||||
return err
|
||||
}
|
||||
|
||||
// readUDLCell reads one cell. exists is false when it was never written.
|
||||
func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) {
|
||||
raw, ok, err := r.Get(ctx, recKey)
|
||||
if err != nil || !ok {
|
||||
return UDLCell{}, false, err
|
||||
}
|
||||
cell, err := decodeUDLCell(raw)
|
||||
if err != nil {
|
||||
return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err)
|
||||
}
|
||||
return cell, true, nil
|
||||
}
|
||||
|
||||
// readUDLCells reads many cells in one multi-key read; absent cells are
|
||||
// omitted from the map.
|
||||
func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) {
|
||||
raws, err := r.MGet(ctx, recKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]UDLCell, len(raws))
|
||||
for k, raw := range raws {
|
||||
cell, err := decodeUDLCell(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %w", k, err)
|
||||
}
|
||||
out[k] = cell
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
|
||||
// one, and maintains the indexes — one transaction, so the gate and the write
|
||||
// cannot interleave with a concurrent upsert of the same cell.
|
||||
// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by
|
||||
// aggregation); pass hasValue=false for private-tier keys and tombstones. Set
|
||||
// tombstone=true when the record deletes the cell.
|
||||
//
|
||||
// A tombstone still writes the cell — that is the CRDT state, and dropping it
|
||||
// would let an older write resurrect the value — but it removes the cid from
|
||||
// the user+key index. That index is what `active_for_user_key` returns, so
|
||||
// leaving deleted cids in it means every My List / Continue Watching read
|
||||
// carries every title ever un-liked or finished, forever. The other four
|
||||
// indexes are left alone: they answer "who has an opinion about this cid",
|
||||
// where a tombstone is still an answer.
|
||||
//
|
||||
// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even
|
||||
// though it looks symmetric with the user+key index. It is the export index: a
|
||||
// cid dropped from it is a cid whose tombstone never travels, and a tombstone
|
||||
// that does not travel is exactly how an un-liked title comes back from the
|
||||
// dead on the other box — the peer still holds the older `like` at a lower
|
||||
// version, and with no tombstone to beat it, nothing ever supersedes it.
|
||||
//
|
||||
// Returns accepted=false when the write is stale.
|
||||
func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
|
||||
accepted := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
recKey := udlRecKey(uid, cid, key)
|
||||
stored, ok, err := readUDLCell(tx.Context(), tx, recKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok && stored.Version >= version {
|
||||
return nil
|
||||
}
|
||||
cell := udlCellJSON{Record: record, Version: version, Ts: ts}
|
||||
if hasValue {
|
||||
v := publicValue
|
||||
cell.Value = &v
|
||||
}
|
||||
body, err := json.Marshal(cell)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Put(recKey, string(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
if tombstone {
|
||||
err = udlRem(tx, udlIdxUserKey(uid, key), cid)
|
||||
} else {
|
||||
err = udlAdd(tx, udlIdxUserKey(uid, key), cid)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, add := range [][2]string{
|
||||
{udlIdxUserCid(uid, cid), key},
|
||||
{udlIdxCidKey(cid, key), uid},
|
||||
{udlIdxCidUsers(cid), uid},
|
||||
{udlIdxUserCids(uid), cid},
|
||||
} {
|
||||
if err := udlAdd(tx, add[0], add[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
accepted = true
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("udl upsert failed: %w", err)
|
||||
}
|
||||
return accepted, nil
|
||||
}
|
||||
|
||||
// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
|
||||
// no record has ever been written for the cell.
|
||||
func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return UDLCell{}, false, err
|
||||
// UDLGetClaim returns the stored claim of uid on (cid, key), tombstone
|
||||
// included (a client needs its ts to write a newer one). ok is false when the
|
||||
// signer never voted on it.
|
||||
func (c *Client) UDLGetClaim(uid, cid, key string) (claim.Claim, bool, error) {
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return claim.Claim{}, false, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
return readUDLCell(ctx, s, udlRecKey(uid, cid, key))
|
||||
cl := claim.Claim{CID: cid, UID: uid, Key: key}
|
||||
var raw any
|
||||
var ts int64
|
||||
err := c.truth.DB().QueryRowContext(ctx,
|
||||
`SELECT c.value, c.ts, c.sig FROM claim c JOIN principal p ON p.pid = c.pid
|
||||
WHERE c.cid = ? AND c.key = ? AND p.uid = ?`, cid, key, uid).Scan(&raw, &ts, &cl.Sig)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return claim.Claim{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return claim.Claim{}, false, err
|
||||
}
|
||||
cl.Value, cl.TS = scanValue(raw), uint64(ts)
|
||||
return cl, true, nil
|
||||
}
|
||||
|
||||
// UDLListUserKey returns every cell for (uid, key) across all cids — the
|
||||
// projection behind active_for_user_key (My List, Continue Watching). Includes
|
||||
// tombstones; the caller decodes each record and filters.
|
||||
func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
|
||||
s, err := c.kv()
|
||||
// UDLClaimsForUser returns uid's live claims, optionally narrowed to one cid
|
||||
// and/or one key, ordered by cid then key.
|
||||
func (c *Client) UDLClaimsForUser(uid, cid, key string) ([]claim.Claim, error) {
|
||||
refs, err := c.LiveClaimsBySigner(uid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recKeys := make([]string, len(cids))
|
||||
for i, cid := range cids {
|
||||
recKeys[i] = udlRecKey(uid, cid, key)
|
||||
}
|
||||
cells, err := readUDLCells(ctx, s, recKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]UDLUserKeyEntry, 0, len(cids))
|
||||
for i, cid := range cids {
|
||||
cell, ok := cells[recKeys[i]]
|
||||
if !ok {
|
||||
var out []claim.Claim
|
||||
for _, r := range refs {
|
||||
if (cid != "" && r[0] != cid) || (key != "" && r[1] != key) {
|
||||
continue
|
||||
}
|
||||
out = append(out, UDLUserKeyEntry{Cid: cid, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLListUserCid returns every cell for (uid, cid) across all keys — the
|
||||
// projection behind list_for_user_cid.
|
||||
func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recKeys := make([]string, len(keys))
|
||||
for i, key := range keys {
|
||||
recKeys[i] = udlRecKey(uid, cid, key)
|
||||
}
|
||||
cells, err := readUDLCells(ctx, s, recKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]UDLUserCidEntry, 0, len(keys))
|
||||
for i, key := range keys {
|
||||
cell, ok := cells[recKeys[i]]
|
||||
if !ok {
|
||||
continue
|
||||
cl, ok, err := c.UDLGetClaim(uid, r[0], r[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok && !cl.Value.IsNull() {
|
||||
out = append(out, cl)
|
||||
}
|
||||
out = append(out, UDLUserCidEntry{Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLUserAllEntry is one row of a whole-profile export.
|
||||
type UDLUserAllEntry struct {
|
||||
Cid string `json:"cid"`
|
||||
Key string `json:"key"`
|
||||
Record string `json:"record"`
|
||||
Version int64 `json:"version"`
|
||||
Ts int64 `json:"ts"`
|
||||
}
|
||||
|
||||
// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
|
||||
//
|
||||
// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never
|
||||
// split across a page: the page ends on a cid boundary once `limit` rows have
|
||||
// been emitted, and `nextCid` is the last cid included — pass it back as
|
||||
// `afterCid` to continue. `nextCid` is empty when the export is complete.
|
||||
//
|
||||
// Tombstones are included. They are the CRDT state that stops a deleted value
|
||||
// resurrecting on the box that receives this export, so omitting them would
|
||||
// make recovery actively wrong rather than merely incomplete.
|
||||
func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
|
||||
s, err := c.kv()
|
||||
// UDLAllForUser pages uid's live claims by cid: claims of cids strictly after
|
||||
// `after`, stopping at a cid boundary once limit claims are collected.
|
||||
// nextCid is "" when the walk is complete.
|
||||
func (c *Client) UDLAllForUser(uid, after string, limit int) ([]claim.Claim, string, error) {
|
||||
all, err := c.UDLClaimsForUser(uid, "", "")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 1000
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
cids, err := udlMembers(ctx, s, udlIdxUserCids(uid))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sort.Strings(cids)
|
||||
|
||||
out := make([]UDLUserAllEntry, 0, limit)
|
||||
nextCid := ""
|
||||
for _, cid := range cids {
|
||||
if afterCid != "" && cid <= afterCid {
|
||||
var out []claim.Claim
|
||||
for _, cl := range all {
|
||||
if after != "" && cl.CID <= after {
|
||||
continue
|
||||
}
|
||||
if len(out) >= limit {
|
||||
nextCid = out[len(out)-1].Cid
|
||||
break
|
||||
}
|
||||
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for _, key := range keys {
|
||||
cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, UDLUserAllEntry{
|
||||
Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts,
|
||||
})
|
||||
if len(out) >= limit && cl.CID != out[len(out)-1].CID {
|
||||
return out, out[len(out)-1].CID, nil
|
||||
}
|
||||
out = append(out, cl)
|
||||
}
|
||||
return out, nextCid, nil
|
||||
return out, "", nil
|
||||
}
|
||||
|
||||
// UDLCountUserCids is the number of distinct cids a uid holds records for.
|
||||
//
|
||||
// One index range count, which is what makes it usable as a per-push quota
|
||||
// check on the gateway mirror — a quota that cost a keyspace walk would be a
|
||||
// denial-of-service in its own right. Counts cids rather than cells because
|
||||
// that is what the index holds; a profile is bounded by the titles it has
|
||||
// touched, not by how many keys it has per title.
|
||||
func (c *Client) UDLCountUserCids(uid string) (int64, error) {
|
||||
s, err := c.kv()
|
||||
// UDLCountUserCids counts the distinct cids uid holds live claims on.
|
||||
func (c *Client) UDLCountUserCids(uid string) (int, error) {
|
||||
refs, err := c.LiveClaimsBySigner(uid)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep)
|
||||
seen := map[string]bool{}
|
||||
for _, r := range refs {
|
||||
seen[r[0]] = true
|
||||
}
|
||||
return len(seen), nil
|
||||
}
|
||||
|
||||
// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
|
||||
// UDLCidUsers lists the signers holding live claims on cid.
|
||||
func (c *Client) UDLCidUsers(cid string) ([]string, error) {
|
||||
s, err := c.kv()
|
||||
claims, err := c.ClaimsForCIDs([]string{cid}, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
return udlMembers(ctx, s, udlIdxCidUsers(cid))
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for _, cl := range claims {
|
||||
if !seen[cl.UID] {
|
||||
seen[cl.UID] = true
|
||||
out = append(out, cl.UID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLPublicValues returns the live plaintext public JSON scalar values for
|
||||
// (cid, key) across all uids — the input to aggregation. Cells without a value
|
||||
// (private-tier or tombstoned) are skipped.
|
||||
// UDLPublicValues returns the plaintext values of every live claim on
|
||||
// (cid, key) — the input of /api/udl/cid/{cid}/aggregate.
|
||||
func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
|
||||
s, err := c.kv()
|
||||
claims, err := c.ClaimsForCIDs([]string{cid}, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []string{}
|
||||
for _, cl := range claims {
|
||||
if cl.Value.Kind == claim.KindText {
|
||||
out = append(out, cl.Value.Text)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLAllUserStats summarises every signer that holds live claims, keyed by
|
||||
// uid. Callers that want only human accounts intersect it with the account
|
||||
// roster.
|
||||
func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := c.truthStore(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key))
|
||||
rows, err := s.DB().QueryContext(ctx,
|
||||
`SELECT pid, count(*), count(DISTINCT cid), count(DISTINCT key) FROM claim_by_pid GROUP BY pid`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recKeys := make([]string, len(uids))
|
||||
for i, uid := range uids {
|
||||
recKeys[i] = udlRecKey(uid, cid, key)
|
||||
type agg struct {
|
||||
pid int64
|
||||
st UDLUserStats
|
||||
}
|
||||
cells, err := readUDLCells(ctx, s, recKeys)
|
||||
if err != nil {
|
||||
var aggs []agg
|
||||
for rows.Next() {
|
||||
var a agg
|
||||
if err := rows.Scan(&a.pid, &a.st.Records, &a.st.Cids, &a.st.Keys); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
aggs = append(aggs, a)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]string, 0, len(uids))
|
||||
for _, k := range recKeys {
|
||||
if cell, ok := cells[k]; ok && cell.HasValue {
|
||||
out = append(out, cell.Value)
|
||||
out := make(map[string]UDLUserStats, len(aggs))
|
||||
for _, a := range aggs {
|
||||
var uid string
|
||||
if err := c.truth.DB().QueryRowContext(ctx, `SELECT uid FROM principal WHERE pid = ?`, a.pid).Scan(&uid); err != nil {
|
||||
continue
|
||||
}
|
||||
if identity.ValidUID(uid) {
|
||||
out[uid] = a.st
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package storage
|
||||
|
||||
// Account-scoped UDL administration: what a uid holds, and removing all of it.
|
||||
//
|
||||
// Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
|
||||
// cell. Keys are ordered, so one account's cells are the key range under
|
||||
// `udl:rec:<uid>/`, and every account's are the range under `udl:rec:`.
|
||||
// UDLAllUserStats stays a bulk call (one walk buckets every account) because
|
||||
// the accounts page needs a number for each.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// UDLUserStats counts what one account holds. Records is the number of cells,
|
||||
// including tombstones — a retracted rating is still a row this account owns
|
||||
// and still something a purge has to remove.
|
||||
type UDLUserStats struct {
|
||||
Records int `json:"records"`
|
||||
Cids int `json:"cids"`
|
||||
Keys int `json:"keys"`
|
||||
}
|
||||
|
||||
// safeUIDForPattern rejects anything that would reach across the key
|
||||
// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric
|
||||
// (identity.validUID), so this only ever fires on a hostile or malformed one —
|
||||
// but it fires here, in the layer that builds the pattern, rather than relying
|
||||
// on every caller having validated first.
|
||||
func safeUIDForPattern(uid string) error {
|
||||
if uid == "" {
|
||||
return fmt.Errorf("empty uid")
|
||||
}
|
||||
if strings.ContainsAny(uid, "*?[]\\/:") {
|
||||
return fmt.Errorf("uid contains characters that are not valid in a uid: %q", uid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitUDLRecSuffix splits the "<uid>/<cid>/<key>" tail of a udl:rec key.
|
||||
//
|
||||
// SplitN with a limit of 3 so a key containing a slash survives intact. cids
|
||||
// never contain one; keys are a controlled vocabulary today, but a namespaced
|
||||
// one arriving later must not silently truncate here — a truncated key would
|
||||
// make the purge SREM the wrong index member and leave the real one behind.
|
||||
func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
|
||||
parts := strings.SplitN(suffix, "/", 3)
|
||||
if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
return parts[0], parts[1], parts[2], true
|
||||
}
|
||||
|
||||
// UDLUserCells returns every (cid, key) pair this account holds a cell for,
|
||||
// keyed by the "cid/key" tail with the cid as value (the purge needs them
|
||||
// grouped).
|
||||
func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
|
||||
if err := safeUIDForPattern(uid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
out := make(map[string]string)
|
||||
err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error {
|
||||
for _, row := range rows {
|
||||
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
|
||||
if !ok || gotUID != uid {
|
||||
continue
|
||||
}
|
||||
out[cid+"/"+key] = cid
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// UDLAllUserStats counts every account's cells in one walk.
|
||||
//
|
||||
// Returns only accounts that actually hold records; a uid absent from the map
|
||||
// holds none. Callers render a zero for those rather than treating it as an
|
||||
// error — an account created a minute ago legitimately has nothing.
|
||||
func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
type acc struct {
|
||||
cells int
|
||||
cids map[string]struct{}
|
||||
keys map[string]struct{}
|
||||
}
|
||||
byUID := make(map[string]*acc)
|
||||
err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error {
|
||||
for _, row := range rows {
|
||||
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
a := byUID[uid]
|
||||
if a == nil {
|
||||
a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})}
|
||||
byUID[uid] = a
|
||||
}
|
||||
a.cells++
|
||||
a.cids[cid] = struct{}{}
|
||||
a.keys[key] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]UDLUserStats, len(byUID))
|
||||
for uid, a := range byUID {
|
||||
out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UDLPurgeUser removes every trace of one account from the User Data Layer:
|
||||
// its cells, its three user-scoped indexes, and its membership in the shared
|
||||
// cid-scoped indexes. Returns what was removed.
|
||||
//
|
||||
// The shared indexes are the part that is easy to miss and expensive to leave.
|
||||
// `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
|
||||
// not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
|
||||
// uid listed as an opinion-holder on every title it ever touched.
|
||||
//
|
||||
// Idempotent: purging an account with nothing stored is a no-op returning
|
||||
// zeroes, so a retry is safe.
|
||||
func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
|
||||
cells, err := c.UDLUserCells(uid)
|
||||
if err != nil {
|
||||
return UDLUserStats{}, err
|
||||
}
|
||||
|
||||
stats := UDLUserStats{Records: len(cells)}
|
||||
distinctCids := make(map[string]struct{})
|
||||
distinctKeys := make(map[string]struct{})
|
||||
err = c.update(func(tx *kv.Tx) error {
|
||||
for cidKey, cid := range cells {
|
||||
key := strings.TrimPrefix(cidKey, cid+"/")
|
||||
distinctCids[cid] = struct{}{}
|
||||
distinctKeys[key] = struct{}{}
|
||||
if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for cid := range distinctCids {
|
||||
if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one
|
||||
// key range, dropped whole — which also catches index entries whose
|
||||
// cell is already gone (the state a half-finished purge leaves).
|
||||
_, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return UDLUserStats{}, fmt.Errorf("purge: %w", err)
|
||||
}
|
||||
stats.Cids = len(distinctCids)
|
||||
stats.Keys = len(distinctKeys)
|
||||
return stats, nil
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seedCell writes one live UDL cell the way a real client would.
|
||||
func seedCell(t *testing.T, c *Client, uid, cid, key string) {
|
||||
t.Helper()
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
|
||||
t.Fatalf("seed %s/%s/%s: ok=%v err=%v", uid, cid, key, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The purge has to reach the cid-scoped indexes, not just the user's own rows.
|
||||
//
|
||||
// This is the failure that motivated the whole endpoint: deleting an account
|
||||
// while leaving `udl:idx:cid:<cid>:users` intact means every aggregate read on
|
||||
// every title that account ever touched keeps naming a uid nobody can resolve,
|
||||
// forever — and the rows are unreachable through any API, so nothing will ever
|
||||
// clean them up.
|
||||
func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
seedCell(t, c, "zAlice", "cidOne", "rating")
|
||||
seedCell(t, c, "zAlice", "cidTwo", "seek")
|
||||
|
||||
stats, err := c.UDLPurgeUser("zAlice")
|
||||
if err != nil {
|
||||
t.Fatalf("purge: %v", err)
|
||||
}
|
||||
if stats.Records != 3 || stats.Cids != 2 || stats.Keys != 3 {
|
||||
t.Fatalf("stats = %+v, want 3 records / 2 cids / 3 keys", stats)
|
||||
}
|
||||
|
||||
// Cells gone.
|
||||
if _, ok, err := c.UDLGetCell("zAlice", "cidOne", "like"); err != nil || ok {
|
||||
t.Fatalf("cell survived the purge: ok=%v err=%v", ok, err)
|
||||
}
|
||||
// User-scoped indexes gone.
|
||||
if entries, err := c.UDLListUserKey("zAlice", "like"); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("user+key index survived: %d entries, err=%v", len(entries), err)
|
||||
}
|
||||
if entries, err := c.UDLListUserCid("zAlice", "cidOne"); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("user+cid index survived: %d entries, err=%v", len(entries), err)
|
||||
}
|
||||
// Cid-scoped index gone — the one that is easy to miss.
|
||||
for _, cid := range []string{"cidOne", "cidTwo"} {
|
||||
uids, err := c.UDLCidUsers(cid)
|
||||
if err != nil {
|
||||
t.Fatalf("cid users %s: %v", cid, err)
|
||||
}
|
||||
for _, u := range uids {
|
||||
if u == "zAlice" {
|
||||
t.Fatalf("%s still lists the deleted account in uids_for_cid", cid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Purging one account must not touch another's rows, including on a cid they
|
||||
// both have an opinion about — where the two share an index set and a careless
|
||||
// DEL of the whole key would take the survivor's membership with it.
|
||||
func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "shared", "rating")
|
||||
seedCell(t, c, "zBob", "shared", "rating")
|
||||
|
||||
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
|
||||
t.Fatalf("purge: %v", err)
|
||||
}
|
||||
|
||||
if _, ok, err := c.UDLGetCell("zBob", "shared", "rating"); err != nil || !ok {
|
||||
t.Fatalf("bob's cell must survive alice's purge: ok=%v err=%v", ok, err)
|
||||
}
|
||||
uids, err := c.UDLCidUsers("shared")
|
||||
if err != nil {
|
||||
t.Fatalf("cid users: %v", err)
|
||||
}
|
||||
if len(uids) != 1 || uids[0] != "zBob" {
|
||||
t.Fatalf("uids_for_cid = %v, want exactly [zBob]", uids)
|
||||
}
|
||||
entries, err := c.UDLListUserKey("zBob", "rating")
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("bob's user+key index must survive: %d entries, err=%v", len(entries), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A uid that is a prefix of another must not drag its neighbour's rows out.
|
||||
// The key layout separates them with '/' and ':', so this passes only as long
|
||||
// as the scan patterns keep those separators.
|
||||
func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
seedCell(t, c, "zAliceExtra", "cidOne", "like")
|
||||
|
||||
stats, err := c.UDLPurgeUser("zAlice")
|
||||
if err != nil {
|
||||
t.Fatalf("purge: %v", err)
|
||||
}
|
||||
if stats.Records != 1 {
|
||||
t.Fatalf("purged %d records, want exactly alice's 1", stats.Records)
|
||||
}
|
||||
if _, ok, err := c.UDLGetCell("zAliceExtra", "cidOne", "like"); err != nil || !ok {
|
||||
t.Fatalf("zAliceExtra's cell must survive: ok=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tombstoned cells still belong to the account. They are absent from the
|
||||
// user+key index by design (see udl.go), so a purge that enumerated *that*
|
||||
// index instead of the cells would leave every retracted rating behind — the
|
||||
// rows most likely to have accumulated over an account's life.
|
||||
func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
|
||||
t.Fatalf("tombstone: ok=%v err=%v", ok, err)
|
||||
}
|
||||
// Precondition: the tombstone left the cell but dropped it from the index.
|
||||
if entries, _ := c.UDLListUserKey("zAlice", "like"); len(entries) != 0 {
|
||||
t.Fatalf("precondition: tombstone should be out of the user+key index")
|
||||
}
|
||||
|
||||
stats, err := c.UDLPurgeUser("zAlice")
|
||||
if err != nil {
|
||||
t.Fatalf("purge: %v", err)
|
||||
}
|
||||
if stats.Records != 1 {
|
||||
t.Fatalf("purged %d records, want the 1 tombstoned cell", stats.Records)
|
||||
}
|
||||
if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); ok {
|
||||
t.Fatal("tombstoned cell survived the purge")
|
||||
}
|
||||
}
|
||||
|
||||
// Retry-safe: the delete handler purges before removing the key, so a caller
|
||||
// that retries after a partial failure runs this a second time.
|
||||
func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
|
||||
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
|
||||
t.Fatalf("first purge: %v", err)
|
||||
}
|
||||
stats, err := c.UDLPurgeUser("zAlice")
|
||||
if err != nil {
|
||||
t.Fatalf("second purge must be a no-op, got: %v", err)
|
||||
}
|
||||
if stats.Records != 0 || stats.Cids != 0 {
|
||||
t.Fatalf("second purge = %+v, want zeroes", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
|
||||
// "*" would otherwise match every account in the keyspace — a wildcard
|
||||
// purge dressed up as a uid.
|
||||
for _, uid := range []string{"*", "z*", "", "z:Alice", "z/Alice"} {
|
||||
if _, err := c.UDLPurgeUser(uid); err == nil {
|
||||
t.Errorf("uid %q must be refused before it reaches a SCAN pattern", uid)
|
||||
}
|
||||
}
|
||||
if _, ok, _ := c.UDLGetCell("zAlice", "cidOne", "like"); !ok {
|
||||
t.Fatal("a refused purge must not have deleted anything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
seedCell(t, c, "zAlice", "cidOne", "rating")
|
||||
seedCell(t, c, "zAlice", "cidTwo", "like")
|
||||
seedCell(t, c, "zBob", "cidOne", "seek")
|
||||
|
||||
stats, err := c.UDLAllUserStats()
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
if got := stats["zAlice"]; got.Records != 3 || got.Cids != 2 || got.Keys != 2 {
|
||||
t.Fatalf("alice = %+v, want 3 records / 2 cids / 2 keys", got)
|
||||
}
|
||||
if got := stats["zBob"]; got.Records != 1 || got.Cids != 1 || got.Keys != 1 {
|
||||
t.Fatalf("bob = %+v, want 1/1/1", got)
|
||||
}
|
||||
// An account with nothing stored is simply absent — the caller renders 0.
|
||||
if _, present := stats["zNobody"]; present {
|
||||
t.Fatal("an account with no records must not appear in the map")
|
||||
}
|
||||
}
|
||||
|
||||
// The stats and the purge must agree: the number shown in the delete
|
||||
// confirmation is the number of rows the delete then removes.
|
||||
func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
seedCell(t, c, "zAlice", "cidOne", "like")
|
||||
seedCell(t, c, "zAlice", "cidTwo", "rating")
|
||||
seedCell(t, c, "zAlice", "cidTwo", "seek")
|
||||
|
||||
before, err := c.UDLAllUserStats()
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
purged, err := c.UDLPurgeUser("zAlice")
|
||||
if err != nil {
|
||||
t.Fatalf("purge: %v", err)
|
||||
}
|
||||
if before["zAlice"] != purged {
|
||||
t.Fatalf("stats said %+v but purge removed %+v", before["zAlice"], purged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
|
||||
// A namespaced key arriving later must not be truncated: the purge SREMs
|
||||
// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong
|
||||
// member and leaves the real one behind.
|
||||
uid, cid, key, ok := splitUDLRecSuffix("zAlice/cidOne/ns/sub:key")
|
||||
if !ok || uid != "zAlice" || cid != "cidOne" || key != "ns/sub:key" {
|
||||
t.Fatalf("got (%q,%q,%q,%v)", uid, cid, key, ok)
|
||||
}
|
||||
for _, bad := range []string{"", "zAlice", "zAlice/cidOne", "zAlice//key", "/cid/key"} {
|
||||
if _, _, _, ok := splitUDLRecSuffix(bad); ok {
|
||||
t.Errorf("%q should not parse", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The whole-profile export is what cross-instance recovery reads: a box that
|
||||
// has just imported a secret key knows the uid and nothing else — not which
|
||||
// cids the profile touched, not which keys. Every one of these tests is about
|
||||
// that one caller.
|
||||
|
||||
func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
const uid = "zAlice"
|
||||
writes := []struct{ cid, key, rec string }{
|
||||
{"bagcsaaaaone", "like", "rec-one-like"},
|
||||
{"bagcsaaaaone", "seek", "rec-one-seek"},
|
||||
{"bagcsaaaatwo", "rating", "rec-two-rating"},
|
||||
{"bagcsaaaatwo", "profile:name", "rec-two-name"},
|
||||
{"bagcsaaathree", "watched", "rec-three-watched"},
|
||||
}
|
||||
for _, w := range writes {
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, w.cid, w.key, 1, 100, w.rec, "", false, false); err != nil || !ok {
|
||||
t.Fatalf("upsert %s/%s: ok=%v err=%v", w.cid, w.key, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
entries, next, err := c.UDLAllForUser(uid, "", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("export: %v", err)
|
||||
}
|
||||
if next != "" {
|
||||
t.Fatalf("export of 5 rows should be complete, got nextCid=%q", next)
|
||||
}
|
||||
if len(entries) != len(writes) {
|
||||
t.Fatalf("want %d rows, got %d: %+v", len(writes), len(entries), entries)
|
||||
}
|
||||
seen := map[string]string{}
|
||||
for _, e := range entries {
|
||||
seen[e.Cid+"/"+e.Key] = e.Record
|
||||
}
|
||||
for _, w := range writes {
|
||||
if got := seen[w.cid+"/"+w.key]; got != w.rec {
|
||||
t.Errorf("%s/%s: want record %q, got %q", w.cid, w.key, w.rec, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The failure this guards is silent and total: a uid whose cids were never
|
||||
// indexed exports as empty, and recovery reports success having restored
|
||||
// nothing.
|
||||
func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
|
||||
t.Fatalf("upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
entries, _, err := c.UDLAllForUser("zBob", "", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("export: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("Bob owns nothing; got %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// ⚠ The tombstone rule, and the reason the cids index is never pruned.
|
||||
//
|
||||
// A tombstone must EXPORT. It is the only thing that can beat the older live
|
||||
// value the other box still holds at a lower version — drop it from the export
|
||||
// and an un-liked title comes back from the dead on every peer that already
|
||||
// saw the `like`. This is the one place the cids index deliberately diverges
|
||||
// from the user+key index, which does prune.
|
||||
func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
const (
|
||||
uid = "zAlice"
|
||||
cid = "bagcsaaaaone"
|
||||
key = "like"
|
||||
)
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-liked", "", false, false); err != nil || !ok {
|
||||
t.Fatalf("live upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-unliked", "", false, true); err != nil || !ok {
|
||||
t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
|
||||
// The user+key index prunes it — that is what keeps My List short.
|
||||
live, err := c.UDLListUserKey(uid, key)
|
||||
if err != nil {
|
||||
t.Fatalf("active_for_user_key: %v", err)
|
||||
}
|
||||
if len(live) != 0 {
|
||||
t.Fatalf("tombstoned cid should leave the user+key index, got %+v", live)
|
||||
}
|
||||
|
||||
// The export must still carry it, at the tombstone's version.
|
||||
entries, _, err := c.UDLAllForUser(uid, "", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("export: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("tombstone must export, got %+v", entries)
|
||||
}
|
||||
if entries[0].Record != "rec-unliked" || entries[0].Version != 2 {
|
||||
t.Fatalf("export must carry the tombstone, not the superseded value: %+v", entries[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
const uid = "zAlice"
|
||||
cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
|
||||
for _, cid := range cids {
|
||||
for _, key := range []string{"like", "seek"} {
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-"+cid+"-"+key, "", false, false); err != nil || !ok {
|
||||
t.Fatalf("upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cid's keys are never split across a page, so a limit of 1 still yields
|
||||
// both of the first cid's rows.
|
||||
page1, next1, err := c.UDLAllForUser(uid, "", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("page 1: %v", err)
|
||||
}
|
||||
if len(page1) != 2 || page1[0].Cid != cids[0] {
|
||||
t.Fatalf("page 1 should hold both rows of %s, got %+v", cids[0], page1)
|
||||
}
|
||||
if next1 != cids[0] {
|
||||
t.Fatalf("cursor should name the last cid emitted, want %q got %q", cids[0], next1)
|
||||
}
|
||||
|
||||
// Walking the cursor must visit every row exactly once.
|
||||
all := append([]UDLUserAllEntry{}, page1...)
|
||||
cursor := next1
|
||||
for cursor != "" {
|
||||
page, next, err := c.UDLAllForUser(uid, cursor, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("page after %q: %v", cursor, err)
|
||||
}
|
||||
all = append(all, page...)
|
||||
cursor = next
|
||||
}
|
||||
if len(all) != 6 {
|
||||
t.Fatalf("want 6 rows across all pages, got %d: %+v", len(all), all)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, e := range all {
|
||||
k := e.Cid + "/" + e.Key
|
||||
if seen[k] {
|
||||
t.Fatalf("%s emitted twice across pages", k)
|
||||
}
|
||||
seen[k] = true
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A tombstone must leave the cell in place but drop the cid from the user+key
|
||||
// index.
|
||||
//
|
||||
// The cell has to stay: it *is* the CRDT state, and deleting it would let an
|
||||
// older write resurrect the value the tombstone retracted. The index entry has
|
||||
// to go: `active_for_user_key` returns every member of that set, so a set that
|
||||
// only ever grows means each My List / Continue Watching read carries every
|
||||
// title the user has ever un-liked or finished — with its full record blob —
|
||||
// forever. The reader filters them out again, so the cost is invisible until
|
||||
// the account is old.
|
||||
func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
|
||||
c := newTestClient(t)
|
||||
|
||||
const (
|
||||
uid = "zAlice"
|
||||
cid = "bagcsaaalocator"
|
||||
key = "seek"
|
||||
)
|
||||
|
||||
// A live position: indexed, readable.
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-v1", "", false, false); err != nil || !ok {
|
||||
t.Fatalf("live upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
entries, err := c.UDLListUserKey(uid, key)
|
||||
if err != nil {
|
||||
t.Fatalf("active_for_user_key: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Cid != cid {
|
||||
t.Fatalf("live position should be indexed, got %+v", entries)
|
||||
}
|
||||
|
||||
// Finishing it tombstones the seek.
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-v2-tombstone", "", false, true); err != nil || !ok {
|
||||
t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
|
||||
entries, err = c.UDLListUserKey(uid, key)
|
||||
if err != nil {
|
||||
t.Fatalf("active_for_user_key after tombstone: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("tombstoned cid must leave the user+key index, got %+v", entries)
|
||||
}
|
||||
|
||||
// ...but the cell itself survives, at the tombstone's version.
|
||||
cell, exists, err := c.UDLGetCell(uid, cid, key)
|
||||
if err != nil {
|
||||
t.Fatalf("get cell: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatal("tombstone must not delete the cell — an older write could then resurrect the value")
|
||||
}
|
||||
if cell.Version != 2 || cell.Record != "rec-v2-tombstone" {
|
||||
t.Fatalf("cell should hold the tombstone record at v2, got %+v", cell)
|
||||
}
|
||||
|
||||
// And the high-water mark still rejects the superseded write.
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 300, "rec-v1-replay", "", false, false); err != nil || ok {
|
||||
t.Fatalf("stale replay must be rejected: ok=%v err=%v", ok, err)
|
||||
}
|
||||
|
||||
// Re-watching re-establishes the cell and puts it back in the index.
|
||||
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 3, 400, "rec-v3", "", false, false); err != nil || !ok {
|
||||
t.Fatalf("re-establish upsert: ok=%v err=%v", ok, err)
|
||||
}
|
||||
entries, err = c.UDLListUserKey(uid, key)
|
||||
if err != nil {
|
||||
t.Fatalf("active_for_user_key after re-establish: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Cid != cid {
|
||||
t.Fatalf("re-established cid must return to the index, got %+v", entries)
|
||||
}
|
||||
}
|
||||
@@ -1,109 +1,66 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
)
|
||||
|
||||
// BenchmarkConcurrentRecordWrites is the ingest shape the single-writer model
|
||||
// has to absorb: many goroutines (feeder flushes, gateway persists, plugin
|
||||
// callbacks) each writing a ~40-field record. SQLite serialises them; this
|
||||
// measures what that costs. Reported as records/s.
|
||||
func BenchmarkConcurrentRecordWrites(b *testing.B) {
|
||||
c := NewClient()
|
||||
if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
|
||||
b.Fatal(err)
|
||||
// benchRecord is a 40-field signed record, the shape gateway hits persist.
|
||||
func benchRecord(b *testing.B, c *Client, i int) []claim.Claim {
|
||||
t := &testing.T{}
|
||||
id := signer(t, byte(1+i%7))
|
||||
cid := fmt.Sprintf("bafkbench%012d", i)
|
||||
out := make([]claim.Claim, 0, 40)
|
||||
for f := 0; f < 40; f++ {
|
||||
out = append(out, vote(t, id, cid, fmt.Sprintf("field%02d", f), fmt.Sprintf("value-%d-%d", i, f), 1000))
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
record := func(i int64) map[string]string {
|
||||
m := map[string]string{
|
||||
"title": fmt.Sprintf("Some Show S01E%02d", i%40),
|
||||
"fileName": fmt.Sprintf("some.show.s01e%02d.1080p.mkv", i%40),
|
||||
"filePath": fmt.Sprintf("/files/watch/some.show.%d.mkv", i),
|
||||
"fileType": "video",
|
||||
"tmdbid": fmt.Sprintf("%d", 1000+i%500),
|
||||
fmt.Sprintf("cids/bafkbench%020d", i): "true",
|
||||
}
|
||||
for f := 0; len(m) < 40; f++ {
|
||||
m[fmt.Sprintf("categories/%d", f)] = "true"
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
const writers = 16
|
||||
var next int64
|
||||
b.ResetTimer()
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < writers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
i := atomic.AddInt64(&next, 1)
|
||||
if i > int64(b.N) {
|
||||
return
|
||||
}
|
||||
if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), record(i)); err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
|
||||
return out
|
||||
}
|
||||
|
||||
// BenchmarkConcurrentRepersistUnchanged is the dominant write on a permanent
|
||||
// store: a gateway re-persisting hits it already holds. The kv layer skips
|
||||
// every unchanged field, so this should cost reads only.
|
||||
func BenchmarkConcurrentRepersistUnchanged(b *testing.B) {
|
||||
c := NewClient()
|
||||
if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// BenchmarkConcurrentClaimWrites measures new 40-claim records, 16 writers.
|
||||
func BenchmarkConcurrentClaimWrites(b *testing.B) {
|
||||
c := openClient(&testing.T{}, b.TempDir())
|
||||
defer c.Close()
|
||||
const distinct = 2000
|
||||
rec := func(i int64) map[string]string {
|
||||
m := map[string]string{
|
||||
"title": fmt.Sprintf("Some Show S01E%02d", i%40),
|
||||
"fileType": "video",
|
||||
fmt.Sprintf("cids/bafkbench%020d", i): "true",
|
||||
}
|
||||
for f := 0; len(m) < 40; f++ {
|
||||
m[fmt.Sprintf("categories/%d", f)] = "true"
|
||||
}
|
||||
return m
|
||||
recs := make([][]claim.Claim, b.N)
|
||||
for i := range recs {
|
||||
recs[i] = benchRecord(b, c, i)
|
||||
}
|
||||
for i := int64(0); i < distinct; i++ {
|
||||
if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), rec(i)); err != nil {
|
||||
b.ResetTimer()
|
||||
b.SetParallelism(16)
|
||||
var next int64
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
i := int(atomicAdd(&next) - 1)
|
||||
if _, err := c.WriteClaims(context.Background(), recs[i]); err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkRepersistUnchangedClaims measures the 94% case: the same signed
|
||||
// claims again (skip-identical before verification).
|
||||
func BenchmarkRepersistUnchangedClaims(b *testing.B) {
|
||||
c := openClient(&testing.T{}, b.TempDir())
|
||||
defer c.Close()
|
||||
const distinct = 200
|
||||
recs := make([][]claim.Claim, distinct)
|
||||
for i := range recs {
|
||||
recs[i] = benchRecord(b, c, i)
|
||||
if _, err := c.WriteClaims(context.Background(), recs[i]); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
const writers = 16
|
||||
var next int64
|
||||
b.ResetTimer()
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < writers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
i := atomic.AddInt64(&next, 1)
|
||||
if i > int64(b.N) {
|
||||
return
|
||||
}
|
||||
if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i%distinct), rec(i%distinct)); err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := c.WriteClaims(context.Background(), recs[i%distinct]); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
|
||||
}
|
||||
|
||||
func atomicAdd(p *int64) int64 { return atomic.AddInt64(p, 1) }
|
||||
|
||||
Reference in New Issue
Block a user