metastar/meta-core
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
Details

Browse Source
This commit is contained in:
Worph
2026-10-19 20:51:00 +00:00
parent c53146e514
commit dc5cbdfb9b
80 changed files with 4601 additions and 6077 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/storage/cid_resolution.go
+172 -284
View File
@@ -5,10 +5,9 @@ import (
"fmt"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/cid"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// This file implements the CID-resolution layer that lives on top of the
@@ -18,7 +17,7 @@ import (
// The UUID is an internal implementation detail; the mental model is
// "N CIDs → one record of keys→metadata".
// - Every known CID for a file is registered as a reverse-index entry:
// cid:<bareCid> → <uuid> (Redis STRING)
// cid:<bareCid> → <uuid>
// and as a member of the per-root bare-CID key-set:
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
@@ -27,14 +26,13 @@ import (
// *derived by rank* from the key-set on read (GetMetadataDocument), never
// stored.
//
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
// any write of a cids/<cid> field and register the reverse-index entry, so
// The record write path (writeFieldsTx) catches any write of a cids/<cid>
// field and registers the reverse-index entry in the same transaction, so
// writers that just emit the key-set members don't need to know about the
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
// already held by the caller, avoiding nested RLock (which can deadlock
// against a waiting writer).
// reverse index at all. The helpers here take the caller's *kv.Tx: a write
// must never open a second transaction (there is one writer connection).
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
// CIDIndexPrefix is the key prefix for the reverse index. Combined
// with a bare CID, it forms the full key: "cid:bafk…".
const CIDIndexPrefix = "cid:"
@@ -45,152 +43,112 @@ const CIDIndexPrefix = "cid:"
// reverse-index entries that need cleanup).
const CIDsKeyPrefix = "cids/"
// DuplicatesField is the per-root SET key holding additional file paths
// whose content hashes to the same CIDs as this root. The watcher writes
// here when it sees a new path for content that's already indexed; meta-dup
// reads from it.
const DuplicatesField = "duplicates"
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
// cidStr is a bare multibase CIDv1 ("bafk…").
func (c *Client) buildCIDIndexKey(cidStr string) string {
return c.buildKey(CIDIndexPrefix + cidStr)
}
// Duplicate paths — other physical locations whose content hashes to the same
// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
// writes one when it sees a new path for content that's already indexed.
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
// in the file index. Returns the new UUID. The caller is expected to follow
// in the root index. Returns the new UUID. The caller is expected to follow
// up with AddAlias for at least one CID — a root without any alias is
// reachable only via the index, which works but defeats the point.
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
var uuid string
err := c.update(func(tx *kv.Tx) error {
var err error
uuid, err = mintTx(tx, filePath, size, mtimeNano)
return err
})
return uuid, err
}
func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
uuid, err := NewUUID()
if err != nil {
return "", fmt.Errorf("uuid: %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
pipe := c.client.Pipeline()
pipe.Set(ctx, prefix+"filePath", filePath, 0)
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
if _, err := pipe.Exec(ctx); err != nil {
return "", fmt.Errorf("mint pipeline: %w", err)
}
return uuid, nil
return uuid, writeFieldsTx(tx, uuid, map[string]string{
"filePath": filePath,
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
})
}
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
// Idempotent: registering the same CID twice is a no-op.
func (c *Client) AddAlias(uuid, cidStr string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.addAliasLocked(ctx, uuid, cidStr)
}
// addAliasLocked is the lock-free body of AddAlias. The caller must already
// hold c.mu (read or write). Split out so the alias-maintenance hooks
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
// the work without re-entering the lock — nested RLock can deadlock against
// a waiting writer.
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
return c.update(func(tx *kv.Tx) error {
return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
})
}
keysetField := CIDsKeyPrefix + cidStr
keysetKey := c.buildKeyPrefix(uuid) + keysetField
// Every branch below writes the key-set member, so index the field name
// once here — including the two guard branches that skip the alias itself.
// Missing it would leave `cids/<cid>` invisible to a record read.
defer c.addFieldLocked(ctx, uuid, keysetField)
// addAliasTx registers the reverse-index entry for a cids/<cid> member the
// caller is writing (writeFieldsTx writes the member itself).
func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
// Self-pointing alias guard. If the caller is trying to register
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
// dual-root failure mode in disguise — it means somebody wrote
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
// write; leave any existing alias alone. Still record the key-set
// membership so the field-level write succeeds.
// write; leave any existing alias alone. The key-set membership is still
// recorded by the caller, so the field-level write succeeds.
if uuid == cidStr {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
return nil
}
// Guard against the dual-root pattern: if cid:<cid> already points at a
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
// just record the key-set membership so it's still discoverable via the
// root → cids enumeration.
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != "" && existing != uuid {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
}
pipe := c.client.Pipeline()
pipe.Set(ctx, indexKey, uuid, 0)
pipe.Set(ctx, keysetKey, "true", 0)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("add alias pipeline: %w", err)
}
return nil
}
// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller must already hold c.mu, and must delete
// the cids/<cid> key-set member itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat
// file:<uuid>/cids/<cid>, the __fields__ index entry, and this reverse alias —
// while the delete path only ever removed the first two. A "deleted" CID
// therefore still resolved: GET /meta/<cid>, ResolveRoot and GetByCID all kept
// answering with the record. The asymmetry is not theoretical. meta-share
// published dag-pb roots computed over RAR volumes and par2 files onto six
// records on watch.nsl.sh; removing the member was expressible, removing the
// claim was not, so the false identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasLocked, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err == redis.Nil {
return nil
}
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
// key-set membership keeps the CID discoverable via root → cids.
indexKey := CIDIndexPrefix + cidStr
existing, _, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != uuid {
// Somebody else owns this alias. Leave it.
if existing != "" && existing != uuid {
return nil
}
return c.client.Del(ctx, indexKey).Err()
if existing == uuid {
return nil
}
return tx.Put(indexKey, uuid)
}
// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
// itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches two keys — the flat
// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
// only removed the first. A "deleted" CID therefore still resolved: GET
// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
// The asymmetry is not theoretical. meta-share published dag-pb roots computed
// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
// the member was expressible, removing the claim was not, so the false
// identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasTx, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := CIDIndexPrefix + cidStr
existing, ok, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if !ok || existing != uuid {
// Absent, or somebody else owns this alias. Leave it.
return nil
}
_, err = tx.Delete(indexKey)
return err
}
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
@@ -210,8 +168,8 @@ func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) err
// - hash matches a reverse-index entry → return UUID
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
//
// Errors are converted to "no resolution" — if Redis is down or the lookup
// fails we let the caller proceed with the bare hash and surface the Redis
// Errors are converted to "no resolution" — if the store is down or the
// lookup fails we let the caller proceed with the bare hash and surface the
// failure on the actual operation.
func (c *Client) ResolveRoot(hash string) string {
if hash == "" {
@@ -228,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string {
// when the CID is unknown; the empty string is the "not found" signal rather
// than a typed error so callers can branch without errors.As.
func (c *Client) GetByCID(cidStr string) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return "", err
}
if cidStr == "" {
return "", nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
ctx, cancel := opCtx()
defer cancel()
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
if err == redis.Nil {
return "", nil
}
uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
if err != nil {
return "", fmt.Errorf("get cid index: %w", err)
}
@@ -252,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
}
// AddDuplicatePath records that path is another physical location whose
// content matches uuid. Returns true if path was newly added (not already
// in the set). The watcher calls this when it sees a file whose midhash
// matches an existing root but at a different filePath.
// content matches uuid. Returns true if path was newly added. The watcher
// calls this when it sees a file whose midhash matches an existing root but
// at a different filePath.
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
if uuid == "" || path == "" {
return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
if err != nil {
return false, fmt.Errorf("sadd duplicates: %w", err)
}
return added > 0, nil
added := false
err := c.update(func(tx *kv.Tx) error {
var err error
added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
return err
})
return added, err
}
// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
// prefix stripped). Caller must hold c.mu.
//
// Reads the field index and filters by prefix; only an un-indexed root falls
// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
// stripped).
func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
prefix := recordPrefix(uuid) + CIDsKeyPrefix
rows, err := r.Range(ctx, prefix, "", 0)
if err != nil {
return nil, err
}
out := make([]string, 0, len(rows))
for _, row := range rows {
out = append(out, row.K[len(prefix):])
}
return out, nil
}
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
// (including the cids/<cid> key-set), every reverse-index entry for those
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
// is a successful no-op.
// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
// with no remaining keys is a successful no-op.
func (c *Client) DeleteRoot(uuid string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
if uuid == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
// the list of reverse-index entries to clean up.
cids, err := c.cidsForRootLocked(ctx, uuid)
if err != nil {
return c.update(func(tx *kv.Tx) error {
_, err := deleteRecordTx(tx, uuid)
return err
}
// 2. Delete every reverse-index entry in one shot.
if len(cids) > 0 {
keys := make([]string, 0, len(cids))
for _, cidStr := range cids {
keys = append(keys, c.buildCIDIndexKey(cidStr))
}
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del cid index: %w", err)
}
}
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return fmt.Errorf("scan root keys: %w", err)
}
if len(keys) > 0 {
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del root keys: %w", err)
}
}
cursor = next
if cursor == 0 {
break
}
}
// 4. Remove from the file index set.
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
return fmt.Errorf("srem index: %w", err)
}
// Marked so the incremental search-index refresh reads it, finds it gone,
// and evicts it — otherwise a deleted root keeps answering searches until
// the next full rebuild.
c.markDirtyLocked(ctx, uuid)
return nil
}
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
// root. If it's a key-set member — field name starts with "cids/" — it
// registers the implied CID in the reverse index. Writers emit cids/<cid>
// members without knowing about the reverse index; this hook keeps it in
// sync transparently.
//
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
// (logged, not returned): an out-of-sync reverse index is recoverable via a
// sweep, while a failed property write is not.
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
cidStr := cidFromKeysetField(field)
if cidStr == "" {
return nil
}
return c.addAliasLocked(ctx, uuid, cidStr)
}
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
// over every entry in a metadata map. Used by SetMetadataFlat /
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
// hold c.mu.
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
for k := range metadata {
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
}
})
}
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
@@ -406,24 +275,23 @@ type MetadataDocument struct {
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
// never existed).
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
if uuid == "" {
return nil, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := opCtx()
defer cancel()
flat, err := c.getMetadataFlatInternal(ctx, uuid)
flat, err := readRecord(ctx, s, uuid)
if err != nil {
return nil, err
}
prefix := c.buildKeyPrefix(uuid)
if flat == nil {
flat = make(map[string]string)
}
// Hoist the cids/<cid> key-set members into a typed array and derive the
// canonical CID by rank. The members remain in Flat too — that's how
@@ -443,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
}
}
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers duplicates: %w", err)
dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
if err != nil {
return nil, fmt.Errorf("read duplicates: %w", err)
}
dups := make([]string, 0, len(dupRows))
for _, row := range dupRows {
dups = append(dups, row.K[len(dupPrefix(uuid)):])
}
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
@@ -474,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas
if midhash == "" {
return "", false, fmt.Errorf("midhash is required")
}
uuid, err := c.GetByCID(midhash)
if err != nil {
return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if uuid == "" {
uuid, err = c.Mint(absPath, size, mtimeNano)
var (
uuid string
minted bool
)
// One transaction: the lookup and the mint cannot interleave with another
// registration of the same content, so two concurrent scans of one file
// can no longer mint two roots for it.
err := c.update(func(tx *kv.Tx) error {
ctx := tx.Context()
existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
if err != nil {
return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
return fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
uuid = existing
if uuid == "" {
if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
return fmt.Errorf("Mint(%s): %w", absPath, err)
}
minted = true
if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
}
return nil
}
return uuid, true, nil
}
existing, _ := c.GetProperty(uuid, "filePath")
if existing != absPath {
if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
if err != nil {
return err
}
return uuid, false, nil
if current != absPath {
if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
}
return nil
}
if err := writeFieldsTx(tx, uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return nil
})
if err != nil {
return "", false, err
}
if _, err := c.MergeMetadataFlat(uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return uuid, false, nil
return uuid, minted, nil
}
internal/storage/cid_retraction_test.go
+18 -17
View File
@@ -6,13 +6,13 @@ import (
)
// Tests for retracting a `cids/<cid>` key-set member (cid_resolution.go's
// removeAliasLocked, reached through DeleteProperty).
// removeAliasTx, reached through DeleteProperty).
//
// Why this is worth its own file. Adding a member writes THREE keys — the flat
// `file:<uuid>/cids/<cid>`, the `__fields__` index entry, and the reverse alias
// `cid:<cid>` → `<uuid>`. Until removeAliasLocked existed, deleting removed only
// the first two, so a retracted CID kept resolving to the record: the member was
// gone from the document and the claim was not.
// Why this is worth its own file. Adding a member writes two keys — the flat
// `file:<uuid>/cids/<cid>` and the reverse alias `cid:<cid>` → `<uuid>`. Until
// removeAliasLocked (now removeAliasTx) existed, deleting removed only the
// first, so a retracted CID kept resolving to the record: the member was gone
// from the document and the claim was not.
//
// That asymmetry had a live consequence. meta-share minted dag-pb roots and
// midhash256s over whatever `archive::pick_media` elected, which on an
@@ -26,19 +26,20 @@ import (
// aliasTarget returns the uuid that cid:<cidStr> currently resolves to, or "".
func aliasTarget(t *testing.T, c *Client, cidStr string) string {
t.Helper()
got, err := c.client.Get(context.Background(), c.buildCIDIndexKey(cidStr)).Result()
got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr)
if err != nil {
return ""
t.Fatalf("get alias: %v", err)
}
return got
}
func hasField(t *testing.T, c *Client, hashID, field string) bool {
t.Helper()
ok, err := c.client.SIsMember(context.Background(), c.buildFieldsKey(hashID), field).Result()
fields, err := c.GetMetadataFlat(hashID)
if err != nil {
t.Fatalf("sismember: %v", err)
t.Fatalf("read record: %v", err)
}
_, ok := fields[field]
return ok
}
@@ -50,7 +51,7 @@ const (
)
func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const uuid = "01KSBHMTESTRECORD"
if err := c.SetMetadataFlat(uuid, map[string]string{
@@ -77,13 +78,13 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
}
// 1. the flat key
if v, err := c.client.Get(context.Background(),
c.buildKeyPrefix(uuid)+"cids/"+poisonedRoot).Result(); err == nil {
if v, ok, _ := c.Store().Get(context.Background(),
recordPrefix(uuid)+"cids/"+poisonedRoot); ok {
t.Fatalf("flat key should be gone, still %q", v)
}
// 2. the field index
// 2. the record read
if hasField(t, c, uuid, "cids/"+poisonedRoot) {
t.Fatal("field index still lists the retracted member")
t.Fatal("record read still lists the retracted member")
}
// 3. the reverse alias — the one that used to survive
if got := aliasTarget(t, c, poisonedRoot); got != "" {
@@ -110,7 +111,7 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
// writer's alias in place. Retracting the loser's member must not unpick the
// winner's index — which is why removeAliasLocked compares before deleting.
func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const winner = "01KSBHMWINNER"
const loser = "01KSBHMLOSER"
const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m"
@@ -140,7 +141,7 @@ func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
// A non-cids field has no alias to clean up, and the delete must not invent one.
func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const uuid = "01KSBHMORDINARY"
if err := c.SetMetadataFlat(uuid, map[string]string{
"cids/" + goodMidhash: "true",
internal/storage/clear_test.go
+105 -91
View File
@@ -1,111 +1,122 @@
package storage
import (
"context"
"path/filepath"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/kv"
)
// newTestClient spins an in-memory Redis and returns a connected Client with
// the given namespace prefix. The server is torn down with the test.
func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
// newTestClient opens a fresh database in the test's temp dir.
func newTestClient(t *testing.T) *Client {
t.Helper()
s := miniredis.RunT(t)
c := NewClient(prefix)
if err := c.Connect("redis://" + s.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
c := NewClient()
if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return c, s
return c
}
// allKeys lists every row (test helper; full scan).
func allKeys(t *testing.T, c *Client) []string {
t.Helper()
var keys []string
if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
for _, r := range rows {
keys = append(keys, r.K)
}
return nil
}); err != nil {
t.Fatalf("scan: %v", err)
}
return keys
}
// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
// leak: ClearAllMetadata used to delete every file:<root>/* key and the
// file:__index__ set, but left the cid:<cid> → root reverse-index entries
// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
// a surviving entry silently redirects a later write into a deleted root.
// leak: ClearAllMetadata used to delete every record key and the root index,
// but left the cid:<cid> → root reverse-index entries behind. Those stale
// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry
// silently redirects a later write into a deleted root.
func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
for _, prefix := range []string{"", "mm:"} {
name := "no-prefix"
if prefix != "" {
name = "prefixed"
c := newTestClient(t)
// Two records, each with several CID aliases (a midhash plus siblings) —
// exactly what the watcher produces.
aliases := map[string][]string{}
for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
if err != nil {
t.Fatalf("mint: %v", err)
}
t.Run(name, func(t *testing.T) {
c, s := newTestClient(t, prefix)
cids := []string{"bafkmid" + uuid, "bafkfull" + uuid}
for _, cidStr := range cids {
if err := c.AddAlias(uuid, cidStr); err != nil {
t.Fatalf("add alias %s: %v", cidStr, err)
}
}
if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil {
t.Fatalf("add duplicate: %v", err)
}
aliases[uuid] = cids
}
// Two records, each with several CID aliases (a midhash plus
// siblings) — exactly what the watcher produces.
aliases := map[string][]string{}
for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
if err != nil {
t.Fatalf("mint: %v", err)
}
cids := []string{
"bafkmid" + uuid,
"bafkfull" + uuid,
}
for _, cidStr := range cids {
if err := c.AddAlias(uuid, cidStr); err != nil {
t.Fatalf("add alias %s: %v", cidStr, err)
}
}
aliases[uuid] = cids
}
// An entry already orphaned by a previous (buggy) clear: a reverse-index
// key whose root no longer exists. The sweep is expected to collect it too.
if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil {
t.Fatalf("seed orphan: %v", err)
}
// An entry already orphaned by a previous (buggy) clear: a
// reverse-index key whose root no longer exists. The sweep is
// expected to collect it too.
if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
t.Fatalf("seed orphan: %v", err)
for uuid, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != uuid {
t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
}
}
}
// Sanity: the aliases resolve before the clear.
for uuid, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != uuid {
t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
}
}
}
deleted, err := c.ClearAllMetadata()
if err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
if deleted != 2 {
t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
}
deleted, err := c.ClearAllMetadata()
if err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
// deletedCount counts files, not keys.
if deleted != 2 {
t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
// Nothing record-shaped may survive: not the records, not their aliases,
// duplicates or search rows, not the pre-existing orphan. (The event
// streams and the schema sentinel are not metadata and stay.)
for _, k := range allKeys(t, c) {
for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} {
if strings.HasPrefix(k, ns) {
t.Errorf("key survived the clear: %q", k)
}
}
}
var ftsRows int
if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 {
t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err)
}
// No reverse-index key may survive — not for the cleared records,
// not the pre-existing orphan.
if leaked := s.Keys(); len(leaked) != 0 {
t.Errorf("keys survived the clear: %v", leaked)
for _, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != cidStr {
t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
}
// And the aliases must no longer resolve: ResolveRoot returns the
// hash unchanged when there is no index entry.
for _, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != cidStr {
t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
}
}
}
if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
}
})
}
}
if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
}
}
// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
// so a clear on a prefixed client cannot reach another namespace's index.
// keys outside the metadata namespaces (magnet cache, coverage markers, a key
// that merely CONTAINS "cid:") must survive a metadata wipe.
func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
c, s := newTestClient(t, "mm:")
c := newTestClient(t)
uuid, err := c.Mint("watch/a.mkv", 100, 1000)
if err != nil {
@@ -114,25 +125,28 @@ func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
if err := c.AddAlias(uuid, "bafkmine"); err != nil {
t.Fatalf("add alias: %v", err)
}
// Another namespace's reverse-index entry, plus an unrelated key.
if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
t.Fatalf("seed foreign cid: %v", err)
}
if err := s.Set("unrelated", "keep-me"); err != nil {
t.Fatalf("seed unrelated: %v", err)
for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} {
if err := c.SetRawValue(k, v); err != nil {
t.Fatalf("seed %s: %v", k, err)
}
}
if _, err := c.ClearAllMetadata(); err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
if !s.Exists(k) {
t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} {
if _, _, ok, _ := c.GetKeyInfo(k); !ok {
t.Errorf("clear deleted foreign key %q", k)
}
}
if s.Exists(c.buildCIDIndexKey("bafkmine")) {
if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok {
t.Errorf("own reverse-index entry survived the clear")
}
}
// withClient runs fn against a fresh client.
func withClient(t *testing.T, fn func(t *testing.T, c *Client)) {
t.Helper()
fn(t, newTestClient(t))
}
internal/storage/client.go
+673 -1234
View File
File diff suppressed because it is too large Load Diff
internal/storage/domain_screen_migration.go
-193
View File
@@ -1,193 +0,0 @@
package storage
import (
"context"
"fmt"
"time"
)
// The one-shot corpus sweep for METADATA_KEYS.md §14.17: the `film` + `tv`
// domains merged into `screen`, and the standalone-vs-serial split they were
// really carrying moved to the new `workForm` key.
//
// # Why a sweep and not a read-side alias
//
// `domain` is **stored, not derived on read** — that is what lets the
// `contentKind` vocabulary stay open (a peer that has never heard of a kind
// still routes the record correctly). The price of storing it is that a
// vocabulary change has to be paid once, in the data: every consumer's filter
// is an exact match, so a record left saying `film` is invisible to a
// `domain:screen` wall, and teaching every consumer to accept both spellings
// forever is exactly the alternation the merge exists to delete.
//
// So the values are rewritten at rest, here, and no alias vocabulary survives
// in query filters. (meta-search additionally normalises on the way into its
// index, because a record can still arrive from a peer that has not been swept
// — that is a courtesy for un-swept peers, not a second source of truth.)
//
// # Contract
//
// Idempotent: a record already carrying `screen` and a `workForm` is skipped,
// so the endpoint is safe to call repeatedly and safe to call on a fresh box.
// Returns the number of records **changed**, not the number examined.
// legacyDomainReplacements maps a retired `domain` value to its replacement.
var legacyDomainReplacements = map[string]string{
"film": "screen",
"tv": "screen",
}
// workFormByContentKind is the `contentKind` → `workForm` table from
// METADATA_KEYS.md §1.
//
// ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain` (and of
// meta-search's copy). It exists here only to backfill the pre-merge corpus —
// meta-core never classifies a record it is given, and nothing else in Go reads
// it. If the tables ever disagree, the writers win: this one is a one-shot.
var workFormByContentKind = map[string]string{
// screen
"movie": "standalone",
"series": "serial",
"episode": "serial",
// music — an album is a closed work, not an ongoing run.
"track": "standalone",
"album": "standalone",
"artist": "standalone",
"musicVideo": "standalone",
"djMix": "standalone",
"liveSet": "standalone",
// literature
"book": "standalone",
"audiobook": "standalone",
"comic": "serial",
"manga": "serial",
"magazine": "serial",
// science
"paper": "standalone",
// ⚠ `pack` is deliberately absent: a season pack is `serial`, an album
// release `standalone`, and only the writer knows which. A pack that
// reaches this sweep without a `workForm` keeps none — inventing one would
// be a guess written permanently into the record.
}
// domainScreenPlan is one record's pending rewrite. Empty strings mean "leave
// this field alone".
type domainScreenPlan struct {
hashID string
domain string
workForm string
}
// MigrateDomainScreen rewrites `domain=film|tv` to `domain=screen` and
// backfills `workForm` from `contentKind` across the whole corpus.
//
// Reads in two batched MGET passes (never a per-record SCAN — a single record
// read costs a full keyspace sweep on this store), then writes only the records
// that actually change.
func (c *Client) MigrateDomainScreen(ctx context.Context) (int, error) {
plans, err := c.planDomainScreen(ctx)
if err != nil {
return 0, err
}
// Writes take the client lock themselves, so they happen after the read
// phase has released it.
fixed := 0
for _, p := range plans {
if p.domain != "" {
if err := c.SetProperty(p.hashID, "domain", p.domain); err != nil {
return fixed, fmt.Errorf("set domain on %s: %w", p.hashID, err)
}
}
if p.workForm != "" {
if err := c.SetProperty(p.hashID, "workForm", p.workForm); err != nil {
return fixed, fmt.Errorf("set workForm on %s: %w", p.hashID, err)
}
}
fixed++
}
return fixed, nil
}
// planDomainScreen is the read half: what would change, without changing it.
func (c *Client) planDomainScreen(ctx context.Context) ([]domainScreenPlan, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
if err != nil {
return nil, fmt.Errorf("smembers index: %w", err)
}
if len(hashIDs) == 0 {
return nil, nil
}
// Three keys per record, in a fixed order so the MGET response decodes by
// position: keys[i*3+n] belongs to hashIDs[i].
const fieldsPerFile = 3
fields := [fieldsPerFile]string{"domain", "workForm", "contentKind"}
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
for _, h := range hashIDs {
prefix := c.buildKeyPrefix(h)
for _, f := range fields {
keys = append(keys, prefix+f)
}
}
// Chunked so one MGET never carries the whole corpus.
const chunk = 3000
values := make([]interface{}, 0, len(keys))
for start := 0; start < len(keys); start += chunk {
end := start + chunk
if end > len(keys) {
end = len(keys)
}
got, err := c.client.MGet(ctx, keys[start:end]...).Result()
if err != nil {
return nil, fmt.Errorf("mget: %w", err)
}
values = append(values, got...)
}
str := func(i int) string {
if i >= len(values) {
return ""
}
s, _ := values[i].(string)
return s
}
plans := make([]domainScreenPlan, 0)
for i, h := range hashIDs {
base := i * fieldsPerFile
domain, workForm, contentKind := str(base), str(base+1), str(base+2)
plan := domainScreenPlan{hashID: h}
if replacement, ok := legacyDomainReplacements[domain]; ok {
plan.domain = replacement
}
// A stored `workForm` is authoritative — a `pack` writer knew something
// the table cannot. Only an absent one is filled in.
if workForm == "" {
if form, ok := workFormByContentKind[contentKind]; ok {
plan.workForm = form
}
}
if plan.domain != "" || plan.workForm != "" {
plans = append(plans, plan)
}
}
return plans, nil
}
// MigrateDomainScreenWithTimeout is the convenience wrapper used by the API
// handler. Generous timeout: the read phase is two batched passes, but the
// write phase is one round-trip per changed field.
func (c *Client) MigrateDomainScreenWithTimeout() (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
return c.MigrateDomainScreen(ctx)
}
internal/storage/domain_screen_migration_test.go
-140
View File
@@ -1,140 +0,0 @@
package storage
import (
"context"
"testing"
)
// seedRecord writes one record's classification fields straight through
// SetProperty, which is what a real writer uses (it maintains the field index
// and the root index the sweep reads from).
func seedRecord(t *testing.T, c *Client, hashID string, fields map[string]string) {
t.Helper()
for k, v := range fields {
if err := c.SetProperty(hashID, k, v); err != nil {
t.Fatalf("seed %s/%s: %v", hashID, k, err)
}
}
}
// The merge (METADATA_KEYS.md §14.17) is paid in the data: `domain` is stored,
// not derived on read, so a record left saying `film` is invisible to a
// `domain:screen` wall. This is the sweep that pays it.
func TestMigrateDomainScreen_RewritesLegacyDomainsAndBackfillsWorkForm(t *testing.T) {
c, _ := newTestClient(t, "")
seedRecord(t, c, "rec-movie", map[string]string{
"domain": "film",
"contentKind": "movie",
})
seedRecord(t, c, "rec-episode", map[string]string{
"domain": "tv",
"contentKind": "episode",
})
// Already current, and already carries both keys: must be left alone.
seedRecord(t, c, "rec-current", map[string]string{
"domain": "screen",
"contentKind": "series",
"workForm": "serial",
})
// A different domain entirely — the sweep must not touch it, but it is
// still owed the `workForm` backfill.
seedRecord(t, c, "rec-book", map[string]string{
"domain": "literature",
"contentKind": "book",
})
fixed, err := c.MigrateDomainScreen(context.Background())
if err != nil {
t.Fatalf("migrate: %v", err)
}
if fixed != 3 {
t.Errorf("fixed = %d, want 3 (the two legacy rows + the book's backfill)", fixed)
}
for _, tc := range []struct {
hashID string
domain string
workForm string
}{
{"rec-movie", "screen", "standalone"},
{"rec-episode", "screen", "serial"},
{"rec-current", "screen", "serial"},
{"rec-book", "literature", "standalone"},
} {
m, err := c.GetMetadataFlat(tc.hashID)
if err != nil {
t.Fatalf("read %s: %v", tc.hashID, err)
}
if m["domain"] != tc.domain {
t.Errorf("%s domain = %q, want %q", tc.hashID, m["domain"], tc.domain)
}
if m["workForm"] != tc.workForm {
t.Errorf("%s workForm = %q, want %q", tc.hashID, m["workForm"], tc.workForm)
}
}
}
// A second pass must find nothing to do — the endpoint is called by hand on
// each box, and "run it twice" must not mean "write the whole corpus twice".
func TestMigrateDomainScreen_IsIdempotent(t *testing.T) {
c, _ := newTestClient(t, "mm:")
seedRecord(t, c, "rec-1", map[string]string{
"domain": "tv",
"contentKind": "episode",
})
if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
t.Fatalf("first pass: %v", err)
}
fixed, err := c.MigrateDomainScreen(context.Background())
if err != nil {
t.Fatalf("second pass: %v", err)
}
if fixed != 0 {
t.Errorf("second pass fixed = %d, want 0", fixed)
}
}
// `pack` is the one kind neither routing table can decide — a season pack is
// `serial`, an album release `standalone`. A pack that reaches the sweep
// without a `workForm` must keep none: inventing one writes a guess into the
// record permanently.
func TestMigrateDomainScreen_NeverGuessesAPacksWorkForm(t *testing.T) {
c, _ := newTestClient(t, "")
seedRecord(t, c, "rec-pack", map[string]string{
"domain": "tv",
"contentKind": "pack",
})
// A writer that DID know keeps what it wrote.
seedRecord(t, c, "rec-album", map[string]string{
"domain": "music",
"contentKind": "pack",
"workForm": "standalone",
})
if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
t.Fatalf("migrate: %v", err)
}
m, err := c.GetMetadataFlat("rec-pack")
if err != nil {
t.Fatalf("read: %v", err)
}
if m["domain"] != "screen" {
t.Errorf("domain = %q, want screen (the domain WAS knowable)", m["domain"])
}
if _, ok := m["workForm"]; ok {
t.Errorf("workForm = %q, want absent", m["workForm"])
}
a, err := c.GetMetadataFlat("rec-album")
if err != nil {
t.Fatalf("read: %v", err)
}
if a["workForm"] != "standalone" {
t.Errorf("workForm = %q, want standalone (writer-supplied, never overwritten)", a["workForm"])
}
}
internal/storage/dual_root_migration.go
-41
View File
@@ -1,41 +0,0 @@
package storage
import (
"context"
"fmt"
)
// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
// legacy storage model: /meta/{hash} write handlers that didn't resolve
// CID → UUID first created a parallel midhash-rooted entry alongside the
// watcher's UUID root.
//
// That pattern is now structurally impossible:
// - writes resolve via ResolveRoot before landing (see handlers), and
// - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
// UUID roots, with the reverse index cid:<cid> → uuid as the single
// resolution path.
//
// The legacy on-disk shape this migration targeted (midhash-rooted entries,
// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
// cut-over wipes Redis, so there is nothing to migrate. The function is kept
// as a no-op so the admin endpoint and its wrappers continue to compile and
// respond.
func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
return 0, nil
}
// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
// handler and the startup hook.
func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
return c.MigrateDualRoots(context.Background())
}
// MigrateDualRootsReport returns a human-readable summary. With the dual-root
// pattern retired (see MigrateDualRoots), this always reports nothing to do.
func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
}
internal/storage/field_index.go
-486
View File
@@ -1,486 +0,0 @@
package storage
import (
"context"
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
// The per-record field-name index.
//
// A record is stored as one flat Redis STRING per leaf
// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
// meta-search can react to one field changing instead of diffing a whole hash
// (see docs/project-architecture/persistent-storage.md, and
// dev/test/suites/flat-keys.bats which asserts records are never hashes).
//
// The cost of that shape was that reading one record meant asking Redis "which
// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
// the byte path for every poster meta-share served, which is how it surfaced.
//
// So we keep the flat keys and index their *names*:
//
// file:<uuid>/__fields__ SET{ "filePath", "sizeByte", "cids/bafk…", … }
//
// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
// This mirrors `file:__index__` (the set of every root) one level down, and the
// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
// from the index rather than scanning per root.
//
// **The set is derived, never authoritative.** The flat keys remain the source
// of truth. A record whose set is missing (anything written before this landed)
// falls back to the old SCAN and populates the set on the way out, so existing
// deployments heal themselves on first touch with no migration and no
// SchemaVersion bump — which matters, because a bump is a flag day that makes
// operators wipe Redis (see schema_version.go).
//
// The consistency argument that makes a derived index safe here is meta-core's
// single-writer rule: leader election publishes an empty `redisUrl` on purpose
// (internal/leader/election.go), so nothing outside this process can write a
// `file:` key behind the index's back.
// FieldsField is the reserved field name holding a record's field-name index.
//
// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
// prefix as the record's own fields. That keeps every existing sweep correct
// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
// The flip side is that every reader which enumerates that prefix has to skip
// it, exactly as they already skip `file:__index__`.
const FieldsField = "__fields__"
// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
const mgetBatch = 1000
// buildFieldsKey constructs the key for a record's field-name index set.
func (c *Client) buildFieldsKey(hashID string) string {
return c.buildKeyPrefix(hashID) + FieldsField
}
// isReservedFieldName reports whether a field name is bookkeeping rather than
// metadata, and so must never be returned to an API caller, exported into a
// snapshot, or folded into the search haystack.
func isReservedFieldName(field string) bool {
return field == FieldsField
}
// readMetadataLocked returns a record's full flat metadata. Caller must hold
// c.mu (either mode).
//
// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
// legacy full-keyspace SCAN, whose result is used to populate the index so the
// next read takes the fast path.
//
// Returns (nil, nil) for a record with no fields — callers treat that as 404.
func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
if len(fields) > 0 {
return c.mgetFieldsLocked(ctx, hashID, fields)
}
// No index yet: a record written before the index existed, or one that
// genuinely has no fields. Fall back to the scan, then backfill.
result, scanned, err := c.scanMetadataLocked(ctx, hashID)
if err != nil {
return nil, err
}
if len(scanned) > 0 {
// Best-effort: a failed backfill costs the next read a scan, it does
// not make the read wrong. Never fail the caller's read for it.
if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
logFieldIndexBackfillFailure(hashID, err)
}
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
//
// A field named in the index whose key is gone reads back as nil and is simply
// skipped: a *stale* index entry is harmless. (A *missing* one would silently
// drop data, which is why every write path maintains the set.)
func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
prefix := c.buildKeyPrefix(hashID)
// Keep the reserved name out of the key list AND out of the result — it is
// a SET, so MGET would error on it in real Redis rather than return nil.
wanted := make([]string, 0, len(fields))
for _, f := range fields {
if !isReservedFieldName(f) {
wanted = append(wanted, f)
}
}
if len(wanted) == 0 {
return nil, nil
}
result := make(map[string]string, len(wanted))
for start := 0; start < len(wanted); start += mgetBatch {
end := start + mgetBatch
if end > len(wanted) {
end = len(wanted)
}
chunk := wanted[start:end]
keys := make([]string, len(chunk))
for i, f := range chunk {
keys[i] = prefix + f
}
values, err := c.client.MGet(ctx, keys...).Result()
if err != nil {
return nil, fmt.Errorf("mget failed: %w", err)
}
for i, v := range values {
if s, ok := v.(string); ok {
result[chunk[i]] = s
}
}
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
// MGET. Retained as the fallback for un-indexed records and as the source for
// the backfill. Returns the metadata plus the field names it saw (which
// includes fields whose value came back nil, so the index stays complete).
func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
prefix := c.buildKeyPrefix(hashID)
result := make(map[string]string)
var fields []string
var cursor uint64
for {
keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, nil, fmt.Errorf("scan failed: %w", err)
}
if len(keys) > 0 {
wanted := make([]string, 0, len(keys))
for _, key := range keys {
field := strings.TrimPrefix(key, prefix)
if isReservedFieldName(field) {
continue
}
fields = append(fields, field)
wanted = append(wanted, key)
}
if len(wanted) > 0 {
values, err := c.client.MGet(ctx, wanted...).Result()
if err != nil {
return nil, nil, fmt.Errorf("mget failed: %w", err)
}
for i, key := range wanted {
if s, ok := values[i].(string); ok {
result[strings.TrimPrefix(key, prefix)] = s
}
}
}
}
cursor = nextCursor
if cursor == 0 {
break
}
}
return result, fields, nil
}
// addFieldsToPipe records field names in the index as part of an existing
// pipeline, so a multi-field write stays one round-trip.
func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
names := filterReserved(fields)
if len(names) == 0 {
return
}
pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
}
// addFieldLocked records a single field name in the index. Caller must hold
// c.mu. Best-effort by the same argument as the backfill: the flat key is the
// source of truth, and a lost index entry is repaired by the next scan
// fallback — so it must not fail the write it accompanies.
func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
if isReservedFieldName(field) {
return
}
if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
logFieldIndexWriteFailure(hashID, field, err)
}
}
// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
//
// A stale entry left behind by a failure is harmless (the MGET reads nil and
// skips it), so this is best-effort too.
func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
if isReservedFieldName(field) {
return
}
if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
logFieldIndexWriteFailure(hashID, field, err)
}
}
// fieldsWithPrefixLocked returns the record's indexed field names under a
// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
// no index yet. Caller must hold c.mu.
func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
if len(fields) == 0 {
// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
// rather than the whole record, and do not backfill from it — a
// partial view must never be mistaken for a complete index.
return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
}
var out []string
for _, f := range fields {
if isReservedFieldName(f) {
continue
}
if strings.HasPrefix(f, fieldPrefix) {
out = append(out, strings.TrimPrefix(f, fieldPrefix))
}
}
// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
// at these sizes and makes the response byte-stable, which matters for
// anything diffing or caching it.
sort.Strings(out)
return out, nil
}
func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
prefix := c.buildKeyPrefix(hashID) + fieldPrefix
var out []string
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, fmt.Errorf("scan fields: %w", err)
}
for _, k := range keys {
out = append(out, strings.TrimPrefix(k, prefix))
}
cursor = next
if cursor == 0 {
break
}
}
sort.Strings(out)
return out, nil
}
// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
//
// Without it the indexes still appear — each record's first read falls back to
// a scan and backfills — but that transition is O(records x keyspace) and it
// runs again from scratch after any wipe. Measured on the real store (12,345
// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
// during which meta-core stays pinned exactly as it was before the fix. Worse,
// `RepublishMetadata` walks every record at startup, so that tail is paid on
// every restart.
//
// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
// for the same reason.
//
// Idempotent, and safe to run against a partially-indexed store: SADD of a
// member that is already there is a no-op. Returns the number of records it
// wrote an index for.
func (c *Client) WarmFieldIndexes() (int, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
indexKey := c.buildIndexKey()
byRecord := make(map[string][]string)
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
if err != nil {
return 0, fmt.Errorf("scan: %w", err)
}
for _, k := range keys {
if k == indexKey {
continue
}
// hashID has no '/'; field may contain them.
rest := strings.TrimPrefix(k, filePrefix)
slash := strings.IndexByte(rest, '/')
if slash < 0 {
continue
}
hashID, field := rest[:slash], rest[slash+1:]
if isReservedFieldName(field) {
// Already indexed — remember the record so we don't treat it
// as missing, but there is nothing to add for this key.
if _, seen := byRecord[hashID]; !seen {
byRecord[hashID] = nil
}
continue
}
byRecord[hashID] = append(byRecord[hashID], field)
}
cursor = next
if cursor == 0 {
break
}
}
written := 0
pipe := c.client.Pipeline()
pending := 0
for hashID, fields := range byRecord {
if len(fields) == 0 {
continue
}
pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
written++
pending++
// Bound the pipeline so a large store doesn't buffer every command.
if pending >= 500 {
if _, err := pipe.Exec(ctx); err != nil {
return written, fmt.Errorf("sadd pipeline: %w", err)
}
pipe = c.client.Pipeline()
pending = 0
}
}
if pending > 0 {
if _, err := pipe.Exec(ctx); err != nil {
return written, fmt.Errorf("sadd pipeline: %w", err)
}
}
return written, nil
}
// recordHasFieldsLocked reports whether a record still has at least one
// property key. Caller must hold c.mu.
//
// `__fields__` must not count — it lives under the record's own prefix, so a
// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
// whose every property has been deleted, and the root could never be
// de-indexed.
func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return false, fmt.Errorf("scard fields: %w", err)
}
if n > 0 {
return true, nil
}
// Either genuinely empty, or an un-indexed record from before the index
// existed — the scan is the only way to tell those apart.
prefix := c.buildKeyPrefix(hashID)
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return false, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
return true, nil
}
}
cursor = next
if cursor == 0 {
return false, nil
}
}
}
// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
// record property key. Reports ok=false for anything else — the index set
// itself, `file:__index__`, and any key outside the `file:` namespace.
//
// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
// the index honest without its callers knowing the layout.
func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
rest := key
if c.prefix != "" {
var found bool
rest, found = strings.CutPrefix(rest, c.prefix)
if !found {
return "", "", false
}
}
rest, found := strings.CutPrefix(rest, "file:")
if !found {
return "", "", false
}
hashID, field, found = strings.Cut(rest, "/")
if !found || hashID == "" || field == "" {
// `file:__index__` has no slash and lands here.
return "", "", false
}
if isReservedFieldName(field) {
return "", "", false
}
return hashID, field, true
}
// The index is derived and self-healing, so a write failure degrades
// performance (the next read falls back to a scan and re-populates) rather
// than correctness. Log it rather than failing the caller's write — but log it
// loudly, because a *persistent* failure means every read is scanning again.
func logFieldIndexWriteFailure(hashID, field string, err error) {
log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
}
func logFieldIndexBackfillFailure(hashID string, err error) {
log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
}
func filterReserved(fields []string) []string {
out := make([]string, 0, len(fields))
for _, f := range fields {
if !isReservedFieldName(f) {
out = append(out, f)
}
}
return out
}
func toAny(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i, s := range ss {
out[i] = s
}
return out
}
internal/storage/field_index_test.go
-464
View File
@@ -1,464 +0,0 @@
package storage
import (
"context"
"reflect"
"sort"
"testing"
)
// Tests for the per-record field-name index (field_index.go).
//
// The index is what makes a record read O(fields) instead of O(keyspace). It is
// derived state, so the properties worth pinning are: (1) a read served from
// the index returns exactly what a scan would have, (2) a record with no index
// still reads correctly and gets one, and (3) every write path keeps it in
// sync — because a *missing* entry silently drops a field from every
// subsequent read, which is the one failure mode that would be invisible.
//
// Every case runs with and without a namespace prefix, matching clear_test.go.
func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
t.Helper()
for _, prefix := range []string{"", "mm:"} {
name := "no-prefix"
if prefix != "" {
name = "prefix-" + prefix
}
t.Run(name, func(t *testing.T) {
c, _ := newTestClient(t, prefix)
fn(t, c)
})
}
}
func fieldNames(t *testing.T, c *Client, hashID string) []string {
t.Helper()
got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
if err != nil {
t.Fatalf("smembers fields: %v", err)
}
sort.Strings(got)
return got
}
func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
t.Helper()
got, err := c.GetMetadataFlat(hashID)
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
return got
}
// SetMetadataFlat must index every field it writes, and the read must come back
// through the index rather than a scan.
func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
want := map[string]string{
"filePath": "/files/watch/a.mkv",
"sizeByte": "123",
"cids/bafkreiaaa": "true",
"titles/jpn/ナルト": "true",
"categories/TV/UHD": "true",
"info/files/0/path": "a.mkv",
"provenance": `{"source":"gateway"}`,
"plot/eng": "a plot with / slashes",
"stream/3": "eng",
"genres/action": "true",
"originaltitle": "Naruto",
"deeply/nested/field/name/here": "x",
}
if err := c.SetMetadataFlat("root1", want); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
}
// The index must name every field — including the ones containing '/',
// which is where a naive "split on the first slash" would go wrong.
wantNames := make([]string, 0, len(want))
for k := range want {
wantNames = append(wantNames, k)
}
sort.Strings(wantNames)
if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
}
})
}
// A record written before the index existed has flat keys but no set. The read
// must still be correct, and must leave an index behind so the next read takes
// the fast path. This is the migration path for the ~12k records already on
// disk: no flag day, no SchemaVersion bump.
func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
prefix := c.buildKeyPrefix("legacy")
want := map[string]string{
"filePath": "/files/watch/old.mkv",
"sizeByte": "42",
"cids/bafkreiold": "true",
}
// Write the flat keys directly, bypassing every index-maintaining path
// — exactly the shape a pre-upgrade store is in.
for f, v := range want {
if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
t.Fatalf("seed %s: %v", f, err)
}
}
if names := fieldNames(t, c, "legacy"); len(names) != 0 {
t.Fatalf("precondition: expected no index, got %v", names)
}
if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
}
wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
}
// Second read, now served from the index, must agree with the first.
if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
}
})
}
// Each single-field write path must add to the index, and each delete path must
// remove from it. A field written but not indexed vanishes from every later
// read — silently — so this is the property that matters most.
func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
t.Fatalf("SetProperty: %v", err)
}
if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
t.Fatalf("AddToSet: %v", err)
}
if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
t.Fatalf("MergeMetadataFlat: %v", err)
}
want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
t.Errorf("after writes\n got %v\nwant %v", got, want)
}
// DeleteProperty drops the key and the index entry.
if err := c.DeleteProperty("root2", "title"); err != nil {
t.Fatalf("DeleteProperty: %v", err)
}
// RemoveFromSet deletes the key once its last member is gone.
if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
t.Fatalf("RemoveFromSet: %v", err)
}
want = map[string]string{"year": "2002"}
if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
t.Errorf("after deletes\n got %v\nwant %v", got, want)
}
if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
t.Errorf("index after deletes = %v, want [year]", got)
}
})
}
// The raw KV editor can mint or drop a record property out of band. It is the
// one write path whose caller does not know the record layout, so the storage
// layer has to keep the index honest for it.
func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
t.Fatalf("SetRawValue: %v", err)
}
if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
t.Errorf("index after SetRawValue = %v", got)
}
if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
t.Errorf("read after SetRawValue = %v", got)
}
if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
t.Fatalf("DeleteRawKey: %v", err)
}
if got := fieldNames(t, c, "root3"); len(got) != 0 {
t.Errorf("index after DeleteRawKey = %v, want empty", got)
}
// Keys outside the record namespace must be left alone entirely.
if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
t.Fatalf("SetRawValue non-record: %v", err)
}
if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
t.Error("cid: key was treated as a record property")
}
if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
t.Error("file:__index__ was treated as a record property")
}
})
}
// Mint and AddAlias write fields through their own pipelines, so they need the
// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
// it and a record's CID key-set becomes invisible to every read.
func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
if err != nil {
t.Fatalf("Mint: %v", err)
}
if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
t.Errorf("index after Mint = %v", got)
}
if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
t.Fatalf("AddAlias: %v", err)
}
got := mustGetMetadata(t, c, uuid)
if got["cids/bafkreiaaa"] != "true" {
t.Errorf("alias key-set member missing from read: %v", got)
}
// And the CID enumeration, which now reads the index instead of
// scanning, must see it.
cids, err := c.cidsForRootLocked(context.Background(), uuid)
if err != nil {
t.Fatalf("cidsForRootLocked: %v", err)
}
if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
}
})
}
// The index lives under the record's own prefix, so the "is this record empty?"
// probe would otherwise always find it and the root could never be de-indexed.
func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetProperty("root4", "title", "x"); err != nil {
t.Fatalf("SetProperty: %v", err)
}
removed, err := c.IndexRemoveIfEmpty("root4")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if removed {
t.Error("de-indexed a record that still has a field")
}
if err := c.DeleteProperty("root4", "title"); err != nil {
t.Fatalf("DeleteProperty: %v", err)
}
removed, err = c.IndexRemoveIfEmpty("root4")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if !removed {
t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
}
// ...and the now-empty index key must not be left orphaned.
n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
if err != nil {
t.Fatalf("exists: %v", err)
}
if n != 0 {
t.Error("orphaned field index left behind after de-indexing")
}
})
}
// The same probe must still work for a record that predates the index.
func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
t.Fatalf("seed: %v", err)
}
if err := c.IndexAdd("legacy2"); err != nil {
t.Fatalf("IndexAdd: %v", err)
}
removed, err := c.IndexRemoveIfEmpty("legacy2")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if removed {
t.Error("de-indexed an un-indexed record that still has a field")
}
})
}
// The reserved name is bookkeeping. It must never reach an API caller as a
// metadata field, and an inbound payload must never be able to overwrite the
// SET with a string — that would break every subsequent read of the record.
func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
err := c.SetMetadataFlat("root5", map[string]string{
"title": "x",
FieldsField: "malicious",
})
if err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
got := mustGetMetadata(t, c, "root5")
if _, present := got[FieldsField]; present {
t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
}
if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
t.Errorf("read = %v, want just title", got)
}
// The index must still be a SET — i.e. not clobbered by the payload.
if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
t.Errorf("index = %v, want [title]", got)
}
if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
t.Error("SetProperty accepted the reserved field name")
}
})
}
// Deleting a record must take its index with it, or a recreated root would
// inherit phantom field names.
func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
if _, err := c.DeleteMetadata("root6"); err != nil {
t.Fatalf("DeleteMetadata: %v", err)
}
if got := fieldNames(t, c, "root6"); len(got) != 0 {
t.Errorf("index survived DeleteMetadata: %v", got)
}
got, err := c.GetMetadataFlat("root6")
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
if got != nil {
t.Errorf("deleted record still reads back: %v", got)
}
})
}
// A stale index entry (name present, key gone) is the benign direction: the
// MGET reads nil and the field is skipped rather than erroring.
func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
// Name a field that has no key behind it.
if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
t.Fatalf("sadd ghost: %v", err)
}
got := mustGetMetadata(t, c, "root7")
if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
t.Errorf("read = %v, want just title", got)
}
})
}
// WarmFieldIndexes is what keeps an upgrade from spending hours reading
// un-indexed records one full-keyspace scan at a time. It must index every
// record in one pass, be idempotent, and leave already-indexed records alone.
func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
// Two legacy records seeded straight into Redis (no index), plus one
// written through the normal path (already indexed).
legacy := map[string]map[string]string{
"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
"old2": {"filePath": "/b.mkv", "titles/jpn/ナルト": "true", "deep/a/b": "x"},
}
for root, fields := range legacy {
for f, v := range fields {
if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
t.Fatalf("seed: %v", err)
}
}
if err := c.IndexAdd(root); err != nil {
t.Fatalf("IndexAdd: %v", err)
}
}
if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
n, err := c.WarmFieldIndexes()
if err != nil {
t.Fatalf("WarmFieldIndexes: %v", err)
}
if n < 2 {
t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
}
if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
t.Errorf("old1 index = %v", got)
}
if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
t.Errorf("old2 index = %v", got)
}
if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
t.Errorf("fresh index disturbed = %v", got)
}
// Reads must now agree with what the records actually hold.
for root, want := range legacy {
if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
t.Errorf("%s read = %v, want %v", root, got, want)
}
}
// Idempotent: a second pass changes nothing.
if _, err := c.WarmFieldIndexes(); err != nil {
t.Fatalf("second WarmFieldIndexes: %v", err)
}
if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
t.Errorf("old2 index after second warm = %v", got)
}
})
}
// The bulk search read enumerates the whole file: keyspace and MGETs what it
// finds. The index keys are SETs living in that same namespace, so it has to
// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetMetadataFlat("root8", map[string]string{
"title": "Naruto Shippuden",
"fileName": "naruto.mkv",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
tuples, err := c.GetSearchIndexTuples()
if err != nil {
t.Fatalf("GetSearchIndexTuples: %v", err)
}
var found bool
for _, tu := range tuples {
if tu.HashID != "root8" {
continue
}
found = true
if _, present := tu.Fields[FieldsField]; present {
t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
}
}
if !found {
t.Fatal("record missing from search tuples")
}
})
}
internal/storage/literature_echo_sweep.go
-326
View File
@@ -1,326 +0,0 @@
package storage
import (
"context"
"fmt"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/cid"
)
// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
// identity and position keys are resolved, never copied from a query, and a
// card carries no position keys and no `workcid`.
//
// # Why a sweep
//
// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
// own filter values onto what it returned. The gateway persists whatever passes
// the filter, so the copy became a permanent binding — and stored records never
// pick up the writer fix: a covered `(upstream, query)` never reaches the
// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
// one. So the copies already at rest are removed here, once.
//
// What the copies cost: an `anilistid` on a MangaDex site card claims an
// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
// anchor — the site card then competes with (and can hide) the real AniList
// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
// join and sort like a position inside one.
//
// # What is swept, and what deliberately is not
//
// - Only `domain=literature` + `fileType=card` records. Nothing else is read
// for writing, whatever keys it carries.
// - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
// go from **every** such card: a card is a work, never a position in one,
// so no legitimate writer puts them there.
// - `anilistid` goes only from a **site card** — a card whose own `cids/`
// key-set holds a site-form card locator and no locator of any other form.
// On an anchor card the id *is* the card's identity (its locator is
// `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
// it: which half the id belongs to is not recoverable at rest, and leaving
// a stamp is the safe direction.
// - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
// Matched by rule, and a query-copied stamp is indistinguishable at rest
// from a title-matched one — deleting would drop real bindings with the
// fake ones.
//
// # Contract
//
// Dry-run unless `apply`: the sweep deletes keys, so the default call only
// reports. Idempotent: a swept record plans nothing on the next pass, so the
// endpoint is safe to call repeatedly and on a fresh box.
// literatureEchoAnchorSources are the card-locator sources that name an id-bag
// work (an anchor), never a site.
//
// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
// excludes them and the two rules agree on every locator minted today. They are
// listed so the rule reads on its own here.
var literatureEchoAnchorSources = map[string]bool{
"anilist": true,
"openlibrary": true,
"tmdb": true,
"imdb": true,
"musicbrainz": true,
"mal": true,
}
// literatureCardEchoFields are removed from every literature card carrying
// them — position keys and the work reference, none of which a card may hold.
var literatureCardEchoFields = []string{
"workcid",
"chapterNumber",
"volumeNumber",
"chapterStart",
"chapterEnd",
}
// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
// then `anilistid`, then the echo fields in literatureCardEchoFields order.
var literatureEchoReadFields = append(
[]string{"domain", "fileType", "anilistid"},
literatureCardEchoFields...,
)
// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
// records; ByKey counts fields per key — removed on an apply pass, that would
// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
type LiteratureEchoSweep struct {
Planned int
Fixed int
ByKey map[string]int
}
// literatureEchoPlan is one record's pending deletes, in a fixed order
// (`anilistid` first, then literatureCardEchoFields order).
type literatureEchoPlan struct {
hashID string
fields []string
}
// isSiteFormCardSource is the site-form rule on an already-decoded source: not
// an anchor source, and shaped like a host.
func isSiteFormCardSource(source string) bool {
return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
}
// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
func isSiteFormCardLocator(cidStr string) bool {
source, _, err := cid.DecodeCard(cidStr)
return err == nil && isSiteFormCardSource(source)
}
// isSiteCard reports whether a card's field names prove it a site card: at
// least one site-form `cids/` card locator and no card locator of another form.
// Non-card members (a content CID) say nothing about which work the card is and
// are ignored. An empty field list — a record with no field-name index — is
// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
func isSiteCard(fields []string) bool {
site := false
for _, f := range fields {
member := cidFromKeysetField(f)
if member == "" {
continue
}
source, _, err := cid.DecodeCard(member)
if err != nil {
continue
}
if !isSiteFormCardSource(source) {
return false
}
site = true
}
return site
}
// SweepLiteratureEchoes removes query echoes stored on literature cards (see
// the file comment). With apply=false it only plans.
//
// Reads in batched passes (never a per-record SCAN — a single record read
// costs a full keyspace sweep on this store), then deletes only the planned
// fields, through DeleteProperty so the field-name index and the search-dirty
// set stay consistent.
func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
res := LiteratureEchoSweep{ByKey: map[string]int{}}
plans, err := c.planLiteratureEchoes(ctx)
if err != nil {
return res, err
}
res.Planned = len(plans)
if !apply {
for _, p := range plans {
for _, f := range p.fields {
res.ByKey[f]++
}
}
return res, nil
}
// Writes take the client lock themselves, so they happen after the read
// phase has released it. ByKey counts what was actually deleted, so a
// failure midway reports the true partial state.
for _, p := range plans {
// DeleteProperty carries its own per-call timeout; checking ctx here is
// what makes the wrapper's deadline bound the whole write phase.
if err := ctx.Err(); err != nil {
return res, err
}
for _, f := range p.fields {
if err := c.DeleteProperty(p.hashID, f); err != nil {
return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
}
res.ByKey[f]++
}
res.Fixed++
}
return res, nil
}
// planLiteratureEchoes is the read half: what would be deleted, without
// deleting it.
func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
if err != nil {
return nil, fmt.Errorf("smembers index: %w", err)
}
if len(hashIDs) == 0 {
return nil, nil
}
// Pass 1: the gate and echo fields for every record, in a fixed order so
// the MGET response decodes by position: keys[i*per+n] belongs to
// hashIDs[i].
per := len(literatureEchoReadFields)
keys := make([]string, 0, len(hashIDs)*per)
for _, h := range hashIDs {
prefix := c.buildKeyPrefix(h)
for _, f := range literatureEchoReadFields {
keys = append(keys, prefix+f)
}
}
// Chunked so one MGET never carries the whole corpus.
const chunk = 3000
values := make([]interface{}, 0, len(keys))
for start := 0; start < len(keys); start += chunk {
end := start + chunk
if end > len(keys) {
end = len(keys)
}
got, err := c.client.MGet(ctx, keys[start:end]...).Result()
if err != nil {
return nil, fmt.Errorf("mget: %w", err)
}
values = append(values, got...)
}
// A nil slot is an absent key. A present key holding "" is still an echo
// and still goes, so presence — not the value — decides.
get := func(i int) (string, bool) {
if i >= len(values) || values[i] == nil {
return "", false
}
s, ok := values[i].(string)
return s, ok
}
type candidate struct {
hashID string
hasAnilist bool
echoes []string
}
var cands []candidate
var probe []string // candidates whose anilistid needs their cids/ key-set
for i, h := range hashIDs {
base := i * per
domain, _ := get(base)
fileType, _ := get(base + 1)
if domain != "literature" || fileType != "card" {
continue
}
cand := candidate{hashID: h}
_, cand.hasAnilist = get(base + 2)
for j, f := range literatureCardEchoFields {
if _, ok := get(base + 3 + j); ok {
cand.echoes = append(cand.echoes, f)
}
}
if !cand.hasAnilist && len(cand.echoes) == 0 {
continue
}
if cand.hasAnilist {
probe = append(probe, h)
}
cands = append(cands, cand)
}
// Pass 2: the field-name index of only the cards carrying an `anilistid`
// — the `cids/` members are field NAMES, so the index is where they are
// listed. Pipelined per chunk; still no SCAN.
//
// A card with no index (never touched since before the index existed, and
// missed by the startup WarmFieldIndexes pass) reads as an empty set and
// keeps its `anilistid`: proving it a site card would need exactly the
// per-record SCAN this sweep refuses, and a stamp left in place is the
// safe direction.
siteCard := make(map[string]bool, len(probe))
for start := 0; start < len(probe); start += mgetBatch {
end := start + mgetBatch
if end > len(probe) {
end = len(probe)
}
pipe := c.client.Pipeline()
cmds := make([]*redis.StringSliceCmd, 0, end-start)
for _, h := range probe[start:end] {
cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
}
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
for k, cmd := range cmds {
fields, err := cmd.Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
}
siteCard[probe[start+k]] = isSiteCard(fields)
}
}
plans := make([]literatureEchoPlan, 0, len(cands))
for _, cand := range cands {
var fields []string
if cand.hasAnilist && siteCard[cand.hashID] {
fields = append(fields, "anilistid")
}
fields = append(fields, cand.echoes...)
if len(fields) > 0 {
plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
}
}
return plans, nil
}
// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
// handler. Generous timeout: the read phase is batched, but the write phase is
// one round-trip per deleted field.
func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
return c.SweepLiteratureEchoes(ctx, apply)
}
internal/storage/literature_echo_sweep_test.go
-249
View File
@@ -1,249 +0,0 @@
package storage
import (
"context"
"encoding/base32"
"encoding/binary"
"reflect"
"strings"
"testing"
)
// Pinned from meta-read `src/card_cid.rs` — literals, not round-trips.
const (
literatureAnchorCardCID = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy" // ("anilist", "manga:105778")
literatureSiteCardCID = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
// A non-card CID a card may also carry.
literatureContentCID = "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
)
// mintCardCID encodes a card locator the way meta-feeder-sdk
// `compute_card_cid` does. Asserted against the pinned literals before use.
func mintCardCID(source, id string) string {
digest := binary.AppendUvarint(nil, uint64(len(source)))
digest = append(digest, source...)
digest = append(digest, id...)
wire := []byte{0x01}
wire = binary.AppendUvarint(wire, 0x1007)
wire = append(wire, 0x00)
wire = binary.AppendUvarint(wire, uint64(len(digest)))
wire = append(wire, digest...)
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(wire))
}
func TestIsSiteFormCardLocator(t *testing.T) {
if got := mintCardCID("anilist", "manga:105778"); got != literatureAnchorCardCID {
t.Fatalf("test encoder drifted: %s != %s", got, literatureAnchorCardCID)
}
if got := mintCardCID("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"); got != literatureSiteCardCID {
t.Fatalf("test encoder drifted: %s != %s", got, literatureSiteCardCID)
}
for _, tc := range []struct {
name string
cid string
want bool
}{
{"anilist anchor (golden)", literatureAnchorCardCID, false},
{"mangadex site (golden)", literatureSiteCardCID, true},
{"openlibrary anchor", mintCardCID("openlibrary", "OL45804W"), false},
{"another site", mintCardCID("weebcentral.com", "/series/01J76XY/chainsaw-man"), true},
// Not an anchor source, but not host-shaped either: meta-read reads it
// as an id-bag key, so it must not be treated as a site.
{"dotless unknown source", mintCardCID("suwayomi", "42"), false},
{"content CID", literatureContentCID, false},
{"title key", "title:chainsaw man", false},
} {
if got := isSiteFormCardLocator(tc.cid); got != tc.want {
t.Errorf("%s: isSiteFormCardLocator = %v, want %v", tc.name, got, tc.want)
}
}
}
// seedLiteratureCorpus writes one record per rule the sweep has to honour.
func seedLiteratureCorpus(t *testing.T, c *Client) {
t.Helper()
// A site card carrying every echo: all of it goes.
seedRecord(t, c, "site-card", map[string]string{
"domain": "literature",
"fileType": "card",
"title": "Chainsaw Man",
"cids/" + literatureSiteCardCID: "true",
"anilistid": "105778",
"workcid": literatureSiteCardCID,
"chapterNumber": "12",
"volumeNumber": "2",
"chapterStart": "1",
"chapterEnd": "20",
})
// An anchor card: its anilistid is its identity and stays; the position
// echo and the workcid still go.
seedRecord(t, c, "anchor-card", map[string]string{
"domain": "literature",
"fileType": "card",
"title": "Chainsaw Man",
"cids/" + literatureAnchorCardCID: "true",
"anilistid": "105778",
"workcid": literatureAnchorCardCID,
"chapterNumber": "3",
})
// A site card that is already clean: nothing planned.
seedRecord(t, c, "clean-site-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureSiteCardCID: "true",
})
// Both locator forms: which one the id belongs to is unknowable, so it
// stays.
seedRecord(t, c, "merged-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureSiteCardCID: "true",
"cids/" + literatureAnchorCardCID: "true",
"anilistid": "105778",
})
// No card locator at all: not provably a site card.
seedRecord(t, c, "unlocated-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureContentCID: "true",
"anilistid": "105778",
})
// A stored empty value is still an echo.
seedRecord(t, c, "blank-echo-card", map[string]string{
"domain": "literature",
"fileType": "card",
"volumeNumber": "",
})
// A chapter is never swept, whatever it carries.
seedRecord(t, c, "chapter", map[string]string{
"domain": "literature",
"fileType": "archive",
"cids/" + literatureContentCID: "true",
"anilistid": "105778",
"workcid": literatureSiteCardCID,
"chapterNumber": "12",
})
// Not literature: out of scope even though it is a card.
seedRecord(t, c, "screen-card", map[string]string{
"domain": "screen",
"fileType": "card",
"workcid": literatureSiteCardCID,
"chapterNumber": "1",
})
}
func TestPlanLiteratureEchoes_ScopesToLiteratureCards(t *testing.T) {
c, _ := newTestClient(t, "")
seedLiteratureCorpus(t, c)
plans, err := c.planLiteratureEchoes(context.Background())
if err != nil {
t.Fatalf("plan: %v", err)
}
got := map[string][]string{}
for _, p := range plans {
got[p.hashID] = p.fields
}
want := map[string][]string{
"site-card": {"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"},
"anchor-card": {"workcid", "chapterNumber"},
"blank-echo-card": {"volumeNumber"},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("plan =\n %v\nwant\n %v", got, want)
}
}
func TestSweepLiteratureEchoes_DryRunThenApply(t *testing.T) {
c, s := newTestClient(t, "mm:")
seedLiteratureCorpus(t, c)
wantByKey := map[string]int{
"anilistid": 1,
"workcid": 2,
"chapterNumber": 2,
"volumeNumber": 2,
"chapterStart": 1,
"chapterEnd": 1,
}
// Dry run: counts, and nothing written.
dry, err := c.SweepLiteratureEchoes(context.Background(), false)
if err != nil {
t.Fatalf("dry run: %v", err)
}
if dry.Planned != 3 || dry.Fixed != 0 {
t.Errorf("dry run planned/fixed = %d/%d, want 3/0", dry.Planned, dry.Fixed)
}
if !reflect.DeepEqual(dry.ByKey, wantByKey) {
t.Errorf("dry run byKey = %v, want %v", dry.ByKey, wantByKey)
}
if m, _ := c.GetMetadataFlat("site-card"); m["anilistid"] != "105778" {
t.Fatalf("dry run deleted anilistid: %v", m)
}
applied, err := c.SweepLiteratureEchoes(context.Background(), true)
if err != nil {
t.Fatalf("apply: %v", err)
}
if applied.Planned != 3 || applied.Fixed != 3 {
t.Errorf("apply planned/fixed = %d/%d, want 3/3", applied.Planned, applied.Fixed)
}
if !reflect.DeepEqual(applied.ByKey, wantByKey) {
t.Errorf("apply byKey = %v, want %v", applied.ByKey, wantByKey)
}
site, err := c.GetMetadataFlat("site-card")
if err != nil {
t.Fatalf("read site-card: %v", err)
}
for _, k := range []string{"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"} {
if _, ok := site[k]; ok {
t.Errorf("site-card still carries %s", k)
}
// The flat key and its field-name index entry must both be gone, or the
// next read would MGET a ghost field.
if s.Exists("mm:file:site-card/" + k) {
t.Errorf("site-card flat key %s survived", k)
}
if member, _ := s.IsMember("mm:file:site-card/"+FieldsField, k); member {
t.Errorf("site-card field index still lists %s", k)
}
}
if site["title"] != "Chainsaw Man" || site["cids/"+literatureSiteCardCID] != "true" {
t.Errorf("site-card lost fields it should keep: %v", site)
}
anchor, _ := c.GetMetadataFlat("anchor-card")
if anchor["anilistid"] != "105778" {
t.Errorf("anchor-card anilistid = %q, want kept", anchor["anilistid"])
}
if _, ok := anchor["workcid"]; ok {
t.Errorf("anchor-card still carries workcid")
}
chapter, _ := c.GetMetadataFlat("chapter")
if chapter["anilistid"] != "105778" || chapter["workcid"] != literatureSiteCardCID || chapter["chapterNumber"] != "12" {
t.Errorf("chapter was touched: %v", chapter)
}
screen, _ := c.GetMetadataFlat("screen-card")
if screen["workcid"] != literatureSiteCardCID {
t.Errorf("screen-card was touched: %v", screen)
}
merged, _ := c.GetMetadataFlat("merged-card")
if merged["anilistid"] != "105778" {
t.Errorf("merged-card anilistid = %q, want kept", merged["anilistid"])
}
// Idempotent: the endpoint is called by hand, and a second call must find
// nothing — with byKey still an empty map, not nil.
again, err := c.SweepLiteratureEchoes(context.Background(), true)
if err != nil {
t.Fatalf("second pass: %v", err)
}
if again.Planned != 0 || again.Fixed != 0 || again.ByKey == nil || len(again.ByKey) != 0 {
t.Errorf("second pass = %+v, want nothing planned and an empty byKey", again)
}
}
internal/storage/schema_version.go
+43 -56
View File
@@ -1,83 +1,70 @@
package storage
import (
"context"
"fmt"
"strconv"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// SchemaVersion is the Redis-layout version this build of meta-core expects.
// SchemaVersion is the key-layout version this build of meta-core expects.
// Distinct from the snapshot manifest version (which is on-disk format);
// this number describes the in-memory Redis schema — primarily, "are roots
// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
// this number describes the record layout — primarily, "are roots UUIDs or
// content-hash tokens, and do reverse-index keys cid:* exist?"
//
// Bump only when a Redis-layout change can't be detected from the data
// itself or healed by the auto-alias hooks. Each bump is a flag day in
// alpha: operators wipe and restart.
// The Redis → SQLite move did not change the record layout (file:<rid>/<field>,
// cid:<cid>), so it did not bump this: the one-time migration copied records
// verbatim.
//
// Bump only when a layout change can't be detected from the data itself or
// healed by the write path. Each bump is a flag day in alpha: operators wipe
// and restart.
const SchemaVersion = 2
// SchemaVersionKey is the sentinel under which we record SchemaVersion in
// Redis. Checked on every meta-core boot.
// SchemaVersionKey is the sentinel under which we record SchemaVersion.
// Checked on every meta-core boot.
const SchemaVersionKey = "meta-core:schema-version"
// EnsureSchemaVersion verifies the Redis store matches the expected schema
// version. Called once at startup, before any other reads/writes.
// EnsureSchemaVersion verifies the store matches the expected schema version.
// Called once at startup, before any other reads/writes.
//
// Behavior:
// - sentinel present and matches expected → OK
// - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
// - sentinel missing AND file:__index__ non-empty → legacy data, refuse
// - sentinel missing AND no roots → fresh store, stamp sentinel
// - sentinel missing AND roots exist → legacy data, refuse
// - sentinel present but mismatched → refuse, point operator at clean:all
//
// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
// The error message names the wipe remediation because in alpha there
// is no automated migration — operators bring up a clean store.
func (c *Client) EnsureSchemaVersion(expected int) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
key := c.buildKey(SchemaVersionKey)
have, err := c.client.Get(ctx, key).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("get schema version: %w", err)
}
if err == redis.Nil {
// Sentinel absent. Fresh store iff there's no file:__index__ membership.
n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
if sErr != nil && sErr != redis.Nil {
return fmt.Errorf("scard file:__index__: %w", sErr)
return c.update(func(tx *kv.Tx) error {
ctx := tx.Context()
have, ok, err := tx.Get(ctx, SchemaVersionKey)
if err != nil {
return fmt.Errorf("get schema version: %w", err)
}
if n > 0 {
return fmt.Errorf(
"redis has %d indexed roots but no %q sentinel — this is alpha "+
"data from before the UUID-rooted schema landed. Run "+
"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
"to wipe and restart, or downgrade meta-core to a compatible build",
n, SchemaVersionKey)
if !ok {
n, err := tx.Count(ctx, rootsNS)
if err != nil {
return fmt.Errorf("count roots: %w", err)
}
if n > 0 {
return fmt.Errorf(
"the store has %d indexed roots but no %q sentinel — this is alpha "+
"data from before the UUID-rooted schema landed. Delete the "+
"database file to wipe and restart, or downgrade meta-core to a compatible build",
n, SchemaVersionKey)
}
return tx.Put(SchemaVersionKey, strconv.Itoa(expected))
}
if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
return fmt.Errorf("set schema version: %w", err)
if have == strconv.Itoa(expected) {
return nil
}
return nil
}
if have == strconv.Itoa(expected) {
return nil
}
return fmt.Errorf(
"redis schema version is %q, this build of meta-core expects %d. "+
"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
"build that matches the existing data",
have, expected)
return fmt.Errorf(
"store schema version is %q, this build of meta-core expects %d. "+
"Delete the database file to wipe and restart, or use a meta-core "+
"build that matches the existing data",
have, expected)
})
}
internal/storage/search.go
+565
View File
@@ -0,0 +1,565 @@
package storage
// Search over the KV store.
//
// The Redis era answered POST /api/metadata/search from an in-memory copy of
// every record (the API's searchIndex), because Redis could not filter
// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
// gone: search now runs against two derived structures inside the database.
//
// - kv_field_v — an expression index over (lowercased field name parsed out
// of the key, lowercased value, value), covering only record keys. It answers "which
// records have field F with a value like V" for ANY field, including ones
// invented tomorrow: the data stays schema-free, the index is over the key
// itself.
// - fts — an FTS5 trigram table holding one row per record: the record's
// haystack (buildHaystack) plus its rid. Trigrams make every substring of
// 3+ characters indexable, which is the matching rule search always had.
//
// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
// semantics are bit-identical to the in-memory matcher and nothing depends on
// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
"unicode/utf8"
"github.com/metazla/meta-core/internal/kv"
)
// The field expression and the partial-index predicate MUST appear verbatim
// (as literals, never bound parameters) in every query meant to use kv_field_v —
// SQLite only matches a partial index whose WHERE clause it can see implied by
// the query text. TestSearchFilterUsesFieldIndex pins this.
const (
fieldExpr = `lower(substr(k, instr(k,'/')+1))`
recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
ridExpr = `substr(k, 6, instr(k,'/')-6)`
)
// searchSchema is the derived DDL applied when the database is opened.
//
// kv_field_v carries the raw value as a third column so the filters' substring
// test (instr(lower(v), …)) is answered from the index alone. Without it,
// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
// lower(v)), which is dropped here; the rebuild costs ~80 s once.
const searchSchema = `
DROP INDEX IF EXISTS kv_field;
CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
`
// searchIndexFields are the metadata fields folded into a record's search
// haystack — must stay in sync with the fields the search handler documents.
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
// affectsHaystack reports whether writing field can change the haystack.
func affectsHaystack(field string) bool {
if strings.HasPrefix(field, "titles/") {
return true
}
for _, f := range searchIndexFields {
if field == f {
return true
}
}
return false
}
// Haystack folds a record's searchable text fields into one lowercased blob —
// what free-text tokens must each be a substring of.
//
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
// record whose `title` is `3%`.
func Haystack(fields map[string]string) string {
var sb strings.Builder
for _, f := range searchIndexFields {
if v, ok := fields[f]; ok && v != "" {
sb.WriteString(strings.ToLower(v))
sb.WriteByte('\n')
}
}
for _, name := range titleMemberNames(fields) {
sb.WriteString(strings.ToLower(name))
sb.WriteByte('\n')
}
return sb.String()
}
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
// back. Sorted so the haystack is deterministic.
func titleMemberNames(fields map[string]string) []string {
var names []string
for k, v := range fields {
rest, ok := strings.CutPrefix(k, "titles/")
if !ok || v == "" || v == "false" {
continue
}
_, name, ok := strings.Cut(rest, "/")
if !ok || name == "" {
continue
}
names = append(names, strings.ReplaceAll(name, "∕", "/"))
}
sort.Strings(names)
return names
}
// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
// "query is a substring of the hashID" rule also has an indexed candidate path.
// Verification checks the two separately (a rid token plus a title token must
// not match together).
func ftsHay(hashID string, fields map[string]string) string {
return Haystack(fields) + strings.ToLower(hashID) + "\n"
}
// refreshSearchTx rewrites a record's FTS row from its current fields, or
// drops it when the record is gone.
func refreshSearchTx(tx *kv.Tx, hashID string) error {
fields, err := readRecord(tx.Context(), tx, hashID)
if err != nil {
return err
}
if len(fields) == 0 {
return deleteSearchRowTx(tx, hashID)
}
return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
}
// ensureSearchRowTx creates the record's FTS row if it has none (a record
// whose writes never touched a haystack field still needs one for the rid
// rule).
func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil || ok {
return err
}
return refreshSearchTx(tx, hashID)
}
func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil {
return err
}
if ok {
if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
return fmt.Errorf("fts update %s: %w", hashID, err)
}
return nil
}
res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
if err != nil {
return fmt.Errorf("fts insert %s: %w", hashID, err)
}
rowid, err := res.LastInsertId()
if err != nil {
return err
}
return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
}
func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil || !ok {
return err
}
if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
return fmt.Errorf("fts delete %s: %w", hashID, err)
}
_, err = tx.Delete(ftsNS + hashID)
return err
}
// SearchFilter is one structured filter: AND across filters, OR across Values.
type SearchFilter struct {
Property string
Values []string
}
// SearchHit is one verified result.
type SearchHit struct {
HashID string
Fields map[string]string
}
// Search returns up to limit records, in hashID order, that satisfy match.
//
// query and filters only NARROW the candidates (SQL, superset semantics);
// match is the authoritative test, run on each hydrated record. Pass the same
// query/filters match evaluates, or results will be missing.
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
s, err := c.kv()
if err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
ctx, cancel := opCtx()
defer cancel()
driver, probes, err := c.planCandidates(ctx, s, query, filters)
if err != nil {
return nil, err
}
page := max(limit*2, 500)
hits := make([]SearchHit, 0, limit)
after := ""
for len(hits) < limit {
ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
if err != nil {
return nil, err
}
survivors := ids
if len(probes) > 0 && len(ids) > 0 {
if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
return nil, err
}
}
for _, id := range survivors {
fields, err := readRecord(ctx, s, id)
if err != nil {
return nil, err
}
if fields == nil || !match(id, fields) {
continue
}
hits = append(hits, SearchHit{HashID: id, Fields: fields})
if len(hits) >= limit {
break
}
}
if len(ids) < page {
break
}
after = ids[len(ids)-1]
}
return hits, nil
}
// queryIDs returns one page of the driver's candidates, in rid order.
func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
if err != nil {
return nil, fmt.Errorf("search candidates: %w", err)
}
defer rows.Close()
var out []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
// probeSQL applies the non-driving filters to one page of candidate rids
// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
func probeSQL(probes []candidatePart) (string, []any) {
where := make([]string, len(probes))
var args []any
for i, p := range probes {
where[i] = p.exists
args = append(args, p.existsArgs...)
}
return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
}
// probeIDs keeps the ids (in their order) that pass every probe.
func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
q, args := probeSQL(probes)
list, err := json.Marshal(ids)
if err != nil {
return nil, err
}
rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
if err != nil {
return nil, fmt.Errorf("search probes: %w", err)
}
defer rows.Close()
pass := make(map[string]bool, len(ids))
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
pass[id] = true
}
if err := rows.Err(); err != nil {
return nil, err
}
out := ids[:0:0]
for _, id := range ids {
if pass[id] {
out = append(out, id)
}
}
return out, nil
}
// candidatePart is one narrowing condition: free text, or one filter.
type candidatePart struct {
sql string // SELECT … AS rid
args []any
// exists is the same filter as a predicate on candidate row c.rid (empty
// for free text, which has no per-record probe: the FTS rid column is not
// indexed).
exists string
existsArgs []any
}
// estimateCap bounds the count used to pick the driving part: it only has to
// tell a 50-row part from a 100k-row one.
const estimateCap = 1000
// planCandidates picks how to narrow a search: one DRIVING part (the most
// selective, by a capped count) that lists candidate rids in order, plus PROBES
// — every other filter, checked as an EXISTS over each candidate's own key
// range (a primary-key range read of ~40 rows).
//
// Search pages the driver in Go and probes one page at a time, so a broad
// query stops as soon as `limit` records verify. Two earlier shapes were
// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
// and probing inside the driver query (SQLite evaluates the probe on every
// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
// both ~100k, 10 s). Free text that does not drive is dropped from SQL
// entirely — the verifier still applies it, and dropping a condition only
// widens the superset.
func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
var parts []candidatePart
if p, ok := textCandidates(query); ok {
parts = append(parts, p)
}
for _, f := range filters {
if p, ok := filterCandidates(f); ok {
parts = append(parts, p)
}
}
if len(parts) == 0 {
return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
}
driver := 0
if len(parts) > 1 {
best := int64(-1)
for i, p := range parts {
var n int64
q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
}
if best < 0 || n < best {
best, driver = n, i
}
}
}
var probes []candidatePart
for i, p := range parts {
if i != driver && p.exists != "" {
probes = append(probes, p)
}
}
return parts[driver], probes, nil
}
// candidateSQL is planCandidates without the store: text plus filters, the
// first part driving. Used by tests that inspect the plan.
func candidateSQL(query string, filters []SearchFilter) (string, []any) {
var parts []candidatePart
if p, ok := textCandidates(query); ok {
parts = append(parts, p)
}
for _, f := range filters {
if p, ok := filterCandidates(f); ok {
parts = append(parts, p)
}
}
if len(parts) == 0 {
return "", nil
}
return parts[0].sql, parts[0].args
}
// textCandidates narrows by free text: every whitespace token must be a
// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
// same rows.
func textCandidates(query string) (candidatePart, bool) {
tokens := strings.Fields(strings.ToLower(query))
if len(tokens) == 0 {
return candidatePart{}, false
}
var match []string
var likes []string
var likeArgs []any
for _, t := range tokens {
if utf8.RuneCountInString(t) >= 3 {
match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
} else {
likes = append(likes, `hay LIKE ? ESCAPE '\'`)
likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
}
}
var where []string
var args []any
if len(match) > 0 {
where = append(where, `fts MATCH ?`)
args = append(args, strings.Join(match, " "))
}
where = append(where, likes...)
args = append(args, likeArgs...)
return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
}
func escapeLike(s string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return r.Replace(s)
}
// filterCandidates narrows by one filter — the UNION, over its values, of the
// two storage shapes recordMatchesFilter probes:
//
// - key-set member `<prop>/<value>` (any case): the lowercased field equals
// lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
// lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
// comparison could miss);
// - flat field `<prop>` whose lowercased value contains the lowercased value
// — covers exact and contains alike, and comma-split csv-set values.
//
// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
// name): it constrains nothing in SQL and is left to the verifier.
func filterCandidates(f SearchFilter) (candidatePart, bool) {
var values []string
for _, v := range f.Values {
if v = strings.TrimSpace(v); v != "" {
values = append(values, v)
}
}
if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
return candidatePart{}, false
}
prop := asciiLower(f.Property)
var conds []string
var args []any
memberRange := false
for _, v := range values {
if isASCII(v) {
lv := asciiLower(v)
conds = append(conds, fieldExpr+` = ?`)
args = append(args, prop+"/"+lv)
conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
args = append(args, prop, lv)
} else {
memberRange = true
conds = append(conds, fieldExpr+` = ?`)
args = append(args, prop)
}
}
if memberRange {
conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
}
// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
// OR inside one SELECT would defeat the partial index; plain UNION would
// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
selects := make([]string, len(conds))
for i, cond := range conds {
selects[i] = base + cond
}
// As a probe: the same conditions over one candidate's key range.
exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
strings.Join(conds, ") OR (") + `)))`
return candidatePart{
sql: strings.Join(selects, " UNION ALL "), args: args,
exists: exists, existsArgs: args,
}, true
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
// asciiLower lowercases A–Z only, exactly like SQLite's lower().
func asciiLower(s string) string {
b := []byte(s)
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + ('a' - 'A')
}
}
return string(b)
}
func envInt(name string, def int) int {
if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
return v
}
return def
}
// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
// empty but records exist — a store restored from a snapshot, a copy made
// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
// change its tokenizer. The rows are derived, so this is a repair, not a
// migration. Runs in bounded batches so writes keep flowing; until it
// finishes, free-text search under-matches (filters are unaffected).
func (c *Client) RebuildSearchIfEmpty() (int, error) {
s, err := c.kv()
if err != nil {
return 0, err
}
ctx, cancel := opCtx()
var rows int64
err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
cancel()
if err != nil {
return 0, fmt.Errorf("count fts rows: %w", err)
}
if rows > 0 {
return 0, nil
}
ids, err := c.GetAllHashIDs()
if err != nil {
return 0, err
}
const batch = 500
for start := 0; start < len(ids); start += batch {
chunk := ids[start:min(start+batch, len(ids))]
if err := c.update(func(tx *kv.Tx) error {
for _, id := range chunk {
if err := refreshSearchTx(tx, id); err != nil {
return err
}
}
return nil
}); err != nil {
return start, fmt.Errorf("rebuild search rows: %w", err)
}
}
return len(ids), nil
}
internal/storage/search_dirty.go
-245
View File
@@ -1,245 +0,0 @@
package storage
import (
"context"
"fmt"
"log"
"sort"
"strings"
"time"
)
// Incremental search-index maintenance.
//
// # Why this exists
//
// The API's in-memory search index used to be rebuilt in full every 60 s:
// GetSearchIndexTuples does a whole-keyspace SCAN plus batched MGETs, builds an
// intermediate map of every record, then allocates a fresh []SearchTuple — all
// while the previous index is still live serving reads. That cost is O(corpus),
// not O(change): adding 200 records in a minute re-read every one of the other
// ~620 000 field keys to notice them.
//
// Measured on a 19 622-record staging corpus (2026-08-30): SCAN was 57.8 % and
// MGET 19.1 % of ALL Redis CPU, and ~90 % of the MGET calls plus ~48 % of the
// SCAN calls were this loop — roughly 45 % of Redis's total work, spent
// continuously whether or not anything changed. The Go side held 783 MB for
// data Redis stores in 135 MB, largely because of the double copy.
//
// That was tolerable while the corpus was small and mostly static. It stops
// being tolerable now that meta-core is the platform's permanent, additive
// store: every search hit on every gateway is retained forever, so the rebuild
// cost is a ratchet.
//
// # The shape
//
// Every write path marks its record in a dirty set; the refresh drains that set
// and re-reads only those records. Cost becomes O(change).
//
// Marking rides along with the field-index maintenance every write path already
// performs, so there is no new bookkeeping discipline to remember — if you are
// adding a write path, the same line that keeps `__fields__` correct keeps this
// correct.
//
// # Correctness
//
// The drain is a destructive SPOP, so a write that lands *during* a refresh is
// either already in the drained batch or still in the set for the next tick —
// never dropped. The cost of the race is one tick of staleness, which is well
// inside the interval the index was already stale by.
//
// A refresh that fails after draining WOULD lose its batch, so drainDirty's
// caller re-marks on error (see MarkDirty / the API's refreshIncremental).
//
// ⚠ **The destructive drain assumes ONE index consumer per Redis.** Two
// meta-core processes sharing a keyspace would steal each other's marks, and
// each would end up with an index missing whatever the other drained. That is
// fine today — multi-instance leader election was deprecated and the dev/prod
// stacks run a single meta-core — but it is the assumption to revisit first if
// a second instance is ever pointed at the same Redis. A per-consumer dirty key
// (`file:__dirty__:<consumer>`) is the shape that would fix it.
//
// The API's periodic full reconcile (searchIndexReconcileEvery) is the backstop
// for both this and for a write path that forgets to mark: a missed mark costs
// staleness until the next reconcile, never permanent divergence.
// dirtyField is the reserved key holding the set of record ids whose metadata
// changed since the last search-index refresh. It sits under the same `file:`
// namespace as `__index__` and is likewise not a record.
const dirtyField = "file:__dirty__"
// dirtyDrainCap bounds one incremental refresh. A burst larger than this is
// drained over several ticks rather than in one long-held pass — the point of
// the change is to keep any single refresh cheap. Sized well above a normal
// search-persist flush (which writes at most a few hundred records).
const dirtyDrainCap = 5000
// buildDirtyKey constructs the key for the dirty-record set.
func (c *Client) buildDirtyKey() string {
return c.buildKey(dirtyField)
}
// markDirtyLocked records that hashID's metadata changed. Caller must hold c.mu.
//
// Best-effort by design: a failure here costs one record its incremental
// refresh, and the periodic full rebuild still repairs it. Failing the caller's
// write instead would trade a stale index entry for a lost write, which is
// strictly worse.
func (c *Client) markDirtyLocked(ctx context.Context, hashID string) {
if hashID == "" {
return
}
if err := c.client.SAdd(ctx, c.buildDirtyKey(), hashID).Err(); err != nil {
log.Printf("[STORAGE] dirty-mark failed for %s: %v (index repairs on the next full rebuild)", hashID, err)
}
}
// MarkDirty records that hashID needs re-indexing. Exported for the API layer,
// which re-marks a batch whose refresh failed after the drain.
func (c *Client) MarkDirty(hashIDs ...string) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil || len(hashIDs) == 0 {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
members := make([]interface{}, 0, len(hashIDs))
for _, h := range hashIDs {
if h != "" {
members = append(members, h)
}
}
if len(members) == 0 {
return
}
if err := c.client.SAdd(ctx, c.buildDirtyKey(), members...).Err(); err != nil {
log.Printf("[STORAGE] dirty re-mark failed for %d records: %v", len(members), err)
}
}
// DrainDirty removes and returns up to dirtyDrainCap record ids from the dirty
// set. `more` reports whether entries remain, so the caller can schedule
// another pass instead of waiting a full interval to catch up on a burst.
//
// Destructive on purpose: SPOP makes concurrent refreshes disjoint and stops a
// record being re-read every tick forever. The caller owns the drained ids
// until it has indexed them — on failure it must MarkDirty them again.
func (c *Client) DrainDirty() (ids []string, more bool, err error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
key := c.buildDirtyKey()
ids, err = c.client.SPopN(ctx, key, dirtyDrainCap).Result()
if err != nil {
return nil, false, fmt.Errorf("spop dirty: %w", err)
}
if len(ids) < dirtyDrainCap {
return ids, false, nil
}
remaining, err := c.client.SCard(ctx, key).Result()
if err != nil {
// We have the batch; not knowing the remainder just means the next
// tick picks it up on schedule.
return ids, false, nil
}
return ids, remaining > 0, nil
}
// GetSearchIndexTuplesFor reads the search tuples for a specific set of records
// — the incremental counterpart to GetSearchIndexTuples.
//
// Reads go through the per-record field index (GetMetadataFlat), so this is two
// round-trips per record rather than a keyspace scan. That trade is what makes
// it O(change): it beats the full pass whenever the changed set is a small
// fraction of the corpus, and the caller bounds the batch (dirtyDrainCap).
//
// A record present in the dirty set but absent from storage was deleted; it is
// reported in `gone` so the caller can evict it from the index rather than
// leaving a tombstone that still answers searches.
func (c *Client) GetSearchIndexTuplesFor(hashIDs []string) (tuples []SearchTuple, gone []string, err error) {
if len(hashIDs) == 0 {
return nil, nil, nil
}
excluded := indexExcludePrefixes()
tuples = make([]SearchTuple, 0, len(hashIDs))
for _, hashID := range hashIDs {
fields, ferr := c.GetMetadataFlat(hashID)
if ferr != nil {
// Re-mark so the record is retried rather than silently missing
// from the index until the next full rebuild.
c.MarkDirty(hashID)
log.Printf("[STORAGE] incremental index read failed for %s: %v", hashID, ferr)
continue
}
if len(fields) == 0 {
gone = append(gone, hashID)
continue
}
for field := range fields {
if hasAnyPrefix(field, excluded) {
delete(fields, field)
}
}
tuples = append(tuples, SearchTuple{
HashID: hashID,
Haystack: buildHaystack(fields),
Fields: fields,
})
}
return tuples, gone, nil
}
// buildHaystack folds a record's searchable text fields into one lowercased
// blob. Shared by the full and incremental index reads so the two can never
// disagree about what is searchable — a drift there would make a record's
// matchability depend on which path happened to index it.
//
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
// record whose `title` is `3%`.
func buildHaystack(fields map[string]string) string {
var sb strings.Builder
for _, f := range searchIndexFields {
if v, ok := fields[f]; ok && v != "" {
sb.WriteString(strings.ToLower(v))
sb.WriteByte('\n')
}
}
for _, name := range titleMemberNames(fields) {
sb.WriteString(strings.ToLower(name))
sb.WriteByte('\n')
}
return sb.String()
}
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
// back. Sorted: the fields map has no order, and the full and incremental
// builds must produce byte-identical haystacks.
func titleMemberNames(fields map[string]string) []string {
var names []string
for k, v := range fields {
rest, ok := strings.CutPrefix(k, "titles/")
if !ok || v == "" || v == "false" {
continue
}
_, name, ok := strings.Cut(rest, "/")
if !ok || name == "" {
continue
}
names = append(names, strings.ReplaceAll(name, "\u2215", "/"))
}
sort.Strings(names)
return names
}
internal/storage/search_dirty_test.go
-230
View File
@@ -1,230 +0,0 @@
package storage
import (
"context"
"sort"
"testing"
)
// Tests for incremental search-index maintenance (search_dirty.go).
//
// The dirty set is derived state on the hot write path, which makes its failure
// mode the dangerous kind: a MISSING mark doesn't error, it just leaves the
// search index quietly serving a stale copy of that record until someone
// triggers a full rebuild. So the property worth pinning is coverage — every
// path that changes what a search would match must mark — and that is what the
// bulk of these tests assert, one write path at a time.
//
// The second property is that a delete is a mark too, not an unmark: the
// refresh has to LEARN the record is gone in order to evict it. Marking on
// delete is counter-intuitive enough to be worth a test of its own.
func dirtyMembers(t *testing.T, c *Client) []string {
t.Helper()
got, err := c.client.SMembers(context.Background(), c.buildDirtyKey()).Result()
if err != nil {
t.Fatalf("smembers dirty: %v", err)
}
sort.Strings(got)
return got
}
func clearDirty(t *testing.T, c *Client) {
t.Helper()
if err := c.client.Del(context.Background(), c.buildDirtyKey()).Err(); err != nil {
t.Fatalf("del dirty: %v", err)
}
}
// Every write path must mark. A path that doesn't leaves the index serving a
// stale record with no error anywhere.
func TestDirty_EveryWritePathMarks(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
cases := []struct {
name string
id string
write func(id string) error
}{
{"SetMetadataFlat", "r-set", func(id string) error {
return c.SetMetadataFlat(id, map[string]string{"title": "A"})
}},
{"SetProperty", "r-prop", func(id string) error {
return c.SetProperty(id, "title", "B")
}},
{"MergeMetadataFlat", "r-merge", func(id string) error {
_, err := c.MergeMetadataFlat(id, map[string]string{"title": "C"})
return err
}},
{"AddToSet", "r-add", func(id string) error {
_, err := c.AddToSet(id, "tags", "x")
return err
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
clearDirty(t, c)
if err := tc.write(tc.id); err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
got := dirtyMembers(t, c)
if len(got) != 1 || got[0] != tc.id {
t.Fatalf("%s must mark %q dirty, got %v", tc.name, tc.id, got)
}
})
}
})
}
// RemoveFromSet's non-empty branch changes a field's VALUE without changing the
// field-name index, so it is the one path that a mark riding on field-index
// maintenance would have missed. Pinned explicitly for that reason.
func TestDirty_RemoveFromSetMarksEvenWhenTheFieldSurvives(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-rem"
for _, v := range []string{"x", "y"} {
if _, err := c.AddToSet(id, "tags", v); err != nil {
t.Fatalf("AddToSet: %v", err)
}
}
clearDirty(t, c)
if _, err := c.RemoveFromSet(id, "tags", "x"); err != nil {
t.Fatalf("RemoveFromSet: %v", err)
}
// `tags` still exists (y remains) — no field-index change, but the
// searchable value changed.
if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
t.Fatalf("a value-only change must still mark dirty, got %v", got)
}
})
}
// A delete MARKS rather than unmarks: the refresh must read the record, find it
// gone, and evict it. Unmarking would leave a deleted record answering searches
// until the next full rebuild.
func TestDirty_DeletesMarkSoTheRefreshCanEvict(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-del"
if err := c.SetMetadataFlat(id, map[string]string{"title": "Doomed"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
clearDirty(t, c)
if _, err := c.DeleteMetadata(id); err != nil {
t.Fatalf("DeleteMetadata: %v", err)
}
if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
t.Fatalf("a delete must mark dirty so the index evicts it, got %v", got)
}
// …and the incremental read reports it as gone rather than as an error
// or an empty-but-present tuple.
tuples, gone, err := c.GetSearchIndexTuplesFor([]string{id})
if err != nil {
t.Fatalf("GetSearchIndexTuplesFor: %v", err)
}
if len(tuples) != 0 {
t.Fatalf("a deleted record must produce no tuple, got %d", len(tuples))
}
if len(gone) != 1 || gone[0] != id {
t.Fatalf("a deleted record must be reported gone, got %v", gone)
}
})
}
// The drain is destructive so concurrent refreshes stay disjoint and a record
// isn't re-read every tick forever. The caller owns what it drained.
func TestDirty_DrainIsDestructive(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
clearDirty(t, c)
c.MarkDirty("a", "b", "c")
ids, more, err := c.DrainDirty()
if err != nil {
t.Fatalf("DrainDirty: %v", err)
}
sort.Strings(ids)
if len(ids) != 3 {
t.Fatalf("expected 3 drained ids, got %v", ids)
}
if more {
t.Fatal("a fully-drained set must not report more")
}
if got := dirtyMembers(t, c); len(got) != 0 {
t.Fatalf("drain must empty the set, got %v", got)
}
// A second drain is empty, not an error — the steady state when nothing
// has been written since the last tick.
ids, _, err = c.DrainDirty()
if err != nil || len(ids) != 0 {
t.Fatalf("draining an empty set must be a clean no-op, got %v / %v", ids, err)
}
})
}
// The incremental read must produce tuples indistinguishable from the full
// rebuild's — same haystack, same fields, same exclusions. A drift here would
// make a record's matchability depend on which path happened to index it, which
// is invisible until someone searches for the one term that differs.
func TestDirty_IncrementalTuplesMatchTheFullRebuild(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-parity"
if err := c.SetMetadataFlat(id, map[string]string{
"title": "Parity Release",
"fileName": "parity.s01e01.mkv",
"fileType": "video",
"titles/eng/3 Percent": "true",
"titles/fra/3 Pourcent": "true",
"categories/XXX": "true",
"categories/newznab/5070": "true",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
full := tuplesByHashID(t, c)[id]
incr, gone, err := c.GetSearchIndexTuplesFor([]string{id})
if err != nil {
t.Fatalf("GetSearchIndexTuplesFor: %v", err)
}
if len(gone) != 0 || len(incr) != 1 {
t.Fatalf("expected exactly one live tuple, got %d live / %d gone", len(incr), len(gone))
}
if incr[0].Haystack != full.Haystack {
t.Errorf("haystack drift:\n incremental %q\n full %q", incr[0].Haystack, full.Haystack)
}
if len(incr[0].Fields) != len(full.Fields) {
t.Errorf("field-count drift: incremental %d, full %d", len(incr[0].Fields), len(full.Fields))
}
for k, v := range full.Fields {
if incr[0].Fields[k] != v {
t.Errorf("field %q drift: incremental %q, full %q", k, incr[0].Fields[k], v)
}
}
// The index projection must apply on both paths.
if _, ok := incr[0].Fields["categories/newznab/5070"]; ok {
t.Error("the incremental path must honour the index exclusion list too")
}
if _, ok := incr[0].Fields["categories/XXX"]; !ok {
t.Error("the incremental path must keep the adult-filter category")
}
})
}
// A `titles/<lang3>/<name>` member contributes its NAME (the key), never its
// "true" value, and the `∕` a writer substituted for `/` reads back as `/`.
func TestBuildHaystack_FoldsTitleMemberNames(t *testing.T) {
got := buildHaystack(map[string]string{
"title": "3%",
"titles/fra/3 Pourcent": "true",
"titles/eng/3 Percent": "true",
"titles/und/Fate\u2215Zero": "true",
"titles/eng": "Retired Scalar",
"titles/eng/": "true",
})
want := "3%\n3 percent\n3 pourcent\nfate/zero\n"
if got != want {
t.Errorf("haystack = %q, want %q", got, want)
}
}
internal/storage/search_index_exclude_test.go
-116
View File
@@ -1,116 +0,0 @@
package storage
import (
"testing"
)
// Tests for the search-index field projection (GetSearchIndexTuples +
// indexExcludePrefixes).
//
// The projection narrows the in-memory index only — Redis keeps every field and
// every per-record read still returns it. What has to be pinned is the boundary
// between the two, because getting it wrong in either direction is silent:
// exclude too little and the index keeps paying for fields nothing reads;
// exclude too much and a record served *from the index* comes back missing a
// field a downstream consumer depends on.
//
// The specific hazard the default guards is adult-content filtering:
// `categories/XXX` is the server-side enforcement point (meta-search's
// `query_eval::fields_are_adult`, meta-gateway's `cards::fields_are_adult`),
// and `exclude_adult` defaults to ON. Excluding the whole `categories/` family
// would make every index-served record read as non-adult and the filter would
// fail OPEN — so the default must keep `categories/XXX` while dropping the
// reader-less `categories/newznab/` mirror.
func tuplesByHashID(t *testing.T, c *Client) map[string]SearchTuple {
t.Helper()
got, err := c.GetSearchIndexTuples()
if err != nil {
t.Fatalf("GetSearchIndexTuples: %v", err)
}
out := make(map[string]SearchTuple, len(got))
for _, tu := range got {
out[tu.HashID] = tu
}
return out
}
func TestSearchIndex_ExcludesNewznabCategoriesButKeepsTheRest(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "root-1"
if err := c.SetMetadataFlat(id, map[string]string{
"title": "Some Release",
"fileType": "video",
"categories/TV/Anime": "true",
"categories/XXX": "true",
"categories/newznab/5070": "true",
"categories/newznab/2040": "true",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
fields := tuplesByHashID(t, c)[id].Fields
if fields == nil {
t.Fatal("record missing from the search index")
}
for _, excluded := range []string{"categories/newznab/5070", "categories/newznab/2040"} {
if _, ok := fields[excluded]; ok {
t.Errorf("%s must not be indexed — it is a reader-less mirror vocabulary", excluded)
}
}
// The load-bearing ones must survive. XXX especially: without it the
// adult filter fails open on every index-served record.
for _, kept := range []string{"categories/XXX", "categories/TV/Anime", "title", "fileType"} {
if _, ok := fields[kept]; !ok {
t.Errorf("%s must stay in the index", kept)
}
}
// The store itself is untouched — the projection is index-only.
stored, err := c.GetMetadataFlat(id)
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
if _, ok := stored["categories/newznab/5070"]; !ok {
t.Error("an index-excluded field must still be stored and readable per-record")
}
})
}
func TestIndexExcludePrefixes_EnvOverride(t *testing.T) {
if got := indexExcludePrefixes(); len(got) != 1 || got[0] != "categories/newznab/" {
t.Fatalf("unset env must yield the default, got %v", got)
}
t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "foo/, bar/ ,")
got := indexExcludePrefixes()
if len(got) != 2 || got[0] != "foo/" || got[1] != "bar/" {
t.Fatalf("override must trim and drop blanks, got %v", got)
}
// An explicitly empty value is the escape hatch: index everything.
t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "")
if got := indexExcludePrefixes(); len(got) != 0 {
t.Fatalf("an empty override must disable exclusion, got %v", got)
}
}
func TestHasAnyPrefix(t *testing.T) {
prefixes := []string{"categories/newznab/", "tmp/"}
cases := map[string]bool{
"categories/newznab/5070": true,
"tmp/scratch": true,
"categories/XXX": false,
"categories/newznab": false, // the family root is not the mirror
"title": false,
}
for field, want := range cases {
if got := hasAnyPrefix(field, prefixes); got != want {
t.Errorf("hasAnyPrefix(%q) = %v, want %v", field, got, want)
}
}
if hasAnyPrefix("anything", nil) {
t.Error("an empty prefix list must exclude nothing")
}
}
internal/storage/search_test.go
+327
View File
@@ -0,0 +1,327 @@
package storage
import (
"context"
"fmt"
"sort"
"strings"
"testing"
"github.com/metazla/meta-core/internal/kv"
)
// These tests pin the one property storage.Search must have: its SQL
// candidates are a SUPERSET of the records the exact matcher accepts. A
// candidate miss is a silently dropped search result, so each case below is a
// shape where SQLite and Go disagree about text (case folding, csv values,
// FTS syntax, trigram length) and the candidate query has to cover for it.
//
// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
// every candidate path; the API tests own the matcher itself.
func seed(t *testing.T, c *Client, records map[string]map[string]string) {
t.Helper()
for id, fields := range records {
if err := c.SetMetadataFlat(id, fields); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
}
// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
for _, v := range values {
if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" {
return true
}
}
prefix := strings.ToLower(prop) + "/"
for k, val := range fields {
if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) {
for _, v := range values {
if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" {
return true
}
}
}
}
fv, ok := fields[prop]
if !ok {
return false
}
for _, piece := range strings.Split(fv, ",") {
piece = strings.TrimSpace(piece)
for _, v := range values {
if exact && strings.EqualFold(piece, v) {
return true
}
if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
return true
}
}
}
return false
}
func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string {
t.Helper()
hits, err := c.Search(query, filters, 100, match)
if err != nil {
t.Fatalf("search: %v", err)
}
ids := make([]string, 0, len(hits))
for _, h := range hits {
ids = append(ids, h.HashID)
}
return ids
}
func textMatch(query string) func(string, map[string]string) bool {
tokens := strings.Fields(strings.ToLower(query))
return func(id string, fields map[string]string) bool {
hay := Haystack(fields)
for _, tok := range tokens {
if !strings.Contains(hay, tok) {
return strings.Contains(strings.ToLower(id), strings.ToLower(query))
}
}
return true
}
}
func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool {
return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) }
}
func equalIDs(t *testing.T, got []string, want ...string) {
t.Helper()
sort.Strings(want)
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("got %v, want %v", got, want)
}
}
func TestSearchFilterCandidates(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"},
"R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"},
"R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"},
"R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"},
})
cases := []struct {
name string
prop string
exact bool
values []string
want []string
}{
{"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}},
{"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}},
{"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}},
{"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}},
{"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}},
{"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}},
{"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}},
{"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}},
{"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...))
equalIDs(t, got, tc.want...)
})
}
}
func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"R1": {"genres/Action": "true", "fileType": "video"},
"R2": {"genres/Action": "true", "fileType": "audio"},
})
match := func(_ string, f map[string]string) bool {
return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video")
}
got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match)
equalIDs(t, got, "R1")
}
func TestSearchFreeTextCandidates(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"},
"01KBBB": {"title": "Dune: Part Two"},
"01KCCC": {"title": "進撃の巨人"},
"01KDDD": {"title": "3%", "titles/eng/3 percent": "true"},
"01KEEE": {"title": "AND -x a:b * \"quoted\""},
"01KFFF": {"title": "Unrelated"},
})
cases := []struct {
query string
want []string
}{
{"naruto", []string{"01KAAA"}},
{"NARUTO s01e02", []string{"01KAAA"}},
{"dune:", []string{"01KBBB"}},
{"巨人", []string{"01KCCC"}}, // 2 runes: below trigram length
{"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set
{"3%", []string{"01KDDD"}}, // LIKE metacharacter in a short token
{"AND", []string{"01KEEE"}}, // FTS operator as a literal
{"-x", []string{"01KEEE"}},
{"a:b", []string{"01KEEE"}},
{"*", []string{"01KEEE"}},
{`"quoted"`, []string{"01KEEE"}},
{"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring
}
for _, tc := range cases {
t.Run(tc.query, func(t *testing.T) {
got := search(t, c, tc.query, nil, textMatch(tc.query))
equalIDs(t, got, tc.want...)
})
}
}
// A rid token and a title token must not match together: the rid is in the
// FTS row only as a candidate path for the "whole query is a hashID substring"
// rule, and verification checks the two separately.
func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}})
if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 {
t.Fatalf("rid + title token matched: %v", got)
}
}
func TestSearchFollowsWritesAndDeletes(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
if err := c.SetProperty("R1", "title", "After"); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
equalIDs(t, search(t, c, "before", nil, textMatch("before")))
if _, err := c.DeleteMetadata("R1"); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "after", nil, textMatch("after")))
equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
}
func TestSearchPagesPastRejectedCandidates(t *testing.T) {
c := newTestClient(t)
records := map[string]map[string]string{}
for i := 0; i < 450; i++ {
id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3)
records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"}
}
records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"}
seed(t, c, records)
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}},
func(_ string, f map[string]string) bool { return f["keep"] == "yes" })
equalIDs(t, got, "RZZZZ")
}
// The filter candidates only scale if SQLite actually uses the partial
// expression index — which it does only when the query repeats the index's
// expression and WHERE clause as literals.
func TestSearchFilterUsesFieldIndex(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}})
q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}})
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") {
t.Fatalf("filter candidates do not use kv_field_v: %s", joined)
}
}
func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) {
if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" {
t.Fatalf("got %q %q %v", h, f, ok)
}
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}})
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video"))
equalIDs(t, got, "bafkreiabcdefghijklmnop")
}
func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}})
if err := c.update(func(tx *kv.Tx) error {
if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
return err
}
_, err := tx.DeletePrefix(ftsNS, 0)
return err
}); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")))
n, err := c.RebuildSearchIfEmpty()
if err != nil || n != 2 {
t.Fatalf("rebuilt %d (err %v), want 2", n, err)
}
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1")
if n, _ := c.RebuildSearchIfEmpty(); n != 0 {
t.Fatalf("second rebuild ran again (%d)", n)
}
}
// Text plus a broad filter: the selective part drives and the filter becomes a
// per-candidate probe over the record's own key range — never a full
// materialisation of the filter's result set.
func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
c := newTestClient(t)
recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}}
for i := 1; i < 300; i++ {
recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"}
}
seed(t, c, recs)
filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 {
t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes))
}
q, pargs := probeSQL(probes)
args := append([]any{`["R0000"]`}, pargs...)
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
t.Fatalf("filter probe is not a primary-key range read: %s", joined)
}
equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool {
return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
}), "R0000")
}
internal/storage/stats.go
+28 -35
View File
@@ -1,52 +1,45 @@
package storage
import (
"context"
"fmt"
"time"
)
// CountRoots returns the number of metadata roots in the file index.
//
// SCARD, not SMEMBERS: GetAllHashIDs ships every member over the wire (~3MB
// and rising on an 88k-record box) just so the caller can take len(). A
// dashboard tile that polls needs the number, never the list.
// CountRoots returns the number of metadata roots in the root index — one
// range count over idx:roots/, never the list itself.
func (c *Client) CountRoots() (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.SCard(ctx, c.buildIndexKey()).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("scard failed: %w", err)
return 0, err
}
return n, nil
ctx, cancel := opCtx()
defer cancel()
return s.Count(ctx, rootsNS)
}
// CountKeys returns DBSIZE — every key in the Redis database, not just the
// ones meta-core owns. It is the denominator for "how much of Redis is
// records": ~3.1M keys against ~89k roots means the flat per-field keys
// (file:{root}/{field}) dominate, which is expected.
// keyCountTTL bounds how stale CountKeys may be. Counting every row is a full
// table scan (~a second at millions of keys), and the only reader is a polled
// dashboard tile — it needs a number, not a fresh one every few seconds.
const keyCountTTL = 60 * time.Second
// CountKeys returns the number of rows in the database — every key, not just
// records. It is the denominator for "how much of the store is records": the
// flat per-field keys (file:{root}/{field}) dominate, which is expected.
func (c *Client) CountKeys() (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
c.keysMu.Lock()
defer c.keysMu.Unlock()
if !c.keysAt.IsZero() && time.Since(c.keysAt) < keyCountTTL {
return c.keysCount, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.DBSize(ctx).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("dbsize failed: %w", err)
return 0, err
}
ctx, cancel := opCtx()
defer cancel()
n, err := s.CountAll(ctx)
if err != nil {
return 0, err
}
c.keysCount, c.keysAt = n, time.Now()
return n, nil
}
internal/storage/udl.go
+238 -277
View File
@@ -12,28 +12,31 @@ package storage
// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
// op-log merge that used to live in meta-watch's store.rs).
//
// Redis model (all under the literal "udl:" prefix, mirroring the "file:" one):
// Key model (all under the literal "udl:" prefix, mirroring the "file:" one):
//
// udl:rec:<uid>/<cid>/<key> HASH { record, version, ts, value? }
// udl:idx:user:<uid>:key:<key> SET of <cid> -> active_for_user_key (My List, Continue Watching)
// udl:idx:user:<uid>:cid:<cid> SET of <key> -> list_for_user_cid
// udl:idx:user:<uid>:cids SET of <cid> -> whole-profile export (recovery)
// udl:idx:cid:<cid>:key:<key> SET of <uid> -> aggregate
// udl:idx:cid:<cid>:users SET of <uid> -> uids_for_cid
// udl:rec:<uid>/<cid>/<key> JSON { record, version, ts, value? }
// udl:idx:user:<uid>:key:<key> set of <cid> -> active_for_user_key (My List, Continue Watching)
// udl:idx:user:<uid>:cid:<cid> set of <key> -> list_for_user_cid
// udl:idx:user:<uid>:cids set of <cid> -> whole-profile export (recovery)
// udl:idx:cid:<cid>:key:<key> set of <uid> -> aggregate
// udl:idx:cid:<cid>:users set of <uid> -> uids_for_cid
//
// The `value` hash field is the plaintext JSON scalar for PUBLIC keys only
// A "set" is a key-set: one key `<set>\x1f<member>` = "" per member, read as
// the key range under `<set>\x1f`. The separator is a control character
// because UDL keys are not validated — with `/` the members of
// `…:key:a/b` would show up as members of `…:key:a`.
//
// The `value` field is the plaintext JSON scalar for PUBLIC keys only
// (like/rating), used for aggregation; it is absent for private-tier keys
// (whose real value rides encrypted inside `record`) and for tombstones.
import (
"context"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// UDLCell is a decoded (uid, cid, key) record cell.
@@ -67,7 +70,7 @@ func udlRecKey(uid, cid, key string) string {
func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
func udlIdxCidKey(cid, key string) string { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
// index that makes "export this whole profile" a set read instead of a
@@ -81,161 +84,196 @@ func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s
// already made `/api/udl/users/stats` admin-only.
func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
// udlUpsertScript performs the version-gated upsert + index maintenance
// atomically. Accepts (returns 1) only when the incoming version strictly
// exceeds the stored one; otherwise returns 0 (stale).
//
// KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
// 6=idx:user:cids
// ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
// 6=cid 7=key 8=uid 9=tombstone("1"/"0")
//
// A tombstone still writes the cell — that is the CRDT state, and dropping it
// would let an older write resurrect the value — but it SREMs the cid from the
// user+key index. That index is what `active_for_user_key` returns, so leaving
// deleted cids in it means every My List / Continue Watching read carries every
// title ever un-liked or finished, forever. The other four indexes are left
// alone: they answer "who has an opinion about this cid", where a tombstone is
// still an answer.
//
// ⚠ `idx:user:cids` (KEYS[6]) in particular must NEVER be pruned on a
// tombstone, even though it looks symmetric with KEYS[2]. It is the export
// index: a cid dropped from it is a cid whose tombstone never travels, and a
// tombstone that does not travel is exactly how an un-liked title comes back
// from the dead on the other box — the peer still holds the older `like` at a
// lower version, and with no tombstone to beat it, nothing ever supersedes it.
var udlUpsertScript = redis.NewScript(`
local stored = redis.call('HGET', KEYS[1], 'version')
if stored and tonumber(stored) >= tonumber(ARGV[1]) then
return 0
end
redis.call('HSET', KEYS[1], 'version', ARGV[1], 'ts', ARGV[2], 'record', ARGV[3])
if ARGV[5] == '1' then
redis.call('HSET', KEYS[1], 'value', ARGV[4])
else
redis.call('HDEL', KEYS[1], 'value')
end
if ARGV[9] == '1' then
redis.call('SREM', KEYS[2], ARGV[6])
else
redis.call('SADD', KEYS[2], ARGV[6])
end
redis.call('SADD', KEYS[3], ARGV[7])
redis.call('SADD', KEYS[4], ARGV[8])
redis.call('SADD', KEYS[5], ARGV[8])
redis.call('SADD', KEYS[6], ARGV[6])
return 1
`)
// udlSetSep separates a set key from a member (see the key model above).
const udlSetSep = "\x1f"
// udlCellJSON is a cell's stored form. Value is a pointer so an absent value
// (private-tier key, tombstone) stays distinguishable from an empty one.
type udlCellJSON struct {
Record string `json:"record"`
Version int64 `json:"version"`
Ts int64 `json:"ts"`
Value *string `json:"value,omitempty"`
}
func decodeUDLCell(raw string) (UDLCell, error) {
var j udlCellJSON
if err := json.Unmarshal([]byte(raw), &j); err != nil {
return UDLCell{}, err
}
cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts}
if j.Value != nil {
cell.Value, cell.HasValue = *j.Value, true
}
return cell, nil
}
// udlMembers lists a key-set's members, sorted.
func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) {
prefix := set + udlSetSep
rows, err := r.Range(ctx, prefix, "", 0)
if err != nil {
return nil, err
}
out := make([]string, 0, len(rows))
for _, row := range rows {
out = append(out, row.K[len(prefix):])
}
return out, nil
}
func udlAdd(tx *kv.Tx, set, member string) error {
_, err := tx.PutIfAbsent(set+udlSetSep+member, "")
return err
}
func udlRem(tx *kv.Tx, set, member string) error {
_, err := tx.Delete(set + udlSetSep + member)
return err
}
// readUDLCell reads one cell. exists is false when it was never written.
func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) {
raw, ok, err := r.Get(ctx, recKey)
if err != nil || !ok {
return UDLCell{}, false, err
}
cell, err := decodeUDLCell(raw)
if err != nil {
return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err)
}
return cell, true, nil
}
// readUDLCells reads many cells in one multi-key read; absent cells are
// omitted from the map.
func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) {
raws, err := r.MGet(ctx, recKeys)
if err != nil {
return nil, err
}
out := make(map[string]UDLCell, len(raws))
for k, raw := range raws {
cell, err := decodeUDLCell(raw)
if err != nil {
return nil, fmt.Errorf("decode %s: %w", k, err)
}
out[k] = cell
}
return out, nil
}
// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
// one. publicValue/hasValue carry the plaintext JSON scalar for public keys
// (used by aggregation); pass hasValue=false for private-tier keys and
// tombstones. Set tombstone=true when the record deletes the cell, which also
// prunes the cid from the user+key index. Returns accepted=false when the write
// is stale.
// one, and maintains the indexes — one transaction, so the gate and the write
// cannot interleave with a concurrent upsert of the same cell.
// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by
// aggregation); pass hasValue=false for private-tier keys and tombstones. Set
// tombstone=true when the record deletes the cell.
//
// A tombstone still writes the cell — that is the CRDT state, and dropping it
// would let an older write resurrect the value — but it removes the cid from
// the user+key index. That index is what `active_for_user_key` returns, so
// leaving deleted cids in it means every My List / Continue Watching read
// carries every title ever un-liked or finished, forever. The other four
// indexes are left alone: they answer "who has an opinion about this cid",
// where a tombstone is still an answer.
//
// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even
// though it looks symmetric with the user+key index. It is the export index: a
// cid dropped from it is a cid whose tombstone never travels, and a tombstone
// that does not travel is exactly how an un-liked title comes back from the
// dead on the other box — the peer still holds the older `like` at a lower
// version, and with no tombstone to beat it, nothing ever supersedes it.
//
// Returns accepted=false when the write is stale.
func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
keys := []string{
c.buildKey(udlRecKey(uid, cid, key)),
c.buildKey(udlIdxUserKey(uid, key)),
c.buildKey(udlIdxUserCid(uid, cid)),
c.buildKey(udlIdxCidKey(cid, key)),
c.buildKey(udlIdxCidUsers(cid)),
c.buildKey(udlIdxUserCids(uid)),
}
hv := "0"
if hasValue {
hv = "1"
}
tomb := "0"
if tombstone {
tomb = "1"
}
res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
accepted := false
err := c.update(func(tx *kv.Tx) error {
recKey := udlRecKey(uid, cid, key)
stored, ok, err := readUDLCell(tx.Context(), tx, recKey)
if err != nil {
return err
}
if ok && stored.Version >= version {
return nil
}
cell := udlCellJSON{Record: record, Version: version, Ts: ts}
if hasValue {
v := publicValue
cell.Value = &v
}
body, err := json.Marshal(cell)
if err != nil {
return err
}
if err := tx.Put(recKey, string(body)); err != nil {
return err
}
if tombstone {
err = udlRem(tx, udlIdxUserKey(uid, key), cid)
} else {
err = udlAdd(tx, udlIdxUserKey(uid, key), cid)
}
if err != nil {
return err
}
for _, add := range [][2]string{
{udlIdxUserCid(uid, cid), key},
{udlIdxCidKey(cid, key), uid},
{udlIdxCidUsers(cid), uid},
{udlIdxUserCids(uid), cid},
} {
if err := udlAdd(tx, add[0], add[1]); err != nil {
return err
}
}
accepted = true
return nil
})
if err != nil {
return false, fmt.Errorf("udl upsert failed: %w", err)
}
n, _ := res.(int64)
return n == 1, nil
return accepted, nil
}
// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
// no record has ever been written for the cell.
func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return UDLCell{}, false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.udlReadCell(ctx, udlRecKey(uid, cid, key))
}
// udlReadCell HMGETs a cell hash. Caller holds the lock. exists is false when
// the version field is absent (i.e. the hash does not exist).
func (c *Client) udlReadCell(ctx context.Context, recKey string) (UDLCell, bool, error) {
vals, err := c.client.HMGet(ctx, c.buildKey(recKey), "record", "version", "ts", "value").Result()
s, err := c.kv()
if err != nil {
return UDLCell{}, false, fmt.Errorf("hmget failed: %w", err)
return UDLCell{}, false, err
}
// vals: [record, version, ts, value] — nil for absent fields.
if len(vals) < 4 || vals[1] == nil {
return UDLCell{}, false, nil
}
cell := UDLCell{}
if s, ok := vals[0].(string); ok {
cell.Record = s
}
if s, ok := vals[1].(string); ok {
cell.Version, _ = strconv.ParseInt(s, 10, 64)
}
if s, ok := vals[2].(string); ok {
cell.Ts, _ = strconv.ParseInt(s, 10, 64)
}
if s, ok := vals[3].(string); ok {
cell.Value = s
cell.HasValue = true
}
return cell, true, nil
ctx, cancel := opCtx()
defer cancel()
return readUDLCell(ctx, s, udlRecKey(uid, cid, key))
}
// UDLListUserKey returns every cell for (uid, key) across all cids — the
// projection behind active_for_user_key (My List, Continue Watching). Includes
// tombstones; the caller decodes each record and filters.
func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserKey(uid, key))).Result()
cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(cids))
for i, cid := range cids {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]UDLUserKeyEntry, 0, len(cids))
for _, cid := range cids {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
if err != nil {
return nil, err
}
for i, cid := range cids {
cell, ok := cells[recKeys[i]]
if !ok {
continue
}
@@ -247,26 +285,28 @@ func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
// UDLListUserCid returns every cell for (uid, cid) across all keys — the
// projection behind list_for_user_cid.
func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(keys))
for i, key := range keys {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]UDLUserCidEntry, 0, len(keys))
for _, key := range keys {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
if err != nil {
return nil, err
}
for i, key := range keys {
cell, ok := cells[recKeys[i]]
if !ok {
continue
}
@@ -286,33 +326,28 @@ type UDLUserAllEntry struct {
// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
//
// Ordered by cid (sorted, so paging is stable across calls even though the
// underlying index is a Redis SET), starting strictly after `afterCid`. A cid's
// keys are never split across a page: the page ends on a cid boundary once
// `limit` rows have been emitted, and `nextCid` is the last cid included — pass
// it back as `afterCid` to continue. `nextCid` is empty when the export is
// complete.
// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never
// split across a page: the page ends on a cid boundary once `limit` rows have
// been emitted, and `nextCid` is the last cid included — pass it back as
// `afterCid` to continue. `nextCid` is empty when the export is complete.
//
// Tombstones are included. They are the CRDT state that stops a deleted value
// resurrecting on the box that receives this export, so omitting them would
// make recovery actively wrong rather than merely incomplete.
func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, "", fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, "", err
}
if limit <= 0 {
limit = 1000
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := opCtx()
defer cancel()
cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
cids, err := udlMembers(ctx, s, udlIdxUserCids(uid))
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
return nil, "", err
}
sort.Strings(cids)
@@ -326,13 +361,12 @@ func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEnt
nextCid = out[len(out)-1].Cid
break
}
keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
return nil, "", err
}
sort.Strings(keys)
for _, key := range keys {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key))
if err != nil {
return nil, "", err
}
@@ -349,133 +383,60 @@ func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEnt
// UDLCountUserCids is the number of distinct cids a uid holds records for.
//
// O(1) (a Redis SCARD), which is what makes it usable as a per-push quota
// One index range count, which is what makes it usable as a per-push quota
// check on the gateway mirror — a quota that cost a keyspace walk would be a
// denial-of-service in its own right. Counts cids rather than cells because
// that is what the index holds; a profile is bounded by the titles it has
// touched, not by how many keys it has per title.
func (c *Client) UDLCountUserCids(uid string) (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.SCard(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("scard failed: %w", err)
return 0, err
}
return n, nil
}
// UDLBackfillUserCids rebuilds the `idx:user:<uid>:cids` sets for data written
// before that index existed, and reports how many cids it added.
//
// One-shot: the upsert script maintains the set from now on. It is a SCAN, but
// a targeted one (`udl:idx:user:*:cid:*`, which is one key per (uid,cid), not
// per record) run once at boot rather than per read — the distinction that
// makes the difference between a migration and the pathology this index exists
// to avoid.
func (c *Client) UDLBackfillUserCids() (int, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
ctx, cancel := opCtx()
defer cancel()
pattern := c.buildKey("udl:idx:user:*:cid:*")
prefix := c.buildKey("udl:idx:user:")
added := 0
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 500).Result()
if err != nil {
return added, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
// <prefix><uid>:cid:<cid> — split on the FIRST separator, not the
// last. A uid is multibase base58btc, whose alphabet contains no
// colon at all, so the first ":cid:" always ends the uid. A cid
// that happens to contain ":cid:" in its own name would make the
// last occurrence split in the wrong place.
rest := strings.TrimPrefix(k, prefix)
i := strings.Index(rest, ":cid:")
if i < 0 {
continue
}
uid, cid := rest[:i], rest[i+len(":cid:"):]
if uid == "" || cid == "" {
continue
}
n, err := c.client.SAdd(ctx, c.buildKey(udlIdxUserCids(uid)), cid).Result()
if err != nil {
return added, fmt.Errorf("sadd failed: %w", err)
}
added += int(n)
}
cursor = next
if cursor == 0 {
break
}
}
return added, nil
return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep)
}
// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
func (c *Client) UDLCidUsers(cid string) ([]string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidUsers(cid))).Result()
s, err := c.kv()
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
return uids, nil
ctx, cancel := opCtx()
defer cancel()
return udlMembers(ctx, s, udlIdxCidUsers(cid))
}
// UDLPublicValues returns the live plaintext public JSON scalar values for
// (cid, key) across all uids — the input to aggregation. Cells without a value
// field (private-tier or tombstoned) are skipped.
// (private-tier or tombstoned) are skipped.
func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidKey(cid, key))).Result()
uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(uids))
for i, uid := range uids {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]string, 0, len(uids))
for _, uid := range uids {
v, err := c.client.HGet(ctx, c.buildKey(udlRecKey(uid, cid, key)), "value").Result()
if err == redis.Nil {
continue // private-tier or tombstoned
for _, k := range recKeys {
if cell, ok := cells[k]; ok && cell.HasValue {
out = append(out, cell.Value)
}
if err != nil {
return nil, fmt.Errorf("hget failed: %w", err)
}
out = append(out, v)
}
return out, nil
}
internal/storage/udl_admin.go
+76 -144
View File
@@ -3,22 +3,16 @@ package storage
// Account-scoped UDL administration: what a uid holds, and removing all of it.
//
// Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
// cell — because there is no per-user cid index to consult. The Redis model
// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
// "what is in My List", not "everything this account ever wrote". Adding a
// third index would put a write on every UDL upsert to serve two admin calls,
// so these SCAN instead and accept being O(keyspace).
//
// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
// one: the accounts page needs a number for every account, and MATCH filters
// server-side without indexing, so N per-uid calls would walk the keyspace N
// times. One walk buckets them all.
// cell. Keys are ordered, so one account's cells are the key range under
// `udl:rec:<uid>/`, and every account's are the range under `udl:rec:`.
// UDLAllUserStats stays a bulk call (one walk buckets every account) because
// the accounts page needs a number for each.
import (
"context"
"fmt"
"strings"
"time"
"github.com/metazla/meta-core/internal/kv"
)
// UDLUserStats counts what one account holds. Records is the number of cells,
@@ -30,11 +24,8 @@ type UDLUserStats struct {
Keys int `json:"keys"`
}
// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
const udlScanBatch = 1000
// safeUIDForPattern rejects anything that would change the meaning of a SCAN
// glob or reach across the key structure. meta-core's uids are alphanumeric
// safeUIDForPattern rejects anything that would reach across the key
// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric
// (identity.validUID), so this only ever fires on a hostile or malformed one —
// but it fires here, in the layer that builds the pattern, rather than relying
// on every caller having validated first.
@@ -62,118 +53,76 @@ func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
return parts[0], parts[1], parts[2], true
}
// scanKeys walks the keyspace for pattern and returns every match. Caller holds
// the lock.
//
// SCAN can return the same key more than once across cursor iterations (it
// guarantees no misses, not no duplicates), so callers that count must
// de-duplicate — both callers below collect into maps or sets for that reason.
func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
var (
out []string
cursor uint64
)
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
if err != nil {
return nil, fmt.Errorf("scan %q: %w", pattern, err)
}
out = append(out, keys...)
cursor = next
if cursor == 0 {
return out, nil
}
}
}
// UDLUserCells returns every (cid, key) pair this account holds a cell for.
// De-duplicated, since SCAN may repeat a key.
// UDLUserCells returns every (cid, key) pair this account holds a cell for,
// keyed by the "cid/key" tail with the cid as value (the purge needs them
// grouped).
func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
// the cid, which the purge needs grouped.
if err := safeUIDForPattern(uid); err != nil {
return nil, err
}
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+uid+"/*")
s, err := c.kv()
if err != nil {
return nil, err
}
out := make(map[string]string, len(keys))
for _, full := range keys {
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
// The glob can only match this uid, but a uid that is a prefix of
// another would still be caught by the '/' in the pattern; check anyway
// so a malformed key can never be attributed to the wrong account.
if !ok || gotUID != uid {
continue
ctx, cancel := opCtx()
defer cancel()
out := make(map[string]string)
err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error {
for _, row := range rows {
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
if !ok || gotUID != uid {
continue
}
out[cid+"/"+key] = cid
}
out[cid+"/"+key] = cid
}
return out, nil
return nil
})
return out, err
}
// UDLAllUserStats counts every account's cells in one keyspace walk.
// UDLAllUserStats counts every account's cells in one walk.
//
// Returns only accounts that actually hold records; a uid absent from the map
// holds none. Callers render a zero for those rather than treating it as an
// error — an account created a minute ago legitimately has nothing.
func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+"*")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := opCtx()
defer cancel()
// Sets rather than counters: SCAN may hand back the same key twice, which
// would otherwise inflate every number on the page.
type acc struct {
cells map[string]struct{}
cells int
cids map[string]struct{}
keys map[string]struct{}
}
byUID := make(map[string]*acc)
for _, full := range keys {
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
if !ok {
continue
}
a := byUID[uid]
if a == nil {
a = &acc{
cells: make(map[string]struct{}),
cids: make(map[string]struct{}),
keys: make(map[string]struct{}),
err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error {
for _, row := range rows {
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
if !ok {
continue
}
byUID[uid] = a
a := byUID[uid]
if a == nil {
a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})}
byUID[uid] = a
}
a.cells++
a.cids[cid] = struct{}{}
a.keys[key] = struct{}{}
}
a.cells[cid+"/"+key] = struct{}{}
a.cids[cid] = struct{}{}
a.keys[key] = struct{}{}
return nil
})
if err != nil {
return nil, err
}
out := make(map[string]UDLUserStats, len(byUID))
for uid, a := range byUID {
out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)}
}
return out, nil
}
@@ -185,63 +134,46 @@ func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
// The shared indexes are the part that is easy to miss and expensive to leave.
// `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
// not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
// uid listed as an opinion-holder on every title it ever touched. Aggregation
// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
// rating average stays right, but uids_for_cid keeps naming a deleted account
// forever and every aggregate read pays for the lookup.
// uid listed as an opinion-holder on every title it ever touched.
//
// Idempotent: purging an account with nothing stored is a no-op returning
// zeroes, so a retry after a partial failure is safe.
// zeroes, so a retry is safe.
func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
cells, err := c.UDLUserCells(uid)
if err != nil {
return UDLUserStats{}, err
}
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return UDLUserStats{}, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
stats := UDLUserStats{Records: len(cells)}
distinctCids := make(map[string]struct{})
distinctKeys := make(map[string]struct{})
pipe := c.client.Pipeline()
for cidKey, cid := range cells {
key := strings.TrimPrefix(cidKey, cid+"/")
distinctCids[cid] = struct{}{}
distinctKeys[key] = struct{}{}
pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
}
for cid := range distinctCids {
pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
err = c.update(func(tx *kv.Tx) error {
for cidKey, cid := range cells {
key := strings.TrimPrefix(cidKey, cid+"/")
distinctCids[cid] = struct{}{}
distinctKeys[key] = struct{}{}
if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil {
return err
}
if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil {
return err
}
}
for cid := range distinctCids {
if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil {
return err
}
}
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one
// key range, dropped whole — which also catches index entries whose
// cell is already gone (the state a half-finished purge leaves).
_, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0)
return err
})
if err != nil {
return UDLUserStats{}, fmt.Errorf("purge: %w", err)
}
stats.Cids = len(distinctCids)
stats.Keys = len(distinctKeys)
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are enumerable
// directly, and are dropped whole rather than member-by-member. The glob is
// what keeps this correct as indexes are added — `:cids` needed no change
// here when it landed. Scanning them separately also catches
// index entries whose cell is already gone — exactly the state a previously
// half-finished purge leaves behind.
idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
if err != nil {
return UDLUserStats{}, err
}
for _, k := range idxKeys {
pipe.Del(ctx, k)
}
if _, err := pipe.Exec(ctx); err != nil {
return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
}
return stats, nil
}
internal/storage/udl_admin_test.go
+8 -28
View File
@@ -20,7 +20,7 @@ func seedCell(t *testing.T, c *Client, uid, cid, key string) {
// forever — and the rows are unreachable through any API, so nothing will ever
// clean them up.
func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -63,7 +63,7 @@ func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
// both have an opinion about — where the two share an index set and a careless
// DEL of the whole key would take the survivor's membership with it.
func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "shared", "rating")
seedCell(t, c, "zBob", "shared", "rating")
@@ -92,7 +92,7 @@ func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
// The key layout separates them with '/' and ':', so this passes only as long
// as the scan patterns keep those separators.
func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAliceExtra", "cidOne", "like")
@@ -114,7 +114,7 @@ func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
// index instead of the cells would leave every retracted rating behind — the
// rows most likely to have accumulated over an account's life.
func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
@@ -140,7 +140,7 @@ func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
// Retry-safe: the delete handler purges before removing the key, so a caller
// that retries after a partial failure runs this a second time.
func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
@@ -156,7 +156,7 @@ func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
}
func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
// "*" would otherwise match every account in the keyspace — a wildcard
@@ -172,7 +172,7 @@ func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
}
func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -198,7 +198,7 @@ func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
// The stats and the purge must agree: the number shown in the delete
// confirmation is the number of rows the delete then removes.
func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidTwo", "rating")
@@ -217,26 +217,6 @@ func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
}
}
// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or
// every account on a prefixed node would be reported under a mangled name and
// the purge would find nothing.
func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) {
c, _ := newTestClient(t, "tenantA:")
seedCell(t, c, "zAlice", "cidOne", "like")
stats, err := c.UDLAllUserStats()
if err != nil {
t.Fatalf("stats: %v", err)
}
if got := stats["zAlice"]; got.Records != 1 {
t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats)
}
if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 {
t.Fatalf("purge under a prefix: %+v err=%v", s, err)
}
}
func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
// A namespaced key arriving later must not be truncated: the purge SREMs
// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong
internal/storage/udl_export_test.go
+4 -78
View File
@@ -1,7 +1,6 @@
package storage
import (
"context"
"testing"
)
@@ -11,7 +10,7 @@ import (
// that one caller.
func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const uid = "zAlice"
writes := []struct{ cid, key, rec string }{
@@ -52,7 +51,7 @@ func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
// indexed exports as empty, and recovery reports success having restored
// nothing.
func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
@@ -74,7 +73,7 @@ func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
// saw the `like`. This is the one place the cids index deliberately diverges
// from the user+key index, which does prune.
func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const (
uid = "zAlice"
@@ -111,7 +110,7 @@ func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
}
func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const uid = "zAlice"
cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
@@ -159,76 +158,3 @@ func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
seen[k] = true
}
}
// Data written before the cids index existed must still export, or every
// profile that predates this feature silently recovers as empty.
func TestUDLBackfillUserCids_MakesLegacyDataExportable(t *testing.T) {
c, _ := newTestClient(t, "")
const (
uid = "zAlice"
cid = "bagcsaaaaone"
key = "like"
)
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
// Simulate the pre-index state: the cell and the other indexes exist, the
// cids set does not.
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if entries, _, err := c.UDLAllForUser(uid, "", 0); err != nil || len(entries) != 0 {
t.Fatalf("precondition: legacy data should export as empty, got %+v err=%v", entries, err)
}
added, err := c.UDLBackfillUserCids()
if err != nil {
t.Fatalf("backfill: %v", err)
}
if added != 1 {
t.Fatalf("want 1 cid backfilled, got %d", added)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export after backfill: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid || entries[0].Key != key {
t.Fatalf("backfilled data should export, got %+v", entries)
}
// Idempotent: a second run adds nothing.
if added, err := c.UDLBackfillUserCids(); err != nil || added != 0 {
t.Fatalf("second backfill should be a no-op, added=%d err=%v", added, err)
}
}
// The backfill splits `<uid>:cid:<cid>` on the FIRST separator, because a uid
// is multibase base58btc and its alphabet has no colon. Splitting on the last
// occurrence — the obvious-looking choice — mis-parses a cid that contains
// ":cid:" in its own name into uid "zAlice:cid:weird", cid "name".
func TestUDLBackfillUserCids_SplitsOnTheFirstSeparator(t *testing.T) {
c, _ := newTestClient(t, "")
const uid = "zAlice"
cid := "weird:cid:name"
if ok, err := c.UDLUpsertIfNewer(uid, cid, "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if _, err := c.UDLBackfillUserCids(); err != nil {
t.Fatalf("backfill: %v", err)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid {
t.Fatalf("want the cid parsed whole, got %+v", entries)
}
}
internal/storage/udl_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
// forever. The reader filters them out again, so the cost is invisible until
// the account is old.
func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const (
uid = "zAlice"
internal/storage/uuid.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
// layout). Used as the root key for file metadata entries.
//
// Why this format over canonical 8-4-4-4-12 hex:
// - 26 chars vs 36 — shorter Redis keys, less prefix space waste
// - 26 chars vs 36 — shorter keys, less prefix space waste
// - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
// given time window, restoring some of the debug-ability lost vs the
// previous midhash256:abc rooting
internal/storage/uuid_test.go
+5 -5
View File
@@ -60,11 +60,11 @@ func TestCIDFromKeysetField(t *testing.T) {
{"cids/bafkmidhash", "bafkmidhash"},
{"cids/baejbeisha256", "baejbeisha256"},
{"cids/bafyipfs", "bafyipfs"},
{"title", ""}, // non-CID field
{"tmdb/title", ""}, // non-CID nested
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
{"midhash256", ""}, // legacy named field — no longer recognized
{"cids", ""}, // prefix without a member
{"title", ""}, // non-CID field
{"tmdb/title", ""}, // non-CID nested
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
{"midhash256", ""}, // legacy named field — no longer recognized
{"cids", ""}, // prefix without a member
}
for _, tc := range cases {
if got := cidFromKeysetField(tc.field); got != tc.want {
internal/storage/write_bench_test.go
+59
View File
@@ -0,0 +1,59 @@
package storage
import (
"fmt"
"path/filepath"
"sync"
"sync/atomic"
"testing"
)
// BenchmarkConcurrentRecordWrites is the ingest shape the single-writer model
// has to absorb: many goroutines (feeder flushes, gateway persists, plugin
// callbacks) each writing a ~40-field record. SQLite serialises them; this
// measures what that costs. Reported as records/s.
func BenchmarkConcurrentRecordWrites(b *testing.B) {
c := NewClient()
if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
b.Fatal(err)
}
defer c.Close()
record := func(i int64) map[string]string {
m := map[string]string{
"title": fmt.Sprintf("Some Show S01E%02d", i%40),
"fileName": fmt.Sprintf("some.show.s01e%02d.1080p.mkv", i%40),
"filePath": fmt.Sprintf("/files/watch/some.show.%d.mkv", i),
"fileType": "video",
"tmdbid": fmt.Sprintf("%d", 1000+i%500),
fmt.Sprintf("cids/bafkbench%020d", i): "true",
}
for f := 0; len(m) < 40; f++ {
m[fmt.Sprintf("categories/%d", f)] = "true"
}
return m
}
const writers = 16
var next int64
b.ResetTimer()
var wg sync.WaitGroup
for w := 0; w < writers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
i := atomic.AddInt64(&next, 1)
if i > int64(b.N) {
return
}
if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), record(i)); err != nil {
b.Error(err)
return
}
}
}()
}
wg.Wait()
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
}