replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s

This commit is contained in:
Worph
2026-10-19 20:51:00 +00:00
parent c53146e514
commit dc5cbdfb9b
80 changed files with 4601 additions and 6077 deletions
+172 -284
View File
@@ -5,10 +5,9 @@ import (
"fmt"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/cid"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// This file implements the CID-resolution layer that lives on top of the
@@ -18,7 +17,7 @@ import (
// The UUID is an internal implementation detail; the mental model is
// "N CIDs → one record of keys→metadata".
// - Every known CID for a file is registered as a reverse-index entry:
// cid:<bareCid> → <uuid> (Redis STRING)
// cid:<bareCid> → <uuid>
// and as a member of the per-root bare-CID key-set:
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
@@ -27,14 +26,13 @@ import (
// *derived by rank* from the key-set on read (GetMetadataDocument), never
// stored.
//
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
// any write of a cids/<cid> field and register the reverse-index entry, so
// The record write path (writeFieldsTx) catches any write of a cids/<cid>
// field and registers the reverse-index entry in the same transaction, so
// writers that just emit the key-set members don't need to know about the
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
// already held by the caller, avoiding nested RLock (which can deadlock
// against a waiting writer).
// reverse index at all. The helpers here take the caller's *kv.Tx: a write
// must never open a second transaction (there is one writer connection).
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
// CIDIndexPrefix is the key prefix for the reverse index. Combined
// with a bare CID, it forms the full key: "cid:bafk…".
const CIDIndexPrefix = "cid:"
@@ -45,152 +43,112 @@ const CIDIndexPrefix = "cid:"
// reverse-index entries that need cleanup).
const CIDsKeyPrefix = "cids/"
// DuplicatesField is the per-root SET key holding additional file paths
// whose content hashes to the same CIDs as this root. The watcher writes
// here when it sees a new path for content that's already indexed; meta-dup
// reads from it.
const DuplicatesField = "duplicates"
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
// cidStr is a bare multibase CIDv1 ("bafk…").
func (c *Client) buildCIDIndexKey(cidStr string) string {
return c.buildKey(CIDIndexPrefix + cidStr)
}
// Duplicate paths — other physical locations whose content hashes to the same
// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
// writes one when it sees a new path for content that's already indexed.
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
// in the file index. Returns the new UUID. The caller is expected to follow
// in the root index. Returns the new UUID. The caller is expected to follow
// up with AddAlias for at least one CID — a root without any alias is
// reachable only via the index, which works but defeats the point.
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
var uuid string
err := c.update(func(tx *kv.Tx) error {
var err error
uuid, err = mintTx(tx, filePath, size, mtimeNano)
return err
})
return uuid, err
}
func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
uuid, err := NewUUID()
if err != nil {
return "", fmt.Errorf("uuid: %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
pipe := c.client.Pipeline()
pipe.Set(ctx, prefix+"filePath", filePath, 0)
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
if _, err := pipe.Exec(ctx); err != nil {
return "", fmt.Errorf("mint pipeline: %w", err)
}
return uuid, nil
return uuid, writeFieldsTx(tx, uuid, map[string]string{
"filePath": filePath,
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
})
}
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
// Idempotent: registering the same CID twice is a no-op.
func (c *Client) AddAlias(uuid, cidStr string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.addAliasLocked(ctx, uuid, cidStr)
}
// addAliasLocked is the lock-free body of AddAlias. The caller must already
// hold c.mu (read or write). Split out so the alias-maintenance hooks
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
// the work without re-entering the lock — nested RLock can deadlock against
// a waiting writer.
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
return c.update(func(tx *kv.Tx) error {
return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
})
}
keysetField := CIDsKeyPrefix + cidStr
keysetKey := c.buildKeyPrefix(uuid) + keysetField
// Every branch below writes the key-set member, so index the field name
// once here — including the two guard branches that skip the alias itself.
// Missing it would leave `cids/<cid>` invisible to a record read.
defer c.addFieldLocked(ctx, uuid, keysetField)
// addAliasTx registers the reverse-index entry for a cids/<cid> member the
// caller is writing (writeFieldsTx writes the member itself).
func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
// Self-pointing alias guard. If the caller is trying to register
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
// dual-root failure mode in disguise — it means somebody wrote
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
// write; leave any existing alias alone. Still record the key-set
// membership so the field-level write succeeds.
// write; leave any existing alias alone. The key-set membership is still
// recorded by the caller, so the field-level write succeeds.
if uuid == cidStr {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
return nil
}
// Guard against the dual-root pattern: if cid:<cid> already points at a
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
// just record the key-set membership so it's still discoverable via the
// root → cids enumeration.
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != "" && existing != uuid {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
}
pipe := c.client.Pipeline()
pipe.Set(ctx, indexKey, uuid, 0)
pipe.Set(ctx, keysetKey, "true", 0)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("add alias pipeline: %w", err)
}
return nil
}
// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller must already hold c.mu, and must delete
// the cids/<cid> key-set member itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat
// file:<uuid>/cids/<cid>, the __fields__ index entry, and this reverse alias —
// while the delete path only ever removed the first two. A "deleted" CID
// therefore still resolved: GET /meta/<cid>, ResolveRoot and GetByCID all kept
// answering with the record. The asymmetry is not theoretical. meta-share
// published dag-pb roots computed over RAR volumes and par2 files onto six
// records on watch.nsl.sh; removing the member was expressible, removing the
// claim was not, so the false identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasLocked, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err == redis.Nil {
return nil
}
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
// key-set membership keeps the CID discoverable via root → cids.
indexKey := CIDIndexPrefix + cidStr
existing, _, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != uuid {
// Somebody else owns this alias. Leave it.
if existing != "" && existing != uuid {
return nil
}
return c.client.Del(ctx, indexKey).Err()
if existing == uuid {
return nil
}
return tx.Put(indexKey, uuid)
}
// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
// itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches two keys — the flat
// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
// only removed the first. A "deleted" CID therefore still resolved: GET
// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
// The asymmetry is not theoretical. meta-share published dag-pb roots computed
// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
// the member was expressible, removing the claim was not, so the false
// identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasTx, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := CIDIndexPrefix + cidStr
existing, ok, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if !ok || existing != uuid {
// Absent, or somebody else owns this alias. Leave it.
return nil
}
_, err = tx.Delete(indexKey)
return err
}
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
@@ -210,8 +168,8 @@ func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) err
// - hash matches a reverse-index entry → return UUID
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
//
// Errors are converted to "no resolution" — if Redis is down or the lookup
// fails we let the caller proceed with the bare hash and surface the Redis
// Errors are converted to "no resolution" — if the store is down or the
// lookup fails we let the caller proceed with the bare hash and surface the
// failure on the actual operation.
func (c *Client) ResolveRoot(hash string) string {
if hash == "" {
@@ -228,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string {
// when the CID is unknown; the empty string is the "not found" signal rather
// than a typed error so callers can branch without errors.As.
func (c *Client) GetByCID(cidStr string) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return "", err
}
if cidStr == "" {
return "", nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
ctx, cancel := opCtx()
defer cancel()
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
if err == redis.Nil {
return "", nil
}
uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
if err != nil {
return "", fmt.Errorf("get cid index: %w", err)
}
@@ -252,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
}
// AddDuplicatePath records that path is another physical location whose
// content matches uuid. Returns true if path was newly added (not already
// in the set). The watcher calls this when it sees a file whose midhash
// matches an existing root but at a different filePath.
// content matches uuid. Returns true if path was newly added. The watcher
// calls this when it sees a file whose midhash matches an existing root but
// at a different filePath.
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
if uuid == "" || path == "" {
return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
if err != nil {
return false, fmt.Errorf("sadd duplicates: %w", err)
}
return added > 0, nil
added := false
err := c.update(func(tx *kv.Tx) error {
var err error
added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
return err
})
return added, err
}
// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
// prefix stripped). Caller must hold c.mu.
//
// Reads the field index and filters by prefix; only an un-indexed root falls
// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
// stripped).
func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
prefix := recordPrefix(uuid) + CIDsKeyPrefix
rows, err := r.Range(ctx, prefix, "", 0)
if err != nil {
return nil, err
}
out := make([]string, 0, len(rows))
for _, row := range rows {
out = append(out, row.K[len(prefix):])
}
return out, nil
}
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
// (including the cids/<cid> key-set), every reverse-index entry for those
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
// is a successful no-op.
// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
// with no remaining keys is a successful no-op.
func (c *Client) DeleteRoot(uuid string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
if uuid == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
// the list of reverse-index entries to clean up.
cids, err := c.cidsForRootLocked(ctx, uuid)
if err != nil {
return c.update(func(tx *kv.Tx) error {
_, err := deleteRecordTx(tx, uuid)
return err
}
// 2. Delete every reverse-index entry in one shot.
if len(cids) > 0 {
keys := make([]string, 0, len(cids))
for _, cidStr := range cids {
keys = append(keys, c.buildCIDIndexKey(cidStr))
}
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del cid index: %w", err)
}
}
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return fmt.Errorf("scan root keys: %w", err)
}
if len(keys) > 0 {
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del root keys: %w", err)
}
}
cursor = next
if cursor == 0 {
break
}
}
// 4. Remove from the file index set.
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
return fmt.Errorf("srem index: %w", err)
}
// Marked so the incremental search-index refresh reads it, finds it gone,
// and evicts it — otherwise a deleted root keeps answering searches until
// the next full rebuild.
c.markDirtyLocked(ctx, uuid)
return nil
}
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
// root. If it's a key-set member — field name starts with "cids/" — it
// registers the implied CID in the reverse index. Writers emit cids/<cid>
// members without knowing about the reverse index; this hook keeps it in
// sync transparently.
//
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
// (logged, not returned): an out-of-sync reverse index is recoverable via a
// sweep, while a failed property write is not.
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
cidStr := cidFromKeysetField(field)
if cidStr == "" {
return nil
}
return c.addAliasLocked(ctx, uuid, cidStr)
}
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
// over every entry in a metadata map. Used by SetMetadataFlat /
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
// hold c.mu.
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
for k := range metadata {
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
}
})
}
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
@@ -406,24 +275,23 @@ type MetadataDocument struct {
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
// never existed).
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
if uuid == "" {
return nil, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := opCtx()
defer cancel()
flat, err := c.getMetadataFlatInternal(ctx, uuid)
flat, err := readRecord(ctx, s, uuid)
if err != nil {
return nil, err
}
prefix := c.buildKeyPrefix(uuid)
if flat == nil {
flat = make(map[string]string)
}
// Hoist the cids/<cid> key-set members into a typed array and derive the
// canonical CID by rank. The members remain in Flat too — that's how
@@ -443,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
}
}
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers duplicates: %w", err)
dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
if err != nil {
return nil, fmt.Errorf("read duplicates: %w", err)
}
dups := make([]string, 0, len(dupRows))
for _, row := range dupRows {
dups = append(dups, row.K[len(dupPrefix(uuid)):])
}
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
@@ -474,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas
if midhash == "" {
return "", false, fmt.Errorf("midhash is required")
}
uuid, err := c.GetByCID(midhash)
if err != nil {
return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if uuid == "" {
uuid, err = c.Mint(absPath, size, mtimeNano)
var (
uuid string
minted bool
)
// One transaction: the lookup and the mint cannot interleave with another
// registration of the same content, so two concurrent scans of one file
// can no longer mint two roots for it.
err := c.update(func(tx *kv.Tx) error {
ctx := tx.Context()
existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
if err != nil {
return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
return fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
uuid = existing
if uuid == "" {
if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
return fmt.Errorf("Mint(%s): %w", absPath, err)
}
minted = true
if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
}
return nil
}
return uuid, true, nil
}
existing, _ := c.GetProperty(uuid, "filePath")
if existing != absPath {
if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
if err != nil {
return err
}
return uuid, false, nil
if current != absPath {
if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
}
return nil
}
if err := writeFieldsTx(tx, uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return nil
})
if err != nil {
return "", false, err
}
if _, err := c.MergeMetadataFlat(uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return uuid, false, nil
return uuid, minted, nil
}