replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
Build and Push Docker Image / build (push) Successful in 2m44s
This commit is contained in:
+172
-284
@@ -5,10 +5,9 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/cid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// This file implements the CID-resolution layer that lives on top of the
|
||||
@@ -18,7 +17,7 @@ import (
|
||||
// The UUID is an internal implementation detail; the mental model is
|
||||
// "N CIDs → one record of keys→metadata".
|
||||
// - Every known CID for a file is registered as a reverse-index entry:
|
||||
// cid:<bareCid> → <uuid> (Redis STRING)
|
||||
// cid:<bareCid> → <uuid>
|
||||
// and as a member of the per-root bare-CID key-set:
|
||||
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
|
||||
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
|
||||
@@ -27,14 +26,13 @@ import (
|
||||
// *derived by rank* from the key-set on read (GetMetadataDocument), never
|
||||
// stored.
|
||||
//
|
||||
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
|
||||
// any write of a cids/<cid> field and register the reverse-index entry, so
|
||||
// The record write path (writeFieldsTx) catches any write of a cids/<cid>
|
||||
// field and registers the reverse-index entry in the same transaction, so
|
||||
// writers that just emit the key-set members don't need to know about the
|
||||
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
|
||||
// already held by the caller, avoiding nested RLock (which can deadlock
|
||||
// against a waiting writer).
|
||||
// reverse index at all. The helpers here take the caller's *kv.Tx: a write
|
||||
// must never open a second transaction (there is one writer connection).
|
||||
|
||||
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
|
||||
// CIDIndexPrefix is the key prefix for the reverse index. Combined
|
||||
// with a bare CID, it forms the full key: "cid:bafk…".
|
||||
const CIDIndexPrefix = "cid:"
|
||||
|
||||
@@ -45,152 +43,112 @@ const CIDIndexPrefix = "cid:"
|
||||
// reverse-index entries that need cleanup).
|
||||
const CIDsKeyPrefix = "cids/"
|
||||
|
||||
// DuplicatesField is the per-root SET key holding additional file paths
|
||||
// whose content hashes to the same CIDs as this root. The watcher writes
|
||||
// here when it sees a new path for content that's already indexed; meta-dup
|
||||
// reads from it.
|
||||
const DuplicatesField = "duplicates"
|
||||
|
||||
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
|
||||
// cidStr is a bare multibase CIDv1 ("bafk…").
|
||||
func (c *Client) buildCIDIndexKey(cidStr string) string {
|
||||
return c.buildKey(CIDIndexPrefix + cidStr)
|
||||
}
|
||||
// Duplicate paths — other physical locations whose content hashes to the same
|
||||
// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
|
||||
// writes one when it sees a new path for content that's already indexed.
|
||||
|
||||
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
|
||||
// in the file index. Returns the new UUID. The caller is expected to follow
|
||||
// in the root index. Returns the new UUID. The caller is expected to follow
|
||||
// up with AddAlias for at least one CID — a root without any alias is
|
||||
// reachable only via the index, which works but defeats the point.
|
||||
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return "", fmt.Errorf("not connected")
|
||||
}
|
||||
var uuid string
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
uuid, err = mintTx(tx, filePath, size, mtimeNano)
|
||||
return err
|
||||
})
|
||||
return uuid, err
|
||||
}
|
||||
|
||||
func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
|
||||
uuid, err := NewUUID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uuid: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
prefix := c.buildKeyPrefix(uuid)
|
||||
pipe := c.client.Pipeline()
|
||||
pipe.Set(ctx, prefix+"filePath", filePath, 0)
|
||||
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
|
||||
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
|
||||
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
|
||||
c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return "", fmt.Errorf("mint pipeline: %w", err)
|
||||
}
|
||||
return uuid, nil
|
||||
return uuid, writeFieldsTx(tx, uuid, map[string]string{
|
||||
"filePath": filePath,
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
})
|
||||
}
|
||||
|
||||
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
|
||||
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
|
||||
// Idempotent: registering the same CID twice is a no-op.
|
||||
func (c *Client) AddAlias(uuid, cidStr string) error {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
return c.addAliasLocked(ctx, uuid, cidStr)
|
||||
}
|
||||
|
||||
// addAliasLocked is the lock-free body of AddAlias. The caller must already
|
||||
// hold c.mu (read or write). Split out so the alias-maintenance hooks
|
||||
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
|
||||
// the work without re-entering the lock — nested RLock can deadlock against
|
||||
// a waiting writer.
|
||||
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return fmt.Errorf("uuid and cid are required")
|
||||
}
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
|
||||
})
|
||||
}
|
||||
|
||||
keysetField := CIDsKeyPrefix + cidStr
|
||||
keysetKey := c.buildKeyPrefix(uuid) + keysetField
|
||||
|
||||
// Every branch below writes the key-set member, so index the field name
|
||||
// once here — including the two guard branches that skip the alias itself.
|
||||
// Missing it would leave `cids/<cid>` invisible to a record read.
|
||||
defer c.addFieldLocked(ctx, uuid, keysetField)
|
||||
|
||||
// addAliasTx registers the reverse-index entry for a cids/<cid> member the
|
||||
// caller is writing (writeFieldsTx writes the member itself).
|
||||
func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return fmt.Errorf("uuid and cid are required")
|
||||
}
|
||||
// Self-pointing alias guard. If the caller is trying to register
|
||||
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
|
||||
// dual-root failure mode in disguise — it means somebody wrote
|
||||
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
|
||||
// write; leave any existing alias alone. Still record the key-set
|
||||
// membership so the field-level write succeeds.
|
||||
// write; leave any existing alias alone. The key-set membership is still
|
||||
// recorded by the caller, so the field-level write succeeds.
|
||||
if uuid == cidStr {
|
||||
return c.client.Set(ctx, keysetKey, "true", 0).Err()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Guard against the dual-root pattern: if cid:<cid> already points at a
|
||||
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
|
||||
// just record the key-set membership so it's still discoverable via the
|
||||
// root → cids enumeration.
|
||||
indexKey := c.buildCIDIndexKey(cidStr)
|
||||
existing, err := c.client.Get(ctx, indexKey).Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
return fmt.Errorf("read existing alias: %w", err)
|
||||
}
|
||||
if existing != "" && existing != uuid {
|
||||
return c.client.Set(ctx, keysetKey, "true", 0).Err()
|
||||
}
|
||||
|
||||
pipe := c.client.Pipeline()
|
||||
pipe.Set(ctx, indexKey, uuid, 0)
|
||||
pipe.Set(ctx, keysetKey, "true", 0)
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("add alias pipeline: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index
|
||||
// entry cid:<cid> → <uuid>. The caller must already hold c.mu, and must delete
|
||||
// the cids/<cid> key-set member itself (DeleteProperty does both).
|
||||
//
|
||||
// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat
|
||||
// file:<uuid>/cids/<cid>, the __fields__ index entry, and this reverse alias —
|
||||
// while the delete path only ever removed the first two. A "deleted" CID
|
||||
// therefore still resolved: GET /meta/<cid>, ResolveRoot and GetByCID all kept
|
||||
// answering with the record. The asymmetry is not theoretical. meta-share
|
||||
// published dag-pb roots computed over RAR volumes and par2 files onto six
|
||||
// records on watch.nsl.sh; removing the member was expressible, removing the
|
||||
// claim was not, so the false identity survived its own retraction.
|
||||
//
|
||||
// Guarded, not blind: the alias is removed only when it currently points at
|
||||
// THIS uuid. Two records can legitimately race to claim a cid (see the
|
||||
// dual-root guard in addAliasLocked, which leaves the first writer's alias in
|
||||
// place); deleting a member from the loser must not unpick the winner's index.
|
||||
func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return nil
|
||||
}
|
||||
indexKey := c.buildCIDIndexKey(cidStr)
|
||||
existing, err := c.client.Get(ctx, indexKey).Result()
|
||||
if err == redis.Nil {
|
||||
return nil
|
||||
}
|
||||
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
|
||||
// key-set membership keeps the CID discoverable via root → cids.
|
||||
indexKey := CIDIndexPrefix + cidStr
|
||||
existing, _, err := tx.Get(tx.Context(), indexKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read existing alias: %w", err)
|
||||
}
|
||||
if existing != uuid {
|
||||
// Somebody else owns this alias. Leave it.
|
||||
if existing != "" && existing != uuid {
|
||||
return nil
|
||||
}
|
||||
return c.client.Del(ctx, indexKey).Err()
|
||||
if existing == uuid {
|
||||
return nil
|
||||
}
|
||||
return tx.Put(indexKey, uuid)
|
||||
}
|
||||
|
||||
// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
|
||||
// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
|
||||
// itself (DeleteProperty does both).
|
||||
//
|
||||
// ⚠ Why this exists. A cids/ member write touches two keys — the flat
|
||||
// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
|
||||
// only removed the first. A "deleted" CID therefore still resolved: GET
|
||||
// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
|
||||
// The asymmetry is not theoretical. meta-share published dag-pb roots computed
|
||||
// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
|
||||
// the member was expressible, removing the claim was not, so the false
|
||||
// identity survived its own retraction.
|
||||
//
|
||||
// Guarded, not blind: the alias is removed only when it currently points at
|
||||
// THIS uuid. Two records can legitimately race to claim a cid (see the
|
||||
// dual-root guard in addAliasTx, which leaves the first writer's alias in
|
||||
// place); deleting a member from the loser must not unpick the winner's index.
|
||||
func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
|
||||
if uuid == "" || cidStr == "" {
|
||||
return nil
|
||||
}
|
||||
indexKey := CIDIndexPrefix + cidStr
|
||||
existing, ok, err := tx.Get(tx.Context(), indexKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read existing alias: %w", err)
|
||||
}
|
||||
if !ok || existing != uuid {
|
||||
// Absent, or somebody else owns this alias. Leave it.
|
||||
return nil
|
||||
}
|
||||
_, err = tx.Delete(indexKey)
|
||||
return err
|
||||
}
|
||||
|
||||
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
|
||||
@@ -210,8 +168,8 @@ func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) err
|
||||
// - hash matches a reverse-index entry → return UUID
|
||||
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
|
||||
//
|
||||
// Errors are converted to "no resolution" — if Redis is down or the lookup
|
||||
// fails we let the caller proceed with the bare hash and surface the Redis
|
||||
// Errors are converted to "no resolution" — if the store is down or the
|
||||
// lookup fails we let the caller proceed with the bare hash and surface the
|
||||
// failure on the actual operation.
|
||||
func (c *Client) ResolveRoot(hash string) string {
|
||||
if hash == "" {
|
||||
@@ -228,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string {
|
||||
// when the CID is unknown; the empty string is the "not found" signal rather
|
||||
// than a typed error so callers can branch without errors.As.
|
||||
func (c *Client) GetByCID(cidStr string) (string, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return "", fmt.Errorf("not connected")
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if cidStr == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get cid index: %w", err)
|
||||
}
|
||||
@@ -252,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
|
||||
}
|
||||
|
||||
// AddDuplicatePath records that path is another physical location whose
|
||||
// content matches uuid. Returns true if path was newly added (not already
|
||||
// in the set). The watcher calls this when it sees a file whose midhash
|
||||
// matches an existing root but at a different filePath.
|
||||
// content matches uuid. Returns true if path was newly added. The watcher
|
||||
// calls this when it sees a file whose midhash matches an existing root but
|
||||
// at a different filePath.
|
||||
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return false, fmt.Errorf("not connected")
|
||||
}
|
||||
if uuid == "" || path == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sadd duplicates: %w", err)
|
||||
}
|
||||
return added > 0, nil
|
||||
added := false
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
var err error
|
||||
added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
|
||||
return err
|
||||
})
|
||||
return added, err
|
||||
}
|
||||
|
||||
// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
|
||||
// prefix stripped). Caller must hold c.mu.
|
||||
//
|
||||
// Reads the field index and filters by prefix; only an un-indexed root falls
|
||||
// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
|
||||
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
|
||||
return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
|
||||
// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
|
||||
// stripped).
|
||||
func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
|
||||
prefix := recordPrefix(uuid) + CIDsKeyPrefix
|
||||
rows, err := r.Range(ctx, prefix, "", 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, row.K[len(prefix):])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
|
||||
// (including the cids/<cid> key-set), every reverse-index entry for those
|
||||
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
|
||||
// is a successful no-op.
|
||||
// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
|
||||
// with no remaining keys is a successful no-op.
|
||||
func (c *Client) DeleteRoot(uuid string) error {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
if uuid == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
prefix := c.buildKeyPrefix(uuid)
|
||||
|
||||
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
|
||||
// the list of reverse-index entries to clean up.
|
||||
cids, err := c.cidsForRootLocked(ctx, uuid)
|
||||
if err != nil {
|
||||
return c.update(func(tx *kv.Tx) error {
|
||||
_, err := deleteRecordTx(tx, uuid)
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. Delete every reverse-index entry in one shot.
|
||||
if len(cids) > 0 {
|
||||
keys := make([]string, 0, len(cids))
|
||||
for _, cidStr := range cids {
|
||||
keys = append(keys, c.buildCIDIndexKey(cidStr))
|
||||
}
|
||||
if err := c.client.Del(ctx, keys...).Err(); err != nil {
|
||||
return fmt.Errorf("del cid index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan root keys: %w", err)
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
if err := c.client.Del(ctx, keys...).Err(); err != nil {
|
||||
return fmt.Errorf("del root keys: %w", err)
|
||||
}
|
||||
}
|
||||
cursor = next
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Remove from the file index set.
|
||||
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
|
||||
return fmt.Errorf("srem index: %w", err)
|
||||
}
|
||||
// Marked so the incremental search-index refresh reads it, finds it gone,
|
||||
// and evicts it — otherwise a deleted root keeps answering searches until
|
||||
// the next full rebuild.
|
||||
c.markDirtyLocked(ctx, uuid)
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
|
||||
// root. If it's a key-set member — field name starts with "cids/" — it
|
||||
// registers the implied CID in the reverse index. Writers emit cids/<cid>
|
||||
// members without knowing about the reverse index; this hook keeps it in
|
||||
// sync transparently.
|
||||
//
|
||||
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
|
||||
// (logged, not returned): an out-of-sync reverse index is recoverable via a
|
||||
// sweep, while a failed property write is not.
|
||||
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
|
||||
cidStr := cidFromKeysetField(field)
|
||||
if cidStr == "" {
|
||||
return nil
|
||||
}
|
||||
return c.addAliasLocked(ctx, uuid, cidStr)
|
||||
}
|
||||
|
||||
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
|
||||
// over every entry in a metadata map. Used by SetMetadataFlat /
|
||||
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
|
||||
// hold c.mu.
|
||||
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
|
||||
for k := range metadata {
|
||||
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
|
||||
@@ -406,24 +275,23 @@ type MetadataDocument struct {
|
||||
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
|
||||
// never existed).
|
||||
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return nil, fmt.Errorf("not connected")
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if uuid == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
flat, err := c.getMetadataFlatInternal(ctx, uuid)
|
||||
flat, err := readRecord(ctx, s, uuid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := c.buildKeyPrefix(uuid)
|
||||
if flat == nil {
|
||||
flat = make(map[string]string)
|
||||
}
|
||||
|
||||
// Hoist the cids/<cid> key-set members into a typed array and derive the
|
||||
// canonical CID by rank. The members remain in Flat too — that's how
|
||||
@@ -443,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
|
||||
}
|
||||
}
|
||||
|
||||
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
return nil, fmt.Errorf("smembers duplicates: %w", err)
|
||||
dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read duplicates: %w", err)
|
||||
}
|
||||
dups := make([]string, 0, len(dupRows))
|
||||
for _, row := range dupRows {
|
||||
dups = append(dups, row.K[len(dupPrefix(uuid)):])
|
||||
}
|
||||
|
||||
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
|
||||
@@ -474,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas
|
||||
if midhash == "" {
|
||||
return "", false, fmt.Errorf("midhash is required")
|
||||
}
|
||||
uuid, err := c.GetByCID(midhash)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
|
||||
}
|
||||
|
||||
if uuid == "" {
|
||||
uuid, err = c.Mint(absPath, size, mtimeNano)
|
||||
var (
|
||||
uuid string
|
||||
minted bool
|
||||
)
|
||||
// One transaction: the lookup and the mint cannot interleave with another
|
||||
// registration of the same content, so two concurrent scans of one file
|
||||
// can no longer mint two roots for it.
|
||||
err := c.update(func(tx *kv.Tx) error {
|
||||
ctx := tx.Context()
|
||||
existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
|
||||
return fmt.Errorf("GetByCID(%s): %w", midhash, err)
|
||||
}
|
||||
if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
|
||||
return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
|
||||
uuid = existing
|
||||
if uuid == "" {
|
||||
if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
|
||||
return fmt.Errorf("Mint(%s): %w", absPath, err)
|
||||
}
|
||||
minted = true
|
||||
if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
|
||||
return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return uuid, true, nil
|
||||
}
|
||||
|
||||
existing, _ := c.GetProperty(uuid, "filePath")
|
||||
if existing != absPath {
|
||||
if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
|
||||
return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
|
||||
current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return uuid, false, nil
|
||||
if current != absPath {
|
||||
if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
|
||||
return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := writeFieldsTx(tx, uuid, map[string]string{
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if _, err := c.MergeMetadataFlat(uuid, map[string]string{
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
}); err != nil {
|
||||
return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
|
||||
}
|
||||
return uuid, false, nil
|
||||
return uuid, minted, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user