new meta-core api, now redis is not exposed as an "API"

internal/storage/cid_resolution.go

@@ -0,0 +1,500 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/metazla/meta-core/internal/cid"
+	"github.com/redis/go-redis/v9"
+)
+
+// This file implements the CID-resolution layer that lives on top of the
+// flat metadata storage in client.go. The shape is described in detail in
+// docs/uuid-rooted-metadata.md; the short version:
+//
+//   - Roots are opaque UUIDv7s — file:<uuid>/<property> for every property
+//   - Every known CID for a file is registered as a reverse-index entry:
+//       cid:<algorithm>:<value> → <uuid>     (Redis STRING)
+//     and mirrored in a per-root set for fast enumeration:
+//       file:<uuid>/cids                     (Redis SET)
+//   - A canonical_cid field tracks the best (rank-wise) CID to advertise
+//     externally; updated by reconcileCanonicalCIDLocked whenever the set
+//     changes
+//
+// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
+// any write of a cid_* or midhash256 field and register the alias, so
+// plugins that just write their result fields don't need to know about
+// the reverse index at all. The hook helpers are *Locked — they assume
+// c.mu is already held by the caller, avoiding nested RLock (which can
+// deadlock against a waiting writer).
+
+// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
+// with a CID token, it forms the full key: "cid:midhash256:bafk…".
+const CIDIndexPrefix = "cid:"
+
+// CIDsField is the per-root SET key that mirrors all CID aliases for a file.
+// Stored at file:<uuid>/cids — read by reconcileCanonicalCIDLocked and by
+// DeleteRoot to find every reverse-index entry that needs to be cleaned up.
+const CIDsField = "cids"
+
+// CanonicalCIDField is the per-root STRING key that stores the externally
+// preferred CID for this file. Picked by cid.Better over the cids set.
+const CanonicalCIDField = "canonical_cid"
+
+// DuplicatesField is the per-root SET key holding additional file paths
+// whose content hashes to the same CIDs as this root. The watcher writes
+// here when it sees a new path for content that's already indexed; meta-dup
+// reads from it.
+const DuplicatesField = "duplicates"
+
+// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
+// token is a CID in prefixed form ("midhash256:bafk…", "sha256:…", etc.).
+func (c *Client) buildCIDIndexKey(token string) string {
+	return c.buildKey(CIDIndexPrefix + token)
+}
+
+// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
+// in the file index. Returns the new UUID. The caller is expected to follow
+// up with AddAlias for at least one CID — a root without any alias is
+// reachable only via the index, which works but defeats the point.
+func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return "", fmt.Errorf("not connected")
+	}
+
+	uuid, err := NewUUID()
+	if err != nil {
+		return "", fmt.Errorf("uuid: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	prefix := c.buildKeyPrefix(uuid)
+	pipe := c.client.Pipeline()
+	pipe.Set(ctx, prefix+"filePath", filePath, 0)
+	pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
+	pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
+	pipe.SAdd(ctx, c.buildIndexKey(), uuid)
+	if _, err := pipe.Exec(ctx); err != nil {
+		return "", fmt.Errorf("mint pipeline: %w", err)
+	}
+	return uuid, nil
+}
+
+// AddAlias registers a CID token as resolving to uuid. Idempotent: writing
+// the same alias twice is a no-op. After the alias is registered, the
+// canonical_cid field is reconciled — picks the highest-ranked token in
+// the cids set and writes it to canonical_cid if it changed.
+func (c *Client) AddAlias(uuid, token string) error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	return c.addAliasLocked(ctx, uuid, token)
+}
+
+// addAliasLocked is the lock-free body of AddAlias. The caller must already
+// hold c.mu (read or write). Splits out so the alias-maintenance hooks
+// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
+// the work without re-entering the lock — nested RLock can deadlock
+// against a waiting writer.
+func (c *Client) addAliasLocked(ctx context.Context, uuid, token string) error {
+	if uuid == "" || token == "" {
+		return fmt.Errorf("uuid and token are required")
+	}
+	if cid.AlgorithmOf(token) == "" {
+		return fmt.Errorf("token %q is not in <algorithm>:<value> form", token)
+	}
+
+	// Self-pointing alias guard. If the caller is trying to register
+	// `cid:<algo>:<value>` → `<value>` (i.e. the root IS the cid value),
+	// that's the dual-root failure mode in disguise — it means somebody
+	// wrote /meta/<value> without first resolving via ResolveRoot. Refuse
+	// the alias write; leave the existing alias (if any) alone. Still
+	// add to this root's cids set so the field-level write succeeds.
+	if uuid == cid.ValueOf(token) {
+		if _, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+CIDsField, token).Result(); err != nil {
+			return fmt.Errorf("sadd cids set: %w", err)
+		}
+		return c.reconcileCanonicalCIDLocked(ctx, uuid)
+	}
+
+	// Guard against the dual-root pattern: if `cid:<token>` already points
+	// at a DIFFERENT uuid, don't silently overwrite. The historical bug:
+	// meta-sort writing cid_midhash256=<v> to root "<v>" caused the hook
+	// to register cid:midhash256:<v> → <v>, severing the watcher's prior
+	// cid:midhash256:<v> → <real-uuid> alias. With write-path resolution
+	// now in place callers shouldn't hit this anymore, but the guard is
+	// cheap and prevents regression if a new caller bypasses ResolveRoot.
+	indexKey := c.buildCIDIndexKey(token)
+	existing, err := c.client.Get(ctx, indexKey).Result()
+	if err != nil && err != redis.Nil {
+		return fmt.Errorf("read existing alias: %w", err)
+	}
+	if existing != "" && existing != uuid {
+		// Existing alias points elsewhere. Leave it alone and just add the
+		// token to this root's cids set so it's still discoverable via the
+		// root → cids enumeration.
+		if _, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+CIDsField, token).Result(); err != nil {
+			return fmt.Errorf("sadd cids set: %w", err)
+		}
+		return c.reconcileCanonicalCIDLocked(ctx, uuid)
+	}
+
+	prefix := c.buildKeyPrefix(uuid)
+	pipe := c.client.Pipeline()
+	pipe.Set(ctx, indexKey, uuid, 0)
+	pipe.SAdd(ctx, prefix+CIDsField, token)
+	if _, err := pipe.Exec(ctx); err != nil {
+		return fmt.Errorf("add alias pipeline: %w", err)
+	}
+	return c.reconcileCanonicalCIDLocked(ctx, uuid)
+}
+
+// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an
+// editor lookup, an SSE consumer) to the actual storage root key.
+//
+// Callers historically send a bare midhash256 value (e.g.
+// "bagacba…") because that's what the watcher publishes on file:events.
+// The UUID-rooted-metadata design (see docs/uuid-rooted-metadata.md)
+// keeps roots opaque and aliases CIDs into the reverse index:
+//
+//	cid:midhash256:bagacba… → 01KSBHM*
+//
+// Without resolution, every /meta/<midhash> write creates a parallel
+// midhash-rooted entry that the watcher's UUID never sees. The editor
+// then shows two entries per file: the (rich) midhash root and the
+// (empty) UUID root containing only `duplicates`.
+//
+// Behaviour:
+//   - empty hash → empty (caller's problem)
+//   - hash matches cid:midhash256:<hash> in the reverse index → return UUID
+//   - otherwise return hash unchanged (legacy / explicit-UUID writes)
+//
+// Errors are converted to "no resolution" — if Redis is down or the
+// lookup fails we let the caller proceed with the bare hash and surface
+// the Redis failure on the actual operation.
+func (c *Client) ResolveRoot(hash string) string {
+	if hash == "" {
+		return hash
+	}
+	uuid, err := c.GetByCID("midhash256:" + hash)
+	if err != nil || uuid == "" {
+		return hash
+	}
+	return uuid
+}
+
+// GetByCID resolves a CID token to the UUID of its root. Returns ("", nil)
+// when the token is unknown; the empty string is the "not found" signal
+// rather than a typed error so callers can branch without errors.As.
+func (c *Client) GetByCID(token string) (string, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return "", fmt.Errorf("not connected")
+	}
+	if token == "" {
+		return "", nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(token)).Result()
+	if err == redis.Nil {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("get cid index: %w", err)
+	}
+	return uuid, nil
+}
+
+// AddDuplicatePath records that path is another physical location whose
+// content matches uuid. Returns true if path was newly added (not already
+// in the set). The watcher calls this when it sees a file whose midhash
+// matches an existing root but at a different filePath.
+func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return false, fmt.Errorf("not connected")
+	}
+	if uuid == "" || path == "" {
+		return false, nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
+	if err != nil {
+		return false, fmt.Errorf("sadd duplicates: %w", err)
+	}
+	return added > 0, nil
+}
+
+// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property,
+// every reverse-index entry listed in the cids set, and the index membership.
+// Idempotent — a uuid with no remaining keys is a successful no-op.
+func (c *Client) DeleteRoot(uuid string) error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return fmt.Errorf("not connected")
+	}
+	if uuid == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	prefix := c.buildKeyPrefix(uuid)
+
+	// 1. Read the cids set first — once we delete file:<uuid>/* we lose the
+	//    list of reverse-index entries to clean up. Tolerate a missing set
+	//    (empty list = nothing to unmap).
+	tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result()
+	if err != nil && err != redis.Nil {
+		return fmt.Errorf("smembers cids: %w", err)
+	}
+
+	// 2. Delete every reverse-index entry. Build the keys upfront, then DEL
+	//    in one shot so we don't pay per-RTT.
+	if len(tokens) > 0 {
+		keys := make([]string, 0, len(tokens))
+		for _, t := range tokens {
+			keys = append(keys, c.buildCIDIndexKey(t))
+		}
+		if err := c.client.Del(ctx, keys...).Err(); err != nil {
+			return fmt.Errorf("del cid index: %w", err)
+		}
+	}
+
+	// 3. Scan and delete every file:<uuid>/* key.
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
+		if err != nil {
+			return fmt.Errorf("scan root keys: %w", err)
+		}
+		if len(keys) > 0 {
+			if err := c.client.Del(ctx, keys...).Err(); err != nil {
+				return fmt.Errorf("del root keys: %w", err)
+			}
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+
+	// 4. Remove from the file index set.
+	if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
+		return fmt.Errorf("srem index: %w", err)
+	}
+	return nil
+}
+
+// reconcileCanonicalCIDLocked reads the cids set, picks the highest-ranked
+// token, and updates the canonical_cid field if it changed. Called from
+// addAliasLocked after a new alias is registered. Caller must hold c.mu.
+func (c *Client) reconcileCanonicalCIDLocked(ctx context.Context, uuid string) error {
+	prefix := c.buildKeyPrefix(uuid)
+
+	tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result()
+	if err != nil {
+		return fmt.Errorf("smembers cids: %w", err)
+	}
+	if len(tokens) == 0 {
+		return nil
+	}
+
+	best := tokens[0]
+	for _, t := range tokens[1:] {
+		best = cid.Better(best, t)
+	}
+
+	canonicalKey := prefix + CanonicalCIDField
+	current, err := c.client.Get(ctx, canonicalKey).Result()
+	if err != nil && err != redis.Nil {
+		return fmt.Errorf("get canonical_cid: %w", err)
+	}
+	if current == best {
+		return nil
+	}
+	if err := c.client.Set(ctx, canonicalKey, best, 0).Err(); err != nil {
+		return fmt.Errorf("set canonical_cid: %w", err)
+	}
+	return nil
+}
+
+// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into
+// a root. If it looks like a CID field — name is "midhash256" or starts
+// with "cid_" — it registers the implied CID token as an alias. Plugins
+// write CID fields without knowing about the reverse index; this hook
+// keeps the index in sync transparently.
+//
+// Caller must hold c.mu. Errors are intentionally swallowed at the call
+// site (logged, not returned): an out-of-sync reverse index is recoverable
+// via a sweep, while a failed property write is not.
+func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
+	token := cidTokenFromField(field, value)
+	if token == "" {
+		return nil
+	}
+	return c.addAliasLocked(ctx, uuid, token)
+}
+
+// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
+// over every entry in a metadata map. Used by SetMetadataFlat /
+// MergeMetadataFlat after the underlying pipeline write succeeds. Caller
+// must hold c.mu.
+func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
+	for k, v := range metadata {
+		_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, v)
+	}
+}
+
+// cidTokenFromField turns a (field-name, raw-value) pair into a normalized
+// CID token "<algorithm>:<value>". Returns "" for non-CID fields.
+//
+// Acceptable inputs:
+//   - field="midhash256", value="bafk…"           → "midhash256:bafk…"
+//   - field="cid_sha256",  value="bafk…"          → "sha256:bafk…"
+//   - field="cid_ipfs",    value="ipfs:bafy…"     → "ipfs:bafy…" (trusted)
+//
+// Plugins that already use the prefixed form get their value passed through;
+// plugins that store the bare CID get prefixed from the field name. The
+// detection heuristic is just "value contains a colon" — base32lower CIDs
+// never contain colons, so this is unambiguous.
+func cidTokenFromField(field, value string) string {
+	if value == "" {
+		return ""
+	}
+	var algo string
+	switch {
+	case field == "midhash256":
+		algo = "midhash256"
+	case strings.HasPrefix(field, "cid_"):
+		algo = strings.TrimPrefix(field, "cid_")
+	default:
+		return ""
+	}
+	if algo == "" {
+		return ""
+	}
+	if strings.IndexByte(value, ':') > 0 {
+		return value
+	}
+	return cid.Token(algo, value)
+}
+
+// MetadataDocument is the complete state of one file as the public API
+// returns it: every flat string field, plus the cids and duplicates sets
+// hoisted into typed arrays. Distinct from GetMetadataFlat (which only
+// returns string keys) — clients querying /api/meta/<cid> want the SETs
+// expanded too, since "what other CIDs identify this file" and "what other
+// paths point at this content" are core to the response.
+type MetadataDocument struct {
+	Flat       map[string]string `json:"metadata"`
+	CIDs       []string          `json:"cids"`
+	Duplicates []string          `json:"duplicates"`
+}
+
+// GetMetadataDocument returns the full document for a root: every string
+// property under file:<uuid>/* plus the cids and duplicates SETs. Returns
+// nil if the root has no keys at all (deleted or never existed).
+func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+	if uuid == "" {
+		return nil, nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	flat, err := c.getMetadataFlatInternal(ctx, uuid)
+	if err != nil {
+		return nil, err
+	}
+	prefix := c.buildKeyPrefix(uuid)
+
+	cids, err := c.client.SMembers(ctx, prefix+CIDsField).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers cids: %w", err)
+	}
+	dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
+	if err != nil && err != redis.Nil {
+		return nil, fmt.Errorf("smembers duplicates: %w", err)
+	}
+
+	if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
+		return nil, nil
+	}
+	return &MetadataDocument{
+		Flat:       flat,
+		CIDs:       cids,
+		Duplicates: dups,
+	}, nil
+}
+
+// lookupSidecarPathByCID is the fallback that LookupPathByCID falls into
+// when the token isn't a known root alias. Used to resolve poster/backdrop
+// CIDs, which are stored as values pointing at sidecar files (posterPath /
+// backdropPath) rather than registered as aliases of the file itself.
+//
+// O(n) — walks every root. Acceptable because (a) sidecar lookups are rare
+// vs. file CID lookups and (b) the alternative is registering sidecars in
+// the reverse index, which conflates "this CID is the file" with "this CID
+// is an image associated with the file."
+func (c *Client) lookupSidecarPathByCID(token string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	hashIDs, err := c.getAllHashIDsInternal(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	for _, hashID := range hashIDs {
+		meta, err := c.getMetadataFlatInternal(ctx, hashID)
+		if err != nil || len(meta) == 0 {
+			continue
+		}
+		if meta["poster"] == token {
+			if p := meta["posterPath"]; p != "" {
+				return normalizeFilesRelativePath(p), nil
+			}
+		}
+		if meta["backdrop"] == token {
+			if p := meta["backdropPath"]; p != "" {
+				return normalizeFilesRelativePath(p), nil
+			}
+		}
+	}
+	return "", nil
+}

internal/storage/client.go

@@ -192,6 +192,10 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
 
+	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
+	// in the payload. See cid_resolution.go for the rules.
+	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
+
 	return nil
 }
 
@@ -263,11 +267,17 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
 
+	// Auto-register reverse-index alias if this is a CID field.
+	_ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value)
+
 	return nil
 }
 
 // DeleteMetadata deletes all metadata for a file
 // Uses SCAN + DEL for flat keys: file:{hashId}/*
+// Also unmaps every reverse-index entry registered for this root, so a
+// subsequent GET /api/meta/<old-cid> correctly returns 404 instead of
+// pointing at a UUID with no keys.
 func (c *Client) DeleteMetadata(hashID string) (int64, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
@@ -281,6 +291,22 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) {
 
 	prefix := c.buildKeyPrefix(hashID)
 
+	// Read the cids set first — once file:<uuid>/* is gone we've lost the
+	// list of reverse-index keys that need cleanup.
+	tokens, err := c.client.SMembers(ctx, prefix+CIDsField).Result()
+	if err != nil && err != redis.Nil {
+		return 0, fmt.Errorf("smembers cids: %w", err)
+	}
+	if len(tokens) > 0 {
+		idxKeys := make([]string, 0, len(tokens))
+		for _, t := range tokens {
+			idxKeys = append(idxKeys, c.buildCIDIndexKey(t))
+		}
+		if err := c.client.Del(ctx, idxKeys...).Err(); err != nil {
+			return 0, fmt.Errorf("del cid index: %w", err)
+		}
+	}
+
 	// Scan and delete all keys with this prefix
 	var deletedCount int64
 	var cursor uint64
@@ -323,72 +349,31 @@ func (c *Client) CountFiles() (int, error) {
 
 // LookupPathByCID resolves a CID to its file path on disk.
 //
-// Resolution covers three cases:
-//   - midhash256 hashID itself: the CID identifies a file's content, return its filePath.
-//   - poster/backdrop fields: the CID is for the image asset, return posterPath/backdropPath.
-//   - any cid_* field: the CID is another digest of the same file, return its filePath.
+// Primary path: the reverse index (cid:<token> → uuid). O(1) lookup followed
+// by a single GET of file:<uuid>/filePath.
 //
-// Walks every indexed hash; acceptable for admin/UI use, but would benefit from
-// a reverse index (cid → hashID/field) if call volume grows.
-func (c *Client) LookupPathByCID(cid string) (string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return "", fmt.Errorf("not connected")
-	}
-
-	if cid == "" {
-		return "", nil
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	hashIDs, err := c.getAllHashIDsInternal(ctx)
+// Fallback path: poster/backdrop CIDs aren't registered as aliases (they
+// refer to sidecar images, not the main file content), so unknown tokens
+// fall through to a scan of poster/backdrop fields. O(n) but rare in
+// practice — sidecar lookups are infrequent compared to file CID lookups.
+//
+// Accepts CID tokens in prefixed form ("midhash256:bafk…", "sha256:…",
+// "ipfs:bafy…"). Bare CIDs without an algorithm prefix won't resolve via
+// the reverse index — callers should pass the prefixed form.
+func (c *Client) LookupPathByCID(token string) (string, error) {
+	uuid, err := c.GetByCID(token)
 	if err != nil {
 		return "", err
 	}
-
-	for _, hashID := range hashIDs {
-		meta, err := c.getMetadataFlatInternal(ctx, hashID)
-		if err != nil || len(meta) == 0 {
-			continue
-		}
-
-		// midhash256 == hashID: the file itself.
-		if hashID == cid {
-			if fp := meta["filePath"]; fp != "" {
-				return normalizeFilesRelativePath(fp), nil
-			}
-		}
-
-		// Poster / backdrop refer to a sidecar image asset.
-		if meta["poster"] == cid {
-			if p := meta["posterPath"]; p != "" {
-				return normalizeFilesRelativePath(p), nil
-			}
-		}
-		if meta["backdrop"] == cid {
-			if p := meta["backdropPath"]; p != "" {
-				return normalizeFilesRelativePath(p), nil
-			}
-		}
-
-		// Any other cid_* field is an additional digest of the file itself.
-		for field, value := range meta {
-			if value != cid {
-				continue
-			}
-			if strings.HasPrefix(field, "cid_") {
-				if fp := meta["filePath"]; fp != "" {
-					return normalizeFilesRelativePath(fp), nil
-				}
-			}
+	if uuid != "" {
+		fp, err := c.GetProperty(uuid, "filePath")
+		if err != nil {
+			return "", err
 		}
+		return normalizeFilesRelativePath(fp), nil
 	}
-
-	return "", nil
+	// Not a file alias — try poster/backdrop sidecars.
+	return c.lookupSidecarPathByCID(token)
 }
 
 // normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
@@ -477,6 +462,10 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 		return len(metadata), fmt.Errorf("sadd index failed: %w", err)
 	}
 
+	// Auto-register reverse-index aliases for any cid_* or midhash256 fields
+	// in the payload. See cid_resolution.go for the rules.
+	c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
+
 	return len(metadata), nil
 }
 
@@ -599,19 +588,27 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
 	return true, c.client.Set(ctx, key, newValue, 0).Err()
 }
 
-// FileTuple holds the (path, size, mtime) record for one file. Used by the
-// watcher hydrator at startup to avoid re-mid-hashing files whose tuple
-// matches what's already in Redis.
+// FileTuple holds the (uuid, path, size, mtime, midhash) record for one
+// file. Used by the watcher hydrator at startup to avoid re-mid-hashing
+// files whose tuple matches what's already in Redis.
+//
+// HashID is the root identifier (UUIDv7 in the current model); MidHash256
+// is the actual midhash CID for the file, present iff the watcher already
+// computed and stored it. Hydration uses HashID for the state-registry
+// key and MidHash256 for the in-memory "I already know this hash" flag.
 type FileTuple struct {
-	HashID    string
-	FilePath  string
-	Size      int64
-	MtimeNano int64
+	HashID     string
+	FilePath   string
+	Size       int64
+	MtimeNano  int64
+	MidHash256 string
 }
 
-// GetTuplesForAllFiles returns one FileTuple per indexed CID, fetched in
-// batched MGETs (no SCAN per CID). Skips files missing any of the three
-// fields. Sub-second on local Redis for ~10K files.
+// GetTuplesForAllFiles returns one FileTuple per indexed root, fetched in
+// batched MGETs (no SCAN per root). Skips files missing any of filePath,
+// sizeByte, or mtimeNano. midhash256 is optional — files that haven't
+// been hashed yet still appear in the result with an empty MidHash256.
+// Sub-second on local Redis for ~10K files.
 func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
@@ -631,11 +628,11 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 		return nil, nil
 	}
 
-	// Build the key list: 3 keys per CID, in a fixed order so we can decode
+	// Build the key list: 4 keys per root, in a fixed order so we can decode
 	// the MGET response by position. fields[i] for hashIDs[j] sits at
-	// keys[j*3+i].
-	const fieldsPerFile = 3
-	fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"}
+	// keys[j*4+i].
+	const fieldsPerFile = 4
+	fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano", "midhash256"}
 	keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
 	for _, h := range hashIDs {
 		prefix := c.buildKeyPrefix(h)
@@ -665,6 +662,7 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 		filePath, _ := values[base].(string)
 		sizeStr, _ := values[base+1].(string)
 		mtimeStr, _ := values[base+2].(string)
+		midhash, _ := values[base+3].(string)
 		if filePath == "" || sizeStr == "" || mtimeStr == "" {
 			continue
 		}
@@ -677,10 +675,11 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 			continue
 		}
 		out = append(out, FileTuple{
-			HashID:    h,
-			FilePath:  filePath,
-			Size:      size,
-			MtimeNano: mtime,
+			HashID:     h,
+			FilePath:   filePath,
+			Size:       size,
+			MtimeNano:  mtime,
+			MidHash256: midhash,
 		})
 	}
 	return out, nil

internal/storage/dual_root_migration.go

@@ -0,0 +1,162 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+)
+
+// MigrateDualRoots scans file:__index__ for hashIds that look like raw
+// midhash256 CID values (i.e. NOT UUIDs/ULIDs) and have a matching alias
+// `cid:midhash256:<hash>` pointing to a different UUID. For each such
+// "stranded" root we:
+//
+//  1. Copy every file:<midhash>/<property> into file:<uuid>/<property>
+//     using MergeMetadataFlat (preserves anything the UUID root already
+//     holds, e.g. duplicates and watcher-owned fields).
+//  2. Delete the entire file:<midhash>/* subtree.
+//  3. Drop the midhash hashId from file:__index__ (kept on the UUID).
+//
+// This is the one-shot fix for the dual-root pattern described in
+// docs/api-mediated-access.md. The pattern was caused by the (now-fixed)
+// /meta/{hash} write handlers not resolving CID → UUID before writing.
+//
+// Returns the number of stranded roots fixed.
+func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
+	c.mu.RLock()
+	if c.client == nil {
+		c.mu.RUnlock()
+		return 0, fmt.Errorf("not connected")
+	}
+	hashIDs, err := c.getAllHashIDsInternal(ctx)
+	c.mu.RUnlock()
+	if err != nil {
+		return 0, fmt.Errorf("get hash ids: %w", err)
+	}
+
+	// Build the orphaned-UUID index: for every UUID-rooted entry, record
+	// midhash256 → UUID. This lets us find the "real" target when an alias
+	// has been overwritten to self-point (the failure mode the in-flight
+	// fix prevents going forward, but a one-shot still needs to repair
+	// existing dual-rooted data).
+	midhashToUUID := make(map[string]string)
+	for _, hashID := range hashIDs {
+		if !looksLikeULID(hashID) {
+			continue
+		}
+		mh, err := c.GetProperty(hashID, "midhash256")
+		if err != nil || mh == "" {
+			continue
+		}
+		midhashToUUID[mh] = hashID
+	}
+
+	fixed := 0
+	for _, hashID := range hashIDs {
+		// Skip likely-UUID hashIds. ULIDs are 26 chars, uppercase Crockford
+		// Base32. We treat anything that's not all-uppercase-alphanumeric
+		// of length ~26 as a CID-shaped hashId worth checking.
+		if looksLikeULID(hashID) {
+			continue
+		}
+
+		// Is there an alias pointing to a different uuid?
+		token := "midhash256:" + hashID
+		uuid, err := c.GetByCID(token)
+		if err != nil {
+			log.Printf("[MigrateDualRoots] GetByCID(%s): %v (skipping)", token, err)
+			continue
+		}
+		// Three cases:
+		//   - uuid empty → no alias, this midhash root is standalone (skip)
+		//   - uuid == hashID → self-pointing alias; check the orphan index
+		//     for a UUID that claims this midhash via its midhash256 field
+		//   - uuid != hashID and != "" → proper alias, migrate to that UUID
+		if uuid == "" || uuid == hashID {
+			if orphan, ok := midhashToUUID[hashID]; ok && orphan != "" {
+				uuid = orphan
+			} else {
+				continue
+			}
+		}
+
+		// Pull every key under file:<hashID>/* and merge into file:<uuid>/.
+		flat, err := c.GetMetadataFlat(hashID)
+		if err != nil {
+			log.Printf("[MigrateDualRoots] GetMetadataFlat(%s): %v (skipping)", hashID, err)
+			continue
+		}
+		if len(flat) > 0 {
+			if _, err := c.MergeMetadataFlat(uuid, flat); err != nil {
+				log.Printf("[MigrateDualRoots] MergeMetadataFlat(%s ← %s): %v", uuid, hashID, err)
+				continue
+			}
+		}
+
+		// Delete the stranded root entirely.
+		if _, err := c.DeleteMetadata(hashID); err != nil {
+			log.Printf("[MigrateDualRoots] DeleteMetadata(%s): %v", hashID, err)
+			continue
+		}
+
+		// Re-point the alias to the surviving UUID. DeleteMetadata wipes
+		// the cid:* entries for the deleted root, so the alias for this
+		// midhash may have been removed. Restore it pointing at the
+		// canonical UUID so future ResolveRoot calls land correctly.
+		if err := c.AddAlias(uuid, "midhash256:"+hashID); err != nil {
+			log.Printf("[MigrateDualRoots] AddAlias(%s → midhash256:%s): %v", uuid, hashID, err)
+			// Not fatal — data is migrated; the alias mismatch can be
+			// re-fixed by another sweep.
+		}
+
+		fixed++
+	}
+
+	if fixed > 0 {
+		log.Printf("[MigrateDualRoots] Reunited %d stranded midhash roots into their UUID counterparts", fixed)
+	}
+	return fixed, nil
+}
+
+// looksLikeULID is a cheap heuristic for "this hashId is opaque internal
+// UUID, not a CID we should resolve." ULIDs are 26-char Crockford Base32;
+// CIDs in this codebase are longer and lowercase. Tolerates dashes for
+// future-proofing.
+func looksLikeULID(s string) bool {
+	if len(s) != 26 {
+		return false
+	}
+	for _, r := range s {
+		if r >= 'A' && r <= 'Z' {
+			continue
+		}
+		if r >= '0' && r <= '9' {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
+// handler and the startup hook. Default to a generous 5-minute deadline —
+// the per-entry work is small but the index can be large.
+func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	return c.MigrateDualRoots(ctx)
+}
+
+// MigrateDualRootsReport returns a human-readable summary of what
+// MigrateDualRoots would do or did. Useful for the admin endpoint.
+func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
+	fixed, err := c.MigrateDualRoots(ctx)
+	if err != nil {
+		return "", fixed, err
+	}
+	parts := []string{}
+	parts = append(parts, fmt.Sprintf("merged %d stranded midhash roots into UUID roots", fixed))
+	return strings.Join(parts, "; "), fixed, nil
+}

internal/storage/schema_version.go

@@ -0,0 +1,83 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// SchemaVersion is the Redis-layout version this build of meta-core expects.
+// Distinct from the snapshot manifest version (which is on-disk format);
+// this number describes the in-memory Redis schema — primarily, "are roots
+// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
+//
+// Bump only when a Redis-layout change can't be detected from the data
+// itself or healed by the auto-alias hooks. Each bump is a flag day in
+// alpha: operators wipe and restart.
+const SchemaVersion = 2
+
+// SchemaVersionKey is the sentinel under which we record SchemaVersion in
+// Redis. Checked on every meta-core boot.
+const SchemaVersionKey = "meta-core:schema-version"
+
+// EnsureSchemaVersion verifies the Redis store matches the expected schema
+// version. Called once at startup, before any other reads/writes.
+//
+// Behavior:
+//   - sentinel present and matches expected → OK
+//   - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
+//   - sentinel missing AND file:__index__ non-empty → legacy data, refuse
+//   - sentinel present but mismatched → refuse, point operator at clean:all
+//
+// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
+// The error message names the wipe remediation because in alpha there
+// is no automated migration — operators bring up a clean store.
+func (c *Client) EnsureSchemaVersion(expected int) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client == nil {
+		return fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	key := c.buildKey(SchemaVersionKey)
+	have, err := c.client.Get(ctx, key).Result()
+	if err != nil && err != redis.Nil {
+		return fmt.Errorf("get schema version: %w", err)
+	}
+
+	if err == redis.Nil {
+		// Sentinel absent. Fresh store iff there's no file:__index__ membership.
+		n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
+		if sErr != nil && sErr != redis.Nil {
+			return fmt.Errorf("scard file:__index__: %w", sErr)
+		}
+		if n > 0 {
+			return fmt.Errorf(
+				"redis has %d indexed roots but no %q sentinel — this is alpha "+
+					"data from before the UUID-rooted schema landed. Run "+
+					"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
+					"to wipe and restart, or downgrade meta-core to a compatible build",
+				n, SchemaVersionKey)
+		}
+		if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
+			return fmt.Errorf("set schema version: %w", err)
+		}
+		return nil
+	}
+
+	if have == strconv.Itoa(expected) {
+		return nil
+	}
+	return fmt.Errorf(
+		"redis schema version is %q, this build of meta-core expects %d. "+
+			"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
+			"build that matches the existing data",
+		have, expected)
+}

internal/storage/uuid.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"github.com/google/uuid"
+)
+
+// NewUUID returns a new UUIDv7 encoded as 26-char Crockford Base32 (ULID
+// layout). Used as the root key for file metadata entries.
+//
+// Why this format over canonical 8-4-4-4-12 hex:
+//   - 26 chars vs 36 — shorter Redis keys, less prefix space waste
+//   - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
+//     given time window, restoring some of the debug-ability lost vs the
+//     previous midhash256:abc rooting
+//   - Case-insensitive — no ambiguity when copying from logs
+//   - Crockford alphabet skips I/L/O/U, so eyeballed inspection of a key
+//     can't be confused for an ambient digit/letter typo
+func NewUUID() (string, error) {
+	u, err := uuid.NewV7()
+	if err != nil {
+		return "", err
+	}
+	return encodeCrockford(u[:]), nil
+}
+
+const crockfordAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+
+// encodeCrockford encodes a 16-byte value as 26 Crockford Base32 chars.
+// 128 bits packs into 130 bits (26 × 5); the top 2 bits of the first char
+// carry the high bits of byte 0, the remaining 3 are implicit zero. This
+// matches the ULID encoding so the output is interchangeable with ULIDs in
+// any tool that accepts them.
+func encodeCrockford(b []byte) string {
+	out := make([]byte, 26)
+	out[0] = crockfordAlphabet[(b[0]&0xE0)>>5]
+	out[1] = crockfordAlphabet[b[0]&0x1F]
+	out[2] = crockfordAlphabet[(b[1]&0xF8)>>3]
+	out[3] = crockfordAlphabet[((b[1]&0x07)<<2)|((b[2]&0xC0)>>6)]
+	out[4] = crockfordAlphabet[(b[2]&0x3E)>>1]
+	out[5] = crockfordAlphabet[((b[2]&0x01)<<4)|((b[3]&0xF0)>>4)]
+	out[6] = crockfordAlphabet[((b[3]&0x0F)<<1)|((b[4]&0x80)>>7)]
+	out[7] = crockfordAlphabet[(b[4]&0x7C)>>2]
+	out[8] = crockfordAlphabet[((b[4]&0x03)<<3)|((b[5]&0xE0)>>5)]
+	out[9] = crockfordAlphabet[b[5]&0x1F]
+	out[10] = crockfordAlphabet[(b[6]&0xF8)>>3]
+	out[11] = crockfordAlphabet[((b[6]&0x07)<<2)|((b[7]&0xC0)>>6)]
+	out[12] = crockfordAlphabet[(b[7]&0x3E)>>1]
+	out[13] = crockfordAlphabet[((b[7]&0x01)<<4)|((b[8]&0xF0)>>4)]
+	out[14] = crockfordAlphabet[((b[8]&0x0F)<<1)|((b[9]&0x80)>>7)]
+	out[15] = crockfordAlphabet[(b[9]&0x7C)>>2]
+	out[16] = crockfordAlphabet[((b[9]&0x03)<<3)|((b[10]&0xE0)>>5)]
+	out[17] = crockfordAlphabet[b[10]&0x1F]
+	out[18] = crockfordAlphabet[(b[11]&0xF8)>>3]
+	out[19] = crockfordAlphabet[((b[11]&0x07)<<2)|((b[12]&0xC0)>>6)]
+	out[20] = crockfordAlphabet[(b[12]&0x3E)>>1]
+	out[21] = crockfordAlphabet[((b[12]&0x01)<<4)|((b[13]&0xF0)>>4)]
+	out[22] = crockfordAlphabet[((b[13]&0x0F)<<1)|((b[14]&0x80)>>7)]
+	out[23] = crockfordAlphabet[(b[14]&0x7C)>>2]
+	out[24] = crockfordAlphabet[((b[14]&0x03)<<3)|((b[15]&0xE0)>>5)]
+	out[25] = crockfordAlphabet[b[15]&0x1F]
+	return string(out)
+}

internal/storage/uuid_test.go

@@ -0,0 +1,77 @@
+package storage
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNewUUID_Shape(t *testing.T) {
+	id, err := NewUUID()
+	if err != nil {
+		t.Fatalf("NewUUID: %v", err)
+	}
+	if len(id) != 26 {
+		t.Errorf("expected 26 chars, got %d (%q)", len(id), id)
+	}
+	const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+	for i, c := range id {
+		if !strings.ContainsRune(alphabet, c) {
+			t.Errorf("char %d (%q) not in Crockford alphabet (uuid=%q)", i, c, id)
+		}
+	}
+}
+
+func TestNewUUID_Unique(t *testing.T) {
+	const n = 1000
+	seen := make(map[string]struct{}, n)
+	for i := 0; i < n; i++ {
+		id, err := NewUUID()
+		if err != nil {
+			t.Fatalf("NewUUID at i=%d: %v", i, err)
+		}
+		if _, dup := seen[id]; dup {
+			t.Fatalf("collision after %d mints: %q", i+1, id)
+		}
+		seen[id] = struct{}{}
+	}
+}
+
+func TestNewUUID_TimeSortable(t *testing.T) {
+	// UUIDv7 has a millisecond timestamp prefix; consecutive mints in the
+	// same millisecond may not be strictly ordered (random tail), but
+	// mints across milliseconds are. Just check that an early ID
+	// lexicographically sorts before a later one when there's a real
+	// time gap. Doesn't test the no-gap case — the v7 spec only
+	// guarantees ordering across millisecond boundaries.
+	a, _ := NewUUID()
+	time.Sleep(2 * time.Millisecond)
+	b, _ := NewUUID()
+	if a >= b {
+		t.Errorf("expected %q < %q (UUIDv7 should be roughly time-sortable)", a, b)
+	}
+}
+
+func TestCIDTokenFromField(t *testing.T) {
+	cases := []struct {
+		field, value string
+		want         string
+	}{
+		{"midhash256", "bafk", "midhash256:bafk"},
+		{"cid_sha256", "bafk", "sha256:bafk"},
+		{"cid_ipfs", "bafy", "ipfs:bafy"},
+		{"cid_sha256", "sha256:bafk", "sha256:bafk"}, // already prefixed
+		{"cid_ipfs", "ipfs:bafy", "ipfs:bafy"},       // already prefixed
+		{"title", "Inception", ""},                    // non-CID field
+		{"tmdb/title", "Inception", ""},               // non-CID nested
+		{"cid_", "bafk", ""},                          // empty algo
+		{"midhash256", "", ""},                        // empty value
+		{"cid_sha256", "", ""},                        // empty value
+	}
+	for _, tc := range cases {
+		if got := cidTokenFromField(tc.field, tc.value); got != tc.want {
+			t.Errorf("cidTokenFromField(%q, %q) = %q, want %q",
+				tc.field, tc.value, got, tc.want)
+		}
+	}
+}