remove in metadata cid_* to put in the cids/ set
This commit is contained in:
Pierre
@@ -3,160 +3,39 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MigrateDualRoots scans file:__index__ for hashIds that look like raw
|
||||
// midhash256 CID values (i.e. NOT UUIDs/ULIDs) and have a matching alias
|
||||
// `cid:midhash256:<hash>` pointing to a different UUID. For each such
|
||||
// "stranded" root we:
|
||||
// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
|
||||
// legacy storage model: /meta/{hash} write handlers that didn't resolve
|
||||
// CID → UUID first created a parallel midhash-rooted entry alongside the
|
||||
// watcher's UUID root.
|
||||
//
|
||||
// 1. Copy every file:<midhash>/<property> into file:<uuid>/<property>
|
||||
// using MergeMetadataFlat (preserves anything the UUID root already
|
||||
// holds, e.g. duplicates and watcher-owned fields).
|
||||
// 2. Delete the entire file:<midhash>/* subtree.
|
||||
// 3. Drop the midhash hashId from file:__index__ (kept on the UUID).
|
||||
// That pattern is now structurally impossible:
|
||||
// - writes resolve via ResolveRoot before landing (see handlers), and
|
||||
// - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
|
||||
// UUID roots, with the reverse index cid:<cid> → uuid as the single
|
||||
// resolution path.
|
||||
//
|
||||
// This is the one-shot fix for the dual-root pattern described in
|
||||
// docs/api-mediated-access.md. The pattern was caused by the (now-fixed)
|
||||
// /meta/{hash} write handlers not resolving CID → UUID before writing.
|
||||
//
|
||||
// Returns the number of stranded roots fixed.
|
||||
// The legacy on-disk shape this migration targeted (midhash-rooted entries,
|
||||
// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
|
||||
// cut-over wipes Redis, so there is nothing to migrate. The function is kept
|
||||
// as a no-op so the admin endpoint and its wrappers continue to compile and
|
||||
// respond.
|
||||
func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
|
||||
c.mu.RLock()
|
||||
if c.client == nil {
|
||||
c.mu.RUnlock()
|
||||
return 0, fmt.Errorf("not connected")
|
||||
}
|
||||
hashIDs, err := c.getAllHashIDsInternal(ctx)
|
||||
c.mu.RUnlock()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("get hash ids: %w", err)
|
||||
}
|
||||
|
||||
// Build the orphaned-UUID index: for every UUID-rooted entry, record
|
||||
// midhash256 → UUID. This lets us find the "real" target when an alias
|
||||
// has been overwritten to self-point (the failure mode the in-flight
|
||||
// fix prevents going forward, but a one-shot still needs to repair
|
||||
// existing dual-rooted data).
|
||||
midhashToUUID := make(map[string]string)
|
||||
for _, hashID := range hashIDs {
|
||||
if !looksLikeULID(hashID) {
|
||||
continue
|
||||
}
|
||||
mh, err := c.GetProperty(hashID, "midhash256")
|
||||
if err != nil || mh == "" {
|
||||
continue
|
||||
}
|
||||
midhashToUUID[mh] = hashID
|
||||
}
|
||||
|
||||
fixed := 0
|
||||
for _, hashID := range hashIDs {
|
||||
// Skip likely-UUID hashIds. ULIDs are 26 chars, uppercase Crockford
|
||||
// Base32. We treat anything that's not all-uppercase-alphanumeric
|
||||
// of length ~26 as a CID-shaped hashId worth checking.
|
||||
if looksLikeULID(hashID) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Is there an alias pointing to a different uuid?
|
||||
token := "midhash256:" + hashID
|
||||
uuid, err := c.GetByCID(token)
|
||||
if err != nil {
|
||||
log.Printf("[MigrateDualRoots] GetByCID(%s): %v (skipping)", token, err)
|
||||
continue
|
||||
}
|
||||
// Three cases:
|
||||
// - uuid empty → no alias, this midhash root is standalone (skip)
|
||||
// - uuid == hashID → self-pointing alias; check the orphan index
|
||||
// for a UUID that claims this midhash via its midhash256 field
|
||||
// - uuid != hashID and != "" → proper alias, migrate to that UUID
|
||||
if uuid == "" || uuid == hashID {
|
||||
if orphan, ok := midhashToUUID[hashID]; ok && orphan != "" {
|
||||
uuid = orphan
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Pull every key under file:<hashID>/* and merge into file:<uuid>/.
|
||||
flat, err := c.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
log.Printf("[MigrateDualRoots] GetMetadataFlat(%s): %v (skipping)", hashID, err)
|
||||
continue
|
||||
}
|
||||
if len(flat) > 0 {
|
||||
if _, err := c.MergeMetadataFlat(uuid, flat); err != nil {
|
||||
log.Printf("[MigrateDualRoots] MergeMetadataFlat(%s ← %s): %v", uuid, hashID, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the stranded root entirely.
|
||||
if _, err := c.DeleteMetadata(hashID); err != nil {
|
||||
log.Printf("[MigrateDualRoots] DeleteMetadata(%s): %v", hashID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Re-point the alias to the surviving UUID. DeleteMetadata wipes
|
||||
// the cid:* entries for the deleted root, so the alias for this
|
||||
// midhash may have been removed. Restore it pointing at the
|
||||
// canonical UUID so future ResolveRoot calls land correctly.
|
||||
if err := c.AddAlias(uuid, "midhash256:"+hashID); err != nil {
|
||||
log.Printf("[MigrateDualRoots] AddAlias(%s → midhash256:%s): %v", uuid, hashID, err)
|
||||
// Not fatal — data is migrated; the alias mismatch can be
|
||||
// re-fixed by another sweep.
|
||||
}
|
||||
|
||||
fixed++
|
||||
}
|
||||
|
||||
if fixed > 0 {
|
||||
log.Printf("[MigrateDualRoots] Reunited %d stranded midhash roots into their UUID counterparts", fixed)
|
||||
}
|
||||
return fixed, nil
|
||||
}
|
||||
|
||||
// looksLikeULID is a cheap heuristic for "this hashId is opaque internal
|
||||
// UUID, not a CID we should resolve." ULIDs are 26-char Crockford Base32;
|
||||
// CIDs in this codebase are longer and lowercase. Tolerates dashes for
|
||||
// future-proofing.
|
||||
func looksLikeULID(s string) bool {
|
||||
if len(s) != 26 {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
continue
|
||||
}
|
||||
if r >= '0' && r <= '9' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
|
||||
// handler and the startup hook. Default to a generous 5-minute deadline —
|
||||
// the per-entry work is small but the index can be large.
|
||||
// handler and the startup hook.
|
||||
func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
return c.MigrateDualRoots(ctx)
|
||||
return c.MigrateDualRoots(context.Background())
|
||||
}
|
||||
|
||||
// MigrateDualRootsReport returns a human-readable summary of what
|
||||
// MigrateDualRoots would do or did. Useful for the admin endpoint.
|
||||
// MigrateDualRootsReport returns a human-readable summary. With the dual-root
|
||||
// pattern retired (see MigrateDualRoots), this always reports nothing to do.
|
||||
func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
|
||||
fixed, err := c.MigrateDualRoots(ctx)
|
||||
if err != nil {
|
||||
return "", fixed, err
|
||||
}
|
||||
parts := []string{}
|
||||
parts = append(parts, fmt.Sprintf("merged %d stranded midhash roots into UUID roots", fixed))
|
||||
return strings.Join(parts, "; "), fixed, nil
|
||||
return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user