package storage import ( "context" "fmt" "strconv" "strings" "github.com/metazla/meta-core/internal/cid" "github.com/metazla/meta-core/internal/kv" ) // This file implements the CID-resolution layer that lives on top of the // flat metadata storage in client.go. The short version: // // - Roots are opaque UUIDv7s — file:/ for every property. // The UUID is an internal implementation detail; the mental model is // "N CIDs → one record of keys→metadata". // - Every known CID for a file is registered as a reverse-index entry: // cid: → // and as a member of the per-root bare-CID key-set: // file:/cids/ = "true" (flat key, sentinel value) // - A CID is self-describing: its algorithm is recoverable from the CIDv1 // multicodec (see internal/cid), so there is no per-algorithm field name // and no : token prefix. The canonical externally-visible CID is // *derived by rank* from the key-set on read (GetMetadataDocument), never // stored. // // The record write path (writeFieldsTx) catches any write of a cids/ // field and registers the reverse-index entry in the same transaction, so // writers that just emit the key-set members don't need to know about the // reverse index at all. The helpers here take the caller's *kv.Tx: a write // must never open a second transaction (there is one writer connection). // CIDIndexPrefix is the key prefix for the reverse index. Combined // with a bare CID, it forms the full key: "cid:bafk…". const CIDIndexPrefix = "cid:" // CIDsKeyPrefix is the field-name prefix for the bare-CID key-set. Sibling // CIDs live as flat keys file:/cids/ = "true"; the set IS the // keyset, the value is the sentinel. Read by GetMetadataDocument (hoisted // into the typed CIDs array) and by DeleteRoot/DeleteMetadata (to find the // reverse-index entries that need cleanup). const CIDsKeyPrefix = "cids/" // Duplicate paths — other physical locations whose content hashes to the same // CIDs as a root — live as dup:/ = "" (a key-set). The watcher // writes one when it sees a new path for content that's already indexed. // Mint creates a new root with filePath/sizeByte/mtimeNano and registers it // in the root index. Returns the new UUID. The caller is expected to follow // up with AddAlias for at least one CID — a root without any alias is // reachable only via the index, which works but defeats the point. func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) { var uuid string err := c.update(func(tx *kv.Tx) error { var err error uuid, err = mintTx(tx, filePath, size, mtimeNano) return err }) return uuid, err } func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) { uuid, err := NewUUID() if err != nil { return "", fmt.Errorf("uuid: %w", err) } return uuid, writeFieldsTx(tx, uuid, map[string]string{ "filePath": filePath, "sizeByte": strconv.FormatInt(size, 10), "mtimeNano": strconv.FormatInt(mtimeNano, 10), }) } // AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse // index cid: → uuid and the key-set member cids/ = "true". // Idempotent: registering the same CID twice is a no-op. func (c *Client) AddAlias(uuid, cidStr string) error { if uuid == "" || cidStr == "" { return fmt.Errorf("uuid and cid are required") } return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"}) }) } // addAliasTx registers the reverse-index entry for a cids/ member the // caller is writing (writeFieldsTx writes the member itself). func addAliasTx(tx *kv.Tx, uuid, cidStr string) error { if uuid == "" || cidStr == "" { return fmt.Errorf("uuid and cid are required") } // Self-pointing alias guard. If the caller is trying to register // cid: → (i.e. the root IS the cid value), that's the // dual-root failure mode in disguise — it means somebody wrote // /meta/ without first resolving via ResolveRoot. Refuse the alias // write; leave any existing alias alone. The key-set membership is still // recorded by the caller, so the field-level write succeeds. if uuid == cidStr { return nil } // Guard against the dual-root pattern: if cid: already points at a // DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the // key-set membership keeps the CID discoverable via root → cids. indexKey := CIDIndexPrefix + cidStr existing, _, err := tx.Get(tx.Context(), indexKey) if err != nil { return fmt.Errorf("read existing alias: %w", err) } if existing != "" && existing != uuid { return nil } if existing == uuid { return nil } return tx.Put(indexKey, uuid) } // removeAliasTx is the inverse of addAliasTx: it drops the reverse-index // entry cid: → . The caller deletes the cids/ key-set member // itself (DeleteProperty does both). // // ⚠ Why this exists. A cids/ member write touches two keys — the flat // file:/cids/ and this reverse alias — while the delete path once // only removed the first. A "deleted" CID therefore still resolved: GET // /meta/, ResolveRoot and GetByCID all kept answering with the record. // The asymmetry is not theoretical. meta-share published dag-pb roots computed // over RAR volumes and par2 files onto six records on watch.nsl.sh; removing // the member was expressible, removing the claim was not, so the false // identity survived its own retraction. // // Guarded, not blind: the alias is removed only when it currently points at // THIS uuid. Two records can legitimately race to claim a cid (see the // dual-root guard in addAliasTx, which leaves the first writer's alias in // place); deleting a member from the loser must not unpick the winner's index. func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error { if uuid == "" || cidStr == "" { return nil } indexKey := CIDIndexPrefix + cidStr existing, ok, err := tx.Get(tx.Context(), indexKey) if err != nil { return fmt.Errorf("read existing alias: %w", err) } if !ok || existing != uuid { // Absent, or somebody else owns this alias. Leave it. return nil } _, err = tx.Delete(indexKey) return err } // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor // lookup, an SSE consumer) to the actual storage root key. // // Callers send a bare CID (e.g. the midhash "bafk…" the watcher publishes on // file:events, or any sibling CID). Roots are kept opaque and every CID is // aliased into the reverse index: // // cid:bafk… → 01KSBHM* // // Without resolution, every /meta/ write creates a parallel cid-rooted // entry that the watcher's UUID never sees. // // Behaviour: // - empty hash → empty (caller's problem) // - hash matches a reverse-index entry → return UUID // - otherwise return hash unchanged (legacy / explicit-UUID writes) // // Errors are converted to "no resolution" — if the store is down or the // lookup fails we let the caller proceed with the bare hash and surface the // failure on the actual operation. func (c *Client) ResolveRoot(hash string) string { if hash == "" { return hash } uuid, err := c.GetByCID(hash) if err != nil || uuid == "" { return hash } return uuid } // GetByCID resolves a bare CID to the UUID of its root. Returns ("", nil) // when the CID is unknown; the empty string is the "not found" signal rather // than a typed error so callers can branch without errors.As. func (c *Client) GetByCID(cidStr string) (string, error) { s, err := c.kv() if err != nil { return "", err } if cidStr == "" { return "", nil } ctx, cancel := opCtx() defer cancel() uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr) if err != nil { return "", fmt.Errorf("get cid index: %w", err) } return uuid, nil } // AddDuplicatePath records that path is another physical location whose // content matches uuid. Returns true if path was newly added. The watcher // calls this when it sees a file whose midhash matches an existing root but // at a different filePath. func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) { if uuid == "" || path == "" { return false, nil } added := false err := c.update(func(tx *kv.Tx) error { var err error added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "") return err }) return added, err } // cidsForRoot returns a root's bare-CID key-set members (cids/ prefix // stripped). func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) { prefix := recordPrefix(uuid) + CIDsKeyPrefix rows, err := r.Range(ctx, prefix, "", 0) if err != nil { return nil, err } out := make([]string, 0, len(rows)) for _, row := range rows { out = append(out, row.K[len(prefix):]) } return out, nil } // DeleteRoot removes everything keyed off uuid: every file:/* property // (including the cids/ key-set), every reverse-index entry for those // CIDs, its duplicates, search row and index membership. Idempotent — a uuid // with no remaining keys is a successful no-op. func (c *Client) DeleteRoot(uuid string) error { if uuid == "" { return nil } return c.update(func(tx *kv.Tx) error { _, err := deleteRecordTx(tx, uuid) return err }) } // cidFromKeysetField returns the bare CID encoded in a cids/ field name, // or "" if the field isn't a key-set member. func cidFromKeysetField(field string) string { if !strings.HasPrefix(field, CIDsKeyPrefix) { return "" } return strings.TrimPrefix(field, CIDsKeyPrefix) } // MetadataDocument is the complete state of one file as the public API // returns it: every flat string field (the cids/ members included), // plus the CIDs hoisted into a typed array, the canonical CID derived by // rank, and the duplicates set. Distinct from GetMetadataFlat (which only // returns the raw string keys). type MetadataDocument struct { Flat map[string]string `json:"metadata"` CIDs []string `json:"cids"` Canonical string `json:"canonical_cid,omitempty"` Duplicates []string `json:"duplicates"` } // GetMetadataDocument returns the full document for a root: every string // property under file:/* (cids/ members included in Flat), the // CIDs hoisted into a typed array, the canonical CID computed by rank, and // the duplicates SET. Returns nil if the root has no keys at all (deleted or // never existed). func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { s, err := c.kv() if err != nil { return nil, err } if uuid == "" { return nil, nil } ctx, cancel := opCtx() defer cancel() flat, err := readRecord(ctx, s, uuid) if err != nil { return nil, err } if flat == nil { flat = make(map[string]string) } // Hoist the cids/ key-set members into a typed array and derive the // canonical CID by rank. The members remain in Flat too — that's how // flat-map consumers (meta-share ingest) see them. cids := make([]string, 0) canonical := "" for k := range flat { if !strings.HasPrefix(k, CIDsKeyPrefix) { continue } cidStr := strings.TrimPrefix(k, CIDsKeyPrefix) cids = append(cids, cidStr) if canonical == "" { canonical = cidStr } else { canonical = cid.Better(canonical, cidStr) } } dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0) if err != nil { return nil, fmt.Errorf("read duplicates: %w", err) } dups := make([]string, 0, len(dupRows)) for _, row := range dupRows { dups = append(dups, row.K[len(dupPrefix(uuid)):]) } if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 { return nil, nil } return &MetadataDocument{ Flat: flat, CIDs: cids, Canonical: canonical, Duplicates: dups, }, nil } // RegisterFileTuple records one physical file under its midhash256 CID, the // same way the file watcher does for every file it scans: mint a UUID root // (filePath/sizeByte/mtimeNano) and write `cids/`, whose hook // registers the cid: reverse alias. Content already known under // another path is recorded as a duplicate; the same path just refreshes // size/mtime. // // Returns the root and whether it was newly minted. Shared by the watcher and // POST /api/files/register, which is how a box running with the watcher off // (a meta-watch client core) still gets files it writes under /files/plugin // resolvable by CID. func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) { if midhash == "" { return "", false, fmt.Errorf("midhash is required") } var ( uuid string minted bool ) // One transaction: the lookup and the mint cannot interleave with another // registration of the same content, so two concurrent scans of one file // can no longer mint two roots for it. err := c.update(func(tx *kv.Tx) error { ctx := tx.Context() existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash) if err != nil { return fmt.Errorf("GetByCID(%s): %w", midhash, err) } uuid = existing if uuid == "" { if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil { return fmt.Errorf("Mint(%s): %w", absPath, err) } minted = true if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil { return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err) } return nil } current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath") if err != nil { return err } if current != absPath { if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil { return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err) } return nil } if err := writeFieldsTx(tx, uuid, map[string]string{ "sizeByte": strconv.FormatInt(size, 10), "mtimeNano": strconv.FormatInt(mtimeNano, 10), }); err != nil { return fmt.Errorf("refresh tuple for %s: %w", uuid, err) } return nil }) if err != nil { return "", false, err } return uuid, minted, nil }