fix cids

packages/shared/meta-hash/src/lib/file-id/CidDecode.ts

@@ -0,0 +1,86 @@
+import {CID} from 'multiformats/cid';
+import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
+
+/**
+ * Decoding a bare CIDv1 back into the algorithm that produced it.
+ *
+ * This is **the** TypeScript CID decoder for the platform. It exists because a
+ * bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
+ * "the sha2-256 of this file" has to ask the multicodec, not a string.
+ *
+ * Before this module, meta-dup carried a private hand-rolled base32+uvarint
+ * decoder, the meta-sort editor carried a second one with a *different* rank
+ * ladder, and meta-fuse had none at all (so it could only read the legacy
+ * `cid_midhash256` field, and silently failed on any record that didn't have
+ * one). One implementation, exported, replaces all three.
+ *
+ * ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
+ * has two encoders that disagree about the codec: meta-hash writes
+ * `codec == algorithm code`, while the fullhash plugin and the gateway wrap
+ * every digest with the raw codec `0x55`. The multihash code is the algorithm in
+ * both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
+ */
+
+/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
+export function cidMultihashCode(cid: string): number | undefined {
+    try {
+        return CID.parse(cid).multihash.code;
+    } catch {
+        return undefined;
+    }
+}
+
+/** The content codec of a bare CID, or `undefined` if it won't parse. */
+export function cidCodec(cid: string): number | undefined {
+    try {
+        return CID.parse(cid).code;
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * The first member of `cids` whose multihash code is `mhCode`.
+ *
+ * Takes the raw code so callers can ask for digests meta-hash doesn't itself
+ * compute (e.g. the gateway's per-file btih codes).
+ */
+export function pickCidByMulticodec(
+    cids: readonly string[] | undefined,
+    mhCode: number
+): string | undefined {
+    return cids?.find(cid => cidMultihashCode(cid) === mhCode);
+}
+
+/** The member of `cids` produced by `algo`, if present. */
+export function pickCidByAlgorithm(
+    cids: readonly string[] | undefined,
+    algo: CidAlgorithm
+): string | undefined {
+    return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
+}
+
+/**
+ * Has `algo` already been computed for this cid set?
+ *
+ * The "skip if already computed" check. It used to be an O(1) property lookup
+ * (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
+ * That is a handful of microseconds against a hash computation that streams the
+ * whole file — the trade is not close.
+ */
+export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
+    return pickCidByAlgorithm(cids, algo) !== undefined;
+}
+
+/**
+ * Add `cid` to `cids` unless an equal string is already there.
+ *
+ * Dedup is on the exact string. Two encoders can spell the same digest
+ * differently (codec 0x55 vs codec == mh), and both spellings are legitimate
+ * members — meta-core's reverse index maps each to the same record.
+ */
+export function addCid(cids: string[], cid: string): void {
+    if (cid && !cids.includes(cid)) {
+        cids.push(cid);
+    }
+}

packages/shared/meta-hash/src/lib/file-id/CodeTable.ts

@@ -1,12 +0,0 @@
-import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
-
-export const codeTable = {
-    [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
-    [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
-    [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
-    [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
-    [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
-    [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
-    [CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
-    [CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
-}

packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts

@@ -1,21 +1,20 @@
-import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
 import {create} from "multiformats/hashes/digest";
 import {CID} from 'multiformats/cid';
 import {SimpleHash} from "./SimpleHash";
-import {codeTable} from "@root/file-id/CodeTable";
 
 /**
  * Compute the CIDs of a file using specific algorithms
  * @param stream The Readable stream of the file
  * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
- * @returns Array of CIDs (in the order of the algorithms)
+ * @returns Array of bare CIDv1 strings, in the order of `algorithms`
  */
-export async function computeCIDs({stream, algorithms,createHasher}: {
+export async function computeCIDs({stream, algorithms, createHasher}: {
     stream: ReadableStream<Uint8Array>;
-    algorithms: CID_ALGORITHM_NAMES[];
-    createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
+    algorithms: CidAlgorithm[];
+    createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
 }): Promise<string[]> {
-    const hashers = await hasherDefiner(algorithms,createHasher);
+    const hashers = await hasherDefiner(algorithms, createHasher);
     const reader = stream.getReader();
     while (true) {
         const {done, value} = await reader.read();
@@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
     return cidFinalize(hashers);
 }
 
-async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
-                             createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
+async function hasherDefiner(algorithms: CidAlgorithm[],
+                             createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
     hasher: SimpleHash,
     code: CID_ALGORITHM_CODES
 }[]> {
-    const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
+    const hashers = algorithms
+        .filter(algo => algo in CID_ALGORITHM_CODES)
         .map(async algo => ({
             hasher: await createHasher(algo),
-            code: codeTable[algo]
+            code: CID_ALGORITHM_CODES[algo]
         }));
     return await Promise.all(hashers);
 }
 
+/**
+ * Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
+ * plugin and the gateway instead use the raw codec (0x55) and carry the
+ * algorithm only in the multihash. Both are valid CIDs for the same digest, and
+ * both must decode to the same algorithm — which is why every consumer selects
+ * on the **multihash code**, never the codec. See `CidDecode.ts`.
+ */
 async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
     return await Promise.all(hashers.map(async ({code, hasher}) => {
         const hashBuffer = await hasher.digest();
         const digest = create(code, hashBuffer);
         return CID.createV1(code, digest).toString();
     }));
-}
+}

packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts

@@ -1,31 +1,32 @@
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 import {SimpleHash} from "./SimpleHash";
 import {createHash} from "crypto";
 import {Crc32Hash} from "./Crc32Hash";
 import {BtihV2Hasher} from "./BtihV2Hasher";
 
 /**
- * Compute the CIDs of a file using specific algorithms
- * @returns Array of CIDs (in the order of the algorithms)
- * @param algo
+ * Build the stream hasher for one algorithm.
+ *
+ * `midhash256` is absent on purpose: it is not a stream digest (it hashes the
+ * file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
  */
-export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
+export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
     switch (algo) {
-        case CID_ALGORITHM_NAMES.sha256:
+        case 'sha256':
             return createHash("sha256");
-        case CID_ALGORITHM_NAMES.sha1:
+        case 'sha1':
             return createHash("sha1");
-        case CID_ALGORITHM_NAMES.md5:
+        case 'md5':
             return createHash("md5");
-        case CID_ALGORITHM_NAMES.sha3_256:
+        case 'sha3_256':
             return createHash("sha3-256");
-        case CID_ALGORITHM_NAMES.sha3_384:
+        case 'sha3_384':
             return createHash("sha3-384");
-        case CID_ALGORITHM_NAMES.crc32:
+        case 'crc32':
             return new Crc32Hash();
-        case CID_ALGORITHM_NAMES.btih_v2:
+        case 'btih_v2':
             return new BtihV2Hasher();
         default:
             throw new Error(`Unsupported algorithm: ${algo}`);
     }
-}
+}

packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts

@@ -1,22 +1,27 @@
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 import {SimpleHash} from "./SimpleHash";
 import {Crc32Hash} from "./Crc32Hash";
 import {BrowserHasher} from "@root/file-id/BrowserHasher";
 
-export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
+/**
+ * Browser build. Note it supports fewer algorithms than the node one: there is
+ * no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
+ * either throws.
+ */
+export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
     const hasher = (() => {
         switch (algo) {
-            case CID_ALGORITHM_NAMES.sha256:
+            case 'sha256':
                 return new BrowserHasher('sha-256');
-            case CID_ALGORITHM_NAMES.sha1:
+            case 'sha1':
                 return new BrowserHasher('sha-1');
-            case CID_ALGORITHM_NAMES.md5:
+            case 'md5':
                 return new BrowserHasher('md5');
-            case CID_ALGORITHM_NAMES.sha3_256:
+            case 'sha3_256':
                 return new BrowserHasher('sha3-256');
-            case CID_ALGORITHM_NAMES.sha3_384:
+            case 'sha3_384':
                 return new BrowserHasher('sha3-384');
-            case CID_ALGORITHM_NAMES.crc32:
+            case 'crc32':
                 return new Crc32Hash();
             default:
                 throw new Error(`Unsupported algorithm: ${algo}`);
@@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHas
     }
 
     return hasher;
-}
+}

packages/shared/meta-hash/src/lib/file-id/FileIDComputerInterface.ts

@@ -1,5 +1,5 @@
-import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
 
 export interface FileIDComputerInterface{
-    computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
+    computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
 }

packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts

@@ -1,10 +1,16 @@
 import {Piscina} from "piscina";
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 
 export class FileIDComputerWorker{
     private piscina: Piscina;
 
     constructor(workerPath?:string) {
+        // `WORKER_URL` is the escape hatch the tests have always set — it just
+        // wasn't read. Honour it before falling back to path derivation, which
+        // only works for a caller running from `dist/`.
+        if (!workerPath && process.env.WORKER_URL) {
+            workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
+        }
         if(!workerPath) {
             // Construct the URL for the current module
             let distFolder = import.meta.dirname;
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
      * @param algorithms Array of algorithms ('sha256', 'sha1')
      * @returns Array of CIDs (in the order of the algorithms)
      */
-    public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
+    public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
         return this.piscina.run({filePath, algorithms});
     }
 }

packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts

@@ -1,4 +1,5 @@
 import {MultiHashData} from '../hash-compute/MultiHashData';
+import {pickCidByAlgorithm} from './CidDecode';
 import {CID} from 'multiformats/cid';
 
 /**
@@ -22,13 +23,13 @@ export class MagnetLinkGenerator {
     /**
      * Generate BitTorrent v2 magnet link from metadata
      *
-     * @param metadata - File metadata containing cid_btih_v2
+     * @param metadata - File metadata whose `cids` include a btih-v2 CID
      * @param fileName - Original filename for display name
      * @param fileSize - File size in bytes
      * @returns Magnet link string, or null if btih_v2 not available
      *
      * @example
-     * const metadata = {cid_btih_v2: 'bafyb...'};
+     * const metadata = {cids: ['bafyb...']};
      * const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
      * // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
      */
@@ -37,7 +38,9 @@ export class MagnetLinkGenerator {
         fileName: string,
         fileSize: number
     ): string | null {
-        const cidString = metadata.cid_btih_v2;
+        // The btih-v2 CID is recovered from the bare list by its multicodec
+        // (0x10B7), not by a field name. See CidDecode.ts.
+        const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
         if (!cidString) {
             return null; // BitTorrent v2 hash not computed
         }
@@ -178,6 +181,6 @@ export class MagnetLinkGenerator {
         fileName: string,
         fileSize: number
     ): string | null {
-        return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
+        return this.generate({cids: [cidString]}, fileName, fileSize);
     }
 }

packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts

@@ -1,4 +1,4 @@
-import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
 import fs from "fs";
 import {Readable} from 'stream';
 import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
@@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher";
  */
 export default async function computeCIDs({filePath, algorithms}: {
     filePath: string;
-    algorithms: CID_ALGORITHM_NAMES[]
+    algorithms: CidAlgorithm[]
 }): Promise<string[]> {
     const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
     return computeCIDsInternal({stream, algorithms, createHasher});

packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts

@@ -1,30 +1,29 @@
-import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, MultiHashData} from "./MultiHashData";
 
 import {computeCIDs} from "@root/file-id/ComputeHash";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 import {SimpleHash} from "@root/file-id/SimpleHash";
 
-export class HashComputer{
+export class HashComputer {
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[],
-                private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
+    constructor(private targetHash: CidAlgorithm[],
+                private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
     }
 
     async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+        const cids = (metadata.cids ??= []);
 
-        // If all hashes are already computed, skip the processing
+        // Which digests are still missing? On a bare-CID list this costs a decode
+        // per member rather than a property lookup — negligible against streaming
+        // the whole file through a hasher.
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
-        const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
-
-        // Map the computed CIDs back to their respective metadata properties
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
+        const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
+        for (const cid of computed) {
+            addCid(cids, cid);
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts

@@ -1,32 +1,28 @@
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
 import {Readable} from "stream";
 
 import {computeCIDs} from "@root/file-id/ComputeHash";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 import fs from "fs";
 import {createHasher} from "@root/file-id/CreateHasher";
 
-export class HashComputerFile implements ComputeInterface{
+export class HashComputerFile implements ComputeInterface {
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
+    constructor(private targetHash: CidAlgorithm[]) {
     }
 
     public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+        const cids = (metadata.cids ??= []);
 
-        // If all hashes are already computed, skip the processing
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
         const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
-        const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
-
-        // Map the computed CIDs back to their respective metadata properties
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
+        const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
+        for (const cid of computed) {
+            addCid(cids, cid);
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts

@@ -1,33 +1,59 @@
 import {HashComputerWorker} from "./HashComputerWorker";
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
 import {HashIndexManager} from "./HashIndexManager";
+import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
 import {stat} from "fs/promises";
 import path from "path";
 
+/**
+ * The translation point between the two shapes.
+ *
+ * `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
+ * persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
+ * **per-algorithm columns**, because its job is the O(1) "already computed?"
+ * check. Neither shape leaks into the other; this class converts.
+ */
 export class HashComputerIndexCache implements ComputeInterface {
     private hashIndexManager: HashIndexManager;
     private hashComputer: HashComputerWorker;
 
-    constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
-        this.hashComputer = new HashComputerWorker(targetHash,workerPath);
-        this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
+    constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
+        this.hashComputer = new HashComputerWorker(targetHash, workerPath);
+        this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
     }
 
     public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
         await this.hashIndexManager.init();
-        let stats = await stat(filePath);
-        if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
+        const stats = await stat(filePath);
+        const cids = (metadata.cids ??= []);
+
+        // Cache hit: lift the cached CIDs into the bare list.
+        if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
             const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
-            if(indexLine) {
-                for (const hash of this.targetHash) {
-                    if (!metadata[hash] && indexLine[hash]) {
-                        metadata[hash] = indexLine[hash];
+            if (indexLine) {
+                for (const algo of this.targetHash) {
+                    const cached = indexLine[algo];
+                    if (cached && !hasAlgorithm(cids, algo)) {
+                        addCid(cids, cached);
                     }
                 }
             }
         }
+
+        // Compute whatever is still missing (a no-op on a full cache hit).
         await this.hashComputer.computeMissingHash(filePath, metadata);
-        this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
+
+        // Write back: project the bare list onto the index's per-algorithm columns.
+        const columns: Partial<Record<CidAlgorithm, string>> = {};
+        for (const algo of this.targetHash) {
+            const cid = pickCidByAlgorithm(cids, algo);
+            if (cid) {
+                columns[algo] = cid;
+            }
+        }
+        if (Object.keys(columns).length > 0) {
+            this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
+        }
     }
 
     async getHashIndexManager(): Promise<HashIndexManager> {

packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts

@@ -1,29 +1,25 @@
 import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 
 export class HashComputerWorker implements ComputeInterface {
     private fileIDComputer: FileIDComputerWorker;
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
+    constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
         this.fileIDComputer = new FileIDComputerWorker(workerPath);
     }
 
     async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+        const cids = (metadata.cids ??= []);
 
-        // If all hashes are already computed, skip the processing
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
-        const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
-
-        // Map the computed CIDs back to their respective metadata properties
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
+        const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
+        for (const cid of computed) {
+            addCid(cids, cid);
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts

@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
 import {existsAsync} from "../utils/ExistsAsync";
 import path from "path";
 import {clearInterval} from "node:timers";
-import {CID_ALGORITHM_NAMES} from "./MultiHashData";
+import {CidAlgorithm} from "./MultiHashData";
 import {stat} from "fs/promises";
 import {Stats} from "node:fs";
 
-interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
+/**
+ * One row of the on-disk hash-index cache.
+ *
+ * This is the one place a **per-algorithm column** is still the right shape, and
+ * it is not the deprecated one. The index answers "have I already computed sha256
+ * for this file?" on every scan, which a column makes an O(1) lookup and a bare
+ * CID list would make a decode-per-member. It is also strictly local: a wipeable
+ * cache under `/data/cache/hash-index/` (see
+ * `docs/project-architecture/cache-persistence.md`), never a record, never on the
+ * wire.
+ *
+ * What changed is the *naming*: columns are keyed by bare algorithm
+ * (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
+ * so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
+ * shape and the cache shape are now separate concerns that cannot leak into each
+ * other — which is the whole point of the split (METADATA_KEYS.md §14.13).
+ */
+interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
     path: string;
     size: string;
     mtime: string;
@@ -21,15 +38,15 @@ export class HashIndexManager {
     private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
     private intervalId: any;
     private intervalTime: number = 30000;
-    private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
-    private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
+    private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
+    private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
     private indexOpsInProgress: boolean = false;
     private hasChanged: boolean = false;
     private initialLoad: Promise<void>;
-    private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
+    private filePaths: { [key in CidAlgorithm]?: string } = {};
 
     constructor(private indexFolderPath: string,
-                private targetHash: CID_ALGORITHM_NAMES[]) {
+                private targetHash: CidAlgorithm[]) {
     }
 
     hasFileInCache(filePath: string,stats:Stats): boolean {
@@ -81,7 +98,7 @@ export class HashIndexManager {
     }
 
     // Function to check CSV headers
-    private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
+    private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
         const records = parseSync(csvContent, {
             bom: true,
             columns: true,
@@ -116,7 +133,7 @@ export class HashIndexManager {
         this.intervalId = setInterval(() => this.saveCacheToFile(), time);
     }
 
-    public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
+    public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
         if (await existsAsync(this.filePaths[hash])) {
             // check the file size and if it did not change, do not read the file
             const stats = await fs.stat(this.filePaths[hash]);
@@ -147,7 +164,7 @@ export class HashIndexManager {
         return Array.from(this.cache.values());
     }
 
-    private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
+    private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
         if (!(await existsAsync(this.filePaths[hash]))) {
             return [];
         }
@@ -178,10 +195,8 @@ export class HashIndexManager {
 
     public async saveCacheToFile(): Promise<void> {
         if (this.indexOpsInProgress || !this.hasChanged) {
-            console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`);
             return;
         }
-        console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`);
         this.hasChanged = false;
         this.indexOpsInProgress = true;
         const start = performance.now();
@@ -274,17 +289,15 @@ export class HashIndexManager {
         return null;
     }
 
-    public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
+    public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
         if (!filePath || !fileSize || !mtime || !hashs) {
             throw new Error('Invalid parameters');
         }
         // Check that at least one target hash is provided
         const providedHashes = this.targetHash.filter(hash => hashs[hash]);
         if (providedHashes.length === 0) {
-            console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`);
             throw new Error('At least one target hash must be provided');
         }
-        console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`);
         const size = fileSize + "";
         const baseName = path.basename(filePath);
         const cacheKey = `${baseName}-${size}-${mtime}`;

packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts

@@ -1,21 +1,19 @@
-export interface MultiHashData {
-    cid_crc32?: string; //0x0132 CRC32 (SFV)
-    cid_md5?: string; // 0xd5 md5
-    cid_sha1?: string; // 0x11 sha1
-    "cid_sha2-256"?: string;//0x12 sha2-256
-    "cid_sha3-256"?: string; // 0x16 sha3-256
-    "cid_sha3_384"?: string; // 0x16 sha3-256
-    cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
-    cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
-}
-
-export interface ComputeInterface {
-    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
-}
-
 /**
- * according to
- * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
+ * Multicodec / multihash codes for every digest meta-hash can compute.
+ *
+ * Source of truth for these numbers, mirrored in:
+ *   - `packages/meta-core/internal/cid/rank.go`
+ *   - the `cid_rank.rs` in each of meta-search and meta-share
+ *   - `/cid-rank-vectors.json` (the golden fixture that pins them all together)
+ *
+ * ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
+ * `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
+ * the gateway wrap every digest with the **raw** codec `0x55` and put the
+ * algorithm in the multihash code. The multihash code is the algorithm in both
+ * conventions — so anything selecting a CID by algorithm must read the
+ * **multihash code**, never the content codec. See `CidDecode.ts`.
+ *
+ * https://github.com/multiformats/multicodec/blob/master/table.csv
  */
 export enum CID_ALGORITHM_CODES {
     crc32 = 0x0132,
@@ -24,28 +22,55 @@ export enum CID_ALGORITHM_CODES {
     sha256 = 0x12,
     sha3_256 = 0x16,
     sha3_384 = 0x15,
-    midhash256 = 0x1000, // Custom code for midhash256
-    btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
+    midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
+    btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
 }
 
-export enum CID_ALGORITHM_NAMES {
-    crc32 = 'cid_crc32',
-    md5 = 'cid_md5',
-    sha1 = 'cid_sha1',
-    sha256 = 'cid_sha2-256',
-    sha3_256 = 'cid_sha3-256',
-    sha3_384 = 'cid_sha3-384',
-    midhash256 = 'cid_midhash256',
-    btih_v2 = 'cid_btih_v2',
+/**
+ * An algorithm *selector*: which digests to compute, and which column of the
+ * local hash-index cache to read.
+ *
+ * This is deliberately **not** a metadata field name. The old
+ * `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
+ * duty — selector, record property name, and CSV column — and the middle duty
+ * was the deprecated one: those `cid_<algo>` fields were stored on records but
+ * never reverse-indexed by meta-core, leaving records unresolvable by their own
+ * CID. meta-core now rejects them at the write boundary (400). See
+ * METADATA_KEYS.md §2/§14.13.
+ *
+ * The triple duty also rotted on its own: the interface declared `cid_sha3_384`
+ * (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
+ * comment on it named the wrong multicodec. A self-describing CID cannot drift
+ * from its own name that way.
+ */
+export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
+
+/** Every algorithm meta-hash knows how to compute. */
+export const CID_ALGORITHMS: CidAlgorithm[] = [
+    'crc32',
+    'md5',
+    'sha1',
+    'sha256',
+    'sha3_256',
+    'sha3_384',
+    'midhash256',
+    'btih_v2',
+];
+
+/**
+ * The output of a hash computation: **bare CIDv1 strings, nothing else**.
+ *
+ * A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
+ * per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
+ * specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
+ *
+ * This is persisted onto a record as the bare-CID key-set
+ * (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
+ */
+export interface MultiHashData {
+    cids?: string[];
 }
 
-export enum CID_ALGORITHM {
-    crc32 = 'crc32',
-    md5 = 'md5',
-    sha1 = 'sha1',
-    sha256 = 'sha2-256',
-    sha3_256 = 'sha3-256',
-    sha3_384 = 'sha3-384',
-    midhash256 = 'midhash256',
-    btih_v2 = 'btih-v2',
-}
+export interface ComputeInterface {
+    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+}

packages/shared/meta-hash/src/lib/index.ts

@@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker
 export * from './file-id/FastHash.js'; // Fast-hash computation
 export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
 export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
+export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
+
+// The CID model: algorithm codes, the algorithm selector, and MultiHashData
+// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
+// key-set needs them to recover a specific digest — see CidDecode.
+export * from './hash-compute/MultiHashData.js';
 
 //hash-compute
 export * from './hash-compute/HashComputerIndexCache.js';