fix cids

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

packages/meta-sort-core/.mocharc.json

@@ -1,5 +1,5 @@
 {
-  "spec": "dist/api/api.test.js",
+  "spec": ["dist/api/api.test.js", "dist/kv/*.test.js"],
   "timeout": 30000,
   "exit": true
 }

packages/meta-sort-core/src/config/TargetHash.ts

@@ -1,21 +1,27 @@
-import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
+import {CidAlgorithm} from "@metazla/meta-hash";
 
 // Target hashes for Piscina worker (full-file hash computation)
 // Note: midhash256 is NOT included here because it's computed in the main thread
 // during light processing, not by the Piscina worker
-export const targetHash = [
+//
-    CID_ALGORITHM_NAMES.sha1,        // For torrent verification
+// These are algorithm *selectors*, not metadata field names. The old
-    CID_ALGORITHM_NAMES.sha256,      // For IPFS compatibility
+// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
-    CID_ALGORITHM_NAMES.md5,         // For compatibility
+// removed `cid_<algo>` record fields kept getting minted in the first place.
-    CID_ALGORITHM_NAMES.sha3_256,    // Additional cryptographic hash
+// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
-    CID_ALGORITHM_NAMES.sha3_384,    // Additional cryptographic hash
+// See METADATA_KEYS.md §2/§14.13.
-    CID_ALGORITHM_NAMES.crc32,       // Fast checksum
+export const targetHash: CidAlgorithm[] = [
-    CID_ALGORITHM_NAMES.btih_v2      // BitTorrent v2 info hash (BEP 52)
+    'sha1',        // For torrent verification
+    'sha256',      // For IPFS compatibility
+    'md5',         // For compatibility
+    'sha3_256',    // Additional cryptographic hash
+    'sha3_384',    // Additional cryptographic hash
+    'crc32',       // Fast checksum
+    'btih_v2',     // BitTorrent v2 info hash (BEP 52)
 ]
 
 // Target hashes for index-cid cache (includes all hashes we want to cache)
 // This includes midhash256 which is computed separately from the worker hashes
-export const targetHashForIndex = [
+export const targetHashForIndex: CidAlgorithm[] = [
     ...targetHash,
-    CID_ALGORITHM_NAMES.midhash256   // Fast hash for file identification (computed in light processing)
+    'midhash256',  // Fast hash for file identification (computed in light processing)
-]
+]

packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts

@@ -15,7 +15,7 @@
  * Reads and stream consumption stay on Redis in PR B; PR C migrates those.
  */
 import {
-    flattenMetadata,
+    buildRecordFields,
     buildFilePrefix,
 } from './MetadataUtils.js';
 
@@ -48,30 +48,12 @@ export class MetaCoreApiWriter {
         metadata: any,
         excludeFields: string[] = []
     ): Promise<void> {
-        // Reuse the same flattening logic the Redis path used so the wire
+        // `buildRecordFields` is the single write-shape boundary, shared with the
-        // payload matches what handler.MergeMetadataFlat expects (flat map
+        // direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
-        // of "a/b/c" → string value).
+        // never a per-algorithm `cid_*` field — which meta-core would now reject
-        const prefix = buildFilePrefix(hashId);
+        // with a 400 anyway.
-        const pairs = flattenMetadata(metadata, prefix, excludeFields);
+        const flat = buildRecordFields(metadata, excludeFields);
-        if (pairs.length === 0) return;
+        if (Object.keys(flat).length === 0) return;
-
-        const flat: Record<string, string> = {};
-        for (const pair of pairs) {
-            // pair.key is "file:<hash>/<field>"; strip the prefix+slash.
-            const field = pair.key.slice(prefix.length + 1);
-            if (!field) continue;
-            // Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
-            // "true"), never as per-algorithm cid_* fields. Rewrite any
-            // legacy cid_* field (e.g. the in-memory cid_midhash256 primary
-            // id) into the key-set form so no deprecated field ever reaches a
-            // record. The CID is self-describing, so the algorithm in the
-            // field name is dropped. See METADATA_KEYS.md §2/§14.13.
-            if (field.startsWith('cid_') && pair.value) {
-                flat[`cids/${pair.value}`] = 'true';
-            } else {
-                flat[field] = pair.value;
-            }
-        }
 
         await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
     }

packages/meta-sort-core/src/kv/MetadataUtils.test.ts

@@ -0,0 +1,86 @@
+import { strict as assert } from 'assert';
+import { buildRecordFields } from './MetadataUtils.js';
+
+/**
+ * `buildRecordFields` is the single boundary where in-memory metadata becomes a
+ * record. It is worth testing precisely because the bug it fixes had no coverage:
+ * meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
+ * fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
+ * The other wrote a literal `cid_midhash256` field, which meta-core stores but
+ * never reverse-indexes — so those records were unresolvable by their own CID,
+ * silently. Both paths now go through this function.
+ */
+describe('buildRecordFields', () => {
+    const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
+    const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
+
+    it('turns cids[] into the bare-CID key-set, not array indices', () => {
+        const flat = buildRecordFields({
+            hashId: MIDHASH,
+            cids: [MIDHASH, SHA256],
+            title: 'Inception',
+        });
+
+        assert.equal(flat[`cids/${MIDHASH}`], 'true');
+        assert.equal(flat[`cids/${SHA256}`], 'true');
+
+        // The trap: the generic flattener would emit `cids/0`, `cids/1` for an
+        // array. meta-core's reverse-index hook only matches `cids/<cid>`, so
+        // those would be stored and never indexed.
+        assert.equal(flat['cids/0'], undefined);
+        assert.equal(flat['cids/1'], undefined);
+    });
+
+    it('never emits a deprecated cid_* or midhash256 field', () => {
+        const flat = buildRecordFields({
+            hashId: MIDHASH,
+            cids: [MIDHASH],
+            title: 'Inception',
+        });
+
+        // meta-core returns 400 for any of these now — a record that carried one
+        // was stored but unindexed.
+        for (const field of Object.keys(flat)) {
+            assert.ok(
+                !field.startsWith('cid_'),
+                `deprecated per-algorithm field reached the record: ${field}`
+            );
+            assert.notEqual(field, 'midhash256');
+            assert.notEqual(field, 'canonical_cid');
+        }
+    });
+
+    it('does not store hashId as a property of its own record', () => {
+        const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
+        // hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
+        assert.equal(flat['hashId'], undefined);
+    });
+
+    it('still flattens ordinary nested metadata', () => {
+        const flat = buildRecordFields({
+            hashId: MIDHASH,
+            cids: [MIDHASH],
+            title: 'Inception',
+            fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
+        });
+
+        assert.equal(flat['title'], 'Inception');
+        assert.equal(flat['fileinfo/duration'], '8878.5');
+        assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
+    });
+
+    it('honours excludeFields', () => {
+        const flat = buildRecordFields(
+            {hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
+            ['processingStatus']
+        );
+        assert.equal(flat['processingStatus'], undefined);
+        assert.equal(flat['title'], 'x');
+    });
+
+    it('tolerates a record with no cids at all', () => {
+        const flat = buildRecordFields({title: 'x'});
+        assert.equal(flat['title'], 'x');
+        assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
+    });
+});

packages/meta-sort-core/src/kv/MetadataUtils.ts

@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
  * //   { key: "/file/abc123/video/codec", value: "h265" }
  * // ]
  */
+/**
+ * Fields that never go onto a record under their own name.
+ *
+ * `cids` is an **array of bare CIDs** and must become the key-set
+ * `cids/<cid> = "true"`. If it fell through the generic flattener below it would
+ * emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
+ * reverse-index hook would not recognise, silently leaving the record
+ * unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
+ *
+ * `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
+ * storing it as a property inside itself is redundant.
+ */
+const STRUCTURAL_FIELDS = ['cids', 'hashId'];
+
+/**
+ * Turn an in-memory `FileMetadata` into the flat `field → value` map that a
+ * record is made of.
+ *
+ * **This is the single write-shape boundary.** Both writers go through it — the
+ * meta-core HTTP writer and the direct-Redis fallback — because they used not
+ * to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
+ * with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
+ * meta-core never indexed. Records looked fine and could not be found by their
+ * own midhash. One function, one shape, no second path to forget.
+ *
+ * Returns field names relative to the record root (no `/file/{hashId}` prefix).
+ */
+export function buildRecordFields(
+  metadata: any,
+  excludeFields: string[] = []
+): Record<string, string> {
+  const prefix = 'r'; // throwaway root; stripped below
+  const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
+
+  const flat: Record<string, string> = {};
+  for (const pair of pairs) {
+    const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
+    if (field) {
+      flat[field] = pair.value;
+    }
+  }
+
+  // Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
+  // The midhash is just the member whose multicodec is 0x1000 — there is no
+  // `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
+  const cids: unknown = metadata?.cids;
+  if (Array.isArray(cids)) {
+    for (const cid of cids) {
+      if (typeof cid === 'string' && cid) {
+        flat[`cids/${cid}`] = 'true';
+      }
+    }
+  }
+
+  return flat;
+}
+
 export function flattenMetadata(
   metadata: any,
   prefix: string,

packages/meta-sort-core/src/kv/RedisClient.ts

@@ -16,7 +16,7 @@ import * as IORedis from 'ioredis';
 import * as os from 'os';
 import type { IKVClient, KeyValuePair } from './IKVClient.js';
 import {
-    flattenMetadata,
+    buildRecordFields,
     reconstructMetadata,
     buildFilePrefix,
     buildPropertyKey,
@@ -331,24 +331,24 @@ export class RedisKVClient implements IKVClient {
             return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
         }
 
-        const prefix = buildFilePrefix(hashId);
+        // Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
-        const pairs = flattenMetadata(metadata, prefix, excludeFields);
+        //
-
+        // This path used to flatten the metadata itself, which meant it skipped
-        if (pairs.length === 0) return;
+        // the `cid_* → cids/` transform that only the HTTP writer applied: it
+        // wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
+        // reverse-index hook only registers aliases for fields named `cids/<cid>`,
+        // so those records were stored but never indexed — unresolvable by their
+        // own midhash, with nothing logged. Both paths now build the record
+        // through the same `buildRecordFields`, so the key-set cannot be skipped.
+        const flat = buildRecordFields(metadata, excludeFields);
+        if (Object.keys(flat).length === 0) return;
 
         // Use pipeline for efficient batch SET operations
         const pipeline = this.redis.pipeline();
-
+        for (const [field, value] of Object.entries(flat)) {
-        for (const pair of pairs) {
+            // Key format: file:{hashId}/{property}
-            // Extract field name by removing the prefix
+            pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
-            const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
-            if (field) {
-                // Key format: file:{hashId}/{property}
-                const redisKey = this.buildKey(`file:${hashId}/${field}`);
-                pipeline.set(redisKey, pair.value);
-            }
         }
-
         await pipeline.exec();
 
         // Maintain index set for getAllHashIds

packages/meta-sort-core/src/logic/WatchedFileProcessor.ts

@@ -282,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
                 );
 
                 let midHash256: string;
-                if (indexLine?.cid_midhash256) {
+                if (indexLine?.midhash256) {
-                    midHash256 = indexLine.cid_midhash256;
+                    midHash256 = indexLine.midhash256;
                     performanceMetrics.recordCacheHit('midhash256');
                 } else {
                     const computeStart = performance.now();
@@ -297,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
                         return;
                     }
                     midHash256 = hash;
-                    performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
+                    performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
                     performanceMetrics.recordCacheMiss('midhash256');
 
                     // Store in cache for future lookups
@@ -305,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
                         filePath,
                         fileStats.size,
                         fileStats.mtime.toISOString(),
-                        { cid_midhash256: midHash256 }
+                        { midhash256: midHash256 }
                     );
                 }
 
-                // STEP 2: Initialize metadata with midhash256
+                // STEP 2: Initialize metadata with midhash256.
+                //
+                // The midhash plays two roles and now says so: it is the record
+                // address (`hashId`, the `/file/{hashId}` key) *and* one member of
+                // the record's bare-CID key-set (`cids`). It used to be a single
+                // `cid_midhash256` property, which conflated the two and leaked a
+                // deprecated field name onto the record. See METADATA_KEYS.md §14.13.
                 const metadata: any = {
-                    cid_midhash256: midHash256,
+                    hashId: midHash256,
+                    cids: [midHash256],
                     filePath,
                     processingStatus: 'processing'
                 };
@@ -441,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
                     const metadata = db.get(filePath);
 
                     // Collect hashId before deletion (for Stremio notification)
-                    if (metadata && metadata.cid_midhash256) {
+                    if (metadata && metadata.hashId) {
-                        removedHashIds.push(metadata.cid_midhash256);
+                        removedHashIds.push(metadata.hashId);
                     }
 
                     db.delete(filePath);

packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts

@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
             let midHash256: string | undefined;
             const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
 
-            if (indexLine && indexLine.cid_midhash256) {
+            if (indexLine && indexLine.midhash256) {
                 // Cache hit! Use cached midhash256 (no network I/O needed)
-                midHash256 = indexLine.cid_midhash256;
+                midHash256 = indexLine.midhash256;
                 performanceMetrics.recordCacheHit('midhash256');
             } else {
                 // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
                 const computeTime = Math.ceil(performance.now() - computeStart);
 
                 // Record midhash256 computation time (excludes cache hits)
-                performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
+                performanceMetrics.recordHashComputation('midhash256', computeTime);
                 performanceMetrics.recordCacheMiss('midhash256');
 
                 // Store in cache for future lookups
-                this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
+                this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
             }
 
-            // Initialize metadata with midhash256
+            // Initialize metadata with midhash256.
+            //
+            // Two roles, now named separately: the record address (`hashId`, the
+            // `/file/{hashId}` key) and a member of the record's bare-CID key-set
+            // (`cids`). The old single `cid_midhash256` property conflated them and
+            // leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
             const metadata: FileMetadata = {
-                cid_midhash256: midHash256,
+                hashId: midHash256,
+                cids: [midHash256],
                 processingStatus: 'processing'
             };
 
@@ -409,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
                 // Check cache or compute locally
                 const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
 
-                if (indexLine && indexLine.cid_midhash256) {
+                if (indexLine && indexLine.midhash256) {
                     // Cache hit! Use cached midhash256 (no network I/O needed)
-                    midHash256 = indexLine.cid_midhash256;
+                    midHash256 = indexLine.midhash256;
                     performanceMetrics.recordCacheHit('midhash256');
                 } else {
                     // Cache miss - compute midhash256 locally (requires file access)
@@ -423,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
                     const computeTime = Math.ceil(performance.now() - computeStart);
 
                     // Record midhash256 computation time (excludes cache hits)
-                    performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
+                    performanceMetrics.recordHashComputation('midhash256', computeTime);
                     performanceMetrics.recordCacheMiss('midhash256');
 
                     // Store in cache for future lookups
-                    this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
+                    this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
                 }
             }
 
-            // Initialize metadata with midhash256
+            // Initialize metadata with midhash256.
+            //
+            // Two roles, now named separately: the record address (`hashId`, the
+            // `/file/{hashId}` key) and a member of the record's bare-CID key-set
+            // (`cids`). The old single `cid_midhash256` property conflated them and
+            // leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
             const metadata: FileMetadata = {
-                cid_midhash256: midHash256,
+                hashId: midHash256,
+                cids: [midHash256],
                 processingStatus: 'processing'
             };
 
@@ -524,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
                 throw new Error('Metadata not found - light processing may have failed');
             }
 
-            const midHash256 = metadata.cid_midhash256;
+            const midHash256 = metadata.hashId;
             if (!midHash256) {
                 throw new Error('midhash256 not found in metadata');
             }
@@ -622,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
         // Delete from KV if configured (using nested key architecture)
         if (this.kvClient && metadata) {
             try {
-                const hashId = metadata.cid_midhash256;
+                const hashId = metadata.hashId;
 
                 // Delete all properties for this file using nested key architecture
                 if (hashId) {

packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts

@@ -1,4 +1,4 @@
-import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
+import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
 import {HashIndexManager} from "@metazla/meta-hash";
 import {config} from "../../config/EnvConfig.js";
 import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
@@ -16,19 +16,34 @@ class SimpleFileToCid{
             if (!stats || !stats.exists) {
                 return null;
             }
-            let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+            const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
-            if(indexLine) {
+            if (indexLine) {
-                return indexLine[CID_ALGORITHM_NAMES.sha256];
+                // The on-disk index keeps per-algorithm columns, keyed by bare
-            }else {
+                // algorithm name (not the removed `cid_<algo>` record fields).
-                let hashs = {};
+                return indexLine['sha256'] ?? null;
-                await globalHashComputer.computeMissingHash(filePath, hashs);
-                globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
-                return hashs[CID_ALGORITHM_NAMES.sha256];
             }
+
+            // The computer returns bare CIDs; the algorithm is recovered from the
+            // multicodec, never from a field name.
+            const hashes: MultiHashData = {};
+            await globalHashComputer.computeMissingHash(filePath, hashes);
+
+            const columns: Partial<Record<CidAlgorithm, string>> = {};
+            for (const algo of targetHash) {
+                const cid = pickCidByAlgorithm(hashes.cids, algo);
+                if (cid) {
+                    columns[algo] = cid;
+                }
+            }
+            if (Object.keys(columns).length > 0) {
+                globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
+            }
+
+            return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
         } catch (e) {
             return null;
         }
     }
 }
 
-export const simpleFileToCid = new SimpleFileToCid();
+export const simpleFileToCid = new SimpleFileToCid();

packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts

@@ -166,14 +166,14 @@ export class StreamingPipeline {
 
             // After light processing, dispatch to plugins and queue background tasks
             const metadata = this.config.fileProcessor.getDatabase().get(filePath);
-            if (metadata && metadata.cid_midhash256) {
+            if (metadata && metadata.hashId) {
                 // Notify Stremio addon to add video
-                this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
+                this.notifyStremioAdd(metadata.hashId).catch(err => {
                     // Silent fail - Stremio addon may not be running
                 });
 
                 // Dispatch tasks to container plugins (fire and forget)
-                this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
+                this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
 
                 // Stage 3: Queue for background processing (SHA-256 computation)
                 this.backgroundQueue.add(() => this.processHashPhase(filePath))
@@ -246,7 +246,7 @@ export class StreamingPipeline {
             // Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
             if (!this.config.containerPluginScheduler) {
                 const metadata = this.config.fileProcessor.getDatabase().get(filePath);
-                const hashId = metadata?.cid_midhash256;
+                const hashId = metadata?.hashId;
                 let virtualPath: string | undefined;
                 try {
                     virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;

packages/meta-sort-core/src/types/ExtendedInterfaces.ts

@@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta {
  */
 export interface IndexManager {
     /**
-     * Get cached CID for a file based on path, size, and mtime
+     * Get cached CID for a file based on path, size, and mtime.
+     *
+     * Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
+     * by bare **algorithm** (`midhash256`, `sha256`), not by the removed
+     * `cid_<algo>` record-field names — the local index and the record shape are
+     * deliberately separate concerns now. See METADATA_KEYS.md §14.13.
      */
     getCidForFile(
         path: string,
         size: number,
         mtime: string
-    ): { cid_midhash256?: string } | undefined;
+    ): { midhash256?: string } | undefined;
 
     /**
      * Add a file CID to the cache
@@ -117,7 +122,7 @@ export interface IndexManager {
         path: string,
         size: number,
         mtime: string,
-        cid: { cid_midhash256: string }
+        cid: { midhash256: string }
     ): void;
 }
 

packages/meta-sort-core/src/types/FileMetadata.ts

@@ -20,12 +20,26 @@ export interface FileMetadata extends HashMeta {
   processingStatus?: ProcessingStatus;
 
   /**
-   * In-memory primary identifier: the file's midhash256 CID, computed in the
+   * The record address: the file's midhash256 CID, computed in the light phase
-   * light phase and used as the record address (hashId) throughout meta-sort.
+   * and used as the `hashId` throughout meta-sort and as the key of the
-   * This is an in-process working field only — it is persisted to the record
+   * `/file/{hashId}` record.
-   * as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
+   *
-   * writer's cid_* → cids/ transform), never as a named `cid_midhash256`
+   * This replaces the old `cid_midhash256` field. The rename is the point: that
-   * field. See METADATA_KEYS.md §1/§14.13.
+   * name was a *metadata property* name, and it leaked — a `cid_*` field written
+   * to a record is stored but never reverse-indexed by meta-core, leaving the
+   * record unresolvable by its own CID. meta-core now rejects such fields with a
+   * 400. See METADATA_KEYS.md §1/§14.13.
    */
-  cid_midhash256?: string;
+  hashId?: string;
+
+  /**
+   * Every digest known for this file, as **bare CIDv1 strings** — the midhash
+   * included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
+   * shape meta-core indexes.
+   *
+   * A CIDv1 is self-describing, so a specific digest is recovered by decoding
+   * the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
+   * field name.
+   */
+  cids?: string[];
 }

packages/meta-sort-editor/src/components/MetadataEditor.tsx

@@ -375,16 +375,32 @@ const CID_LABELS: Record<number, string> = {
   0xd5: 'MD5',
   0x132: 'CRC32',
   0x1000: 'MidHash256',
-  0x10b7: 'BitTorrent v2',
   0xb702: 'BT Pieces Root',
+  0x70: 'IPFS (dag-pb)',
+  0x10b7: 'BitTorrent v2',
   0x1001: 'BitTorrent v1 (file)',
   0x1002: 'BitTorrent v2 (file)',
+  0x1005: 'Usenet release',
+  0x1006: 'URL',
 };
 
+/**
+ * Display order for a record's CIDs. This mirrors the canonical rank ladder
+ * (`packages/meta-core/internal/cid/rank.go`, the two `cid_rank.rs`, and the
+ * `/cid-rank-vectors.json` golden fixture) — same ordering, scaled down.
+ *
+ * It used to be its own invented ladder (5/4/3/2) with no dag-pb entry at all,
+ * which meant an IPFS root fell through to 0 and sorted *below* MD5. Display-only,
+ * so it never broke anything — but it was a seventh, silently divergent copy of a
+ * rule that already has too many. Keep it in step: a code with no entry sorts last.
+ */
 const RANK_BY_LABEL: Record<string, number> = {
+  'IPFS (dag-pb)': 6,
   'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5,
   'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4,
-  MidHash256: 3, 'SHA-1': 2, MD5: 2, CRC32: 2,
+  MidHash256: 3,
+  'Usenet release': 2, URL: 2,
+  'SHA-1': 1, MD5: 1, CRC32: 1,
 };
 
 const B32 = 'abcdefghijklmnopqrstuvwxyz234567';

packages/shared/meta-hash/src/lib/file-id/CidDecode.ts

@@ -0,0 +1,86 @@
+import {CID} from 'multiformats/cid';
+import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
+
+/**
+ * Decoding a bare CIDv1 back into the algorithm that produced it.
+ *
+ * This is **the** TypeScript CID decoder for the platform. It exists because a
+ * bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
+ * "the sha2-256 of this file" has to ask the multicodec, not a string.
+ *
+ * Before this module, meta-dup carried a private hand-rolled base32+uvarint
+ * decoder, the meta-sort editor carried a second one with a *different* rank
+ * ladder, and meta-fuse had none at all (so it could only read the legacy
+ * `cid_midhash256` field, and silently failed on any record that didn't have
+ * one). One implementation, exported, replaces all three.
+ *
+ * ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
+ * has two encoders that disagree about the codec: meta-hash writes
+ * `codec == algorithm code`, while the fullhash plugin and the gateway wrap
+ * every digest with the raw codec `0x55`. The multihash code is the algorithm in
+ * both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
+ */
+
+/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
+export function cidMultihashCode(cid: string): number | undefined {
+    try {
+        return CID.parse(cid).multihash.code;
+    } catch {
+        return undefined;
+    }
+}
+
+/** The content codec of a bare CID, or `undefined` if it won't parse. */
+export function cidCodec(cid: string): number | undefined {
+    try {
+        return CID.parse(cid).code;
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * The first member of `cids` whose multihash code is `mhCode`.
+ *
+ * Takes the raw code so callers can ask for digests meta-hash doesn't itself
+ * compute (e.g. the gateway's per-file btih codes).
+ */
+export function pickCidByMulticodec(
+    cids: readonly string[] | undefined,
+    mhCode: number
+): string | undefined {
+    return cids?.find(cid => cidMultihashCode(cid) === mhCode);
+}
+
+/** The member of `cids` produced by `algo`, if present. */
+export function pickCidByAlgorithm(
+    cids: readonly string[] | undefined,
+    algo: CidAlgorithm
+): string | undefined {
+    return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
+}
+
+/**
+ * Has `algo` already been computed for this cid set?
+ *
+ * The "skip if already computed" check. It used to be an O(1) property lookup
+ * (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
+ * That is a handful of microseconds against a hash computation that streams the
+ * whole file — the trade is not close.
+ */
+export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
+    return pickCidByAlgorithm(cids, algo) !== undefined;
+}
+
+/**
+ * Add `cid` to `cids` unless an equal string is already there.
+ *
+ * Dedup is on the exact string. Two encoders can spell the same digest
+ * differently (codec 0x55 vs codec == mh), and both spellings are legitimate
+ * members — meta-core's reverse index maps each to the same record.
+ */
+export function addCid(cids: string[], cid: string): void {
+    if (cid && !cids.includes(cid)) {
+        cids.push(cid);
+    }
+}

packages/shared/meta-hash/src/lib/file-id/CodeTable.ts

@@ -1,12 +0,0 @@
-import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
-
-export const codeTable = {
-    [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
-    [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
-    [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
-    [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
-    [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
-    [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
-    [CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
-    [CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
-}

packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts

@@ -1,21 +1,20 @@
-import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
 import {create} from "multiformats/hashes/digest";
 import {CID} from 'multiformats/cid';
 import {SimpleHash} from "./SimpleHash";
-import {codeTable} from "@root/file-id/CodeTable";
 
 /**
  * Compute the CIDs of a file using specific algorithms
  * @param stream The Readable stream of the file
  * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
- * @returns Array of CIDs (in the order of the algorithms)
+ * @returns Array of bare CIDv1 strings, in the order of `algorithms`
  */
-export async function computeCIDs({stream, algorithms,createHasher}: {
+export async function computeCIDs({stream, algorithms, createHasher}: {
     stream: ReadableStream<Uint8Array>;
-    algorithms: CID_ALGORITHM_NAMES[];
+    algorithms: CidAlgorithm[];
-    createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
+    createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
 }): Promise<string[]> {
-    const hashers = await hasherDefiner(algorithms,createHasher);
+    const hashers = await hasherDefiner(algorithms, createHasher);
     const reader = stream.getReader();
     while (true) {
         const {done, value} = await reader.read();
@@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
     return cidFinalize(hashers);
 }
 
-async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
+async function hasherDefiner(algorithms: CidAlgorithm[],
-                             createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
+                             createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
     hasher: SimpleHash,
     code: CID_ALGORITHM_CODES
 }[]> {
-    const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
+    const hashers = algorithms
+        .filter(algo => algo in CID_ALGORITHM_CODES)
         .map(async algo => ({
             hasher: await createHasher(algo),
-            code: codeTable[algo]
+            code: CID_ALGORITHM_CODES[algo]
         }));
     return await Promise.all(hashers);
 }
 
+/**
+ * Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
+ * plugin and the gateway instead use the raw codec (0x55) and carry the
+ * algorithm only in the multihash. Both are valid CIDs for the same digest, and
+ * both must decode to the same algorithm — which is why every consumer selects
+ * on the **multihash code**, never the codec. See `CidDecode.ts`.
+ */
 async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
     return await Promise.all(hashers.map(async ({code, hasher}) => {
         const hashBuffer = await hasher.digest();
         const digest = create(code, hashBuffer);
         return CID.createV1(code, digest).toString();
     }));
-}
+}

packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts

@@ -1,31 +1,32 @@
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 import {SimpleHash} from "./SimpleHash";
 import {createHash} from "crypto";
 import {Crc32Hash} from "./Crc32Hash";
 import {BtihV2Hasher} from "./BtihV2Hasher";
 
 /**
- * Compute the CIDs of a file using specific algorithms
+ * Build the stream hasher for one algorithm.
- * @returns Array of CIDs (in the order of the algorithms)
+ *
- * @param algo
+ * `midhash256` is absent on purpose: it is not a stream digest (it hashes the
+ * file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
  */
-export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
+export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
     switch (algo) {
-        case CID_ALGORITHM_NAMES.sha256:
+        case 'sha256':
             return createHash("sha256");
-        case CID_ALGORITHM_NAMES.sha1:
+        case 'sha1':
             return createHash("sha1");
-        case CID_ALGORITHM_NAMES.md5:
+        case 'md5':
             return createHash("md5");
-        case CID_ALGORITHM_NAMES.sha3_256:
+        case 'sha3_256':
             return createHash("sha3-256");
-        case CID_ALGORITHM_NAMES.sha3_384:
+        case 'sha3_384':
             return createHash("sha3-384");
-        case CID_ALGORITHM_NAMES.crc32:
+        case 'crc32':
             return new Crc32Hash();
-        case CID_ALGORITHM_NAMES.btih_v2:
+        case 'btih_v2':
             return new BtihV2Hasher();
         default:
             throw new Error(`Unsupported algorithm: ${algo}`);
     }
-}
+}

packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts

@@ -1,22 +1,27 @@
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 import {SimpleHash} from "./SimpleHash";
 import {Crc32Hash} from "./Crc32Hash";
 import {BrowserHasher} from "@root/file-id/BrowserHasher";
 
-export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
+/**
+ * Browser build. Note it supports fewer algorithms than the node one: there is
+ * no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
+ * either throws.
+ */
+export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
     const hasher = (() => {
         switch (algo) {
-            case CID_ALGORITHM_NAMES.sha256:
+            case 'sha256':
                 return new BrowserHasher('sha-256');
-            case CID_ALGORITHM_NAMES.sha1:
+            case 'sha1':
                 return new BrowserHasher('sha-1');
-            case CID_ALGORITHM_NAMES.md5:
+            case 'md5':
                 return new BrowserHasher('md5');
-            case CID_ALGORITHM_NAMES.sha3_256:
+            case 'sha3_256':
                 return new BrowserHasher('sha3-256');
-            case CID_ALGORITHM_NAMES.sha3_384:
+            case 'sha3_384':
                 return new BrowserHasher('sha3-384');
-            case CID_ALGORITHM_NAMES.crc32:
+            case 'crc32':
                 return new Crc32Hash();
             default:
                 throw new Error(`Unsupported algorithm: ${algo}`);
@@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHas
     }
 
     return hasher;
-}
+}

packages/shared/meta-hash/src/lib/file-id/FileIDComputerInterface.ts

@@ -1,5 +1,5 @@
-import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
 
 export interface FileIDComputerInterface{
-    computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
+    computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
 }

packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts

@@ -1,10 +1,16 @@
 import {Piscina} from "piscina";
-import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {CidAlgorithm} from "../hash-compute/MultiHashData";
 
 export class FileIDComputerWorker{
     private piscina: Piscina;
 
     constructor(workerPath?:string) {
+        // `WORKER_URL` is the escape hatch the tests have always set — it just
+        // wasn't read. Honour it before falling back to path derivation, which
+        // only works for a caller running from `dist/`.
+        if (!workerPath && process.env.WORKER_URL) {
+            workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
+        }
         if(!workerPath) {
             // Construct the URL for the current module
             let distFolder = import.meta.dirname;
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
      * @param algorithms Array of algorithms ('sha256', 'sha1')
      * @returns Array of CIDs (in the order of the algorithms)
      */
-    public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
+    public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
         return this.piscina.run({filePath, algorithms});
     }
 }

packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts

@@ -1,4 +1,5 @@
 import {MultiHashData} from '../hash-compute/MultiHashData';
+import {pickCidByAlgorithm} from './CidDecode';
 import {CID} from 'multiformats/cid';
 
 /**
@@ -22,13 +23,13 @@ export class MagnetLinkGenerator {
     /**
      * Generate BitTorrent v2 magnet link from metadata
      *
-     * @param metadata - File metadata containing cid_btih_v2
+     * @param metadata - File metadata whose `cids` include a btih-v2 CID
      * @param fileName - Original filename for display name
      * @param fileSize - File size in bytes
      * @returns Magnet link string, or null if btih_v2 not available
      *
      * @example
-     * const metadata = {cid_btih_v2: 'bafyb...'};
+     * const metadata = {cids: ['bafyb...']};
      * const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
      * // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
      */
@@ -37,7 +38,9 @@ export class MagnetLinkGenerator {
         fileName: string,
         fileSize: number
     ): string | null {
-        const cidString = metadata.cid_btih_v2;
+        // The btih-v2 CID is recovered from the bare list by its multicodec
+        // (0x10B7), not by a field name. See CidDecode.ts.
+        const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
         if (!cidString) {
             return null; // BitTorrent v2 hash not computed
         }
@@ -178,6 +181,6 @@ export class MagnetLinkGenerator {
         fileName: string,
         fileSize: number
     ): string | null {
-        return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
+        return this.generate({cids: [cidString]}, fileName, fileSize);
     }
 }

packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts

@@ -1,4 +1,4 @@
-import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
 import fs from "fs";
 import {Readable} from 'stream';
 import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
@@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher";
  */
 export default async function computeCIDs({filePath, algorithms}: {
     filePath: string;
-    algorithms: CID_ALGORITHM_NAMES[]
+    algorithms: CidAlgorithm[]
 }): Promise<string[]> {
     const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
     return computeCIDsInternal({stream, algorithms, createHasher});

packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts

@@ -1,30 +1,29 @@
-import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, MultiHashData} from "./MultiHashData";
 
 import {computeCIDs} from "@root/file-id/ComputeHash";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 import {SimpleHash} from "@root/file-id/SimpleHash";
 
-export class HashComputer{
+export class HashComputer {
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[],
+    constructor(private targetHash: CidAlgorithm[],
-                private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
+                private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
     }
 
     async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
+        const cids = (metadata.cids ??= []);
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
 
-        // If all hashes are already computed, skip the processing
+        // Which digests are still missing? On a bare-CID list this costs a decode
+        // per member rather than a property lookup — negligible against streaming
+        // the whole file through a hasher.
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
+        const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
-        const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
+        for (const cid of computed) {
-
+            addCid(cids, cid);
-        // Map the computed CIDs back to their respective metadata properties
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts

@@ -1,32 +1,28 @@
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
 import {Readable} from "stream";
 
 import {computeCIDs} from "@root/file-id/ComputeHash";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 import fs from "fs";
 import {createHasher} from "@root/file-id/CreateHasher";
 
-export class HashComputerFile implements ComputeInterface{
+export class HashComputerFile implements ComputeInterface {
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
+    constructor(private targetHash: CidAlgorithm[]) {
     }
 
     public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
+        const cids = (metadata.cids ??= []);
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
 
-        // If all hashes are already computed, skip the processing
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
         const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
-        const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
+        const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
-
+        for (const cid of computed) {
-        // Map the computed CIDs back to their respective metadata properties
+            addCid(cids, cid);
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts

@@ -1,33 +1,59 @@
 import {HashComputerWorker} from "./HashComputerWorker";
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
 import {HashIndexManager} from "./HashIndexManager";
+import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
 import {stat} from "fs/promises";
 import path from "path";
 
+/**
+ * The translation point between the two shapes.
+ *
+ * `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
+ * persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
+ * **per-algorithm columns**, because its job is the O(1) "already computed?"
+ * check. Neither shape leaks into the other; this class converts.
+ */
 export class HashComputerIndexCache implements ComputeInterface {
     private hashIndexManager: HashIndexManager;
     private hashComputer: HashComputerWorker;
 
-    constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
+    constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
-        this.hashComputer = new HashComputerWorker(targetHash,workerPath);
+        this.hashComputer = new HashComputerWorker(targetHash, workerPath);
-        this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
+        this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
     }
 
     public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
         await this.hashIndexManager.init();
-        let stats = await stat(filePath);
+        const stats = await stat(filePath);
-        if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
+        const cids = (metadata.cids ??= []);
+
+        // Cache hit: lift the cached CIDs into the bare list.
+        if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
             const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
-            if(indexLine) {
+            if (indexLine) {
-                for (const hash of this.targetHash) {
+                for (const algo of this.targetHash) {
-                    if (!metadata[hash] && indexLine[hash]) {
+                    const cached = indexLine[algo];
-                        metadata[hash] = indexLine[hash];
+                    if (cached && !hasAlgorithm(cids, algo)) {
+                        addCid(cids, cached);
                     }
                 }
             }
         }
+
+        // Compute whatever is still missing (a no-op on a full cache hit).
         await this.hashComputer.computeMissingHash(filePath, metadata);
-        this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
+
+        // Write back: project the bare list onto the index's per-algorithm columns.
+        const columns: Partial<Record<CidAlgorithm, string>> = {};
+        for (const algo of this.targetHash) {
+            const cid = pickCidByAlgorithm(cids, algo);
+            if (cid) {
+                columns[algo] = cid;
+            }
+        }
+        if (Object.keys(columns).length > 0) {
+            this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
+        }
     }
 
     async getHashIndexManager(): Promise<HashIndexManager> {

packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts

@@ -1,29 +1,25 @@
 import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
-import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
 
 export class HashComputerWorker implements ComputeInterface {
     private fileIDComputer: FileIDComputerWorker;
 
-    constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
+    constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
         this.fileIDComputer = new FileIDComputerWorker(workerPath);
     }
 
     async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
-        // Dynamically determine which hashes are needed
+        const cids = (metadata.cids ??= []);
-        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
 
-        // If all hashes are already computed, skip the processing
+        const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
         if (neededHashes.length === 0) {
             return;
         }
 
-        // Compute only the needed CIDs
+        const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
-        const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
+        for (const cid of computed) {
-
+            addCid(cids, cid);
-        // Map the computed CIDs back to their respective metadata properties
-        for (const [index, cid] of cids.entries()) {
-            const hashType = neededHashes[index];
-            metadata[hashType] = cid;
         }
     }
 }

packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts

@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
 import {existsAsync} from "../utils/ExistsAsync";
 import path from "path";
 import {clearInterval} from "node:timers";
-import {CID_ALGORITHM_NAMES} from "./MultiHashData";
+import {CidAlgorithm} from "./MultiHashData";
 import {stat} from "fs/promises";
 import {Stats} from "node:fs";
 
-interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
+/**
+ * One row of the on-disk hash-index cache.
+ *
+ * This is the one place a **per-algorithm column** is still the right shape, and
+ * it is not the deprecated one. The index answers "have I already computed sha256
+ * for this file?" on every scan, which a column makes an O(1) lookup and a bare
+ * CID list would make a decode-per-member. It is also strictly local: a wipeable
+ * cache under `/data/cache/hash-index/` (see
+ * `docs/project-architecture/cache-persistence.md`), never a record, never on the
+ * wire.
+ *
+ * What changed is the *naming*: columns are keyed by bare algorithm
+ * (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
+ * so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
+ * shape and the cache shape are now separate concerns that cannot leak into each
+ * other — which is the whole point of the split (METADATA_KEYS.md §14.13).
+ */
+interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
     path: string;
     size: string;
     mtime: string;
@@ -21,15 +38,15 @@ export class HashIndexManager {
     private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
     private intervalId: any;
     private intervalTime: number = 30000;
-    private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
+    private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
-    private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
+    private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
     private indexOpsInProgress: boolean = false;
     private hasChanged: boolean = false;
     private initialLoad: Promise<void>;
-    private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
+    private filePaths: { [key in CidAlgorithm]?: string } = {};
 
     constructor(private indexFolderPath: string,
-                private targetHash: CID_ALGORITHM_NAMES[]) {
+                private targetHash: CidAlgorithm[]) {
     }
 
     hasFileInCache(filePath: string,stats:Stats): boolean {
@@ -81,7 +98,7 @@ export class HashIndexManager {
     }
 
     // Function to check CSV headers
-    private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
+    private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
         const records = parseSync(csvContent, {
             bom: true,
             columns: true,
@@ -116,7 +133,7 @@ export class HashIndexManager {
         this.intervalId = setInterval(() => this.saveCacheToFile(), time);
     }
 
-    public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
+    public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
         if (await existsAsync(this.filePaths[hash])) {
             // check the file size and if it did not change, do not read the file
             const stats = await fs.stat(this.filePaths[hash]);
@@ -147,7 +164,7 @@ export class HashIndexManager {
         return Array.from(this.cache.values());
     }
 
-    private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
+    private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
         if (!(await existsAsync(this.filePaths[hash]))) {
             return [];
         }
@@ -178,10 +195,8 @@ export class HashIndexManager {
 
     public async saveCacheToFile(): Promise<void> {
         if (this.indexOpsInProgress || !this.hasChanged) {
-            console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`);
             return;
         }
-        console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`);
         this.hasChanged = false;
         this.indexOpsInProgress = true;
         const start = performance.now();
@@ -274,17 +289,15 @@ export class HashIndexManager {
         return null;
     }
 
-    public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
+    public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
         if (!filePath || !fileSize || !mtime || !hashs) {
             throw new Error('Invalid parameters');
         }
         // Check that at least one target hash is provided
         const providedHashes = this.targetHash.filter(hash => hashs[hash]);
         if (providedHashes.length === 0) {
-            console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`);
             throw new Error('At least one target hash must be provided');
         }
-        console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`);
         const size = fileSize + "";
         const baseName = path.basename(filePath);
         const cacheKey = `${baseName}-${size}-${mtime}`;

packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts

@@ -1,21 +1,19 @@
-export interface MultiHashData {
-    cid_crc32?: string; //0x0132 CRC32 (SFV)
-    cid_md5?: string; // 0xd5 md5
-    cid_sha1?: string; // 0x11 sha1
-    "cid_sha2-256"?: string;//0x12 sha2-256
-    "cid_sha3-256"?: string; // 0x16 sha3-256
-    "cid_sha3_384"?: string; // 0x16 sha3-256
-    cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
-    cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
-}
-
-export interface ComputeInterface {
-    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
-}
-
 /**
- * according to
+ * Multicodec / multihash codes for every digest meta-hash can compute.
- * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
+ *
+ * Source of truth for these numbers, mirrored in:
+ *   - `packages/meta-core/internal/cid/rank.go`
+ *   - the `cid_rank.rs` in each of meta-search and meta-share
+ *   - `/cid-rank-vectors.json` (the golden fixture that pins them all together)
+ *
+ * ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
+ * `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
+ * the gateway wrap every digest with the **raw** codec `0x55` and put the
+ * algorithm in the multihash code. The multihash code is the algorithm in both
+ * conventions — so anything selecting a CID by algorithm must read the
+ * **multihash code**, never the content codec. See `CidDecode.ts`.
+ *
+ * https://github.com/multiformats/multicodec/blob/master/table.csv
  */
 export enum CID_ALGORITHM_CODES {
     crc32 = 0x0132,
@@ -24,28 +22,55 @@ export enum CID_ALGORITHM_CODES {
     sha256 = 0x12,
     sha3_256 = 0x16,
     sha3_384 = 0x15,
-    midhash256 = 0x1000, // Custom code for midhash256
+    midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
-    btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
+    btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
 }
 
-export enum CID_ALGORITHM_NAMES {
+/**
-    crc32 = 'cid_crc32',
+ * An algorithm *selector*: which digests to compute, and which column of the
-    md5 = 'cid_md5',
+ * local hash-index cache to read.
-    sha1 = 'cid_sha1',
+ *
-    sha256 = 'cid_sha2-256',
+ * This is deliberately **not** a metadata field name. The old
-    sha3_256 = 'cid_sha3-256',
+ * `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
-    sha3_384 = 'cid_sha3-384',
+ * duty — selector, record property name, and CSV column — and the middle duty
-    midhash256 = 'cid_midhash256',
+ * was the deprecated one: those `cid_<algo>` fields were stored on records but
-    btih_v2 = 'cid_btih_v2',
+ * never reverse-indexed by meta-core, leaving records unresolvable by their own
+ * CID. meta-core now rejects them at the write boundary (400). See
+ * METADATA_KEYS.md §2/§14.13.
+ *
+ * The triple duty also rotted on its own: the interface declared `cid_sha3_384`
+ * (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
+ * comment on it named the wrong multicodec. A self-describing CID cannot drift
+ * from its own name that way.
+ */
+export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
+
+/** Every algorithm meta-hash knows how to compute. */
+export const CID_ALGORITHMS: CidAlgorithm[] = [
+    'crc32',
+    'md5',
+    'sha1',
+    'sha256',
+    'sha3_256',
+    'sha3_384',
+    'midhash256',
+    'btih_v2',
+];
+
+/**
+ * The output of a hash computation: **bare CIDv1 strings, nothing else**.
+ *
+ * A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
+ * per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
+ * specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
+ *
+ * This is persisted onto a record as the bare-CID key-set
+ * (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
+ */
+export interface MultiHashData {
+    cids?: string[];
 }
 
-export enum CID_ALGORITHM {
+export interface ComputeInterface {
-    crc32 = 'crc32',
+    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
-    md5 = 'md5',
+}
-    sha1 = 'sha1',
-    sha256 = 'sha2-256',
-    sha3_256 = 'sha3-256',
-    sha3_384 = 'sha3-384',
-    midhash256 = 'midhash256',
-    btih_v2 = 'btih-v2',
-}

packages/shared/meta-hash/src/lib/index.ts

@@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker
 export * from './file-id/FastHash.js'; // Fast-hash computation
 export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
 export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
+export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
+
+// The CID model: algorithm codes, the algorithm selector, and MultiHashData
+// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
+// key-set needs them to recover a specific digest — see CidDecode.
+export * from './hash-compute/MultiHashData.js';
 
 //hash-compute
 export * from './hash-compute/HashComputerIndexCache.js';

packages/shared/meta-hash/src/tests/btihv2.test.ts

@@ -1,6 +1,6 @@
 import {describe, expect, it} from 'vitest';
 import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher';
-import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib";
+import {HashComputerWorker, pickCidByAlgorithm} from "../lib";
 import {readFile} from 'fs/promises';
 import {createHash} from 'crypto';
 import * as process from "process";
@@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
     });
 
     it('should integrate with HashComputerWorker', async () => {
-        const hashComputer = new HashComputerWorker([
+        const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']);
-            CID_ALGORITHM_NAMES.btih_v2,
-            CID_ALGORITHM_NAMES.sha256
-        ]);
 
         const metadata: any = {};
         await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
 
-        console.log('Computed metadata:', metadata);
+        // The digests land in a bare CID list; each is recovered by its
+        // multicodec, not by a field name.
+        const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2');
+        expect(btih).toBeDefined();
+        expect(typeof btih).toBe('string');
+        expect(btih!.startsWith('b')).toBe(true);
 
-        // Verify btih_v2 CID was computed
+        // Should also have SHA-256 for comparison.
-        expect(metadata).toHaveProperty('cid_btih_v2');
+        expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined();
-        expect(typeof metadata.cid_btih_v2).toBe('string');
-        expect(metadata.cid_btih_v2.length).toBeGreaterThan(0);
-
-        // Verify CID format (should start with 'b' for base32 CIDv1)
-        expect(metadata.cid_btih_v2.startsWith('b')).toBe(true);
-
-        // Should also have SHA-256 for comparison
-        expect(metadata).toHaveProperty('cid_sha2-256');
     });
 
     it('should compute piece length correctly', () => {
@@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
     });
 
     it('should produce valid CID format with codec 0x10B7', async () => {
-        const hashComputer = new HashComputerWorker([
+        const hashComputer = new HashComputerWorker(['btih_v2']);
-            CID_ALGORITHM_NAMES.btih_v2
-        ]);
 
         const metadata: any = {};
         await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
 
-        const cid = metadata.cid_btih_v2;
+        const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2');
-        console.log('BitTorrent v2 CID:', cid);
+
+        // The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on
+        // multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7"
+        // assertion this test is named for.
 
         // Verify CID format
         expect(cid).toBeDefined();

packages/shared/meta-hash/src/tests/cid-rank-vectors.json

@@ -0,0 +1,153 @@
+{
+  "$schema": "cid-rank-vectors/v1",
+  "$comment": [
+    "CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
+    "The rank decides which of a record's several CIDs is elected canonical - the cid a",
+    "record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
+    "they cannot share code (independent submodules, no cargo path-dep). These vectors are",
+    "the only thing holding them in sync: every implementation's test suite loads this file.",
+    "",
+    "This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
+    "own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
+    "",
+    "Ranking is by UTILITY for external retrieval, not cryptographic strength.",
+    "Ties break on the lexicographically smaller cid, so the election is deterministic",
+    "across peers and does not churn.",
+    "",
+    "See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
+  ],
+  "ladder": {
+    "40": "dag-pb (IPFS block)",
+    "30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
+    "20": "BitTorrent infohashes - swarm interop",
+    "10": "midhash256 - the record address",
+    "5": "opaque locators (nzb-release, url) - never outrank a real digest",
+    "0": "weak/unknown digests, unparseable input"
+  },
+  "tieBreak": "lexicographically smaller cid wins",
+  "vectors": [
+    {
+      "name": "ipfs-dagpb",
+      "cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 40,
+      "note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
+    },
+    {
+      "name": "sha2-256-raw-codec",
+      "cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 30,
+      "note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
+    },
+    {
+      "name": "sha2-256-codec-eq-mh",
+      "cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
+      "rank": 30,
+      "note": "REAL meta-hash fixture; meta-hash sets codec == mh"
+    },
+    {
+      "name": "sha3-256",
+      "cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "sha3-384",
+      "cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
+      "rank": 30,
+      "note": "REAL meta-hash fixture"
+    },
+    {
+      "name": "btih-v2-codec-eq-mh",
+      "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "meta-hash encoder convention"
+    },
+    {
+      "name": "btih-v2-raw-codec",
+      "cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "fullhash encoder convention - SAME tier, both conventions must match"
+    },
+    {
+      "name": "btih-v1-file",
+      "cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v1"
+    },
+    {
+      "name": "btih-v2-file",
+      "cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 20,
+      "note": "gateway per-file BitTorrent v2"
+    },
+    {
+      "name": "midhash256-codec-eq-mh",
+      "cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10,
+      "note": "the record address; internally meaningful, externally invisible"
+    },
+    {
+      "name": "midhash256-raw-codec",
+      "cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 10
+    },
+    {
+      "name": "nzb-release-locator",
+      "cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
+      "rank": 5,
+      "note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
+    },
+    {
+      "name": "url-locator",
+      "cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
+      "rank": 5,
+      "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
+    },
+    {
+      "name": "sha1",
+      "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "md5",
+      "cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "crc32",
+      "cid": "bagzafmqcathsbnrk",
+      "rank": 0,
+      "note": "REAL meta-hash fixture; weak digest"
+    },
+    {
+      "name": "bt-pieces-root",
+      "cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "torrent-internal merkle root, NOT a file-identity cid"
+    },
+    {
+      "name": "cidv0-rejected",
+      "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
+      "rank": 0,
+      "note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
+    },
+    {
+      "name": "token-form-rejected",
+      "cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "rank": 0,
+      "note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
+    },
+    {
+      "name": "garbage",
+      "cid": "not-a-cid",
+      "rank": 0
+    },
+    {
+      "name": "empty",
+      "cid": "",
+      "rank": 0
+    }
+  ]
+}

packages/shared/meta-hash/src/tests/main.test.ts

@@ -1,8 +1,13 @@
-import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
+import {
+    CidAlgorithm,
+    HashComputerIndexCache,
+    HashComputerWorker,
+    pickCidByAlgorithm,
+} from "../lib";
 
 import {describe, expect, it} from 'vitest';
 import * as process from "process";
-import { mkdir, rm, rmdir } from "fs/promises";
+import {mkdir, rm, stat} from "fs/promises";
 
 process.env.WORKER_URL = "./dist/worker.js";
 
@@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js";
  * Remember to build the worker each time you change the worker code.
  */
 
+const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384'];
+
+/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */
+const FIXTURE_BYTES = 490;
+
+/**
+ * The expected digests of `src/tests/test.txt`, as bare CIDv1 strings.
+ *
+ * Independently reproducible — these are not "whatever the code printed":
+ *
+ *   $ sha256sum src/tests/test.txt
+ *   be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec
+ *   $ md5sum  → c8971ace120d1393b658eabb45d08c88
+ *   $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7
+ *
+ * and each CID below wraps exactly that digest under its multicodec (meta-hash's
+ * encoder sets `codec == multihash code`).
+ *
+ * ⚠ These values were **re-pinned**. The previous ones described a 499-byte
+ * CRLF copy of the fixture, from before `.gitattributes` normalised it to LF —
+ * so on any LF checkout the digests could never match and this suite was red.
+ * The mismatch was invisible because the worker path was also broken, so the
+ * assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the
+ * fixture's bytes ever shift again, the test says *that*, instead of printing
+ * six inscrutable CID diffs.
+ */
+const EXPECTED: Record<string, string> = {
+    crc32: 'bagzafmqcat2hnot3',
+    md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea',
+    sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y',
+    sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q',
+    sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi',
+    sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq',
+};
+
+/**
+ * Every expected digest is present in the bare list, and each is recoverable by
+ * its algorithm **through its multicodec** — there is no field name to look up.
+ * That is the whole point of the `cids: string[]` shape: a CIDv1 is
+ * self-describing, so the record never carries a per-algorithm key that could
+ * drift from the CID it names (METADATA_KEYS.md §14.13).
+ */
+function expectAllDigests(cids: string[] | undefined) {
+    expect(cids).toBeDefined();
+    expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort());
+
+    for (const algo of ALGOS) {
+        expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]);
+    }
+}
+
 describe('add', () => {
-    it('meta data compute', async () => {
+    it('the fixture has the bytes these CIDs were pinned against', async () => {
-        let hashComputer = new HashComputerWorker([
+        const {size} = await stat('./src/tests/test.txt');
-            CID_ALGORITHM_NAMES.sha1,
+        expect(size).toBe(FIXTURE_BYTES);
-            CID_ALGORITHM_NAMES.sha256,
-            CID_ALGORITHM_NAMES.md5,
-            CID_ALGORITHM_NAMES.sha3_256,
-            CID_ALGORITHM_NAMES.crc32,
-            CID_ALGORITHM_NAMES.sha3_384
-        ]);
-        let metadata = {};
-        await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
-        console.log(metadata);
-        /**
-         * test.txt CF20B62A (CRC32)
-         * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
-         * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
-         * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
-         * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
-         */
-        expect(metadata).toEqual({
-            "cid_crc32": "bagzafmqcathsbnrk",
-            "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
-            "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
-            "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
-            "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
-            "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
-        });
     });
-    it('meta data compute with index', async () => {
+
-        await rm("./tmp",{recursive:true});
+    it('meta data compute', async () => {
-        await mkdir("./tmp",{recursive:true});
+        const hashComputer = new HashComputerWorker(ALGOS);
-        let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
+        const metadata = {};
-            CID_ALGORITHM_NAMES.sha1,
-            CID_ALGORITHM_NAMES.sha256,
-            CID_ALGORITHM_NAMES.md5,
-            CID_ALGORITHM_NAMES.sha3_256,
-            CID_ALGORITHM_NAMES.crc32,
-            CID_ALGORITHM_NAMES.sha3_384
-        ]);
-        let metadata = {};
         await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
-        console.log(metadata);
+        expectAllDigests((metadata as { cids?: string[] }).cids);
-        /**
+    });
-         * test.txt CF20B62A (CRC32)
+
-         * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
+    it('meta data compute with index', async () => {
-         * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
+        await rm("./tmp", {recursive: true, force: true});
-         * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
+        await mkdir("./tmp", {recursive: true});
-         * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
+        const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
-         */
+        const metadata = {};
-        expect(metadata).toEqual({
+        await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
-            "cid_crc32": "bagzafmqcathsbnrk",
+        expectAllDigests((metadata as { cids?: string[] }).cids);
-            "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
-            "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
-            "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
-            "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
-            "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
-        });
         await (await hashComputer.getHashIndexManager()).saveCacheToFile();
     });
+
     it('cache', async () => {
-        await mkdir("./tmp",{recursive:true});
+        await mkdir("./tmp", {recursive: true});
-        let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
+        const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
-            CID_ALGORITHM_NAMES.sha1,
+
-            CID_ALGORITHM_NAMES.sha256,
+        // The on-disk index is the one place per-algorithm columns survive — it
-            CID_ALGORITHM_NAMES.md5,
+        // answers "already computed?" in O(1) and never leaves this machine. Note
-            CID_ALGORITHM_NAMES.sha3_256,
+        // the columns are bare algorithm names now (`sha256`), not the removed
-            CID_ALGORITHM_NAMES.crc32,
+        // record-field names (`cid_sha2-256`).
-            CID_ALGORITHM_NAMES.sha3_384
+        const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
-        ]);
-        let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
         delete indexLine['mtime'];
         expect(indexLine).toEqual({
-            "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
+            ...EXPECTED,
-            "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
+            path: "test.txt",
-            "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
+            size: String(FIXTURE_BYTES),
-            "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
-            "cid_crc32": "bagzafmqcathsbnrk",
-            "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
-            "path": "test.txt",
-            "size": "499",
         });
     });
-});
+
+    it('a second pass reuses the cache and recomputes nothing', async () => {
+        const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
+        const metadata = {};
+        await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
+        // Same digests, recovered from the index rather than by re-hashing.
+        expectAllDigests((metadata as { cids?: string[] }).cids);
+    });
+});