From 1eef338f796071e30f5401003d9ace78fd18baea Mon Sep 17 00:00:00 2001 From: Worph Date: Tue, 14 Jul 2026 16:08:36 +0200 Subject: [PATCH] fix cids --- pnpm-workspace.yaml | 2 + src/lib/file-id/CidDecode.ts | 86 +++++++++ src/lib/file-id/CodeTable.ts | 12 -- src/lib/file-id/ComputeHash.ts | 31 ++-- src/lib/file-id/CreateHasher.ts | 27 +-- src/lib/file-id/CreateHasherBrowser.ts | 23 ++- src/lib/file-id/FileIDComputerInterface.ts | 4 +- src/lib/file-id/FileIDComputerWorker.ts | 10 +- src/lib/file-id/MagnetLinkGenerator.ts | 11 +- src/lib/file-id/ShaComputeWorker.ts | 4 +- src/lib/hash-compute/HashComputer.ts | 27 ++- src/lib/hash-compute/HashComputerFile.ts | 22 +-- .../hash-compute/HashComputerIndexCache.ts | 48 +++-- src/lib/hash-compute/HashComputerWorker.ts | 20 +- src/lib/hash-compute/HashIndexManager.ts | 37 +++- src/lib/hash-compute/MultiHashData.ts | 101 ++++++---- src/lib/index.ts | 6 + src/tests/btihv2.test.ts | 37 ++-- src/tests/cid-rank-vectors.json | 153 ++++++++++++++++ src/tests/main.test.ts | 172 ++++++++++-------- tmp/index.csv/index-crc32.csv | 2 + tmp/index.csv/index-md5.csv | 2 + tmp/index.csv/index-sha1.csv | 2 + tmp/index.csv/index-sha256.csv | 2 + tmp/index.csv/index-sha3_256.csv | 2 + tmp/index.csv/index-sha3_384.csv | 2 + 26 files changed, 594 insertions(+), 251 deletions(-) create mode 100644 pnpm-workspace.yaml create mode 100644 src/lib/file-id/CidDecode.ts delete mode 100644 src/lib/file-id/CodeTable.ts create mode 100644 src/tests/cid-rank-vectors.json create mode 100644 tmp/index.csv/index-crc32.csv create mode 100644 tmp/index.csv/index-md5.csv create mode 100644 tmp/index.csv/index-sha1.csv create mode 100644 tmp/index.csv/index-sha256.csv create mode 100644 tmp/index.csv/index-sha3_256.csv create mode 100644 tmp/index.csv/index-sha3_384.csv diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..00f6fc4 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: set this to true or false diff --git a/src/lib/file-id/CidDecode.ts b/src/lib/file-id/CidDecode.ts new file mode 100644 index 0000000..6cfb0d8 --- /dev/null +++ b/src/lib/file-id/CidDecode.ts @@ -0,0 +1,86 @@ +import {CID} from 'multiformats/cid'; +import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData'; + +/** + * Decoding a bare CIDv1 back into the algorithm that produced it. + * + * This is **the** TypeScript CID decoder for the platform. It exists because a + * bare-CID key-set (`cids/`) carries no field names — a consumer that wants + * "the sha2-256 of this file" has to ask the multicodec, not a string. + * + * Before this module, meta-dup carried a private hand-rolled base32+uvarint + * decoder, the meta-sort editor carried a second one with a *different* rank + * ladder, and meta-fuse had none at all (so it could only read the legacy + * `cid_midhash256` field, and silently failed on any record that didn't have + * one). One implementation, exported, replaces all three. + * + * ⚠ **Always match on the multihash code, never the content codec.** MetaMesh + * has two encoders that disagree about the codec: meta-hash writes + * `codec == algorithm code`, while the fullhash plugin and the gateway wrap + * every digest with the raw codec `0x55`. The multihash code is the algorithm in + * both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.) + */ + +/** The multihash function code of a bare CID, or `undefined` if it won't parse. */ +export function cidMultihashCode(cid: string): number | undefined { + try { + return CID.parse(cid).multihash.code; + } catch { + return undefined; + } +} + +/** The content codec of a bare CID, or `undefined` if it won't parse. */ +export function cidCodec(cid: string): number | undefined { + try { + return CID.parse(cid).code; + } catch { + return undefined; + } +} + +/** + * The first member of `cids` whose multihash code is `mhCode`. + * + * Takes the raw code so callers can ask for digests meta-hash doesn't itself + * compute (e.g. the gateway's per-file btih codes). + */ +export function pickCidByMulticodec( + cids: readonly string[] | undefined, + mhCode: number +): string | undefined { + return cids?.find(cid => cidMultihashCode(cid) === mhCode); +} + +/** The member of `cids` produced by `algo`, if present. */ +export function pickCidByAlgorithm( + cids: readonly string[] | undefined, + algo: CidAlgorithm +): string | undefined { + return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]); +} + +/** + * Has `algo` already been computed for this cid set? + * + * The "skip if already computed" check. It used to be an O(1) property lookup + * (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member. + * That is a handful of microseconds against a hash computation that streams the + * whole file — the trade is not close. + */ +export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean { + return pickCidByAlgorithm(cids, algo) !== undefined; +} + +/** + * Add `cid` to `cids` unless an equal string is already there. + * + * Dedup is on the exact string. Two encoders can spell the same digest + * differently (codec 0x55 vs codec == mh), and both spellings are legitimate + * members — meta-core's reverse index maps each to the same record. + */ +export function addCid(cids: string[], cid: string): void { + if (cid && !cids.includes(cid)) { + cids.push(cid); + } +} diff --git a/src/lib/file-id/CodeTable.ts b/src/lib/file-id/CodeTable.ts deleted file mode 100644 index 13ac685..0000000 --- a/src/lib/file-id/CodeTable.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; - -export const codeTable = { - [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32, - [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5, - [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, - [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, - [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, - [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384, - [CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256, - [CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2 -} \ No newline at end of file diff --git a/src/lib/file-id/ComputeHash.ts b/src/lib/file-id/ComputeHash.ts index beaaa74..41a0868 100644 --- a/src/lib/file-id/ComputeHash.ts +++ b/src/lib/file-id/ComputeHash.ts @@ -1,21 +1,20 @@ -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData"; import {create} from "multiformats/hashes/digest"; import {CID} from 'multiformats/cid'; import {SimpleHash} from "./SimpleHash"; -import {codeTable} from "@root/file-id/CodeTable"; /** * Compute the CIDs of a file using specific algorithms * @param stream The Readable stream of the file * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) - * @returns Array of CIDs (in the order of the algorithms) + * @returns Array of bare CIDv1 strings, in the order of `algorithms` */ -export async function computeCIDs({stream, algorithms,createHasher}: { +export async function computeCIDs({stream, algorithms, createHasher}: { stream: ReadableStream; - algorithms: CID_ALGORITHM_NAMES[]; - createHasher : (algo: CID_ALGORITHM_NAMES) => Promise + algorithms: CidAlgorithm[]; + createHasher: (algo: CidAlgorithm) => Promise }): Promise { - const hashers = await hasherDefiner(algorithms,createHasher); + const hashers = await hasherDefiner(algorithms, createHasher); const reader = stream.getReader(); while (true) { const {done, value} = await reader.read(); @@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: { return cidFinalize(hashers); } -async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[], - createHasher : (algo: CID_ALGORITHM_NAMES) => Promise): Promise<{ +async function hasherDefiner(algorithms: CidAlgorithm[], + createHasher: (algo: CidAlgorithm) => Promise): Promise<{ hasher: SimpleHash, code: CID_ALGORITHM_CODES }[]> { - const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo)) + const hashers = algorithms + .filter(algo => algo in CID_ALGORITHM_CODES) .map(async algo => ({ hasher: await createHasher(algo), - code: codeTable[algo] + code: CID_ALGORITHM_CODES[algo] })); return await Promise.all(hashers); } +/** + * Encoder convention: meta-hash sets `codec == multihash code`. The fullhash + * plugin and the gateway instead use the raw codec (0x55) and carry the + * algorithm only in the multihash. Both are valid CIDs for the same digest, and + * both must decode to the same algorithm — which is why every consumer selects + * on the **multihash code**, never the codec. See `CidDecode.ts`. + */ async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise { return await Promise.all(hashers.map(async ({code, hasher}) => { const hashBuffer = await hasher.digest(); const digest = create(code, hashBuffer); return CID.createV1(code, digest).toString(); })); -} \ No newline at end of file +} diff --git a/src/lib/file-id/CreateHasher.ts b/src/lib/file-id/CreateHasher.ts index 978dae8..f379817 100644 --- a/src/lib/file-id/CreateHasher.ts +++ b/src/lib/file-id/CreateHasher.ts @@ -1,31 +1,32 @@ -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; import {SimpleHash} from "./SimpleHash"; import {createHash} from "crypto"; import {Crc32Hash} from "./Crc32Hash"; import {BtihV2Hasher} from "./BtihV2Hasher"; /** - * Compute the CIDs of a file using specific algorithms - * @returns Array of CIDs (in the order of the algorithms) - * @param algo + * Build the stream hasher for one algorithm. + * + * `midhash256` is absent on purpose: it is not a stream digest (it hashes the + * file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead. */ -export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { +export async function createHasher(algo: CidAlgorithm): Promise { switch (algo) { - case CID_ALGORITHM_NAMES.sha256: + case 'sha256': return createHash("sha256"); - case CID_ALGORITHM_NAMES.sha1: + case 'sha1': return createHash("sha1"); - case CID_ALGORITHM_NAMES.md5: + case 'md5': return createHash("md5"); - case CID_ALGORITHM_NAMES.sha3_256: + case 'sha3_256': return createHash("sha3-256"); - case CID_ALGORITHM_NAMES.sha3_384: + case 'sha3_384': return createHash("sha3-384"); - case CID_ALGORITHM_NAMES.crc32: + case 'crc32': return new Crc32Hash(); - case CID_ALGORITHM_NAMES.btih_v2: + case 'btih_v2': return new BtihV2Hasher(); default: throw new Error(`Unsupported algorithm: ${algo}`); } -} \ No newline at end of file +} diff --git a/src/lib/file-id/CreateHasherBrowser.ts b/src/lib/file-id/CreateHasherBrowser.ts index 4ea320b..1a3ca85 100644 --- a/src/lib/file-id/CreateHasherBrowser.ts +++ b/src/lib/file-id/CreateHasherBrowser.ts @@ -1,22 +1,27 @@ -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; import {SimpleHash} from "./SimpleHash"; import {Crc32Hash} from "./Crc32Hash"; import {BrowserHasher} from "@root/file-id/BrowserHasher"; -export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { +/** + * Browser build. Note it supports fewer algorithms than the node one: there is + * no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for + * either throws. + */ +export async function createHasher(algo: CidAlgorithm): Promise { const hasher = (() => { switch (algo) { - case CID_ALGORITHM_NAMES.sha256: + case 'sha256': return new BrowserHasher('sha-256'); - case CID_ALGORITHM_NAMES.sha1: + case 'sha1': return new BrowserHasher('sha-1'); - case CID_ALGORITHM_NAMES.md5: + case 'md5': return new BrowserHasher('md5'); - case CID_ALGORITHM_NAMES.sha3_256: + case 'sha3_256': return new BrowserHasher('sha3-256'); - case CID_ALGORITHM_NAMES.sha3_384: + case 'sha3_384': return new BrowserHasher('sha3-384'); - case CID_ALGORITHM_NAMES.crc32: + case 'crc32': return new Crc32Hash(); default: throw new Error(`Unsupported algorithm: ${algo}`); @@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise, algorithms: CID_ALGORITHM_NAMES[]): Promise; + computeCIDs(filePath: ReadableStream, algorithms: CidAlgorithm[]): Promise; } \ No newline at end of file diff --git a/src/lib/file-id/FileIDComputerWorker.ts b/src/lib/file-id/FileIDComputerWorker.ts index 0ec6ce4..360e439 100644 --- a/src/lib/file-id/FileIDComputerWorker.ts +++ b/src/lib/file-id/FileIDComputerWorker.ts @@ -1,10 +1,16 @@ import {Piscina} from "piscina"; -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; export class FileIDComputerWorker{ private piscina: Piscina; constructor(workerPath?:string) { + // `WORKER_URL` is the escape hatch the tests have always set — it just + // wasn't read. Honour it before falling back to path derivation, which + // only works for a caller running from `dist/`. + if (!workerPath && process.env.WORKER_URL) { + workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href; + } if(!workerPath) { // Construct the URL for the current module let distFolder = import.meta.dirname; @@ -25,7 +31,7 @@ export class FileIDComputerWorker{ * @param algorithms Array of algorithms ('sha256', 'sha1') * @returns Array of CIDs (in the order of the algorithms) */ - public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise { + public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise { return this.piscina.run({filePath, algorithms}); } } diff --git a/src/lib/file-id/MagnetLinkGenerator.ts b/src/lib/file-id/MagnetLinkGenerator.ts index f03a1c4..5466626 100644 --- a/src/lib/file-id/MagnetLinkGenerator.ts +++ b/src/lib/file-id/MagnetLinkGenerator.ts @@ -1,4 +1,5 @@ import {MultiHashData} from '../hash-compute/MultiHashData'; +import {pickCidByAlgorithm} from './CidDecode'; import {CID} from 'multiformats/cid'; /** @@ -22,13 +23,13 @@ export class MagnetLinkGenerator { /** * Generate BitTorrent v2 magnet link from metadata * - * @param metadata - File metadata containing cid_btih_v2 + * @param metadata - File metadata whose `cids` include a btih-v2 CID * @param fileName - Original filename for display name * @param fileSize - File size in bytes * @returns Magnet link string, or null if btih_v2 not available * * @example - * const metadata = {cid_btih_v2: 'bafyb...'}; + * const metadata = {cids: ['bafyb...']}; * const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000); * // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000 */ @@ -37,7 +38,9 @@ export class MagnetLinkGenerator { fileName: string, fileSize: number ): string | null { - const cidString = metadata.cid_btih_v2; + // The btih-v2 CID is recovered from the bare list by its multicodec + // (0x10B7), not by a field name. See CidDecode.ts. + const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2'); if (!cidString) { return null; // BitTorrent v2 hash not computed } @@ -178,6 +181,6 @@ export class MagnetLinkGenerator { fileName: string, fileSize: number ): string | null { - return this.generate({cid_btih_v2: cidString}, fileName, fileSize); + return this.generate({cids: [cidString]}, fileName, fileSize); } } diff --git a/src/lib/file-id/ShaComputeWorker.ts b/src/lib/file-id/ShaComputeWorker.ts index 3e05a49..a4188f1 100644 --- a/src/lib/file-id/ShaComputeWorker.ts +++ b/src/lib/file-id/ShaComputeWorker.ts @@ -1,4 +1,4 @@ -import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; +import {CidAlgorithm} from "@root/hash-compute/MultiHashData"; import fs from "fs"; import {Readable} from 'stream'; import {computeCIDs as computeCIDsInternal} from "./ComputeHash"; @@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher"; */ export default async function computeCIDs({filePath, algorithms}: { filePath: string; - algorithms: CID_ALGORITHM_NAMES[] + algorithms: CidAlgorithm[] }): Promise { const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream; return computeCIDsInternal({stream, algorithms, createHasher}); diff --git a/src/lib/hash-compute/HashComputer.ts b/src/lib/hash-compute/HashComputer.ts index 256a216..456ac74 100644 --- a/src/lib/hash-compute/HashComputer.ts +++ b/src/lib/hash-compute/HashComputer.ts @@ -1,30 +1,29 @@ -import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, MultiHashData} from "./MultiHashData"; import {computeCIDs} from "@root/file-id/ComputeHash"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; import {SimpleHash} from "@root/file-id/SimpleHash"; -export class HashComputer{ +export class HashComputer { - constructor(private targetHash: CID_ALGORITHM_NAMES[], - private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise) { + constructor(private targetHash: CidAlgorithm[], + private createHasher: (algo: CidAlgorithm) => Promise) { } async computeMissingHash(stream: ReadableStream, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + // Which digests are still missing? On a bare-CID list this costs a decode + // per member rather than a property lookup — negligible against streaming + // the whole file through a hasher. + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs - const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher}); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher}); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/src/lib/hash-compute/HashComputerFile.ts b/src/lib/hash-compute/HashComputerFile.ts index 676b7e3..78cca1d 100644 --- a/src/lib/hash-compute/HashComputerFile.ts +++ b/src/lib/hash-compute/HashComputerFile.ts @@ -1,32 +1,28 @@ -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; import {Readable} from "stream"; import {computeCIDs} from "@root/file-id/ComputeHash"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; import fs from "fs"; import {createHasher} from "@root/file-id/CreateHasher"; -export class HashComputerFile implements ComputeInterface{ +export class HashComputerFile implements ComputeInterface { - constructor(private targetHash: CID_ALGORITHM_NAMES[]) { + constructor(private targetHash: CidAlgorithm[]) { } public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream; - const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher}); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher}); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/src/lib/hash-compute/HashComputerIndexCache.ts b/src/lib/hash-compute/HashComputerIndexCache.ts index 00547a7..82b5dca 100644 --- a/src/lib/hash-compute/HashComputerIndexCache.ts +++ b/src/lib/hash-compute/HashComputerIndexCache.ts @@ -1,33 +1,59 @@ import {HashComputerWorker} from "./HashComputerWorker"; -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; import {HashIndexManager} from "./HashIndexManager"; +import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode"; import {stat} from "fs/promises"; import path from "path"; +/** + * The translation point between the two shapes. + * + * `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets + * persisted onto a record as the `cids/` key-set. The on-disk index is + * **per-algorithm columns**, because its job is the O(1) "already computed?" + * check. Neither shape leaks into the other; this class converts. + */ export class HashComputerIndexCache implements ComputeInterface { private hashIndexManager: HashIndexManager; private hashComputer: HashComputerWorker; - constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { - this.hashComputer = new HashComputerWorker(targetHash,workerPath); - this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash); + constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) { + this.hashComputer = new HashComputerWorker(targetHash, workerPath); + this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash); } public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { await this.hashIndexManager.init(); - let stats = await stat(filePath); - if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) { + const stats = await stat(filePath); + const cids = (metadata.cids ??= []); + + // Cache hit: lift the cached CIDs into the bare list. + if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) { const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); - if(indexLine) { - for (const hash of this.targetHash) { - if (!metadata[hash] && indexLine[hash]) { - metadata[hash] = indexLine[hash]; + if (indexLine) { + for (const algo of this.targetHash) { + const cached = indexLine[algo]; + if (cached && !hasAlgorithm(cids, algo)) { + addCid(cids, cached); } } } } + + // Compute whatever is still missing (a no-op on a full cache hit). await this.hashComputer.computeMissingHash(filePath, metadata); - this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata); + + // Write back: project the bare list onto the index's per-algorithm columns. + const columns: Partial> = {}; + for (const algo of this.targetHash) { + const cid = pickCidByAlgorithm(cids, algo); + if (cid) { + columns[algo] = cid; + } + } + if (Object.keys(columns).length > 0) { + this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns); + } } async getHashIndexManager(): Promise { diff --git a/src/lib/hash-compute/HashComputerWorker.ts b/src/lib/hash-compute/HashComputerWorker.ts index 659658a..399f5c9 100644 --- a/src/lib/hash-compute/HashComputerWorker.ts +++ b/src/lib/hash-compute/HashComputerWorker.ts @@ -1,29 +1,25 @@ import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker"; -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; export class HashComputerWorker implements ComputeInterface { private fileIDComputer: FileIDComputerWorker; - constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) { + constructor(private targetHash: CidAlgorithm[], workerPath?: string) { this.fileIDComputer = new FileIDComputerWorker(workerPath); } async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs - const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/src/lib/hash-compute/HashIndexManager.ts b/src/lib/hash-compute/HashIndexManager.ts index 54424bd..0dcb3d4 100644 --- a/src/lib/hash-compute/HashIndexManager.ts +++ b/src/lib/hash-compute/HashIndexManager.ts @@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync'; import {existsAsync} from "../utils/ExistsAsync"; import path from "path"; import {clearInterval} from "node:timers"; -import {CID_ALGORITHM_NAMES} from "./MultiHashData"; +import {CidAlgorithm} from "./MultiHashData"; import {stat} from "fs/promises"; import {Stats} from "node:fs"; -interface IndexLine extends Partial> { +/** + * One row of the on-disk hash-index cache. + * + * This is the one place a **per-algorithm column** is still the right shape, and + * it is not the deprecated one. The index answers "have I already computed sha256 + * for this file?" on every scan, which a column makes an O(1) lookup and a bare + * CID list would make a decode-per-member. It is also strictly local: a wipeable + * cache under `/data/cache/hash-index/` (see + * `docs/project-architecture/cache-persistence.md`), never a record, never on the + * wire. + * + * What changed is the *naming*: columns are keyed by bare algorithm + * (`sha256`, `midhash256`) instead of by the old `cid_` record-field names, + * so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record + * shape and the cache shape are now separate concerns that cannot leak into each + * other — which is the whole point of the split (METADATA_KEYS.md §14.13). + */ +interface IndexLine extends Partial> { path: string; size: string; mtime: string; @@ -21,15 +38,15 @@ export class HashIndexManager { private cache: Map = new Map();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}` private intervalId: any; private intervalTime: number = 30000; - private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read - private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read + private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read + private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read private indexOpsInProgress: boolean = false; private hasChanged: boolean = false; private initialLoad: Promise; - private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {}; + private filePaths: { [key in CidAlgorithm]?: string } = {}; constructor(private indexFolderPath: string, - private targetHash: CID_ALGORITHM_NAMES[]) { + private targetHash: CidAlgorithm[]) { } hasFileInCache(filePath: string,stats:Stats): boolean { @@ -81,7 +98,7 @@ export class HashIndexManager { } // Function to check CSV headers - private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean { + private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean { const records = parseSync(csvContent, { bom: true, columns: true, @@ -116,7 +133,7 @@ export class HashIndexManager { this.intervalId = setInterval(() => this.saveCacheToFile(), time); } - public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise { + public async loadIndex(hash: CidAlgorithm): Promise { if (await existsAsync(this.filePaths[hash])) { // check the file size and if it did not change, do not read the file const stats = await fs.stat(this.filePaths[hash]); @@ -147,7 +164,7 @@ export class HashIndexManager { return Array.from(this.cache.values()); } - private async readCsv(hash: CID_ALGORITHM_NAMES): Promise { + private async readCsv(hash: CidAlgorithm): Promise { if (!(await existsAsync(this.filePaths[hash]))) { return []; } @@ -272,7 +289,7 @@ export class HashIndexManager { return null; } - public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void { + public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void { if (!filePath || !fileSize || !mtime || !hashs) { throw new Error('Invalid parameters'); } diff --git a/src/lib/hash-compute/MultiHashData.ts b/src/lib/hash-compute/MultiHashData.ts index 8f40b78..986fe20 100644 --- a/src/lib/hash-compute/MultiHashData.ts +++ b/src/lib/hash-compute/MultiHashData.ts @@ -1,21 +1,19 @@ -export interface MultiHashData { - cid_crc32?: string; //0x0132 CRC32 (SFV) - cid_md5?: string; // 0xd5 md5 - cid_sha1?: string; // 0x11 sha1 - "cid_sha2-256"?: string;//0x12 sha2-256 - "cid_sha3-256"?: string; // 0x16 sha3-256 - "cid_sha3_384"?: string; // 0x16 sha3-256 - cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format - cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format -} - -export interface ComputeInterface { - computeMissingHash(filePath: string, metadata: MultiHashData): Promise; -} - /** - * according to - * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv + * Multicodec / multihash codes for every digest meta-hash can compute. + * + * Source of truth for these numbers, mirrored in: + * - `packages/meta-core/internal/cid/rank.go` + * - the `cid_rank.rs` in each of meta-search and meta-share + * - `/cid-rank-vectors.json` (the golden fixture that pins them all together) + * + * ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`: + * `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and + * the gateway wrap every digest with the **raw** codec `0x55` and put the + * algorithm in the multihash code. The multihash code is the algorithm in both + * conventions — so anything selecting a CID by algorithm must read the + * **multihash code**, never the content codec. See `CidDecode.ts`. + * + * https://github.com/multiformats/multicodec/blob/master/table.csv */ export enum CID_ALGORITHM_CODES { crc32 = 0x0132, @@ -24,28 +22,55 @@ export enum CID_ALGORITHM_CODES { sha256 = 0x12, sha3_256 = 0x16, sha3_384 = 0x15, - midhash256 = 0x1000, // Custom code for midhash256 - btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash + midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB) + btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52) } -export enum CID_ALGORITHM_NAMES { - crc32 = 'cid_crc32', - md5 = 'cid_md5', - sha1 = 'cid_sha1', - sha256 = 'cid_sha2-256', - sha3_256 = 'cid_sha3-256', - sha3_384 = 'cid_sha3-384', - midhash256 = 'cid_midhash256', - btih_v2 = 'cid_btih_v2', +/** + * An algorithm *selector*: which digests to compute, and which column of the + * local hash-index cache to read. + * + * This is deliberately **not** a metadata field name. The old + * `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple + * duty — selector, record property name, and CSV column — and the middle duty + * was the deprecated one: those `cid_` fields were stored on records but + * never reverse-indexed by meta-core, leaving records unresolvable by their own + * CID. meta-core now rejects them at the write boundary (400). See + * METADATA_KEYS.md §2/§14.13. + * + * The triple duty also rotted on its own: the interface declared `cid_sha3_384` + * (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc + * comment on it named the wrong multicodec. A self-describing CID cannot drift + * from its own name that way. + */ +export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES; + +/** Every algorithm meta-hash knows how to compute. */ +export const CID_ALGORITHMS: CidAlgorithm[] = [ + 'crc32', + 'md5', + 'sha1', + 'sha256', + 'sha3_256', + 'sha3_384', + 'midhash256', + 'btih_v2', +]; + +/** + * The output of a hash computation: **bare CIDv1 strings, nothing else**. + * + * A CIDv1 is self-describing — its algorithm is the multicodec — so there is no + * per-algorithm field name to keep in sync, and no `:` prefix. To find a + * specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`). + * + * This is persisted onto a record as the bare-CID key-set + * (`cids/ = "true"`), which is the only shape meta-core indexes. + */ +export interface MultiHashData { + cids?: string[]; } -export enum CID_ALGORITHM { - crc32 = 'crc32', - md5 = 'md5', - sha1 = 'sha1', - sha256 = 'sha2-256', - sha3_256 = 'sha3-256', - sha3_384 = 'sha3-384', - midhash256 = 'midhash256', - btih_v2 = 'btih-v2', -} \ No newline at end of file +export interface ComputeInterface { + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} diff --git a/src/lib/index.ts b/src/lib/index.ts index 425eebc..1661c71 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker export * from './file-id/FastHash.js'; // Fast-hash computation export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation +export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder) + +// The CID model: algorithm codes, the algorithm selector, and MultiHashData +// (bare `cids: string[]`). Exported because every consumer of a `cids/` +// key-set needs them to recover a specific digest — see CidDecode. +export * from './hash-compute/MultiHashData.js'; //hash-compute export * from './hash-compute/HashComputerIndexCache.js'; diff --git a/src/tests/btihv2.test.ts b/src/tests/btihv2.test.ts index 8a32716..9eed921 100644 --- a/src/tests/btihv2.test.ts +++ b/src/tests/btihv2.test.ts @@ -1,6 +1,6 @@ import {describe, expect, it} from 'vitest'; import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher'; -import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib"; +import {HashComputerWorker, pickCidByAlgorithm} from "../lib"; import {readFile} from 'fs/promises'; import {createHash} from 'crypto'; import * as process from "process"; @@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => { }); it('should integrate with HashComputerWorker', async () => { - const hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.btih_v2, - CID_ALGORITHM_NAMES.sha256 - ]); + const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']); const metadata: any = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log('Computed metadata:', metadata); + // The digests land in a bare CID list; each is recovered by its + // multicodec, not by a field name. + const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2'); + expect(btih).toBeDefined(); + expect(typeof btih).toBe('string'); + expect(btih!.startsWith('b')).toBe(true); - // Verify btih_v2 CID was computed - expect(metadata).toHaveProperty('cid_btih_v2'); - expect(typeof metadata.cid_btih_v2).toBe('string'); - expect(metadata.cid_btih_v2.length).toBeGreaterThan(0); - - // Verify CID format (should start with 'b' for base32 CIDv1) - expect(metadata.cid_btih_v2.startsWith('b')).toBe(true); - - // Should also have SHA-256 for comparison - expect(metadata).toHaveProperty('cid_sha2-256'); + // Should also have SHA-256 for comparison. + expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined(); }); it('should compute piece length correctly', () => { @@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => { }); it('should produce valid CID format with codec 0x10B7', async () => { - const hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.btih_v2 - ]); + const hashComputer = new HashComputerWorker(['btih_v2']); const metadata: any = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - const cid = metadata.cid_btih_v2; - console.log('BitTorrent v2 CID:', cid); + const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2'); + + // The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on + // multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7" + // assertion this test is named for. // Verify CID format expect(cid).toBeDefined(); diff --git a/src/tests/cid-rank-vectors.json b/src/tests/cid-rank-vectors.json new file mode 100644 index 0000000..a772ee7 --- /dev/null +++ b/src/tests/cid-rank-vectors.json @@ -0,0 +1,153 @@ +{ + "$schema": "cid-rank-vectors/v1", + "$comment": [ + "CANONICAL golden vectors for the CID utility rank. THE single source of truth.", + "The rank decides which of a record's several CIDs is elected canonical - the cid a", + "record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;", + "they cannot share code (independent submodules, no cargo path-dep). These vectors are", + "the only thing holding them in sync: every implementation's test suite loads this file.", + "", + "This file is VENDORED into each consuming submodule (submodule CI cannot read above its", + "own tree). scripts/check-cid-vectors.sh diffs every copy against this one.", + "", + "Ranking is by UTILITY for external retrieval, not cryptographic strength.", + "Ties break on the lexicographically smaller cid, so the election is deterministic", + "across peers and does not churn.", + "", + "See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md." + ], + "ladder": { + "40": "dag-pb (IPFS block)", + "30": "sha2-256 / sha3-256 / sha3-384 - real content digests", + "20": "BitTorrent infohashes - swarm interop", + "10": "midhash256 - the record address", + "5": "opaque locators (nzb-release, url) - never outrank a real digest", + "0": "weak/unknown digests, unparseable input" + }, + "tieBreak": "lexicographically smaller cid wins", + "vectors": [ + { + "name": "ipfs-dagpb", + "cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 40, + "note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach" + }, + { + "name": "sha2-256-raw-codec", + "cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 30, + "note": "fullhash/gateway convention: raw codec 0x55, mh names the algo" + }, + { + "name": "sha2-256-codec-eq-mh", + "cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua", + "rank": 30, + "note": "REAL meta-hash fixture; meta-hash sets codec == mh" + }, + { + "name": "sha3-256", + "cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq", + "rank": 30, + "note": "REAL meta-hash fixture" + }, + { + "name": "sha3-384", + "cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga", + "rank": 30, + "note": "REAL meta-hash fixture" + }, + { + "name": "btih-v2-codec-eq-mh", + "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "meta-hash encoder convention" + }, + { + "name": "btih-v2-raw-codec", + "cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "fullhash encoder convention - SAME tier, both conventions must match" + }, + { + "name": "btih-v1-file", + "cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "gateway per-file BitTorrent v1" + }, + { + "name": "btih-v2-file", + "cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "gateway per-file BitTorrent v2" + }, + { + "name": "midhash256-codec-eq-mh", + "cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 10, + "note": "the record address; internally meaningful, externally invisible" + }, + { + "name": "midhash256-raw-codec", + "cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 10 + }, + { + "name": "nzb-release-locator", + "cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab", + "rank": 5, + "note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid." + }, + { + "name": "url-locator", + "cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg", + "rank": 5, + "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release." + }, + { + "name": "sha1", + "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "md5", + "cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "crc32", + "cid": "bagzafmqcathsbnrk", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "bt-pieces-root", + "cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 0, + "note": "torrent-internal merkle root, NOT a file-identity cid" + }, + { + "name": "cidv0-rejected", + "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", + "rank": 0, + "note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix." + }, + { + "name": "token-form-rejected", + "cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 0, + "note": "TRAP: the deprecated : token form. Rust used to strip the prefix and rank it 10. It must now be rejected." + }, + { + "name": "garbage", + "cid": "not-a-cid", + "rank": 0 + }, + { + "name": "empty", + "cid": "", + "rank": 0 + } + ] +} diff --git a/src/tests/main.test.ts b/src/tests/main.test.ts index 944e785..c4e29be 100644 --- a/src/tests/main.test.ts +++ b/src/tests/main.test.ts @@ -1,8 +1,13 @@ -import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib"; +import { + CidAlgorithm, + HashComputerIndexCache, + HashComputerWorker, + pickCidByAlgorithm, +} from "../lib"; import {describe, expect, it} from 'vitest'; import * as process from "process"; -import { mkdir, rm, rmdir } from "fs/promises"; +import {mkdir, rm, stat} from "fs/promises"; process.env.WORKER_URL = "./dist/worker.js"; @@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js"; * Remember to build the worker each time you change the worker code. */ +const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384']; + +/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */ +const FIXTURE_BYTES = 490; + +/** + * The expected digests of `src/tests/test.txt`, as bare CIDv1 strings. + * + * Independently reproducible — these are not "whatever the code printed": + * + * $ sha256sum src/tests/test.txt + * be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec + * $ md5sum → c8971ace120d1393b658eabb45d08c88 + * $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7 + * + * and each CID below wraps exactly that digest under its multicodec (meta-hash's + * encoder sets `codec == multihash code`). + * + * ⚠ These values were **re-pinned**. The previous ones described a 499-byte + * CRLF copy of the fixture, from before `.gitattributes` normalised it to LF — + * so on any LF checkout the digests could never match and this suite was red. + * The mismatch was invisible because the worker path was also broken, so the + * assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the + * fixture's bytes ever shift again, the test says *that*, instead of printing + * six inscrutable CID diffs. + */ +const EXPECTED: Record = { + crc32: 'bagzafmqcat2hnot3', + md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea', + sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y', + sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q', + sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi', + sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq', +}; + +/** + * Every expected digest is present in the bare list, and each is recoverable by + * its algorithm **through its multicodec** — there is no field name to look up. + * That is the whole point of the `cids: string[]` shape: a CIDv1 is + * self-describing, so the record never carries a per-algorithm key that could + * drift from the CID it names (METADATA_KEYS.md §14.13). + */ +function expectAllDigests(cids: string[] | undefined) { + expect(cids).toBeDefined(); + expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort()); + + for (const algo of ALGOS) { + expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]); + } +} + describe('add', () => { - it('meta data compute', async () => { - let hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let metadata = {}; - await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log(metadata); - /** - * test.txt CF20B62A (CRC32) - * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) - * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt - * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt - * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt - */ - expect(metadata).toEqual({ - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' - }); + it('the fixture has the bytes these CIDs were pinned against', async () => { + const {size} = await stat('./src/tests/test.txt'); + expect(size).toBe(FIXTURE_BYTES); }); - it('meta data compute with index', async () => { - await rm("./tmp",{recursive:true}); - await mkdir("./tmp",{recursive:true}); - let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let metadata = {}; + + it('meta data compute', async () => { + const hashComputer = new HashComputerWorker(ALGOS); + const metadata = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log(metadata); - /** - * test.txt CF20B62A (CRC32) - * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) - * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt - * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt - * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt - */ - expect(metadata).toEqual({ - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' - }); + expectAllDigests((metadata as { cids?: string[] }).cids); + }); + + it('meta data compute with index', async () => { + await rm("./tmp", {recursive: true, force: true}); + await mkdir("./tmp", {recursive: true}); + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + const metadata = {}; + await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); + expectAllDigests((metadata as { cids?: string[] }).cids); await (await hashComputer.getHashIndexManager()).saveCacheToFile(); }); + it('cache', async () => { - await mkdir("./tmp",{recursive:true}); - let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt'); + await mkdir("./tmp", {recursive: true}); + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + + // The on-disk index is the one place per-algorithm columns survive — it + // answers "already computed?" in O(1) and never leaves this machine. Note + // the columns are bare algorithm names now (`sha256`), not the removed + // record-field names (`cid_sha2-256`). + const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt'); delete indexLine['mtime']; expect(indexLine).toEqual({ - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga', - "path": "test.txt", - "size": "499", + ...EXPECTED, + path: "test.txt", + size: String(FIXTURE_BYTES), }); }); -}); \ No newline at end of file + + it('a second pass reuses the cache and recomputes nothing', async () => { + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + const metadata = {}; + await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); + // Same digests, recovered from the index rather than by re-hashing. + expectAllDigests((metadata as { cids?: string[] }).cids); + }); +}); diff --git a/tmp/index.csv/index-crc32.csv b/tmp/index.csv/index-crc32.csv new file mode 100644 index 0000000..331993a --- /dev/null +++ b/tmp/index.csv/index-crc32.csv @@ -0,0 +1,2 @@ +path,size,mtime,crc32 +test.txt,490,2025-12-31T11:56:56.014Z,bagzafmqcat2hnot3 diff --git a/tmp/index.csv/index-md5.csv b/tmp/index.csv/index-md5.csv new file mode 100644 index 0000000..7d225ff --- /dev/null +++ b/tmp/index.csv/index-md5.csv @@ -0,0 +1,2 @@ +path,size,mtime,md5 +test.txt,490,2025-12-31T11:56:56.014Z,bahkqdvibcdejogwocigrhe5wldvlwroqrsea diff --git a/tmp/index.csv/index-sha1.csv b/tmp/index.csv/index-sha1.csv new file mode 100644 index 0000000..4054e47 --- /dev/null +++ b/tmp/index.csv/index-sha1.csv @@ -0,0 +1,2 @@ +path,size,mtime,sha1 +test.txt,490,2025-12-31T11:56:56.014Z,baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y diff --git a/tmp/index.csv/index-sha256.csv b/tmp/index.csv/index-sha256.csv new file mode 100644 index 0000000..2945d49 --- /dev/null +++ b/tmp/index.csv/index-sha256.csv @@ -0,0 +1,2 @@ +path,size,mtime,sha256 +test.txt,490,2025-12-31T11:56:56.014Z,baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q diff --git a/tmp/index.csv/index-sha3_256.csv b/tmp/index.csv/index-sha3_256.csv new file mode 100644 index 0000000..881da01 --- /dev/null +++ b/tmp/index.csv/index-sha3_256.csv @@ -0,0 +1,2 @@ +path,size,mtime,sha3_256 +test.txt,490,2025-12-31T11:56:56.014Z,baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi diff --git a/tmp/index.csv/index-sha3_384.csv b/tmp/index.csv/index-sha3_384.csv new file mode 100644 index 0000000..ae8f4e4 --- /dev/null +++ b/tmp/index.csv/index-sha3_384.csv @@ -0,0 +1,2 @@ +path,size,mtime,sha3_384 +test.txt,490,2025-12-31T11:56:56.014Z,baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq