From a7fef61c307cbf18af78b24c7a4575848c7a1edb Mon Sep 17 00:00:00 2001 From: Worph Date: Thu, 20 Feb 2025 16:41:04 +0100 Subject: [PATCH] bug fix --- package.json | 2 +- src/lib/file-id/BrowserHasher.ts | 49 ++++++++++++++++++++++ src/lib/file-id/CodeTable.ts | 10 +++++ src/lib/file-id/ComputeHash.ts | 42 +++++++------------ src/lib/file-id/CreateHasher.ts | 11 +---- src/lib/file-id/CreateHasherBrowser.ts | 31 ++++++++++++++ src/lib/file-id/FileIDComputerInterface.ts | 3 +- src/lib/file-id/ShaComputeWorker.ts | 8 ++-- src/lib/file-id/SimpleHash.ts | 4 +- src/lib/hash-compute/HashComputer.ts | 9 ++-- src/lib/hash-compute/HashComputerFile.ts | 5 ++- src/lib/index-browser.ts | 2 +- 12 files changed, 124 insertions(+), 52 deletions(-) create mode 100644 src/lib/file-id/BrowserHasher.ts create mode 100644 src/lib/file-id/CodeTable.ts create mode 100644 src/lib/file-id/CreateHasherBrowser.ts diff --git a/package.json b/package.json index 2855a95..39b4f9d 100644 --- a/package.json +++ b/package.json @@ -42,13 +42,13 @@ "typescript": "^5.4.3", "vitest": "^1.4.0" }, - "packageManager": "yarn@4.1.1", "dependencies": { "chokidar": "^3.6.0", "crc-32": "^1.2.2", "csv-parse": "^5.5.5", "csv-stringify": "^6.4.6", "debounce": "^2.0.0", + "hash-wasm": "^4.12.0", "multiformats": "^13.1.0", "p-queue": "^8.0.1", "piscina": "^4.4.0" diff --git a/src/lib/file-id/BrowserHasher.ts b/src/lib/file-id/BrowserHasher.ts new file mode 100644 index 0000000..c2149bb --- /dev/null +++ b/src/lib/file-id/BrowserHasher.ts @@ -0,0 +1,49 @@ +// Browser-compatible hash implementation +import {SimpleHash} from "@root/file-id/SimpleHash"; +import {createMD5, createSHA1, createSHA256, createSHA3} from "hash-wasm"; + +export class BrowserHasher implements SimpleHash { + private algorithm: string; + private wasmHasher: any; + private isWasm: boolean; + + constructor(algorithm: string) { + this.algorithm = algorithm; + this.isWasm = true; + } + + async initialize(): Promise { + if (this.isWasm) { + switch (this.algorithm) { + case 'md5': + this.wasmHasher = await createMD5(); + break; + case 'sha-1': + this.wasmHasher = await createSHA1(); + break; + case 'sha-256': + this.wasmHasher = await createSHA256(); + break; + case 'sha3-256': + this.wasmHasher = await createSHA3(256); + break; + case 'sha3-384': + this.wasmHasher = await createSHA3(384); + break; + default: + throw new Error(`Unsupported algorithm: ${this.algorithm}`); + } + this.wasmHasher.init(); + } + } + + update(data: Uint8Array): SimpleHash { + this.wasmHasher.update(data); + return this; + } + + async digest(): Promise { + const hash = await this.wasmHasher.digest('binary'); + return new Uint8Array(hash); + } +} \ No newline at end of file diff --git a/src/lib/file-id/CodeTable.ts b/src/lib/file-id/CodeTable.ts new file mode 100644 index 0000000..e0e0885 --- /dev/null +++ b/src/lib/file-id/CodeTable.ts @@ -0,0 +1,10 @@ +import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; + +export const codeTable = { + [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32, + [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5, + [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, + [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, + [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, + [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384 +} \ No newline at end of file diff --git a/src/lib/file-id/ComputeHash.ts b/src/lib/file-id/ComputeHash.ts index 39abb4d..beaaa74 100644 --- a/src/lib/file-id/ComputeHash.ts +++ b/src/lib/file-id/ComputeHash.ts @@ -1,9 +1,8 @@ import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; -import {Readable} from "stream"; import {create} from "multiformats/hashes/digest"; import {CID} from 'multiformats/cid'; -import {createHasher, codeTable} from "./CreateHasher"; import {SimpleHash} from "./SimpleHash"; +import {codeTable} from "@root/file-id/CodeTable"; /** * Compute the CIDs of a file using specific algorithms @@ -11,37 +10,26 @@ import {SimpleHash} from "./SimpleHash"; * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) * @returns Array of CIDs (in the order of the algorithms) */ -export async function computeCIDs({stream, algorithms}: { - stream: Readable | ReadableStream; - algorithms: CID_ALGORITHM_NAMES[] +export async function computeCIDs({stream, algorithms,createHasher}: { + stream: ReadableStream; + algorithms: CID_ALGORITHM_NAMES[]; + createHasher : (algo: CID_ALGORITHM_NAMES) => Promise }): Promise { - const hashers = await hasherDefiner(algorithms); - - // Check if the environment is Node.js or Browser and handle the stream accordingly - if (stream instanceof Readable) { - // Node.js environment - for await (const chunk of stream) { - for (const item of hashers) { - await item.hasher.update(chunk as Buffer); - } - } - } else { - // Browser environment - const reader = stream.getReader(); - while (true) { - const {done, value} = await reader.read(); - if (done) break; - // Assuming the hasher can handle Uint8Array directly - for (const item of hashers) { - await item.hasher.update(Buffer.from(value)); - } + const hashers = await hasherDefiner(algorithms,createHasher); + const reader = stream.getReader(); + while (true) { + const {done, value} = await reader.read(); + if (done) break; + // Assuming the hasher can handle Uint8Array directly + for (const item of hashers) { + await item.hasher.update(value); } } - return cidFinalize(hashers); } -async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): Promise<{ +async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[], + createHasher : (algo: CID_ALGORITHM_NAMES) => Promise): Promise<{ hasher: SimpleHash, code: CID_ALGORITHM_CODES }[]> { diff --git a/src/lib/file-id/CreateHasher.ts b/src/lib/file-id/CreateHasher.ts index 887ad19..743faa3 100644 --- a/src/lib/file-id/CreateHasher.ts +++ b/src/lib/file-id/CreateHasher.ts @@ -1,17 +1,8 @@ -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; import {SimpleHash} from "./SimpleHash"; import {createHash} from "crypto"; import {Crc32Hash} from "./Crc32Hash"; -export const codeTable = { - [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32, - [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5, - [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, - [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, - [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, - [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384 -} - /** * Compute the CIDs of a file using specific algorithms * @returns Array of CIDs (in the order of the algorithms) diff --git a/src/lib/file-id/CreateHasherBrowser.ts b/src/lib/file-id/CreateHasherBrowser.ts new file mode 100644 index 0000000..4ea320b --- /dev/null +++ b/src/lib/file-id/CreateHasherBrowser.ts @@ -0,0 +1,31 @@ +import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {SimpleHash} from "./SimpleHash"; +import {Crc32Hash} from "./Crc32Hash"; +import {BrowserHasher} from "@root/file-id/BrowserHasher"; + +export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { + const hasher = (() => { + switch (algo) { + case CID_ALGORITHM_NAMES.sha256: + return new BrowserHasher('sha-256'); + case CID_ALGORITHM_NAMES.sha1: + return new BrowserHasher('sha-1'); + case CID_ALGORITHM_NAMES.md5: + return new BrowserHasher('md5'); + case CID_ALGORITHM_NAMES.sha3_256: + return new BrowserHasher('sha3-256'); + case CID_ALGORITHM_NAMES.sha3_384: + return new BrowserHasher('sha3-384'); + case CID_ALGORITHM_NAMES.crc32: + return new Crc32Hash(); + default: + throw new Error(`Unsupported algorithm: ${algo}`); + } + })(); + + if (hasher instanceof BrowserHasher) { + await hasher.initialize(); + } + + return hasher; +} \ No newline at end of file diff --git a/src/lib/file-id/FileIDComputerInterface.ts b/src/lib/file-id/FileIDComputerInterface.ts index 7d3a97d..eec3f97 100644 --- a/src/lib/file-id/FileIDComputerInterface.ts +++ b/src/lib/file-id/FileIDComputerInterface.ts @@ -1,6 +1,5 @@ import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; -import * as Stream from "stream"; export interface FileIDComputerInterface{ - computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise; + computeCIDs(filePath: ReadableStream, algorithms: CID_ALGORITHM_NAMES[]): Promise; } \ No newline at end of file diff --git a/src/lib/file-id/ShaComputeWorker.ts b/src/lib/file-id/ShaComputeWorker.ts index 7e3f7ea..4c3bf8e 100644 --- a/src/lib/file-id/ShaComputeWorker.ts +++ b/src/lib/file-id/ShaComputeWorker.ts @@ -1,6 +1,8 @@ import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; import fs from "fs"; +import {Readable} from 'stream'; import {computeCIDs as computeCIDsInternal} from "./ComputeHash"; +import {createHasher} from "@root/file-id/CreateHasher"; /** * Compute the CIDs of a file using specific algorithms @@ -12,6 +14,6 @@ export default async function computeCIDs({filePath, algorithms}: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise { - const stream = fs.createReadStream(filePath); - return computeCIDsInternal({stream, algorithms}); -} + const stream = Readable.toWeb(fs.createReadStream(filePath)); + return computeCIDsInternal({stream, algorithms, createHasher}); +} \ No newline at end of file diff --git a/src/lib/file-id/SimpleHash.ts b/src/lib/file-id/SimpleHash.ts index e33cbfd..3cca912 100644 --- a/src/lib/file-id/SimpleHash.ts +++ b/src/lib/file-id/SimpleHash.ts @@ -1,5 +1,5 @@ export interface SimpleHash { - update(data: Buffer): SimpleHash | Promise; + update(data: Uint8Array): SimpleHash | Promise; - digest(): Buffer | Promise; + digest(): Uint8Array | Promise; } \ No newline at end of file diff --git a/src/lib/hash-compute/HashComputer.ts b/src/lib/hash-compute/HashComputer.ts index b7f7dea..256a216 100644 --- a/src/lib/hash-compute/HashComputer.ts +++ b/src/lib/hash-compute/HashComputer.ts @@ -1,14 +1,15 @@ import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData"; -import {Readable} from "stream"; import {computeCIDs} from "@root/file-id/ComputeHash"; +import {SimpleHash} from "@root/file-id/SimpleHash"; export class HashComputer{ - constructor(private targetHash: CID_ALGORITHM_NAMES[]) { + constructor(private targetHash: CID_ALGORITHM_NAMES[], + private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise) { } - async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise { + async computeMissingHash(stream: ReadableStream, metadata: MultiHashData): Promise { // Dynamically determine which hashes are needed const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); @@ -18,7 +19,7 @@ export class HashComputer{ } // Compute only the needed CIDs - const cids = await computeCIDs({stream, algorithms:neededHashes}); + const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher}); // Map the computed CIDs back to their respective metadata properties for (const [index, cid] of cids.entries()) { diff --git a/src/lib/hash-compute/HashComputerFile.ts b/src/lib/hash-compute/HashComputerFile.ts index 8684b2e..d766e8b 100644 --- a/src/lib/hash-compute/HashComputerFile.ts +++ b/src/lib/hash-compute/HashComputerFile.ts @@ -3,6 +3,7 @@ import {Readable} from "stream"; import {computeCIDs} from "@root/file-id/ComputeHash"; import fs from "fs"; +import {createHasher} from "@root/file-id/CreateHasher"; export class HashComputerFile implements ComputeInterface{ @@ -19,8 +20,8 @@ export class HashComputerFile implements ComputeInterface{ } // Compute only the needed CIDs - const stream = fs.createReadStream(filePath); - const cids = await computeCIDs({stream, algorithms:neededHashes}); + const stream = Readable.toWeb(fs.createReadStream(filePath)); + const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher}); // Map the computed CIDs back to their respective metadata properties for (const [index, cid] of cids.entries()) { diff --git a/src/lib/index-browser.ts b/src/lib/index-browser.ts index ff303d4..12e6f05 100644 --- a/src/lib/index-browser.ts +++ b/src/lib/index-browser.ts @@ -1,4 +1,4 @@ export * from './index-interface.js'; -export * from './file-id/ComputeHash.js'; +export * from './file-id/CreateHasherBrowser.js'; export * from './hash-compute/HashComputer.js'; \ No newline at end of file