metastar/meta-hash
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bug fix

Browse Source
This commit is contained in:
Worph
2025-02-20 16:41:04 +01:00
parent b64bc034c6
commit a7fef61c30
12 changed files with 124 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1 -1
View File
@@ -42,13 +42,13 @@
"typescript": "^5.4.3",
"vitest": "^1.4.0"
},
"packageManager": "[email protected]",
"dependencies": {
"chokidar": "^3.6.0",
"crc-32": "^1.2.2",
"csv-parse": "^5.5.5",
"csv-stringify": "^6.4.6",
"debounce": "^2.0.0",
"hash-wasm": "^4.12.0",
"multiformats": "^13.1.0",
"p-queue": "^8.0.1",
"piscina": "^4.4.0"
src/lib/file-id/BrowserHasher.ts
+49
View File
@@ -0,0 +1,49 @@
// Browser-compatible hash implementation
import {SimpleHash} from "@root/file-id/SimpleHash";
import {createMD5, createSHA1, createSHA256, createSHA3} from "hash-wasm";
export class BrowserHasher implements SimpleHash {
private algorithm: string;
private wasmHasher: any;
private isWasm: boolean;
constructor(algorithm: string) {
this.algorithm = algorithm;
this.isWasm = true;
}
async initialize(): Promise<void> {
if (this.isWasm) {
switch (this.algorithm) {
case 'md5':
this.wasmHasher = await createMD5();
break;
case 'sha-1':
this.wasmHasher = await createSHA1();
break;
case 'sha-256':
this.wasmHasher = await createSHA256();
break;
case 'sha3-256':
this.wasmHasher = await createSHA3(256);
break;
case 'sha3-384':
this.wasmHasher = await createSHA3(384);
break;
default:
throw new Error(`Unsupported algorithm: ${this.algorithm}`);
}
this.wasmHasher.init();
}
}
update(data: Uint8Array): SimpleHash {
this.wasmHasher.update(data);
return this;
}
async digest(): Promise<Uint8Array> {
const hash = await this.wasmHasher.digest('binary');
return new Uint8Array(hash);
}
}
src/lib/file-id/CodeTable.ts
+10
View File
@@ -0,0 +1,10 @@
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
export const codeTable = {
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384
}
src/lib/file-id/ComputeHash.ts
+15 -27
View File
@@ -1,9 +1,8 @@
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {Readable} from "stream";
import {create} from "multiformats/hashes/digest";
import {CID} from 'multiformats/cid';
import {createHasher, codeTable} from "./CreateHasher";
import {SimpleHash} from "./SimpleHash";
import {codeTable} from "@root/file-id/CodeTable";
/**
* Compute the CIDs of a file using specific algorithms
@@ -11,37 +10,26 @@ import {SimpleHash} from "./SimpleHash";
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
* @returns Array of CIDs (in the order of the algorithms)
*/
export async function computeCIDs({stream, algorithms}: {
stream: Readable | ReadableStream;
algorithms: CID_ALGORITHM_NAMES[]
export async function computeCIDs({stream, algorithms,createHasher}: {
stream: ReadableStream<Uint8Array>;
algorithms: CID_ALGORITHM_NAMES[];
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
}): Promise<string[]> {
const hashers = await hasherDefiner(algorithms);
// Check if the environment is Node.js or Browser and handle the stream accordingly
if (stream instanceof Readable) {
// Node.js environment
for await (const chunk of stream) {
for (const item of hashers) {
await item.hasher.update(chunk as Buffer);
}
}
} else {
// Browser environment
const reader = stream.getReader();
while (true) {
const {done, value} = await reader.read();
if (done) break;
// Assuming the hasher can handle Uint8Array directly
for (const item of hashers) {
await item.hasher.update(Buffer.from(value));
}
const hashers = await hasherDefiner(algorithms,createHasher);
const reader = stream.getReader();
while (true) {
const {done, value} = await reader.read();
if (done) break;
// Assuming the hasher can handle Uint8Array directly
for (const item of hashers) {
await item.hasher.update(value);
}
}
return cidFinalize(hashers);
}
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): Promise<{
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
hasher: SimpleHash,
code: CID_ALGORITHM_CODES
}[]> {
src/lib/file-id/CreateHasher.ts
+1 -10
View File
@@ -1,17 +1,8 @@
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {SimpleHash} from "./SimpleHash";
import {createHash} from "crypto";
import {Crc32Hash} from "./Crc32Hash";
export const codeTable = {
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384
}
/**
* Compute the CIDs of a file using specific algorithms
* @returns Array of CIDs (in the order of the algorithms)
src/lib/file-id/CreateHasherBrowser.ts
+31
View File
@@ -0,0 +1,31 @@
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {SimpleHash} from "./SimpleHash";
import {Crc32Hash} from "./Crc32Hash";
import {BrowserHasher} from "@root/file-id/BrowserHasher";
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
const hasher = (() => {
switch (algo) {
case CID_ALGORITHM_NAMES.sha256:
return new BrowserHasher('sha-256');
case CID_ALGORITHM_NAMES.sha1:
return new BrowserHasher('sha-1');
case CID_ALGORITHM_NAMES.md5:
return new BrowserHasher('md5');
case CID_ALGORITHM_NAMES.sha3_256:
return new BrowserHasher('sha3-256');
case CID_ALGORITHM_NAMES.sha3_384:
return new BrowserHasher('sha3-384');
case CID_ALGORITHM_NAMES.crc32:
return new Crc32Hash();
default:
throw new Error(`Unsupported algorithm: ${algo}`);
}
})();
if (hasher instanceof BrowserHasher) {
await hasher.initialize();
}
return hasher;
}
src/lib/file-id/FileIDComputerInterface.ts
+1 -2
View File
@@ -1,6 +1,5 @@
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
import * as Stream from "stream";
export interface FileIDComputerInterface{
computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
}
src/lib/file-id/ShaComputeWorker.ts
+4 -2
View File
@@ -1,6 +1,8 @@
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
import fs from "fs";
import {Readable} from 'stream';
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
import {createHasher} from "@root/file-id/CreateHasher";
/**
* Compute the CIDs of a file using specific algorithms
@@ -12,6 +14,6 @@ export default async function computeCIDs({filePath, algorithms}: {
filePath: string;
algorithms: CID_ALGORITHM_NAMES[]
}): Promise<string[]> {
const stream = fs.createReadStream(filePath);
return computeCIDsInternal({stream, algorithms});
const stream = Readable.toWeb(fs.createReadStream(filePath));
return computeCIDsInternal({stream, algorithms, createHasher});
}
src/lib/file-id/SimpleHash.ts
+2 -2
View File
@@ -1,5 +1,5 @@
export interface SimpleHash {
update(data: Buffer): SimpleHash | Promise<SimpleHash>;
update(data: Uint8Array): SimpleHash | Promise<SimpleHash>;
digest(): Buffer | Promise<Buffer>;
digest(): Uint8Array | Promise<Uint8Array>;
}
src/lib/hash-compute/HashComputer.ts
+5 -4
View File
@@ -1,14 +1,15 @@
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
import {Readable} from "stream";
import {computeCIDs} from "@root/file-id/ComputeHash";
import {SimpleHash} from "@root/file-id/SimpleHash";
export class HashComputer{
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
constructor(private targetHash: CID_ALGORITHM_NAMES[],
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
}
async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void> {
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
// Dynamically determine which hashes are needed
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
@@ -18,7 +19,7 @@ export class HashComputer{
}
// Compute only the needed CIDs
const cids = await computeCIDs({stream, algorithms:neededHashes});
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
// Map the computed CIDs back to their respective metadata properties
for (const [index, cid] of cids.entries()) {
src/lib/hash-compute/HashComputerFile.ts
+3 -2
View File
@@ -3,6 +3,7 @@ import {Readable} from "stream";
import {computeCIDs} from "@root/file-id/ComputeHash";
import fs from "fs";
import {createHasher} from "@root/file-id/CreateHasher";
export class HashComputerFile implements ComputeInterface{
@@ -19,8 +20,8 @@ export class HashComputerFile implements ComputeInterface{
}
// Compute only the needed CIDs
const stream = fs.createReadStream(filePath);
const cids = await computeCIDs({stream, algorithms:neededHashes});
const stream = Readable.toWeb(fs.createReadStream(filePath));
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
// Map the computed CIDs back to their respective metadata properties
for (const [index, cid] of cids.entries()) {
src/lib/index-browser.ts
+1 -1
View File
@@ -1,4 +1,4 @@
export * from './index-interface.js';
export * from './file-id/ComputeHash.js';
export * from './file-id/CreateHasherBrowser.js';
export * from './hash-compute/HashComputer.js';