add browser dist

src/lib/Properties.ts

@@ -1,5 +0,0 @@
-
-export interface Properties{
-    TARGET_HASHES:string;
-    INDEX_FILE_PATH:string;
-}

src/lib/file-id/ComputeHash.ts

@@ -0,0 +1,58 @@
+import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {Readable} from "stream";
+import {create} from "multiformats/hashes/digest";
+import {CID} from 'multiformats/cid';
+import {createHasher, codeTable} from "./CreateHasher";
+import {SimpleHash} from "./SimpleHash";
+
+/**
+ * Compute the CIDs of a file using specific algorithms
+ * @param stream The Readable stream of the file
+ * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
+ * @returns Array of CIDs (in the order of the algorithms)
+ */
+export async function computeCIDs({stream, algorithms}: {
+    stream: Readable | ReadableStream;
+    algorithms: CID_ALGORITHM_NAMES[]
+}): Promise<string[]> {
+    const hashers = hasherDefiner(algorithms);
+
+    // Check if the environment is Node.js or Browser and handle the stream accordingly
+    if (stream instanceof Readable) {
+        // Node.js environment
+        for await (const chunk of stream) {
+            hashers.forEach(item => item.hasher.update(chunk as Buffer));
+        }
+    } else {
+        // Browser environment
+        const reader = stream.getReader();
+        while (true) {
+            const {done, value} = await reader.read();
+            if (done) break;
+            // Assuming the hasher can handle Uint8Array directly
+            hashers.forEach(item => item.hasher.update(Buffer.from(value)));
+        }
+    }
+
+    return cidFinalize(hashers);
+}
+
+function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): {
+    hasher: SimpleHash,
+    code: CID_ALGORITHM_CODES
+}[] {
+    const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
+        .map(algo => ({
+            hasher: createHasher(algo),
+            code: codeTable[algo]
+        }));
+    return hashers;
+}
+
+async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
+    return await Promise.all(hashers.map(async ({code, hasher}) => {
+        const hashBuffer = hasher.digest();
+        const digest = create(code, hashBuffer);
+        return CID.createV1(code, digest).toString();
+    }));
+}

src/lib/file-id/Crc32Hash.ts

@@ -0,0 +1,18 @@
+import crc32 from "crc-32";
+import {SimpleHash} from "@root/file-id/SimpleHash";
+
+export class Crc32Hash implements SimpleHash {
+    private _crc: number = undefined;
+
+    update(data: Buffer): Crc32Hash {
+        // Update the CRC-32 checksum with the new chunk of data
+        this._crc = crc32.buf(data, this._crc);
+        return this;
+    }
+
+    digest(): Buffer {
+        const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
+        buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
+        return buffer;
+    }
+}

src/lib/file-id/CreateHasher.ts

@@ -0,0 +1,39 @@
+import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {Readable} from "stream";
+import {ReadableStream} from "stream/web";
+import {SimpleHash} from "./SimpleHash";
+import {createHash} from "crypto";
+import {Crc32Hash} from "./Crc32Hash";
+
+export const codeTable = {
+    [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
+    [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
+    [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
+    [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
+    [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
+    [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384
+}
+
+/**
+ * Compute the CIDs of a file using specific algorithms
+ * @returns Array of CIDs (in the order of the algorithms)
+ * @param algo
+ */
+export function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash {
+    switch (algo) {
+        case CID_ALGORITHM_NAMES.sha256:
+            return createHash("sha256");
+        case CID_ALGORITHM_NAMES.sha1:
+            return createHash("sha1");
+        case CID_ALGORITHM_NAMES.md5:
+            return createHash("md5");
+        case CID_ALGORITHM_NAMES.sha3_256:
+            return createHash("sha3-256");
+        case CID_ALGORITHM_NAMES.sha3_384:
+            return createHash("sha3-384");
+        case CID_ALGORITHM_NAMES.crc32:
+            return new Crc32Hash();
+        default:
+            throw new Error(`Unsupported algorithm: ${algo}`);
+    }
+}

src/lib/file-id/FileIDComputerInterface.ts

@@ -0,0 +1,6 @@
+import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import * as Stream from "stream";
+
+export interface FileIDComputerInterface{
+    computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
+}

src/lib/file-id/FileIDComputerWorker.ts

@@ -1,5 +1,6 @@
 import {Piscina} from "piscina";
-import {CID_ALGORITHM_NAMES} from "../MultiHashData";
+import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
+import {FileIDComputerInterface} from "@root/file-id/FileIDComputerInterface";
 
 // Construct the URL for the current module
 let distFolder = import.meta.dirname;
@@ -8,7 +9,7 @@ distFolder = distFolder + "/worker.js";
 const workerUrl = new URL(distFolder).href;
 console.log("Worker URL: ", workerUrl);
 
-export class FileIDComputer {
+export class FileIDComputerWorker{
     private piscina: Piscina;
 
     constructor(workerPath?:string) {

src/lib/file-id/ShaComputeWorker.ts

@@ -0,0 +1,17 @@
+import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
+import fs from "fs";
+import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
+
+/**
+ * Compute the CIDs of a file using specific algorithms
+ * @param filePath The path to the file
+ * @param algorithms Array of algorithms ('sha256', 'sha1')
+ * @returns Array of CIDs (in the order of the algorithms)
+ */
+export default async function computeCIDs({filePath, algorithms}: {
+    filePath: string;
+    algorithms: CID_ALGORITHM_NAMES[]
+}): Promise<string[]> {
+    const stream = fs.createReadStream(filePath);
+    return computeCIDsInternal({stream, algorithms});
+}

src/lib/file-id/SimpleHash.ts

@@ -0,0 +1,5 @@
+export interface SimpleHash {
+    update(data: Buffer),
+
+    digest(): Buffer;
+}

src/lib/fileID/ShaComputeWorker.ts

@@ -1,71 +0,0 @@
-import {createReadStream} from "fs";
-import {createHash} from 'crypto';
-import {CID} from 'multiformats/cid';
-import {create} from "multiformats/hashes/digest";
-import crc32 from 'crc-32';
-import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData";
-
-export interface SimpleHash {
-    update(data: Buffer),
-
-    digest(): Buffer;
-}
-
-class Crc32Hash implements SimpleHash {
-    private _crc: number = undefined;
-
-    update(data: Buffer): Crc32Hash {
-        // Update the CRC-32 checksum with the new chunk of data
-        this._crc = crc32.buf(data, this._crc);
-        return this;
-    }
-
-    digest(): Buffer {
-        const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
-        buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
-        return buffer;
-    }
-}
-
-
-/**
- * Compute the CIDs of a file using specific algorithms
- * @param filePath The path to the file
- * @param algorithms Array of algorithms ('sha256', 'sha1')
- * @returns Array of CIDs (in the order of the algorithms)
- */
-export default async function computeCIDs({filePath, algorithms}: {
-    filePath: string;
-    algorithms: CID_ALGORITHM_NAMES[]
-}): Promise<string[]> {
-    const hashFunctions: Record<string, { code: number, hasher: SimpleHash }>
-        = {
-        [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},
-        [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},
-        [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},
-        [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},
-        [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},
-        [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}
-
-    };
-
-    const hashers = algorithms.map(algo => ({
-        hasher: hashFunctions[algo].hasher,
-        code: hashFunctions[algo].code
-    }));
-
-    const stream = createReadStream(filePath);
-    for await (const chunk of stream) {
-        for (const item of hashers) {
-            item.hasher.update(chunk as Buffer);
-        }
-    }
-
-    const cids = await Promise.all(hashers.map(async ({code, hasher}) => {
-        const hashBuffer = hasher.digest();
-        const digest = create(code, hashBuffer);
-        return CID.createV1(code, digest).toString();
-    }));
-
-    return cids;
-}

src/lib/hash-compute/HashComputer.ts

@@ -0,0 +1,29 @@
+import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
+import {Readable} from "stream";
+
+import {computeCIDs} from "@root/file-id/ComputeHash";
+
+export class HashComputer{
+
+    constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
+    }
+
+    async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void> {
+        // Dynamically determine which hashes are needed
+        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+
+        // If all hashes are already computed, skip the processing
+        if (neededHashes.length === 0) {
+            return;
+        }
+
+        // Compute only the needed CIDs
+        const cids = await computeCIDs({stream, algorithms:neededHashes});
+
+        // Map the computed CIDs back to their respective metadata properties
+        for (const [index, cid] of cids.entries()) {
+            const hashType = neededHashes[index];
+            metadata[hashType] = cid;
+        }
+    }
+}

src/lib/hash-compute/HashComputerIndexCache.ts

@@ -1,15 +1,15 @@
-import {HashComputer} from "./HashComputer";
+import {HashComputerWorker} from "./HashComputerWorker";
 import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
 import {HashIndexManager} from "./HashIndexManager";
 import {stat} from "fs/promises";
 import path from "path";
 
-export class ComputeHashIndexCache implements ComputeInterface {
+export class HashComputerIndexCache implements ComputeInterface {
     private hashIndexManager: HashIndexManager;
-    private hashComputer: HashComputer;
+    private hashComputer: HashComputerWorker;
 
     constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
-        this.hashComputer = new HashComputer(targetHash,workerPath);
+        this.hashComputer = new HashComputerWorker(targetHash,workerPath);
         this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);
     }
 

src/lib/hash-compute/HashComputerWorker.ts

@@ -1,11 +1,11 @@
-import {FileIDComputer} from "./fileID/FileIDComputer";
+import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
 import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
 
-export class HashComputer implements ComputeInterface {
-    private fileIDComputer: FileIDComputer;
+export class HashComputerWorker implements ComputeInterface {
+    private fileIDComputer: FileIDComputerWorker;
 
     constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
-        this.fileIDComputer = new FileIDComputer(workerPath);
+        this.fileIDComputer = new FileIDComputerWorker(workerPath);
     }
 
     async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {

src/lib/hash-compute/HashIndexManager.ts

@@ -2,7 +2,7 @@ import {createReadStream, promises as fs} from 'fs';
 import {parse} from 'csv-parse';
 import {parse as parseSync} from 'csv-parse/sync';
 import {stringify} from 'csv-stringify/sync';
-import {existsAsync} from "./ExistsAsync";
+import {existsAsync} from "../utils/ExistsAsync";
 import path from "path";
 import {clearInterval} from "node:timers";
 import {CID_ALGORITHM_NAMES} from "./MultiHashData";

src/lib/index-browser.ts

@@ -0,0 +1,4 @@
+
+export * from './index-interface.js';
+export * from './file-id/ComputeHash.js';
+export * from './hash-compute/HashComputer.js';

src/lib/index-interface.ts

@@ -0,0 +1,3 @@
+export * from './hash-compute/MultiHashData.js';
+export * from './folder-watcher/FileProcessorInterface.js';
+export * from './file-id/FileIDComputerInterface.js';

src/lib/index.ts

@@ -1,13 +1,12 @@
-// Import and export directly to simplify access to library components
-export * from './ComputeHashIndexCache.js';
-export * from './ExistsAsync.js'; // Assuming 'async' is a function; adjust as necessary
-export * from './fileID/FileIDComputer.js';
-export * from './fileID/ShaComputeWorker.js'; // Assuming this exports a class or main function; adjust as needed
-export * from './HashComputer.js';
-export * from './HashIndexManager.js';
-// For MultiHashData.ts, assuming interfaces and enums need to be exported; you might need to adjust based on actual content
-export * from './MultiHashData.js';
-// Assuming Properties.ts exports one or more interfaces; adjust as needed
-export * from './Properties.js';
-export * from './folder_watcher/FolderWatcher.js';
-export * from './folder_watcher/FileProcessorInterface.js';
+//fileID
+export * from './file-id/FileIDComputerWorker.js';
+export * from './file-id/ShaComputeWorker.js';//worker
+
+//hash-compute
+export * from './hash-compute/HashComputerIndexCache.js';
+export * from './hash-compute/HashComputerWorker.js';
+export * from './hash-compute/HashIndexManager.js';
+
+//folder-watcher
+export * from './folder-watcher/FolderWatcher.js';
+export * from './index-interface.js';

src/tests/main.test.ts

@@ -1,4 +1,4 @@
-import {CID_ALGORITHM_NAMES, ComputeHashIndexCache, HashComputer} from "../lib";
+import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
 
 import {describe, expect, it} from 'vitest';
 import * as process from "process";
@@ -13,7 +13,7 @@ process.env.WORKER_URL = "./dist/worker.js";
 
 describe('add', () => {
     it('meta data compute', async () => {
-        let hashComputer = new HashComputer([
+        let hashComputer = new HashComputerWorker([
             CID_ALGORITHM_NAMES.sha1,
             CID_ALGORITHM_NAMES.sha256,
             CID_ALGORITHM_NAMES.md5,
@@ -43,7 +43,7 @@ describe('add', () => {
     it('meta data compute with index', async () => {
         await rm("./tmp",{recursive:true});
         await mkdir("./tmp",{recursive:true});
-        let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[
+        let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
             CID_ALGORITHM_NAMES.sha1,
             CID_ALGORITHM_NAMES.sha256,
             CID_ALGORITHM_NAMES.md5,
@@ -73,7 +73,7 @@ describe('add', () => {
     });
     it('cache', async () => {
         await mkdir("./tmp",{recursive:true});
-        let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[
+        let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
             CID_ALGORITHM_NAMES.sha1,
             CID_ALGORITHM_NAMES.sha256,
             CID_ALGORITHM_NAMES.md5,