initial commit

src/lib/ComputeHashIndexCache.ts

@@ -0,0 +1,32 @@
+import {HashComputer} from "./HashComputer.js";
+import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData.js";
+import {HashIndexManager} from "./HashIndexManager.js";
+import {stat} from "fs/promises";
+import path from "path";
+
+export class ComputeHashIndexCache implements ComputeInterface {
+    hashIndexManager: HashIndexManager;
+    hashComputer: HashComputer;
+
+    constructor(indexFilePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {
+        this.hashComputer = new HashComputer(this.targetHash);
+        this.hashIndexManager = new HashIndexManager(indexFilePath);
+    }
+
+    async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
+        await this.hashIndexManager.init();
+        let stats = await stat(filePath);
+        if (this.hashIndexManager.getCache().has(path.basename(filePath))) {
+            const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+            if(indexLine) {
+                for (const hash of this.targetHash) {
+                    if (!metadata[hash] && indexLine[hash]) {
+                        metadata[hash] = indexLine[hash];
+                    }
+                }
+            }
+        }
+        await this.hashComputer.computeMissingHash(filePath, metadata);
+        this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
+    }
+}

src/lib/ExistsAsync.ts

@@ -0,0 +1,12 @@
+// Example of checking if a file exists asynchronously
+// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.
+import {access,constants} from "fs/promises";
+
+export async function existsAsync(filePath: string): Promise<boolean> {
+    try {
+        await access(filePath, constants.F_OK);
+        return true; // File exists
+    } catch {
+        return false; // File does not exist
+    }
+}

src/lib/HashComputer.ts

@@ -0,0 +1,28 @@
+import { FileIDComputer } from "./fileID/FileIDComputer.js";
+import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData.js";
+
+export class HashComputer implements ComputeInterface{
+    private fileIDComputer = new FileIDComputer();
+
+    constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
+    }
+
+    async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
+        // Dynamically determine which hashes are needed
+        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+
+        // If all hashes are already computed, skip the processing
+        if (neededHashes.length === 0) {
+            return;
+        }
+
+        // Compute only the needed CIDs
+        const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
+
+        // Map the computed CIDs back to their respective metadata properties
+        cids.forEach((cid, index) => {
+            const hashType = neededHashes[index];
+            metadata[hashType] = cid;
+        });
+    }
+}

src/lib/HashIndexManager.ts

@@ -0,0 +1,229 @@
+import {createReadStream, promises as fs} from 'fs';
+import {parse} from 'csv-parse';
+import {parse as parseSync} from 'csv-parse/sync';
+import {stringify} from 'csv-stringify/sync';
+import {existsAsync} from "./ExistsAsync.js";
+import path from "path";
+import {clearInterval} from "node:timers";
+import {CID_ALGORITHM_NAMES} from "./MultiHashData.js";
+
+interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
+    path: string;
+    size: string;
+    mtime: string;
+}
+export const INDEX_HEADERS = ['path', 'size', 'mtime'];
+
+export class HashIndexManager {
+    private cache: Map<string, IndexLine> = new Map<string, IndexLine>();
+    private intervalId: NodeJS.Timeout;
+    private intervalTime: number = 30000;
+    private lastIndexFileSize: number = 0; //size of the index file last time it was read
+    private lastCacheFile: IndexLine[]; //state of the file last time it was read
+    private indexOpsInProgress: boolean = false;
+    private hasChanged: boolean = false;
+    private initialLoad;
+
+    constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {
+        if (!filePath) {
+            throw new Error('Invalid index file path');
+        }
+    }
+
+    public getCache(): Map<string, IndexLine> {
+        return new Map(this.cache);
+    }
+
+    /**
+     * After init consseutively calls to this method will not reload the index
+     * @param autosave
+     */
+    async init(autosave = true) {
+        if (!this.initialLoad) {
+            this.initialLoad = new Promise<void>(async (resolve, reject) => {
+                try {
+                    if (!this.checkCSVHeaders(this.filePath)) {
+                        throw new Error('Invalid index file headers');
+                    }
+                    await this.loadIndex();
+                    if (autosave) {
+                        this.start();
+                    }
+                    resolve();
+                }catch (e) {
+                    reject(e);
+                }
+            });
+        }
+        return this.initialLoad;
+    }
+
+    // Function to check CSV headers
+    checkCSVHeaders (csvContent: string): boolean {
+        const records = parseSync(csvContent, {
+            bom: true,
+            columns: true,
+            skip_empty_lines: true,
+        });
+
+        if (records.length === 0) {
+            //we will write the headers
+            return true;
+        }
+
+        // Extract headers
+        const headers = Object.keys(records[0]);
+
+        // Define required headers
+        //from the IndexLine interface
+        const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];
+
+        // Check if all required headers are present
+        return requiredHeaders.every(header => headers.includes(header));
+    };
+
+    public start() {
+        this.startAutoSave(this.intervalTime);
+    }
+
+    public stopAutoSave() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+        }
+    }
+
+    private startAutoSave(time: number) {
+        this.stopAutoSave();
+        this.intervalTime = time;
+        this.intervalId = setInterval(() => this.saveCacheToFile(), time);
+    }
+
+    public async loadIndex(): Promise<IndexLine[]> {
+        if (await existsAsync(this.filePath)) {
+            // check the file size and if it did not change, do not read the file
+            const stats = await fs.stat(this.filePath);
+            if (this.lastIndexFileSize !== stats.size) {
+                // Read existing file content and parse it
+                const records: IndexLine[] = await this.readCsv();
+                for (const record of records) {
+                    this.cache.set(record.path, record);
+                }
+                this.lastIndexFileSize = stats.size;
+                this.lastCacheFile = records;
+                return records;
+            } else {
+                return this.lastCacheFile;
+            }
+        }
+        return [];
+    }
+
+    private loadIndexFromCache(): IndexLine[] {
+        return Array.from(this.cache.values());
+    }
+
+    private async readCsv(): Promise<IndexLine[]> {
+        if (!(await existsAsync(this.filePath))) {
+            return [];
+        }
+        const start = performance.now();
+
+        const parser = parse({
+            columns: true,
+            skip_empty_lines: true,
+        });
+
+        const records: IndexLine[] = [];
+
+        return new Promise((resolve, reject) => {
+            createReadStream(this.filePath)
+                .pipe(parser)
+                .on('data', (record) => {
+                    records.push(record);
+                }).on('end', () => {
+                resolve(records);
+                console.log(`Index read time ${performance.now() - start}ms`);
+            }).on('error', (err) => {
+                reject(err);
+            });
+        });
+    }
+
+    private async saveCacheToFile(): Promise<void> {
+        if (this.indexOpsInProgress || !this.hasChanged) {
+            return;
+        }
+        this.hasChanged = false;
+        this.indexOpsInProgress = true;
+        const start = performance.now();
+        let existingRows: IndexLine[] = await this.loadIndex();
+
+        if (this.cache.size !== 0) {
+            const cacheRows: IndexLine[] = this.loadIndexFromCache();
+            // Filter out cacheRows that are already in the file
+            const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));
+
+            if (newRows.length !== 0) {
+                // Serialize new cacheRows to CSV string
+                const csvString = stringify(newRows, {
+                    header: existingRows.length === 0, // Only add header if the file was empty
+                    columns: [
+                        {key: 'path', header: 'path'},
+                        {key: 'size', header: 'size'},
+                        {key: 'mtime', header: 'mtime'},
+                        ...this.targetHash.map(hash => ({key: hash, header: hash})),
+                    ],
+                });
+
+                // Append new cacheRows to the file
+                await fs.appendFile(this.filePath, csvString);
+            }
+        }
+        const totalTime = performance.now() - start;
+        console.log(`Index saved in ${totalTime}ms`);
+        // Check if the time to save the index is greater than the interval time. increase the interval time if needed
+        if (totalTime * 10 > this.intervalTime) {
+            this.startAutoSave(totalTime * 10);
+            console.log(`Index save interval increased to ${totalTime * 10}ms`);
+        }
+        this.indexOpsInProgress = false;
+    }
+
+    public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {
+        const fileName = path.basename(filePath);
+        let fileNameIndex = this.cache.get(fileName);
+        if(fileNameIndex) {
+            if(fileNameIndex.mtime) {
+                //if we have a mtime, we need to check it
+                if (fileNameIndex.size === (fileSize + "") && fileNameIndex.mtime === mtime) {
+                    return fileNameIndex;
+                }
+            }else {
+                //mtime is optional
+                if (fileNameIndex.size === (fileSize + "")) {
+                    return fileNameIndex;
+                }
+            }
+        }
+
+        // 3 - if not found, delete the entry (keeps the index clean)
+        /*if (fileNameIndex && fileNameIndex.size !== (fileSize + "") && pathIndex && pathIndex.size !== (fileSize + "")) {
+            this.cache.delete(fileName);
+            this.cache.delete(filePath);
+        }*/
+
+        return null;
+    }
+
+    public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
+        if (!filePath || !fileSize || !mtime || !hashs) {
+            throw new Error('Invalid parameters');
+        }
+        const size = fileSize + "";
+        const baseName = path.basename(filePath);
+        const data = {path: baseName, size: size, mtime: mtime, ...hashs};
+        this.cache.set(baseName, data);
+        this.hasChanged = true;
+    }
+
+}

src/lib/MultiHashData.ts

@@ -0,0 +1,35 @@
+export interface MultiHashData {
+    "cid_sha2-256"?: string;//0x12 sha2-256
+    cid_sha1?: string; // 0x11 sha1
+    cid_md5?: string; // 0xd5 md5
+    "cid_sha3-256"?: string; // 0x16 sha3-256
+    //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO
+}
+
+export interface ComputeInterface {
+    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+}
+
+export enum CID_ALGORITHM_CODES {
+    sha256 = 0x12,
+    sha1 = 0x11,
+    md5 = 0xd5,
+    sha3_256 = 0x16,
+    //crc32 = 0x0132
+}
+
+export enum CID_ALGORITHM_NAMES {
+    sha256 = 'cid_sha2-256',
+    sha1 = 'cid_sha1',
+    md5 = 'cid_md5',
+    sha3_256 = 'cid_sha3-256',
+    //crc32 = 'cid_crc32'
+}
+
+export enum CID_ALGORITHM {
+    sha256 = 'sha2-256',
+    sha1 = 'sha1',
+    md5 = 'md5',
+    sha3_256 = 'sha3-256',
+    //crc32 = 'crc32'
+}

src/lib/Properties.ts

@@ -0,0 +1,5 @@
+
+export interface Properties{
+    TARGET_HASHES:string;
+    INDEX_FILE_PATH:string;
+}

src/lib/fileID/FileIDComputer.ts

@@ -0,0 +1,23 @@
+import {Piscina} from "piscina";
+import {CID_ALGORITHM_NAMES} from "../MultiHashData.js";
+
+export class FileIDComputer {
+    private piscina: Piscina;
+
+    constructor() {
+        this.piscina = new Piscina({
+            maxThreads: 4,
+            filename: new URL('./ShaComputeWorker.js', import.meta.url).href
+        });
+    }
+
+    /**
+     * Compute the CIDs of a file using specific algorithms
+     * @param filePath The path to the file
+     * @param algorithms Array of algorithms ('sha256', 'sha1')
+     * @returns Array of CIDs (in the order of the algorithms)
+     */
+    public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
+        return this.piscina.run({filePath, algorithms});
+    }
+}

src/lib/fileID/ShaComputeWorker.ts

@@ -0,0 +1,47 @@
+import { createReadStream } from "fs";
+import {BinaryLike, createHash} from 'crypto';
+import { CID } from 'multiformats/cid';
+import { create } from "multiformats/hashes/digest";
+import {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData.js";
+
+export interface SimpleHash {
+    update(data: BinaryLike),
+    digest(): Buffer;
+}
+
+
+/**
+ * Compute the CIDs of a file using specific algorithms
+ * @param filePath The path to the file
+ * @param algorithms Array of algorithms ('sha256', 'sha1')
+ * @returns Array of CIDs (in the order of the algorithms)
+ */
+export default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise<string[]> {
+    const hashFunctions: Record<string, { code: number, hasher: SimpleHash }>
+        = {
+        [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },
+        [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },
+        [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },
+        [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }
+    };
+
+    const hashers = algorithms.map(algo => ({
+        hasher: hashFunctions[algo].hasher,
+        code: hashFunctions[algo].code
+    }));
+
+    const stream = createReadStream(filePath);
+    for await (const chunk of stream) {
+        for (const item of hashers) {
+            item.hasher.update(chunk);
+        }
+    }
+
+    const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {
+        const hashBuffer = hasher.digest();
+        const digest = create(code, hashBuffer);
+        return CID.createV1(code, digest).toString();
+    }));
+
+    return cids;
+}

src/lib/index.ts

@@ -0,0 +1,11 @@
+// Import and export directly to simplify access to library components
+export * from './ComputeHashIndexCache';
+export * from './ExistsAsync'; // Assuming 'async' is a function; adjust as necessary
+export * from './fileID/FileIDComputer';
+export * from './fileID/ShaComputeWorker'; // Assuming this exports a class or main function; adjust as needed
+export * from './HashComputer';
+export * from './HashIndexManager';
+// For MultiHashData.ts, assuming interfaces and enums need to be exported; you might need to adjust based on actual content
+export * from './MultiHashData';
+// Assuming Properties.ts exports one or more interfaces; adjust as needed
+export * from './Properties';