improvement

dist/ShaComputeWorker-BzBitaFi.d.cts

@@ -52,4 +52,4 @@ declare function computeCIDs({ filePath, algorithms }: {
     algorithms: CID_ALGORITHM_NAMES[];
 }): Promise<string[]>;
 
-export { type ComputeInterface as C, type MultiHashData as M, type SimpleHash as S, CID_ALGORITHM_NAMES as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d };
+export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d };

dist/ShaComputeWorker-BzBitaFi.d.ts

@@ -52,4 +52,4 @@ declare function computeCIDs({ filePath, algorithms }: {
     algorithms: CID_ALGORITHM_NAMES[];
 }): Promise<string[]>;
 
-export { type ComputeInterface as C, type MultiHashData as M, type SimpleHash as S, CID_ALGORITHM_NAMES as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d };
+export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d };

dist/index.d.cts

@@ -1,16 +1,9 @@
-import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.cjs';
-export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.cjs';
+import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.cjs';
+export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.cjs';
 import * as PQueue from 'p-queue';
 import PQueue__default from 'p-queue';
 import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
 
-declare class HashComputer implements ComputeInterface {
-    private targetHash;
-    private fileIDComputer;
-    constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
-    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
-}
-
 interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
     path: string;
     size: string;
@@ -42,17 +35,19 @@ declare class HashIndexManager {
     loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]>;
     private loadIndexFromCache;
     private readCsv;
-    private saveCacheToFile;
+    saveCacheToFile(): Promise<void>;
+    getCidForFileAsync(filePath: string): Promise<IndexLine>;
     getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine;
     addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void;
 }
 
 declare class ComputeHashIndexCache implements ComputeInterface {
     private targetHash;
-    hashIndexManager: HashIndexManager;
-    hashComputer: HashComputer;
+    private hashIndexManager;
+    private hashComputer;
     constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string);
     computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+    getHashIndexManager(): Promise<HashIndexManager>;
 }
 
 declare function existsAsync(filePath: string): Promise<boolean>;
@@ -69,6 +64,13 @@ declare class FileIDComputer {
     computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
 }
 
+declare class HashComputer implements ComputeInterface {
+    private targetHash;
+    private fileIDComputer;
+    constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
+    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+}
+
 interface Properties {
     TARGET_HASHES: string;
     INDEX_FILE_PATH: string;

dist/index.d.ts

@@ -1,16 +1,9 @@
-import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.js';
-export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.js';
+import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.js';
+export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.js';
 import * as PQueue from 'p-queue';
 import PQueue__default from 'p-queue';
 import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
 
-declare class HashComputer implements ComputeInterface {
-    private targetHash;
-    private fileIDComputer;
-    constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
-    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
-}
-
 interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
     path: string;
     size: string;
@@ -42,17 +35,19 @@ declare class HashIndexManager {
     loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]>;
     private loadIndexFromCache;
     private readCsv;
-    private saveCacheToFile;
+    saveCacheToFile(): Promise<void>;
+    getCidForFileAsync(filePath: string): Promise<IndexLine>;
     getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine;
     addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void;
 }
 
 declare class ComputeHashIndexCache implements ComputeInterface {
     private targetHash;
-    hashIndexManager: HashIndexManager;
-    hashComputer: HashComputer;
+    private hashIndexManager;
+    private hashComputer;
     constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string);
     computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+    getHashIndexManager(): Promise<HashIndexManager>;
 }
 
 declare function existsAsync(filePath: string): Promise<boolean>;
@@ -69,6 +64,13 @@ declare class FileIDComputer {
     computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
 }
 
+declare class HashComputer implements ComputeInterface {
+    private targetHash;
+    private fileIDComputer;
+    constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
+    computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+}
+
 interface Properties {
     TARGET_HASHES: string;
     INDEX_FILE_PATH: string;

dist/worker.d.cts

@@ -1 +1 @@
-export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.cjs';
+export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.cjs';

dist/worker.d.ts

@@ -1 +1 @@
-export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.js';
+export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.js';

jest.config.ts

@@ -1,205 +0,0 @@
-/*
- * For a detailed explanation regarding each configuration property and type check, visit:
- * https://jestjs.io/docs/configuration
- */
-
-export default {
-  // All imported modules in your tests should be mocked automatically
-  // automock: false,
-
-  // Stop running tests after `n` failures
-  // bail: 0,
-
-  // The directory where Jest should store its cached dependency information
-  // cacheDirectory: "C:\\Users\\Pierre-HenriFRICOT\\AppData\\Local\\Temp\\jest",
-
-  // Automatically clear mock calls, instances, contexts and results before every test
-  // clearMocks: false,
-
-  // Indicates whether the coverage information should be collected while executing the test
-  //collectCoverage: true,
-
-  // An array of glob patterns indicating a set of files for which coverage information should be collected
-  // collectCoverageFrom: undefined,
-
-  // The directory where Jest should output its coverage files
-  //coverageDirectory: "coverage",
-
-  // An array of regexp pattern strings used to skip coverage collection
-  // coveragePathIgnorePatterns: [
-  //   "\\\\node_modules\\\\"
-  // ],
-
-  // Indicates which provider should be used to instrument code for coverage
-  // coverageProvider: "babel",
-
-  // A list of reporter names that Jest uses when writing coverage reports
-  // coverageReporters: [
-  //   "json",
-  //   "text",
-  //   "lcov",
-  //   "clover"
-  // ],
-
-  // An object that configures minimum threshold enforcement for coverage results
-  // coverageThreshold: undefined,
-
-  // A path to a custom dependency extractor
-  // dependencyExtractor: undefined,
-
-  // Make calling deprecated APIs throw helpful error messages
-  // errorOnDeprecated: false,
-
-  // The default configuration for fake timers
-  // fakeTimers: {
-  //   "enableGlobally": false
-  // },
-
-  // Force coverage collection from ignored files using an array of glob patterns
-  // forceCoverageMatch: [],
-
-  // A path to a module which exports an async function that is triggered once before all test suites
-  // globalSetup: undefined,
-
-  // A path to a module which exports an async function that is triggered once after all test suites
-  // globalTeardown: undefined,
-
-  // A set of global variables that need to be available in all test environments
-  // globals: {},
-
-  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
-  // maxWorkers: "50%",
-
-  // An array of directory names to be searched recursively up from the requiring module's location
-  moduleDirectories: [
-    "node_modules",
-    "src"
-  ],
-
-  // An array of file extensions your modules use
-  moduleFileExtensions: [
-    "js",
-    "mjs",
-    "cjs",
-    "jsx",
-    "ts",
-    "tsx",
-    "json",
-    "node",
-    "web.js",
-    "web.ts",
-    "web.tsx",
-    "web.jsx",
-  ],
-
-  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
-  "moduleNameMapper": {
-    "@root(.*)$": "<rootDir>/src/lib/$1",
-  },
-
-  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
-  // modulePathIgnorePatterns: [],
-
-  // Activates notifications for test results
-  // notify: false,
-
-  // An enum that specifies notification mode. Requires { notify: true }
-  // notifyMode: "failure-change",
-
-  // A preset that is used as a base for Jest's configuration
-  // preset: undefined,
-
-  // Run tests from one or more projects
-  // projects: undefined,
-
-  // Use this configuration option to add custom reporters to Jest
-  // reporters: undefined,
-
-  // Automatically reset mock state before every test
-  // resetMocks: false,
-
-  // Reset the module registry before running each individual test
-  // resetModules: false,
-
-  // A path to a custom resolver
-  // resolver: undefined,
-
-  // Automatically restore mock state and implementation before every test
-  // restoreMocks: false,
-
-  // The root directory that Jest should scan for tests and modules within
-  // rootDir: undefined,
-
-  // A list of paths to directories that Jest should use to search for files in
-  // roots: [
-  //   "<rootDir>"
-  // ],
-
-  // Allows you to use a custom runner instead of Jest's default test runner
-  // runner: "jest-runner",
-
-  // The paths to modules that run some code to configure or set up the testing environment before each test
-  // setupFiles: [],
-
-  // A list of paths to modules that run some code to configure or set up the testing framework before each test
-  // setupFilesAfterEnv: [],
-
-  // The number of seconds after which a test is considered as slow and reported as such in the results.
-  // slowTestThreshold: 5,
-
-  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
-  // snapshotSerializers: [],
-
-  // The test environment that will be used for testing
-  // testEnvironment: "jest-environment-node",
-
-  // Options that will be passed to the testEnvironment
-  // testEnvironmentOptions: {},
-
-  // Adds a location field to test results
-  // testLocationInResults: false,
-
-  // The glob patterns Jest uses to detect test files
-  testMatch: [
-    "**/src/tests/**/*.[jt]s?(x)",
-    "**/?(*.)+(spec|test).[tj]s?(x)"
-  ],
-
-  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
-  // testPathIgnorePatterns: [
-  //   "\\\\node_modules\\\\"
-  // ],
-
-  // The regexp pattern or array of patterns that Jest uses to detect test files
-  // testRegex: [],
-
-  // This option allows the use of a custom results processor
-  // testResultsProcessor: undefined,
-
-  // This option allows use of a custom test runner
-  // testRunner: "jest-circus/runner",
-
-  // A map from regular expressions to paths to transformers
-  // transform: undefined,
-
-  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
-  // transformIgnorePatterns: [
-  //   "\\\\node_modules\\\\",
-  //   "\\.pnp\\.[^\\\\]+$"
-  // ],
-  "transformIgnorePatterns": [
-    "node_modules/(?!@ngrx|(?!deck.gl)|ng-dynamic)"
-  ],
-
-  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
-  // unmockedModulePathPatterns: undefined,
-
-  // Indicates whether each individual test should be reported during the run
-  // verbose: undefined,
-
-  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
-  // watchPathIgnorePatterns: [],
-
-  // Whether to use watchman for file crawling
-  // watchman: true,
-};

src/lib/ComputeHashIndexCache.ts

@@ -5,15 +5,15 @@ import {stat} from "fs/promises";
 import path from "path";
 
 export class ComputeHashIndexCache implements ComputeInterface {
-    hashIndexManager: HashIndexManager;
-    hashComputer: HashComputer;
+    private hashIndexManager: HashIndexManager;
+    private hashComputer: HashComputer;
 
     constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
         this.hashComputer = new HashComputer(targetHash,workerPath);
         this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);
     }
 
-    async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
+    public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
         await this.hashIndexManager.init();
         let stats = await stat(filePath);
         if (this.hashIndexManager.getCache().has(path.basename(filePath))) {
@@ -29,4 +29,9 @@ export class ComputeHashIndexCache implements ComputeInterface {
         await this.hashComputer.computeMissingHash(filePath, metadata);
         this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
     }
+
+    async getHashIndexManager(): Promise<HashIndexManager> {
+        await this.hashIndexManager.init();
+        return this.hashIndexManager;
+    }
 }

src/lib/HashIndexManager.ts

@@ -6,6 +6,7 @@ import {existsAsync} from "./ExistsAsync";
 import path from "path";
 import {clearInterval} from "node:timers";
 import {CID_ALGORITHM_NAMES} from "./MultiHashData";
+import {stat} from "fs/promises";
 
 interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
     path: string;
@@ -163,7 +164,7 @@ export class HashIndexManager {
         });
     }
 
-    private async saveCacheToFile(): Promise<void> {
+    public async saveCacheToFile(): Promise<void> {
         if (this.indexOpsInProgress || !this.hasChanged) {
             return;
         }
@@ -205,6 +206,34 @@ export class HashIndexManager {
         this.indexOpsInProgress = false;
     }
 
+    public async getCidForFileAsync(filePath: string): Promise<IndexLine> {
+        const fileName = path.basename(filePath);
+        const stats = await stat(filePath);
+        let fileNameIndex = this.cache.get(fileName);
+        if (fileNameIndex) {
+            if (fileNameIndex.mtime) {
+                //if we have a mtime, we need to check it
+                if (fileNameIndex.size === (stats.size + "") && fileNameIndex.mtime === stats.mtime.toISOString()) {
+                    return fileNameIndex;
+                }
+            } else {
+                //mtime is optional
+                if (fileNameIndex.size === (stats.size + "")) {
+                    return fileNameIndex;
+                }
+            }
+        }
+
+        // 3 - if not found, delete the entry (keeps the index clean)
+        /*if (fileNameIndex && fileNameIndex.size !== (fileSize + "") && pathIndex && pathIndex.size !== (fileSize + "")) {
+            this.cache.delete(fileName);
+            this.cache.delete(filePath);
+        }*/
+
+        return null;
+    }
+
+
     public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {
         const fileName = path.basename(filePath);
         let fileNameIndex = this.cache.get(fileName);

src/tests/main.test.ts

@@ -1,7 +1,8 @@
-import {CID_ALGORITHM_NAMES, HashComputer} from "../lib";
+import {CID_ALGORITHM_NAMES, ComputeHashIndexCache, HashComputer} from "../lib";
 
 import {describe, expect, it} from 'vitest';
 import * as process from "process";
+import { mkdir, rm, rmdir } from "fs/promises";
 
 process.env.WORKER_URL = "./dist/worker.js";
 
@@ -39,4 +40,58 @@ describe('add', () => {
             "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
         });
     });
+    it('meta data compute with index', async () => {
+        await rm("./tmp",{recursive:true});
+        await mkdir("./tmp",{recursive:true});
+        let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[
+            CID_ALGORITHM_NAMES.sha1,
+            CID_ALGORITHM_NAMES.sha256,
+            CID_ALGORITHM_NAMES.md5,
+            CID_ALGORITHM_NAMES.sha3_256,
+            CID_ALGORITHM_NAMES.crc32,
+            CID_ALGORITHM_NAMES.sha3_384
+        ]);
+        let metadata = {};
+        await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
+        console.log(metadata);
+        /**
+         * test.txt CF20B62A (CRC32)
+         * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
+         * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
+         * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
+         * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
+         */
+        expect(metadata).toEqual({
+            "cid_crc32": "bagzafmqcathsbnrk",
+            "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
+            "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
+            "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
+            "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
+            "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
+        });
+        await (await hashComputer.getHashIndexManager()).saveCacheToFile();
+    });
+    it('cache', async () => {
+        await mkdir("./tmp",{recursive:true});
+        let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[
+            CID_ALGORITHM_NAMES.sha1,
+            CID_ALGORITHM_NAMES.sha256,
+            CID_ALGORITHM_NAMES.md5,
+            CID_ALGORITHM_NAMES.sha3_256,
+            CID_ALGORITHM_NAMES.crc32,
+            CID_ALGORITHM_NAMES.sha3_384
+        ]);
+        let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
+        delete indexLine['mtime'];
+        expect(indexLine).toEqual({
+            "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
+            "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
+            "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
+            "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
+            "cid_crc32": "bagzafmqcathsbnrk",
+            "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
+            "path": "test.txt",
+            "size": "499",
+        });
+    });
 });

tmp/index-cid_crc32.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_crc32
+test.txt,499,2024-04-02T16:35:33.190Z,bagzafmqcathsbnrk

tmp/index-cid_md5.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_md5
+test.txt,499,2024-04-02T16:35:33.190Z,bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa

tmp/index-cid_sha1.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_sha1
+test.txt,499,2024-04-02T16:35:33.190Z,baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra

tmp/index-cid_sha2-256.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_sha2-256
+test.txt,499,2024-04-02T16:35:33.190Z,baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua

tmp/index-cid_sha3-256.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_sha3-256
+test.txt,499,2024-04-02T16:35:33.190Z,baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq

tmp/index-cid_sha3-384.csv

@@ -0,0 +1,2 @@
+path,size,mtime,cid_sha3-384
+test.txt,499,2024-04-02T16:35:33.190Z,baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga

vitest.config.ts

@@ -4,6 +4,7 @@ export default defineConfig({
     test: {
         globals: true,
         environment: 'node',
+        testTimeout: 300000, // Set global timeout to 30 seconds
         // Use ts-jest or esbuild to handle TypeScript files
         // Specify other configurations as needed
     },