add finalize method to folder watch

.yarnrc.yml

@@ -1,2 +0,0 @@
-nmHoistingLimits: dependencies
-nodeLinker: node-modules

package.json

@@ -31,8 +31,7 @@
     }
   },
   "scripts": {
-    "build": "cross-env rimraf ./dist & tsup",
+    "build": "tsup",
-    "ci": "yarn run coverage",
     "test": "vitest"
   },
   "devDependencies": {
@@ -49,6 +48,7 @@
     "crc-32": "^1.2.2",
     "csv-parse": "^5.5.5",
     "csv-stringify": "^6.4.6",
+    "debounce": "^2.0.0",
     "multiformats": "^13.1.0",
     "p-queue": "^8.0.1",
     "piscina": "^4.4.0"

src/lib/folder-watcher/FileProcessorInterface.ts

@@ -3,5 +3,7 @@ export interface FileProcessorInterface {
 
     canProcessFile(filePath: string): Promise<boolean>;
 
-    deleteFile(filePath: string): Promise<void>
+    deleteFile?(filePath: string): Promise<void>;
+
+    finalize?(): Promise<void>;
 }

src/lib/folder-watcher/FolderWatcher.ts

@@ -2,6 +2,7 @@ import * as path from 'path';
 import chokidar from 'chokidar';
 import {readdir} from "fs/promises";
 import PQueue from "p-queue";
+import debounce from 'debounce';
 import {FileProcessorInterface} from "./FileProcessorInterface.js";
 
 export class FolderWatcher {
@@ -11,13 +12,24 @@ export class FolderWatcher {
     queueSize = -1;
     current = 0;
     processing = new Set<string>();
+    private readonly updateDebounced: () => Promise<void>;
 
     constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{
         interval?: number,
         stabilityThreshold?: number,
         pollInterval?: number,
+        outputFolderUpdateIntervalMs?: number
     }) {
         this.fileProcessor = fileProcessor;
+        if(this.fileProcessor.finalize) {
+            const debounced = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: true});
+            this.updateDebounced = async () => {
+                if (this.initialized) {
+                    return debounced();
+                }
+            }
+        }
+
     }
 
     /**
@@ -108,6 +120,7 @@ export class FolderWatcher {
             .on('add', async (filePath) => {
                 try {
                     await this.processFileExtended(filePath);
+                    await this.updateDebounced();
                 } catch (error) {
                     console.error(`Error processing file ${filePath}:`, error);
                 }
@@ -115,14 +128,18 @@ export class FolderWatcher {
             .on('change', async (filePath) => {
                 try {
                     await this.processFileExtended(filePath);
+                    await this.updateDebounced();
                 } catch (e) {
                     console.error(`Error processing file ${filePath}:`, e);
                 }
             })
             .on('unlink', async (filePath) => {
                 try {
-                    await this.fileProcessor.deleteFile(filePath);
+                    if(this.fileProcessor.deleteFile) {
+                        await this.fileProcessor.deleteFile(filePath);
+                    }
                     await this.processFileExtended(filePath);
+                    await this.updateDebounced();
                 } catch (e) {
                     console.error(`Error processing file deletion:`, e);
                 }
@@ -158,6 +175,7 @@ export class FolderWatcher {
         }
         await Promise.all(promises);
         this.initialized = true;
+        await this.updateDebounced();
 
         console.log(`Watcher ready`);
     }

src/lib/hash-compute/HashComputerFile.ts

@@ -0,0 +1,31 @@
+import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
+import {Readable} from "stream";
+
+import {computeCIDs} from "@root/file-id/ComputeHash";
+import fs from "fs";
+
+export class HashComputerFile implements ComputeInterface{
+
+    constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
+    }
+
+    public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
+        // Dynamically determine which hashes are needed
+        const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
+
+        // If all hashes are already computed, skip the processing
+        if (neededHashes.length === 0) {
+            return;
+        }
+
+        // Compute only the needed CIDs
+        const stream = fs.createReadStream(filePath);
+        const cids = await computeCIDs({stream, algorithms:neededHashes});
+
+        // Map the computed CIDs back to their respective metadata properties
+        for (const [index, cid] of cids.entries()) {
+            const hashType = neededHashes[index];
+            metadata[hashType] = cid;
+        }
+    }
+}

src/lib/hash-compute/HashIndexManager.ts

@@ -42,7 +42,8 @@ export class HashIndexManager {
     }
 
     /**
-     * After init consseutively calls to this method will not reload the index
+     * After init consecutively calls to this method will not reload the index
+     * readThe index for the first time and enable autosave
      * @param autosave
      */
     public async init(autosave = true) {
@@ -56,7 +57,7 @@ export class HashIndexManager {
                         await this.loadIndex(hash);
                     }
                     if (autosave) {
-                        this.start();
+                        this.startAutoSave()
                     }
                     resolve();
                 } catch (e) {
@@ -91,17 +92,13 @@ export class HashIndexManager {
         return requiredHeaders.every(header => headers.includes(header));
     };
 
-    public start() {
-        this.startAutoSave(this.intervalTime);
-    }
-
     public stopAutoSave() {
         if (this.intervalId) {
             clearInterval(this.intervalId);
         }
     }
 
-    private startAutoSave(time: number) {
+    public startAutoSave(time: number = this.intervalTime) {
         this.stopAutoSave();
         this.intervalTime = time;
         this.intervalId = setInterval(() => this.saveCacheToFile(), time);
@@ -216,6 +213,10 @@ export class HashIndexManager {
         this.indexOpsInProgress = false;
     }
 
+    /**
+     * this.getCidForFile(fileName, stats.size, stats.mtime.toISOString())
+     * @param filePath
+     */
     public async getCidForFileAsync(filePath: string): Promise<IndexLine> {
         const fileName = path.basename(filePath);
         const stats = await stat(filePath);

src/lib/index.ts

@@ -6,6 +6,8 @@ export * from './file-id/ShaComputeWorker.js';//worker
 export * from './hash-compute/HashComputerIndexCache.js';
 export * from './hash-compute/HashComputerWorker.js';
 export * from './hash-compute/HashIndexManager.js';
+export * from './hash-compute/HashComputer.js';
+export * from './hash-compute/HashComputerFile.js';
 
 //folder-watcher
 export * from './folder-watcher/FolderWatcher.js';