improve the debounce finalize

src/lib/folder-watcher/FolderWatcher.ts

@@ -9,7 +9,7 @@ export class FolderWatcher {
     initialized = false;
     queue = new PQueue({concurrency: 100, autoStart: true});
     fileProcessor: FileProcessorInterface;
-    queueSize = -1;
+    queueSize = 0;
     current = 0;
     processing = new Set<string>();
     private readonly finalizeDebounced: () => Promise<void>;
@@ -22,10 +22,15 @@ export class FolderWatcher {
     }) {
         this.fileProcessor = fileProcessor;
         if (this.fileProcessor.finalize) {
-            const debounced = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: true});
+            const debouncedImmediate = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: true});
+            const debouncedLate = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: false});
             this.finalizeDebounced = async () => {
                 if (this.initialized) {
-                    return debounced();
+                    const ret1 = debouncedImmediate();
+                    const ret2 = debouncedLate();
+                    this.current = 0;
+                    this.queueSize = 0;
+                    await Promise.all([ret1, ret2]);
                 }
             }
         }
@@ -74,15 +79,15 @@ export class FolderWatcher {
     }
 
     private async processFile(filePath: string): Promise<void> {
-        if(await this.fileProcessor.canProcessFile(filePath)) {
         if (!this.processing.has(filePath)) {
             this.processing.add(filePath);//avoid double processing in the queue
+            if (await this.fileProcessor.canProcessFile(filePath)) {
                 const current = ++this.current;
                 await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));
+            }
+        }
         this.processing.delete(filePath);
     }
-        }
-    }
 
     /**
      * Process the file and its sibling file included other folder it is a recursive function
@@ -91,7 +96,24 @@ export class FolderWatcher {
      */
     private async processFileExtended(filePath: string): Promise<void> {
         let dirname = path.dirname(filePath);
-        await this.processDirectory(dirname, []);
+        await this.processDirWithCount([dirname]);
+    }
+
+    private async processDirWithCount(folderList: string[]): Promise<void> {
+        let countPromises = [];
+        for (let i = 0; i < folderList.length; i++) {
+            folderList[i] = path.normalize(folderList[i]);
+            countPromises.push(this.countFile(folderList[i]));
+        }
+        const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);
+        console.log(`Found ${count} files to process`);
+        this.queueSize += count;
+        let promises: Promise<any>[] = [];
+        for (let i = 0; i < folderList.length; i++) {
+            folderList[i] = path.normalize(folderList[i]);
+            await this.processDirectory(folderList[i], promises);
+        }
+        await Promise.all(promises);
     }
 
     private chokidarWatch(folderList: string[]) {
@@ -142,6 +164,7 @@ export class FolderWatcher {
                     if (this.fileProcessor.deleteFile) {
                         await this.fileProcessor.deleteFile(filePath);
                     }
+                    this.processing.delete(filePath);
                     await this.processFileExtended(filePath);
                     if (this.finalizeDebounced) {
                         await this.finalizeDebounced();
@@ -170,20 +193,7 @@ export class FolderWatcher {
         this.chokidarWatch(folderList);
 
         //start the initial scan
-        let countPromises = [];
+        await this.processDirWithCount(folderList);
-        for (let i = 0; i < folderList.length; i++) {
-            folderList[i] = path.normalize(folderList[i]);
-            countPromises.push(this.countFile(folderList[i]));
-        }
-        const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);
-        console.log(`Found ${count} files to process`);
-        this.queueSize = count;
-        let promises: Promise<any>[] = [];
-        for (let i = 0; i < folderList.length; i++) {
-            folderList[i] = path.normalize(folderList[i]);
-            await this.processDirectory(folderList[i], promises);
-        }
-        await Promise.all(promises);
         this.initialized = true;
         if (this.finalizeDebounced) {
             await this.finalizeDebounced();

src/lib/hash-compute/HashComputerIndexCache.ts

@@ -16,7 +16,7 @@ export class HashComputerIndexCache implements ComputeInterface {
     public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
         await this.hashIndexManager.init();
         let stats = await stat(filePath);
-        if (this.hashIndexManager.getCache().has(path.basename(filePath))) {
+        if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
             const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
             if(indexLine) {
                 for (const hash of this.targetHash) {

src/lib/hash-compute/HashIndexManager.ts

@@ -7,6 +7,7 @@ import path from "path";
 import {clearInterval} from "node:timers";
 import {CID_ALGORITHM_NAMES} from "./MultiHashData";
 import {stat} from "fs/promises";
+import {Stats} from "node:fs";
 
 interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
     path: string;
@@ -17,7 +18,7 @@ interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
 export const INDEX_HEADERS = ['path', 'size', 'mtime'];
 
 export class HashIndexManager {
-    private cache: Map<string, IndexLine> = new Map<string, IndexLine>();
+    private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
     private intervalId: any;
     private intervalTime: number = 30000;
     private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
@@ -31,8 +32,12 @@ export class HashIndexManager {
                 private targetHash: CID_ALGORITHM_NAMES[]) {
     }
 
-    public getCache(): Map<string, IndexLine> {
+    hasFileInCache(filePath: string,stats:Stats): boolean {
-        return new Map(this.cache);
+        return this.cache.has(`${path.basename(filePath)}-${stats.size}-${stats.mtime.toISOString()}`);
+    }
+
+    getCacheSize(): number {
+        return this.cache.size;
     }
 
     /**
@@ -117,9 +122,10 @@ export class HashIndexManager {
                 // Read existing file content and parse it
                 const records: IndexLine[] = await this.readCsv(hash);
                 for (const record of records) {
-                    let indexLine = this.cache.get(record.path);
+                    const cacheKey = `${record.path}-${record.size}-${record.mtime}`;
+                    let indexLine = this.cache.get(cacheKey);
                     if (!indexLine) {
-                        this.cache.set(record.path, record);
+                        this.cache.set(cacheKey, record);
                     } else if(record[hash]){
                         //update the cache with the latest data
                         indexLine[hash] = record[hash];
@@ -183,12 +189,12 @@ export class HashIndexManager {
         let didWrite=false;
         for (const hash of this.targetHash) {
             let existingRows: IndexLine[] = await this.loadIndex(hash);
-            let existingRowsAsMap: Map<string, IndexLine> = new Map(existingRows.map(row => [row.path, row]));
+            let existingRowsAsMap: Map<string, IndexLine> = new Map(existingRows.map(row => [row.path + '-' + row.size + '-' + row.mtime, row]));
             if (this.cache.size !== 0) {
                 // Filter out cacheRows that are already in the file
                 const newRows = cacheRows.filter(row => {
                     //to be added a row must not exist in the file and must exist in the cache (with a hash)
-                    const newRow = !existingRowsAsMap.has(row.path);
+                    const newRow = !existingRowsAsMap.has(row.path + '-' + row.size + '-' + row.mtime);
                     return newRow && !!row[hash];
                 });
 
@@ -235,7 +241,7 @@ export class HashIndexManager {
 
     public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {
         const fileName = path.basename(filePath);
-        let fileNameIndex = this.cache.get(fileName);
+        let fileNameIndex = this.cache.get(`${fileName}-${fileSize}-${mtime}`);
         if (fileNameIndex) {
             for (const hash of this.targetHash) {
                 if (!fileNameIndex[hash]) {
@@ -275,7 +281,8 @@ export class HashIndexManager {
         }
         const size = fileSize + "";
         const baseName = path.basename(filePath);
-        let indexLine = this.cache.get(baseName);
+        const cacheKey = `${baseName}-${size}-${mtime}`;
+        let indexLine = this.cache.get(cacheKey);
         if (!indexLine) {
             //filter only the hashes we need
             let filteredHash = {};
@@ -283,7 +290,7 @@ export class HashIndexManager {
                 filteredHash[hash] = hashs[hash];
             }
             const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};
-            this.cache.set(baseName, data);
+            this.cache.set(cacheKey, data);
         } else {
             //update the cache with the latest data
             for (const hash of this.targetHash) {