From 14ba90f521b34c7ff53265fe90725c451a5d6291 Mon Sep 17 00:00:00 2001 From: Metazla Date: Sat, 6 Apr 2024 17:39:42 +0200 Subject: [PATCH] improvement --- dist/index.cjs | 10 ++++----- dist/index.cjs.map | 2 +- dist/index.js | 6 +++--- dist/index.js.map | 2 +- src/lib/HashIndexManager.ts | 43 ++++++++++++++++--------------------- 5 files changed, 28 insertions(+), 35 deletions(-) diff --git a/dist/index.cjs b/dist/index.cjs index af475eb..1cf02af 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -9,7 +9,7 @@ var sync$1 = require('csv-stringify/sync'); var promises = require('fs/promises'); var c = require('path'); var timers = require('timers'); -var y = require('chokidar'); +var M = require('chokidar'); var z = require('p-queue'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } @@ -33,10 +33,10 @@ function _interopNamespace(e) { } var c__namespace = /*#__PURE__*/_interopNamespace(c); -var y__default = /*#__PURE__*/_interopDefault(y); +var M__default = /*#__PURE__*/_interopDefault(M); var z__default = /*#__PURE__*/_interopDefault(z); -var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var v=new URL(l).href;console.log("Worker URL: ",v);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||v});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;let i=await this.fileIDComputer.computeCIDs(e,r);for(let[s,a]of i.entries()){let n=r[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...E,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=sync$1.stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),r=await promises.stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={};for(let g of this.targetHash)o[g]=i[g];let F={path:a,size:s,mtime:r,...o};this.cache.set(a,F);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=y__default.default.watch(e,t):r=y__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=csvParse.parse({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=sync$1.stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),i=await promises.stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__namespace.default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await promises.readdir(e,{withFileTypes:!0});for(let r of i){let s=c__namespace.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M__default.default.watch(e,t):i=M__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"uDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA+C,CAGlD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index 12070ed..12aec21 100644 --- a/dist/index.js +++ b/dist/index.js @@ -8,11 +8,11 @@ import { access, constants, stat, readdir } from 'fs/promises'; import * as c from 'path'; import c__default from 'path'; import { clearInterval } from 'node:timers'; -import y from 'chokidar'; +import M from 'chokidar'; import z from 'p-queue'; -var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var v=new URL(l).href;console.log("Worker URL: ",v);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||v});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;let i=await this.fileIDComputer.computeCIDs(e,r);for(let[s,a]of i.entries()){let n=r[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...E,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),r=await stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={};for(let g of this.targetHash)o[g]=i[g];let F={path:a,size:s,mtime:r,...o};this.cache.set(a,F);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=y.watch(e,t):r=y.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=parse$1({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),i=await stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await readdir(e,{withFileTypes:!0});for(let r of i){let s=c.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M.watch(e,t):i=M.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"sDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA+C,CAGlD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/src/lib/HashIndexManager.ts b/src/lib/HashIndexManager.ts index 08382e2..49186a1 100644 --- a/src/lib/HashIndexManager.ts +++ b/src/lib/HashIndexManager.ts @@ -172,12 +172,21 @@ export class HashIndexManager { this.indexOpsInProgress = true; const start = performance.now(); + let cacheRows: IndexLine[]; + if (this.cache.size !== 0) { + cacheRows = this.loadIndexFromCache(); + } + for (const hash of this.targetHash) { let existingRows: IndexLine[] = await this.loadIndex(hash); + let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row])); if (this.cache.size !== 0) { - const cacheRows: IndexLine[] = this.loadIndexFromCache(); // Filter out cacheRows that are already in the file - const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path)); + const newRows = cacheRows.filter(row => { + //to be added a row must not exist in the file and must exist in the cache (with a hash) + const newRow = !existingRowsAsMap.has(row.path); + return newRow && !!row[hash]; + }); if (newRows.length !== 0) { // Serialize new cacheRows to CSV string @@ -196,6 +205,7 @@ export class HashIndexManager { } } } + const totalTime = performance.now() - start; console.log(`Index saved in ${totalTime}ms`); // Check if the time to save the index is greater than the interval time. increase the interval time if needed @@ -209,34 +219,17 @@ export class HashIndexManager { public async getCidForFileAsync(filePath: string): Promise { const fileName = path.basename(filePath); const stats = await stat(filePath); - let fileNameIndex = this.cache.get(fileName); - if (fileNameIndex) { - if (fileNameIndex.mtime) { - //if we have a mtime, we need to check it - if (fileNameIndex.size === (stats.size + "") && fileNameIndex.mtime === stats.mtime.toISOString()) { - return fileNameIndex; - } - } else { - //mtime is optional - if (fileNameIndex.size === (stats.size + "")) { - return fileNameIndex; - } - } - } - - // 3 - if not found, delete the entry (keeps the index clean) - /*if (fileNameIndex && fileNameIndex.size !== (fileSize + "") && pathIndex && pathIndex.size !== (fileSize + "")) { - this.cache.delete(fileName); - this.cache.delete(filePath); - }*/ - - return null; + return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString()); } - public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine { const fileName = path.basename(filePath); let fileNameIndex = this.cache.get(fileName); + for (const hash of this.targetHash) { + if (!fileNameIndex[hash]) { + delete fileNameIndex[hash]; + } + } if (fileNameIndex) { if (fileNameIndex.mtime) { //if we have a mtime, we need to check it