From 5045120fbbd41d0eef0c194fda7ff2dee6612936 Mon Sep 17 00:00:00 2001 From: Metazla Date: Sat, 6 Apr 2024 16:16:16 +0200 Subject: [PATCH] improvement --- ....d.cts => ShaComputeWorker-BzBitaFi.d.cts} | 2 +- ...X4.d.ts => ShaComputeWorker-BzBitaFi.d.ts} | 2 +- dist/index.cjs | 14 +- dist/index.cjs.map | 2 +- dist/index.d.cts | 26 ++- dist/index.d.ts | 26 ++- dist/index.js | 8 +- dist/index.js.map | 2 +- dist/worker.d.cts | 2 +- dist/worker.d.ts | 2 +- jest.config.ts | 205 ------------------ src/lib/ComputeHashIndexCache.ts | 11 +- src/lib/HashIndexManager.ts | 31 ++- src/tests/main.test.ts | 57 ++++- tmp/index-cid_crc32.csv | 2 + tmp/index-cid_md5.csv | 2 + tmp/index-cid_sha1.csv | 2 + tmp/index-cid_sha2-256.csv | 2 + tmp/index-cid_sha3-256.csv | 2 + tmp/index-cid_sha3-384.csv | 2 + vitest.config.ts | 1 + 21 files changed, 152 insertions(+), 251 deletions(-) rename dist/{ShaComputeWorker-DxWHpwX4.d.cts => ShaComputeWorker-BzBitaFi.d.cts} (91%) rename dist/{ShaComputeWorker-DxWHpwX4.d.ts => ShaComputeWorker-BzBitaFi.d.ts} (91%) delete mode 100644 jest.config.ts create mode 100644 tmp/index-cid_crc32.csv create mode 100644 tmp/index-cid_md5.csv create mode 100644 tmp/index-cid_sha1.csv create mode 100644 tmp/index-cid_sha2-256.csv create mode 100644 tmp/index-cid_sha3-256.csv create mode 100644 tmp/index-cid_sha3-384.csv diff --git a/dist/ShaComputeWorker-DxWHpwX4.d.cts b/dist/ShaComputeWorker-BzBitaFi.d.cts similarity index 91% rename from dist/ShaComputeWorker-DxWHpwX4.d.cts rename to dist/ShaComputeWorker-BzBitaFi.d.cts index 12d1e21..a44621f 100644 --- a/dist/ShaComputeWorker-DxWHpwX4.d.cts +++ b/dist/ShaComputeWorker-BzBitaFi.d.cts @@ -52,4 +52,4 @@ declare function computeCIDs({ filePath, algorithms }: { algorithms: CID_ALGORITHM_NAMES[]; }): Promise; -export { type ComputeInterface as C, type MultiHashData as M, type SimpleHash as S, CID_ALGORITHM_NAMES as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; +export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; diff --git a/dist/ShaComputeWorker-DxWHpwX4.d.ts b/dist/ShaComputeWorker-BzBitaFi.d.ts similarity index 91% rename from dist/ShaComputeWorker-DxWHpwX4.d.ts rename to dist/ShaComputeWorker-BzBitaFi.d.ts index 12d1e21..a44621f 100644 --- a/dist/ShaComputeWorker-DxWHpwX4.d.ts +++ b/dist/ShaComputeWorker-BzBitaFi.d.ts @@ -52,4 +52,4 @@ declare function computeCIDs({ filePath, algorithms }: { algorithms: CID_ALGORITHM_NAMES[]; }): Promise; -export { type ComputeInterface as C, type MultiHashData as M, type SimpleHash as S, CID_ALGORITHM_NAMES as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; +export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; diff --git a/dist/index.cjs b/dist/index.cjs index 33b65ca..1632eb1 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -9,8 +9,8 @@ var sync$1 = require('csv-stringify/sync'); var promises = require('fs/promises'); var c = require('path'); var timers = require('timers'); -var x = require('chokidar'); -var b = require('p-queue'); +var w = require('chokidar'); +var R = require('p-queue'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } @@ -33,10 +33,10 @@ function _interopNamespace(e) { } var c__namespace = /*#__PURE__*/_interopNamespace(c); -var x__default = /*#__PURE__*/_interopDefault(x); -var b__default = /*#__PURE__*/_interopDefault(b); +var w__default = /*#__PURE__*/_interopDefault(w); +var R__default = /*#__PURE__*/_interopDefault(R); -var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var g=new URL(l).href;console.log("Worker URL: ",g);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||g});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,o)=>{let a=r[o];t[a]=s;});}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(o=>i.includes(o))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,o)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",a=>{o(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let o=this.loadIndexFromCache().filter(a=>!i.find(n=>n.path===a.path));if(o.length!==0){let a=sync$1.stringify(o,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],a);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let n of this.targetHash)if(!i[n])throw new Error(`Missing hash ${n}`);let s=t+"",o=c__namespace.default.basename(e),a=this.cache.get(o);if(a)for(let n of this.targetHash)a[n]=i[n];else {let n={path:o,size:s,mtime:r,...i};this.cache.set(o,n);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let o=this.processFile(s);t.push(o);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let o=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(o)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(o)?1:0));}return (await Promise.all(r)).reduce((s,o)=>s+o,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=x__default.default.watch(e,t):r=x__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let o=0;oo+a,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let o=0;o!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,a)=>{let n=r[a];t[n]=s;});}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=sync$1.stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),r=await promises.stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={path:a,size:s,mtime:r,...i};this.cache.set(a,o);}this.hasChanged=!0;}};var x=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var M=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new R__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w__default.default.watch(e,t):r=w__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileName","fileNameIndex","fileSize","mtime","hashs","size","baseName","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"uDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA8C,CAGjD,YAAoBC,EAAkCJ,EAAoB,CAAtD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYL,EAAUK,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,CACJ,EC5BA,OAAQ,oBAAAG,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYjB,EAAoC,CAClE,GAAI,CACA,aAAMe,EAAOf,EAAUgB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYtB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWuB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGvB,CAAQ,IAAIuB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY7B,EAAU0B,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB5C,EAAsC,CAClE,IAAM6C,EAAW3B,EAAK,SAASlB,CAAQ,EACjCiC,EAAQ,MAAMb,EAAKpB,CAAQ,EAC7B8C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUb,EAAM,KAAO,IAAOa,EAAc,QAAUb,EAAM,MAAM,YAAY,EAC5F,OAAOa,UAIPA,EAAc,OAAUb,EAAM,KAAO,GACrC,OAAOa,EAWnB,OAAO,IACX,CAGO,cAAc9C,EAAkB+C,EAAkBC,EAA0B,CAC/E,IAAMH,EAAW3B,EAAK,SAASlB,CAAQ,EACnC8C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUC,EAAW,IAAOD,EAAc,QAAUE,EAClE,OAAOF,UAIPA,EAAc,OAAUC,EAAW,GACnC,OAAOD,EAWnB,OAAO,IACX,CAEO,WAAW9C,EAAkB+C,EAAkBC,EAAeC,EAA2D,CAC5H,GAAI,CAACjD,GAAY,CAAC+C,GAAY,CAACC,GAAS,CAACC,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOH,EAAW,GAClBI,EAAWjC,EAAK,SAASlB,CAAQ,EACnCmC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAKD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MANpB,CACZ,IAAM6B,EAAO,CAAC,KAAMD,EAAU,KAAMD,EAAM,MAAOF,EAAO,GAAGC,CAAK,EAChE,KAAK,MAAM,IAAIE,EAAUC,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EE3RA,OAAQ,QAAAhC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BnD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIuB,EAAiBgC,EAAcnD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI6B,EAAQ,MAAMb,EAAKpB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIkB,EAAK,SAASlB,CAAQ,CAAC,EAAG,CAC/D,IAAMmC,EAAY,KAAK,iBAAiB,cAAcnC,EAAUiC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACnB,EAASmB,CAAI,GAAKY,EAAUZ,CAAI,IACjCnB,EAASmB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBvB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUiC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG7B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYc,MAAU,OACtB,OAAOqC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYtE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMsE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWtE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIuE,EAAe,UAAQvE,CAAQ,EACnC,MAAM,KAAK,iBAAiBuE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO1E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2E,EAAO,CACZ,QAAQ,MAAM,yBAAyB3E,CAAQ,IAAK2E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2B,EAAG,CACR,QAAQ,MAAM,yBAAyB3B,CAAQ,IAAK2B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUgD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.d.cts b/dist/index.d.cts index 88aae43..619a746 100644 --- a/dist/index.d.cts +++ b/dist/index.d.cts @@ -1,16 +1,9 @@ -import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.cjs'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.cjs'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.cjs'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; -declare class HashComputer implements ComputeInterface { - private targetHash; - private fileIDComputer; - constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); - computeMissingHash(filePath: string, metadata: MultiHashData): Promise; -} - interface IndexLine extends Partial> { path: string; size: string; @@ -42,17 +35,19 @@ declare class HashIndexManager { loadIndex(hash: CID_ALGORITHM_NAMES): Promise; private loadIndexFromCache; private readCsv; - private saveCacheToFile; + saveCacheToFile(): Promise; + getCidForFileAsync(filePath: string): Promise; getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine; addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; } declare class ComputeHashIndexCache implements ComputeInterface { private targetHash; - hashIndexManager: HashIndexManager; - hashComputer: HashComputer; + private hashIndexManager; + private hashComputer; constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string); computeMissingHash(filePath: string, metadata: MultiHashData): Promise; + getHashIndexManager(): Promise; } declare function existsAsync(filePath: string): Promise; @@ -69,6 +64,13 @@ declare class FileIDComputer { computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; } +declare class HashComputer implements ComputeInterface { + private targetHash; + private fileIDComputer; + constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + interface Properties { TARGET_HASHES: string; INDEX_FILE_PATH: string; diff --git a/dist/index.d.ts b/dist/index.d.ts index 0c9237f..8cc028b 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,16 +1,9 @@ -import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.js'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.js'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.js'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; -declare class HashComputer implements ComputeInterface { - private targetHash; - private fileIDComputer; - constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); - computeMissingHash(filePath: string, metadata: MultiHashData): Promise; -} - interface IndexLine extends Partial> { path: string; size: string; @@ -42,17 +35,19 @@ declare class HashIndexManager { loadIndex(hash: CID_ALGORITHM_NAMES): Promise; private loadIndexFromCache; private readCsv; - private saveCacheToFile; + saveCacheToFile(): Promise; + getCidForFileAsync(filePath: string): Promise; getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine; addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; } declare class ComputeHashIndexCache implements ComputeInterface { private targetHash; - hashIndexManager: HashIndexManager; - hashComputer: HashComputer; + private hashIndexManager; + private hashComputer; constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string); computeMissingHash(filePath: string, metadata: MultiHashData): Promise; + getHashIndexManager(): Promise; } declare function existsAsync(filePath: string): Promise; @@ -69,6 +64,13 @@ declare class FileIDComputer { computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; } +declare class HashComputer implements ComputeInterface { + private targetHash; + private fileIDComputer; + constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + interface Properties { TARGET_HASHES: string; INDEX_FILE_PATH: string; diff --git a/dist/index.js b/dist/index.js index e7bd031..625d269 100644 --- a/dist/index.js +++ b/dist/index.js @@ -8,11 +8,11 @@ import { access, constants, stat, readdir } from 'fs/promises'; import * as c from 'path'; import c__default from 'path'; import { clearInterval } from 'node:timers'; -import x from 'chokidar'; -import b from 'p-queue'; +import w from 'chokidar'; +import R from 'p-queue'; -var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var g=new URL(l).href;console.log("Worker URL: ",g);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||g});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,o)=>{let a=r[o];t[a]=s;});}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(o=>i.includes(o))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,o)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",a=>{o(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let o=this.loadIndexFromCache().filter(a=>!i.find(n=>n.path===a.path));if(o.length!==0){let a=stringify(o,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],a);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let n of this.targetHash)if(!i[n])throw new Error(`Missing hash ${n}`);let s=t+"",o=c__default.basename(e),a=this.cache.get(o);if(a)for(let n of this.targetHash)a[n]=i[n];else {let n={path:o,size:s,mtime:r,...i};this.cache.set(o,n);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let o=this.processFile(s);t.push(o);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let o=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(o)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(o)?1:0));}return (await Promise.all(r)).reduce((s,o)=>s+o,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=x.watch(e,t):r=x.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let o=0;oo+a,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let o=0;o!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,a)=>{let n=r[a];t[n]=s;});}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),r=await stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={path:a,size:s,mtime:r,...i};this.cache.set(a,o);}this.hasChanged=!0;}};var x=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var M=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new R({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w.watch(e,t):r=w.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileName","fileNameIndex","fileSize","mtime","hashs","size","baseName","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"sDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA8C,CAGjD,YAAoBC,EAAkCJ,EAAoB,CAAtD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYL,EAAUK,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,CACJ,EC5BA,OAAQ,oBAAAG,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYjB,EAAoC,CAClE,GAAI,CACA,aAAMe,EAAOf,EAAUgB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYtB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWuB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGvB,CAAQ,IAAIuB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY7B,EAAU0B,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB5C,EAAsC,CAClE,IAAM6C,EAAW3B,EAAK,SAASlB,CAAQ,EACjCiC,EAAQ,MAAMb,EAAKpB,CAAQ,EAC7B8C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUb,EAAM,KAAO,IAAOa,EAAc,QAAUb,EAAM,MAAM,YAAY,EAC5F,OAAOa,UAIPA,EAAc,OAAUb,EAAM,KAAO,GACrC,OAAOa,EAWnB,OAAO,IACX,CAGO,cAAc9C,EAAkB+C,EAAkBC,EAA0B,CAC/E,IAAMH,EAAW3B,EAAK,SAASlB,CAAQ,EACnC8C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUC,EAAW,IAAOD,EAAc,QAAUE,EAClE,OAAOF,UAIPA,EAAc,OAAUC,EAAW,GACnC,OAAOD,EAWnB,OAAO,IACX,CAEO,WAAW9C,EAAkB+C,EAAkBC,EAAeC,EAA2D,CAC5H,GAAI,CAACjD,GAAY,CAAC+C,GAAY,CAACC,GAAS,CAACC,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOH,EAAW,GAClBI,EAAWjC,EAAK,SAASlB,CAAQ,EACnCmC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAKD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MANpB,CACZ,IAAM6B,EAAO,CAAC,KAAMD,EAAU,KAAMD,EAAM,MAAOF,EAAO,GAAGC,CAAK,EAChE,KAAK,MAAM,IAAIE,EAAUC,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EE3RA,OAAQ,QAAAhC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BnD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIuB,EAAiBgC,EAAcnD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI6B,EAAQ,MAAMb,EAAKpB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIkB,EAAK,SAASlB,CAAQ,CAAC,EAAG,CAC/D,IAAMmC,EAAY,KAAK,iBAAiB,cAAcnC,EAAUiC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACnB,EAASmB,CAAI,GAAKY,EAAUZ,CAAI,IACjCnB,EAASmB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBvB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUiC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG7B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYc,MAAU,OACtB,OAAOqC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYtE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMsE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWtE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIuE,EAAe,UAAQvE,CAAQ,EACnC,MAAM,KAAK,iBAAiBuE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO1E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2E,EAAO,CACZ,QAAQ,MAAM,yBAAyB3E,CAAQ,IAAK2E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2B,EAAG,CACR,QAAQ,MAAM,yBAAyB3B,CAAQ,IAAK2B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUgD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/worker.d.cts b/dist/worker.d.cts index 416bf67..16d7e99 100644 --- a/dist/worker.d.cts +++ b/dist/worker.d.cts @@ -1 +1 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.cjs'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.cjs'; diff --git a/dist/worker.d.ts b/dist/worker.d.ts index e6a7c0f..7404512 100644 --- a/dist/worker.d.ts +++ b/dist/worker.d.ts @@ -1 +1 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.js'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.js'; diff --git a/jest.config.ts b/jest.config.ts deleted file mode 100644 index 29e0859..0000000 --- a/jest.config.ts +++ /dev/null @@ -1,205 +0,0 @@ -/* - * For a detailed explanation regarding each configuration property and type check, visit: - * https://jestjs.io/docs/configuration - */ - -export default { - // All imported modules in your tests should be mocked automatically - // automock: false, - - // Stop running tests after `n` failures - // bail: 0, - - // The directory where Jest should store its cached dependency information - // cacheDirectory: "C:\\Users\\Pierre-HenriFRICOT\\AppData\\Local\\Temp\\jest", - - // Automatically clear mock calls, instances, contexts and results before every test - // clearMocks: false, - - // Indicates whether the coverage information should be collected while executing the test - //collectCoverage: true, - - // An array of glob patterns indicating a set of files for which coverage information should be collected - // collectCoverageFrom: undefined, - - // The directory where Jest should output its coverage files - //coverageDirectory: "coverage", - - // An array of regexp pattern strings used to skip coverage collection - // coveragePathIgnorePatterns: [ - // "\\\\node_modules\\\\" - // ], - - // Indicates which provider should be used to instrument code for coverage - // coverageProvider: "babel", - - // A list of reporter names that Jest uses when writing coverage reports - // coverageReporters: [ - // "json", - // "text", - // "lcov", - // "clover" - // ], - - // An object that configures minimum threshold enforcement for coverage results - // coverageThreshold: undefined, - - // A path to a custom dependency extractor - // dependencyExtractor: undefined, - - // Make calling deprecated APIs throw helpful error messages - // errorOnDeprecated: false, - - // The default configuration for fake timers - // fakeTimers: { - // "enableGlobally": false - // }, - - // Force coverage collection from ignored files using an array of glob patterns - // forceCoverageMatch: [], - - // A path to a module which exports an async function that is triggered once before all test suites - // globalSetup: undefined, - - // A path to a module which exports an async function that is triggered once after all test suites - // globalTeardown: undefined, - - // A set of global variables that need to be available in all test environments - // globals: {}, - - // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. - // maxWorkers: "50%", - - // An array of directory names to be searched recursively up from the requiring module's location - moduleDirectories: [ - "node_modules", - "src" - ], - - // An array of file extensions your modules use - moduleFileExtensions: [ - "js", - "mjs", - "cjs", - "jsx", - "ts", - "tsx", - "json", - "node", - "web.js", - "web.ts", - "web.tsx", - "web.jsx", - ], - - // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module - "moduleNameMapper": { - "@root(.*)$": "/src/lib/$1", - }, - - // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader - // modulePathIgnorePatterns: [], - - // Activates notifications for test results - // notify: false, - - // An enum that specifies notification mode. Requires { notify: true } - // notifyMode: "failure-change", - - // A preset that is used as a base for Jest's configuration - // preset: undefined, - - // Run tests from one or more projects - // projects: undefined, - - // Use this configuration option to add custom reporters to Jest - // reporters: undefined, - - // Automatically reset mock state before every test - // resetMocks: false, - - // Reset the module registry before running each individual test - // resetModules: false, - - // A path to a custom resolver - // resolver: undefined, - - // Automatically restore mock state and implementation before every test - // restoreMocks: false, - - // The root directory that Jest should scan for tests and modules within - // rootDir: undefined, - - // A list of paths to directories that Jest should use to search for files in - // roots: [ - // "" - // ], - - // Allows you to use a custom runner instead of Jest's default test runner - // runner: "jest-runner", - - // The paths to modules that run some code to configure or set up the testing environment before each test - // setupFiles: [], - - // A list of paths to modules that run some code to configure or set up the testing framework before each test - // setupFilesAfterEnv: [], - - // The number of seconds after which a test is considered as slow and reported as such in the results. - // slowTestThreshold: 5, - - // A list of paths to snapshot serializer modules Jest should use for snapshot testing - // snapshotSerializers: [], - - // The test environment that will be used for testing - // testEnvironment: "jest-environment-node", - - // Options that will be passed to the testEnvironment - // testEnvironmentOptions: {}, - - // Adds a location field to test results - // testLocationInResults: false, - - // The glob patterns Jest uses to detect test files - testMatch: [ - "**/src/tests/**/*.[jt]s?(x)", - "**/?(*.)+(spec|test).[tj]s?(x)" - ], - - // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped - // testPathIgnorePatterns: [ - // "\\\\node_modules\\\\" - // ], - - // The regexp pattern or array of patterns that Jest uses to detect test files - // testRegex: [], - - // This option allows the use of a custom results processor - // testResultsProcessor: undefined, - - // This option allows use of a custom test runner - // testRunner: "jest-circus/runner", - - // A map from regular expressions to paths to transformers - // transform: undefined, - - // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation - // transformIgnorePatterns: [ - // "\\\\node_modules\\\\", - // "\\.pnp\\.[^\\\\]+$" - // ], - "transformIgnorePatterns": [ - "node_modules/(?!@ngrx|(?!deck.gl)|ng-dynamic)" - ], - - // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them - // unmockedModulePathPatterns: undefined, - - // Indicates whether each individual test should be reported during the run - // verbose: undefined, - - // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode - // watchPathIgnorePatterns: [], - - // Whether to use watchman for file crawling - // watchman: true, -}; diff --git a/src/lib/ComputeHashIndexCache.ts b/src/lib/ComputeHashIndexCache.ts index e512bad..ec71957 100644 --- a/src/lib/ComputeHashIndexCache.ts +++ b/src/lib/ComputeHashIndexCache.ts @@ -5,15 +5,15 @@ import {stat} from "fs/promises"; import path from "path"; export class ComputeHashIndexCache implements ComputeInterface { - hashIndexManager: HashIndexManager; - hashComputer: HashComputer; + private hashIndexManager: HashIndexManager; + private hashComputer: HashComputer; constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { this.hashComputer = new HashComputer(targetHash,workerPath); this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash); } - async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { + public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { await this.hashIndexManager.init(); let stats = await stat(filePath); if (this.hashIndexManager.getCache().has(path.basename(filePath))) { @@ -29,4 +29,9 @@ export class ComputeHashIndexCache implements ComputeInterface { await this.hashComputer.computeMissingHash(filePath, metadata); this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata); } + + async getHashIndexManager(): Promise { + await this.hashIndexManager.init(); + return this.hashIndexManager; + } } \ No newline at end of file diff --git a/src/lib/HashIndexManager.ts b/src/lib/HashIndexManager.ts index 9f6f296..481466a 100644 --- a/src/lib/HashIndexManager.ts +++ b/src/lib/HashIndexManager.ts @@ -6,6 +6,7 @@ import {existsAsync} from "./ExistsAsync"; import path from "path"; import {clearInterval} from "node:timers"; import {CID_ALGORITHM_NAMES} from "./MultiHashData"; +import {stat} from "fs/promises"; interface IndexLine extends Partial> { path: string; @@ -163,7 +164,7 @@ export class HashIndexManager { }); } - private async saveCacheToFile(): Promise { + public async saveCacheToFile(): Promise { if (this.indexOpsInProgress || !this.hasChanged) { return; } @@ -205,6 +206,34 @@ export class HashIndexManager { this.indexOpsInProgress = false; } + public async getCidForFileAsync(filePath: string): Promise { + const fileName = path.basename(filePath); + const stats = await stat(filePath); + let fileNameIndex = this.cache.get(fileName); + if (fileNameIndex) { + if (fileNameIndex.mtime) { + //if we have a mtime, we need to check it + if (fileNameIndex.size === (stats.size + "") && fileNameIndex.mtime === stats.mtime.toISOString()) { + return fileNameIndex; + } + } else { + //mtime is optional + if (fileNameIndex.size === (stats.size + "")) { + return fileNameIndex; + } + } + } + + // 3 - if not found, delete the entry (keeps the index clean) + /*if (fileNameIndex && fileNameIndex.size !== (fileSize + "") && pathIndex && pathIndex.size !== (fileSize + "")) { + this.cache.delete(fileName); + this.cache.delete(filePath); + }*/ + + return null; + } + + public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine { const fileName = path.basename(filePath); let fileNameIndex = this.cache.get(fileName); diff --git a/src/tests/main.test.ts b/src/tests/main.test.ts index d48b349..39a3ab1 100644 --- a/src/tests/main.test.ts +++ b/src/tests/main.test.ts @@ -1,7 +1,8 @@ -import {CID_ALGORITHM_NAMES, HashComputer} from "../lib"; +import {CID_ALGORITHM_NAMES, ComputeHashIndexCache, HashComputer} from "../lib"; import {describe, expect, it} from 'vitest'; import * as process from "process"; +import { mkdir, rm, rmdir } from "fs/promises"; process.env.WORKER_URL = "./dist/worker.js"; @@ -39,4 +40,58 @@ describe('add', () => { "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' }); }); + it('meta data compute with index', async () => { + await rm("./tmp",{recursive:true}); + await mkdir("./tmp",{recursive:true}); + let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[ + CID_ALGORITHM_NAMES.sha1, + CID_ALGORITHM_NAMES.sha256, + CID_ALGORITHM_NAMES.md5, + CID_ALGORITHM_NAMES.sha3_256, + CID_ALGORITHM_NAMES.crc32, + CID_ALGORITHM_NAMES.sha3_384 + ]); + let metadata = {}; + await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); + console.log(metadata); + /** + * test.txt CF20B62A (CRC32) + * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) + * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt + * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt + * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt + */ + expect(metadata).toEqual({ + "cid_crc32": "bagzafmqcathsbnrk", + "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', + "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', + "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', + "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', + "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' + }); + await (await hashComputer.getHashIndexManager()).saveCacheToFile(); + }); + it('cache', async () => { + await mkdir("./tmp",{recursive:true}); + let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[ + CID_ALGORITHM_NAMES.sha1, + CID_ALGORITHM_NAMES.sha256, + CID_ALGORITHM_NAMES.md5, + CID_ALGORITHM_NAMES.sha3_256, + CID_ALGORITHM_NAMES.crc32, + CID_ALGORITHM_NAMES.sha3_384 + ]); + let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt'); + delete indexLine['mtime']; + expect(indexLine).toEqual({ + "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', + "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', + "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', + "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', + "cid_crc32": "bagzafmqcathsbnrk", + "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga', + "path": "test.txt", + "size": "499", + }); + }); }); \ No newline at end of file diff --git a/tmp/index-cid_crc32.csv b/tmp/index-cid_crc32.csv new file mode 100644 index 0000000..0b7bd3f --- /dev/null +++ b/tmp/index-cid_crc32.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_crc32 +test.txt,499,2024-04-02T16:35:33.190Z,bagzafmqcathsbnrk diff --git a/tmp/index-cid_md5.csv b/tmp/index-cid_md5.csv new file mode 100644 index 0000000..c5c76c2 --- /dev/null +++ b/tmp/index-cid_md5.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_md5 +test.txt,499,2024-04-02T16:35:33.190Z,bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa diff --git a/tmp/index-cid_sha1.csv b/tmp/index-cid_sha1.csv new file mode 100644 index 0000000..2c1f1a8 --- /dev/null +++ b/tmp/index-cid_sha1.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_sha1 +test.txt,499,2024-04-02T16:35:33.190Z,baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra diff --git a/tmp/index-cid_sha2-256.csv b/tmp/index-cid_sha2-256.csv new file mode 100644 index 0000000..3df0898 --- /dev/null +++ b/tmp/index-cid_sha2-256.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_sha2-256 +test.txt,499,2024-04-02T16:35:33.190Z,baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua diff --git a/tmp/index-cid_sha3-256.csv b/tmp/index-cid_sha3-256.csv new file mode 100644 index 0000000..c899d08 --- /dev/null +++ b/tmp/index-cid_sha3-256.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_sha3-256 +test.txt,499,2024-04-02T16:35:33.190Z,baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq diff --git a/tmp/index-cid_sha3-384.csv b/tmp/index-cid_sha3-384.csv new file mode 100644 index 0000000..e69ad9b --- /dev/null +++ b/tmp/index-cid_sha3-384.csv @@ -0,0 +1,2 @@ +path,size,mtime,cid_sha3-384 +test.txt,499,2024-04-02T16:35:33.190Z,baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga diff --git a/vitest.config.ts b/vitest.config.ts index 519169b..293a3f5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,7 @@ export default defineConfig({ test: { globals: true, environment: 'node', + testTimeout: 300000, // Set global timeout to 30 seconds // Use ts-jest or esbuild to handle TypeScript files // Specify other configurations as needed },