export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-FGATABDV.js'; import { Piscina } from 'piscina'; import { promises, createReadStream } from 'fs'; import { parse as parse$1 } from 'csv-parse'; import { parse } from 'csv-parse/sync'; import { stringify } from 'csv-stringify/sync'; import { access, constants, stat, readdir } from 'fs/promises'; import * as c from 'path'; import c__default from 'path'; import { clearInterval } from 'node:timers'; import x from 'chokidar'; import b from 'p-queue'; var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var g=new URL(l).href;console.log("Worker URL: ",g);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||g});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,o)=>{let a=r[o];t[a]=s;});}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(o=>i.includes(o))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,o)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",a=>{o(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let o=this.loadIndexFromCache().filter(a=>!i.find(n=>n.path===a.path));if(o.length!==0){let a=stringify(o,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],a);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let n of this.targetHash)if(!i[n])throw new Error(`Missing hash ${n}`);let s=t+"",o=c__default.basename(e),a=this.cache.get(o);if(a)for(let n of this.targetHash)a[n]=i[n];else {let n={path:o,size:s,mtime:r,...i};this.cache.set(o,n);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let o=this.processFile(s);t.push(o);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let o=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(o)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(o)?1:0));}return (await Promise.all(r)).reduce((s,o)=>s+o,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=x.watch(e,t):r=x.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let o=0;oo+a,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let o=0;o