61 lines
8.7 KiB
JavaScript
61 lines
8.7 KiB
JavaScript
'use strict';
|
|
|
|
var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs');
|
|
var piscina = require('piscina');
|
|
var fs = require('fs');
|
|
var csvParse = require('csv-parse');
|
|
var sync = require('csv-parse/sync');
|
|
var sync$1 = require('csv-stringify/sync');
|
|
var promises = require('fs/promises');
|
|
var c = require('path');
|
|
var timers = require('timers');
|
|
var w = require('chokidar');
|
|
var R = require('p-queue');
|
|
|
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
|
function _interopNamespace(e) {
|
|
if (e && e.__esModule) return e;
|
|
var n = Object.create(null);
|
|
if (e) {
|
|
Object.keys(e).forEach(function (k) {
|
|
if (k !== 'default') {
|
|
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
Object.defineProperty(n, k, d.get ? d : {
|
|
enumerable: true,
|
|
get: function () { return e[k]; }
|
|
});
|
|
}
|
|
});
|
|
}
|
|
n.default = e;
|
|
return Object.freeze(n);
|
|
}
|
|
|
|
var c__namespace = /*#__PURE__*/_interopNamespace(c);
|
|
var w__default = /*#__PURE__*/_interopDefault(w);
|
|
var R__default = /*#__PURE__*/_interopDefault(R);
|
|
|
|
var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var I=new URL(l).href;console.log("Worker URL: ",I);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||I});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,a)=>{let n=r[a];t[n]=s;});}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=sync$1.stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),r=await promises.stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={path:a,size:s,mtime:r,...i};this.cache.set(a,o);}this.hasChanged=!0;}};var x=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var M=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new R__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w__default.default.watch(e,t):r=w__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;a<t.length;a++)t[a]=c__namespace.normalize(t[a]),r.push(this.countFile(t[a]));let i=(await Promise.all(r)).reduce((a,n)=>a+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a<t.length;a++)t[a]=c__namespace.normalize(t[a]),await this.processDirectory(t[a],s);await Promise.all(s),this.initialized=!0,console.log("Watcher ready");}};
|
|
|
|
Object.defineProperty(exports, "CID_ALGORITHM", {
|
|
enumerable: true,
|
|
get: function () { return chunkA5PD5ID4_cjs.c; }
|
|
});
|
|
Object.defineProperty(exports, "CID_ALGORITHM_CODES", {
|
|
enumerable: true,
|
|
get: function () { return chunkA5PD5ID4_cjs.a; }
|
|
});
|
|
Object.defineProperty(exports, "CID_ALGORITHM_NAMES", {
|
|
enumerable: true,
|
|
get: function () { return chunkA5PD5ID4_cjs.b; }
|
|
});
|
|
exports.ComputeHashIndexCache = x;
|
|
exports.FileIDComputer = p;
|
|
exports.FolderWatcher = M;
|
|
exports.HashComputer = m;
|
|
exports.HashIndexManager = d;
|
|
exports.INDEX_HEADERS = D;
|
|
exports.existsAsync = f;
|
|
//# sourceMappingURL=out.js.map
|
|
//# sourceMappingURL=index.cjs.map
|