diff --git a/.gitignore b/.gitignore index a35a66d..bf61c15 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,5 @@ yarn-error.log* /junit.xml /.nyc_output/ /coverage/ -/dist/ +#/dist/ .idea diff --git a/dist/index.cjs b/dist/index.cjs new file mode 100644 index 0000000..bd7a1b2 --- /dev/null +++ b/dist/index.cjs @@ -0,0 +1,30 @@ +'use strict'; + +var piscina = require('piscina'); +var fs = require('fs'); +var csvParse = require('csv-parse'); +var sync = require('csv-parse/sync'); +var sync$1 = require('csv-stringify/sync'); +var promises = require('fs/promises'); +var y = require('path'); +var timers = require('timers'); +require('multiformats/cid'); +require('multiformats/hashes/digest'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var y__default = /*#__PURE__*/_interopDefault(y); + +var C=Object.defineProperty;var g=Object.getOwnPropertySymbols;var H=Object.prototype.hasOwnProperty,b=Object.prototype.propertyIsEnumerable;var x=(a,e,t)=>e in a?C(a,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):a[e]=t,I=(a,e)=>{for(var t in e||(e={}))H.call(e,t)&&x(a,t,e[t]);if(g)for(var t of g(e))b.call(e,t)&&x(a,t,e[t]);return a};var n=(a,e,t)=>new Promise((i,s)=>{var r=c=>{try{h(t.next(c));}catch(f){s(f);}},o=c=>{try{h(t.throw(c));}catch(f){s(f);}},h=c=>c.done?i(c.value):Promise.resolve(c.value).then(r,o);h((t=t.apply(a,e)).next());});var S={};var m=class{constructor(){this.piscina=new piscina.Piscina({maxThreads:4,filename:new URL("./ShaComputeWorker.js",S.url).href});}computeCIDs(e,t){return n(this,null,function*(){return this.piscina.run({filePath:e,algorithms:t})})}};var d=class{constructor(e){this.targetHash=e;this.fileIDComputer=new m;}computeMissingHash(e,t){return n(this,null,function*(){let i=this.targetHash.filter(r=>!t[r]);if(i.length===0)return;(yield this.fileIDComputer.computeCIDs(e,i)).forEach((r,o)=>{let h=i[o];t[h]=r;});})}};var v=(s=>(s[s.sha256=18]="sha256",s[s.sha1=17]="sha1",s[s.md5=213]="md5",s[s.sha3_256=22]="sha3_256",s))(v||{}),p=(s=>(s.sha256="cid_sha2-256",s.sha1="cid_sha1",s.md5="cid_md5",s.sha3_256="cid_sha3-256",s))(p||{}),F=(s=>(s.sha256="sha2-256",s.sha1="sha1",s.md5="md5",s.sha3_256="sha3-256",s))(F||{});function u(a){return n(this,null,function*(){try{return yield promises.access(a,promises.constants.F_OK),!0}catch(e){return !1}})}var ie=["path","size","mtime"],l=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;this.cache=new Map;this.intervalTime=3e4;this.lastIndexFileSize=0;this.indexOpsInProgress=!1;this.hasChanged=!1;if(!e)throw new Error("Invalid index file path")}getCache(){return new Map(this.cache)}init(e=!0){return n(this,null,function*(){return this.initialLoad||(this.initialLoad=new Promise((t,i)=>n(this,null,function*(){try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");yield this.loadIndex(),e&&this.start(),t();}catch(s){i(s);}}))),this.initialLoad})}checkCSVHeaders(e){let t=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let i=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(r=>i.includes(r))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}loadIndex(){return n(this,null,function*(){if(yield u(this.filePath)){let e=yield fs.promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=yield this.readCsv();for(let i of t)this.cache.set(i.path,i);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []})}loadIndexFromCache(){return Array.from(this.cache.values())}readCsv(){return n(this,null,function*(){if(!(yield u(this.filePath)))return [];let e=performance.now(),t=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,r)=>{fs.createReadStream(this.filePath).pipe(t).on("data",o=>{i.push(o);}).on("end",()=>{s(i),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",o=>{r(o);});})})}saveCacheToFile(){return n(this,null,function*(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=yield this.loadIndex();if(this.cache.size!==0){let r=this.loadIndexFromCache().filter(o=>!t.find(h=>h.path===o.path));if(r.length!==0){let o=sync$1.stringify(r,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(h=>({key:h,header:h}))]});yield fs.promises.appendFile(this.filePath,o);}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;})}getCidForFile(e,t,i){let s=y__default.default.basename(e),r=this.cache.get(s);if(r){if(r.mtime){if(r.size===t+""&&r.mtime===i)return r}else if(r.size===t+"")return r}return null}addFileCid(e,t,i,s){if(!e||!t||!i||!s)throw new Error("Invalid parameters");let r=t+"",o=y__default.default.basename(e),h=I({path:o,size:r,mtime:i},s);this.cache.set(o,h),this.hasChanged=!0;}};var M=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;this.hashComputer=new d(this.targetHash),this.hashIndexManager=new l(e);}computeMissingHash(e,t){return n(this,null,function*(){yield this.hashIndexManager.init();let i=yield promises.stat(e);if(this.hashIndexManager.getCache().has(y__default.default.basename(e))){let s=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(s)for(let r of this.targetHash)!t[r]&&s[r]&&(t[r]=s[r]);}yield this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);})}}; + +exports.CID_ALGORITHM = F; +exports.CID_ALGORITHM_CODES = v; +exports.CID_ALGORITHM_NAMES = p; +exports.ComputeHashIndexCache = M; +exports.FileIDComputer = m; +exports.HashComputer = d; +exports.HashIndexManager = l; +exports.INDEX_HEADERS = ie; +exports.existsAsync = u; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/dist/index.cjs.map b/dist/index.cjs.map new file mode 100644 index 0000000..91596b6 --- /dev/null +++ b/dist/index.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/MultiHashData.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/fileID/ShaComputeWorker.ts"],"names":["Piscina","import_meta","FileIDComputer","filePath","algorithms","__async","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","INDEX_HEADERS","HashIndexManager","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","start","parser","err","existingRows","newRows","row","existingRow","csvString","hash","totalTime","fileSize","mtime","fileName","fileNameIndex","hashs","size","baseName","data","__spreadValues","stat","ComputeHashIndexCache","indexFilePath","indexLine","CID","create"],"mappings":"siBAAA,OAAQ,WAAAA,MAAc,UAAtB,IAAAC,EAAA,GAGO,IAAMC,EAAN,KAAqB,CAGxB,aAAc,CACV,KAAK,QAAU,IAAIF,EAAQ,CACvB,WAAY,EACZ,SAAU,IAAI,IAAI,wBAAyBC,EAAY,GAAG,EAAE,IAChE,CAAC,CACL,CAQa,YAAYE,EAAkBC,EAAsD,QAAAC,EAAA,sBAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAF,EAAU,WAAAC,CAAU,CAAC,CAClD,GACJ,ECnBO,IAAME,EAAN,KAA8C,CAGjD,YAAoBC,EAAmC,CAAnC,gBAAAA,EAFpB,KAAQ,eAAiB,IAAIL,CAG7B,CAEM,mBAAmBC,EAAkBK,EAAwC,QAAAH,EAAA,sBAE/E,IAAMI,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYN,EAAUM,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,GACJ,ECfO,IAAKG,OACRA,IAAA,OAAS,IAAT,SACAA,IAAA,KAAO,IAAP,OACAA,IAAA,IAAM,KAAN,MACAA,IAAA,SAAW,IAAX,WAJQA,OAAA,IAQAC,OACRA,EAAA,OAAS,eACTA,EAAA,KAAO,WACPA,EAAA,IAAM,UACNA,EAAA,SAAW,eAJHA,OAAA,IAQAC,OACRA,EAAA,OAAS,WACTA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,SAAW,WAJHA,OAAA,IC5BZ,OAAQ,oBAAAC,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,SAAsBC,EAAYrB,EAAoC,QAAAE,EAAA,sBAClE,GAAI,CACA,aAAMiB,EAAOnB,EAAUoB,EAAU,IAAI,EAC9B,EACX,OAAQ,GACJ,MAAO,EACX,CACJ,GDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAQrB,IAAMC,GAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAU1B,YAAoBzB,EAAyBI,EAAkC,0BAAoD,EAAG,CAAlH,cAAAJ,EAAyB,gBAAAI,EAT7C,KAAQ,MAAgC,IAAI,IAE5C,KAAQ,aAAuB,IAC/B,KAAQ,kBAA4B,EAEpC,KAAQ,mBAA8B,GACtC,KAAQ,WAAsB,GAI1B,GAAI,CAACJ,EACD,MAAM,IAAI,MAAM,yBAAyB,CAEjD,CAEO,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMM,KAAK0B,EAAW,GAAM,QAAAxB,EAAA,sBACxB,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,CAAOyB,EAASC,IAAW1B,EAAA,sBAC5D,GAAI,CACA,GAAI,CAAC,KAAK,gBAAgB,KAAK,QAAQ,EACnC,MAAM,IAAI,MAAM,4BAA4B,EAEhD,MAAM,KAAK,UAAU,EACjBwB,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAAQE,EAAG,CACPD,EAAOC,CAAC,CACZ,CACJ,EAAC,GAEE,KAAK,WAChB,GAGA,gBAAiBC,EAA6B,CAC1C,IAAMC,EAAUd,EAAUa,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,OAAQ,OAAQ,QAAS,GAAG,KAAK,UAAU,EAG7C,MAAME,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLV,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAcW,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEa,WAAkC,QAAAhC,EAAA,sBAC3C,GAAI,MAAMmB,EAAY,KAAK,QAAQ,EAAG,CAElC,IAAMc,EAAQ,MAAMpB,EAAG,KAAK,KAAK,QAAQ,EACzC,GAAI,KAAK,oBAAsBoB,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQ,EAChD,QAAWK,KAAUL,EACjB,KAAK,MAAM,IAAIK,EAAO,KAAMA,CAAM,EAEtC,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,GAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEc,SAAgC,QAAA7B,EAAA,sBAC1C,GAAI,EAAE,MAAMmB,EAAY,KAAK,QAAQ,GACjC,MAAO,CAAC,EAEZ,IAAMgB,EAAQ,YAAY,IAAI,EAExBC,EAAStB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKe,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpCd,EAAiB,KAAK,QAAQ,EACzB,KAAKwB,CAAM,EACX,GAAG,OAASF,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,mBAAmB,YAAY,IAAI,EAAIM,CAAK,IAAI,CAChE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBX,EAAOW,CAAG,CACd,CAAC,CACL,CAAC,CACL,GAEc,iBAAiC,QAAArC,EAAA,sBAC3C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMmC,EAAQ,YAAY,IAAI,EAC1BG,EAA4B,MAAM,KAAK,UAAU,EAErD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMC,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY1B,EAAUuB,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,GAAG,KAAK,WAAW,IAAIK,IAAS,CAAC,IAAKA,EAAM,OAAQA,CAAI,EAAE,CAC9D,CACJ,CAAC,EAGD,MAAM9B,EAAG,WAAW,KAAK,SAAU6B,CAAS,CAChD,CACJ,CACA,IAAME,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,GAEO,cAAc9C,EAAkB+C,EAAkBC,EAA0B,CAC/E,IAAMC,EAAW3B,EAAK,SAAStB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAGC,GACC,GAAGA,EAAc,OAEb,GAAIA,EAAc,OAAUH,EAAW,IAAOG,EAAc,QAAUF,EAClE,OAAOE,UAIPA,EAAc,OAAUH,EAAW,GACnC,OAAOG,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkB+C,EAAkBC,EAAeG,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAAC+C,GAAY,CAACC,GAAS,CAACG,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAMC,EAAOL,EAAW,GAClBM,EAAW/B,EAAK,SAAStB,CAAQ,EACjCsD,EAAOC,EAAA,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOJ,GAAUG,GAC3D,KAAK,MAAM,IAAIE,EAAUC,CAAI,EAC7B,KAAK,WAAa,EACtB,CAEJ,EEjOA,OAAQ,QAAAE,MAAW,cACnB,OAAOlC,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA+BtD,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EACvC,KAAK,aAAe,IAAID,EAAa,KAAK,UAAU,EACpD,KAAK,iBAAmB,IAAIsB,EAAiBiC,CAAa,CAC9D,CAEM,mBAAmB1D,EAAkBK,EAAwC,QAAAH,EAAA,sBAC/E,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAIiC,EAAQ,MAAMqB,EAAKxD,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIsB,EAAK,SAAStB,CAAQ,CAAC,EAAG,CAC/D,IAAM2D,EAAY,KAAK,iBAAiB,cAAc3D,EAAUmC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGwB,EACC,QAAWd,KAAQ,KAAK,WAChB,CAACxC,EAASwC,CAAI,GAAKc,EAAUd,CAAI,IACjCxC,EAASwC,CAAI,EAAIc,EAAUd,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmB7C,EAAUK,CAAQ,EAC7D,KAAK,iBAAiB,WAAWL,EAAUmC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,GACJ,EC7BA,OAAS,OAAAuD,OAAW,mBACpB,OAAS,UAAAC,OAAc","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData.js\";\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor() {\n this.piscina = new Piscina({\n maxThreads: 4,\n filename: new URL('./ShaComputeWorker.js', import.meta.url).href\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer.js\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData.js\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer = new FileIDComputer();\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\nexport enum CID_ALGORITHM_CODES {\n sha256 = 0x12,\n sha1 = 0x11,\n md5 = 0xd5,\n sha3_256 = 0x16,\n //crc32 = 0x0132\n}\n\nexport enum CID_ALGORITHM_NAMES {\n sha256 = 'cid_sha2-256',\n sha1 = 'cid_sha1',\n md5 = 'cid_md5',\n sha3_256 = 'cid_sha3-256',\n //crc32 = 'cid_crc32'\n}\n\nexport enum CID_ALGORITHM {\n sha256 = 'sha2-256',\n sha1 = 'sha1',\n md5 = 'md5',\n sha3_256 = 'sha3-256',\n //crc32 = 'crc32'\n}","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync.js\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData.js\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad;\n\n constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {\n if (!filePath) {\n throw new Error('Invalid index file path');\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n if (!this.checkCSVHeaders(this.filePath)) {\n throw new Error('Invalid index file headers');\n }\n await this.loadIndex();\n if (autosave) {\n this.start();\n }\n resolve();\n }catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n checkCSVHeaders (csvContent: string): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(): Promise {\n if (await existsAsync(this.filePath)) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePath);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv();\n for (const record of records) {\n this.cache.set(record.path, record);\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(): Promise {\n if (!(await existsAsync(this.filePath))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePath)\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n let existingRows: IndexLine[] = await this.loadIndex();\n\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n ...this.targetHash.map(hash => ({key: hash, header: hash})),\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePath, csvString);\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if(fileNameIndex) {\n if(fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n }else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer.js\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData.js\";\nimport {HashIndexManager} from \"./HashIndexManager.js\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n this.hashComputer = new HashComputer(this.targetHash);\n this.hashIndexManager = new HashIndexManager(indexFilePath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import { createReadStream } from \"fs\";\nimport {BinaryLike, createHash} from 'crypto';\nimport { CID } from 'multiformats/cid';\nimport { create } from \"multiformats/hashes/digest\";\nimport {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData.js\";\n\nexport interface SimpleHash {\n update(data: BinaryLike),\n digest(): Buffer;\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },\n [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },\n [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },\n [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n"]} \ No newline at end of file diff --git a/dist/index.d.cts b/dist/index.d.cts new file mode 100644 index 0000000..06c8d44 --- /dev/null +++ b/dist/index.d.cts @@ -0,0 +1,106 @@ +import { BinaryLike } from 'crypto'; + +interface MultiHashData { + "cid_sha2-256"?: string; + cid_sha1?: string; + cid_md5?: string; + "cid_sha3-256"?: string; +} +interface ComputeInterface { + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} +declare enum CID_ALGORITHM_CODES { + sha256 = 18, + sha1 = 17, + md5 = 213, + sha3_256 = 22 +} +declare enum CID_ALGORITHM_NAMES { + sha256 = "cid_sha2-256", + sha1 = "cid_sha1", + md5 = "cid_md5", + sha3_256 = "cid_sha3-256" +} +declare enum CID_ALGORITHM { + sha256 = "sha2-256", + sha1 = "sha1", + md5 = "md5", + sha3_256 = "sha3-256" +} + +declare class HashComputer implements ComputeInterface { + private targetHash; + private fileIDComputer; + constructor(targetHash: CID_ALGORITHM_NAMES[]); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + +interface IndexLine extends Partial> { + path: string; + size: string; + mtime: string; +} +declare const INDEX_HEADERS: string[]; +declare class HashIndexManager { + private filePath; + private targetHash; + private cache; + private intervalId; + private intervalTime; + private lastIndexFileSize; + private lastCacheFile; + private indexOpsInProgress; + private hasChanged; + private initialLoad; + constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]); + getCache(): Map; + /** + * After init consseutively calls to this method will not reload the index + * @param autosave + */ + init(autosave?: boolean): Promise; + checkCSVHeaders(csvContent: string): boolean; + start(): void; + stopAutoSave(): void; + private startAutoSave; + loadIndex(): Promise; + private loadIndexFromCache; + private readCsv; + private saveCacheToFile; + getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine; + addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; +} + +declare class ComputeHashIndexCache implements ComputeInterface { + private targetHash; + hashIndexManager: HashIndexManager; + hashComputer: HashComputer; + constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[]); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + +declare function existsAsync(filePath: string): Promise; + +declare class FileIDComputer { + private piscina; + constructor(); + /** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ + computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} + +interface SimpleHash { + update(data: BinaryLike): any; + digest(): Buffer; +} + +interface Properties { + TARGET_HASHES: string; + INDEX_FILE_PATH: string; +} + +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES, ComputeHashIndexCache, type ComputeInterface, FileIDComputer, HashComputer, HashIndexManager, INDEX_HEADERS, type MultiHashData, type Properties, type SimpleHash, existsAsync }; diff --git a/dist/index.d.ts b/dist/index.d.ts new file mode 100644 index 0000000..06c8d44 --- /dev/null +++ b/dist/index.d.ts @@ -0,0 +1,106 @@ +import { BinaryLike } from 'crypto'; + +interface MultiHashData { + "cid_sha2-256"?: string; + cid_sha1?: string; + cid_md5?: string; + "cid_sha3-256"?: string; +} +interface ComputeInterface { + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} +declare enum CID_ALGORITHM_CODES { + sha256 = 18, + sha1 = 17, + md5 = 213, + sha3_256 = 22 +} +declare enum CID_ALGORITHM_NAMES { + sha256 = "cid_sha2-256", + sha1 = "cid_sha1", + md5 = "cid_md5", + sha3_256 = "cid_sha3-256" +} +declare enum CID_ALGORITHM { + sha256 = "sha2-256", + sha1 = "sha1", + md5 = "md5", + sha3_256 = "sha3-256" +} + +declare class HashComputer implements ComputeInterface { + private targetHash; + private fileIDComputer; + constructor(targetHash: CID_ALGORITHM_NAMES[]); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + +interface IndexLine extends Partial> { + path: string; + size: string; + mtime: string; +} +declare const INDEX_HEADERS: string[]; +declare class HashIndexManager { + private filePath; + private targetHash; + private cache; + private intervalId; + private intervalTime; + private lastIndexFileSize; + private lastCacheFile; + private indexOpsInProgress; + private hasChanged; + private initialLoad; + constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]); + getCache(): Map; + /** + * After init consseutively calls to this method will not reload the index + * @param autosave + */ + init(autosave?: boolean): Promise; + checkCSVHeaders(csvContent: string): boolean; + start(): void; + stopAutoSave(): void; + private startAutoSave; + loadIndex(): Promise; + private loadIndexFromCache; + private readCsv; + private saveCacheToFile; + getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine; + addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; +} + +declare class ComputeHashIndexCache implements ComputeInterface { + private targetHash; + hashIndexManager: HashIndexManager; + hashComputer: HashComputer; + constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[]); + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} + +declare function existsAsync(filePath: string): Promise; + +declare class FileIDComputer { + private piscina; + constructor(); + /** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ + computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} + +interface SimpleHash { + update(data: BinaryLike): any; + digest(): Buffer; +} + +interface Properties { + TARGET_HASHES: string; + INDEX_FILE_PATH: string; +} + +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES, ComputeHashIndexCache, type ComputeInterface, FileIDComputer, HashComputer, HashIndexManager, INDEX_HEADERS, type MultiHashData, type Properties, type SimpleHash, existsAsync }; diff --git a/dist/index.js b/dist/index.js new file mode 100644 index 0000000..02f8b2a --- /dev/null +++ b/dist/index.js @@ -0,0 +1,16 @@ +import { Piscina } from 'piscina'; +import { promises, createReadStream } from 'fs'; +import { parse as parse$1 } from 'csv-parse'; +import { parse } from 'csv-parse/sync'; +import { stringify } from 'csv-stringify/sync'; +import { access, constants, stat } from 'fs/promises'; +import y from 'path'; +import { clearInterval } from 'timers'; +import 'multiformats/cid'; +import 'multiformats/hashes/digest'; + +var C=Object.defineProperty;var g=Object.getOwnPropertySymbols;var H=Object.prototype.hasOwnProperty,b=Object.prototype.propertyIsEnumerable;var x=(a,e,t)=>e in a?C(a,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):a[e]=t,I=(a,e)=>{for(var t in e||(e={}))H.call(e,t)&&x(a,t,e[t]);if(g)for(var t of g(e))b.call(e,t)&&x(a,t,e[t]);return a};var n=(a,e,t)=>new Promise((i,s)=>{var r=c=>{try{h(t.next(c));}catch(f){s(f);}},o=c=>{try{h(t.throw(c));}catch(f){s(f);}},h=c=>c.done?i(c.value):Promise.resolve(c.value).then(r,o);h((t=t.apply(a,e)).next());});var S={};var m=class{constructor(){this.piscina=new Piscina({maxThreads:4,filename:new URL("./ShaComputeWorker.js",S.url).href});}computeCIDs(e,t){return n(this,null,function*(){return this.piscina.run({filePath:e,algorithms:t})})}};var d=class{constructor(e){this.targetHash=e;this.fileIDComputer=new m;}computeMissingHash(e,t){return n(this,null,function*(){let i=this.targetHash.filter(r=>!t[r]);if(i.length===0)return;(yield this.fileIDComputer.computeCIDs(e,i)).forEach((r,o)=>{let h=i[o];t[h]=r;});})}};var v=(s=>(s[s.sha256=18]="sha256",s[s.sha1=17]="sha1",s[s.md5=213]="md5",s[s.sha3_256=22]="sha3_256",s))(v||{}),p=(s=>(s.sha256="cid_sha2-256",s.sha1="cid_sha1",s.md5="cid_md5",s.sha3_256="cid_sha3-256",s))(p||{}),F=(s=>(s.sha256="sha2-256",s.sha1="sha1",s.md5="md5",s.sha3_256="sha3-256",s))(F||{});function u(a){return n(this,null,function*(){try{return yield access(a,constants.F_OK),!0}catch(e){return !1}})}var ie=["path","size","mtime"],l=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;this.cache=new Map;this.intervalTime=3e4;this.lastIndexFileSize=0;this.indexOpsInProgress=!1;this.hasChanged=!1;if(!e)throw new Error("Invalid index file path")}getCache(){return new Map(this.cache)}init(e=!0){return n(this,null,function*(){return this.initialLoad||(this.initialLoad=new Promise((t,i)=>n(this,null,function*(){try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");yield this.loadIndex(),e&&this.start(),t();}catch(s){i(s);}}))),this.initialLoad})}checkCSVHeaders(e){let t=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let i=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(r=>i.includes(r))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}loadIndex(){return n(this,null,function*(){if(yield u(this.filePath)){let e=yield promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=yield this.readCsv();for(let i of t)this.cache.set(i.path,i);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []})}loadIndexFromCache(){return Array.from(this.cache.values())}readCsv(){return n(this,null,function*(){if(!(yield u(this.filePath)))return [];let e=performance.now(),t=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,r)=>{createReadStream(this.filePath).pipe(t).on("data",o=>{i.push(o);}).on("end",()=>{s(i),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",o=>{r(o);});})})}saveCacheToFile(){return n(this,null,function*(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=yield this.loadIndex();if(this.cache.size!==0){let r=this.loadIndexFromCache().filter(o=>!t.find(h=>h.path===o.path));if(r.length!==0){let o=stringify(r,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(h=>({key:h,header:h}))]});yield promises.appendFile(this.filePath,o);}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;})}getCidForFile(e,t,i){let s=y.basename(e),r=this.cache.get(s);if(r){if(r.mtime){if(r.size===t+""&&r.mtime===i)return r}else if(r.size===t+"")return r}return null}addFileCid(e,t,i,s){if(!e||!t||!i||!s)throw new Error("Invalid parameters");let r=t+"",o=y.basename(e),h=I({path:o,size:r,mtime:i},s);this.cache.set(o,h),this.hasChanged=!0;}};var M=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;this.hashComputer=new d(this.targetHash),this.hashIndexManager=new l(e);}computeMissingHash(e,t){return n(this,null,function*(){yield this.hashIndexManager.init();let i=yield stat(e);if(this.hashIndexManager.getCache().has(y.basename(e))){let s=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(s)for(let r of this.targetHash)!t[r]&&s[r]&&(t[r]=s[r]);}yield this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);})}}; + +export { F as CID_ALGORITHM, v as CID_ALGORITHM_CODES, p as CID_ALGORITHM_NAMES, M as ComputeHashIndexCache, m as FileIDComputer, d as HashComputer, l as HashIndexManager, ie as INDEX_HEADERS, u as existsAsync }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/index.js.map b/dist/index.js.map new file mode 100644 index 0000000..91596b6 --- /dev/null +++ b/dist/index.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/MultiHashData.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/fileID/ShaComputeWorker.ts"],"names":["Piscina","import_meta","FileIDComputer","filePath","algorithms","__async","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","INDEX_HEADERS","HashIndexManager","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","start","parser","err","existingRows","newRows","row","existingRow","csvString","hash","totalTime","fileSize","mtime","fileName","fileNameIndex","hashs","size","baseName","data","__spreadValues","stat","ComputeHashIndexCache","indexFilePath","indexLine","CID","create"],"mappings":"siBAAA,OAAQ,WAAAA,MAAc,UAAtB,IAAAC,EAAA,GAGO,IAAMC,EAAN,KAAqB,CAGxB,aAAc,CACV,KAAK,QAAU,IAAIF,EAAQ,CACvB,WAAY,EACZ,SAAU,IAAI,IAAI,wBAAyBC,EAAY,GAAG,EAAE,IAChE,CAAC,CACL,CAQa,YAAYE,EAAkBC,EAAsD,QAAAC,EAAA,sBAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAF,EAAU,WAAAC,CAAU,CAAC,CAClD,GACJ,ECnBO,IAAME,EAAN,KAA8C,CAGjD,YAAoBC,EAAmC,CAAnC,gBAAAA,EAFpB,KAAQ,eAAiB,IAAIL,CAG7B,CAEM,mBAAmBC,EAAkBK,EAAwC,QAAAH,EAAA,sBAE/E,IAAMI,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYN,EAAUM,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,GACJ,ECfO,IAAKG,OACRA,IAAA,OAAS,IAAT,SACAA,IAAA,KAAO,IAAP,OACAA,IAAA,IAAM,KAAN,MACAA,IAAA,SAAW,IAAX,WAJQA,OAAA,IAQAC,OACRA,EAAA,OAAS,eACTA,EAAA,KAAO,WACPA,EAAA,IAAM,UACNA,EAAA,SAAW,eAJHA,OAAA,IAQAC,OACRA,EAAA,OAAS,WACTA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,SAAW,WAJHA,OAAA,IC5BZ,OAAQ,oBAAAC,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,SAAsBC,EAAYrB,EAAoC,QAAAE,EAAA,sBAClE,GAAI,CACA,aAAMiB,EAAOnB,EAAUoB,EAAU,IAAI,EAC9B,EACX,OAAQ,GACJ,MAAO,EACX,CACJ,GDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAQrB,IAAMC,GAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAU1B,YAAoBzB,EAAyBI,EAAkC,0BAAoD,EAAG,CAAlH,cAAAJ,EAAyB,gBAAAI,EAT7C,KAAQ,MAAgC,IAAI,IAE5C,KAAQ,aAAuB,IAC/B,KAAQ,kBAA4B,EAEpC,KAAQ,mBAA8B,GACtC,KAAQ,WAAsB,GAI1B,GAAI,CAACJ,EACD,MAAM,IAAI,MAAM,yBAAyB,CAEjD,CAEO,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMM,KAAK0B,EAAW,GAAM,QAAAxB,EAAA,sBACxB,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,CAAOyB,EAASC,IAAW1B,EAAA,sBAC5D,GAAI,CACA,GAAI,CAAC,KAAK,gBAAgB,KAAK,QAAQ,EACnC,MAAM,IAAI,MAAM,4BAA4B,EAEhD,MAAM,KAAK,UAAU,EACjBwB,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAAQE,EAAG,CACPD,EAAOC,CAAC,CACZ,CACJ,EAAC,GAEE,KAAK,WAChB,GAGA,gBAAiBC,EAA6B,CAC1C,IAAMC,EAAUd,EAAUa,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,OAAQ,OAAQ,QAAS,GAAG,KAAK,UAAU,EAG7C,MAAME,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLV,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAcW,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEa,WAAkC,QAAAhC,EAAA,sBAC3C,GAAI,MAAMmB,EAAY,KAAK,QAAQ,EAAG,CAElC,IAAMc,EAAQ,MAAMpB,EAAG,KAAK,KAAK,QAAQ,EACzC,GAAI,KAAK,oBAAsBoB,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQ,EAChD,QAAWK,KAAUL,EACjB,KAAK,MAAM,IAAIK,EAAO,KAAMA,CAAM,EAEtC,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,GAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEc,SAAgC,QAAA7B,EAAA,sBAC1C,GAAI,EAAE,MAAMmB,EAAY,KAAK,QAAQ,GACjC,MAAO,CAAC,EAEZ,IAAMgB,EAAQ,YAAY,IAAI,EAExBC,EAAStB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKe,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpCd,EAAiB,KAAK,QAAQ,EACzB,KAAKwB,CAAM,EACX,GAAG,OAASF,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,mBAAmB,YAAY,IAAI,EAAIM,CAAK,IAAI,CAChE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBX,EAAOW,CAAG,CACd,CAAC,CACL,CAAC,CACL,GAEc,iBAAiC,QAAArC,EAAA,sBAC3C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMmC,EAAQ,YAAY,IAAI,EAC1BG,EAA4B,MAAM,KAAK,UAAU,EAErD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMC,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY1B,EAAUuB,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,GAAG,KAAK,WAAW,IAAIK,IAAS,CAAC,IAAKA,EAAM,OAAQA,CAAI,EAAE,CAC9D,CACJ,CAAC,EAGD,MAAM9B,EAAG,WAAW,KAAK,SAAU6B,CAAS,CAChD,CACJ,CACA,IAAME,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,GAEO,cAAc9C,EAAkB+C,EAAkBC,EAA0B,CAC/E,IAAMC,EAAW3B,EAAK,SAAStB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAGC,GACC,GAAGA,EAAc,OAEb,GAAIA,EAAc,OAAUH,EAAW,IAAOG,EAAc,QAAUF,EAClE,OAAOE,UAIPA,EAAc,OAAUH,EAAW,GACnC,OAAOG,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkB+C,EAAkBC,EAAeG,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAAC+C,GAAY,CAACC,GAAS,CAACG,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAMC,EAAOL,EAAW,GAClBM,EAAW/B,EAAK,SAAStB,CAAQ,EACjCsD,EAAOC,EAAA,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOJ,GAAUG,GAC3D,KAAK,MAAM,IAAIE,EAAUC,CAAI,EAC7B,KAAK,WAAa,EACtB,CAEJ,EEjOA,OAAQ,QAAAE,MAAW,cACnB,OAAOlC,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA+BtD,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EACvC,KAAK,aAAe,IAAID,EAAa,KAAK,UAAU,EACpD,KAAK,iBAAmB,IAAIsB,EAAiBiC,CAAa,CAC9D,CAEM,mBAAmB1D,EAAkBK,EAAwC,QAAAH,EAAA,sBAC/E,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAIiC,EAAQ,MAAMqB,EAAKxD,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIsB,EAAK,SAAStB,CAAQ,CAAC,EAAG,CAC/D,IAAM2D,EAAY,KAAK,iBAAiB,cAAc3D,EAAUmC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGwB,EACC,QAAWd,KAAQ,KAAK,WAChB,CAACxC,EAASwC,CAAI,GAAKc,EAAUd,CAAI,IACjCxC,EAASwC,CAAI,EAAIc,EAAUd,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmB7C,EAAUK,CAAQ,EAC7D,KAAK,iBAAiB,WAAWL,EAAUmC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,GACJ,EC7BA,OAAS,OAAAuD,OAAW,mBACpB,OAAS,UAAAC,OAAc","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData.js\";\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor() {\n this.piscina = new Piscina({\n maxThreads: 4,\n filename: new URL('./ShaComputeWorker.js', import.meta.url).href\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer.js\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData.js\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer = new FileIDComputer();\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\nexport enum CID_ALGORITHM_CODES {\n sha256 = 0x12,\n sha1 = 0x11,\n md5 = 0xd5,\n sha3_256 = 0x16,\n //crc32 = 0x0132\n}\n\nexport enum CID_ALGORITHM_NAMES {\n sha256 = 'cid_sha2-256',\n sha1 = 'cid_sha1',\n md5 = 'cid_md5',\n sha3_256 = 'cid_sha3-256',\n //crc32 = 'cid_crc32'\n}\n\nexport enum CID_ALGORITHM {\n sha256 = 'sha2-256',\n sha1 = 'sha1',\n md5 = 'md5',\n sha3_256 = 'sha3-256',\n //crc32 = 'crc32'\n}","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync.js\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData.js\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad;\n\n constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {\n if (!filePath) {\n throw new Error('Invalid index file path');\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n if (!this.checkCSVHeaders(this.filePath)) {\n throw new Error('Invalid index file headers');\n }\n await this.loadIndex();\n if (autosave) {\n this.start();\n }\n resolve();\n }catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n checkCSVHeaders (csvContent: string): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(): Promise {\n if (await existsAsync(this.filePath)) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePath);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv();\n for (const record of records) {\n this.cache.set(record.path, record);\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(): Promise {\n if (!(await existsAsync(this.filePath))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePath)\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n let existingRows: IndexLine[] = await this.loadIndex();\n\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n ...this.targetHash.map(hash => ({key: hash, header: hash})),\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePath, csvString);\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if(fileNameIndex) {\n if(fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n }else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer.js\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData.js\";\nimport {HashIndexManager} from \"./HashIndexManager.js\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n this.hashComputer = new HashComputer(this.targetHash);\n this.hashIndexManager = new HashIndexManager(indexFilePath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import { createReadStream } from \"fs\";\nimport {BinaryLike, createHash} from 'crypto';\nimport { CID } from 'multiformats/cid';\nimport { create } from \"multiformats/hashes/digest\";\nimport {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData.js\";\n\nexport interface SimpleHash {\n update(data: BinaryLike),\n digest(): Buffer;\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },\n [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },\n [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },\n [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n"]} \ No newline at end of file