diff --git a/dist/ShaComputeWorker-BzBitaFi.d.cts b/dist/ShaComputeWorker-CNdGUbtT.d.cts similarity index 98% rename from dist/ShaComputeWorker-BzBitaFi.d.cts rename to dist/ShaComputeWorker-CNdGUbtT.d.cts index a44621f..d5d4b0a 100644 --- a/dist/ShaComputeWorker-BzBitaFi.d.cts +++ b/dist/ShaComputeWorker-CNdGUbtT.d.cts @@ -1,9 +1,10 @@ interface MultiHashData { - "cid_sha2-256"?: string; - cid_sha1?: string; - cid_md5?: string; - "cid_sha3-256"?: string; cid_crc32?: string; + cid_md5?: string; + cid_sha1?: string; + "cid_sha2-256"?: string; + "cid_sha3-256"?: string; + "cid_sha3_384"?: string; } interface ComputeInterface { computeMissingHash(filePath: string, metadata: MultiHashData): Promise; diff --git a/dist/ShaComputeWorker-BzBitaFi.d.ts b/dist/ShaComputeWorker-CNdGUbtT.d.ts similarity index 98% rename from dist/ShaComputeWorker-BzBitaFi.d.ts rename to dist/ShaComputeWorker-CNdGUbtT.d.ts index a44621f..d5d4b0a 100644 --- a/dist/ShaComputeWorker-BzBitaFi.d.ts +++ b/dist/ShaComputeWorker-CNdGUbtT.d.ts @@ -1,9 +1,10 @@ interface MultiHashData { - "cid_sha2-256"?: string; - cid_sha1?: string; - cid_md5?: string; - "cid_sha3-256"?: string; cid_crc32?: string; + cid_md5?: string; + cid_sha1?: string; + "cid_sha2-256"?: string; + "cid_sha3-256"?: string; + "cid_sha3_384"?: string; } interface ComputeInterface { computeMissingHash(filePath: string, metadata: MultiHashData): Promise; diff --git a/dist/chunk-4EZ47WAR.js b/dist/chunk-4EZ47WAR.js new file mode 100644 index 0000000..820b61c --- /dev/null +++ b/dist/chunk-4EZ47WAR.js @@ -0,0 +1,11 @@ +import { createReadStream } from 'fs'; +import { createHash } from 'crypto'; +import { CID } from 'multiformats/cid'; +import { create } from 'multiformats/hashes/digest'; +import x from 'crc-32'; + +var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))} + +export { o as a, d as b, u as c, B as d }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-4EZ47WAR.js.map \ No newline at end of file diff --git a/dist/chunk-4EZ47WAR.js.map b/dist/chunk-4EZ47WAR.js.map new file mode 100644 index 0000000..b0aefc2 --- /dev/null +++ b/dist/chunk-4EZ47WAR.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCaX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDtBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-A5PD5ID4.cjs b/dist/chunk-A5PD5ID4.cjs deleted file mode 100644 index 4bd5c5b..0000000 --- a/dist/chunk-A5PD5ID4.cjs +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var crypto = require('crypto'); -var cid = require('multiformats/cid'); -var digest = require('multiformats/hashes/digest'); -var x = require('crc-32'); - -function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } - -var x__default = /*#__PURE__*/_interopDefault(x); - -var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=fs.createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))} - -exports.a = o; -exports.b = n; -exports.c = u; -exports.d = B; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-A5PD5ID4.cjs.map \ No newline at end of file diff --git a/dist/chunk-A5PD5ID4.cjs.map b/dist/chunk-A5PD5ID4.cjs.map deleted file mode 100644 index b21a764..0000000 --- a/dist/chunk-A5PD5ID4.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCYX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDrBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,QAAQ,KAAK,KAAK,OAAO,GAAG,SAAS,EAAE,CAAC,EACxC,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n console.log((this._crc>>>0).toString(16)) ;\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-ASEKTFOG.cjs b/dist/chunk-ASEKTFOG.cjs new file mode 100644 index 0000000..26585eb --- /dev/null +++ b/dist/chunk-ASEKTFOG.cjs @@ -0,0 +1,20 @@ +'use strict'; + +var fs = require('fs'); +var crypto = require('crypto'); +var cid = require('multiformats/cid'); +var digest = require('multiformats/hashes/digest'); +var x = require('crc-32'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var x__default = /*#__PURE__*/_interopDefault(x); + +var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=fs.createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))} + +exports.a = o; +exports.b = d; +exports.c = u; +exports.d = B; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-ASEKTFOG.cjs.map \ No newline at end of file diff --git a/dist/chunk-ASEKTFOG.cjs.map b/dist/chunk-ASEKTFOG.cjs.map new file mode 100644 index 0000000..b0aefc2 --- /dev/null +++ b/dist/chunk-ASEKTFOG.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCaX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDtBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-FGATABDV.js b/dist/chunk-FGATABDV.js deleted file mode 100644 index 90de17d..0000000 --- a/dist/chunk-FGATABDV.js +++ /dev/null @@ -1,11 +0,0 @@ -import { createReadStream } from 'fs'; -import { createHash } from 'crypto'; -import { CID } from 'multiformats/cid'; -import { create } from 'multiformats/hashes/digest'; -import x from 'crc-32'; - -var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))} - -export { o as a, n as b, u as c, B as d }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-FGATABDV.js.map \ No newline at end of file diff --git a/dist/chunk-FGATABDV.js.map b/dist/chunk-FGATABDV.js.map deleted file mode 100644 index b21a764..0000000 --- a/dist/chunk-FGATABDV.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCYX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDrBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,QAAQ,KAAK,KAAK,OAAO,GAAG,SAAS,EAAE,CAAC,EACxC,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n console.log((this._crc>>>0).toString(16)) ;\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/index.cjs b/dist/index.cjs index 1632eb1..af475eb 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -1,6 +1,6 @@ 'use strict'; -var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs'); +var chunkASEKTFOG_cjs = require('./chunk-ASEKTFOG.cjs'); var piscina = require('piscina'); var fs = require('fs'); var csvParse = require('csv-parse'); @@ -9,8 +9,8 @@ var sync$1 = require('csv-stringify/sync'); var promises = require('fs/promises'); var c = require('path'); var timers = require('timers'); -var w = require('chokidar'); -var R = require('p-queue'); +var y = require('chokidar'); +var z = require('p-queue'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } @@ -33,29 +33,29 @@ function _interopNamespace(e) { } var c__namespace = /*#__PURE__*/_interopNamespace(c); -var w__default = /*#__PURE__*/_interopDefault(w); -var R__default = /*#__PURE__*/_interopDefault(R); +var y__default = /*#__PURE__*/_interopDefault(y); +var z__default = /*#__PURE__*/_interopDefault(z); -var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var I=new URL(l).href;console.log("Worker URL: ",I);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||I});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,a)=>{let n=r[a];t[n]=s;});}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=sync$1.stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),r=await promises.stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={path:a,size:s,mtime:r,...i};this.cache.set(a,o);}this.hasChanged=!0;}};var x=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var M=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new R__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w__default.default.watch(e,t):r=w__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a!t[s]);if(r.length===0)return;let i=await this.fileIDComputer.computeCIDs(e,r);for(let[s,a]of i.entries()){let n=r[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...E,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=sync$1.stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),r=await promises.stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={};for(let g of this.targetHash)o[g]=i[g];let F={path:a,size:s,mtime:r,...o};this.cache.set(a,F);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=y__default.default.watch(e,t):r=y__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileName","fileNameIndex","fileSize","mtime","hashs","size","baseName","filteredHash","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"uDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA+C,CAGlD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY7B,EAAU0B,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB7C,EAAsC,CAClE,IAAM8C,EAAW3B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EAC7B+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUb,EAAM,KAAO,IAAOa,EAAc,QAAUb,EAAM,MAAM,YAAY,EAC5F,OAAOa,UAIPA,EAAc,OAAUb,EAAM,KAAO,GACrC,OAAOa,EAWnB,OAAO,IACX,CAGO,cAAc/C,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMH,EAAW3B,EAAK,SAASnB,CAAQ,EACnC+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUC,EAAW,IAAOD,EAAc,QAAUE,EAClE,OAAOF,UAIPA,EAAc,OAAUC,EAAW,GACnC,OAAOD,EAWnB,OAAO,IACX,CAEO,WAAW/C,EAAkBgD,EAAkBC,EAAeC,EAA2D,CAC5H,GAAI,CAAClD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACC,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOH,EAAW,GAClBI,EAAWjC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MAXpB,CAEZ,IAAI6B,EAAe,CAAC,EACpB,QAAW7B,KAAQ,KAAK,WACpB6B,EAAa7B,CAAI,EAAI0B,EAAM1B,CAAI,EAEnC,IAAM8B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOF,EAAO,GAAGI,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEhSA,OAAQ,QAAAjC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMoC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BrD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIwB,EAAiBiC,EAAcrD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOsC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYxE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMwE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWxE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIyE,EAAe,UAAQzE,CAAQ,EACnC,MAAM,KAAK,iBAAiByE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO5E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS6E,EAAO,CACZ,QAAQ,MAAM,yBAAyB7E,CAAQ,IAAK6E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO7E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUiD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.d.cts b/dist/index.d.cts index 619a746..1b454b4 100644 --- a/dist/index.d.cts +++ b/dist/index.d.cts @@ -1,5 +1,5 @@ -import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.cjs'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.cjs'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.cjs'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; diff --git a/dist/index.d.ts b/dist/index.d.ts index 8cc028b..20e15b3 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,5 +1,5 @@ -import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.js'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.js'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.js'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; diff --git a/dist/index.js b/dist/index.js index 625d269..12070ed 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,4 +1,4 @@ -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-FGATABDV.js'; +export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-4EZ47WAR.js'; import { Piscina } from 'piscina'; import { promises, createReadStream } from 'fs'; import { parse as parse$1 } from 'csv-parse'; @@ -8,11 +8,11 @@ import { access, constants, stat, readdir } from 'fs/promises'; import * as c from 'path'; import c__default from 'path'; import { clearInterval } from 'node:timers'; -import w from 'chokidar'; -import R from 'p-queue'; +import y from 'chokidar'; +import z from 'p-queue'; -var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var I=new URL(l).href;console.log("Worker URL: ",I);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||I});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(s=>!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,a)=>{let n=r[a];t[n]=s;});}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),r=await stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={path:a,size:s,mtime:r,...i};this.cache.set(a,o);}this.hasChanged=!0;}};var x=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var M=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new R({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w.watch(e,t):r=w.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a!t[s]);if(r.length===0)return;let i=await this.fileIDComputer.computeCIDs(e,r);for(let[s,a]of i.entries()){let n=r[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...E,t].every(a=>i.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",n=>{i.push(n);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let a=this.loadIndexFromCache().filter(n=>!i.find(o=>o.path===n.path));if(a.length!==0){let n=stringify(a,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],n);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),r=await stat(e),i=this.cache.get(t);if(i){if(i.mtime){if(i.size===r.size+""&&i.mtime===r.mtime.toISOString())return i}else if(i.size===r.size+"")return i}return null}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!i[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=i[o];else {let o={};for(let g of this.targetHash)o[g]=i[g];let F={path:a,size:s,mtime:r,...o};this.cache.set(a,F);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=y.watch(e,t):r=y.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+n,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileName","fileNameIndex","fileSize","mtime","hashs","size","baseName","filteredHash","data","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"sDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA+C,CAGlD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY7B,EAAU0B,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB7C,EAAsC,CAClE,IAAM8C,EAAW3B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EAC7B+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUb,EAAM,KAAO,IAAOa,EAAc,QAAUb,EAAM,MAAM,YAAY,EAC5F,OAAOa,UAIPA,EAAc,OAAUb,EAAM,KAAO,GACrC,OAAOa,EAWnB,OAAO,IACX,CAGO,cAAc/C,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMH,EAAW3B,EAAK,SAASnB,CAAQ,EACnC+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUC,EAAW,IAAOD,EAAc,QAAUE,EAClE,OAAOF,UAIPA,EAAc,OAAUC,EAAW,GACnC,OAAOD,EAWnB,OAAO,IACX,CAEO,WAAW/C,EAAkBgD,EAAkBC,EAAeC,EAA2D,CAC5H,GAAI,CAAClD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACC,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOH,EAAW,GAClBI,EAAWjC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MAXpB,CAEZ,IAAI6B,EAAe,CAAC,EACpB,QAAW7B,KAAQ,KAAK,WACpB6B,EAAa7B,CAAI,EAAI0B,EAAM1B,CAAI,EAEnC,IAAM8B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOF,EAAO,GAAGI,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEhSA,OAAQ,QAAAjC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMoC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BrD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIwB,EAAiBiC,EAAcrD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOsC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYxE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMwE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWxE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIyE,EAAe,UAAQzE,CAAQ,EACnC,MAAM,KAAK,iBAAiByE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO5E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS6E,EAAO,CACZ,QAAQ,MAAM,yBAAyB7E,CAAQ,IAAK6E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO7E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUiD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (stats.size + \"\") && fileNameIndex.mtime === stats.mtime.toISOString()) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (stats.size + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/worker.cjs b/dist/worker.cjs index 6b29ae1..0e30495 100644 --- a/dist/worker.cjs +++ b/dist/worker.cjs @@ -1,9 +1,9 @@ 'use strict'; -var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs'); +var chunkASEKTFOG_cjs = require('./chunk-ASEKTFOG.cjs'); -module.exports = chunkA5PD5ID4_cjs.d; +module.exports = chunkASEKTFOG_cjs.d; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.cjs.map \ No newline at end of file diff --git a/dist/worker.d.cts b/dist/worker.d.cts index 16d7e99..6ce4749 100644 --- a/dist/worker.d.cts +++ b/dist/worker.d.cts @@ -1 +1 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.cjs'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.cjs'; diff --git a/dist/worker.d.ts b/dist/worker.d.ts index 7404512..dd6aeb4 100644 --- a/dist/worker.d.ts +++ b/dist/worker.d.ts @@ -1 +1 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.js'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.js'; diff --git a/dist/worker.js b/dist/worker.js index d0a108d..6eea1a7 100644 --- a/dist/worker.js +++ b/dist/worker.js @@ -1,3 +1,3 @@ -export { d as default } from './chunk-FGATABDV.js'; +export { d as default } from './chunk-4EZ47WAR.js'; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/src/lib/HashComputer.ts b/src/lib/HashComputer.ts index db8c7eb..0c901ee 100644 --- a/src/lib/HashComputer.ts +++ b/src/lib/HashComputer.ts @@ -1,10 +1,10 @@ -import { FileIDComputer } from "./fileID/FileIDComputer"; +import {FileIDComputer} from "./fileID/FileIDComputer"; import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; -export class HashComputer implements ComputeInterface{ - private fileIDComputer:FileIDComputer; +export class HashComputer implements ComputeInterface { + private fileIDComputer: FileIDComputer; - constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) { + constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) { this.fileIDComputer = new FileIDComputer(workerPath); } @@ -21,9 +21,9 @@ export class HashComputer implements ComputeInterface{ const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes); // Map the computed CIDs back to their respective metadata properties - cids.forEach((cid, index) => { + for (const [index, cid] of cids.entries()) { const hashType = neededHashes[index]; metadata[hashType] = cid; - }); + } } } diff --git a/src/lib/HashIndexManager.ts b/src/lib/HashIndexManager.ts index 481466a..08382e2 100644 --- a/src/lib/HashIndexManager.ts +++ b/src/lib/HashIndexManager.ts @@ -273,7 +273,12 @@ export class HashIndexManager { const baseName = path.basename(filePath); let indexLine = this.cache.get(baseName); if (!indexLine) { - const data = {path: baseName, size: size, mtime: mtime, ...hashs}; + //filter only the hashes we need + let filteredHash = {}; + for (const hash of this.targetHash) { + filteredHash[hash] = hashs[hash]; + } + const data = {path: baseName, size: size, mtime: mtime, ...filteredHash}; this.cache.set(baseName, data); } else { //update the cache with the latest data diff --git a/src/lib/MultiHashData.ts b/src/lib/MultiHashData.ts index bc3248e..e8afe7f 100644 --- a/src/lib/MultiHashData.ts +++ b/src/lib/MultiHashData.ts @@ -1,9 +1,10 @@ export interface MultiHashData { - "cid_sha2-256"?: string;//0x12 sha2-256 - cid_sha1?: string; // 0x11 sha1 - cid_md5?: string; // 0xd5 md5 - "cid_sha3-256"?: string; // 0x16 sha3-256 cid_crc32?: string; //0x0132 CRC32 (SFV) + cid_md5?: string; // 0xd5 md5 + cid_sha1?: string; // 0x11 sha1 + "cid_sha2-256"?: string;//0x12 sha2-256 + "cid_sha3-256"?: string; // 0x16 sha3-256 + "cid_sha3_384"?: string; // 0x16 sha3-256 } export interface ComputeInterface { diff --git a/src/lib/fileID/ShaComputeWorker.ts b/src/lib/fileID/ShaComputeWorker.ts index d9aaa78..12d5586 100644 --- a/src/lib/fileID/ShaComputeWorker.ts +++ b/src/lib/fileID/ShaComputeWorker.ts @@ -21,7 +21,6 @@ class Crc32Hash implements SimpleHash { } digest(): Buffer { - console.log((this._crc>>>0).toString(16)) ; const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits) buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format return buffer;