diff --git a/.yarnrc.yml b/.yarnrc.yml new file mode 100644 index 0000000..3186f3f --- /dev/null +++ b/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/dist/chunk-33LRP2HS.cjs b/dist/chunk-33LRP2HS.cjs deleted file mode 100644 index 495fe17..0000000 --- a/dist/chunk-33LRP2HS.cjs +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); -var stream = require('stream'); -var digest = require('multiformats/hashes/digest'); -var cid = require('multiformats/cid'); -var crypto = require('crypto'); -var u = require('crc-32'); - -function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } - -var u__default = /*#__PURE__*/_interopDefault(u); - -var c=class{_crc=void 0;update(e){return this._crc=u__default.default.buf(e,this._crc),this}digest(){let e=Buffer.alloc(4);return e.writeInt32BE(this._crc,0),e}};var i={cid_crc32:306,cid_md5:213,cid_sha1:17,"cid_sha2-256":18,"cid_sha3-256":22,"cid_sha3-384":21};function m(r){switch(r){case"cid_sha2-256":return crypto.createHash("sha256");case"cid_sha1":return crypto.createHash("sha1");case"cid_md5":return crypto.createHash("md5");case"cid_sha3-256":return crypto.createHash("sha3-256");case"cid_sha3-384":return crypto.createHash("sha3-384");case"cid_crc32":return new c;default:throw new Error(`Unsupported algorithm: ${r}`)}}async function L({stream:r,algorithms:e}){let a=_(e);if(r instanceof stream.Readable)for await(let t of r)a.forEach(s=>s.hasher.update(t));else {let t=r.getReader();for(;;){let{done:s,value:n}=await t.read();if(s)break;a.forEach(f=>f.hasher.update(Buffer.from(n)));}}return H(a)}function _(r){return r.filter(a=>Object.values(chunkPAKJNZBJ_cjs.b).includes(a)).map(a=>({hasher:m(a),code:i[a]}))}async function H(r){return await Promise.all(r.map(async({code:e,hasher:a})=>{let t=a.digest(),s=digest.create(e,t);return cid.CID.createV1(e,s).toString()}))} - -exports.a = L; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-33LRP2HS.cjs.map \ No newline at end of file diff --git a/dist/chunk-33LRP2HS.cjs.map b/dist/chunk-33LRP2HS.cjs.map deleted file mode 100644 index d776c10..0000000 --- a/dist/chunk-33LRP2HS.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":["Readable","create","CID","createHash","crc32","Crc32Hash","data","buffer","codeTable","createHasher","algo","computeCIDs","stream","algorithms","hashers","hasherDefiner","chunk","item","reader","done","value","cidFinalize","CID_ALGORITHM_NAMES","code","hasher","hashBuffer","digest"],"mappings":"yCACA,OAAQ,YAAAA,MAAe,SACvB,OAAQ,UAAAC,MAAa,6BACrB,OAAQ,OAAAC,MAAU,mBCClB,OAAQ,cAAAC,MAAiB,SCJzB,OAAOC,MAAW,SAGX,IAAMC,EAAN,KAAsC,CACjC,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOF,EAAM,IAAIE,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EDVO,IAAMC,EAAY,CACpB,cACA,YACA,YACA,kBACA,kBACA,iBACL,EAOO,SAASC,EAAaC,EAAuC,CAChE,OAAQA,EAAM,CACV,mBACI,OAAOP,EAAW,QAAQ,EAC9B,eACI,OAAOA,EAAW,MAAM,EAC5B,cACI,OAAOA,EAAW,KAAK,EAC3B,mBACI,OAAOA,EAAW,UAAU,EAChC,mBACI,OAAOA,EAAW,UAAU,EAChC,gBACI,OAAO,IAAIE,EACf,QACI,MAAM,IAAI,MAAM,0BAA0BK,CAAI,EAAE,CACxD,CACJ,CDzBA,eAAsBC,EAAY,CAAC,OAAAC,EAAQ,WAAAC,CAAU,EAG/B,CAClB,IAAMC,EAAUC,EAAcF,CAAU,EAGxC,GAAID,aAAkBZ,EAElB,cAAiBgB,KAASJ,EACtBE,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAOD,CAAe,CAAC,MAE5D,CAEH,IAAME,EAASN,EAAO,UAAU,EAChC,OAAa,CACT,GAAM,CAAC,KAAAO,EAAM,MAAAC,CAAK,EAAI,MAAMF,EAAO,KAAK,EACxC,GAAIC,EAAM,MAEVL,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAO,OAAO,KAAKG,CAAK,CAAC,CAAC,CAClE,CACJ,CAEA,OAAOC,EAAYP,CAAO,CAC9B,CAEA,SAASC,EAAcF,EAGnB,CAMA,OALgBA,EAAW,OAAOH,GAAQ,OAAO,OAAOY,CAAmB,EAAE,SAASZ,CAAI,CAAC,EACtF,IAAIA,IAAS,CACV,OAAQD,EAAaC,CAAI,EACzB,KAAMF,EAAUE,CAAI,CACxB,EAAE,CAEV,CAEA,eAAeW,EAAYP,EAAoE,CAC3F,OAAO,MAAM,QAAQ,IAAIA,EAAQ,IAAI,MAAO,CAAC,KAAAS,EAAM,OAAAC,CAAM,IAAM,CAC3D,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASzB,EAAOsB,EAAME,CAAU,EACtC,OAAOvB,EAAI,SAASqB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n hashers.forEach(item => item.hasher.update(chunk as Buffer));\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n hashers.forEach(item => item.hasher.update(Buffer.from(value)));\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nfunction hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): {\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[] {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(algo => ({\r\n hasher: createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return hashers;\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {ReadableStream} from \"stream/web\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): Crc32Hash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-6UBSWOQL.js b/dist/chunk-6UBSWOQL.js deleted file mode 100644 index 1a4c3c1..0000000 --- a/dist/chunk-6UBSWOQL.js +++ /dev/null @@ -1,8 +0,0 @@ -import { a } from './chunk-A6J5HDTV.js'; -import m from 'fs'; - -async function s({filePath:r,algorithms:e}){let o=m.createReadStream(r);return a({stream:o,algorithms:e})} - -export { s as a }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-6UBSWOQL.js.map \ No newline at end of file diff --git a/dist/chunk-6UBSWOQL.js.map b/dist/chunk-6UBSWOQL.js.map deleted file mode 100644 index b6523a8..0000000 --- a/dist/chunk-6UBSWOQL.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["fs","computeCIDs","filePath","algorithms","stream"],"mappings":"wCACA,OAAOA,MAAQ,KASf,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EAASJ,EAAG,iBAAiBE,CAAQ,EAC3C,OAAOD,EAAoB,CAAC,OAAAG,EAAQ,WAAAD,CAAU,CAAC,CACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-A6J5HDTV.js b/dist/chunk-A6J5HDTV.js deleted file mode 100644 index 6123139..0000000 --- a/dist/chunk-A6J5HDTV.js +++ /dev/null @@ -1,12 +0,0 @@ -import { b } from './chunk-YCFZVUZC.js'; -import { Readable } from 'stream'; -import { create } from 'multiformats/hashes/digest'; -import { CID } from 'multiformats/cid'; -import { createHash } from 'crypto'; -import u from 'crc-32'; - -var c=class{_crc=void 0;update(e){return this._crc=u.buf(e,this._crc),this}digest(){let e=Buffer.alloc(4);return e.writeInt32BE(this._crc,0),e}};var i={cid_crc32:306,cid_md5:213,cid_sha1:17,"cid_sha2-256":18,"cid_sha3-256":22,"cid_sha3-384":21};function m(r){switch(r){case"cid_sha2-256":return createHash("sha256");case"cid_sha1":return createHash("sha1");case"cid_md5":return createHash("md5");case"cid_sha3-256":return createHash("sha3-256");case"cid_sha3-384":return createHash("sha3-384");case"cid_crc32":return new c;default:throw new Error(`Unsupported algorithm: ${r}`)}}async function L({stream:r,algorithms:e}){let a=_(e);if(r instanceof Readable)for await(let t of r)a.forEach(s=>s.hasher.update(t));else {let t=r.getReader();for(;;){let{done:s,value:n}=await t.read();if(s)break;a.forEach(f=>f.hasher.update(Buffer.from(n)));}}return H(a)}function _(r){return r.filter(a=>Object.values(b).includes(a)).map(a=>({hasher:m(a),code:i[a]}))}async function H(r){return await Promise.all(r.map(async({code:e,hasher:a})=>{let t=a.digest(),s=create(e,t);return CID.createV1(e,s).toString()}))} - -export { L as a }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-A6J5HDTV.js.map \ No newline at end of file diff --git a/dist/chunk-A6J5HDTV.js.map b/dist/chunk-A6J5HDTV.js.map deleted file mode 100644 index b1a71a1..0000000 --- a/dist/chunk-A6J5HDTV.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":["Readable","create","CID","createHash","crc32","Crc32Hash","data","buffer","codeTable","createHasher","algo","computeCIDs","stream","algorithms","hashers","hasherDefiner","chunk","item","reader","done","value","cidFinalize","CID_ALGORITHM_NAMES","code","hasher","hashBuffer","digest"],"mappings":"wCACA,OAAQ,YAAAA,MAAe,SACvB,OAAQ,UAAAC,MAAa,6BACrB,OAAQ,OAAAC,MAAU,mBCClB,OAAQ,cAAAC,MAAiB,SCJzB,OAAOC,MAAW,SAGX,IAAMC,EAAN,KAAsC,CACjC,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOF,EAAM,IAAIE,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EDVO,IAAMC,EAAY,CACpB,cACA,YACA,YACA,kBACA,kBACA,iBACL,EAOO,SAASC,EAAaC,EAAuC,CAChE,OAAQA,EAAM,CACV,mBACI,OAAOP,EAAW,QAAQ,EAC9B,eACI,OAAOA,EAAW,MAAM,EAC5B,cACI,OAAOA,EAAW,KAAK,EAC3B,mBACI,OAAOA,EAAW,UAAU,EAChC,mBACI,OAAOA,EAAW,UAAU,EAChC,gBACI,OAAO,IAAIE,EACf,QACI,MAAM,IAAI,MAAM,0BAA0BK,CAAI,EAAE,CACxD,CACJ,CDzBA,eAAsBC,EAAY,CAAC,OAAAC,EAAQ,WAAAC,CAAU,EAG/B,CAClB,IAAMC,EAAUC,EAAcF,CAAU,EAGxC,GAAID,aAAkBZ,EAElB,cAAiBgB,KAASJ,EACtBE,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAOD,CAAe,CAAC,MAE5D,CAEH,IAAME,EAASN,EAAO,UAAU,EAChC,OAAa,CACT,GAAM,CAAC,KAAAO,EAAM,MAAAC,CAAK,EAAI,MAAMF,EAAO,KAAK,EACxC,GAAIC,EAAM,MAEVL,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAO,OAAO,KAAKG,CAAK,CAAC,CAAC,CAClE,CACJ,CAEA,OAAOC,EAAYP,CAAO,CAC9B,CAEA,SAASC,EAAcF,EAGnB,CAMA,OALgBA,EAAW,OAAOH,GAAQ,OAAO,OAAOY,CAAmB,EAAE,SAASZ,CAAI,CAAC,EACtF,IAAIA,IAAS,CACV,OAAQD,EAAaC,CAAI,EACzB,KAAMF,EAAUE,CAAI,CACxB,EAAE,CAEV,CAEA,eAAeW,EAAYP,EAAoE,CAC3F,OAAO,MAAM,QAAQ,IAAIA,EAAQ,IAAI,MAAO,CAAC,KAAAS,EAAM,OAAAC,CAAM,IAAM,CAC3D,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASzB,EAAOsB,EAAME,CAAU,EACtC,OAAOvB,EAAI,SAASqB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n hashers.forEach(item => item.hasher.update(chunk as Buffer));\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n hashers.forEach(item => item.hasher.update(Buffer.from(value)));\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nfunction hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): {\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[] {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(algo => ({\r\n hasher: createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return hashers;\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {ReadableStream} from \"stream/web\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): Crc32Hash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-HTZ763NS.js b/dist/chunk-HTZ763NS.js new file mode 100644 index 0000000..1931d3c --- /dev/null +++ b/dist/chunk-HTZ763NS.js @@ -0,0 +1,32 @@ +// src/lib/hash-compute/MultiHashData.ts +var CID_ALGORITHM_CODES = /* @__PURE__ */ ((CID_ALGORITHM_CODES2) => { + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["crc32"] = 306] = "crc32"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["md5"] = 213] = "md5"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha1"] = 17] = "sha1"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha256"] = 18] = "sha256"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_256"] = 22] = "sha3_256"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_384"] = 21] = "sha3_384"; + return CID_ALGORITHM_CODES2; +})(CID_ALGORITHM_CODES || {}); +var CID_ALGORITHM_NAMES = /* @__PURE__ */ ((CID_ALGORITHM_NAMES2) => { + CID_ALGORITHM_NAMES2["crc32"] = "cid_crc32"; + CID_ALGORITHM_NAMES2["md5"] = "cid_md5"; + CID_ALGORITHM_NAMES2["sha1"] = "cid_sha1"; + CID_ALGORITHM_NAMES2["sha256"] = "cid_sha2-256"; + CID_ALGORITHM_NAMES2["sha3_256"] = "cid_sha3-256"; + CID_ALGORITHM_NAMES2["sha3_384"] = "cid_sha3-384"; + return CID_ALGORITHM_NAMES2; +})(CID_ALGORITHM_NAMES || {}); +var CID_ALGORITHM = /* @__PURE__ */ ((CID_ALGORITHM2) => { + CID_ALGORITHM2["crc32"] = "crc32"; + CID_ALGORITHM2["md5"] = "md5"; + CID_ALGORITHM2["sha1"] = "sha1"; + CID_ALGORITHM2["sha256"] = "sha2-256"; + CID_ALGORITHM2["sha3_256"] = "sha3-256"; + CID_ALGORITHM2["sha3_384"] = "sha3-384"; + return CID_ALGORITHM2; +})(CID_ALGORITHM || {}); + +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-HTZ763NS.js.map \ No newline at end of file diff --git a/dist/chunk-HTZ763NS.js.map b/dist/chunk-HTZ763NS.js.map new file mode 100644 index 0000000..9fda71c --- /dev/null +++ b/dist/chunk-HTZ763NS.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":";AAiBO,IAAK,sBAAL,kBAAKA,yBAAL;AACH,EAAAA,0CAAA,WAAQ,OAAR;AACA,EAAAA,0CAAA,SAAM,OAAN;AACA,EAAAA,0CAAA,UAAO,MAAP;AACA,EAAAA,0CAAA,YAAS,MAAT;AACA,EAAAA,0CAAA,cAAW,MAAX;AACA,EAAAA,0CAAA,cAAW,MAAX;AANQ,SAAAA;AAAA,GAAA;AASL,IAAK,sBAAL,kBAAKC,yBAAL;AACH,EAAAA,qBAAA,WAAQ;AACR,EAAAA,qBAAA,SAAM;AACN,EAAAA,qBAAA,UAAO;AACP,EAAAA,qBAAA,YAAS;AACT,EAAAA,qBAAA,cAAW;AACX,EAAAA,qBAAA,cAAW;AANH,SAAAA;AAAA,GAAA;AASL,IAAK,gBAAL,kBAAKC,mBAAL;AACH,EAAAA,eAAA,WAAQ;AACR,EAAAA,eAAA,SAAM;AACN,EAAAA,eAAA,UAAO;AACP,EAAAA,eAAA,YAAS;AACT,EAAAA,eAAA,cAAW;AACX,EAAAA,eAAA,cAAW;AANH,SAAAA;AAAA,GAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-KZVIGNTH.cjs b/dist/chunk-KZVIGNTH.cjs deleted file mode 100644 index 5b7d3cf..0000000 --- a/dist/chunk-KZVIGNTH.cjs +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -var chunk33LRP2HS_cjs = require('./chunk-33LRP2HS.cjs'); -var m = require('fs'); - -function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } - -var m__default = /*#__PURE__*/_interopDefault(m); - -async function s({filePath:r,algorithms:e}){let o=m__default.default.createReadStream(r);return chunk33LRP2HS_cjs.a({stream:o,algorithms:e})} - -exports.a = s; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-KZVIGNTH.cjs.map \ No newline at end of file diff --git a/dist/chunk-KZVIGNTH.cjs.map b/dist/chunk-KZVIGNTH.cjs.map deleted file mode 100644 index d2bb68b..0000000 --- a/dist/chunk-KZVIGNTH.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["fs","computeCIDs","filePath","algorithms","stream"],"mappings":"yCACA,OAAOA,MAAQ,KASf,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EAASJ,EAAG,iBAAiBE,CAAQ,EAC3C,OAAOD,EAAoB,CAAC,OAAAG,EAAQ,WAAAD,CAAU,CAAC,CACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-LTWPBXM5.js b/dist/chunk-LTWPBXM5.js new file mode 100644 index 0000000..07c4de8 --- /dev/null +++ b/dist/chunk-LTWPBXM5.js @@ -0,0 +1,11 @@ +import { computeCIDs } from './chunk-SV45D2HN.js'; +import fs from 'fs'; + +async function computeCIDs2({ filePath, algorithms }) { + const stream = fs.createReadStream(filePath); + return computeCIDs({ stream, algorithms }); +} + +export { computeCIDs2 as computeCIDs }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-LTWPBXM5.js.map \ No newline at end of file diff --git a/dist/chunk-LTWPBXM5.js.map b/dist/chunk-LTWPBXM5.js.map new file mode 100644 index 0000000..1012106 --- /dev/null +++ b/dist/chunk-LTWPBXM5.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["computeCIDs"],"mappings":";;;;;AACA,OAAO,QAAQ;AASf,eAAOA,aAAmC,EAAC,UAAU,WAAU,GAGzC;AAClB,QAAM,SAAS,GAAG,iBAAiB,QAAQ;AAC3C,SAAO,YAAoB,EAAC,QAAQ,WAAU,CAAC;AACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-OHUFSPVO.cjs b/dist/chunk-OHUFSPVO.cjs new file mode 100644 index 0000000..9cf0f7a --- /dev/null +++ b/dist/chunk-OHUFSPVO.cjs @@ -0,0 +1,94 @@ +'use strict'; + +var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs'); +var stream = require('stream'); +var digest = require('multiformats/hashes/digest'); +var cid = require('multiformats/cid'); +var crypto = require('crypto'); +var crc32 = require('crc-32'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var crc32__default = /*#__PURE__*/_interopDefault(crc32); + +var Crc32Hash = class { + _crc = void 0; + update(data) { + this._crc = crc32__default.default.buf(data, this._crc); + return this; + } + digest() { + const buffer = Buffer.alloc(4); + buffer.writeInt32BE(this._crc, 0); + return buffer; + } +}; + +// src/lib/file-id/CreateHasher.ts +var codeTable = { + ["cid_crc32" /* crc32 */]: 306 /* crc32 */, + ["cid_md5" /* md5 */]: 213 /* md5 */, + ["cid_sha1" /* sha1 */]: 17 /* sha1 */, + ["cid_sha2-256" /* sha256 */]: 18 /* sha256 */, + ["cid_sha3-256" /* sha3_256 */]: 22 /* sha3_256 */, + ["cid_sha3-384" /* sha3_384 */]: 21 /* sha3_384 */ +}; +async function createHasher(algo) { + switch (algo) { + case "cid_sha2-256" /* sha256 */: + return crypto.createHash("sha256"); + case "cid_sha1" /* sha1 */: + return crypto.createHash("sha1"); + case "cid_md5" /* md5 */: + return crypto.createHash("md5"); + case "cid_sha3-256" /* sha3_256 */: + return crypto.createHash("sha3-256"); + case "cid_sha3-384" /* sha3_384 */: + return crypto.createHash("sha3-384"); + case "cid_crc32" /* crc32 */: + return new Crc32Hash(); + default: + throw new Error(`Unsupported algorithm: ${algo}`); + } +} + +// src/lib/file-id/ComputeHash.ts +async function computeCIDs({ stream: stream$1, algorithms }) { + const hashers = await hasherDefiner(algorithms); + if (stream$1 instanceof stream.Readable) { + for await (const chunk of stream$1) { + for (const item of hashers) { + await item.hasher.update(chunk); + } + } + } else { + const reader = stream$1.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) + break; + for (const item of hashers) { + await item.hasher.update(Buffer.from(value)); + } + } + } + return cidFinalize(hashers); +} +async function hasherDefiner(algorithms) { + const hashers = algorithms.filter((algo) => Object.values(chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES).includes(algo)).map(async (algo) => ({ + hasher: await createHasher(algo), + code: codeTable[algo] + })); + return await Promise.all(hashers); +} +async function cidFinalize(hashers) { + return await Promise.all(hashers.map(async ({ code, hasher }) => { + const hashBuffer = await hasher.digest(); + const digest$1 = digest.create(code, hashBuffer); + return cid.CID.createV1(code, digest$1).toString(); + })); +} + +exports.computeCIDs = computeCIDs; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-OHUFSPVO.cjs.map \ No newline at end of file diff --git a/dist/chunk-OHUFSPVO.cjs.map b/dist/chunk-OHUFSPVO.cjs.map new file mode 100644 index 0000000..d6be70c --- /dev/null +++ b/dist/chunk-OHUFSPVO.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":[],"mappings":";;;;;AACA,SAAQ,gBAAe;AACvB,SAAQ,cAAa;AACrB,SAAQ,WAAU;;;ACDlB,SAAQ,kBAAiB;;;ACFzB,OAAO,WAAW;AAGX,IAAM,YAAN,MAAsC;AAAA,EACjC,OAAe;AAAA,EAEvB,OAAO,MAA0B;AAE7B,SAAK,OAAO,MAAM,IAAI,MAAM,KAAK,IAAI;AACrC,WAAO;AAAA,EACX;AAAA,EAEA,SAAiB;AACb,UAAM,SAAS,OAAO,MAAM,CAAC;AAC7B,WAAO,aAAa,KAAK,MAAM,CAAC;AAChC,WAAO;AAAA,EACX;AACJ;;;ADZO,IAAM,YAAY;AAAA,EACrB,wBAA0B;AAAA,EAC1B,oBAAwB;AAAA,EACxB,sBAAyB;AAAA,EACzB,4BAA2B;AAAA,EAC3B,8BAA6B;AAAA,EAC7B,8BAA6B;AACjC;AAOA,eAAsB,aAAa,MAAgD;AAC/E,UAAQ,MAAM;AAAA,IACV;AACI,aAAO,WAAW,QAAQ;AAAA,IAC9B;AACI,aAAO,WAAW,MAAM;AAAA,IAC5B;AACI,aAAO,WAAW,KAAK;AAAA,IAC3B;AACI,aAAO,WAAW,UAAU;AAAA,IAChC;AACI,aAAO,WAAW,UAAU;AAAA,IAChC;AACI,aAAO,IAAI,UAAU;AAAA,IACzB;AACI,YAAM,IAAI,MAAM,0BAA0B,IAAI,EAAE;AAAA,EACxD;AACJ;;;ADvBA,eAAsB,YAAY,EAAC,QAAQ,WAAU,GAG/B;AAClB,QAAM,UAAU,MAAM,cAAc,UAAU;AAG9C,MAAI,kBAAkB,UAAU;AAE5B,qBAAiB,SAAS,QAAQ;AAC9B,iBAAW,QAAQ,SAAS;AACxB,cAAM,KAAK,OAAO,OAAO,KAAe;AAAA,MAC5C;AAAA,IACJ;AAAA,EACJ,OAAO;AAEH,UAAM,SAAS,OAAO,UAAU;AAChC,WAAO,MAAM;AACT,YAAM,EAAC,MAAM,MAAK,IAAI,MAAM,OAAO,KAAK;AACxC,UAAI;AAAM;AAEV,iBAAW,QAAQ,SAAS;AACxB,cAAM,KAAK,OAAO,OAAO,OAAO,KAAK,KAAK,CAAC;AAAA,MAC/C;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO,YAAY,OAAO;AAC9B;AAEA,eAAe,cAAc,YAGxB;AACD,QAAM,UAAU,WAAW,OAAO,UAAQ,OAAO,OAAO,mBAAmB,EAAE,SAAS,IAAI,CAAC,EACtF,IAAI,OAAM,UAAS;AAAA,IAChB,QAAQ,MAAM,aAAa,IAAI;AAAA,IAC/B,MAAM,UAAU,IAAI;AAAA,EACxB,EAAE;AACN,SAAO,MAAM,QAAQ,IAAI,OAAO;AACpC;AAEA,eAAe,YAAY,SAAoE;AAC3F,SAAO,MAAM,QAAQ,IAAI,QAAQ,IAAI,OAAO,EAAC,MAAM,OAAM,MAAM;AAC3D,UAAM,aAAa,MAAM,OAAO,OAAO;AACvC,UAAM,SAAS,OAAO,MAAM,UAAU;AACtC,WAAO,IAAI,SAAS,MAAM,MAAM,EAAE,SAAS;AAAA,EAC/C,CAAC,CAAC;AACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = await hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n for (const item of hashers) {\r\n await item.hasher.update(chunk as Buffer);\r\n }\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n for (const item of hashers) {\r\n await item.hasher.update(Buffer.from(value));\r\n }\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nasync function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): Promise<{\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[]> {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(async algo => ({\r\n hasher: await createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return await Promise.all(hashers);\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = await hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport async function createHasher(algo: CID_ALGORITHM_NAMES): Promise {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): SimpleHash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-PAKJNZBJ.cjs b/dist/chunk-PAKJNZBJ.cjs deleted file mode 100644 index 140f26c..0000000 --- a/dist/chunk-PAKJNZBJ.cjs +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -var s=(a=>(a[a.crc32=306]="crc32",a[a.md5=213]="md5",a[a.sha1=17]="sha1",a[a.sha256=18]="sha256",a[a.sha3_256=22]="sha3_256",a[a.sha3_384=21]="sha3_384",a))(s||{}),c=(a=>(a.crc32="cid_crc32",a.md5="cid_md5",a.sha1="cid_sha1",a.sha256="cid_sha2-256",a.sha3_256="cid_sha3-256",a.sha3_384="cid_sha3-384",a))(c||{}),h=(a=>(a.crc32="crc32",a.md5="md5",a.sha1="sha1",a.sha256="sha2-256",a.sha3_256="sha3-256",a.sha3_384="sha3-384",a))(h||{}); - -exports.a = s; -exports.b = c; -exports.c = h; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-PAKJNZBJ.cjs.map \ No newline at end of file diff --git a/dist/chunk-PAKJNZBJ.cjs.map b/dist/chunk-PAKJNZBJ.cjs.map deleted file mode 100644 index 442e48d..0000000 --- a/dist/chunk-PAKJNZBJ.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":"AAiBO,IAAKA,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-S7BSPELE.cjs b/dist/chunk-S7BSPELE.cjs new file mode 100644 index 0000000..cd918a4 --- /dev/null +++ b/dist/chunk-S7BSPELE.cjs @@ -0,0 +1,36 @@ +'use strict'; + +// src/lib/hash-compute/MultiHashData.ts +var CID_ALGORITHM_CODES = /* @__PURE__ */ ((CID_ALGORITHM_CODES2) => { + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["crc32"] = 306] = "crc32"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["md5"] = 213] = "md5"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha1"] = 17] = "sha1"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha256"] = 18] = "sha256"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_256"] = 22] = "sha3_256"; + CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_384"] = 21] = "sha3_384"; + return CID_ALGORITHM_CODES2; +})(CID_ALGORITHM_CODES || {}); +var CID_ALGORITHM_NAMES = /* @__PURE__ */ ((CID_ALGORITHM_NAMES2) => { + CID_ALGORITHM_NAMES2["crc32"] = "cid_crc32"; + CID_ALGORITHM_NAMES2["md5"] = "cid_md5"; + CID_ALGORITHM_NAMES2["sha1"] = "cid_sha1"; + CID_ALGORITHM_NAMES2["sha256"] = "cid_sha2-256"; + CID_ALGORITHM_NAMES2["sha3_256"] = "cid_sha3-256"; + CID_ALGORITHM_NAMES2["sha3_384"] = "cid_sha3-384"; + return CID_ALGORITHM_NAMES2; +})(CID_ALGORITHM_NAMES || {}); +var CID_ALGORITHM = /* @__PURE__ */ ((CID_ALGORITHM2) => { + CID_ALGORITHM2["crc32"] = "crc32"; + CID_ALGORITHM2["md5"] = "md5"; + CID_ALGORITHM2["sha1"] = "sha1"; + CID_ALGORITHM2["sha256"] = "sha2-256"; + CID_ALGORITHM2["sha3_256"] = "sha3-256"; + CID_ALGORITHM2["sha3_384"] = "sha3-384"; + return CID_ALGORITHM2; +})(CID_ALGORITHM || {}); + +exports.CID_ALGORITHM = CID_ALGORITHM; +exports.CID_ALGORITHM_CODES = CID_ALGORITHM_CODES; +exports.CID_ALGORITHM_NAMES = CID_ALGORITHM_NAMES; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-S7BSPELE.cjs.map \ No newline at end of file diff --git a/dist/chunk-S7BSPELE.cjs.map b/dist/chunk-S7BSPELE.cjs.map new file mode 100644 index 0000000..9fda71c --- /dev/null +++ b/dist/chunk-S7BSPELE.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":";AAiBO,IAAK,sBAAL,kBAAKA,yBAAL;AACH,EAAAA,0CAAA,WAAQ,OAAR;AACA,EAAAA,0CAAA,SAAM,OAAN;AACA,EAAAA,0CAAA,UAAO,MAAP;AACA,EAAAA,0CAAA,YAAS,MAAT;AACA,EAAAA,0CAAA,cAAW,MAAX;AACA,EAAAA,0CAAA,cAAW,MAAX;AANQ,SAAAA;AAAA,GAAA;AASL,IAAK,sBAAL,kBAAKC,yBAAL;AACH,EAAAA,qBAAA,WAAQ;AACR,EAAAA,qBAAA,SAAM;AACN,EAAAA,qBAAA,UAAO;AACP,EAAAA,qBAAA,YAAS;AACT,EAAAA,qBAAA,cAAW;AACX,EAAAA,qBAAA,cAAW;AANH,SAAAA;AAAA,GAAA;AASL,IAAK,gBAAL,kBAAKC,mBAAL;AACH,EAAAA,eAAA,WAAQ;AACR,EAAAA,eAAA,SAAM;AACN,EAAAA,eAAA,UAAO;AACP,EAAAA,eAAA,YAAS;AACT,EAAAA,eAAA,cAAW;AACX,EAAAA,eAAA,cAAW;AANH,SAAAA;AAAA,GAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-SV45D2HN.js b/dist/chunk-SV45D2HN.js new file mode 100644 index 0000000..e919d7d --- /dev/null +++ b/dist/chunk-SV45D2HN.js @@ -0,0 +1,88 @@ +import { CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js'; +import { Readable } from 'stream'; +import { create } from 'multiformats/hashes/digest'; +import { CID } from 'multiformats/cid'; +import { createHash } from 'crypto'; +import crc32 from 'crc-32'; + +var Crc32Hash = class { + _crc = void 0; + update(data) { + this._crc = crc32.buf(data, this._crc); + return this; + } + digest() { + const buffer = Buffer.alloc(4); + buffer.writeInt32BE(this._crc, 0); + return buffer; + } +}; + +// src/lib/file-id/CreateHasher.ts +var codeTable = { + ["cid_crc32" /* crc32 */]: 306 /* crc32 */, + ["cid_md5" /* md5 */]: 213 /* md5 */, + ["cid_sha1" /* sha1 */]: 17 /* sha1 */, + ["cid_sha2-256" /* sha256 */]: 18 /* sha256 */, + ["cid_sha3-256" /* sha3_256 */]: 22 /* sha3_256 */, + ["cid_sha3-384" /* sha3_384 */]: 21 /* sha3_384 */ +}; +async function createHasher(algo) { + switch (algo) { + case "cid_sha2-256" /* sha256 */: + return createHash("sha256"); + case "cid_sha1" /* sha1 */: + return createHash("sha1"); + case "cid_md5" /* md5 */: + return createHash("md5"); + case "cid_sha3-256" /* sha3_256 */: + return createHash("sha3-256"); + case "cid_sha3-384" /* sha3_384 */: + return createHash("sha3-384"); + case "cid_crc32" /* crc32 */: + return new Crc32Hash(); + default: + throw new Error(`Unsupported algorithm: ${algo}`); + } +} + +// src/lib/file-id/ComputeHash.ts +async function computeCIDs({ stream, algorithms }) { + const hashers = await hasherDefiner(algorithms); + if (stream instanceof Readable) { + for await (const chunk of stream) { + for (const item of hashers) { + await item.hasher.update(chunk); + } + } + } else { + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) + break; + for (const item of hashers) { + await item.hasher.update(Buffer.from(value)); + } + } + } + return cidFinalize(hashers); +} +async function hasherDefiner(algorithms) { + const hashers = algorithms.filter((algo) => Object.values(CID_ALGORITHM_NAMES).includes(algo)).map(async (algo) => ({ + hasher: await createHasher(algo), + code: codeTable[algo] + })); + return await Promise.all(hashers); +} +async function cidFinalize(hashers) { + return await Promise.all(hashers.map(async ({ code, hasher }) => { + const hashBuffer = await hasher.digest(); + const digest = create(code, hashBuffer); + return CID.createV1(code, digest).toString(); + })); +} + +export { computeCIDs }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-SV45D2HN.js.map \ No newline at end of file diff --git a/dist/chunk-SV45D2HN.js.map b/dist/chunk-SV45D2HN.js.map new file mode 100644 index 0000000..d6be70c --- /dev/null +++ b/dist/chunk-SV45D2HN.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":[],"mappings":";;;;;AACA,SAAQ,gBAAe;AACvB,SAAQ,cAAa;AACrB,SAAQ,WAAU;;;ACDlB,SAAQ,kBAAiB;;;ACFzB,OAAO,WAAW;AAGX,IAAM,YAAN,MAAsC;AAAA,EACjC,OAAe;AAAA,EAEvB,OAAO,MAA0B;AAE7B,SAAK,OAAO,MAAM,IAAI,MAAM,KAAK,IAAI;AACrC,WAAO;AAAA,EACX;AAAA,EAEA,SAAiB;AACb,UAAM,SAAS,OAAO,MAAM,CAAC;AAC7B,WAAO,aAAa,KAAK,MAAM,CAAC;AAChC,WAAO;AAAA,EACX;AACJ;;;ADZO,IAAM,YAAY;AAAA,EACrB,wBAA0B;AAAA,EAC1B,oBAAwB;AAAA,EACxB,sBAAyB;AAAA,EACzB,4BAA2B;AAAA,EAC3B,8BAA6B;AAAA,EAC7B,8BAA6B;AACjC;AAOA,eAAsB,aAAa,MAAgD;AAC/E,UAAQ,MAAM;AAAA,IACV;AACI,aAAO,WAAW,QAAQ;AAAA,IAC9B;AACI,aAAO,WAAW,MAAM;AAAA,IAC5B;AACI,aAAO,WAAW,KAAK;AAAA,IAC3B;AACI,aAAO,WAAW,UAAU;AAAA,IAChC;AACI,aAAO,WAAW,UAAU;AAAA,IAChC;AACI,aAAO,IAAI,UAAU;AAAA,IACzB;AACI,YAAM,IAAI,MAAM,0BAA0B,IAAI,EAAE;AAAA,EACxD;AACJ;;;ADvBA,eAAsB,YAAY,EAAC,QAAQ,WAAU,GAG/B;AAClB,QAAM,UAAU,MAAM,cAAc,UAAU;AAG9C,MAAI,kBAAkB,UAAU;AAE5B,qBAAiB,SAAS,QAAQ;AAC9B,iBAAW,QAAQ,SAAS;AACxB,cAAM,KAAK,OAAO,OAAO,KAAe;AAAA,MAC5C;AAAA,IACJ;AAAA,EACJ,OAAO;AAEH,UAAM,SAAS,OAAO,UAAU;AAChC,WAAO,MAAM;AACT,YAAM,EAAC,MAAM,MAAK,IAAI,MAAM,OAAO,KAAK;AACxC,UAAI;AAAM;AAEV,iBAAW,QAAQ,SAAS;AACxB,cAAM,KAAK,OAAO,OAAO,OAAO,KAAK,KAAK,CAAC;AAAA,MAC/C;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO,YAAY,OAAO;AAC9B;AAEA,eAAe,cAAc,YAGxB;AACD,QAAM,UAAU,WAAW,OAAO,UAAQ,OAAO,OAAO,mBAAmB,EAAE,SAAS,IAAI,CAAC,EACtF,IAAI,OAAM,UAAS;AAAA,IAChB,QAAQ,MAAM,aAAa,IAAI;AAAA,IAC/B,MAAM,UAAU,IAAI;AAAA,EACxB,EAAE;AACN,SAAO,MAAM,QAAQ,IAAI,OAAO;AACpC;AAEA,eAAe,YAAY,SAAoE;AAC3F,SAAO,MAAM,QAAQ,IAAI,QAAQ,IAAI,OAAO,EAAC,MAAM,OAAM,MAAM;AAC3D,UAAM,aAAa,MAAM,OAAO,OAAO;AACvC,UAAM,SAAS,OAAO,MAAM,UAAU;AACtC,WAAO,IAAI,SAAS,MAAM,MAAM,EAAE,SAAS;AAAA,EAC/C,CAAC,CAAC;AACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = await hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n for (const item of hashers) {\r\n await item.hasher.update(chunk as Buffer);\r\n }\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n for (const item of hashers) {\r\n await item.hasher.update(Buffer.from(value));\r\n }\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nasync function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): Promise<{\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[]> {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(async algo => ({\r\n hasher: await createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return await Promise.all(hashers);\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = await hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport async function createHasher(algo: CID_ALGORITHM_NAMES): Promise {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): SimpleHash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-Y5FIE5OK.cjs b/dist/chunk-Y5FIE5OK.cjs new file mode 100644 index 0000000..41ac39e --- /dev/null +++ b/dist/chunk-Y5FIE5OK.cjs @@ -0,0 +1,17 @@ +'use strict'; + +var chunkOHUFSPVO_cjs = require('./chunk-OHUFSPVO.cjs'); +var fs = require('fs'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var fs__default = /*#__PURE__*/_interopDefault(fs); + +async function computeCIDs2({ filePath, algorithms }) { + const stream = fs__default.default.createReadStream(filePath); + return chunkOHUFSPVO_cjs.computeCIDs({ stream, algorithms }); +} + +exports.computeCIDs = computeCIDs2; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-Y5FIE5OK.cjs.map \ No newline at end of file diff --git a/dist/chunk-Y5FIE5OK.cjs.map b/dist/chunk-Y5FIE5OK.cjs.map new file mode 100644 index 0000000..1012106 --- /dev/null +++ b/dist/chunk-Y5FIE5OK.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["computeCIDs"],"mappings":";;;;;AACA,OAAO,QAAQ;AASf,eAAOA,aAAmC,EAAC,UAAU,WAAU,GAGzC;AAClB,QAAM,SAAS,GAAG,iBAAiB,QAAQ;AAC3C,SAAO,YAAoB,EAAC,QAAQ,WAAU,CAAC;AACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-YCFZVUZC.js b/dist/chunk-YCFZVUZC.js deleted file mode 100644 index 8bf590a..0000000 --- a/dist/chunk-YCFZVUZC.js +++ /dev/null @@ -1,5 +0,0 @@ -var s=(a=>(a[a.crc32=306]="crc32",a[a.md5=213]="md5",a[a.sha1=17]="sha1",a[a.sha256=18]="sha256",a[a.sha3_256=22]="sha3_256",a[a.sha3_384=21]="sha3_384",a))(s||{}),c=(a=>(a.crc32="cid_crc32",a.md5="cid_md5",a.sha1="cid_sha1",a.sha256="cid_sha2-256",a.sha3_256="cid_sha3-256",a.sha3_384="cid_sha3-384",a))(c||{}),h=(a=>(a.crc32="crc32",a.md5="md5",a.sha1="sha1",a.sha256="sha2-256",a.sha3_256="sha3-256",a.sha3_384="sha3-384",a))(h||{}); - -export { s as a, c as b, h as c }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-YCFZVUZC.js.map \ No newline at end of file diff --git a/dist/chunk-YCFZVUZC.js.map b/dist/chunk-YCFZVUZC.js.map deleted file mode 100644 index 442e48d..0000000 --- a/dist/chunk-YCFZVUZC.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":"AAiBO,IAAKA,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/index-browser.cjs b/dist/index-browser.cjs index 8bceef3..ee856dd 100644 --- a/dist/index-browser.cjs +++ b/dist/index-browser.cjs @@ -1,27 +1,43 @@ 'use strict'; require('./chunk-KWYLOVWT.cjs'); -var chunk33LRP2HS_cjs = require('./chunk-33LRP2HS.cjs'); -var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); +var chunkOHUFSPVO_cjs = require('./chunk-OHUFSPVO.cjs'); +var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs'); -var a=class{constructor(r){this.targetHash=r;}async computeMissingHash(r,e){let t=this.targetHash.filter(o=>!e[o]);if(t.length===0)return;let i=await chunk33LRP2HS_cjs.a({stream:r,algorithms:t});for(let[o,c]of i.entries()){let n=t[o];e[n]=c;}}}; +// src/lib/hash-compute/HashComputer.ts +var HashComputer = class { + constructor(targetHash) { + this.targetHash = targetHash; + } + async computeMissingHash(stream, metadata) { + const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]); + if (neededHashes.length === 0) { + return; + } + const cids = await chunkOHUFSPVO_cjs.computeCIDs({ stream, algorithms: neededHashes }); + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +}; Object.defineProperty(exports, "computeCIDs", { - enumerable: true, - get: function () { return chunk33LRP2HS_cjs.a; } + enumerable: true, + get: function () { return chunkOHUFSPVO_cjs.computeCIDs; } }); Object.defineProperty(exports, "CID_ALGORITHM", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.c; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; } }); Object.defineProperty(exports, "CID_ALGORITHM_CODES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.a; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; } }); Object.defineProperty(exports, "CID_ALGORITHM_NAMES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.b; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; } }); -exports.HashComputer = a; +exports.HashComputer = HashComputer; //# sourceMappingURL=out.js.map //# sourceMappingURL=index-browser.cjs.map \ No newline at end of file diff --git a/dist/index-browser.cjs.map b/dist/index-browser.cjs.map index dbdadfa..cc141d7 100644 --- a/dist/index-browser.cjs.map +++ b/dist/index-browser.cjs.map @@ -1 +1 @@ -{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":["HashComputer","targetHash","stream","metadata","neededHashes","hashName","cids","computeCIDs","index","cid","hashType"],"mappings":"6HAKO,IAAMA,EAAN,KAAkB,CAErB,YAAoBC,EAAmC,CAAnC,gBAAAA,CACpB,CAEA,MAAM,mBAAmBC,EAAkBC,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAMC,EAAY,CAAC,OAAAL,EAAQ,WAAWE,CAAY,CAAC,EAGhE,OAAW,CAACI,EAAOC,CAAG,IAAKH,EAAK,QAAQ,EAAG,CACvC,IAAMI,EAAWN,EAAaI,CAAK,EACnCL,EAASO,CAAQ,EAAID,CACzB,CACJ,CACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":[],"mappings":";;;;;;;;;;;AAKO,IAAM,eAAN,MAAkB;AAAA,EAErB,YAAoB,YAAmC;AAAnC;AAAA,EACpB;AAAA,EAEA,MAAM,mBAAmB,QAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,YAAY,EAAC,QAAQ,YAAW,aAAY,CAAC;AAGhE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index-browser.js b/dist/index-browser.js index 06dbccf..94a0d92 100644 --- a/dist/index-browser.js +++ b/dist/index-browser.js @@ -1,10 +1,26 @@ import './chunk-XCJZALP2.js'; -import { a as a$1 } from './chunk-A6J5HDTV.js'; -export { a as computeCIDs } from './chunk-A6J5HDTV.js'; -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; +import { computeCIDs } from './chunk-SV45D2HN.js'; +export { computeCIDs } from './chunk-SV45D2HN.js'; +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js'; -var a=class{constructor(r){this.targetHash=r;}async computeMissingHash(r,e){let t=this.targetHash.filter(o=>!e[o]);if(t.length===0)return;let i=await a$1({stream:r,algorithms:t});for(let[o,c]of i.entries()){let n=t[o];e[n]=c;}}}; +// src/lib/hash-compute/HashComputer.ts +var HashComputer = class { + constructor(targetHash) { + this.targetHash = targetHash; + } + async computeMissingHash(stream, metadata) { + const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]); + if (neededHashes.length === 0) { + return; + } + const cids = await computeCIDs({ stream, algorithms: neededHashes }); + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +}; -export { a as HashComputer }; +export { HashComputer }; //# sourceMappingURL=out.js.map //# sourceMappingURL=index-browser.js.map \ No newline at end of file diff --git a/dist/index-browser.js.map b/dist/index-browser.js.map index defc454..cc141d7 100644 --- a/dist/index-browser.js.map +++ b/dist/index-browser.js.map @@ -1 +1 @@ -{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":["HashComputer","targetHash","stream","metadata","neededHashes","hashName","cids","computeCIDs","index","cid","hashType"],"mappings":"0HAKO,IAAMA,EAAN,KAAkB,CAErB,YAAoBC,EAAmC,CAAnC,gBAAAA,CACpB,CAEA,MAAM,mBAAmBC,EAAkBC,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAMC,EAAY,CAAC,OAAAL,EAAQ,WAAWE,CAAY,CAAC,EAGhE,OAAW,CAACI,EAAOC,CAAG,IAAKH,EAAK,QAAQ,EAAG,CACvC,IAAMI,EAAWN,EAAaI,CAAK,EACnCL,EAASO,CAAQ,EAAID,CACzB,CACJ,CACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":[],"mappings":";;;;;;;;;;;AAKO,IAAM,eAAN,MAAkB;AAAA,EAErB,YAAoB,YAAmC;AAAnC;AAAA,EACpB;AAAA,EAEA,MAAM,mBAAmB,QAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,YAAY,EAAC,QAAQ,YAAW,aAAY,CAAC;AAGhE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index-interface.cjs b/dist/index-interface.cjs index da4efea..35a4f5b 100644 --- a/dist/index-interface.cjs +++ b/dist/index-interface.cjs @@ -1,21 +1,21 @@ 'use strict'; require('./chunk-KWYLOVWT.cjs'); -var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); +var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs'); Object.defineProperty(exports, "CID_ALGORITHM", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.c; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; } }); Object.defineProperty(exports, "CID_ALGORITHM_CODES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.a; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; } }); Object.defineProperty(exports, "CID_ALGORITHM_NAMES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.b; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; } }); //# sourceMappingURL=out.js.map //# sourceMappingURL=index-interface.cjs.map \ No newline at end of file diff --git a/dist/index-interface.js b/dist/index-interface.js index ea6cd85..6a63d84 100644 --- a/dist/index-interface.js +++ b/dist/index-interface.js @@ -1,4 +1,4 @@ import './chunk-XCJZALP2.js'; -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js'; //# sourceMappingURL=out.js.map //# sourceMappingURL=index-interface.js.map \ No newline at end of file diff --git a/dist/index.cjs b/dist/index.cjs index c6d6472..0562de6 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -1,63 +1,496 @@ 'use strict'; require('./chunk-KWYLOVWT.cjs'); -require('./chunk-KZVIGNTH.cjs'); -require('./chunk-33LRP2HS.cjs'); -var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); +require('./chunk-Y5FIE5OK.cjs'); +require('./chunk-OHUFSPVO.cjs'); +var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs'); var piscina = require('piscina'); var fs = require('fs'); var csvParse = require('csv-parse'); var sync = require('csv-parse/sync'); var sync$1 = require('csv-stringify/sync'); var promises = require('fs/promises'); -var c = require('path'); +var path3 = require('path'); var timers = require('timers'); -var M = require('chokidar'); -var z = require('p-queue'); +var chokidar = require('chokidar'); +var PQueue = require('p-queue'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } function _interopNamespace(e) { - if (e && e.__esModule) return e; - var n = Object.create(null); - if (e) { - Object.keys(e).forEach(function (k) { - if (k !== 'default') { - var d = Object.getOwnPropertyDescriptor(e, k); - Object.defineProperty(n, k, d.get ? d : { - enumerable: true, - get: function () { return e[k]; } - }); - } - }); - } - n.default = e; - return Object.freeze(n); + if (e && e.__esModule) return e; + var n = Object.create(null); + if (e) { + Object.keys(e).forEach(function (k) { + if (k !== 'default') { + var d = Object.getOwnPropertyDescriptor(e, k); + Object.defineProperty(n, k, d.get ? d : { + enumerable: true, + get: function () { return e[k]; } + }); + } + }); + } + n.default = e; + return Object.freeze(n); } -var c__namespace = /*#__PURE__*/_interopNamespace(c); -var M__default = /*#__PURE__*/_interopDefault(M); -var z__default = /*#__PURE__*/_interopDefault(z); +var path3__namespace = /*#__PURE__*/_interopNamespace(path3); +var chokidar__default = /*#__PURE__*/_interopDefault(chokidar); +var PQueue__default = /*#__PURE__*/_interopDefault(PQueue); -var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var w=new URL(l).href;console.log("Worker URL: ",w);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||w});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let i=this.targetHash.filter(s=>!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=csvParse.parse({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=sync$1.stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),i=await promises.stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__namespace.default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await promises.readdir(e,{withFileTypes:!0});for(let r of i){let s=c__namespace.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M__default.default.watch(e,t):i=M__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a !metadata[hashName]); + if (neededHashes.length === 0) { + return; + } + const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes); + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +}; +async function existsAsync(filePath) { + try { + await promises.access(filePath, promises.constants.F_OK); + return true; + } catch { + return false; + } +} +var INDEX_HEADERS = ["path", "size", "mtime"]; +var HashIndexManager = class { + constructor(filePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */]) { + this.targetHash = targetHash; + filePath = filePath.replace(".csv", ""); + for (const hash of this.targetHash) { + this.filePaths[hash] = `${filePath}-${hash}.csv`; + if (!this.filePaths[hash]) { + throw new Error(`Invalid index file path for ${hash}`); + } + } + } + cache = /* @__PURE__ */ new Map(); + intervalId; + intervalTime = 3e4; + lastIndexFileSize = 0; + //size of the index file last time it was read + lastCacheFile; + //state of the file last time it was read + indexOpsInProgress = false; + hasChanged = false; + initialLoad; + filePaths = {}; + getCache() { + return new Map(this.cache); + } + /** + * After init consseutively calls to this method will not reload the index + * @param autosave + */ + async init(autosave = true) { + if (!this.initialLoad) { + this.initialLoad = new Promise(async (resolve, reject) => { + try { + for (const hash of this.targetHash) { + if (!this.checkCSVHeaders(this.filePaths[hash], hash)) { + throw new Error(`Invalid index file headers for ${hash}`); + } + await this.loadIndex(hash); + } + if (autosave) { + this.start(); + } + resolve(); + } catch (e) { + reject(e); + } + }); + } + return this.initialLoad; + } + // Function to check CSV headers + checkCSVHeaders(csvContent, hash) { + const records = sync.parse(csvContent, { + bom: true, + columns: true, + skip_empty_lines: true + }); + if (records.length === 0) { + return true; + } + const headers = Object.keys(records[0]); + const requiredHeaders = [...INDEX_HEADERS, hash]; + return requiredHeaders.every((header) => headers.includes(header)); + } + start() { + this.startAutoSave(this.intervalTime); + } + stopAutoSave() { + if (this.intervalId) { + timers.clearInterval(this.intervalId); + } + } + startAutoSave(time) { + this.stopAutoSave(); + this.intervalTime = time; + this.intervalId = setInterval(() => this.saveCacheToFile(), time); + } + async loadIndex(hash) { + if (await existsAsync(this.filePaths[hash])) { + const stats = await fs.promises.stat(this.filePaths[hash]); + if (this.lastIndexFileSize !== stats.size) { + const records = await this.readCsv(hash); + for (const record of records) { + let indexLine = this.cache.get(record.path); + if (!indexLine) { + this.cache.set(record.path, record); + } else { + indexLine[hash] = record[hash]; + } + } + this.lastIndexFileSize = stats.size; + this.lastCacheFile = records; + return records; + } else { + return this.lastCacheFile; + } + } + return []; + } + loadIndexFromCache() { + return Array.from(this.cache.values()); + } + async readCsv(hash) { + if (!await existsAsync(this.filePaths[hash])) { + return []; + } + const start = performance.now(); + const parser = csvParse.parse({ + columns: true, + skip_empty_lines: true + }); + const records = []; + return new Promise((resolve, reject) => { + fs.createReadStream(this.filePaths[hash]).pipe(parser).on("data", (record) => { + records.push(record); + }).on("end", () => { + resolve(records); + console.log(`Index read ${hash} time ${performance.now() - start}ms`); + }).on("error", (err) => { + reject(err); + }); + }); + } + async saveCacheToFile() { + if (this.indexOpsInProgress || !this.hasChanged) { + return; + } + this.hasChanged = false; + this.indexOpsInProgress = true; + const start = performance.now(); + let cacheRows; + if (this.cache.size !== 0) { + cacheRows = this.loadIndexFromCache(); + } + for (const hash of this.targetHash) { + let existingRows = await this.loadIndex(hash); + let existingRowsAsMap = new Map(existingRows.map((row) => [row.path, row])); + if (this.cache.size !== 0) { + const newRows = cacheRows.filter((row) => { + const newRow = !existingRowsAsMap.has(row.path); + return newRow && !!row[hash]; + }); + if (newRows.length !== 0) { + const csvString = sync$1.stringify(newRows, { + header: existingRows.length === 0, + // Only add header if the file was empty + columns: [ + { key: "path", header: "path" }, + { key: "size", header: "size" }, + { key: "mtime", header: "mtime" }, + { key: hash, header: hash } + ] + }); + await fs.promises.appendFile(this.filePaths[hash], csvString); + } + } + } + const totalTime = performance.now() - start; + console.log(`Index saved in ${totalTime}ms`); + if (totalTime * 10 > this.intervalTime) { + this.startAutoSave(totalTime * 10); + console.log(`Index save interval increased to ${totalTime * 10}ms`); + } + this.indexOpsInProgress = false; + } + async getCidForFileAsync(filePath) { + const fileName = path3__namespace.default.basename(filePath); + const stats = await promises.stat(filePath); + return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString()); + } + getCidForFile(filePath, fileSize, mtime) { + const fileName = path3__namespace.default.basename(filePath); + let fileNameIndex = this.cache.get(fileName); + for (const hash of this.targetHash) { + if (!fileNameIndex[hash]) { + delete fileNameIndex[hash]; + } + } + if (fileNameIndex) { + if (fileNameIndex.mtime) { + if (fileNameIndex.size === fileSize + "" && fileNameIndex.mtime === mtime) { + return fileNameIndex; + } + } else { + if (fileNameIndex.size === fileSize + "") { + return fileNameIndex; + } + } + } + return null; + } + addFileCid(filePath, fileSize, mtime, hashs) { + if (!filePath || !fileSize || !mtime || !hashs) { + throw new Error("Invalid parameters"); + } + for (const hash of this.targetHash) { + if (!hashs[hash]) { + throw new Error(`Missing hash ${hash}`); + } + } + const size = fileSize + ""; + const baseName = path3__namespace.default.basename(filePath); + let indexLine = this.cache.get(baseName); + if (!indexLine) { + let filteredHash = {}; + for (const hash of this.targetHash) { + filteredHash[hash] = hashs[hash]; + } + const data = { path: baseName, size, mtime, ...filteredHash }; + this.cache.set(baseName, data); + } else { + for (const hash of this.targetHash) { + indexLine[hash] = hashs[hash]; + } + } + this.hasChanged = true; + } +}; +var HashComputerIndexCache = class { + constructor(indexFilePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */], workerPath) { + this.targetHash = targetHash; + this.hashComputer = new HashComputerWorker(targetHash, workerPath); + this.hashIndexManager = new HashIndexManager(indexFilePath, targetHash); + } + hashIndexManager; + hashComputer; + async computeMissingHash(filePath, metadata) { + await this.hashIndexManager.init(); + let stats = await promises.stat(filePath); + if (this.hashIndexManager.getCache().has(path3__namespace.default.basename(filePath))) { + const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + if (indexLine) { + for (const hash of this.targetHash) { + if (!metadata[hash] && indexLine[hash]) { + metadata[hash] = indexLine[hash]; + } + } + } + } + await this.hashComputer.computeMissingHash(filePath, metadata); + this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata); + } + async getHashIndexManager() { + await this.hashIndexManager.init(); + return this.hashIndexManager; + } +}; +var FolderWatcher = class { + constructor(fileProcessor, WATCH_FOLDER_LIST, config) { + this.WATCH_FOLDER_LIST = WATCH_FOLDER_LIST; + this.config = config; + this.fileProcessor = fileProcessor; + } + initialized = false; + queue = new PQueue__default.default({ concurrency: 100, autoStart: true }); + fileProcessor; + queueSize = -1; + current = 0; + processing = /* @__PURE__ */ new Set(); + /** + * Process the directory and its subdirectories + * @param directory + * @param promises + * @private + */ + async processDirectory(directory, promises$1) { + const entries = await promises.readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path3__namespace.join(directory, entry.name); + if (entry.isDirectory()) { + await this.processDirectory(fullPath, promises$1); + } else { + const promise = this.processFile(fullPath); + promises$1.push(promise); + } + } + } + async countFile(directory) { + const entries = await promises.readdir(directory, { withFileTypes: true }); + const countPromises = []; + for (const entry of entries) { + const fullPath = path3__namespace.join(directory, entry.name); + if (entry.isDirectory()) { + countPromises.push(this.countFile(fullPath)); + } else { + countPromises.push(this.queue.add( + async () => await this.fileProcessor.canProcessFile(fullPath) ? 1 : 0 + )); + } + } + const counts = await Promise.all(countPromises); + return counts.reduce((acc, current) => acc + current, 0); + } + async processFile(filePath) { + if (await this.fileProcessor.canProcessFile(filePath)) { + if (!this.processing.has(filePath)) { + this.processing.add(filePath); + const current = ++this.current; + await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath)); + this.processing.delete(filePath); + } + } + } + /** + * Process the file and its sibling file included other folder it is a recursive function + * @param filePath + * @private + */ + async processFileExtended(filePath) { + let dirname2 = path3__namespace.dirname(filePath); + await this.processDirectory(dirname2, []); + } + chokidarWatch(folderList) { + const chokidarconfig = { + ignoreInitial: true, + //ignore the initial scan we have our own function + persistent: true, + depth: Infinity, + awaitWriteFinish: { + stabilityThreshold: this.config.stabilityThreshold || 3e4, + pollInterval: this.config.pollInterval || 5e3 + } + }; + let watcher; + if ((this.config.interval || 0) <= 0) { + watcher = chokidar__default.default.watch(folderList, chokidarconfig); + } else { + watcher = chokidar__default.default.watch(folderList, { + ...chokidarconfig, + usePolling: true, + interval: this.config.interval || 0 + }); + } + watcher.on("add", async (filePath) => { + try { + await this.processFileExtended(filePath); + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } + }).on("change", async (filePath) => { + try { + await this.processFileExtended(filePath); + } catch (e) { + console.error(`Error processing file ${filePath}:`, e); + } + }).on("unlink", async (filePath) => { + try { + await this.fileProcessor.deleteFile(filePath); + await this.processFileExtended(filePath); + } catch (e) { + console.error(`Error processing file deletion:`, e); + } + }).on("error", (error) => console.error(`Watcher error: ${error}`)).on("ready", () => { + console.log(`Watching for file changes on ${folderList}`); + }); + } + async watch() { + let folders = this.WATCH_FOLDER_LIST; + if (!folders) { + throw new Error(`No folder to watch`); + } + let folderList = folders.split(","); + this.chokidarWatch(folderList); + let countPromises = []; + for (let i = 0; i < folderList.length; i++) { + folderList[i] = path3__namespace.normalize(folderList[i]); + countPromises.push(this.countFile(folderList[i])); + } + const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0); + console.log(`Found ${count} files to process`); + this.queueSize = count; + let promises = []; + for (let i = 0; i < folderList.length; i++) { + folderList[i] = path3__namespace.normalize(folderList[i]); + await this.processDirectory(folderList[i], promises); + } + await Promise.all(promises); + this.initialized = true; + console.log(`Watcher ready`); + } +}; Object.defineProperty(exports, "CID_ALGORITHM", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.c; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; } }); Object.defineProperty(exports, "CID_ALGORITHM_CODES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.a; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; } }); Object.defineProperty(exports, "CID_ALGORITHM_NAMES", { - enumerable: true, - get: function () { return chunkPAKJNZBJ_cjs.b; } + enumerable: true, + get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; } }); -exports.FileIDComputerWorker = p; -exports.FolderWatcher = F; -exports.HashComputerIndexCache = y; -exports.HashComputerWorker = m; -exports.HashIndexManager = d; -exports.INDEX_HEADERS = E; +exports.FileIDComputerWorker = FileIDComputerWorker; +exports.FolderWatcher = FolderWatcher; +exports.HashComputerIndexCache = HashComputerIndexCache; +exports.HashComputerWorker = HashComputerWorker; +exports.HashIndexManager = HashIndexManager; +exports.INDEX_HEADERS = INDEX_HEADERS; //# sourceMappingURL=out.js.map //# sourceMappingURL=index.cjs.map \ No newline at end of file diff --git a/dist/index.cjs.map b/dist/index.cjs.map index d811376..904756a 100644 --- a/dist/index.cjs.map +++ b/dist/index.cjs.map @@ -1 +1 @@ -{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputerWorker","workerPath","filePath","algorithms","HashComputerWorker","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","HashComputerIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"8IAAA,OAAQ,WAAAA,MAAc,UAKtB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAA0B,CACrB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC5BO,IAAMC,EAAN,KAAqD,CAGxD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAqBC,CAAU,CAC7D,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAyD,CAI5D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAmBC,EAAWJ,CAAU,EAChE,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["stat","path","dirname"],"mappings":";;;;;;;;;;AAAA,SAAQ,eAAc;AAKtB,IAAI,aAAa,YAAY;AAC7B,aAAa,WAAW,QAAQ,OAAO,MAAM;AAC7C,aAAa,aAAa;AAC1B,IAAM,YAAY,IAAI,IAAI,UAAU,EAAE;AACtC,QAAQ,IAAI,gBAAgB,SAAS;AAE9B,IAAM,uBAAN,MAA0B;AAAA,EACrB;AAAA,EAER,YAAY,YAAoB;AAC5B,SAAK,UAAU,IAAI,QAAQ;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,UAAU,cAAc,QAAQ,IAAI,cAAc;AAAA,IACtD,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAa,YAAY,UAAkB,YAAsD;AAC7F,WAAO,KAAK,QAAQ,IAAI,EAAC,UAAU,WAAU,CAAC;AAAA,EAClD;AACJ;;;AC5BO,IAAM,qBAAN,MAAqD;AAAA,EAGxD,YAAoB,YAAmC,YAAqB;AAAxD;AAChB,SAAK,iBAAiB,IAAI,qBAAqB,UAAU;AAAA,EAC7D;AAAA,EAJQ;AAAA,EAMR,MAAM,mBAAmB,UAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,KAAK,eAAe,YAAY,UAAU,YAAY;AAGzE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ;;;AC5BA,SAAQ,kBAAkB,YAAY,UAAS;AAC/C,SAAQ,aAAY;AACpB,SAAQ,SAAS,iBAAgB;AACjC,SAAQ,iBAAgB;;;ACDxB,SAAQ,QAAO,iBAAgB;AAE/B,eAAsB,YAAY,UAAoC;AAClE,MAAI;AACA,UAAM,OAAO,UAAU,UAAU,IAAI;AACrC,WAAO;AAAA,EACX,QAAQ;AACJ,WAAO;AAAA,EACX;AACJ;;;ADNA,OAAO,UAAU;AACjB,SAAQ,qBAAoB;AAE5B,SAAQ,YAAW;AAQZ,IAAM,gBAAgB,CAAC,QAAQ,QAAQ,OAAO;AAE9C,IAAM,mBAAN,MAAuB;AAAA,EAW1B,YAAY,UAA0B,aAAoC,mDAAqD,GAAG;AAA5F;AAClC,eAAW,SAAS,QAAQ,QAAQ,EAAE;AACtC,eAAW,QAAQ,KAAK,YAAY;AAChC,WAAK,UAAU,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI;AAC1C,UAAI,CAAC,KAAK,UAAU,IAAI,GAAG;AACvB,cAAM,IAAI,MAAM,+BAA+B,IAAI,EAAE;AAAA,MACzD;AAAA,IACJ;AAAA,EACJ;AAAA,EAlBQ,QAAgC,oBAAI,IAAuB;AAAA,EAC3D;AAAA,EACA,eAAuB;AAAA,EACvB,oBAA4B;AAAA;AAAA,EAC5B;AAAA;AAAA,EACA,qBAA8B;AAAA,EAC9B,aAAsB;AAAA,EACtB;AAAA,EACA,YAAuD,CAAC;AAAA,EAYzD,WAAmC;AACtC,WAAO,IAAI,IAAI,KAAK,KAAK;AAAA,EAC7B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAa,KAAK,WAAW,MAAM;AAC/B,QAAI,CAAC,KAAK,aAAa;AACnB,WAAK,cAAc,IAAI,QAAc,OAAO,SAAS,WAAW;AAC5D,YAAI;AACA,qBAAW,QAAQ,KAAK,YAAY;AAChC,gBAAI,CAAC,KAAK,gBAAgB,KAAK,UAAU,IAAI,GAAG,IAAI,GAAG;AACnD,oBAAM,IAAI,MAAM,kCAAkC,IAAI,EAAE;AAAA,YAC5D;AACA,kBAAM,KAAK,UAAU,IAAI;AAAA,UAC7B;AACA,cAAI,UAAU;AACV,iBAAK,MAAM;AAAA,UACf;AACA,kBAAQ;AAAA,QACZ,SAAS,GAAG;AACR,iBAAO,CAAC;AAAA,QACZ;AAAA,MACJ,CAAC;AAAA,IACL;AACA,WAAO,KAAK;AAAA,EAChB;AAAA;AAAA,EAGQ,gBAAgB,YAAoB,MAAoC;AAC5E,UAAM,UAAU,UAAU,YAAY;AAAA,MAClC,KAAK;AAAA,MACL,SAAS;AAAA,MACT,kBAAkB;AAAA,IACtB,CAAC;AAED,QAAI,QAAQ,WAAW,GAAG;AAEtB,aAAO;AAAA,IACX;AAGA,UAAM,UAAU,OAAO,KAAK,QAAQ,CAAC,CAAC;AAItC,UAAM,kBAAkB,CAAC,GAAG,eAAe,IAAI;AAG/C,WAAO,gBAAgB,MAAM,YAAU,QAAQ,SAAS,MAAM,CAAC;AAAA,EACnE;AAAA,EAEO,QAAQ;AACX,SAAK,cAAc,KAAK,YAAY;AAAA,EACxC;AAAA,EAEO,eAAe;AAClB,QAAI,KAAK,YAAY;AACjB,oBAAc,KAAK,UAAU;AAAA,IACjC;AAAA,EACJ;AAAA,EAEQ,cAAc,MAAc;AAChC,SAAK,aAAa;AAClB,SAAK,eAAe;AACpB,SAAK,aAAa,YAAY,MAAM,KAAK,gBAAgB,GAAG,IAAI;AAAA,EACpE;AAAA,EAEA,MAAa,UAAU,MAAiD;AACpE,QAAI,MAAM,YAAY,KAAK,UAAU,IAAI,CAAC,GAAG;AAEzC,YAAM,QAAQ,MAAM,GAAG,KAAK,KAAK,UAAU,IAAI,CAAC;AAChD,UAAI,KAAK,sBAAsB,MAAM,MAAM;AAEvC,cAAM,UAAuB,MAAM,KAAK,QAAQ,IAAI;AACpD,mBAAW,UAAU,SAAS;AAC1B,cAAI,YAAY,KAAK,MAAM,IAAI,OAAO,IAAI;AAC1C,cAAI,CAAC,WAAW;AACZ,iBAAK,MAAM,IAAI,OAAO,MAAM,MAAM;AAAA,UACtC,OAAO;AAEH,sBAAU,IAAI,IAAI,OAAO,IAAI;AAAA,UACjC;AAAA,QACJ;AACA,aAAK,oBAAoB,MAAM;AAC/B,aAAK,gBAAgB;AACrB,eAAO;AAAA,MACX,OAAO;AACH,eAAO,KAAK;AAAA,MAChB;AAAA,IACJ;AACA,WAAO,CAAC;AAAA,EACZ;AAAA,EAEQ,qBAAkC;AACtC,WAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC;AAAA,EACzC;AAAA,EAEA,MAAc,QAAQ,MAAiD;AACnE,QAAI,CAAE,MAAM,YAAY,KAAK,UAAU,IAAI,CAAC,GAAI;AAC5C,aAAO,CAAC;AAAA,IACZ;AACA,UAAM,QAAQ,YAAY,IAAI;AAE9B,UAAM,SAAS,MAAM;AAAA,MACjB,SAAS;AAAA,MACT,kBAAkB;AAAA,IACtB,CAAC;AAED,UAAM,UAAuB,CAAC;AAE9B,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACpC,uBAAiB,KAAK,UAAU,IAAI,CAAC,EAChC,KAAK,MAAM,EACX,GAAG,QAAQ,CAAC,WAAW;AACpB,gBAAQ,KAAK,MAAM;AAAA,MACvB,CAAC,EAAE,GAAG,OAAO,MAAM;AACnB,gBAAQ,OAAO;AACf,gBAAQ,IAAI,cAAc,IAAI,SAAS,YAAY,IAAI,IAAI,KAAK,IAAI;AAAA,MACxE,CAAC,EAAE,GAAG,SAAS,CAAC,QAAQ;AACpB,eAAO,GAAG;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAAA,EACL;AAAA,EAEA,MAAa,kBAAiC;AAC1C,QAAI,KAAK,sBAAsB,CAAC,KAAK,YAAY;AAC7C;AAAA,IACJ;AACA,SAAK,aAAa;AAClB,SAAK,qBAAqB;AAC1B,UAAM,QAAQ,YAAY,IAAI;AAE9B,QAAI;AACJ,QAAI,KAAK,MAAM,SAAS,GAAG;AACvB,kBAAY,KAAK,mBAAmB;AAAA,IACxC;AAEA,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,eAA4B,MAAM,KAAK,UAAU,IAAI;AACzD,UAAI,oBAA4C,IAAI,IAAI,aAAa,IAAI,SAAO,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;AAChG,UAAI,KAAK,MAAM,SAAS,GAAG;AAEvB,cAAM,UAAU,UAAU,OAAO,SAAO;AAEpC,gBAAM,SAAS,CAAC,kBAAkB,IAAI,IAAI,IAAI;AAC9C,iBAAO,UAAU,CAAC,CAAC,IAAI,IAAI;AAAA,QAC/B,CAAC;AAED,YAAI,QAAQ,WAAW,GAAG;AAEtB,gBAAM,YAAY,UAAU,SAAS;AAAA,YACjC,QAAQ,aAAa,WAAW;AAAA;AAAA,YAChC,SAAS;AAAA,cACL,EAAC,KAAK,QAAQ,QAAQ,OAAM;AAAA,cAC5B,EAAC,KAAK,QAAQ,QAAQ,OAAM;AAAA,cAC5B,EAAC,KAAK,SAAS,QAAQ,QAAO;AAAA,cAC9B,EAAC,KAAK,MAAM,QAAQ,KAAI;AAAA,YAC5B;AAAA,UACJ,CAAC;AAGD,gBAAM,GAAG,WAAW,KAAK,UAAU,IAAI,GAAG,SAAS;AAAA,QACvD;AAAA,MACJ;AAAA,IACJ;AAEA,UAAM,YAAY,YAAY,IAAI,IAAI;AACtC,YAAQ,IAAI,kBAAkB,SAAS,IAAI;AAE3C,QAAI,YAAY,KAAK,KAAK,cAAc;AACpC,WAAK,cAAc,YAAY,EAAE;AACjC,cAAQ,IAAI,oCAAoC,YAAY,EAAE,IAAI;AAAA,IACtE;AACA,SAAK,qBAAqB;AAAA,EAC9B;AAAA,EAEA,MAAa,mBAAmB,UAAsC;AAClE,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,UAAM,QAAQ,MAAM,KAAK,QAAQ;AACjC,WAAO,KAAK,cAAc,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,CAAC;AAAA,EAC7E;AAAA,EAEO,cAAc,UAAkB,UAAkB,OAA0B;AAC/E,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,QAAI,gBAAgB,KAAK,MAAM,IAAI,QAAQ;AAC3C,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,CAAC,cAAc,IAAI,GAAG;AACtB,eAAO,cAAc,IAAI;AAAA,MAC7B;AAAA,IACJ;AACA,QAAI,eAAe;AACf,UAAI,cAAc,OAAO;AAErB,YAAI,cAAc,SAAU,WAAW,MAAO,cAAc,UAAU,OAAO;AACzE,iBAAO;AAAA,QACX;AAAA,MACJ,OAAO;AAEH,YAAI,cAAc,SAAU,WAAW,IAAK;AACxC,iBAAO;AAAA,QACX;AAAA,MACJ;AAAA,IACJ;AAQA,WAAO;AAAA,EACX;AAAA,EAEO,WAAW,UAAkB,UAAkB,OAAe,OAA2D;AAC5H,QAAI,CAAC,YAAY,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO;AAC5C,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACxC;AACA,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,CAAC,MAAM,IAAI,GAAG;AACd,cAAM,IAAI,MAAM,gBAAgB,IAAI,EAAE;AAAA,MAC1C;AAAA,IACJ;AACA,UAAM,OAAO,WAAW;AACxB,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,QAAI,YAAY,KAAK,MAAM,IAAI,QAAQ;AACvC,QAAI,CAAC,WAAW;AAEZ,UAAI,eAAe,CAAC;AACpB,iBAAW,QAAQ,KAAK,YAAY;AAChC,qBAAa,IAAI,IAAI,MAAM,IAAI;AAAA,MACnC;AACA,YAAM,OAAO,EAAC,MAAM,UAAU,MAAY,OAAc,GAAG,aAAY;AACvE,WAAK,MAAM,IAAI,UAAU,IAAI;AAAA,IACjC,OAAO;AAEH,iBAAW,QAAQ,KAAK,YAAY;AAChC,kBAAU,IAAI,IAAI,MAAM,IAAI;AAAA,MAChC;AAAA,IACJ;AACA,SAAK,aAAa;AAAA,EACtB;AAEJ;;;AEzRA,SAAQ,QAAAA,aAAW;AACnB,OAAOC,WAAU;AAEV,IAAM,yBAAN,MAAyD;AAAA,EAI5D,YAAY,eAA8B,aAAoC,mDAAqD,GAAE,YAAoB;AAA/G;AACtC,SAAK,eAAe,IAAI,mBAAmB,YAAW,UAAU;AAChE,SAAK,mBAAmB,IAAI,iBAAiB,eAAc,UAAU;AAAA,EACzE;AAAA,EANQ;AAAA,EACA;AAAA,EAOR,MAAa,mBAAmB,UAAkB,UAAwC;AACtF,UAAM,KAAK,iBAAiB,KAAK;AACjC,QAAI,QAAQ,MAAMD,MAAK,QAAQ;AAC/B,QAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIC,MAAK,SAAS,QAAQ,CAAC,GAAG;AAC/D,YAAM,YAAY,KAAK,iBAAiB,cAAc,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,CAAC;AACrG,UAAG,WAAW;AACV,mBAAW,QAAQ,KAAK,YAAY;AAChC,cAAI,CAAC,SAAS,IAAI,KAAK,UAAU,IAAI,GAAG;AACpC,qBAAS,IAAI,IAAI,UAAU,IAAI;AAAA,UACnC;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AACA,UAAM,KAAK,aAAa,mBAAmB,UAAU,QAAQ;AAC7D,SAAK,iBAAiB,WAAW,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,GAAG,QAAQ;AAAA,EAC9F;AAAA,EAEA,MAAM,sBAAiD;AACnD,UAAM,KAAK,iBAAiB,KAAK;AACjC,WAAO,KAAK;AAAA,EAChB;AACJ;;;ACpCA,YAAYA,WAAU;AACtB,OAAO,cAAc;AACrB,SAAQ,eAAc;AACtB,OAAO,YAAY;AAGZ,IAAM,gBAAN,MAAoB;AAAA,EAQvB,YAAY,eAA6C,mBAAmC,QAIzF;AAJsD;AAAmC;AAKxF,SAAK,gBAAgB;AAAA,EACzB;AAAA,EAbA,cAAc;AAAA,EACd,QAAQ,IAAI,OAAO,EAAC,aAAa,KAAK,WAAW,KAAI,CAAC;AAAA,EACtD;AAAA,EACA,YAAY;AAAA,EACZ,UAAU;AAAA,EACV,aAAa,oBAAI,IAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgB7B,MAAc,iBAAiB,WAAmB,UAA0B;AACxE,UAAM,UAAU,MAAM,QAAQ,WAAW,EAAC,eAAe,KAAI,CAAC;AAC9D,eAAW,SAAS,SAAS;AACzB,YAAM,WAAgB,WAAK,WAAW,MAAM,IAAI;AAChD,UAAI,MAAM,YAAY,GAAG;AACrB,cAAM,KAAK,iBAAiB,UAAU,QAAQ;AAAA,MAClD,OAAO;AACH,cAAM,UAAU,KAAK,YAAY,QAAQ;AACzC,iBAAS,KAAK,OAAO;AAAA,MACzB;AAAA,IACJ;AAAA,EACJ;AAAA,EAEA,MAAc,UAAU,WAAoC;AACxD,UAAM,UAAU,MAAM,QAAQ,WAAW,EAAC,eAAe,KAAI,CAAC;AAC9D,UAAM,gBAAgC,CAAC;AAEvC,eAAW,SAAS,SAAS;AACzB,YAAM,WAAgB,WAAK,WAAW,MAAM,IAAI;AAChD,UAAI,MAAM,YAAY,GAAG;AAErB,sBAAc,KAAK,KAAK,UAAU,QAAQ,CAAC;AAAA,MAC/C,OAAO;AAEH,sBAAc,KAAK,KAAK,MAAM;AAAA,UAC1B,YAAa,MAAM,KAAK,cAAc,eAAe,QAAQ,IAAK,IAAI;AAAA,QAC1E,CAAC;AAAA,MACL;AAAA,IACJ;AAGA,UAAM,SAAS,MAAM,QAAQ,IAAI,aAAa;AAC9C,WAAO,OAAO,OAAO,CAAC,KAAK,YAAY,MAAM,SAAS,CAAC;AAAA,EAC3D;AAAA,EAEA,MAAc,YAAY,UAAgC;AACtD,QAAG,MAAM,KAAK,cAAc,eAAe,QAAQ,GAAG;AAClD,UAAG,CAAC,KAAK,WAAW,IAAI,QAAQ,GAAG;AAC/B,aAAK,WAAW,IAAI,QAAQ;AAC5B,cAAM,UAAU,EAAE,KAAK;AACvB,cAAM,KAAK,MAAM,IAAI,MAAM,KAAK,cAAc,YAAY,SAAS,KAAK,WAAW,QAAQ,CAAC;AAC5F,aAAK,WAAW,OAAO,QAAQ;AAAA,MACnC;AAAA,IACJ;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB,UAAgC;AAC9D,QAAIC,WAAe,cAAQ,QAAQ;AACnC,UAAM,KAAK,iBAAiBA,UAAS,CAAC,CAAC;AAAA,EAC3C;AAAA,EAEQ,cAAc,YAAqB;AACvC,UAAM,iBAAiB;AAAA,MACnB,eAAe;AAAA;AAAA,MACf,YAAY;AAAA,MACZ,OAAO;AAAA,MACP,kBAAkB;AAAA,QACd,oBAAoB,KAAK,OAAO,sBAAsB;AAAA,QACtD,cAAc,KAAK,OAAO,gBAAgB;AAAA,MAC9C;AAAA,IACJ;AAEA,QAAI;AACJ,SAAK,KAAK,OAAO,YAAY,MAAM,GAAG;AAClC,gBAAU,SAAS,MAAM,YAAY,cAAc;AAAA,IACvD,OAAO;AACH,gBAAU,SAAS,MAAM,YAAY;AAAA,QACjC,GAAG;AAAA,QACH,YAAY;AAAA,QACZ,UAAU,KAAK,OAAO,YAAY;AAAA,MACtC,CAAC;AAAA,IACL;AAEA,YACK,GAAG,OAAO,OAAO,aAAa;AAC3B,UAAI;AACA,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,OAAO;AACZ,gBAAQ,MAAM,yBAAyB,QAAQ,KAAK,KAAK;AAAA,MAC7D;AAAA,IACJ,CAAC,EACA,GAAG,UAAU,OAAO,aAAa;AAC9B,UAAI;AACA,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,GAAG;AACR,gBAAQ,MAAM,yBAAyB,QAAQ,KAAK,CAAC;AAAA,MACzD;AAAA,IACJ,CAAC,EACA,GAAG,UAAU,OAAO,aAAa;AAC9B,UAAI;AACA,cAAM,KAAK,cAAc,WAAW,QAAQ;AAC5C,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,GAAG;AACR,gBAAQ,MAAM,mCAAmC,CAAC;AAAA,MACtD;AAAA,IACJ,CAAC,EACA,GAAG,SAAS,CAAC,UAAU,QAAQ,MAAM,kBAAkB,KAAK,EAAE,CAAC,EAC/D,GAAG,SAAS,MAAM;AACf,cAAQ,IAAI,gCAAgC,UAAU,EAAE;AAAA,IAC5D,CAAC;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACV,QAAI,UAAU,KAAK;AACnB,QAAI,CAAC,SAAS;AACV,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACxC;AACA,QAAI,aAAa,QAAQ,MAAM,GAAG;AAElC,SAAK,cAAc,UAAU;AAG7B,QAAI,gBAAgB,CAAC;AACrB,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AACxC,iBAAW,CAAC,IAAS,gBAAU,WAAW,CAAC,CAAC;AAC5C,oBAAc,KAAK,KAAK,UAAU,WAAW,CAAC,CAAC,CAAC;AAAA,IACpD;AACA,UAAM,SAAS,MAAM,QAAQ,IAAI,aAAa,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,SAAS,CAAC;AAC1F,YAAQ,IAAI,SAAS,KAAK,mBAAmB;AAC7C,SAAK,YAAY;AACjB,QAAI,WAA2B,CAAC;AAChC,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AACxC,iBAAW,CAAC,IAAS,gBAAU,WAAW,CAAC,CAAC;AAC5C,YAAM,KAAK,iBAAiB,WAAW,CAAC,GAAG,QAAQ;AAAA,IACvD;AACA,UAAM,QAAQ,IAAI,QAAQ;AAC1B,SAAK,cAAc;AAEnB,YAAQ,IAAI,eAAe;AAAA,EAC/B;AACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index 271c5f8..84ba4af 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,21 +1,454 @@ import './chunk-XCJZALP2.js'; -import './chunk-6UBSWOQL.js'; -import './chunk-A6J5HDTV.js'; -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; +import './chunk-LTWPBXM5.js'; +import './chunk-SV45D2HN.js'; +export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js'; import { Piscina } from 'piscina'; import { promises, createReadStream } from 'fs'; import { parse as parse$1 } from 'csv-parse'; import { parse } from 'csv-parse/sync'; import { stringify } from 'csv-stringify/sync'; import { stat, readdir, access, constants } from 'fs/promises'; -import * as c from 'path'; -import c__default from 'path'; +import * as path3 from 'path'; +import path3__default from 'path'; import { clearInterval } from 'node:timers'; -import M from 'chokidar'; -import z from 'p-queue'; +import chokidar from 'chokidar'; +import PQueue from 'p-queue'; -var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var w=new URL(l).href;console.log("Worker URL: ",w);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||w});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let i=this.targetHash.filter(s=>!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=parse$1({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),i=await stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await readdir(e,{withFileTypes:!0});for(let r of i){let s=c.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M.watch(e,t):i=M.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a !metadata[hashName]); + if (neededHashes.length === 0) { + return; + } + const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes); + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +}; +async function existsAsync(filePath) { + try { + await access(filePath, constants.F_OK); + return true; + } catch { + return false; + } +} +var INDEX_HEADERS = ["path", "size", "mtime"]; +var HashIndexManager = class { + constructor(filePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */]) { + this.targetHash = targetHash; + filePath = filePath.replace(".csv", ""); + for (const hash of this.targetHash) { + this.filePaths[hash] = `${filePath}-${hash}.csv`; + if (!this.filePaths[hash]) { + throw new Error(`Invalid index file path for ${hash}`); + } + } + } + cache = /* @__PURE__ */ new Map(); + intervalId; + intervalTime = 3e4; + lastIndexFileSize = 0; + //size of the index file last time it was read + lastCacheFile; + //state of the file last time it was read + indexOpsInProgress = false; + hasChanged = false; + initialLoad; + filePaths = {}; + getCache() { + return new Map(this.cache); + } + /** + * After init consseutively calls to this method will not reload the index + * @param autosave + */ + async init(autosave = true) { + if (!this.initialLoad) { + this.initialLoad = new Promise(async (resolve, reject) => { + try { + for (const hash of this.targetHash) { + if (!this.checkCSVHeaders(this.filePaths[hash], hash)) { + throw new Error(`Invalid index file headers for ${hash}`); + } + await this.loadIndex(hash); + } + if (autosave) { + this.start(); + } + resolve(); + } catch (e) { + reject(e); + } + }); + } + return this.initialLoad; + } + // Function to check CSV headers + checkCSVHeaders(csvContent, hash) { + const records = parse(csvContent, { + bom: true, + columns: true, + skip_empty_lines: true + }); + if (records.length === 0) { + return true; + } + const headers = Object.keys(records[0]); + const requiredHeaders = [...INDEX_HEADERS, hash]; + return requiredHeaders.every((header) => headers.includes(header)); + } + start() { + this.startAutoSave(this.intervalTime); + } + stopAutoSave() { + if (this.intervalId) { + clearInterval(this.intervalId); + } + } + startAutoSave(time) { + this.stopAutoSave(); + this.intervalTime = time; + this.intervalId = setInterval(() => this.saveCacheToFile(), time); + } + async loadIndex(hash) { + if (await existsAsync(this.filePaths[hash])) { + const stats = await promises.stat(this.filePaths[hash]); + if (this.lastIndexFileSize !== stats.size) { + const records = await this.readCsv(hash); + for (const record of records) { + let indexLine = this.cache.get(record.path); + if (!indexLine) { + this.cache.set(record.path, record); + } else { + indexLine[hash] = record[hash]; + } + } + this.lastIndexFileSize = stats.size; + this.lastCacheFile = records; + return records; + } else { + return this.lastCacheFile; + } + } + return []; + } + loadIndexFromCache() { + return Array.from(this.cache.values()); + } + async readCsv(hash) { + if (!await existsAsync(this.filePaths[hash])) { + return []; + } + const start = performance.now(); + const parser = parse$1({ + columns: true, + skip_empty_lines: true + }); + const records = []; + return new Promise((resolve, reject) => { + createReadStream(this.filePaths[hash]).pipe(parser).on("data", (record) => { + records.push(record); + }).on("end", () => { + resolve(records); + console.log(`Index read ${hash} time ${performance.now() - start}ms`); + }).on("error", (err) => { + reject(err); + }); + }); + } + async saveCacheToFile() { + if (this.indexOpsInProgress || !this.hasChanged) { + return; + } + this.hasChanged = false; + this.indexOpsInProgress = true; + const start = performance.now(); + let cacheRows; + if (this.cache.size !== 0) { + cacheRows = this.loadIndexFromCache(); + } + for (const hash of this.targetHash) { + let existingRows = await this.loadIndex(hash); + let existingRowsAsMap = new Map(existingRows.map((row) => [row.path, row])); + if (this.cache.size !== 0) { + const newRows = cacheRows.filter((row) => { + const newRow = !existingRowsAsMap.has(row.path); + return newRow && !!row[hash]; + }); + if (newRows.length !== 0) { + const csvString = stringify(newRows, { + header: existingRows.length === 0, + // Only add header if the file was empty + columns: [ + { key: "path", header: "path" }, + { key: "size", header: "size" }, + { key: "mtime", header: "mtime" }, + { key: hash, header: hash } + ] + }); + await promises.appendFile(this.filePaths[hash], csvString); + } + } + } + const totalTime = performance.now() - start; + console.log(`Index saved in ${totalTime}ms`); + if (totalTime * 10 > this.intervalTime) { + this.startAutoSave(totalTime * 10); + console.log(`Index save interval increased to ${totalTime * 10}ms`); + } + this.indexOpsInProgress = false; + } + async getCidForFileAsync(filePath) { + const fileName = path3__default.basename(filePath); + const stats = await stat(filePath); + return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString()); + } + getCidForFile(filePath, fileSize, mtime) { + const fileName = path3__default.basename(filePath); + let fileNameIndex = this.cache.get(fileName); + for (const hash of this.targetHash) { + if (!fileNameIndex[hash]) { + delete fileNameIndex[hash]; + } + } + if (fileNameIndex) { + if (fileNameIndex.mtime) { + if (fileNameIndex.size === fileSize + "" && fileNameIndex.mtime === mtime) { + return fileNameIndex; + } + } else { + if (fileNameIndex.size === fileSize + "") { + return fileNameIndex; + } + } + } + return null; + } + addFileCid(filePath, fileSize, mtime, hashs) { + if (!filePath || !fileSize || !mtime || !hashs) { + throw new Error("Invalid parameters"); + } + for (const hash of this.targetHash) { + if (!hashs[hash]) { + throw new Error(`Missing hash ${hash}`); + } + } + const size = fileSize + ""; + const baseName = path3__default.basename(filePath); + let indexLine = this.cache.get(baseName); + if (!indexLine) { + let filteredHash = {}; + for (const hash of this.targetHash) { + filteredHash[hash] = hashs[hash]; + } + const data = { path: baseName, size, mtime, ...filteredHash }; + this.cache.set(baseName, data); + } else { + for (const hash of this.targetHash) { + indexLine[hash] = hashs[hash]; + } + } + this.hasChanged = true; + } +}; +var HashComputerIndexCache = class { + constructor(indexFilePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */], workerPath) { + this.targetHash = targetHash; + this.hashComputer = new HashComputerWorker(targetHash, workerPath); + this.hashIndexManager = new HashIndexManager(indexFilePath, targetHash); + } + hashIndexManager; + hashComputer; + async computeMissingHash(filePath, metadata) { + await this.hashIndexManager.init(); + let stats = await stat(filePath); + if (this.hashIndexManager.getCache().has(path3__default.basename(filePath))) { + const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + if (indexLine) { + for (const hash of this.targetHash) { + if (!metadata[hash] && indexLine[hash]) { + metadata[hash] = indexLine[hash]; + } + } + } + } + await this.hashComputer.computeMissingHash(filePath, metadata); + this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata); + } + async getHashIndexManager() { + await this.hashIndexManager.init(); + return this.hashIndexManager; + } +}; +var FolderWatcher = class { + constructor(fileProcessor, WATCH_FOLDER_LIST, config) { + this.WATCH_FOLDER_LIST = WATCH_FOLDER_LIST; + this.config = config; + this.fileProcessor = fileProcessor; + } + initialized = false; + queue = new PQueue({ concurrency: 100, autoStart: true }); + fileProcessor; + queueSize = -1; + current = 0; + processing = /* @__PURE__ */ new Set(); + /** + * Process the directory and its subdirectories + * @param directory + * @param promises + * @private + */ + async processDirectory(directory, promises) { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path3.join(directory, entry.name); + if (entry.isDirectory()) { + await this.processDirectory(fullPath, promises); + } else { + const promise = this.processFile(fullPath); + promises.push(promise); + } + } + } + async countFile(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + const countPromises = []; + for (const entry of entries) { + const fullPath = path3.join(directory, entry.name); + if (entry.isDirectory()) { + countPromises.push(this.countFile(fullPath)); + } else { + countPromises.push(this.queue.add( + async () => await this.fileProcessor.canProcessFile(fullPath) ? 1 : 0 + )); + } + } + const counts = await Promise.all(countPromises); + return counts.reduce((acc, current) => acc + current, 0); + } + async processFile(filePath) { + if (await this.fileProcessor.canProcessFile(filePath)) { + if (!this.processing.has(filePath)) { + this.processing.add(filePath); + const current = ++this.current; + await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath)); + this.processing.delete(filePath); + } + } + } + /** + * Process the file and its sibling file included other folder it is a recursive function + * @param filePath + * @private + */ + async processFileExtended(filePath) { + let dirname2 = path3.dirname(filePath); + await this.processDirectory(dirname2, []); + } + chokidarWatch(folderList) { + const chokidarconfig = { + ignoreInitial: true, + //ignore the initial scan we have our own function + persistent: true, + depth: Infinity, + awaitWriteFinish: { + stabilityThreshold: this.config.stabilityThreshold || 3e4, + pollInterval: this.config.pollInterval || 5e3 + } + }; + let watcher; + if ((this.config.interval || 0) <= 0) { + watcher = chokidar.watch(folderList, chokidarconfig); + } else { + watcher = chokidar.watch(folderList, { + ...chokidarconfig, + usePolling: true, + interval: this.config.interval || 0 + }); + } + watcher.on("add", async (filePath) => { + try { + await this.processFileExtended(filePath); + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } + }).on("change", async (filePath) => { + try { + await this.processFileExtended(filePath); + } catch (e) { + console.error(`Error processing file ${filePath}:`, e); + } + }).on("unlink", async (filePath) => { + try { + await this.fileProcessor.deleteFile(filePath); + await this.processFileExtended(filePath); + } catch (e) { + console.error(`Error processing file deletion:`, e); + } + }).on("error", (error) => console.error(`Watcher error: ${error}`)).on("ready", () => { + console.log(`Watching for file changes on ${folderList}`); + }); + } + async watch() { + let folders = this.WATCH_FOLDER_LIST; + if (!folders) { + throw new Error(`No folder to watch`); + } + let folderList = folders.split(","); + this.chokidarWatch(folderList); + let countPromises = []; + for (let i = 0; i < folderList.length; i++) { + folderList[i] = path3.normalize(folderList[i]); + countPromises.push(this.countFile(folderList[i])); + } + const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0); + console.log(`Found ${count} files to process`); + this.queueSize = count; + let promises = []; + for (let i = 0; i < folderList.length; i++) { + folderList[i] = path3.normalize(folderList[i]); + await this.processDirectory(folderList[i], promises); + } + await Promise.all(promises); + this.initialized = true; + console.log(`Watcher ready`); + } +}; + +export { FileIDComputerWorker, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS }; //# sourceMappingURL=out.js.map //# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/index.js.map b/dist/index.js.map index afeace5..904756a 100644 --- a/dist/index.js.map +++ b/dist/index.js.map @@ -1 +1 @@ -{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputerWorker","workerPath","filePath","algorithms","HashComputerWorker","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","HashComputerIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"0IAAA,OAAQ,WAAAA,MAAc,UAKtB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAA0B,CACrB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC5BO,IAAMC,EAAN,KAAqD,CAGxD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAqBC,CAAU,CAC7D,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAyD,CAI5D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAmBC,EAAWJ,CAAU,EAChE,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["stat","path","dirname"],"mappings":";;;;;;;;;;AAAA,SAAQ,eAAc;AAKtB,IAAI,aAAa,YAAY;AAC7B,aAAa,WAAW,QAAQ,OAAO,MAAM;AAC7C,aAAa,aAAa;AAC1B,IAAM,YAAY,IAAI,IAAI,UAAU,EAAE;AACtC,QAAQ,IAAI,gBAAgB,SAAS;AAE9B,IAAM,uBAAN,MAA0B;AAAA,EACrB;AAAA,EAER,YAAY,YAAoB;AAC5B,SAAK,UAAU,IAAI,QAAQ;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,UAAU,cAAc,QAAQ,IAAI,cAAc;AAAA,IACtD,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAa,YAAY,UAAkB,YAAsD;AAC7F,WAAO,KAAK,QAAQ,IAAI,EAAC,UAAU,WAAU,CAAC;AAAA,EAClD;AACJ;;;AC5BO,IAAM,qBAAN,MAAqD;AAAA,EAGxD,YAAoB,YAAmC,YAAqB;AAAxD;AAChB,SAAK,iBAAiB,IAAI,qBAAqB,UAAU;AAAA,EAC7D;AAAA,EAJQ;AAAA,EAMR,MAAM,mBAAmB,UAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,KAAK,eAAe,YAAY,UAAU,YAAY;AAGzE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ;;;AC5BA,SAAQ,kBAAkB,YAAY,UAAS;AAC/C,SAAQ,aAAY;AACpB,SAAQ,SAAS,iBAAgB;AACjC,SAAQ,iBAAgB;;;ACDxB,SAAQ,QAAO,iBAAgB;AAE/B,eAAsB,YAAY,UAAoC;AAClE,MAAI;AACA,UAAM,OAAO,UAAU,UAAU,IAAI;AACrC,WAAO;AAAA,EACX,QAAQ;AACJ,WAAO;AAAA,EACX;AACJ;;;ADNA,OAAO,UAAU;AACjB,SAAQ,qBAAoB;AAE5B,SAAQ,YAAW;AAQZ,IAAM,gBAAgB,CAAC,QAAQ,QAAQ,OAAO;AAE9C,IAAM,mBAAN,MAAuB;AAAA,EAW1B,YAAY,UAA0B,aAAoC,mDAAqD,GAAG;AAA5F;AAClC,eAAW,SAAS,QAAQ,QAAQ,EAAE;AACtC,eAAW,QAAQ,KAAK,YAAY;AAChC,WAAK,UAAU,IAAI,IAAI,GAAG,QAAQ,IAAI,IAAI;AAC1C,UAAI,CAAC,KAAK,UAAU,IAAI,GAAG;AACvB,cAAM,IAAI,MAAM,+BAA+B,IAAI,EAAE;AAAA,MACzD;AAAA,IACJ;AAAA,EACJ;AAAA,EAlBQ,QAAgC,oBAAI,IAAuB;AAAA,EAC3D;AAAA,EACA,eAAuB;AAAA,EACvB,oBAA4B;AAAA;AAAA,EAC5B;AAAA;AAAA,EACA,qBAA8B;AAAA,EAC9B,aAAsB;AAAA,EACtB;AAAA,EACA,YAAuD,CAAC;AAAA,EAYzD,WAAmC;AACtC,WAAO,IAAI,IAAI,KAAK,KAAK;AAAA,EAC7B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAa,KAAK,WAAW,MAAM;AAC/B,QAAI,CAAC,KAAK,aAAa;AACnB,WAAK,cAAc,IAAI,QAAc,OAAO,SAAS,WAAW;AAC5D,YAAI;AACA,qBAAW,QAAQ,KAAK,YAAY;AAChC,gBAAI,CAAC,KAAK,gBAAgB,KAAK,UAAU,IAAI,GAAG,IAAI,GAAG;AACnD,oBAAM,IAAI,MAAM,kCAAkC,IAAI,EAAE;AAAA,YAC5D;AACA,kBAAM,KAAK,UAAU,IAAI;AAAA,UAC7B;AACA,cAAI,UAAU;AACV,iBAAK,MAAM;AAAA,UACf;AACA,kBAAQ;AAAA,QACZ,SAAS,GAAG;AACR,iBAAO,CAAC;AAAA,QACZ;AAAA,MACJ,CAAC;AAAA,IACL;AACA,WAAO,KAAK;AAAA,EAChB;AAAA;AAAA,EAGQ,gBAAgB,YAAoB,MAAoC;AAC5E,UAAM,UAAU,UAAU,YAAY;AAAA,MAClC,KAAK;AAAA,MACL,SAAS;AAAA,MACT,kBAAkB;AAAA,IACtB,CAAC;AAED,QAAI,QAAQ,WAAW,GAAG;AAEtB,aAAO;AAAA,IACX;AAGA,UAAM,UAAU,OAAO,KAAK,QAAQ,CAAC,CAAC;AAItC,UAAM,kBAAkB,CAAC,GAAG,eAAe,IAAI;AAG/C,WAAO,gBAAgB,MAAM,YAAU,QAAQ,SAAS,MAAM,CAAC;AAAA,EACnE;AAAA,EAEO,QAAQ;AACX,SAAK,cAAc,KAAK,YAAY;AAAA,EACxC;AAAA,EAEO,eAAe;AAClB,QAAI,KAAK,YAAY;AACjB,oBAAc,KAAK,UAAU;AAAA,IACjC;AAAA,EACJ;AAAA,EAEQ,cAAc,MAAc;AAChC,SAAK,aAAa;AAClB,SAAK,eAAe;AACpB,SAAK,aAAa,YAAY,MAAM,KAAK,gBAAgB,GAAG,IAAI;AAAA,EACpE;AAAA,EAEA,MAAa,UAAU,MAAiD;AACpE,QAAI,MAAM,YAAY,KAAK,UAAU,IAAI,CAAC,GAAG;AAEzC,YAAM,QAAQ,MAAM,GAAG,KAAK,KAAK,UAAU,IAAI,CAAC;AAChD,UAAI,KAAK,sBAAsB,MAAM,MAAM;AAEvC,cAAM,UAAuB,MAAM,KAAK,QAAQ,IAAI;AACpD,mBAAW,UAAU,SAAS;AAC1B,cAAI,YAAY,KAAK,MAAM,IAAI,OAAO,IAAI;AAC1C,cAAI,CAAC,WAAW;AACZ,iBAAK,MAAM,IAAI,OAAO,MAAM,MAAM;AAAA,UACtC,OAAO;AAEH,sBAAU,IAAI,IAAI,OAAO,IAAI;AAAA,UACjC;AAAA,QACJ;AACA,aAAK,oBAAoB,MAAM;AAC/B,aAAK,gBAAgB;AACrB,eAAO;AAAA,MACX,OAAO;AACH,eAAO,KAAK;AAAA,MAChB;AAAA,IACJ;AACA,WAAO,CAAC;AAAA,EACZ;AAAA,EAEQ,qBAAkC;AACtC,WAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC;AAAA,EACzC;AAAA,EAEA,MAAc,QAAQ,MAAiD;AACnE,QAAI,CAAE,MAAM,YAAY,KAAK,UAAU,IAAI,CAAC,GAAI;AAC5C,aAAO,CAAC;AAAA,IACZ;AACA,UAAM,QAAQ,YAAY,IAAI;AAE9B,UAAM,SAAS,MAAM;AAAA,MACjB,SAAS;AAAA,MACT,kBAAkB;AAAA,IACtB,CAAC;AAED,UAAM,UAAuB,CAAC;AAE9B,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACpC,uBAAiB,KAAK,UAAU,IAAI,CAAC,EAChC,KAAK,MAAM,EACX,GAAG,QAAQ,CAAC,WAAW;AACpB,gBAAQ,KAAK,MAAM;AAAA,MACvB,CAAC,EAAE,GAAG,OAAO,MAAM;AACnB,gBAAQ,OAAO;AACf,gBAAQ,IAAI,cAAc,IAAI,SAAS,YAAY,IAAI,IAAI,KAAK,IAAI;AAAA,MACxE,CAAC,EAAE,GAAG,SAAS,CAAC,QAAQ;AACpB,eAAO,GAAG;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AAAA,EACL;AAAA,EAEA,MAAa,kBAAiC;AAC1C,QAAI,KAAK,sBAAsB,CAAC,KAAK,YAAY;AAC7C;AAAA,IACJ;AACA,SAAK,aAAa;AAClB,SAAK,qBAAqB;AAC1B,UAAM,QAAQ,YAAY,IAAI;AAE9B,QAAI;AACJ,QAAI,KAAK,MAAM,SAAS,GAAG;AACvB,kBAAY,KAAK,mBAAmB;AAAA,IACxC;AAEA,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,eAA4B,MAAM,KAAK,UAAU,IAAI;AACzD,UAAI,oBAA4C,IAAI,IAAI,aAAa,IAAI,SAAO,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;AAChG,UAAI,KAAK,MAAM,SAAS,GAAG;AAEvB,cAAM,UAAU,UAAU,OAAO,SAAO;AAEpC,gBAAM,SAAS,CAAC,kBAAkB,IAAI,IAAI,IAAI;AAC9C,iBAAO,UAAU,CAAC,CAAC,IAAI,IAAI;AAAA,QAC/B,CAAC;AAED,YAAI,QAAQ,WAAW,GAAG;AAEtB,gBAAM,YAAY,UAAU,SAAS;AAAA,YACjC,QAAQ,aAAa,WAAW;AAAA;AAAA,YAChC,SAAS;AAAA,cACL,EAAC,KAAK,QAAQ,QAAQ,OAAM;AAAA,cAC5B,EAAC,KAAK,QAAQ,QAAQ,OAAM;AAAA,cAC5B,EAAC,KAAK,SAAS,QAAQ,QAAO;AAAA,cAC9B,EAAC,KAAK,MAAM,QAAQ,KAAI;AAAA,YAC5B;AAAA,UACJ,CAAC;AAGD,gBAAM,GAAG,WAAW,KAAK,UAAU,IAAI,GAAG,SAAS;AAAA,QACvD;AAAA,MACJ;AAAA,IACJ;AAEA,UAAM,YAAY,YAAY,IAAI,IAAI;AACtC,YAAQ,IAAI,kBAAkB,SAAS,IAAI;AAE3C,QAAI,YAAY,KAAK,KAAK,cAAc;AACpC,WAAK,cAAc,YAAY,EAAE;AACjC,cAAQ,IAAI,oCAAoC,YAAY,EAAE,IAAI;AAAA,IACtE;AACA,SAAK,qBAAqB;AAAA,EAC9B;AAAA,EAEA,MAAa,mBAAmB,UAAsC;AAClE,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,UAAM,QAAQ,MAAM,KAAK,QAAQ;AACjC,WAAO,KAAK,cAAc,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,CAAC;AAAA,EAC7E;AAAA,EAEO,cAAc,UAAkB,UAAkB,OAA0B;AAC/E,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,QAAI,gBAAgB,KAAK,MAAM,IAAI,QAAQ;AAC3C,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,CAAC,cAAc,IAAI,GAAG;AACtB,eAAO,cAAc,IAAI;AAAA,MAC7B;AAAA,IACJ;AACA,QAAI,eAAe;AACf,UAAI,cAAc,OAAO;AAErB,YAAI,cAAc,SAAU,WAAW,MAAO,cAAc,UAAU,OAAO;AACzE,iBAAO;AAAA,QACX;AAAA,MACJ,OAAO;AAEH,YAAI,cAAc,SAAU,WAAW,IAAK;AACxC,iBAAO;AAAA,QACX;AAAA,MACJ;AAAA,IACJ;AAQA,WAAO;AAAA,EACX;AAAA,EAEO,WAAW,UAAkB,UAAkB,OAAe,OAA2D;AAC5H,QAAI,CAAC,YAAY,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO;AAC5C,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACxC;AACA,eAAW,QAAQ,KAAK,YAAY;AAChC,UAAI,CAAC,MAAM,IAAI,GAAG;AACd,cAAM,IAAI,MAAM,gBAAgB,IAAI,EAAE;AAAA,MAC1C;AAAA,IACJ;AACA,UAAM,OAAO,WAAW;AACxB,UAAM,WAAW,KAAK,SAAS,QAAQ;AACvC,QAAI,YAAY,KAAK,MAAM,IAAI,QAAQ;AACvC,QAAI,CAAC,WAAW;AAEZ,UAAI,eAAe,CAAC;AACpB,iBAAW,QAAQ,KAAK,YAAY;AAChC,qBAAa,IAAI,IAAI,MAAM,IAAI;AAAA,MACnC;AACA,YAAM,OAAO,EAAC,MAAM,UAAU,MAAY,OAAc,GAAG,aAAY;AACvE,WAAK,MAAM,IAAI,UAAU,IAAI;AAAA,IACjC,OAAO;AAEH,iBAAW,QAAQ,KAAK,YAAY;AAChC,kBAAU,IAAI,IAAI,MAAM,IAAI;AAAA,MAChC;AAAA,IACJ;AACA,SAAK,aAAa;AAAA,EACtB;AAEJ;;;AEzRA,SAAQ,QAAAA,aAAW;AACnB,OAAOC,WAAU;AAEV,IAAM,yBAAN,MAAyD;AAAA,EAI5D,YAAY,eAA8B,aAAoC,mDAAqD,GAAE,YAAoB;AAA/G;AACtC,SAAK,eAAe,IAAI,mBAAmB,YAAW,UAAU;AAChE,SAAK,mBAAmB,IAAI,iBAAiB,eAAc,UAAU;AAAA,EACzE;AAAA,EANQ;AAAA,EACA;AAAA,EAOR,MAAa,mBAAmB,UAAkB,UAAwC;AACtF,UAAM,KAAK,iBAAiB,KAAK;AACjC,QAAI,QAAQ,MAAMD,MAAK,QAAQ;AAC/B,QAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIC,MAAK,SAAS,QAAQ,CAAC,GAAG;AAC/D,YAAM,YAAY,KAAK,iBAAiB,cAAc,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,CAAC;AACrG,UAAG,WAAW;AACV,mBAAW,QAAQ,KAAK,YAAY;AAChC,cAAI,CAAC,SAAS,IAAI,KAAK,UAAU,IAAI,GAAG;AACpC,qBAAS,IAAI,IAAI,UAAU,IAAI;AAAA,UACnC;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AACA,UAAM,KAAK,aAAa,mBAAmB,UAAU,QAAQ;AAC7D,SAAK,iBAAiB,WAAW,UAAU,MAAM,MAAM,MAAM,MAAM,YAAY,GAAG,QAAQ;AAAA,EAC9F;AAAA,EAEA,MAAM,sBAAiD;AACnD,UAAM,KAAK,iBAAiB,KAAK;AACjC,WAAO,KAAK;AAAA,EAChB;AACJ;;;ACpCA,YAAYA,WAAU;AACtB,OAAO,cAAc;AACrB,SAAQ,eAAc;AACtB,OAAO,YAAY;AAGZ,IAAM,gBAAN,MAAoB;AAAA,EAQvB,YAAY,eAA6C,mBAAmC,QAIzF;AAJsD;AAAmC;AAKxF,SAAK,gBAAgB;AAAA,EACzB;AAAA,EAbA,cAAc;AAAA,EACd,QAAQ,IAAI,OAAO,EAAC,aAAa,KAAK,WAAW,KAAI,CAAC;AAAA,EACtD;AAAA,EACA,YAAY;AAAA,EACZ,UAAU;AAAA,EACV,aAAa,oBAAI,IAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgB7B,MAAc,iBAAiB,WAAmB,UAA0B;AACxE,UAAM,UAAU,MAAM,QAAQ,WAAW,EAAC,eAAe,KAAI,CAAC;AAC9D,eAAW,SAAS,SAAS;AACzB,YAAM,WAAgB,WAAK,WAAW,MAAM,IAAI;AAChD,UAAI,MAAM,YAAY,GAAG;AACrB,cAAM,KAAK,iBAAiB,UAAU,QAAQ;AAAA,MAClD,OAAO;AACH,cAAM,UAAU,KAAK,YAAY,QAAQ;AACzC,iBAAS,KAAK,OAAO;AAAA,MACzB;AAAA,IACJ;AAAA,EACJ;AAAA,EAEA,MAAc,UAAU,WAAoC;AACxD,UAAM,UAAU,MAAM,QAAQ,WAAW,EAAC,eAAe,KAAI,CAAC;AAC9D,UAAM,gBAAgC,CAAC;AAEvC,eAAW,SAAS,SAAS;AACzB,YAAM,WAAgB,WAAK,WAAW,MAAM,IAAI;AAChD,UAAI,MAAM,YAAY,GAAG;AAErB,sBAAc,KAAK,KAAK,UAAU,QAAQ,CAAC;AAAA,MAC/C,OAAO;AAEH,sBAAc,KAAK,KAAK,MAAM;AAAA,UAC1B,YAAa,MAAM,KAAK,cAAc,eAAe,QAAQ,IAAK,IAAI;AAAA,QAC1E,CAAC;AAAA,MACL;AAAA,IACJ;AAGA,UAAM,SAAS,MAAM,QAAQ,IAAI,aAAa;AAC9C,WAAO,OAAO,OAAO,CAAC,KAAK,YAAY,MAAM,SAAS,CAAC;AAAA,EAC3D;AAAA,EAEA,MAAc,YAAY,UAAgC;AACtD,QAAG,MAAM,KAAK,cAAc,eAAe,QAAQ,GAAG;AAClD,UAAG,CAAC,KAAK,WAAW,IAAI,QAAQ,GAAG;AAC/B,aAAK,WAAW,IAAI,QAAQ;AAC5B,cAAM,UAAU,EAAE,KAAK;AACvB,cAAM,KAAK,MAAM,IAAI,MAAM,KAAK,cAAc,YAAY,SAAS,KAAK,WAAW,QAAQ,CAAC;AAC5F,aAAK,WAAW,OAAO,QAAQ;AAAA,MACnC;AAAA,IACJ;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB,UAAgC;AAC9D,QAAIC,WAAe,cAAQ,QAAQ;AACnC,UAAM,KAAK,iBAAiBA,UAAS,CAAC,CAAC;AAAA,EAC3C;AAAA,EAEQ,cAAc,YAAqB;AACvC,UAAM,iBAAiB;AAAA,MACnB,eAAe;AAAA;AAAA,MACf,YAAY;AAAA,MACZ,OAAO;AAAA,MACP,kBAAkB;AAAA,QACd,oBAAoB,KAAK,OAAO,sBAAsB;AAAA,QACtD,cAAc,KAAK,OAAO,gBAAgB;AAAA,MAC9C;AAAA,IACJ;AAEA,QAAI;AACJ,SAAK,KAAK,OAAO,YAAY,MAAM,GAAG;AAClC,gBAAU,SAAS,MAAM,YAAY,cAAc;AAAA,IACvD,OAAO;AACH,gBAAU,SAAS,MAAM,YAAY;AAAA,QACjC,GAAG;AAAA,QACH,YAAY;AAAA,QACZ,UAAU,KAAK,OAAO,YAAY;AAAA,MACtC,CAAC;AAAA,IACL;AAEA,YACK,GAAG,OAAO,OAAO,aAAa;AAC3B,UAAI;AACA,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,OAAO;AACZ,gBAAQ,MAAM,yBAAyB,QAAQ,KAAK,KAAK;AAAA,MAC7D;AAAA,IACJ,CAAC,EACA,GAAG,UAAU,OAAO,aAAa;AAC9B,UAAI;AACA,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,GAAG;AACR,gBAAQ,MAAM,yBAAyB,QAAQ,KAAK,CAAC;AAAA,MACzD;AAAA,IACJ,CAAC,EACA,GAAG,UAAU,OAAO,aAAa;AAC9B,UAAI;AACA,cAAM,KAAK,cAAc,WAAW,QAAQ;AAC5C,cAAM,KAAK,oBAAoB,QAAQ;AAAA,MAC3C,SAAS,GAAG;AACR,gBAAQ,MAAM,mCAAmC,CAAC;AAAA,MACtD;AAAA,IACJ,CAAC,EACA,GAAG,SAAS,CAAC,UAAU,QAAQ,MAAM,kBAAkB,KAAK,EAAE,CAAC,EAC/D,GAAG,SAAS,MAAM;AACf,cAAQ,IAAI,gCAAgC,UAAU,EAAE;AAAA,IAC5D,CAAC;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACV,QAAI,UAAU,KAAK;AACnB,QAAI,CAAC,SAAS;AACV,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACxC;AACA,QAAI,aAAa,QAAQ,MAAM,GAAG;AAElC,SAAK,cAAc,UAAU;AAG7B,QAAI,gBAAgB,CAAC;AACrB,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AACxC,iBAAW,CAAC,IAAS,gBAAU,WAAW,CAAC,CAAC;AAC5C,oBAAc,KAAK,KAAK,UAAU,WAAW,CAAC,CAAC,CAAC;AAAA,IACpD;AACA,UAAM,SAAS,MAAM,QAAQ,IAAI,aAAa,GAAG,OAAO,CAAC,KAAK,YAAY,MAAM,SAAS,CAAC;AAC1F,YAAQ,IAAI,SAAS,KAAK,mBAAmB;AAC7C,SAAK,YAAY;AACjB,QAAI,WAA2B,CAAC;AAChC,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AACxC,iBAAW,CAAC,IAAS,gBAAU,WAAW,CAAC,CAAC;AAC5C,YAAM,KAAK,iBAAiB,WAAW,CAAC,GAAG,QAAQ;AAAA,IACvD;AACA,UAAM,QAAQ,IAAI,QAAQ;AAC1B,SAAK,cAAc;AAEnB,YAAQ,IAAI,eAAe;AAAA,EAC/B;AACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/worker.cjs b/dist/worker.cjs index caf03ce..ba4251f 100644 --- a/dist/worker.cjs +++ b/dist/worker.cjs @@ -1,11 +1,11 @@ 'use strict'; -var chunkKZVIGNTH_cjs = require('./chunk-KZVIGNTH.cjs'); -require('./chunk-33LRP2HS.cjs'); -require('./chunk-PAKJNZBJ.cjs'); +var chunkY5FIE5OK_cjs = require('./chunk-Y5FIE5OK.cjs'); +require('./chunk-OHUFSPVO.cjs'); +require('./chunk-S7BSPELE.cjs'); -module.exports = chunkKZVIGNTH_cjs.a; +module.exports = chunkY5FIE5OK_cjs.computeCIDs; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.cjs.map \ No newline at end of file diff --git a/dist/worker.js b/dist/worker.js index 24b9409..4e842c7 100644 --- a/dist/worker.js +++ b/dist/worker.js @@ -1,5 +1,5 @@ -export { a as default } from './chunk-6UBSWOQL.js'; -import './chunk-A6J5HDTV.js'; -import './chunk-YCFZVUZC.js'; +export { computeCIDs as default } from './chunk-LTWPBXM5.js'; +import './chunk-SV45D2HN.js'; +import './chunk-HTZ763NS.js'; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/package.json b/package.json index c9b46b7..25fe615 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test": "vitest" }, "devDependencies": { + "@libp2p/crypto": "^4.0.5", "@types/node": "^20.12.2", "ts-node": "^10.9.2", "tsup": "^8.0.2", diff --git a/src/lib/file-id/ComputeHash.ts b/src/lib/file-id/ComputeHash.ts index 4e39401..39abb4d 100644 --- a/src/lib/file-id/ComputeHash.ts +++ b/src/lib/file-id/ComputeHash.ts @@ -15,13 +15,15 @@ export async function computeCIDs({stream, algorithms}: { stream: Readable | ReadableStream; algorithms: CID_ALGORITHM_NAMES[] }): Promise { - const hashers = hasherDefiner(algorithms); + const hashers = await hasherDefiner(algorithms); // Check if the environment is Node.js or Browser and handle the stream accordingly if (stream instanceof Readable) { // Node.js environment for await (const chunk of stream) { - hashers.forEach(item => item.hasher.update(chunk as Buffer)); + for (const item of hashers) { + await item.hasher.update(chunk as Buffer); + } } } else { // Browser environment @@ -30,28 +32,30 @@ export async function computeCIDs({stream, algorithms}: { const {done, value} = await reader.read(); if (done) break; // Assuming the hasher can handle Uint8Array directly - hashers.forEach(item => item.hasher.update(Buffer.from(value))); + for (const item of hashers) { + await item.hasher.update(Buffer.from(value)); + } } } return cidFinalize(hashers); } -function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): { +async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): Promise<{ hasher: SimpleHash, code: CID_ALGORITHM_CODES -}[] { +}[]> { const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo)) - .map(algo => ({ - hasher: createHasher(algo), + .map(async algo => ({ + hasher: await createHasher(algo), code: codeTable[algo] })); - return hashers; + return await Promise.all(hashers); } async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise { return await Promise.all(hashers.map(async ({code, hasher}) => { - const hashBuffer = hasher.digest(); + const hashBuffer = await hasher.digest(); const digest = create(code, hashBuffer); return CID.createV1(code, digest).toString(); })); diff --git a/src/lib/file-id/Crc32Hash.ts b/src/lib/file-id/Crc32Hash.ts index f3efdbf..77eeb81 100644 --- a/src/lib/file-id/Crc32Hash.ts +++ b/src/lib/file-id/Crc32Hash.ts @@ -4,7 +4,7 @@ import {SimpleHash} from "@root/file-id/SimpleHash"; export class Crc32Hash implements SimpleHash { private _crc: number = undefined; - update(data: Buffer): Crc32Hash { + update(data: Buffer): SimpleHash { // Update the CRC-32 checksum with the new chunk of data this._crc = crc32.buf(data, this._crc); return this; diff --git a/src/lib/file-id/CreateHasher.ts b/src/lib/file-id/CreateHasher.ts index 312c0d6..887ad19 100644 --- a/src/lib/file-id/CreateHasher.ts +++ b/src/lib/file-id/CreateHasher.ts @@ -1,6 +1,4 @@ import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; -import {Readable} from "stream"; -import {ReadableStream} from "stream/web"; import {SimpleHash} from "./SimpleHash"; import {createHash} from "crypto"; import {Crc32Hash} from "./Crc32Hash"; @@ -19,7 +17,7 @@ export const codeTable = { * @returns Array of CIDs (in the order of the algorithms) * @param algo */ -export function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash { +export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { switch (algo) { case CID_ALGORITHM_NAMES.sha256: return createHash("sha256"); diff --git a/src/lib/file-id/SimpleHash.ts b/src/lib/file-id/SimpleHash.ts index d1c4297..e33cbfd 100644 --- a/src/lib/file-id/SimpleHash.ts +++ b/src/lib/file-id/SimpleHash.ts @@ -1,5 +1,5 @@ export interface SimpleHash { - update(data: Buffer), + update(data: Buffer): SimpleHash | Promise; - digest(): Buffer; + digest(): Buffer | Promise; } \ No newline at end of file diff --git a/tsup.config.js b/tsup.config.js index 42215d6..d71281a 100644 --- a/tsup.config.js +++ b/tsup.config.js @@ -12,6 +12,6 @@ export default defineConfig({ sourcemap: true, clean: true, dts:true, - minify:true, + minify:false, format: ['cjs', 'esm'], }) diff --git a/yarn.lock b/yarn.lock index 91b3ca2..ad2d31f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,6 +5,22 @@ __metadata: version: 8 cacheKey: 10c0 +"@chainsafe/is-ip@npm:^2.0.1": + version: 2.0.2 + resolution: "@chainsafe/is-ip@npm:2.0.2" + checksum: 10c0/0bb8b9d0babe583642d31ffafad603ac5e5dc48884266feae57479d81f4e81ef903628527d81b39d5305657a957bf435bd2ef38b98a4526a7aab366febf793ad + languageName: node + linkType: hard + +"@chainsafe/netmask@npm:^2.0.0": + version: 2.0.0 + resolution: "@chainsafe/netmask@npm:2.0.0" + dependencies: + "@chainsafe/is-ip": "npm:^2.0.1" + checksum: 10c0/a9069e52b0a1470b00c88d3fb16ff4fe14274e5055770faab974b29f7bc69ebf76172775461da1e88b7fe73539a9228f6af0406253d46e987193ddf21a073da1 + languageName: node + linkType: hard + "@cspotcode/source-map-support@npm:^0.8.0": version: 0.8.1 resolution: "@cspotcode/source-map-support@npm:0.8.1" @@ -411,10 +427,48 @@ __metadata: languageName: node linkType: hard +"@leichtgewicht/ip-codec@npm:^2.0.1": + version: 2.0.5 + resolution: "@leichtgewicht/ip-codec@npm:2.0.5" + checksum: 10c0/14a0112bd59615eef9e3446fea018045720cd3da85a98f801a685a818b0d96ef2a1f7227e8d271def546b2e2a0fe91ef915ba9dc912ab7967d2317b1a051d66b + languageName: node + linkType: hard + +"@libp2p/crypto@npm:^4.0.5": + version: 4.0.5 + resolution: "@libp2p/crypto@npm:4.0.5" + dependencies: + "@libp2p/interface": "npm:^1.1.6" + "@noble/curves": "npm:^1.4.0" + "@noble/hashes": "npm:^1.4.0" + asn1js: "npm:^3.0.5" + multiformats: "npm:^13.1.0" + protons-runtime: "npm:^5.4.0" + uint8arraylist: "npm:^2.4.8" + uint8arrays: "npm:^5.0.3" + checksum: 10c0/d0abefa43da0a6bc9037c844257ff385e08d4be49bf67d4026c48e7866528b74a10a98ae1f2cb71525f2b02fa4e3b12f7b1d99977e2914a5dc0b05c08483e763 + languageName: node + linkType: hard + +"@libp2p/interface@npm:^1.0.0, @libp2p/interface@npm:^1.1.6": + version: 1.1.6 + resolution: "@libp2p/interface@npm:1.1.6" + dependencies: + "@multiformats/multiaddr": "npm:^12.2.1" + it-pushable: "npm:^3.2.3" + it-stream-types: "npm:^2.0.1" + multiformats: "npm:^13.1.0" + progress-events: "npm:^1.0.0" + uint8arraylist: "npm:^2.4.8" + checksum: 10c0/7aa5fb619e2d5d8a468f105427587962a92c1fb1167e201a92d0fd11f166617d9abf138e6fdb27076613ade790224f7d93b68c5c5e7ce61be8d1f3d5b1d79711 + languageName: node + linkType: hard + "@metazla/meta-hash@workspace:.": version: 0.0.0-use.local resolution: "@metazla/meta-hash@workspace:." dependencies: + "@libp2p/crypto": "npm:^4.0.5" "@types/node": "npm:^20.12.2" chokidar: "npm:^3.6.0" crc-32: "npm:^1.2.2" @@ -430,6 +484,52 @@ __metadata: languageName: unknown linkType: soft +"@multiformats/dns@npm:^1.0.3": + version: 1.0.6 + resolution: "@multiformats/dns@npm:1.0.6" + dependencies: + "@types/dns-packet": "npm:^5.6.5" + buffer: "npm:^6.0.3" + dns-packet: "npm:^5.6.1" + hashlru: "npm:^2.3.0" + p-queue: "npm:^8.0.1" + progress-events: "npm:^1.0.0" + uint8arrays: "npm:^5.0.2" + checksum: 10c0/ab0323ec9e697fb345a47b68e9e6ee5e2def2f00e99467ac6c53c9b6f613cff2dc2b792e9270cfe385500b6cdcc565a1c6e6e7871c584a6bc49366441bc4fa7f + languageName: node + linkType: hard + +"@multiformats/multiaddr@npm:^12.2.1": + version: 12.2.1 + resolution: "@multiformats/multiaddr@npm:12.2.1" + dependencies: + "@chainsafe/is-ip": "npm:^2.0.1" + "@chainsafe/netmask": "npm:^2.0.0" + "@libp2p/interface": "npm:^1.0.0" + "@multiformats/dns": "npm:^1.0.3" + multiformats: "npm:^13.0.0" + uint8-varint: "npm:^2.0.1" + uint8arrays: "npm:^5.0.0" + checksum: 10c0/b32026406ee3635ce3a8e919663bf2e911e139890f1c18c74e64f5043966f14d6396280f198b7aa11d0faf2c0b27f3e6db0d546167832657b01b9769a7143abc + languageName: node + linkType: hard + +"@noble/curves@npm:^1.4.0": + version: 1.4.0 + resolution: "@noble/curves@npm:1.4.0" + dependencies: + "@noble/hashes": "npm:1.4.0" + checksum: 10c0/31fbc370df91bcc5a920ca3f2ce69c8cf26dc94775a36124ed8a5a3faf0453badafd2ee4337061ffea1b43c623a90ee8b286a5a81604aaf9563bdad7ff795d18 + languageName: node + linkType: hard + +"@noble/hashes@npm:1.4.0, @noble/hashes@npm:^1.4.0": + version: 1.4.0 + resolution: "@noble/hashes@npm:1.4.0" + checksum: 10c0/8c3f005ee72e7b8f9cff756dfae1241485187254e3f743873e22073d63906863df5d4f13d441b7530ea614b7a093f0d889309f28b59850f33b66cb26a779a4a5 + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -458,15 +558,15 @@ __metadata: linkType: hard "@npmcli/agent@npm:^2.0.0": - version: 2.2.1 - resolution: "@npmcli/agent@npm:2.2.1" + version: 2.2.2 + resolution: "@npmcli/agent@npm:2.2.2" dependencies: agent-base: "npm:^7.1.0" http-proxy-agent: "npm:^7.0.0" https-proxy-agent: "npm:^7.0.1" lru-cache: "npm:^10.0.1" - socks-proxy-agent: "npm:^8.0.1" - checksum: 10c0/38ee5cbe8f3cde13be916e717bfc54fd1a7605c07af056369ff894e244c221e0b56b08ca5213457477f9bc15bca9e729d51a4788829b5c3cf296b3c996147f76 + socks-proxy-agent: "npm:^8.0.3" + checksum: 10c0/325e0db7b287d4154ecd164c0815c08007abfb07653cc57bceded17bb7fd240998a3cbdbe87d700e30bef494885eccc725ab73b668020811d56623d145b524ae languageName: node linkType: hard @@ -486,107 +586,107 @@ __metadata: languageName: node linkType: hard -"@rollup/rollup-android-arm-eabi@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-android-arm-eabi@npm:4.13.2" +"@rollup/rollup-android-arm-eabi@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-android-arm-eabi@npm:4.14.2" conditions: os=android & cpu=arm languageName: node linkType: hard -"@rollup/rollup-android-arm64@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-android-arm64@npm:4.13.2" +"@rollup/rollup-android-arm64@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-android-arm64@npm:4.14.2" conditions: os=android & cpu=arm64 languageName: node linkType: hard -"@rollup/rollup-darwin-arm64@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-darwin-arm64@npm:4.13.2" +"@rollup/rollup-darwin-arm64@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-darwin-arm64@npm:4.14.2" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@rollup/rollup-darwin-x64@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-darwin-x64@npm:4.13.2" +"@rollup/rollup-darwin-x64@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-darwin-x64@npm:4.14.2" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@rollup/rollup-linux-arm-gnueabihf@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-arm-gnueabihf@npm:4.13.2" +"@rollup/rollup-linux-arm-gnueabihf@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-arm-gnueabihf@npm:4.14.2" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@rollup/rollup-linux-arm64-gnu@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-arm64-gnu@npm:4.13.2" +"@rollup/rollup-linux-arm64-gnu@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-arm64-gnu@npm:4.14.2" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@rollup/rollup-linux-arm64-musl@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-arm64-musl@npm:4.13.2" +"@rollup/rollup-linux-arm64-musl@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-arm64-musl@npm:4.14.2" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@rollup/rollup-linux-powerpc64le-gnu@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-powerpc64le-gnu@npm:4.13.2" - conditions: os=linux & cpu=ppc64le & libc=glibc +"@rollup/rollup-linux-powerpc64le-gnu@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-powerpc64le-gnu@npm:4.14.2" + conditions: os=linux & cpu=ppc64 & libc=glibc languageName: node linkType: hard -"@rollup/rollup-linux-riscv64-gnu@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-riscv64-gnu@npm:4.13.2" +"@rollup/rollup-linux-riscv64-gnu@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-riscv64-gnu@npm:4.14.2" conditions: os=linux & cpu=riscv64 & libc=glibc languageName: node linkType: hard -"@rollup/rollup-linux-s390x-gnu@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-s390x-gnu@npm:4.13.2" +"@rollup/rollup-linux-s390x-gnu@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-s390x-gnu@npm:4.14.2" conditions: os=linux & cpu=s390x & libc=glibc languageName: node linkType: hard -"@rollup/rollup-linux-x64-gnu@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-x64-gnu@npm:4.13.2" +"@rollup/rollup-linux-x64-gnu@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-x64-gnu@npm:4.14.2" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@rollup/rollup-linux-x64-musl@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-linux-x64-musl@npm:4.13.2" +"@rollup/rollup-linux-x64-musl@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-linux-x64-musl@npm:4.14.2" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@rollup/rollup-win32-arm64-msvc@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-win32-arm64-msvc@npm:4.13.2" +"@rollup/rollup-win32-arm64-msvc@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-win32-arm64-msvc@npm:4.14.2" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@rollup/rollup-win32-ia32-msvc@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-win32-ia32-msvc@npm:4.13.2" +"@rollup/rollup-win32-ia32-msvc@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-win32-ia32-msvc@npm:4.14.2" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@rollup/rollup-win32-x64-msvc@npm:4.13.2": - version: 4.13.2 - resolution: "@rollup/rollup-win32-x64-msvc@npm:4.13.2" +"@rollup/rollup-win32-x64-msvc@npm:4.14.2": + version: 4.14.2 + resolution: "@rollup/rollup-win32-x64-msvc@npm:4.14.2" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -626,6 +726,15 @@ __metadata: languageName: node linkType: hard +"@types/dns-packet@npm:^5.6.5": + version: 5.6.5 + resolution: "@types/dns-packet@npm:5.6.5" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/70fa9cb77a614f65288a48749d28cc9f40ce6c60980e2b75b25eac0b4e1e4109d14edf5151fa5e7b10aae1ec6b1094a2f171b9941191ff4c9a7afe23116b9edc + languageName: node + linkType: hard + "@types/estree@npm:1.0.5, @types/estree@npm:^1.0.0": version: 1.0.5 resolution: "@types/estree@npm:1.0.5" @@ -633,66 +742,66 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^20.12.2": - version: 20.12.2 - resolution: "@types/node@npm:20.12.2" +"@types/node@npm:*, @types/node@npm:^20.12.2": + version: 20.12.7 + resolution: "@types/node@npm:20.12.7" dependencies: undici-types: "npm:~5.26.4" - checksum: 10c0/bb47d115a8f95aadebdd6403fee16d149d6b5568ad53bacc42df34c2d572561e36ba47b05f7f68bada43752d7369b470a90df65b646af7502dfb799a21b4fca7 + checksum: 10c0/dce80d63a3b91892b321af823d624995c61e39c6a223cc0ac481a44d337640cc46931d33efb3beeed75f5c85c3bda1d97cef4c5cd4ec333caf5dee59cff6eca0 languageName: node linkType: hard -"@vitest/expect@npm:1.4.0": - version: 1.4.0 - resolution: "@vitest/expect@npm:1.4.0" +"@vitest/expect@npm:1.5.0": + version: 1.5.0 + resolution: "@vitest/expect@npm:1.5.0" dependencies: - "@vitest/spy": "npm:1.4.0" - "@vitest/utils": "npm:1.4.0" + "@vitest/spy": "npm:1.5.0" + "@vitest/utils": "npm:1.5.0" chai: "npm:^4.3.10" - checksum: 10c0/2d6a657afc674adb78ad6609ecf61a94355b080cf90f922e05193b5b33b37d486c9b66a52270f1f367c16d626bcb8323368519dae096a992190898e03280b5e0 + checksum: 10c0/12138caf0831a9bcdf475750fdff27588f03f11ee5101124f9d39951bbcdec39aa6a19a1a30f3ab0ca17374ddfc7399b861e2fcb1548bba302943e8e88512a9c languageName: node linkType: hard -"@vitest/runner@npm:1.4.0": - version: 1.4.0 - resolution: "@vitest/runner@npm:1.4.0" +"@vitest/runner@npm:1.5.0": + version: 1.5.0 + resolution: "@vitest/runner@npm:1.5.0" dependencies: - "@vitest/utils": "npm:1.4.0" + "@vitest/utils": "npm:1.5.0" p-limit: "npm:^5.0.0" pathe: "npm:^1.1.1" - checksum: 10c0/87a5bdde5c48e3258ecd2716994da20c8eec63acaf63a0db724513a42701bc644728009a7301d78b8775d8004c7ce1ddb8bde6495066d864c532bc117783aa91 + checksum: 10c0/a7b693e4121b6159a77f6cbffc03c228cd18c3734e27a968fe26579777f6d7a4d55a02fba396d564df996a91881ee76de80ee5a526d27a4b508f44a9af62d95a languageName: node linkType: hard -"@vitest/snapshot@npm:1.4.0": - version: 1.4.0 - resolution: "@vitest/snapshot@npm:1.4.0" +"@vitest/snapshot@npm:1.5.0": + version: 1.5.0 + resolution: "@vitest/snapshot@npm:1.5.0" dependencies: magic-string: "npm:^0.30.5" pathe: "npm:^1.1.1" pretty-format: "npm:^29.7.0" - checksum: 10c0/6f089d1dbe43556779479bc309b0a8fc7e0229843c40efb4dabcf99ccf9a6fa859dd38c13674616a955801442730aca55151cbd52bb22d41d9a335060e03759b + checksum: 10c0/d897070b3a7c008eb58d44ca0dc324c69dfc1f787335e59fb930ae2e8d02a148e6b3c13497a588083a0654bdc430281fdd84276ead4888230314a2984dd9f93d languageName: node linkType: hard -"@vitest/spy@npm:1.4.0": - version: 1.4.0 - resolution: "@vitest/spy@npm:1.4.0" +"@vitest/spy@npm:1.5.0": + version: 1.5.0 + resolution: "@vitest/spy@npm:1.5.0" dependencies: tinyspy: "npm:^2.2.0" - checksum: 10c0/847bc3085d0aa2e039aa42d803cf2dc94596aab3a63de7d364933d24ed9e0781b7d3d4bd222df202f92bae83e9c37b2893b9f24a2de7d83b6930b7b1acf43516 + checksum: 10c0/0206f1e8431f543474dc6d252b553227d0f286b27226b987b63babb18865e8320e6c3d822c67782aae68555728a57ecdeb7c4e16283dbad49791608a691f26d1 languageName: node linkType: hard -"@vitest/utils@npm:1.4.0": - version: 1.4.0 - resolution: "@vitest/utils@npm:1.4.0" +"@vitest/utils@npm:1.5.0": + version: 1.5.0 + resolution: "@vitest/utils@npm:1.5.0" dependencies: diff-sequences: "npm:^29.6.3" estree-walker: "npm:^3.0.3" loupe: "npm:^2.3.7" pretty-format: "npm:^29.7.0" - checksum: 10c0/cfa352484f0ea2614444a94fc35979bea94fac64e9756238c685ae74bcd027893a1798b9d6d92c1cdd454b1f7f08f453d0cca108274f0449b6f5efd345822a4c + checksum: 10c0/b9d779ea5c1a5759df4f59e3ba3f3a94816d6c600afe7a2d14b963ea114ce1acedbfe678cbfacb0a20d33cabcce890a08b2ce3fd52e3465f2e0969cc39f7686b languageName: node linkType: hard @@ -806,6 +915,17 @@ __metadata: languageName: node linkType: hard +"asn1js@npm:^3.0.5": + version: 3.0.5 + resolution: "asn1js@npm:3.0.5" + dependencies: + pvtsutils: "npm:^1.3.2" + pvutils: "npm:^1.1.3" + tslib: "npm:^2.4.0" + checksum: 10c0/bb8eaf4040c8f49dd475566874986f5976b81bae65a6b5526e2208a13cdca323e69ce297bcd435fdda3eb6933defe888e71974d705b6fcb14f2734a907f8aed4 + languageName: node + linkType: hard + "assertion-error@npm:^1.1.0": version: 1.1.0 resolution: "assertion-error@npm:1.1.0" @@ -820,6 +940,13 @@ __metadata: languageName: node linkType: hard +"base64-js@npm:^1.3.1": + version: 1.5.1 + resolution: "base64-js@npm:1.5.1" + checksum: 10c0/f23823513b63173a001030fae4f2dabe283b99a9d324ade3ad3d148e218134676f1ee8568c877cd79ec1c53158dcf2d2ba527a97c606618928ba99dd930102bf + languageName: node + linkType: hard + "binary-extensions@npm:^2.0.0": version: 2.3.0 resolution: "binary-extensions@npm:2.3.0" @@ -845,6 +972,16 @@ __metadata: languageName: node linkType: hard +"buffer@npm:^6.0.3": + version: 6.0.3 + resolution: "buffer@npm:6.0.3" + dependencies: + base64-js: "npm:^1.3.1" + ieee754: "npm:^1.2.1" + checksum: 10c0/2a905fbbcde73cc5d8bd18d1caa23715d5f83a5935867c2329f0ac06104204ba7947be098fe1317fbd8830e26090ff8e764f08cd14fefc977bb248c3487bcbd0 + languageName: node + linkType: hard + "bundle-require@npm:^4.0.0": version: 4.0.2 resolution: "bundle-require@npm:4.0.2" @@ -1048,6 +1185,15 @@ __metadata: languageName: node linkType: hard +"dns-packet@npm:^5.6.1": + version: 5.6.1 + resolution: "dns-packet@npm:5.6.1" + dependencies: + "@leichtgewicht/ip-codec": "npm:^2.0.1" + checksum: 10c0/8948d3d03063fb68e04a1e386875f8c3bcc398fc375f535f2b438fad8f41bf1afa6f5e70893ba44f4ae884c089247e0a31045722fa6ff0f01d228da103f1811d + languageName: node + linkType: hard + "eastasianwidth@npm:^0.2.0": version: 0.2.0 resolution: "eastasianwidth@npm:0.2.0" @@ -1453,6 +1599,13 @@ __metadata: languageName: node linkType: hard +"hashlru@npm:^2.3.0": + version: 2.3.0 + resolution: "hashlru@npm:2.3.0" + checksum: 10c0/90f351db1a320c43aeb681c7e806f35af6877a86d3fa9b970636ea21f111d0f0b819e046dc9b5fe6a8ab211a3d98178eb37aacbad29ea4f6f53554a4541a6f0d + languageName: node + linkType: hard + "http-cache-semantics@npm:^4.1.1": version: 4.1.1 resolution: "http-cache-semantics@npm:4.1.1" @@ -1503,6 +1656,13 @@ __metadata: languageName: node linkType: hard +"ieee754@npm:^1.2.1": + version: 1.2.1 + resolution: "ieee754@npm:1.2.1" + checksum: 10c0/b0782ef5e0935b9f12883a2e2aa37baa75da6e66ce6515c168697b42160807d9330de9a32ec1ed73149aea02e0d822e572bca6f1e22bdcbd2149e13b050b17bb + languageName: node + linkType: hard + "ignore@npm:^5.2.0": version: 5.3.1 resolution: "ignore@npm:5.3.1" @@ -1608,6 +1768,22 @@ __metadata: languageName: node linkType: hard +"it-pushable@npm:^3.2.3": + version: 3.2.3 + resolution: "it-pushable@npm:3.2.3" + dependencies: + p-defer: "npm:^4.0.0" + checksum: 10c0/ba99744f0a6fe9e555bdde76ce2144a3e35c37a33830c69d65f5512129fb5a58272f1bbb4b9741fca69be868c301109fc910b26629c8eb0961f0503dd80a4830 + languageName: node + linkType: hard + +"it-stream-types@npm:^2.0.1": + version: 2.0.1 + resolution: "it-stream-types@npm:2.0.1" + checksum: 10c0/f76119277679807195a5852fbf3e8e2245e0ed0e8c6a68174049ca521119fe0933fe8fb60c3f7883558e46b3ef8f7ae0852ae8d397f0474858415ed366090174 + languageName: node + linkType: hard + "jackspeak@npm:^2.3.6": version: 2.3.6 resolution: "jackspeak@npm:2.3.6" @@ -1713,11 +1889,11 @@ __metadata: linkType: hard "magic-string@npm:^0.30.5": - version: 0.30.8 - resolution: "magic-string@npm:0.30.8" + version: 0.30.9 + resolution: "magic-string@npm:0.30.9" dependencies: "@jridgewell/sourcemap-codec": "npm:^1.4.15" - checksum: 10c0/51a1f06f678c082aceddfb5943de9b6bdb88f2ea1385a1c2adf116deb73dfcfa50df6c222901d691b529455222d4d68d0b28be5689ac6f69b3baa3462861f922 + checksum: 10c0/edbeea35b4f90b58815d8b13899fa412b5bc1e81cae14fe6d24d5c383c5f04331fce2c5a75bfb7926203ab6fc8c71290cdab56703a5b82432d8a1e144d6042e1 languageName: node linkType: hard @@ -1906,7 +2082,7 @@ __metadata: languageName: node linkType: hard -"multiformats@npm:^13.1.0": +"multiformats@npm:^13.0.0, multiformats@npm:^13.1.0": version: 13.1.0 resolution: "multiformats@npm:13.1.0" checksum: 10c0/b04e51a443a6b5c0b9ca50aec8ef137bb1f46ca28a3827ca96336e7d7cb2a3367cfafcccd618b0c4147ceed674498635fb1ce115ab94e8b33c763b8168f0ab78 @@ -2052,6 +2228,13 @@ __metadata: languageName: node linkType: hard +"p-defer@npm:^4.0.0": + version: 4.0.1 + resolution: "p-defer@npm:4.0.1" + checksum: 10c0/592f5bd32f8c6a57f892b00976e5272b3bbbd792b503f4cf3bc22094d08d7a973413c59c15deccff4759d860b38467a08b5b3363e865da6f00f44a031777118c + languageName: node + linkType: hard + "p-limit@npm:^5.0.0": version: 5.0.0 resolution: "p-limit@npm:5.0.0" @@ -2223,6 +2406,13 @@ __metadata: languageName: node linkType: hard +"progress-events@npm:^1.0.0": + version: 1.0.0 + resolution: "progress-events@npm:1.0.0" + checksum: 10c0/b634e9c0796bf2d7f9ccda469029cbb456c1cf3d18e6ab1c2ad6a6a6b9163b2669112f3217a6490d2ac039cae82486b4806e05db246776cb644d3bbc0e060805 + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -2233,6 +2423,17 @@ __metadata: languageName: node linkType: hard +"protons-runtime@npm:^5.4.0": + version: 5.4.0 + resolution: "protons-runtime@npm:5.4.0" + dependencies: + uint8-varint: "npm:^2.0.2" + uint8arraylist: "npm:^2.4.3" + uint8arrays: "npm:^5.0.1" + checksum: 10c0/5a3c462aa6a637971b67f3ec3f82bd7857996fb9c5feb34cba608a102410235f484ad0f19cc8eec76524f1c112a1964c5132683de0d5edcc2f0d6cab4dcd0e36 + languageName: node + linkType: hard + "punycode@npm:^2.1.0": version: 2.3.1 resolution: "punycode@npm:2.3.1" @@ -2240,6 +2441,22 @@ __metadata: languageName: node linkType: hard +"pvtsutils@npm:^1.3.2": + version: 1.3.5 + resolution: "pvtsutils@npm:1.3.5" + dependencies: + tslib: "npm:^2.6.1" + checksum: 10c0/d425aed316907e0b447a459bfb97c55d22270c3cfdba5a07ec90da0737b0e40f4f1771a444636f85bb6a453de90ff8c6b5f4f6ddba7597977166af49974b4534 + languageName: node + linkType: hard + +"pvutils@npm:^1.1.3": + version: 1.1.3 + resolution: "pvutils@npm:1.1.3" + checksum: 10c0/23489e6b3c76b6afb6964a20f891d6bef092939f401c78bba186b2bfcdc7a13904a0af0a78f7933346510f8c1228d5ab02d3c80e968fd84d3c76ff98d8ec9aac + languageName: node + linkType: hard + "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -2285,24 +2502,24 @@ __metadata: linkType: hard "rollup@npm:^4.0.2, rollup@npm:^4.13.0": - version: 4.13.2 - resolution: "rollup@npm:4.13.2" + version: 4.14.2 + resolution: "rollup@npm:4.14.2" dependencies: - "@rollup/rollup-android-arm-eabi": "npm:4.13.2" - "@rollup/rollup-android-arm64": "npm:4.13.2" - "@rollup/rollup-darwin-arm64": "npm:4.13.2" - "@rollup/rollup-darwin-x64": "npm:4.13.2" - "@rollup/rollup-linux-arm-gnueabihf": "npm:4.13.2" - "@rollup/rollup-linux-arm64-gnu": "npm:4.13.2" - "@rollup/rollup-linux-arm64-musl": "npm:4.13.2" - "@rollup/rollup-linux-powerpc64le-gnu": "npm:4.13.2" - "@rollup/rollup-linux-riscv64-gnu": "npm:4.13.2" - "@rollup/rollup-linux-s390x-gnu": "npm:4.13.2" - "@rollup/rollup-linux-x64-gnu": "npm:4.13.2" - "@rollup/rollup-linux-x64-musl": "npm:4.13.2" - "@rollup/rollup-win32-arm64-msvc": "npm:4.13.2" - "@rollup/rollup-win32-ia32-msvc": "npm:4.13.2" - "@rollup/rollup-win32-x64-msvc": "npm:4.13.2" + "@rollup/rollup-android-arm-eabi": "npm:4.14.2" + "@rollup/rollup-android-arm64": "npm:4.14.2" + "@rollup/rollup-darwin-arm64": "npm:4.14.2" + "@rollup/rollup-darwin-x64": "npm:4.14.2" + "@rollup/rollup-linux-arm-gnueabihf": "npm:4.14.2" + "@rollup/rollup-linux-arm64-gnu": "npm:4.14.2" + "@rollup/rollup-linux-arm64-musl": "npm:4.14.2" + "@rollup/rollup-linux-powerpc64le-gnu": "npm:4.14.2" + "@rollup/rollup-linux-riscv64-gnu": "npm:4.14.2" + "@rollup/rollup-linux-s390x-gnu": "npm:4.14.2" + "@rollup/rollup-linux-x64-gnu": "npm:4.14.2" + "@rollup/rollup-linux-x64-musl": "npm:4.14.2" + "@rollup/rollup-win32-arm64-msvc": "npm:4.14.2" + "@rollup/rollup-win32-ia32-msvc": "npm:4.14.2" + "@rollup/rollup-win32-x64-msvc": "npm:4.14.2" "@types/estree": "npm:1.0.5" fsevents: "npm:~2.3.2" dependenciesMeta: @@ -2340,7 +2557,7 @@ __metadata: optional: true bin: rollup: dist/bin/rollup - checksum: 10c0/bde1fa8893a27422b3f9b34857b75a1ee383ccd89d13d539658909f2204e2c465a7b441e53027c40564d288e8fd2c6608e5e756d259ce28dec13e69c68a43479 + checksum: 10c0/3826aaa847a16e8ac0459e20e3ec624295110c0aac2c7b4f970f75ca804d7437bbd002a37079410f71137691aab64649e65361018647b0412911358a17b97902 languageName: node linkType: hard @@ -2422,7 +2639,7 @@ __metadata: languageName: node linkType: hard -"socks-proxy-agent@npm:^8.0.1": +"socks-proxy-agent@npm:^8.0.3": version: 8.0.3 resolution: "socks-proxy-agent@npm:8.0.3" dependencies: @@ -2434,12 +2651,12 @@ __metadata: linkType: hard "socks@npm:^2.7.1": - version: 2.8.1 - resolution: "socks@npm:2.8.1" + version: 2.8.3 + resolution: "socks@npm:2.8.3" dependencies: ip-address: "npm:^9.0.5" smart-buffer: "npm:^4.2.0" - checksum: 10c0/ac77b515c260473cc7c4452f09b20939e22510ce3ae48385c516d1d5784374d5cc75be3cb18ff66cc985a7f4f2ef8fef84e984c5ec70aad58355ed59241f40a8 + checksum: 10c0/d54a52bf9325165770b674a67241143a3d8b4e4c8884560c4e0e078aace2a728dffc7f70150660f51b85797c4e1a3b82f9b7aa25e0a0ceae1a243365da5c51a7 languageName: node linkType: hard @@ -2609,7 +2826,7 @@ __metadata: languageName: node linkType: hard -"tinypool@npm:^0.8.2": +"tinypool@npm:^0.8.3": version: 0.8.3 resolution: "tinypool@npm:0.8.3" checksum: 10c0/c219d0cfb69de8e3cf17403034a508d773f2fccaad79a13cdbad68600c4fb10186ad814d2320bcaa8f6e774fff5666d2a3d3b241dc8a7ad9d970ee63fe620a32 @@ -2695,6 +2912,13 @@ __metadata: languageName: node linkType: hard +"tslib@npm:^2.4.0, tslib@npm:^2.6.1": + version: 2.6.2 + resolution: "tslib@npm:2.6.2" + checksum: 10c0/e03a8a4271152c8b26604ed45535954c0a45296e32445b4b87f8a5abdb2421f40b59b4ca437c4346af0f28179780d604094eb64546bee2019d903d01c6c19bdb + languageName: node + linkType: hard + "tsup@npm:^8.0.2": version: 8.0.2 resolution: "tsup@npm:8.0.2" @@ -2742,22 +2966,22 @@ __metadata: linkType: hard "typescript@npm:^5.4.3": - version: 5.4.3 - resolution: "typescript@npm:5.4.3" + version: 5.4.5 + resolution: "typescript@npm:5.4.5" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: 10c0/22443a8760c3668e256c0b34b6b45c359ef6cecc10c42558806177a7d500ab1a7d7aac1f976d712e26989ddf6731d2fbdd3212b7c73290a45127c1c43ba2005a + checksum: 10c0/2954022ada340fd3d6a9e2b8e534f65d57c92d5f3989a263754a78aba549f7e6529acc1921913560a4b816c46dce7df4a4d29f9f11a3dc0d4213bb76d043251e languageName: node linkType: hard "typescript@patch:typescript@npm%3A^5.4.3#optional!builtin": - version: 5.4.3 - resolution: "typescript@patch:typescript@npm%3A5.4.3#optional!builtin::version=5.4.3&hash=5adc0c" + version: 5.4.5 + resolution: "typescript@patch:typescript@npm%3A5.4.5#optional!builtin::version=5.4.5&hash=5adc0c" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: 10c0/6e51f8b7e6ec55b897b9e56b67e864fe8f44e30f4a14357aad5dc0f7432db2f01efc0522df0b6c36d361c51f2dc3dcac5c832efd96a404cfabf884e915d38828 + checksum: 10c0/db2ad2a16ca829f50427eeb1da155e7a45e598eec7b086d8b4e8ba44e5a235f758e606d681c66992230d3fc3b8995865e5fd0b22a2c95486d0b3200f83072ec9 languageName: node linkType: hard @@ -2768,6 +2992,34 @@ __metadata: languageName: node linkType: hard +"uint8-varint@npm:^2.0.1, uint8-varint@npm:^2.0.2": + version: 2.0.4 + resolution: "uint8-varint@npm:2.0.4" + dependencies: + uint8arraylist: "npm:^2.0.0" + uint8arrays: "npm:^5.0.0" + checksum: 10c0/850bce72c2b639d317db6af2c30544f7f8c6451fa328d674aca74d7e263a9510acd3af555ed04ffbc20b9ff68fd5d2e06b91a5c9ffde781cf6ee6233606f34dc + languageName: node + linkType: hard + +"uint8arraylist@npm:^2.0.0, uint8arraylist@npm:^2.4.3, uint8arraylist@npm:^2.4.8": + version: 2.4.8 + resolution: "uint8arraylist@npm:2.4.8" + dependencies: + uint8arrays: "npm:^5.0.1" + checksum: 10c0/a997289ad66d4ffff24d85524b70af9fb6914ef1657400907a792afbff4c4600e7dde7ab3de83da8309c7b22c093c93bc8a95b3d288eeac4c935133bfb925bac + languageName: node + linkType: hard + +"uint8arrays@npm:^5.0.0, uint8arrays@npm:^5.0.1, uint8arrays@npm:^5.0.2, uint8arrays@npm:^5.0.3": + version: 5.0.3 + resolution: "uint8arrays@npm:5.0.3" + dependencies: + multiformats: "npm:^13.0.0" + checksum: 10c0/2c0d0bbb6eff6912c32d5ffb8f35fcc2fa3643b90bde06848222889aaaf869e973e684c8731cc2797b4117121c86cda83063c1363927d6bb882965e355785058 + languageName: node + linkType: hard + "undici-types@npm:~5.26.4": version: 5.26.5 resolution: "undici-types@npm:5.26.5" @@ -2800,9 +3052,9 @@ __metadata: languageName: node linkType: hard -"vite-node@npm:1.4.0": - version: 1.4.0 - resolution: "vite-node@npm:1.4.0" +"vite-node@npm:1.5.0": + version: 1.5.0 + resolution: "vite-node@npm:1.5.0" dependencies: cac: "npm:^6.7.14" debug: "npm:^4.3.4" @@ -2811,13 +3063,13 @@ __metadata: vite: "npm:^5.0.0" bin: vite-node: vite-node.mjs - checksum: 10c0/bc8eb01dd03c2cc306be2bf35efe789d6a3e8ca1d89d635d3154a9af0213f7609c94ef849f30a01f04535b31e729aee49468275e267693a42c32845fbd2a6721 + checksum: 10c0/85f9e8616f3612c71193a2bbdfb00f610685108a65eaa7ac535655ca1bd65dda5436ee577465436f715966f34e4b25b54140bbd584fe2c95ea0726acf4ecd351 languageName: node linkType: hard "vite@npm:^5.0.0": - version: 5.2.7 - resolution: "vite@npm:5.2.7" + version: 5.2.8 + resolution: "vite@npm:5.2.8" dependencies: esbuild: "npm:^0.20.1" fsevents: "npm:~2.3.3" @@ -2851,19 +3103,19 @@ __metadata: optional: true bin: vite: bin/vite.js - checksum: 10c0/ca927a8df388f75df194d5a5ba2be4ee46dc1d99d5be277f13c6d1ed4a4df833cc953741ef8e984061ea38b531df84e15e2a9f5deea1626317bcbec63c8ca01c + checksum: 10c0/b5717bb00c2570c08ff6d8ed917655e79184efcafa9dd62d52eea19c5d6dfc5a708ec3de9ebc670a7165fc5d401c2bdf1563bb39e2748d8e51e1593d286a9a13 languageName: node linkType: hard "vitest@npm:^1.4.0": - version: 1.4.0 - resolution: "vitest@npm:1.4.0" + version: 1.5.0 + resolution: "vitest@npm:1.5.0" dependencies: - "@vitest/expect": "npm:1.4.0" - "@vitest/runner": "npm:1.4.0" - "@vitest/snapshot": "npm:1.4.0" - "@vitest/spy": "npm:1.4.0" - "@vitest/utils": "npm:1.4.0" + "@vitest/expect": "npm:1.5.0" + "@vitest/runner": "npm:1.5.0" + "@vitest/snapshot": "npm:1.5.0" + "@vitest/spy": "npm:1.5.0" + "@vitest/utils": "npm:1.5.0" acorn-walk: "npm:^8.3.2" chai: "npm:^4.3.10" debug: "npm:^4.3.4" @@ -2875,15 +3127,15 @@ __metadata: std-env: "npm:^3.5.0" strip-literal: "npm:^2.0.0" tinybench: "npm:^2.5.1" - tinypool: "npm:^0.8.2" + tinypool: "npm:^0.8.3" vite: "npm:^5.0.0" - vite-node: "npm:1.4.0" + vite-node: "npm:1.5.0" why-is-node-running: "npm:^2.2.2" peerDependencies: "@edge-runtime/vm": "*" "@types/node": ^18.0.0 || >=20.0.0 - "@vitest/browser": 1.4.0 - "@vitest/ui": 1.4.0 + "@vitest/browser": 1.5.0 + "@vitest/ui": 1.5.0 happy-dom: "*" jsdom: "*" peerDependenciesMeta: @@ -2901,7 +3153,7 @@ __metadata: optional: true bin: vitest: vitest.mjs - checksum: 10c0/732ce229341f6777350d36020dc00ccf5dd2ac0da39424cf5c9f6f4116ed1b6f7bb56de5a11270c693214d817b6d121d3d326e8f5a73437ec3f4c65aa07e1f52 + checksum: 10c0/87f6666cccd52678a0e1dbf32e7492e4003580a168114221e4c2d404d53097d886a8be4c5c379e347d0e749affa0ea5416e4e3a0356aeca228a77ee54891ca3c languageName: node linkType: hard