From 6123e10c07dc40a59b4a27977905b2fac1b07808 Mon Sep 17 00:00:00 2001 From: Metazla Date: Mon, 8 Apr 2024 16:13:08 +0200 Subject: [PATCH] add browser dist --- ...btT.d.cts => MultiHashData-B-scpa5G.d.cts} | 17 +---- ...GUbtT.d.ts => MultiHashData-B-scpa5G.d.ts} | 17 +---- dist/chunk-33LRP2HS.cjs | 18 +++++ dist/chunk-33LRP2HS.cjs.map | 1 + dist/chunk-4EZ47WAR.js | 11 --- dist/chunk-4EZ47WAR.js.map | 1 - dist/chunk-6UBSWOQL.js | 8 +++ dist/chunk-6UBSWOQL.js.map | 1 + dist/chunk-A6J5HDTV.js | 12 ++++ dist/chunk-A6J5HDTV.js.map | 1 + dist/chunk-ASEKTFOG.cjs | 20 ------ dist/chunk-ASEKTFOG.cjs.map | 1 - dist/chunk-KWYLOVWT.cjs | 4 ++ dist/chunk-KWYLOVWT.cjs.map | 1 + dist/chunk-KZVIGNTH.cjs | 14 ++++ dist/chunk-KZVIGNTH.cjs.map | 1 + dist/chunk-PAKJNZBJ.cjs | 9 +++ dist/chunk-PAKJNZBJ.cjs.map | 1 + dist/chunk-XCJZALP2.js | 3 + dist/chunk-XCJZALP2.js.map | 1 + dist/chunk-YCFZVUZC.js | 5 ++ dist/chunk-YCFZVUZC.js.map | 1 + dist/index-browser.cjs | 27 +++++++ dist/index-browser.cjs.map | 1 + dist/index-browser.d.cts | 23 ++++++ dist/index-browser.d.ts | 23 ++++++ dist/index-browser.js | 10 +++ dist/index-browser.js.map | 1 + dist/index-interface.cjs | 21 ++++++ dist/index-interface.cjs.map | 1 + dist/index-interface.d.cts | 15 ++++ dist/index-interface.d.ts | 15 ++++ dist/index-interface.js | 4 ++ dist/index-interface.js.map | 1 + dist/index.cjs | 20 +++--- dist/index.cjs.map | 2 +- dist/index.d.cts | 50 ++++++------- dist/index.d.ts | 50 ++++++------- dist/index.js | 11 +-- dist/index.js.map | 2 +- dist/worker.cjs | 6 +- dist/worker.d.cts | 15 +++- dist/worker.d.ts | 15 +++- dist/worker.js | 4 +- package.json | 12 ++-- src/lib/Properties.ts | 5 -- src/lib/file-id/ComputeHash.ts | 58 +++++++++++++++ src/lib/file-id/Crc32Hash.ts | 18 +++++ src/lib/file-id/CreateHasher.ts | 39 ++++++++++ src/lib/file-id/FileIDComputerInterface.ts | 6 ++ .../FileIDComputerWorker.ts} | 5 +- src/lib/file-id/ShaComputeWorker.ts | 17 +++++ src/lib/file-id/SimpleHash.ts | 5 ++ src/lib/fileID/ShaComputeWorker.ts | 71 ------------------- .../FileProcessorInterface.ts | 0 .../FolderWatcher.ts | 0 src/lib/hash-compute/HashComputer.ts | 29 ++++++++ .../HashComputerIndexCache.ts} | 8 +-- .../HashComputerWorker.ts} | 8 +-- .../{ => hash-compute}/HashIndexManager.ts | 2 +- src/lib/{ => hash-compute}/MultiHashData.ts | 0 src/lib/index-browser.ts | 4 ++ src/lib/index-interface.ts | 3 + src/lib/index.ts | 25 ++++--- src/lib/{ => utils}/ExistsAsync.ts | 0 src/tests/main.test.ts | 8 +-- tsup.config.js | 4 +- 67 files changed, 538 insertions(+), 254 deletions(-) rename dist/{ShaComputeWorker-CNdGUbtT.d.cts => MultiHashData-B-scpa5G.d.cts} (65%) rename dist/{ShaComputeWorker-CNdGUbtT.d.ts => MultiHashData-B-scpa5G.d.ts} (65%) create mode 100644 dist/chunk-33LRP2HS.cjs create mode 100644 dist/chunk-33LRP2HS.cjs.map delete mode 100644 dist/chunk-4EZ47WAR.js delete mode 100644 dist/chunk-4EZ47WAR.js.map create mode 100644 dist/chunk-6UBSWOQL.js create mode 100644 dist/chunk-6UBSWOQL.js.map create mode 100644 dist/chunk-A6J5HDTV.js create mode 100644 dist/chunk-A6J5HDTV.js.map delete mode 100644 dist/chunk-ASEKTFOG.cjs delete mode 100644 dist/chunk-ASEKTFOG.cjs.map create mode 100644 dist/chunk-KWYLOVWT.cjs create mode 100644 dist/chunk-KWYLOVWT.cjs.map create mode 100644 dist/chunk-KZVIGNTH.cjs create mode 100644 dist/chunk-KZVIGNTH.cjs.map create mode 100644 dist/chunk-PAKJNZBJ.cjs create mode 100644 dist/chunk-PAKJNZBJ.cjs.map create mode 100644 dist/chunk-XCJZALP2.js create mode 100644 dist/chunk-XCJZALP2.js.map create mode 100644 dist/chunk-YCFZVUZC.js create mode 100644 dist/chunk-YCFZVUZC.js.map create mode 100644 dist/index-browser.cjs create mode 100644 dist/index-browser.cjs.map create mode 100644 dist/index-browser.d.cts create mode 100644 dist/index-browser.d.ts create mode 100644 dist/index-browser.js create mode 100644 dist/index-browser.js.map create mode 100644 dist/index-interface.cjs create mode 100644 dist/index-interface.cjs.map create mode 100644 dist/index-interface.d.cts create mode 100644 dist/index-interface.d.ts create mode 100644 dist/index-interface.js create mode 100644 dist/index-interface.js.map delete mode 100644 src/lib/Properties.ts create mode 100644 src/lib/file-id/ComputeHash.ts create mode 100644 src/lib/file-id/Crc32Hash.ts create mode 100644 src/lib/file-id/CreateHasher.ts create mode 100644 src/lib/file-id/FileIDComputerInterface.ts rename src/lib/{fileID/FileIDComputer.ts => file-id/FileIDComputerWorker.ts} (84%) create mode 100644 src/lib/file-id/ShaComputeWorker.ts create mode 100644 src/lib/file-id/SimpleHash.ts delete mode 100644 src/lib/fileID/ShaComputeWorker.ts rename src/lib/{folder_watcher => folder-watcher}/FileProcessorInterface.ts (100%) rename src/lib/{folder_watcher => folder-watcher}/FolderWatcher.ts (100%) create mode 100644 src/lib/hash-compute/HashComputer.ts rename src/lib/{ComputeHashIndexCache.ts => hash-compute/HashComputerIndexCache.ts} (85%) rename src/lib/{HashComputer.ts => hash-compute/HashComputerWorker.ts} (78%) rename src/lib/{ => hash-compute}/HashIndexManager.ts (99%) rename src/lib/{ => hash-compute}/MultiHashData.ts (100%) create mode 100644 src/lib/index-browser.ts create mode 100644 src/lib/index-interface.ts rename src/lib/{ => utils}/ExistsAsync.ts (100%) diff --git a/dist/ShaComputeWorker-CNdGUbtT.d.cts b/dist/MultiHashData-B-scpa5G.d.cts similarity index 65% rename from dist/ShaComputeWorker-CNdGUbtT.d.cts rename to dist/MultiHashData-B-scpa5G.d.cts index d5d4b0a..3c163d0 100644 --- a/dist/ShaComputeWorker-CNdGUbtT.d.cts +++ b/dist/MultiHashData-B-scpa5G.d.cts @@ -38,19 +38,4 @@ declare enum CID_ALGORITHM { sha3_384 = "sha3-384" } -interface SimpleHash { - update(data: Buffer): any; - digest(): Buffer; -} -/** - * Compute the CIDs of a file using specific algorithms - * @param filePath The path to the file - * @param algorithms Array of algorithms ('sha256', 'sha1') - * @returns Array of CIDs (in the order of the algorithms) - */ -declare function computeCIDs({ filePath, algorithms }: { - filePath: string; - algorithms: CID_ALGORITHM_NAMES[]; -}): Promise; - -export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; +export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c }; diff --git a/dist/ShaComputeWorker-CNdGUbtT.d.ts b/dist/MultiHashData-B-scpa5G.d.ts similarity index 65% rename from dist/ShaComputeWorker-CNdGUbtT.d.ts rename to dist/MultiHashData-B-scpa5G.d.ts index d5d4b0a..3c163d0 100644 --- a/dist/ShaComputeWorker-CNdGUbtT.d.ts +++ b/dist/MultiHashData-B-scpa5G.d.ts @@ -38,19 +38,4 @@ declare enum CID_ALGORITHM { sha3_384 = "sha3-384" } -interface SimpleHash { - update(data: Buffer): any; - digest(): Buffer; -} -/** - * Compute the CIDs of a file using specific algorithms - * @param filePath The path to the file - * @param algorithms Array of algorithms ('sha256', 'sha1') - * @returns Array of CIDs (in the order of the algorithms) - */ -declare function computeCIDs({ filePath, algorithms }: { - filePath: string; - algorithms: CID_ALGORITHM_NAMES[]; -}): Promise; - -export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type SimpleHash as S, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c, computeCIDs as d }; +export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c }; diff --git a/dist/chunk-33LRP2HS.cjs b/dist/chunk-33LRP2HS.cjs new file mode 100644 index 0000000..495fe17 --- /dev/null +++ b/dist/chunk-33LRP2HS.cjs @@ -0,0 +1,18 @@ +'use strict'; + +var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); +var stream = require('stream'); +var digest = require('multiformats/hashes/digest'); +var cid = require('multiformats/cid'); +var crypto = require('crypto'); +var u = require('crc-32'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var u__default = /*#__PURE__*/_interopDefault(u); + +var c=class{_crc=void 0;update(e){return this._crc=u__default.default.buf(e,this._crc),this}digest(){let e=Buffer.alloc(4);return e.writeInt32BE(this._crc,0),e}};var i={cid_crc32:306,cid_md5:213,cid_sha1:17,"cid_sha2-256":18,"cid_sha3-256":22,"cid_sha3-384":21};function m(r){switch(r){case"cid_sha2-256":return crypto.createHash("sha256");case"cid_sha1":return crypto.createHash("sha1");case"cid_md5":return crypto.createHash("md5");case"cid_sha3-256":return crypto.createHash("sha3-256");case"cid_sha3-384":return crypto.createHash("sha3-384");case"cid_crc32":return new c;default:throw new Error(`Unsupported algorithm: ${r}`)}}async function L({stream:r,algorithms:e}){let a=_(e);if(r instanceof stream.Readable)for await(let t of r)a.forEach(s=>s.hasher.update(t));else {let t=r.getReader();for(;;){let{done:s,value:n}=await t.read();if(s)break;a.forEach(f=>f.hasher.update(Buffer.from(n)));}}return H(a)}function _(r){return r.filter(a=>Object.values(chunkPAKJNZBJ_cjs.b).includes(a)).map(a=>({hasher:m(a),code:i[a]}))}async function H(r){return await Promise.all(r.map(async({code:e,hasher:a})=>{let t=a.digest(),s=digest.create(e,t);return cid.CID.createV1(e,s).toString()}))} + +exports.a = L; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-33LRP2HS.cjs.map \ No newline at end of file diff --git a/dist/chunk-33LRP2HS.cjs.map b/dist/chunk-33LRP2HS.cjs.map new file mode 100644 index 0000000..d776c10 --- /dev/null +++ b/dist/chunk-33LRP2HS.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":["Readable","create","CID","createHash","crc32","Crc32Hash","data","buffer","codeTable","createHasher","algo","computeCIDs","stream","algorithms","hashers","hasherDefiner","chunk","item","reader","done","value","cidFinalize","CID_ALGORITHM_NAMES","code","hasher","hashBuffer","digest"],"mappings":"yCACA,OAAQ,YAAAA,MAAe,SACvB,OAAQ,UAAAC,MAAa,6BACrB,OAAQ,OAAAC,MAAU,mBCClB,OAAQ,cAAAC,MAAiB,SCJzB,OAAOC,MAAW,SAGX,IAAMC,EAAN,KAAsC,CACjC,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOF,EAAM,IAAIE,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EDVO,IAAMC,EAAY,CACpB,cACA,YACA,YACA,kBACA,kBACA,iBACL,EAOO,SAASC,EAAaC,EAAuC,CAChE,OAAQA,EAAM,CACV,mBACI,OAAOP,EAAW,QAAQ,EAC9B,eACI,OAAOA,EAAW,MAAM,EAC5B,cACI,OAAOA,EAAW,KAAK,EAC3B,mBACI,OAAOA,EAAW,UAAU,EAChC,mBACI,OAAOA,EAAW,UAAU,EAChC,gBACI,OAAO,IAAIE,EACf,QACI,MAAM,IAAI,MAAM,0BAA0BK,CAAI,EAAE,CACxD,CACJ,CDzBA,eAAsBC,EAAY,CAAC,OAAAC,EAAQ,WAAAC,CAAU,EAG/B,CAClB,IAAMC,EAAUC,EAAcF,CAAU,EAGxC,GAAID,aAAkBZ,EAElB,cAAiBgB,KAASJ,EACtBE,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAOD,CAAe,CAAC,MAE5D,CAEH,IAAME,EAASN,EAAO,UAAU,EAChC,OAAa,CACT,GAAM,CAAC,KAAAO,EAAM,MAAAC,CAAK,EAAI,MAAMF,EAAO,KAAK,EACxC,GAAIC,EAAM,MAEVL,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAO,OAAO,KAAKG,CAAK,CAAC,CAAC,CAClE,CACJ,CAEA,OAAOC,EAAYP,CAAO,CAC9B,CAEA,SAASC,EAAcF,EAGnB,CAMA,OALgBA,EAAW,OAAOH,GAAQ,OAAO,OAAOY,CAAmB,EAAE,SAASZ,CAAI,CAAC,EACtF,IAAIA,IAAS,CACV,OAAQD,EAAaC,CAAI,EACzB,KAAMF,EAAUE,CAAI,CACxB,EAAE,CAEV,CAEA,eAAeW,EAAYP,EAAoE,CAC3F,OAAO,MAAM,QAAQ,IAAIA,EAAQ,IAAI,MAAO,CAAC,KAAAS,EAAM,OAAAC,CAAM,IAAM,CAC3D,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASzB,EAAOsB,EAAME,CAAU,EACtC,OAAOvB,EAAI,SAASqB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n hashers.forEach(item => item.hasher.update(chunk as Buffer));\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n hashers.forEach(item => item.hasher.update(Buffer.from(value)));\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nfunction hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): {\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[] {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(algo => ({\r\n hasher: createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return hashers;\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {ReadableStream} from \"stream/web\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): Crc32Hash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-4EZ47WAR.js b/dist/chunk-4EZ47WAR.js deleted file mode 100644 index 820b61c..0000000 --- a/dist/chunk-4EZ47WAR.js +++ /dev/null @@ -1,11 +0,0 @@ -import { createReadStream } from 'fs'; -import { createHash } from 'crypto'; -import { CID } from 'multiformats/cid'; -import { create } from 'multiformats/hashes/digest'; -import x from 'crc-32'; - -var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))} - -export { o as a, d as b, u as c, B as d }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-4EZ47WAR.js.map \ No newline at end of file diff --git a/dist/chunk-4EZ47WAR.js.map b/dist/chunk-4EZ47WAR.js.map deleted file mode 100644 index b0aefc2..0000000 --- a/dist/chunk-4EZ47WAR.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCaX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDtBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-6UBSWOQL.js b/dist/chunk-6UBSWOQL.js new file mode 100644 index 0000000..1a4c3c1 --- /dev/null +++ b/dist/chunk-6UBSWOQL.js @@ -0,0 +1,8 @@ +import { a } from './chunk-A6J5HDTV.js'; +import m from 'fs'; + +async function s({filePath:r,algorithms:e}){let o=m.createReadStream(r);return a({stream:o,algorithms:e})} + +export { s as a }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-6UBSWOQL.js.map \ No newline at end of file diff --git a/dist/chunk-6UBSWOQL.js.map b/dist/chunk-6UBSWOQL.js.map new file mode 100644 index 0000000..b6523a8 --- /dev/null +++ b/dist/chunk-6UBSWOQL.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["fs","computeCIDs","filePath","algorithms","stream"],"mappings":"wCACA,OAAOA,MAAQ,KASf,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EAASJ,EAAG,iBAAiBE,CAAQ,EAC3C,OAAOD,EAAoB,CAAC,OAAAG,EAAQ,WAAAD,CAAU,CAAC,CACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-A6J5HDTV.js b/dist/chunk-A6J5HDTV.js new file mode 100644 index 0000000..6123139 --- /dev/null +++ b/dist/chunk-A6J5HDTV.js @@ -0,0 +1,12 @@ +import { b } from './chunk-YCFZVUZC.js'; +import { Readable } from 'stream'; +import { create } from 'multiformats/hashes/digest'; +import { CID } from 'multiformats/cid'; +import { createHash } from 'crypto'; +import u from 'crc-32'; + +var c=class{_crc=void 0;update(e){return this._crc=u.buf(e,this._crc),this}digest(){let e=Buffer.alloc(4);return e.writeInt32BE(this._crc,0),e}};var i={cid_crc32:306,cid_md5:213,cid_sha1:17,"cid_sha2-256":18,"cid_sha3-256":22,"cid_sha3-384":21};function m(r){switch(r){case"cid_sha2-256":return createHash("sha256");case"cid_sha1":return createHash("sha1");case"cid_md5":return createHash("md5");case"cid_sha3-256":return createHash("sha3-256");case"cid_sha3-384":return createHash("sha3-384");case"cid_crc32":return new c;default:throw new Error(`Unsupported algorithm: ${r}`)}}async function L({stream:r,algorithms:e}){let a=_(e);if(r instanceof Readable)for await(let t of r)a.forEach(s=>s.hasher.update(t));else {let t=r.getReader();for(;;){let{done:s,value:n}=await t.read();if(s)break;a.forEach(f=>f.hasher.update(Buffer.from(n)));}}return H(a)}function _(r){return r.filter(a=>Object.values(b).includes(a)).map(a=>({hasher:m(a),code:i[a]}))}async function H(r){return await Promise.all(r.map(async({code:e,hasher:a})=>{let t=a.digest(),s=create(e,t);return CID.createV1(e,s).toString()}))} + +export { L as a }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-A6J5HDTV.js.map \ No newline at end of file diff --git a/dist/chunk-A6J5HDTV.js.map b/dist/chunk-A6J5HDTV.js.map new file mode 100644 index 0000000..b1a71a1 --- /dev/null +++ b/dist/chunk-A6J5HDTV.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ComputeHash.ts","../src/lib/file-id/CreateHasher.ts","../src/lib/file-id/Crc32Hash.ts"],"names":["Readable","create","CID","createHash","crc32","Crc32Hash","data","buffer","codeTable","createHasher","algo","computeCIDs","stream","algorithms","hashers","hasherDefiner","chunk","item","reader","done","value","cidFinalize","CID_ALGORITHM_NAMES","code","hasher","hashBuffer","digest"],"mappings":"wCACA,OAAQ,YAAAA,MAAe,SACvB,OAAQ,UAAAC,MAAa,6BACrB,OAAQ,OAAAC,MAAU,mBCClB,OAAQ,cAAAC,MAAiB,SCJzB,OAAOC,MAAW,SAGX,IAAMC,EAAN,KAAsC,CACjC,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOF,EAAM,IAAIE,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EDVO,IAAMC,EAAY,CACpB,cACA,YACA,YACA,kBACA,kBACA,iBACL,EAOO,SAASC,EAAaC,EAAuC,CAChE,OAAQA,EAAM,CACV,mBACI,OAAOP,EAAW,QAAQ,EAC9B,eACI,OAAOA,EAAW,MAAM,EAC5B,cACI,OAAOA,EAAW,KAAK,EAC3B,mBACI,OAAOA,EAAW,UAAU,EAChC,mBACI,OAAOA,EAAW,UAAU,EAChC,gBACI,OAAO,IAAIE,EACf,QACI,MAAM,IAAI,MAAM,0BAA0BK,CAAI,EAAE,CACxD,CACJ,CDzBA,eAAsBC,EAAY,CAAC,OAAAC,EAAQ,WAAAC,CAAU,EAG/B,CAClB,IAAMC,EAAUC,EAAcF,CAAU,EAGxC,GAAID,aAAkBZ,EAElB,cAAiBgB,KAASJ,EACtBE,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAOD,CAAe,CAAC,MAE5D,CAEH,IAAME,EAASN,EAAO,UAAU,EAChC,OAAa,CACT,GAAM,CAAC,KAAAO,EAAM,MAAAC,CAAK,EAAI,MAAMF,EAAO,KAAK,EACxC,GAAIC,EAAM,MAEVL,EAAQ,QAAQG,GAAQA,EAAK,OAAO,OAAO,OAAO,KAAKG,CAAK,CAAC,CAAC,CAClE,CACJ,CAEA,OAAOC,EAAYP,CAAO,CAC9B,CAEA,SAASC,EAAcF,EAGnB,CAMA,OALgBA,EAAW,OAAOH,GAAQ,OAAO,OAAOY,CAAmB,EAAE,SAASZ,CAAI,CAAC,EACtF,IAAIA,IAAS,CACV,OAAQD,EAAaC,CAAI,EACzB,KAAMF,EAAUE,CAAI,CACxB,EAAE,CAEV,CAEA,eAAeW,EAAYP,EAAoE,CAC3F,OAAO,MAAM,QAAQ,IAAIA,EAAQ,IAAI,MAAO,CAAC,KAAAS,EAAM,OAAAC,CAAM,IAAM,CAC3D,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASzB,EAAOsB,EAAME,CAAU,EACtC,OAAOvB,EAAI,SAASqB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CACN","sourcesContent":["import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {create} from \"multiformats/hashes/digest\";\r\nimport {CID} from 'multiformats/cid';\r\nimport {createHasher, codeTable} from \"./CreateHasher\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @param stream The Readable stream of the file\r\n * @param algorithms Array of algorithms ('sha256', 'sha1', etc.)\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n */\r\nexport async function computeCIDs({stream, algorithms}: {\r\n stream: Readable | ReadableStream;\r\n algorithms: CID_ALGORITHM_NAMES[]\r\n}): Promise {\r\n const hashers = hasherDefiner(algorithms);\r\n\r\n // Check if the environment is Node.js or Browser and handle the stream accordingly\r\n if (stream instanceof Readable) {\r\n // Node.js environment\r\n for await (const chunk of stream) {\r\n hashers.forEach(item => item.hasher.update(chunk as Buffer));\r\n }\r\n } else {\r\n // Browser environment\r\n const reader = stream.getReader();\r\n while (true) {\r\n const {done, value} = await reader.read();\r\n if (done) break;\r\n // Assuming the hasher can handle Uint8Array directly\r\n hashers.forEach(item => item.hasher.update(Buffer.from(value)));\r\n }\r\n }\r\n\r\n return cidFinalize(hashers);\r\n}\r\n\r\nfunction hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): {\r\n hasher: SimpleHash,\r\n code: CID_ALGORITHM_CODES\r\n}[] {\r\n const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))\r\n .map(algo => ({\r\n hasher: createHasher(algo),\r\n code: codeTable[algo]\r\n }));\r\n return hashers;\r\n}\r\n\r\nasync function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise {\r\n return await Promise.all(hashers.map(async ({code, hasher}) => {\r\n const hashBuffer = hasher.digest();\r\n const digest = create(code, hashBuffer);\r\n return CID.createV1(code, digest).toString();\r\n }));\r\n}","import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\r\nimport {Readable} from \"stream\";\r\nimport {ReadableStream} from \"stream/web\";\r\nimport {SimpleHash} from \"./SimpleHash\";\r\nimport {createHash} from \"crypto\";\r\nimport {Crc32Hash} from \"./Crc32Hash\";\r\n\r\nexport const codeTable = {\r\n [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,\r\n [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,\r\n [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,\r\n [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,\r\n [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,\r\n [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384\r\n}\r\n\r\n/**\r\n * Compute the CIDs of a file using specific algorithms\r\n * @returns Array of CIDs (in the order of the algorithms)\r\n * @param algo\r\n */\r\nexport function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash {\r\n switch (algo) {\r\n case CID_ALGORITHM_NAMES.sha256:\r\n return createHash(\"sha256\");\r\n case CID_ALGORITHM_NAMES.sha1:\r\n return createHash(\"sha1\");\r\n case CID_ALGORITHM_NAMES.md5:\r\n return createHash(\"md5\");\r\n case CID_ALGORITHM_NAMES.sha3_256:\r\n return createHash(\"sha3-256\");\r\n case CID_ALGORITHM_NAMES.sha3_384:\r\n return createHash(\"sha3-384\");\r\n case CID_ALGORITHM_NAMES.crc32:\r\n return new Crc32Hash();\r\n default:\r\n throw new Error(`Unsupported algorithm: ${algo}`);\r\n }\r\n}","import crc32 from \"crc-32\";\r\nimport {SimpleHash} from \"@root/file-id/SimpleHash\";\r\n\r\nexport class Crc32Hash implements SimpleHash {\r\n private _crc: number = undefined;\r\n\r\n update(data: Buffer): Crc32Hash {\r\n // Update the CRC-32 checksum with the new chunk of data\r\n this._crc = crc32.buf(data, this._crc);\r\n return this;\r\n }\r\n\r\n digest(): Buffer {\r\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\r\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\r\n return buffer;\r\n }\r\n}"]} \ No newline at end of file diff --git a/dist/chunk-ASEKTFOG.cjs b/dist/chunk-ASEKTFOG.cjs deleted file mode 100644 index 26585eb..0000000 --- a/dist/chunk-ASEKTFOG.cjs +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var crypto = require('crypto'); -var cid = require('multiformats/cid'); -var digest = require('multiformats/hashes/digest'); -var x = require('crc-32'); - -function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } - -var x__default = /*#__PURE__*/_interopDefault(x); - -var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=fs.createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))} - -exports.a = o; -exports.b = d; -exports.c = u; -exports.d = B; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-ASEKTFOG.cjs.map \ No newline at end of file diff --git a/dist/chunk-ASEKTFOG.cjs.map b/dist/chunk-ASEKTFOG.cjs.map deleted file mode 100644 index b0aefc2..0000000 --- a/dist/chunk-ASEKTFOG.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCaX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDtBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-KWYLOVWT.cjs b/dist/chunk-KWYLOVWT.cjs new file mode 100644 index 0000000..33dff88 --- /dev/null +++ b/dist/chunk-KWYLOVWT.cjs @@ -0,0 +1,4 @@ +'use strict'; + +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-KWYLOVWT.cjs.map \ No newline at end of file diff --git a/dist/chunk-KWYLOVWT.cjs.map b/dist/chunk-KWYLOVWT.cjs.map new file mode 100644 index 0000000..a464c67 --- /dev/null +++ b/dist/chunk-KWYLOVWT.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/chunk-KZVIGNTH.cjs b/dist/chunk-KZVIGNTH.cjs new file mode 100644 index 0000000..5b7d3cf --- /dev/null +++ b/dist/chunk-KZVIGNTH.cjs @@ -0,0 +1,14 @@ +'use strict'; + +var chunk33LRP2HS_cjs = require('./chunk-33LRP2HS.cjs'); +var m = require('fs'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var m__default = /*#__PURE__*/_interopDefault(m); + +async function s({filePath:r,algorithms:e}){let o=m__default.default.createReadStream(r);return chunk33LRP2HS_cjs.a({stream:o,algorithms:e})} + +exports.a = s; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-KZVIGNTH.cjs.map \ No newline at end of file diff --git a/dist/chunk-KZVIGNTH.cjs.map b/dist/chunk-KZVIGNTH.cjs.map new file mode 100644 index 0000000..d2bb68b --- /dev/null +++ b/dist/chunk-KZVIGNTH.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["fs","computeCIDs","filePath","algorithms","stream"],"mappings":"yCACA,OAAOA,MAAQ,KASf,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EAASJ,EAAG,iBAAiBE,CAAQ,EAC3C,OAAOD,EAAoB,CAAC,OAAAG,EAAQ,WAAAD,CAAU,CAAC,CACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]} \ No newline at end of file diff --git a/dist/chunk-PAKJNZBJ.cjs b/dist/chunk-PAKJNZBJ.cjs new file mode 100644 index 0000000..140f26c --- /dev/null +++ b/dist/chunk-PAKJNZBJ.cjs @@ -0,0 +1,9 @@ +'use strict'; + +var s=(a=>(a[a.crc32=306]="crc32",a[a.md5=213]="md5",a[a.sha1=17]="sha1",a[a.sha256=18]="sha256",a[a.sha3_256=22]="sha3_256",a[a.sha3_384=21]="sha3_384",a))(s||{}),c=(a=>(a.crc32="cid_crc32",a.md5="cid_md5",a.sha1="cid_sha1",a.sha256="cid_sha2-256",a.sha3_256="cid_sha3-256",a.sha3_384="cid_sha3-384",a))(c||{}),h=(a=>(a.crc32="crc32",a.md5="md5",a.sha1="sha1",a.sha256="sha2-256",a.sha3_256="sha3-256",a.sha3_384="sha3-384",a))(h||{}); + +exports.a = s; +exports.b = c; +exports.c = h; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-PAKJNZBJ.cjs.map \ No newline at end of file diff --git a/dist/chunk-PAKJNZBJ.cjs.map b/dist/chunk-PAKJNZBJ.cjs.map new file mode 100644 index 0000000..442e48d --- /dev/null +++ b/dist/chunk-PAKJNZBJ.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":"AAiBO,IAAKA,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-XCJZALP2.js b/dist/chunk-XCJZALP2.js new file mode 100644 index 0000000..70fc393 --- /dev/null +++ b/dist/chunk-XCJZALP2.js @@ -0,0 +1,3 @@ + +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-XCJZALP2.js.map \ No newline at end of file diff --git a/dist/chunk-XCJZALP2.js.map b/dist/chunk-XCJZALP2.js.map new file mode 100644 index 0000000..a464c67 --- /dev/null +++ b/dist/chunk-XCJZALP2.js.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/chunk-YCFZVUZC.js b/dist/chunk-YCFZVUZC.js new file mode 100644 index 0000000..8bf590a --- /dev/null +++ b/dist/chunk-YCFZVUZC.js @@ -0,0 +1,5 @@ +var s=(a=>(a[a.crc32=306]="crc32",a[a.md5=213]="md5",a[a.sha1=17]="sha1",a[a.sha256=18]="sha256",a[a.sha3_256=22]="sha3_256",a[a.sha3_384=21]="sha3_384",a))(s||{}),c=(a=>(a.crc32="cid_crc32",a.md5="cid_md5",a.sha1="cid_sha1",a.sha256="cid_sha2-256",a.sha3_256="cid_sha3-256",a.sha3_384="cid_sha3-384",a))(c||{}),h=(a=>(a.crc32="crc32",a.md5="md5",a.sha1="sha1",a.sha256="sha2-256",a.sha3_256="sha3-256",a.sha3_384="sha3-384",a))(h||{}); + +export { s as a, c as b, h as c }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-YCFZVUZC.js.map \ No newline at end of file diff --git a/dist/chunk-YCFZVUZC.js.map b/dist/chunk-YCFZVUZC.js.map new file mode 100644 index 0000000..442e48d --- /dev/null +++ b/dist/chunk-YCFZVUZC.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":"AAiBO,IAAKA,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/index-browser.cjs b/dist/index-browser.cjs new file mode 100644 index 0000000..8bceef3 --- /dev/null +++ b/dist/index-browser.cjs @@ -0,0 +1,27 @@ +'use strict'; + +require('./chunk-KWYLOVWT.cjs'); +var chunk33LRP2HS_cjs = require('./chunk-33LRP2HS.cjs'); +var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); + +var a=class{constructor(r){this.targetHash=r;}async computeMissingHash(r,e){let t=this.targetHash.filter(o=>!e[o]);if(t.length===0)return;let i=await chunk33LRP2HS_cjs.a({stream:r,algorithms:t});for(let[o,c]of i.entries()){let n=t[o];e[n]=c;}}}; + +Object.defineProperty(exports, "computeCIDs", { + enumerable: true, + get: function () { return chunk33LRP2HS_cjs.a; } +}); +Object.defineProperty(exports, "CID_ALGORITHM", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.c; } +}); +Object.defineProperty(exports, "CID_ALGORITHM_CODES", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.a; } +}); +Object.defineProperty(exports, "CID_ALGORITHM_NAMES", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.b; } +}); +exports.HashComputer = a; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index-browser.cjs.map \ No newline at end of file diff --git a/dist/index-browser.cjs.map b/dist/index-browser.cjs.map new file mode 100644 index 0000000..dbdadfa --- /dev/null +++ b/dist/index-browser.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":["HashComputer","targetHash","stream","metadata","neededHashes","hashName","cids","computeCIDs","index","cid","hashType"],"mappings":"6HAKO,IAAMA,EAAN,KAAkB,CAErB,YAAoBC,EAAmC,CAAnC,gBAAAA,CACpB,CAEA,MAAM,mBAAmBC,EAAkBC,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAMC,EAAY,CAAC,OAAAL,EAAQ,WAAWE,CAAY,CAAC,EAGhE,OAAW,CAACI,EAAOC,CAAG,IAAKH,EAAK,QAAQ,EAAG,CACvC,IAAMI,EAAWN,EAAaI,CAAK,EACnCL,EAASO,CAAQ,EAAID,CACzB,CACJ,CACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index-browser.d.cts b/dist/index-browser.d.cts new file mode 100644 index 0000000..22304bb --- /dev/null +++ b/dist/index-browser.d.cts @@ -0,0 +1,23 @@ +import { C as CID_ALGORITHM_NAMES, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface } from './MultiHashData-B-scpa5G.cjs'; +export { FileIDComputerInterface, FileProcessorInterface } from './index-interface.cjs'; +import { Readable } from 'stream'; + +/** + * Compute the CIDs of a file using specific algorithms + * @param stream The Readable stream of the file + * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) + * @returns Array of CIDs (in the order of the algorithms) + */ +declare function computeCIDs({ stream, algorithms }: { + stream: Readable | ReadableStream; + algorithms: CID_ALGORITHM_NAMES[]; +}): Promise; + +declare class HashComputer { + private targetHash; + constructor(targetHash: CID_ALGORITHM_NAMES[]); + computeMissingHash(stream: Readable, metadata: MultiHashData): Promise; +} + +export { CID_ALGORITHM_NAMES, HashComputer, MultiHashData, computeCIDs }; diff --git a/dist/index-browser.d.ts b/dist/index-browser.d.ts new file mode 100644 index 0000000..6227163 --- /dev/null +++ b/dist/index-browser.d.ts @@ -0,0 +1,23 @@ +import { C as CID_ALGORITHM_NAMES, M as MultiHashData } from './MultiHashData-B-scpa5G.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface } from './MultiHashData-B-scpa5G.js'; +export { FileIDComputerInterface, FileProcessorInterface } from './index-interface.js'; +import { Readable } from 'stream'; + +/** + * Compute the CIDs of a file using specific algorithms + * @param stream The Readable stream of the file + * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) + * @returns Array of CIDs (in the order of the algorithms) + */ +declare function computeCIDs({ stream, algorithms }: { + stream: Readable | ReadableStream; + algorithms: CID_ALGORITHM_NAMES[]; +}): Promise; + +declare class HashComputer { + private targetHash; + constructor(targetHash: CID_ALGORITHM_NAMES[]); + computeMissingHash(stream: Readable, metadata: MultiHashData): Promise; +} + +export { CID_ALGORITHM_NAMES, HashComputer, MultiHashData, computeCIDs }; diff --git a/dist/index-browser.js b/dist/index-browser.js new file mode 100644 index 0000000..06dbccf --- /dev/null +++ b/dist/index-browser.js @@ -0,0 +1,10 @@ +import './chunk-XCJZALP2.js'; +import { a as a$1 } from './chunk-A6J5HDTV.js'; +export { a as computeCIDs } from './chunk-A6J5HDTV.js'; +export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; + +var a=class{constructor(r){this.targetHash=r;}async computeMissingHash(r,e){let t=this.targetHash.filter(o=>!e[o]);if(t.length===0)return;let i=await a$1({stream:r,algorithms:t});for(let[o,c]of i.entries()){let n=t[o];e[n]=c;}}}; + +export { a as HashComputer }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index-browser.js.map \ No newline at end of file diff --git a/dist/index-browser.js.map b/dist/index-browser.js.map new file mode 100644 index 0000000..defc454 --- /dev/null +++ b/dist/index-browser.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":["HashComputer","targetHash","stream","metadata","neededHashes","hashName","cids","computeCIDs","index","cid","hashType"],"mappings":"0HAKO,IAAMA,EAAN,KAAkB,CAErB,YAAoBC,EAAmC,CAAnC,gBAAAA,CACpB,CAEA,MAAM,mBAAmBC,EAAkBC,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAMC,EAAY,CAAC,OAAAL,EAAQ,WAAWE,CAAY,CAAC,EAGhE,OAAW,CAACI,EAAOC,CAAG,IAAKH,EAAK,QAAQ,EAAG,CACvC,IAAMI,EAAWN,EAAaI,CAAK,EACnCL,EAASO,CAAQ,EAAID,CACzB,CACJ,CACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index-interface.cjs b/dist/index-interface.cjs new file mode 100644 index 0000000..da4efea --- /dev/null +++ b/dist/index-interface.cjs @@ -0,0 +1,21 @@ +'use strict'; + +require('./chunk-KWYLOVWT.cjs'); +var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); + + + +Object.defineProperty(exports, "CID_ALGORITHM", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.c; } +}); +Object.defineProperty(exports, "CID_ALGORITHM_CODES", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.a; } +}); +Object.defineProperty(exports, "CID_ALGORITHM_NAMES", { + enumerable: true, + get: function () { return chunkPAKJNZBJ_cjs.b; } +}); +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index-interface.cjs.map \ No newline at end of file diff --git a/dist/index-interface.cjs.map b/dist/index-interface.cjs.map new file mode 100644 index 0000000..a464c67 --- /dev/null +++ b/dist/index-interface.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/index-interface.d.cts b/dist/index-interface.d.cts new file mode 100644 index 0000000..0c5815b --- /dev/null +++ b/dist/index-interface.d.cts @@ -0,0 +1,15 @@ +import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs'; +import * as Stream from 'stream'; + +interface FileProcessorInterface { + processFile(current: number, queueSize: number, nfoFilePath: string): Promise; + canProcessFile(filePath: string): Promise; + deleteFile(filePath: string): Promise; +} + +interface FileIDComputerInterface { + computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} + +export { CID_ALGORITHM_NAMES, type FileIDComputerInterface, type FileProcessorInterface }; diff --git a/dist/index-interface.d.ts b/dist/index-interface.d.ts new file mode 100644 index 0000000..d4abaf5 --- /dev/null +++ b/dist/index-interface.d.ts @@ -0,0 +1,15 @@ +import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.js'; +import * as Stream from 'stream'; + +interface FileProcessorInterface { + processFile(current: number, queueSize: number, nfoFilePath: string): Promise; + canProcessFile(filePath: string): Promise; + deleteFile(filePath: string): Promise; +} + +interface FileIDComputerInterface { + computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} + +export { CID_ALGORITHM_NAMES, type FileIDComputerInterface, type FileProcessorInterface }; diff --git a/dist/index-interface.js b/dist/index-interface.js new file mode 100644 index 0000000..ea6cd85 --- /dev/null +++ b/dist/index-interface.js @@ -0,0 +1,4 @@ +import './chunk-XCJZALP2.js'; +export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=index-interface.js.map \ No newline at end of file diff --git a/dist/index-interface.js.map b/dist/index-interface.js.map new file mode 100644 index 0000000..a464c67 --- /dev/null +++ b/dist/index-interface.js.map @@ -0,0 +1 @@ +{"version":3,"sources":[],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/index.cjs b/dist/index.cjs index 1cf02af..c6d6472 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -1,6 +1,9 @@ 'use strict'; -var chunkASEKTFOG_cjs = require('./chunk-ASEKTFOG.cjs'); +require('./chunk-KWYLOVWT.cjs'); +require('./chunk-KZVIGNTH.cjs'); +require('./chunk-33LRP2HS.cjs'); +var chunkPAKJNZBJ_cjs = require('./chunk-PAKJNZBJ.cjs'); var piscina = require('piscina'); var fs = require('fs'); var csvParse = require('csv-parse'); @@ -36,26 +39,25 @@ var c__namespace = /*#__PURE__*/_interopNamespace(c); var M__default = /*#__PURE__*/_interopDefault(M); var z__default = /*#__PURE__*/_interopDefault(z); -var l=undefined;l=l.replace("src","dist");l=l+"/worker.js";var v=new URL(l).href;console.log("Worker URL: ",v);var p=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||v});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let i=this.targetHash.filter(s=>!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=csvParse.parse({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=sync$1.stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),i=await promises.stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__namespace.default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await promises.readdir(e,{withFileTypes:!0});for(let r of i){let s=c__namespace.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M__default.default.watch(e,t):i=M__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=csvParse.parse({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{fs.createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=sync$1.stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__namespace.default.basename(e),i=await promises.stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__namespace.default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__namespace.default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await promises.readdir(e,{withFileTypes:!0});for(let r of i){let s=c__namespace.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c__namespace.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M__default.default.watch(e,t):i=M__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputerWorker","workerPath","filePath","algorithms","HashComputerWorker","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","HashComputerIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"8IAAA,OAAQ,WAAAA,MAAc,UAKtB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAA0B,CACrB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC5BO,IAAMC,EAAN,KAAqD,CAGxD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAqBC,CAAU,CAC7D,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAyD,CAI5D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAmBC,EAAWJ,CAAU,EAChE,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.d.cts b/dist/index.d.cts index 1b454b4..531ca1f 100644 --- a/dist/index.d.cts +++ b/dist/index.d.cts @@ -1,8 +1,23 @@ -import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.cjs'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.cjs'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES } from './MultiHashData-B-scpa5G.cjs'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; +import { FileProcessorInterface } from './index-interface.cjs'; +export { FileIDComputerInterface } from './index-interface.cjs'; +import 'stream'; + +declare class FileIDComputerWorker { + private piscina; + constructor(workerPath?: string); + /** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ + computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} interface IndexLine extends Partial> { path: string; @@ -41,7 +56,7 @@ declare class HashIndexManager { addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; } -declare class ComputeHashIndexCache implements ComputeInterface { +declare class HashComputerIndexCache implements ComputeInterface { private targetHash; private hashIndexManager; private hashComputer; @@ -50,38 +65,13 @@ declare class ComputeHashIndexCache implements ComputeInterface { getHashIndexManager(): Promise; } -declare function existsAsync(filePath: string): Promise; - -declare class FileIDComputer { - private piscina; - constructor(workerPath?: string); - /** - * Compute the CIDs of a file using specific algorithms - * @param filePath The path to the file - * @param algorithms Array of algorithms ('sha256', 'sha1') - * @returns Array of CIDs (in the order of the algorithms) - */ - computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; -} - -declare class HashComputer implements ComputeInterface { +declare class HashComputerWorker implements ComputeInterface { private targetHash; private fileIDComputer; constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); computeMissingHash(filePath: string, metadata: MultiHashData): Promise; } -interface Properties { - TARGET_HASHES: string; - INDEX_FILE_PATH: string; -} - -interface FileProcessorInterface { - processFile(current: number, queueSize: number, nfoFilePath: string): Promise; - canProcessFile(filePath: string): Promise; - deleteFile(filePath: string): Promise; -} - declare class FolderWatcher { private WATCH_FOLDER_LIST; private config; @@ -115,4 +105,4 @@ declare class FolderWatcher { watch(): Promise; } -export { CID_ALGORITHM_NAMES, ComputeHashIndexCache, ComputeInterface, FileIDComputer, type FileProcessorInterface, FolderWatcher, HashComputer, HashIndexManager, INDEX_HEADERS, MultiHashData, type Properties, existsAsync }; +export { CID_ALGORITHM_NAMES, ComputeInterface, FileIDComputerWorker, FileProcessorInterface, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS, MultiHashData }; diff --git a/dist/index.d.ts b/dist/index.d.ts index 20e15b3..632172d 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,8 +1,23 @@ -import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.js'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.js'; +import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES } from './MultiHashData-B-scpa5G.js'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; +import { FileProcessorInterface } from './index-interface.js'; +export { FileIDComputerInterface } from './index-interface.js'; +import 'stream'; + +declare class FileIDComputerWorker { + private piscina; + constructor(workerPath?: string); + /** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ + computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} interface IndexLine extends Partial> { path: string; @@ -41,7 +56,7 @@ declare class HashIndexManager { addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void; } -declare class ComputeHashIndexCache implements ComputeInterface { +declare class HashComputerIndexCache implements ComputeInterface { private targetHash; private hashIndexManager; private hashComputer; @@ -50,38 +65,13 @@ declare class ComputeHashIndexCache implements ComputeInterface { getHashIndexManager(): Promise; } -declare function existsAsync(filePath: string): Promise; - -declare class FileIDComputer { - private piscina; - constructor(workerPath?: string); - /** - * Compute the CIDs of a file using specific algorithms - * @param filePath The path to the file - * @param algorithms Array of algorithms ('sha256', 'sha1') - * @returns Array of CIDs (in the order of the algorithms) - */ - computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise; -} - -declare class HashComputer implements ComputeInterface { +declare class HashComputerWorker implements ComputeInterface { private targetHash; private fileIDComputer; constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string); computeMissingHash(filePath: string, metadata: MultiHashData): Promise; } -interface Properties { - TARGET_HASHES: string; - INDEX_FILE_PATH: string; -} - -interface FileProcessorInterface { - processFile(current: number, queueSize: number, nfoFilePath: string): Promise; - canProcessFile(filePath: string): Promise; - deleteFile(filePath: string): Promise; -} - declare class FolderWatcher { private WATCH_FOLDER_LIST; private config; @@ -115,4 +105,4 @@ declare class FolderWatcher { watch(): Promise; } -export { CID_ALGORITHM_NAMES, ComputeHashIndexCache, ComputeInterface, FileIDComputer, type FileProcessorInterface, FolderWatcher, HashComputer, HashIndexManager, INDEX_HEADERS, MultiHashData, type Properties, existsAsync }; +export { CID_ALGORITHM_NAMES, ComputeInterface, FileIDComputerWorker, FileProcessorInterface, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS, MultiHashData }; diff --git a/dist/index.js b/dist/index.js index 12aec21..271c5f8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,18 +1,21 @@ -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-4EZ47WAR.js'; +import './chunk-XCJZALP2.js'; +import './chunk-6UBSWOQL.js'; +import './chunk-A6J5HDTV.js'; +export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-YCFZVUZC.js'; import { Piscina } from 'piscina'; import { promises, createReadStream } from 'fs'; import { parse as parse$1 } from 'csv-parse'; import { parse } from 'csv-parse/sync'; import { stringify } from 'csv-stringify/sync'; -import { access, constants, stat, readdir } from 'fs/promises'; +import { stat, readdir, access, constants } from 'fs/promises'; import * as c from 'path'; import c__default from 'path'; import { clearInterval } from 'node:timers'; import M from 'chokidar'; import z from 'p-queue'; -var l=import.meta.dirname;l=l.replace("src","dist");l=l+"/worker.js";var v=new URL(l).href;console.log("Worker URL: ",v);var p=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||v});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new p(t);}fileIDComputer;async computeMissingHash(e,t){let i=this.targetHash.filter(s=>!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=parse$1({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),i=await stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await readdir(e,{withFileTypes:!0});for(let r of i){let s=c.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M.watch(e,t):i=M.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a!t[s]);if(i.length===0)return;let r=await this.fileIDComputer.computeCIDs(e,i);for(let[s,a]of r.entries()){let n=i[s];t[n]=a;}}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var E=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let i of this.targetHash)if(this.filePaths[i]=`${e}-${i}.csv`,!this.filePaths[i])throw new Error(`Invalid index file path for ${i}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{for(let r of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[r],r))throw new Error(`Invalid index file headers for ${r}`);await this.loadIndex(r);}e&&this.start(),t();}catch(r){i(r);}})),this.initialLoad}checkCSVHeaders(e,t){let i=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(i.length===0)return !0;let r=Object.keys(i[0]);return [...E,t].every(a=>r.includes(a))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let i=await this.readCsv(e);for(let r of i){let s=this.cache.get(r.path);s?s[e]=r[e]:this.cache.set(r.path,r);}return this.lastIndexFileSize=t.size,this.lastCacheFile=i,i}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),i=parse$1({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,a)=>{createReadStream(this.filePaths[e]).pipe(i).on("data",n=>{r.push(n);}).on("end",()=>{s(r),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",n=>{a(n);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t;this.cache.size!==0&&(t=this.loadIndexFromCache());for(let r of this.targetHash){let s=await this.loadIndex(r),a=new Map(s.map(n=>[n.path,n]));if(this.cache.size!==0){let n=t.filter(o=>!a.has(o.path)&&!!o[r]);if(n.length!==0){let o=stringify(n,{header:s.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],o);}}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}async getCidForFileAsync(e){let t=c__default.basename(e),i=await stat(e);return this.getCidForFile(t,i.size,i.mtime.toISOString())}getCidForFile(e,t,i){let r=c__default.basename(e),s=this.cache.get(r);for(let a of this.targetHash)s[a]||delete s[a];if(s){if(s.mtime){if(s.size===t+""&&s.mtime===i)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,i,r){if(!e||!t||!i||!r)throw new Error("Invalid parameters");for(let o of this.targetHash)if(!r[o])throw new Error(`Missing hash ${o}`);let s=t+"",a=c__default.basename(e),n=this.cache.get(a);if(n)for(let o of this.targetHash)n[o]=r[o];else {let o={};for(let I of this.targetHash)o[I]=r[I];let g={path:a,size:s,mtime:i,...o};this.cache.set(a,g);}this.hasChanged=!0;}};var y=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(t,i),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let r=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(r)for(let s of this.targetHash)!t[s]&&r[s]&&(t[s]=r[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}async getHashIndexManager(){return await this.hashIndexManager.init(),this.hashIndexManager}};var F=class{constructor(e,t,i){this.WATCH_FOLDER_LIST=t;this.config=i;this.fileProcessor=e;}initialized=!1;queue=new z({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let i=await readdir(e,{withFileTypes:!0});for(let r of i){let s=c.join(e,r.name);if(r.isDirectory())await this.processDirectory(s,t);else {let a=this.processFile(s);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),i=[];for(let s of t){let a=c.join(e,s.name);s.isDirectory()?i.push(this.countFile(a)):i.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(i)).reduce((s,a)=>s+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},i;(this.config.interval||0)<=0?i=M.watch(e,t):i=M.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),i.on("add",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("change",async r=>{try{await this.processFileExtended(r);}catch(s){console.error(`Error processing file ${r}:`,s);}}).on("unlink",async r=>{try{await this.fileProcessor.deleteFile(r),await this.processFileExtended(r);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",r=>console.error(`Watcher error: ${r}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let i=[];for(let a=0;aa+n,0);console.log(`Found ${r} files to process`),this.queueSize=r;let s=[];for(let a=0;a {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputer} from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface {\n private fileIDComputer: FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/file-id/FileIDComputerWorker.ts","../src/lib/hash-compute/HashComputerWorker.ts","../src/lib/hash-compute/HashIndexManager.ts","../src/lib/utils/ExistsAsync.ts","../src/lib/hash-compute/HashComputerIndexCache.ts","../src/lib/folder-watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputerWorker","workerPath","filePath","algorithms","HashComputerWorker","targetHash","metadata","neededHashes","hashName","cids","index","cid","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","stat","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","cacheRows","existingRows","existingRowsAsMap","row","newRows","csvString","totalTime","fileName","fileSize","mtime","fileNameIndex","hashs","size","baseName","filteredHash","data","HashComputerIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"0IAAA,OAAQ,WAAAA,MAAc,UAKtB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAA0B,CACrB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC5BO,IAAMC,EAAN,KAAqD,CAGxD,YAAoBC,EAAmCJ,EAAqB,CAAxD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAqBC,CAAU,CAC7D,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,OAIJ,IAAME,EAAO,MAAM,KAAK,eAAe,YAAYP,EAAUK,CAAY,EAGzE,OAAW,CAACG,EAAOC,CAAG,IAAKF,EAAK,QAAQ,EAAG,CACvC,IAAMG,EAAWL,EAAaG,CAAK,EACnCJ,EAASM,CAAQ,EAAID,CACzB,CACJ,CACJ,EC5BA,OAAQ,oBAAAE,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYlB,EAAoC,CAClE,GAAI,CACA,aAAMgB,EAAOhB,EAAUiB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cAE5B,OAAQ,QAAAC,MAAW,cAQZ,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYvB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWwB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGxB,CAAQ,IAAIwB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUhB,EAAUe,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLZ,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAca,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMtB,EAAG,KAAK,KAAK,UAAUY,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAMN,EAAY,KAAK,UAAUM,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASzB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKiB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpChB,EAAiB,KAAK,UAAUa,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAa,iBAAiC,CAC1C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE1BG,EACA,KAAK,MAAM,OAAS,IACpBA,EAAY,KAAK,mBAAmB,GAGxC,QAAWhB,KAAQ,KAAK,WAAY,CAChC,IAAIiB,EAA4B,MAAM,KAAK,UAAUjB,CAAI,EACrDkB,EAA4C,IAAI,IAAID,EAAa,IAAIE,GAAO,CAACA,EAAI,KAAMA,CAAG,CAAC,CAAC,EAChG,GAAI,KAAK,MAAM,OAAS,EAAG,CAEvB,IAAMC,EAAUJ,EAAU,OAAOG,GAEd,CAACD,EAAkB,IAAIC,EAAI,IAAI,GAC7B,CAAC,CAACA,EAAInB,CAAI,CAC9B,EAED,GAAIoB,EAAQ,SAAW,EAAG,CAEtB,IAAMC,EAAY9B,EAAU6B,EAAS,CACjC,OAAQH,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKjB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMZ,EAAG,WAAW,KAAK,UAAUY,CAAI,EAAGqB,CAAS,CACvD,CACJ,CACJ,CAEA,IAAMC,EAAY,YAAY,IAAI,EAAIT,EACtC,QAAQ,IAAI,kBAAkBS,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEA,MAAa,mBAAmB9C,EAAsC,CAClE,IAAM+C,EAAW5B,EAAK,SAASnB,CAAQ,EACjCkC,EAAQ,MAAMb,EAAKrB,CAAQ,EACjC,OAAO,KAAK,cAAc+C,EAAUb,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,CAC7E,CAEO,cAAclC,EAAkBgD,EAAkBC,EAA0B,CAC/E,IAAMF,EAAW5B,EAAK,SAASnB,CAAQ,EACnCkD,EAAgB,KAAK,MAAM,IAAIH,CAAQ,EAC3C,QAAWvB,KAAQ,KAAK,WACf0B,EAAc1B,CAAI,GACnB,OAAO0B,EAAc1B,CAAI,EAGjC,GAAI0B,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUF,EAAW,IAAOE,EAAc,QAAUD,EAClE,OAAOC,UAIPA,EAAc,OAAUF,EAAW,GACnC,OAAOE,EAWnB,OAAO,IACX,CAEO,WAAWlD,EAAkBgD,EAAkBC,EAAeE,EAA2D,CAC5H,GAAI,CAACnD,GAAY,CAACgD,GAAY,CAACC,GAAS,CAACE,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW3B,KAAQ,KAAK,WACpB,GAAI,CAAC2B,EAAM3B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM4B,EAAOJ,EAAW,GAClBK,EAAWlC,EAAK,SAASnB,CAAQ,EACnCoC,EAAY,KAAK,MAAM,IAAIiB,CAAQ,EACvC,GAAKjB,EAUD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI2B,EAAM3B,CAAI,MAXpB,CAEZ,IAAI8B,EAAe,CAAC,EACpB,QAAW9B,KAAQ,KAAK,WACpB8B,EAAa9B,CAAI,EAAI2B,EAAM3B,CAAI,EAEnC,IAAM+B,EAAO,CAAC,KAAMF,EAAU,KAAMD,EAAM,MAAOH,EAAO,GAAGK,CAAY,EACvE,KAAK,MAAM,IAAID,EAAUE,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EEzRA,OAAQ,QAAAlC,MAAW,cACnB,OAAOF,MAAU,OAEV,IAAMqC,EAAN,KAAyD,CAI5D,YAAYC,EAA8BtD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAmBC,EAAWJ,CAAU,EAChE,KAAK,iBAAmB,IAAIwB,EAAiBkC,EAActD,CAAU,CACzE,CANQ,iBACA,aAOR,MAAa,mBAAmBH,EAAkBI,EAAwC,CACtF,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI8B,EAAQ,MAAMb,EAAKrB,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAImB,EAAK,SAASnB,CAAQ,CAAC,EAAG,CAC/D,IAAMoC,EAAY,KAAK,iBAAiB,cAAcpC,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAACpB,EAASoB,CAAI,GAAKY,EAAUZ,CAAI,IACjCpB,EAASoB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBxB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUkC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG9B,CAAQ,CAC9F,CAEA,MAAM,qBAAiD,CACnD,aAAM,KAAK,iBAAiB,KAAK,EAC1B,KAAK,gBAChB,CACJ,ECpCA,UAAYe,MAAU,OACtB,OAAOuC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYzE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMyE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWzE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAI0E,EAAe,UAAQ1E,CAAQ,EACnC,MAAM,KAAK,iBAAiB0E,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO7E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS8E,EAAO,CACZ,QAAQ,MAAM,yBAAyB9E,CAAQ,IAAK8E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO9E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,yBAAyB5B,CAAQ,IAAK4B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO5B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS4B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUkD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../hash-compute/MultiHashData\";\nimport {FileIDComputerInterface} from \"@root/file-id/FileIDComputerInterface\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputerWorker{\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import {FileIDComputerWorker} from \"../file-id/FileIDComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputerWorker implements ComputeInterface {\n private fileIDComputer: FileIDComputerWorker;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {\n this.fileIDComputer = new FileIDComputerWorker(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"../utils/ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\nimport {stat} from \"fs/promises\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n public async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n let cacheRows: IndexLine[];\n if (this.cache.size !== 0) {\n cacheRows = this.loadIndexFromCache();\n }\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n let existingRowsAsMap: Map = new Map(existingRows.map(row => [row.path, row]));\n if (this.cache.size !== 0) {\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => {\n //to be added a row must not exist in the file and must exist in the cache (with a hash)\n const newRow = !existingRowsAsMap.has(row.path);\n return newRow && !!row[hash];\n });\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public async getCidForFileAsync(filePath: string): Promise {\n const fileName = path.basename(filePath);\n const stats = await stat(filePath);\n return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n for (const hash of this.targetHash) {\n if (!fileNameIndex[hash]) {\n delete fileNameIndex[hash];\n }\n }\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n //filter only the hashes we need\n let filteredHash = {};\n for (const hash of this.targetHash) {\n filteredHash[hash] = hashs[hash];\n }\n const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputerWorker} from \"./HashComputerWorker\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class HashComputerIndexCache implements ComputeInterface {\n private hashIndexManager: HashIndexManager;\n private hashComputer: HashComputerWorker;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputerWorker(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n\n async getHashIndexManager(): Promise {\n await this.hashIndexManager.init();\n return this.hashIndexManager;\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/worker.cjs b/dist/worker.cjs index 0e30495..caf03ce 100644 --- a/dist/worker.cjs +++ b/dist/worker.cjs @@ -1,9 +1,11 @@ 'use strict'; -var chunkASEKTFOG_cjs = require('./chunk-ASEKTFOG.cjs'); +var chunkKZVIGNTH_cjs = require('./chunk-KZVIGNTH.cjs'); +require('./chunk-33LRP2HS.cjs'); +require('./chunk-PAKJNZBJ.cjs'); -module.exports = chunkASEKTFOG_cjs.d; +module.exports = chunkKZVIGNTH_cjs.a; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.cjs.map \ No newline at end of file diff --git a/dist/worker.d.cts b/dist/worker.d.cts index 6ce4749..365be37 100644 --- a/dist/worker.d.cts +++ b/dist/worker.d.cts @@ -1 +1,14 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.cjs'; +import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.cjs'; + +/** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ +declare function computeCIDs({ filePath, algorithms }: { + filePath: string; + algorithms: CID_ALGORITHM_NAMES[]; +}): Promise; + +export { computeCIDs as default }; diff --git a/dist/worker.d.ts b/dist/worker.d.ts index dd6aeb4..c03d40f 100644 --- a/dist/worker.d.ts +++ b/dist/worker.d.ts @@ -1 +1,14 @@ -export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.js'; +import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.js'; + +/** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ +declare function computeCIDs({ filePath, algorithms }: { + filePath: string; + algorithms: CID_ALGORITHM_NAMES[]; +}): Promise; + +export { computeCIDs as default }; diff --git a/dist/worker.js b/dist/worker.js index 6eea1a7..24b9409 100644 --- a/dist/worker.js +++ b/dist/worker.js @@ -1,3 +1,5 @@ -export { d as default } from './chunk-4EZ47WAR.js'; +export { a as default } from './chunk-6UBSWOQL.js'; +import './chunk-A6J5HDTV.js'; +import './chunk-YCFZVUZC.js'; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/package.json b/package.json index 1904975..c9b46b7 100644 --- a/package.json +++ b/package.json @@ -21,13 +21,17 @@ "import": "./dist/index.js", "require": "./dist/index.cjs" }, - "./index.async": { - "import": "./dist/index.async.js", - "require": "./dist/index.async.cjs" + "./index-interface": { + "import": "./dist/index-interface.js", + "require": "./dist/index-interface.cjs" + }, + "./index-browser": { + "import": "./dist/index-browser.js", + "require": "./dist/index-browser.cjs" } }, "scripts": { - "build": "cross-env rimraf dist & tsup", + "build": "cross-env rimraf ./dist & tsup", "ci": "yarn run coverage", "test": "vitest" }, diff --git a/src/lib/Properties.ts b/src/lib/Properties.ts deleted file mode 100644 index 4521dac..0000000 --- a/src/lib/Properties.ts +++ /dev/null @@ -1,5 +0,0 @@ - -export interface Properties{ - TARGET_HASHES:string; - INDEX_FILE_PATH:string; -} \ No newline at end of file diff --git a/src/lib/file-id/ComputeHash.ts b/src/lib/file-id/ComputeHash.ts new file mode 100644 index 0000000..4e39401 --- /dev/null +++ b/src/lib/file-id/ComputeHash.ts @@ -0,0 +1,58 @@ +import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {Readable} from "stream"; +import {create} from "multiformats/hashes/digest"; +import {CID} from 'multiformats/cid'; +import {createHasher, codeTable} from "./CreateHasher"; +import {SimpleHash} from "./SimpleHash"; + +/** + * Compute the CIDs of a file using specific algorithms + * @param stream The Readable stream of the file + * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) + * @returns Array of CIDs (in the order of the algorithms) + */ +export async function computeCIDs({stream, algorithms}: { + stream: Readable | ReadableStream; + algorithms: CID_ALGORITHM_NAMES[] +}): Promise { + const hashers = hasherDefiner(algorithms); + + // Check if the environment is Node.js or Browser and handle the stream accordingly + if (stream instanceof Readable) { + // Node.js environment + for await (const chunk of stream) { + hashers.forEach(item => item.hasher.update(chunk as Buffer)); + } + } else { + // Browser environment + const reader = stream.getReader(); + while (true) { + const {done, value} = await reader.read(); + if (done) break; + // Assuming the hasher can handle Uint8Array directly + hashers.forEach(item => item.hasher.update(Buffer.from(value))); + } + } + + return cidFinalize(hashers); +} + +function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[]): { + hasher: SimpleHash, + code: CID_ALGORITHM_CODES +}[] { + const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo)) + .map(algo => ({ + hasher: createHasher(algo), + code: codeTable[algo] + })); + return hashers; +} + +async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise { + return await Promise.all(hashers.map(async ({code, hasher}) => { + const hashBuffer = hasher.digest(); + const digest = create(code, hashBuffer); + return CID.createV1(code, digest).toString(); + })); +} \ No newline at end of file diff --git a/src/lib/file-id/Crc32Hash.ts b/src/lib/file-id/Crc32Hash.ts new file mode 100644 index 0000000..f3efdbf --- /dev/null +++ b/src/lib/file-id/Crc32Hash.ts @@ -0,0 +1,18 @@ +import crc32 from "crc-32"; +import {SimpleHash} from "@root/file-id/SimpleHash"; + +export class Crc32Hash implements SimpleHash { + private _crc: number = undefined; + + update(data: Buffer): Crc32Hash { + // Update the CRC-32 checksum with the new chunk of data + this._crc = crc32.buf(data, this._crc); + return this; + } + + digest(): Buffer { + const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits) + buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format + return buffer; + } +} \ No newline at end of file diff --git a/src/lib/file-id/CreateHasher.ts b/src/lib/file-id/CreateHasher.ts new file mode 100644 index 0000000..312c0d6 --- /dev/null +++ b/src/lib/file-id/CreateHasher.ts @@ -0,0 +1,39 @@ +import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {Readable} from "stream"; +import {ReadableStream} from "stream/web"; +import {SimpleHash} from "./SimpleHash"; +import {createHash} from "crypto"; +import {Crc32Hash} from "./Crc32Hash"; + +export const codeTable = { + [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32, + [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5, + [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, + [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, + [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, + [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384 +} + +/** + * Compute the CIDs of a file using specific algorithms + * @returns Array of CIDs (in the order of the algorithms) + * @param algo + */ +export function createHasher(algo: CID_ALGORITHM_NAMES): SimpleHash { + switch (algo) { + case CID_ALGORITHM_NAMES.sha256: + return createHash("sha256"); + case CID_ALGORITHM_NAMES.sha1: + return createHash("sha1"); + case CID_ALGORITHM_NAMES.md5: + return createHash("md5"); + case CID_ALGORITHM_NAMES.sha3_256: + return createHash("sha3-256"); + case CID_ALGORITHM_NAMES.sha3_384: + return createHash("sha3-384"); + case CID_ALGORITHM_NAMES.crc32: + return new Crc32Hash(); + default: + throw new Error(`Unsupported algorithm: ${algo}`); + } +} \ No newline at end of file diff --git a/src/lib/file-id/FileIDComputerInterface.ts b/src/lib/file-id/FileIDComputerInterface.ts new file mode 100644 index 0000000..7d3a97d --- /dev/null +++ b/src/lib/file-id/FileIDComputerInterface.ts @@ -0,0 +1,6 @@ +import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; +import * as Stream from "stream"; + +export interface FileIDComputerInterface{ + computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise; +} \ No newline at end of file diff --git a/src/lib/fileID/FileIDComputer.ts b/src/lib/file-id/FileIDComputerWorker.ts similarity index 84% rename from src/lib/fileID/FileIDComputer.ts rename to src/lib/file-id/FileIDComputerWorker.ts index 2ab712a..c52f793 100644 --- a/src/lib/fileID/FileIDComputer.ts +++ b/src/lib/file-id/FileIDComputerWorker.ts @@ -1,5 +1,6 @@ import {Piscina} from "piscina"; -import {CID_ALGORITHM_NAMES} from "../MultiHashData"; +import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {FileIDComputerInterface} from "@root/file-id/FileIDComputerInterface"; // Construct the URL for the current module let distFolder = import.meta.dirname; @@ -8,7 +9,7 @@ distFolder = distFolder + "/worker.js"; const workerUrl = new URL(distFolder).href; console.log("Worker URL: ", workerUrl); -export class FileIDComputer { +export class FileIDComputerWorker{ private piscina: Piscina; constructor(workerPath?:string) { diff --git a/src/lib/file-id/ShaComputeWorker.ts b/src/lib/file-id/ShaComputeWorker.ts new file mode 100644 index 0000000..7e3f7ea --- /dev/null +++ b/src/lib/file-id/ShaComputeWorker.ts @@ -0,0 +1,17 @@ +import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; +import fs from "fs"; +import {computeCIDs as computeCIDsInternal} from "./ComputeHash"; + +/** + * Compute the CIDs of a file using specific algorithms + * @param filePath The path to the file + * @param algorithms Array of algorithms ('sha256', 'sha1') + * @returns Array of CIDs (in the order of the algorithms) + */ +export default async function computeCIDs({filePath, algorithms}: { + filePath: string; + algorithms: CID_ALGORITHM_NAMES[] +}): Promise { + const stream = fs.createReadStream(filePath); + return computeCIDsInternal({stream, algorithms}); +} diff --git a/src/lib/file-id/SimpleHash.ts b/src/lib/file-id/SimpleHash.ts new file mode 100644 index 0000000..d1c4297 --- /dev/null +++ b/src/lib/file-id/SimpleHash.ts @@ -0,0 +1,5 @@ +export interface SimpleHash { + update(data: Buffer), + + digest(): Buffer; +} \ No newline at end of file diff --git a/src/lib/fileID/ShaComputeWorker.ts b/src/lib/fileID/ShaComputeWorker.ts deleted file mode 100644 index 12d5586..0000000 --- a/src/lib/fileID/ShaComputeWorker.ts +++ /dev/null @@ -1,71 +0,0 @@ -import {createReadStream} from "fs"; -import {createHash} from 'crypto'; -import {CID} from 'multiformats/cid'; -import {create} from "multiformats/hashes/digest"; -import crc32 from 'crc-32'; -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData"; - -export interface SimpleHash { - update(data: Buffer), - - digest(): Buffer; -} - -class Crc32Hash implements SimpleHash { - private _crc: number = undefined; - - update(data: Buffer): Crc32Hash { - // Update the CRC-32 checksum with the new chunk of data - this._crc = crc32.buf(data, this._crc); - return this; - } - - digest(): Buffer { - const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits) - buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format - return buffer; - } -} - - -/** - * Compute the CIDs of a file using specific algorithms - * @param filePath The path to the file - * @param algorithms Array of algorithms ('sha256', 'sha1') - * @returns Array of CIDs (in the order of the algorithms) - */ -export default async function computeCIDs({filePath, algorithms}: { - filePath: string; - algorithms: CID_ALGORITHM_NAMES[] -}): Promise { - const hashFunctions: Record - = { - [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')}, - [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')}, - [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')}, - [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')}, - [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')}, - [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()} - - }; - - const hashers = algorithms.map(algo => ({ - hasher: hashFunctions[algo].hasher, - code: hashFunctions[algo].code - })); - - const stream = createReadStream(filePath); - for await (const chunk of stream) { - for (const item of hashers) { - item.hasher.update(chunk as Buffer); - } - } - - const cids = await Promise.all(hashers.map(async ({code, hasher}) => { - const hashBuffer = hasher.digest(); - const digest = create(code, hashBuffer); - return CID.createV1(code, digest).toString(); - })); - - return cids; -} diff --git a/src/lib/folder_watcher/FileProcessorInterface.ts b/src/lib/folder-watcher/FileProcessorInterface.ts similarity index 100% rename from src/lib/folder_watcher/FileProcessorInterface.ts rename to src/lib/folder-watcher/FileProcessorInterface.ts diff --git a/src/lib/folder_watcher/FolderWatcher.ts b/src/lib/folder-watcher/FolderWatcher.ts similarity index 100% rename from src/lib/folder_watcher/FolderWatcher.ts rename to src/lib/folder-watcher/FolderWatcher.ts diff --git a/src/lib/hash-compute/HashComputer.ts b/src/lib/hash-compute/HashComputer.ts new file mode 100644 index 0000000..b7f7dea --- /dev/null +++ b/src/lib/hash-compute/HashComputer.ts @@ -0,0 +1,29 @@ +import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData"; +import {Readable} from "stream"; + +import {computeCIDs} from "@root/file-id/ComputeHash"; + +export class HashComputer{ + + constructor(private targetHash: CID_ALGORITHM_NAMES[]) { + } + + async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise { + // Dynamically determine which hashes are needed + const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + + // If all hashes are already computed, skip the processing + if (neededHashes.length === 0) { + return; + } + + // Compute only the needed CIDs + const cids = await computeCIDs({stream, algorithms:neededHashes}); + + // Map the computed CIDs back to their respective metadata properties + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +} diff --git a/src/lib/ComputeHashIndexCache.ts b/src/lib/hash-compute/HashComputerIndexCache.ts similarity index 85% rename from src/lib/ComputeHashIndexCache.ts rename to src/lib/hash-compute/HashComputerIndexCache.ts index ec71957..c81a7b6 100644 --- a/src/lib/ComputeHashIndexCache.ts +++ b/src/lib/hash-compute/HashComputerIndexCache.ts @@ -1,15 +1,15 @@ -import {HashComputer} from "./HashComputer"; +import {HashComputerWorker} from "./HashComputerWorker"; import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; import {HashIndexManager} from "./HashIndexManager"; import {stat} from "fs/promises"; import path from "path"; -export class ComputeHashIndexCache implements ComputeInterface { +export class HashComputerIndexCache implements ComputeInterface { private hashIndexManager: HashIndexManager; - private hashComputer: HashComputer; + private hashComputer: HashComputerWorker; constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { - this.hashComputer = new HashComputer(targetHash,workerPath); + this.hashComputer = new HashComputerWorker(targetHash,workerPath); this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash); } diff --git a/src/lib/HashComputer.ts b/src/lib/hash-compute/HashComputerWorker.ts similarity index 78% rename from src/lib/HashComputer.ts rename to src/lib/hash-compute/HashComputerWorker.ts index 0c901ee..659658a 100644 --- a/src/lib/HashComputer.ts +++ b/src/lib/hash-compute/HashComputerWorker.ts @@ -1,11 +1,11 @@ -import {FileIDComputer} from "./fileID/FileIDComputer"; +import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker"; import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; -export class HashComputer implements ComputeInterface { - private fileIDComputer: FileIDComputer; +export class HashComputerWorker implements ComputeInterface { + private fileIDComputer: FileIDComputerWorker; constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) { - this.fileIDComputer = new FileIDComputer(workerPath); + this.fileIDComputer = new FileIDComputerWorker(workerPath); } async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { diff --git a/src/lib/HashIndexManager.ts b/src/lib/hash-compute/HashIndexManager.ts similarity index 99% rename from src/lib/HashIndexManager.ts rename to src/lib/hash-compute/HashIndexManager.ts index 49186a1..149a78b 100644 --- a/src/lib/HashIndexManager.ts +++ b/src/lib/hash-compute/HashIndexManager.ts @@ -2,7 +2,7 @@ import {createReadStream, promises as fs} from 'fs'; import {parse} from 'csv-parse'; import {parse as parseSync} from 'csv-parse/sync'; import {stringify} from 'csv-stringify/sync'; -import {existsAsync} from "./ExistsAsync"; +import {existsAsync} from "../utils/ExistsAsync"; import path from "path"; import {clearInterval} from "node:timers"; import {CID_ALGORITHM_NAMES} from "./MultiHashData"; diff --git a/src/lib/MultiHashData.ts b/src/lib/hash-compute/MultiHashData.ts similarity index 100% rename from src/lib/MultiHashData.ts rename to src/lib/hash-compute/MultiHashData.ts diff --git a/src/lib/index-browser.ts b/src/lib/index-browser.ts new file mode 100644 index 0000000..ff303d4 --- /dev/null +++ b/src/lib/index-browser.ts @@ -0,0 +1,4 @@ + +export * from './index-interface.js'; +export * from './file-id/ComputeHash.js'; +export * from './hash-compute/HashComputer.js'; \ No newline at end of file diff --git a/src/lib/index-interface.ts b/src/lib/index-interface.ts new file mode 100644 index 0000000..0008c5f --- /dev/null +++ b/src/lib/index-interface.ts @@ -0,0 +1,3 @@ +export * from './hash-compute/MultiHashData.js'; +export * from './folder-watcher/FileProcessorInterface.js'; +export * from './file-id/FileIDComputerInterface.js'; \ No newline at end of file diff --git a/src/lib/index.ts b/src/lib/index.ts index c6a79dc..50d2a60 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,13 +1,12 @@ -// Import and export directly to simplify access to library components -export * from './ComputeHashIndexCache.js'; -export * from './ExistsAsync.js'; // Assuming 'async' is a function; adjust as necessary -export * from './fileID/FileIDComputer.js'; -export * from './fileID/ShaComputeWorker.js'; // Assuming this exports a class or main function; adjust as needed -export * from './HashComputer.js'; -export * from './HashIndexManager.js'; -// For MultiHashData.ts, assuming interfaces and enums need to be exported; you might need to adjust based on actual content -export * from './MultiHashData.js'; -// Assuming Properties.ts exports one or more interfaces; adjust as needed -export * from './Properties.js'; -export * from './folder_watcher/FolderWatcher.js'; -export * from './folder_watcher/FileProcessorInterface.js'; \ No newline at end of file +//fileID +export * from './file-id/FileIDComputerWorker.js'; +export * from './file-id/ShaComputeWorker.js';//worker + +//hash-compute +export * from './hash-compute/HashComputerIndexCache.js'; +export * from './hash-compute/HashComputerWorker.js'; +export * from './hash-compute/HashIndexManager.js'; + +//folder-watcher +export * from './folder-watcher/FolderWatcher.js'; +export * from './index-interface.js'; \ No newline at end of file diff --git a/src/lib/ExistsAsync.ts b/src/lib/utils/ExistsAsync.ts similarity index 100% rename from src/lib/ExistsAsync.ts rename to src/lib/utils/ExistsAsync.ts diff --git a/src/tests/main.test.ts b/src/tests/main.test.ts index 39a3ab1..944e785 100644 --- a/src/tests/main.test.ts +++ b/src/tests/main.test.ts @@ -1,4 +1,4 @@ -import {CID_ALGORITHM_NAMES, ComputeHashIndexCache, HashComputer} from "../lib"; +import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib"; import {describe, expect, it} from 'vitest'; import * as process from "process"; @@ -13,7 +13,7 @@ process.env.WORKER_URL = "./dist/worker.js"; describe('add', () => { it('meta data compute', async () => { - let hashComputer = new HashComputer([ + let hashComputer = new HashComputerWorker([ CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.md5, @@ -43,7 +43,7 @@ describe('add', () => { it('meta data compute with index', async () => { await rm("./tmp",{recursive:true}); await mkdir("./tmp",{recursive:true}); - let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[ + let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.md5, @@ -73,7 +73,7 @@ describe('add', () => { }); it('cache', async () => { await mkdir("./tmp",{recursive:true}); - let hashComputer = new ComputeHashIndexCache("./tmp/index.csv",[ + let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.md5, diff --git a/tsup.config.js b/tsup.config.js index 514f0b3..42215d6 100644 --- a/tsup.config.js +++ b/tsup.config.js @@ -3,7 +3,9 @@ import {defineConfig} from 'tsup' export default defineConfig({ entry: { 'index': 'src/lib/index.ts', - 'worker': 'src/lib/fileID/ShaComputeWorker.ts', + 'index-interface': 'src/lib/index-interface.ts', + 'index-browser': 'src/lib/index-browser.ts', + 'worker': 'src/lib/file-id/ShaComputeWorker.ts', }, splitting: true, treeshake: true,