From e28f9248371929acbbc0fa247a30c1856045d462 Mon Sep 17 00:00:00 2001 From: Metazla Date: Sat, 6 Apr 2024 11:48:18 +0200 Subject: [PATCH] adding suport for CRC32 SFV --- README.md | 74 +++++++++++ ....d.cts => ShaComputeWorker-DxWHpwX4.d.cts} | 33 +++-- ...aH.d.ts => ShaComputeWorker-DxWHpwX4.d.ts} | 33 +++-- dist/chunk-A5PD5ID4.cjs | 20 +++ dist/chunk-A5PD5ID4.cjs.map | 1 + dist/chunk-DL4KI2PZ.cjs | 15 --- dist/chunk-DL4KI2PZ.cjs.map | 1 - dist/chunk-FGATABDV.js | 11 ++ dist/chunk-FGATABDV.js.map | 1 + dist/chunk-VGBIRFOR.js | 10 -- dist/chunk-VGBIRFOR.js.map | 1 - dist/index.cjs | 32 ++--- dist/index.cjs.map | 2 +- dist/index.d.cts | 13 +- dist/index.d.ts | 13 +- dist/index.js | 12 +- dist/index.js.map | 2 +- dist/worker.cjs | 4 +- dist/worker.d.cts | 3 +- dist/worker.d.ts | 3 +- dist/worker.js | 2 +- package.json | 1 + src/lib/ComputeHashIndexCache.ts | 4 +- src/lib/HashIndexManager.ts | 115 +++++++++++------- src/lib/MultiHashData.ts | 27 ++-- src/lib/fileID/ShaComputeWorker.ts | 51 ++++++-- src/tests/main.test.ts | 31 +++-- yarn.lock | 10 ++ 28 files changed, 353 insertions(+), 172 deletions(-) rename dist/{ShaComputeWorker-DWOF-qaH.d.cts => ShaComputeWorker-DxWHpwX4.d.cts} (70%) rename dist/{ShaComputeWorker-DWOF-qaH.d.ts => ShaComputeWorker-DxWHpwX4.d.ts} (70%) create mode 100644 dist/chunk-A5PD5ID4.cjs create mode 100644 dist/chunk-A5PD5ID4.cjs.map delete mode 100644 dist/chunk-DL4KI2PZ.cjs delete mode 100644 dist/chunk-DL4KI2PZ.cjs.map create mode 100644 dist/chunk-FGATABDV.js create mode 100644 dist/chunk-FGATABDV.js.map delete mode 100644 dist/chunk-VGBIRFOR.js delete mode 100644 dist/chunk-VGBIRFOR.js.map diff --git a/README.md b/README.md index 8b13789..c8b2b64 100644 --- a/README.md +++ b/README.md @@ -1 +1,75 @@ +# Meta-Hash +Meta-Hash is a TypeScript library designed to monitor a specified folder and compute hashes for any new files detected within it. This library leverages an index system to avoid recomputing hashes if the program is restarted, ensuring efficiency and continuity in hash computation. The hashes are computed in the Content Identifier (CID) format, making them compatible with systems like IPFS (InterPlanetary File System). + +## Features + +- **Efficient Hash Computation**: Automatically computes hashes for new files using multiple algorithms, including SHA2-256, SHA1, MD5, SHA3-256 and more. +- **Persistent Indexing**: Utilizes an index system to record file hashes, sizes, and modification times, preventing the need for rehashing on program restarts. +- **CID Format**: Hashes are stored in the CID format, facilitating integration with IPFS and similar distributed file systems. + +## CID Format and Inspection + +The Content Identifier (CID) is a label used to point to material in IPFS. It doesn't indicate where the content is stored, but it forms a kind of address based on the content itself. CIDs are used as a standard way of pointing to pieces of information. A single CID can represent a piece of content that is distributed across multiple systems. + +The list of CID formats used in this project can be found at the following link: +[Multicodecs Table](https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv) + +To inspect the details of a CID, including its version, codec, multihash and more, you can use the CID Inspector tool. This tool allows you to enter a CID and it will break it down into its component parts and provide a detailed explanation of each part. + +[CID Inspector Tool](https://cid.ipfs.tech/) + +## Installation + +To use Meta-Hash in your project, clone this repository and install the required dependencies. + +```bash +#to install +yarn add metazla@git@github.com:Metazla/meta-hash.git + +#to update +yarn up -i @metazla/meta-hash +``` + +## Usage of ComputeHashIndexCache + +To integrate Meta-Hash into your application, follow these steps: + +1. **Import the Library**: Import the necessary components from the library. + +```typescript +import { ComputeHashIndexCache, CID_ALGORITHM_NAMES } from "./folder-watch-hasher"; +``` + +2. **Initialize the Watcher**: Create an instance of `ComputeHashIndexCache`, specifying the path to the folder you want to monitor and the hash algorithms you wish to use. + +```typescript +//note that the csv file will be derived from the path provided into multiple files (one for each algorithm) +const computer = new ComputeHashIndexCache("path/to/your/indexfile/index.csv", [CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.sha1]); +``` + +3. **Compute Hashes**: Invoke the `computeMissingHash` method for each file you wish to hash. This method will automatically check the index for existing hashes and compute missing ones. + +```typescript +let hash:MultiHashData = {}; +await computer.computeMissingHash("path/to/your/file.txt", hash); +console.log(hash[CID_ALGORITHM_NAMES.sha256]); +``` + +## Example of Index File + +An example index file (`index-cid_sha2-256.csv`) is shown below, demonstrating how file paths, sizes, modification times, and computed hashes are stored: + +``` +path,size,mtime,cid_sha2-256 +file.mkv,230265591,2023-01-17T18:53:37.692Z,baejbeif45glg263244x7cbjzyd3ssp5edzae7bxurzxjhnogazcwmj32wa +... +``` + +## Contributing + +Contributions to Meta-Hash are welcome! Please read our contributing guidelines for more information on how to participate. + +## License + +Meta-Hash is released under the [MIT License](LICENSE.md). diff --git a/dist/ShaComputeWorker-DWOF-qaH.d.cts b/dist/ShaComputeWorker-DxWHpwX4.d.cts similarity index 70% rename from dist/ShaComputeWorker-DWOF-qaH.d.cts rename to dist/ShaComputeWorker-DxWHpwX4.d.cts index dce89ee..12d1e21 100644 --- a/dist/ShaComputeWorker-DWOF-qaH.d.cts +++ b/dist/ShaComputeWorker-DxWHpwX4.d.cts @@ -1,35 +1,44 @@ -import { BinaryLike } from 'crypto'; - interface MultiHashData { "cid_sha2-256"?: string; cid_sha1?: string; cid_md5?: string; "cid_sha3-256"?: string; + cid_crc32?: string; } interface ComputeInterface { computeMissingHash(filePath: string, metadata: MultiHashData): Promise; } +/** + * according to + * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv + */ declare enum CID_ALGORITHM_CODES { - sha256 = 18, - sha1 = 17, + crc32 = 306, md5 = 213, - sha3_256 = 22 + sha1 = 17, + sha256 = 18, + sha3_256 = 22, + sha3_384 = 21 } declare enum CID_ALGORITHM_NAMES { - sha256 = "cid_sha2-256", - sha1 = "cid_sha1", + crc32 = "cid_crc32", md5 = "cid_md5", - sha3_256 = "cid_sha3-256" + sha1 = "cid_sha1", + sha256 = "cid_sha2-256", + sha3_256 = "cid_sha3-256", + sha3_384 = "cid_sha3-384" } declare enum CID_ALGORITHM { - sha256 = "sha2-256", - sha1 = "sha1", + crc32 = "crc32", md5 = "md5", - sha3_256 = "sha3-256" + sha1 = "sha1", + sha256 = "sha2-256", + sha3_256 = "sha3-256", + sha3_384 = "sha3-384" } interface SimpleHash { - update(data: BinaryLike): any; + update(data: Buffer): any; digest(): Buffer; } /** diff --git a/dist/ShaComputeWorker-DWOF-qaH.d.ts b/dist/ShaComputeWorker-DxWHpwX4.d.ts similarity index 70% rename from dist/ShaComputeWorker-DWOF-qaH.d.ts rename to dist/ShaComputeWorker-DxWHpwX4.d.ts index dce89ee..12d1e21 100644 --- a/dist/ShaComputeWorker-DWOF-qaH.d.ts +++ b/dist/ShaComputeWorker-DxWHpwX4.d.ts @@ -1,35 +1,44 @@ -import { BinaryLike } from 'crypto'; - interface MultiHashData { "cid_sha2-256"?: string; cid_sha1?: string; cid_md5?: string; "cid_sha3-256"?: string; + cid_crc32?: string; } interface ComputeInterface { computeMissingHash(filePath: string, metadata: MultiHashData): Promise; } +/** + * according to + * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv + */ declare enum CID_ALGORITHM_CODES { - sha256 = 18, - sha1 = 17, + crc32 = 306, md5 = 213, - sha3_256 = 22 + sha1 = 17, + sha256 = 18, + sha3_256 = 22, + sha3_384 = 21 } declare enum CID_ALGORITHM_NAMES { - sha256 = "cid_sha2-256", - sha1 = "cid_sha1", + crc32 = "cid_crc32", md5 = "cid_md5", - sha3_256 = "cid_sha3-256" + sha1 = "cid_sha1", + sha256 = "cid_sha2-256", + sha3_256 = "cid_sha3-256", + sha3_384 = "cid_sha3-384" } declare enum CID_ALGORITHM { - sha256 = "sha2-256", - sha1 = "sha1", + crc32 = "crc32", md5 = "md5", - sha3_256 = "sha3-256" + sha1 = "sha1", + sha256 = "sha2-256", + sha3_256 = "sha3-256", + sha3_384 = "sha3-384" } interface SimpleHash { - update(data: BinaryLike): any; + update(data: Buffer): any; digest(): Buffer; } /** diff --git a/dist/chunk-A5PD5ID4.cjs b/dist/chunk-A5PD5ID4.cjs new file mode 100644 index 0000000..4bd5c5b --- /dev/null +++ b/dist/chunk-A5PD5ID4.cjs @@ -0,0 +1,20 @@ +'use strict'; + +var fs = require('fs'); +var crypto = require('crypto'); +var cid = require('multiformats/cid'); +var digest = require('multiformats/hashes/digest'); +var x = require('crc-32'); + +function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } + +var x__default = /*#__PURE__*/_interopDefault(x); + +var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=fs.createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))} + +exports.a = o; +exports.b = n; +exports.c = u; +exports.d = B; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-A5PD5ID4.cjs.map \ No newline at end of file diff --git a/dist/chunk-A5PD5ID4.cjs.map b/dist/chunk-A5PD5ID4.cjs.map new file mode 100644 index 0000000..b21a764 --- /dev/null +++ b/dist/chunk-A5PD5ID4.cjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCYX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDrBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,QAAQ,KAAK,KAAK,OAAO,GAAG,SAAS,EAAE,CAAC,EACxC,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n console.log((this._crc>>>0).toString(16)) ;\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-DL4KI2PZ.cjs b/dist/chunk-DL4KI2PZ.cjs deleted file mode 100644 index 1a8bb60..0000000 --- a/dist/chunk-DL4KI2PZ.cjs +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -var fs = require('fs'); -var crypto = require('crypto'); -var cid = require('multiformats/cid'); -var digest = require('multiformats/hashes/digest'); - -var i=(a=>(a[a.sha256=18]="sha256",a[a.sha1=17]="sha1",a[a.md5=213]="md5",a[a.sha3_256=22]="sha3_256",a))(i||{}),o=(a=>(a.sha256="cid_sha2-256",a.sha1="cid_sha1",a.md5="cid_md5",a.sha3_256="cid_sha3-256",a))(o||{}),f=(a=>(a.sha256="sha2-256",a.sha1="sha1",a.md5="md5",a.sha3_256="sha3-256",a))(f||{});async function x({filePath:c,algorithms:m}){let r={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")}},h=m.map(s=>({hasher:r[s].hasher,code:r[s].code})),a=fs.createReadStream(c);for await(let s of a)for(let t of h)t.hasher.update(s);return await Promise.all(h.map(async({code:s,hasher:t})=>{let d=t.digest(),n=digest.create(s,d);return cid.CID.createV1(s,n).toString()}))} - -exports.a = i; -exports.b = o; -exports.c = f; -exports.d = x; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-DL4KI2PZ.cjs.map \ No newline at end of file diff --git a/dist/chunk-DL4KI2PZ.cjs.map b/dist/chunk-DL4KI2PZ.cjs.map deleted file mode 100644 index df6769f..0000000 --- a/dist/chunk-DL4KI2PZ.cjs.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAS,oBAAAA,MAAwB,KACjC,OAAoB,cAAAC,MAAiB,SACrC,OAAS,OAAAC,MAAW,mBACpB,OAAS,UAAAC,MAAc,6BCShB,IAAKC,OACRA,IAAA,OAAS,IAAT,SACAA,IAAA,KAAO,IAAP,OACAA,IAAA,IAAM,KAAN,MACAA,IAAA,SAAW,IAAX,WAJQA,OAAA,IAQAC,OACRA,EAAA,OAAS,eACTA,EAAA,KAAO,WACPA,EAAA,IAAM,UACNA,EAAA,SAAW,eAJHA,OAAA,IAQAC,OACRA,EAAA,OAAS,WACTA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,SAAW,WAJHA,OAAA,IDVZ,eAAOC,EAAmC,CAAE,SAAAC,EAAU,WAAAC,CAAW,EAA+E,CAC5I,IAAMC,EACA,CACD,eAA6B,CAAE,QAAkC,OAAQT,EAAW,QAAQ,CAAE,EAC9F,SAA2B,CAAE,QAAgC,OAAQA,EAAW,MAAM,CAAE,EACxF,QAA0B,CAAE,SAA+B,OAAQA,EAAW,KAAK,CAAE,EACrF,eAA+B,CAAE,QAAoC,OAAOA,EAAW,UAAU,CAAE,CACxG,EAEMU,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASb,EAAiBQ,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAK,EAUhC,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAE,KAAAK,EAAM,OAAAC,CAAO,IAAM,CACnE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAAShB,EAAOa,EAAME,CAAU,EACtC,OAAOhB,EAAI,SAASc,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import { createReadStream } from \"fs\";\nimport {BinaryLike, createHash} from 'crypto';\nimport { CID } from 'multiformats/cid';\nimport { create } from \"multiformats/hashes/digest\";\nimport {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: BinaryLike),\n digest(): Buffer;\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },\n [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },\n [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },\n [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\nexport enum CID_ALGORITHM_CODES {\n sha256 = 0x12,\n sha1 = 0x11,\n md5 = 0xd5,\n sha3_256 = 0x16,\n //crc32 = 0x0132\n}\n\nexport enum CID_ALGORITHM_NAMES {\n sha256 = 'cid_sha2-256',\n sha1 = 'cid_sha1',\n md5 = 'cid_md5',\n sha3_256 = 'cid_sha3-256',\n //crc32 = 'cid_crc32'\n}\n\nexport enum CID_ALGORITHM {\n sha256 = 'sha2-256',\n sha1 = 'sha1',\n md5 = 'md5',\n sha3_256 = 'sha3-256',\n //crc32 = 'crc32'\n}"]} \ No newline at end of file diff --git a/dist/chunk-FGATABDV.js b/dist/chunk-FGATABDV.js new file mode 100644 index 0000000..90de17d --- /dev/null +++ b/dist/chunk-FGATABDV.js @@ -0,0 +1,11 @@ +import { createReadStream } from 'fs'; +import { createHash } from 'crypto'; +import { CID } from 'multiformats/cid'; +import { create } from 'multiformats/hashes/digest'; +import x from 'crc-32'; + +var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))} + +export { o as a, n as b, u as c, B as d }; +//# sourceMappingURL=out.js.map +//# sourceMappingURL=chunk-FGATABDV.js.map \ No newline at end of file diff --git a/dist/chunk-FGATABDV.js.map b/dist/chunk-FGATABDV.js.map new file mode 100644 index 0000000..b21a764 --- /dev/null +++ b/dist/chunk-FGATABDV.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","crc32","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","Crc32Hash","data","buffer","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAQ,oBAAAA,MAAuB,KAC/B,OAAQ,cAAAC,MAAiB,SACzB,OAAQ,OAAAC,MAAU,mBAClB,OAAQ,UAAAC,MAAa,6BACrB,OAAOC,MAAW,SCYX,IAAKC,OACRA,IAAA,MAAQ,KAAR,QACAA,IAAA,IAAM,KAAN,MACAA,IAAA,KAAO,IAAP,OACAA,IAAA,OAAS,IAAT,SACAA,IAAA,SAAW,IAAX,WACAA,IAAA,SAAW,IAAX,WANQA,OAAA,IASAC,OACRA,EAAA,MAAQ,YACRA,EAAA,IAAM,UACNA,EAAA,KAAO,WACPA,EAAA,OAAS,eACTA,EAAA,SAAW,eACXA,EAAA,SAAW,eANHA,OAAA,IASAC,OACRA,EAAA,MAAQ,QACRA,EAAA,IAAM,MACNA,EAAA,KAAO,OACPA,EAAA,OAAS,WACTA,EAAA,SAAW,WACXA,EAAA,SAAW,WANHA,OAAA,IDrBZ,IAAMC,EAAN,KAAsC,CAC1B,KAAe,OAEvB,OAAOC,EAAyB,CAE5B,YAAK,KAAOL,EAAM,IAAIK,EAAM,KAAK,IAAI,EAC9B,IACX,CAEA,QAAiB,CACb,QAAQ,KAAK,KAAK,OAAO,GAAG,SAAS,EAAE,CAAC,EACxC,IAAMC,EAAS,OAAO,MAAM,CAAC,EAC7B,OAAAA,EAAO,aAAa,KAAK,KAAM,CAAC,EACzBA,CACX,CACJ,EASA,eAAOC,EAAmC,CAAC,SAAAC,EAAU,WAAAC,CAAU,EAGzC,CAClB,IAAMC,EACA,CACD,eAA6B,CAAC,QAAkC,OAAQb,EAAW,QAAQ,CAAC,EAC5F,SAA2B,CAAC,QAAgC,OAAQA,EAAW,MAAM,CAAC,EACtF,QAA0B,CAAC,SAA+B,OAAQA,EAAW,KAAK,CAAC,EACnF,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,eAA+B,CAAC,QAAoC,OAAQA,EAAW,UAAU,CAAC,EAClG,UAA4B,CAAC,SAAiC,OAAQ,IAAIO,CAAW,CAE1F,EAEMO,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASjB,EAAiBY,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAe,EAU1C,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAC,KAAAK,EAAM,OAAAC,CAAM,IAAM,CACjE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAASpB,EAAOiB,EAAME,CAAU,EACtC,OAAOpB,EAAI,SAASkB,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import {createReadStream} from \"fs\";\nimport {createHash} from 'crypto';\nimport {CID} from 'multiformats/cid';\nimport {create} from \"multiformats/hashes/digest\";\nimport crc32 from 'crc-32';\nimport {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: Buffer),\n\n digest(): Buffer;\n}\n\nclass Crc32Hash implements SimpleHash {\n private _crc: number = undefined;\n\n update(data: Buffer): Crc32Hash {\n // Update the CRC-32 checksum with the new chunk of data\n this._crc = crc32.buf(data, this._crc);\n return this;\n }\n\n digest(): Buffer {\n console.log((this._crc>>>0).toString(16)) ;\n const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)\n buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format\n return buffer;\n }\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},\n [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},\n [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},\n [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},\n [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},\n [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}\n\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk as Buffer);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({code, hasher}) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]} \ No newline at end of file diff --git a/dist/chunk-VGBIRFOR.js b/dist/chunk-VGBIRFOR.js deleted file mode 100644 index cc3826f..0000000 --- a/dist/chunk-VGBIRFOR.js +++ /dev/null @@ -1,10 +0,0 @@ -import { createReadStream } from 'fs'; -import { createHash } from 'crypto'; -import { CID } from 'multiformats/cid'; -import { create } from 'multiformats/hashes/digest'; - -var i=(a=>(a[a.sha256=18]="sha256",a[a.sha1=17]="sha1",a[a.md5=213]="md5",a[a.sha3_256=22]="sha3_256",a))(i||{}),o=(a=>(a.sha256="cid_sha2-256",a.sha1="cid_sha1",a.md5="cid_md5",a.sha3_256="cid_sha3-256",a))(o||{}),f=(a=>(a.sha256="sha2-256",a.sha1="sha1",a.md5="md5",a.sha3_256="sha3-256",a))(f||{});async function x({filePath:c,algorithms:m}){let r={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")}},h=m.map(s=>({hasher:r[s].hasher,code:r[s].code})),a=createReadStream(c);for await(let s of a)for(let t of h)t.hasher.update(s);return await Promise.all(h.map(async({code:s,hasher:t})=>{let d=t.digest(),n=create(s,d);return CID.createV1(s,n).toString()}))} - -export { i as a, o as b, f as c, x as d }; -//# sourceMappingURL=out.js.map -//# sourceMappingURL=chunk-VGBIRFOR.js.map \ No newline at end of file diff --git a/dist/chunk-VGBIRFOR.js.map b/dist/chunk-VGBIRFOR.js.map deleted file mode 100644 index df6769f..0000000 --- a/dist/chunk-VGBIRFOR.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sources":["../src/lib/fileID/ShaComputeWorker.ts","../src/lib/MultiHashData.ts"],"names":["createReadStream","createHash","CID","create","CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM","computeCIDs","filePath","algorithms","hashFunctions","hashers","algo","stream","chunk","item","code","hasher","hashBuffer","digest"],"mappings":"AAAA,OAAS,oBAAAA,MAAwB,KACjC,OAAoB,cAAAC,MAAiB,SACrC,OAAS,OAAAC,MAAW,mBACpB,OAAS,UAAAC,MAAc,6BCShB,IAAKC,OACRA,IAAA,OAAS,IAAT,SACAA,IAAA,KAAO,IAAP,OACAA,IAAA,IAAM,KAAN,MACAA,IAAA,SAAW,IAAX,WAJQA,OAAA,IAQAC,OACRA,EAAA,OAAS,eACTA,EAAA,KAAO,WACPA,EAAA,IAAM,UACNA,EAAA,SAAW,eAJHA,OAAA,IAQAC,OACRA,EAAA,OAAS,WACTA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,SAAW,WAJHA,OAAA,IDVZ,eAAOC,EAAmC,CAAE,SAAAC,EAAU,WAAAC,CAAW,EAA+E,CAC5I,IAAMC,EACA,CACD,eAA6B,CAAE,QAAkC,OAAQT,EAAW,QAAQ,CAAE,EAC9F,SAA2B,CAAE,QAAgC,OAAQA,EAAW,MAAM,CAAE,EACxF,QAA0B,CAAE,SAA+B,OAAQA,EAAW,KAAK,CAAE,EACrF,eAA+B,CAAE,QAAoC,OAAOA,EAAW,UAAU,CAAE,CACxG,EAEMU,EAAUF,EAAW,IAAIG,IAAS,CACpC,OAAQF,EAAcE,CAAI,EAAE,OAC5B,KAAMF,EAAcE,CAAI,EAAE,IAC9B,EAAE,EAEIC,EAASb,EAAiBQ,CAAQ,EACxC,cAAiBM,KAASD,EACtB,QAAWE,KAAQJ,EACfI,EAAK,OAAO,OAAOD,CAAK,EAUhC,OANa,MAAM,QAAQ,IAAIH,EAAQ,IAAI,MAAO,CAAE,KAAAK,EAAM,OAAAC,CAAO,IAAM,CACnE,IAAMC,EAAaD,EAAO,OAAO,EAC3BE,EAAShB,EAAOa,EAAME,CAAU,EACtC,OAAOhB,EAAI,SAASc,EAAMG,CAAM,EAAE,SAAS,CAC/C,CAAC,CAAC,CAGN","sourcesContent":["import { createReadStream } from \"fs\";\nimport {BinaryLike, createHash} from 'crypto';\nimport { CID } from 'multiformats/cid';\nimport { create } from \"multiformats/hashes/digest\";\nimport {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\nexport interface SimpleHash {\n update(data: BinaryLike),\n digest(): Buffer;\n}\n\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise {\n const hashFunctions: Record\n = {\n [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },\n [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },\n [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },\n [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }\n };\n\n const hashers = algorithms.map(algo => ({\n hasher: hashFunctions[algo].hasher,\n code: hashFunctions[algo].code\n }));\n\n const stream = createReadStream(filePath);\n for await (const chunk of stream) {\n for (const item of hashers) {\n item.hasher.update(chunk);\n }\n }\n\n const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {\n const hashBuffer = hasher.digest();\n const digest = create(code, hashBuffer);\n return CID.createV1(code, digest).toString();\n }));\n\n return cids;\n}\n","export interface MultiHashData {\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n cid_sha1?: string; // 0x11 sha1\n cid_md5?: string; // 0xd5 md5\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise;\n}\n\nexport enum CID_ALGORITHM_CODES {\n sha256 = 0x12,\n sha1 = 0x11,\n md5 = 0xd5,\n sha3_256 = 0x16,\n //crc32 = 0x0132\n}\n\nexport enum CID_ALGORITHM_NAMES {\n sha256 = 'cid_sha2-256',\n sha1 = 'cid_sha1',\n md5 = 'cid_md5',\n sha3_256 = 'cid_sha3-256',\n //crc32 = 'cid_crc32'\n}\n\nexport enum CID_ALGORITHM {\n sha256 = 'sha2-256',\n sha1 = 'sha1',\n md5 = 'md5',\n sha3_256 = 'sha3-256',\n //crc32 = 'crc32'\n}"]} \ No newline at end of file diff --git a/dist/index.cjs b/dist/index.cjs index 8b5f764..33b65ca 100644 --- a/dist/index.cjs +++ b/dist/index.cjs @@ -1,15 +1,15 @@ 'use strict'; -var chunkDL4KI2PZ_cjs = require('./chunk-DL4KI2PZ.cjs'); +var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs'); var piscina = require('piscina'); var fs = require('fs'); var csvParse = require('csv-parse'); var sync = require('csv-parse/sync'); var sync$1 = require('csv-stringify/sync'); var promises = require('fs/promises'); -var n = require('path'); +var c = require('path'); var timers = require('timers'); -var w = require('chokidar'); +var x = require('chokidar'); var b = require('p-queue'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } @@ -32,30 +32,30 @@ function _interopNamespace(e) { return Object.freeze(n); } -var n__namespace = /*#__PURE__*/_interopNamespace(n); -var w__default = /*#__PURE__*/_interopDefault(w); +var c__namespace = /*#__PURE__*/_interopNamespace(c); +var x__default = /*#__PURE__*/_interopDefault(x); var b__default = /*#__PURE__*/_interopDefault(b); -var h=undefined;h=h.replace("src","dist");h=h+"/worker.js";var f=new URL(h).href;console.log("Worker URL: ",f);var l=class{piscina;constructor(e){this.piscina=new piscina.Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||f});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var p=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new l(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(i=>!t[i]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((i,a)=>{let o=r[a];t[o]=i;});}};async function d(c){try{return await promises.access(c,promises.constants.F_OK),!0}catch{return !1}}var Q=["path","size","mtime"],m=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;if(!e)throw new Error("Invalid index file path")}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");await this.loadIndex(),e&&this.start(),t();}catch(s){r(s);}})),this.initialLoad}checkCSVHeaders(e){let t=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let r=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(i=>r.includes(i))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(){if(await d(this.filePath)){let e=await fs.promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=await this.readCsv();for(let r of t)this.cache.set(r.path,r);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(){if(!await d(this.filePath))return [];let e=performance.now(),t=csvParse.parse({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,i)=>{fs.createReadStream(this.filePath).pipe(t).on("data",a=>{r.push(a);}).on("end",()=>{s(r),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",a=>{i(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=await this.loadIndex();if(this.cache.size!==0){let i=this.loadIndexFromCache().filter(a=>!t.find(o=>o.path===a.path));if(i.length!==0){let a=sync$1.stringify(i,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(o=>({key:o,header:o}))]});await fs.promises.appendFile(this.filePath,a);}}let r=performance.now()-e;console.log(`Index saved in ${r}ms`),r*10>this.intervalTime&&(this.startAutoSave(r*10),console.log(`Index save interval increased to ${r*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let s=n__namespace.default.basename(e),i=this.cache.get(s);if(i){if(i.mtime){if(i.size===t+""&&i.mtime===r)return i}else if(i.size===t+"")return i}return null}addFileCid(e,t,r,s){if(!e||!t||!r||!s)throw new Error("Invalid parameters");let i=t+"",a=n__namespace.default.basename(e),o={path:a,size:i,mtime:r,...s};this.cache.set(a,o),this.hasChanged=!0;}};var v=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new p(this.targetHash,r),this.hashIndexManager=new m(e);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(n__namespace.default.basename(e))){let s=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(s)for(let i of this.targetHash)!t[i]&&s[i]&&(t[i]=s[i]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var y=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let s of r){let i=n__namespace.join(e,s.name);if(s.isDirectory())await this.processDirectory(i,t);else {let a=this.processFile(i);t.push(a);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let i of t){let a=n__namespace.join(e,i.name);i.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((i,a)=>i+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=n__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w__default.default.watch(e,t):r=w__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async s=>{try{await this.processFileExtended(s);}catch(i){console.error(`Error processing file ${s}:`,i);}}).on("change",async s=>{try{await this.processFileExtended(s);}catch(i){console.error(`Error processing file ${s}:`,i);}}).on("unlink",async s=>{try{await this.fileProcessor.deleteFile(s),await this.processFileExtended(s);}catch(i){console.error("Error processing file deletion:",i);}}).on("error",s=>console.error(`Watcher error: ${s}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+o,0);console.log(`Found ${s} files to process`),this.queueSize=s;let i=[];for(let a=0;a!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,o)=>{let a=r[o];t[a]=s;});}};async function f(h){try{return await promises.access(h,promises.constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=sync.parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(o=>i.includes(o))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&timers.clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await fs.promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=csvParse.parse({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,o)=>{fs.createReadStream(this.filePaths[e]).pipe(r).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",a=>{o(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let o=this.loadIndexFromCache().filter(a=>!i.find(n=>n.path===a.path));if(o.length!==0){let a=sync$1.stringify(o,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await fs.promises.appendFile(this.filePaths[r],a);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let i=c__namespace.default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let n of this.targetHash)if(!i[n])throw new Error(`Missing hash ${n}`);let s=t+"",o=c__namespace.default.basename(e),a=this.cache.get(o);if(a)for(let n of this.targetHash)a[n]=i[n];else {let n={path:o,size:s,mtime:r,...i};this.cache.set(o,n);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await promises.stat(e);if(this.hashIndexManager.getCache().has(c__namespace.default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b__default.default({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await promises.readdir(e,{withFileTypes:!0});for(let i of r){let s=c__namespace.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let o=this.processFile(s);t.push(o);}}}async countFile(e){let t=await promises.readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let o=c__namespace.join(e,s.name);s.isDirectory()?r.push(this.countFile(o)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(o)?1:0));}return (await Promise.all(r)).reduce((s,o)=>s+o,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c__namespace.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=x__default.default.watch(e,t):r=x__default.default.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let o=0;oo+a,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let o=0;o {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad;\n\n constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {\n if (!filePath) {\n throw new Error('Invalid index file path');\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n if (!this.checkCSVHeaders(this.filePath)) {\n throw new Error('Invalid index file headers');\n }\n await this.loadIndex();\n if (autosave) {\n this.start();\n }\n resolve();\n }catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n checkCSVHeaders (csvContent: string): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(): Promise {\n if (await existsAsync(this.filePath)) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePath);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv();\n for (const record of records) {\n this.cache.set(record.path, record);\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(): Promise {\n if (!(await existsAsync(this.filePath))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePath)\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n let existingRows: IndexLine[] = await this.loadIndex();\n\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n ...this.targetHash.map(hash => ({key: hash, header: hash})),\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePath, csvString);\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if(fileNameIndex) {\n if(fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n }else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(this.targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileSize","mtime","fileName","fileNameIndex","hashs","size","baseName","data","stat","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"uDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA8C,CAGjD,YAAoBC,EAAkCJ,EAAoB,CAAtD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYL,EAAUK,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,CACJ,EC5BA,OAAQ,oBAAAG,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYjB,EAAoC,CAClE,GAAI,CACA,aAAMe,EAAOf,EAAUgB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,SASrB,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYrB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWsB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGtB,CAAQ,IAAIsB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUf,EAAUc,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLX,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAcY,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAML,EAAY,KAAK,UAAUK,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMrB,EAAG,KAAK,KAAK,UAAUW,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAML,EAAY,KAAK,UAAUK,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASxB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKgB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpCf,EAAiB,KAAK,UAAUY,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAc,iBAAiC,CAC3C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY5B,EAAUyB,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMX,EAAG,WAAW,KAAK,UAAUW,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEO,cAAc3C,EAAkB4C,EAAkBC,EAA0B,CAC/E,IAAMC,EAAW5B,EAAK,SAASlB,CAAQ,EACnC+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUH,EAAW,IAAOG,EAAc,QAAUF,EAClE,OAAOE,UAIPA,EAAc,OAAUH,EAAW,GACnC,OAAOG,EAWnB,OAAO,IACX,CAEO,WAAW/C,EAAkB4C,EAAkBC,EAAeG,EAA2D,CAC5H,GAAI,CAAChD,GAAY,CAAC4C,GAAY,CAACC,GAAS,CAACG,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOL,EAAW,GAClBM,EAAWhC,EAAK,SAASlB,CAAQ,EACnCkC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAKD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MANpB,CACZ,IAAM6B,EAAO,CAAC,KAAMD,EAAU,KAAMD,EAAM,MAAOJ,EAAO,GAAGG,CAAK,EAChE,KAAK,MAAM,IAAIE,EAAUC,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EE9PA,OAAQ,QAAAC,MAAW,cACnB,OAAOlC,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BnD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIsB,EAAiBiC,EAAcnD,CAAU,CACzE,CANA,iBACA,aAOA,MAAM,mBAAmBH,EAAkBI,EAAwC,CAC/E,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI4B,EAAQ,MAAMoB,EAAKpD,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIkB,EAAK,SAASlB,CAAQ,CAAC,EAAG,CAC/D,IAAMkC,EAAY,KAAK,iBAAiB,cAAclC,EAAUgC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAAClB,EAASkB,CAAI,GAAKY,EAAUZ,CAAI,IACjClB,EAASkB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBtB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUgC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG5B,CAAQ,CAC9F,CACJ,EC/BA,UAAYc,MAAU,OACtB,OAAOqC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYtE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMsE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWtE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIuE,EAAe,UAAQvE,CAAQ,EACnC,MAAM,KAAK,iBAAiBuE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO1E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2E,EAAO,CACZ,QAAQ,MAAM,yBAAyB3E,CAAQ,IAAK2E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS0B,EAAG,CACR,QAAQ,MAAM,yBAAyB1B,CAAQ,IAAK0B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO1B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS0B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUiD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/index.d.cts b/dist/index.d.cts index 40ffe02..88aae43 100644 --- a/dist/index.d.cts +++ b/dist/index.d.cts @@ -1,9 +1,8 @@ -import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DWOF-qaH.cjs'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DWOF-qaH.cjs'; +import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.cjs'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.cjs'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; -import 'crypto'; declare class HashComputer implements ComputeInterface { private targetHash; @@ -19,7 +18,6 @@ interface IndexLine extends Partial> { } declare const INDEX_HEADERS: string[]; declare class HashIndexManager { - private filePath; private targetHash; private cache; private intervalId; @@ -29,18 +27,19 @@ declare class HashIndexManager { private indexOpsInProgress; private hasChanged; private initialLoad; + private filePaths; constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]); getCache(): Map; /** * After init consseutively calls to this method will not reload the index * @param autosave */ - init(autosave?: boolean): Promise; - checkCSVHeaders(csvContent: string): boolean; + init(autosave?: boolean): Promise; + private checkCSVHeaders; start(): void; stopAutoSave(): void; private startAutoSave; - loadIndex(): Promise; + loadIndex(hash: CID_ALGORITHM_NAMES): Promise; private loadIndexFromCache; private readCsv; private saveCacheToFile; diff --git a/dist/index.d.ts b/dist/index.d.ts index 940af05..0c9237f 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,9 +1,8 @@ -import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DWOF-qaH.js'; -export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DWOF-qaH.js'; +import { C as ComputeInterface, a as CID_ALGORITHM_NAMES, M as MultiHashData } from './ShaComputeWorker-DxWHpwX4.js'; +export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-DxWHpwX4.js'; import * as PQueue from 'p-queue'; import PQueue__default from 'p-queue'; import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js'; -import 'crypto'; declare class HashComputer implements ComputeInterface { private targetHash; @@ -19,7 +18,6 @@ interface IndexLine extends Partial> { } declare const INDEX_HEADERS: string[]; declare class HashIndexManager { - private filePath; private targetHash; private cache; private intervalId; @@ -29,18 +27,19 @@ declare class HashIndexManager { private indexOpsInProgress; private hasChanged; private initialLoad; + private filePaths; constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]); getCache(): Map; /** * After init consseutively calls to this method will not reload the index * @param autosave */ - init(autosave?: boolean): Promise; - checkCSVHeaders(csvContent: string): boolean; + init(autosave?: boolean): Promise; + private checkCSVHeaders; start(): void; stopAutoSave(): void; private startAutoSave; - loadIndex(): Promise; + loadIndex(hash: CID_ALGORITHM_NAMES): Promise; private loadIndexFromCache; private readCsv; private saveCacheToFile; diff --git a/dist/index.js b/dist/index.js index 6e75969..e7bd031 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,18 +1,18 @@ -export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-VGBIRFOR.js'; +export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-FGATABDV.js'; import { Piscina } from 'piscina'; import { promises, createReadStream } from 'fs'; import { parse as parse$1 } from 'csv-parse'; import { parse } from 'csv-parse/sync'; import { stringify } from 'csv-stringify/sync'; import { access, constants, stat, readdir } from 'fs/promises'; -import * as n from 'path'; -import n__default from 'path'; +import * as c from 'path'; +import c__default from 'path'; import { clearInterval } from 'node:timers'; -import w from 'chokidar'; +import x from 'chokidar'; import b from 'p-queue'; -var h=import.meta.dirname;h=h.replace("src","dist");h=h+"/worker.js";var f=new URL(h).href;console.log("Worker URL: ",f);var l=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||f});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var p=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new l(t);}fileIDComputer;async computeMissingHash(e,t){let r=this.targetHash.filter(i=>!t[i]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((i,a)=>{let o=r[a];t[o]=i;});}};async function d(c){try{return await access(c,constants.F_OK),!0}catch{return !1}}var Q=["path","size","mtime"],m=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;if(!e)throw new Error("Invalid index file path")}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");await this.loadIndex(),e&&this.start(),t();}catch(s){r(s);}})),this.initialLoad}checkCSVHeaders(e){let t=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let r=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(i=>r.includes(i))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(){if(await d(this.filePath)){let e=await promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=await this.readCsv();for(let r of t)this.cache.set(r.path,r);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(){if(!await d(this.filePath))return [];let e=performance.now(),t=parse$1({columns:!0,skip_empty_lines:!0}),r=[];return new Promise((s,i)=>{createReadStream(this.filePath).pipe(t).on("data",a=>{r.push(a);}).on("end",()=>{s(r),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",a=>{i(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=await this.loadIndex();if(this.cache.size!==0){let i=this.loadIndexFromCache().filter(a=>!t.find(o=>o.path===a.path));if(i.length!==0){let a=stringify(i,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(o=>({key:o,header:o}))]});await promises.appendFile(this.filePath,a);}}let r=performance.now()-e;console.log(`Index saved in ${r}ms`),r*10>this.intervalTime&&(this.startAutoSave(r*10),console.log(`Index save interval increased to ${r*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let s=n__default.basename(e),i=this.cache.get(s);if(i){if(i.mtime){if(i.size===t+""&&i.mtime===r)return i}else if(i.size===t+"")return i}return null}addFileCid(e,t,r,s){if(!e||!t||!r||!s)throw new Error("Invalid parameters");let i=t+"",a=n__default.basename(e),o={path:a,size:i,mtime:r,...s};this.cache.set(a,o),this.hasChanged=!0;}};var v=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new p(this.targetHash,r),this.hashIndexManager=new m(e);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(n__default.basename(e))){let s=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(s)for(let i of this.targetHash)!t[i]&&s[i]&&(t[i]=s[i]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var y=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let s of r){let i=n.join(e,s.name);if(s.isDirectory())await this.processDirectory(i,t);else {let a=this.processFile(i);t.push(a);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let i of t){let a=n.join(e,i.name);i.isDirectory()?r.push(this.countFile(a)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(a)?1:0));}return (await Promise.all(r)).reduce((i,a)=>i+a,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=n.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=w.watch(e,t):r=w.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async s=>{try{await this.processFileExtended(s);}catch(i){console.error(`Error processing file ${s}:`,i);}}).on("change",async s=>{try{await this.processFileExtended(s);}catch(i){console.error(`Error processing file ${s}:`,i);}}).on("unlink",async s=>{try{await this.fileProcessor.deleteFile(s),await this.processFileExtended(s);}catch(i){console.error("Error processing file deletion:",i);}}).on("error",s=>console.error(`Watcher error: ${s}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let a=0;aa+o,0);console.log(`Found ${s} files to process`),this.queueSize=s;let i=[];for(let a=0;a!t[s]);if(r.length===0)return;(await this.fileIDComputer.computeCIDs(e,r)).forEach((s,o)=>{let a=r[o];t[a]=s;});}};async function f(h){try{return await access(h,constants.F_OK),!0}catch{return !1}}var D=["path","size","mtime"],d=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;e=e.replace(".csv","");for(let r of this.targetHash)if(this.filePaths[r]=`${e}-${r}.csv`,!this.filePaths[r])throw new Error(`Invalid index file path for ${r}`)}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;filePaths={};getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,r)=>{try{for(let i of this.targetHash){if(!this.checkCSVHeaders(this.filePaths[i],i))throw new Error(`Invalid index file headers for ${i}`);await this.loadIndex(i);}e&&this.start(),t();}catch(i){r(i);}})),this.initialLoad}checkCSVHeaders(e,t){let r=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(r.length===0)return !0;let i=Object.keys(r[0]);return [...D,t].every(o=>i.includes(o))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(e){if(await f(this.filePaths[e])){let t=await promises.stat(this.filePaths[e]);if(this.lastIndexFileSize!==t.size){let r=await this.readCsv(e);for(let i of r){let s=this.cache.get(i.path);s?s[e]=i[e]:this.cache.set(i.path,i);}return this.lastIndexFileSize=t.size,this.lastCacheFile=r,r}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(e){if(!await f(this.filePaths[e]))return [];let t=performance.now(),r=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,o)=>{createReadStream(this.filePaths[e]).pipe(r).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read ${e} time ${performance.now()-t}ms`);}).on("error",a=>{o(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now();for(let r of this.targetHash){let i=await this.loadIndex(r);if(this.cache.size!==0){let o=this.loadIndexFromCache().filter(a=>!i.find(n=>n.path===a.path));if(o.length!==0){let a=stringify(o,{header:i.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},{key:r,header:r}]});await promises.appendFile(this.filePaths[r],a);}}}let t=performance.now()-e;console.log(`Index saved in ${t}ms`),t*10>this.intervalTime&&(this.startAutoSave(t*10),console.log(`Index save interval increased to ${t*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,r){let i=c__default.basename(e),s=this.cache.get(i);if(s){if(s.mtime){if(s.size===t+""&&s.mtime===r)return s}else if(s.size===t+"")return s}return null}addFileCid(e,t,r,i){if(!e||!t||!r||!i)throw new Error("Invalid parameters");for(let n of this.targetHash)if(!i[n])throw new Error(`Missing hash ${n}`);let s=t+"",o=c__default.basename(e),a=this.cache.get(o);if(a)for(let n of this.targetHash)a[n]=i[n];else {let n={path:o,size:s,mtime:r,...i};this.cache.set(o,n);}this.hasChanged=!0;}};var w=class{constructor(e,t=["cid_sha1","cid_sha2-256"],r){this.targetHash=t;this.hashComputer=new m(t,r),this.hashIndexManager=new d(e,t);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let r=await stat(e);if(this.hashIndexManager.getCache().has(c__default.basename(e))){let i=this.hashIndexManager.getCidForFile(e,r.size,r.mtime.toISOString());if(i)for(let s of this.targetHash)!t[s]&&i[s]&&(t[s]=i[s]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,r.size,r.mtime.toISOString(),t);}};var C=class{constructor(e,t,r){this.WATCH_FOLDER_LIST=t;this.config=r;this.fileProcessor=e;}initialized=!1;queue=new b({concurrency:100,autoStart:!0});fileProcessor;queueSize=-1;current=0;processing=new Set;async processDirectory(e,t){let r=await readdir(e,{withFileTypes:!0});for(let i of r){let s=c.join(e,i.name);if(i.isDirectory())await this.processDirectory(s,t);else {let o=this.processFile(s);t.push(o);}}}async countFile(e){let t=await readdir(e,{withFileTypes:!0}),r=[];for(let s of t){let o=c.join(e,s.name);s.isDirectory()?r.push(this.countFile(o)):r.push(this.queue.add(async()=>await this.fileProcessor.canProcessFile(o)?1:0));}return (await Promise.all(r)).reduce((s,o)=>s+o,0)}async processFile(e){if(await this.fileProcessor.canProcessFile(e)&&!this.processing.has(e)){this.processing.add(e);let t=++this.current;await this.queue.add(()=>this.fileProcessor.processFile(t,this.queueSize,e)),this.processing.delete(e);}}async processFileExtended(e){let t=c.dirname(e);await this.processDirectory(t,[]);}chokidarWatch(e){let t={ignoreInitial:!0,persistent:!0,depth:1/0,awaitWriteFinish:{stabilityThreshold:this.config.stabilityThreshold||3e4,pollInterval:this.config.pollInterval||5e3}},r;(this.config.interval||0)<=0?r=x.watch(e,t):r=x.watch(e,{...t,usePolling:!0,interval:this.config.interval||0}),r.on("add",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("change",async i=>{try{await this.processFileExtended(i);}catch(s){console.error(`Error processing file ${i}:`,s);}}).on("unlink",async i=>{try{await this.fileProcessor.deleteFile(i),await this.processFileExtended(i);}catch(s){console.error("Error processing file deletion:",s);}}).on("error",i=>console.error(`Watcher error: ${i}`)).on("ready",()=>{console.log(`Watching for file changes on ${e}`);});}async watch(){let e=this.WATCH_FOLDER_LIST;if(!e)throw new Error("No folder to watch");let t=e.split(",");this.chokidarWatch(t);let r=[];for(let o=0;oo+a,0);console.log(`Found ${i} files to process`),this.queueSize=i;let s=[];for(let o=0;o {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad;\n\n constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {\n if (!filePath) {\n throw new Error('Invalid index file path');\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n if (!this.checkCSVHeaders(this.filePath)) {\n throw new Error('Invalid index file headers');\n }\n await this.loadIndex();\n if (autosave) {\n this.start();\n }\n resolve();\n }catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n checkCSVHeaders (csvContent: string): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(): Promise {\n if (await existsAsync(this.filePath)) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePath);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv();\n for (const record of records) {\n this.cache.set(record.path, record);\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(): Promise {\n if (!(await existsAsync(this.filePath))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePath)\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n let existingRows: IndexLine[] = await this.loadIndex();\n\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n ...this.targetHash.map(hash => ({key: hash, header: hash})),\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePath, csvString);\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if(fileNameIndex) {\n if(fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n }else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(this.targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file +{"version":3,"sources":["../src/lib/fileID/FileIDComputer.ts","../src/lib/HashComputer.ts","../src/lib/HashIndexManager.ts","../src/lib/ExistsAsync.ts","../src/lib/ComputeHashIndexCache.ts","../src/lib/folder_watcher/FolderWatcher.ts"],"names":["Piscina","distFolder","workerUrl","FileIDComputer","workerPath","filePath","algorithms","HashComputer","targetHash","metadata","neededHashes","hashName","cid","index","hashType","createReadStream","fs","parse","parseSync","stringify","access","constants","existsAsync","path","clearInterval","INDEX_HEADERS","HashIndexManager","hash","autosave","resolve","reject","e","csvContent","records","headers","header","time","stats","record","indexLine","start","parser","err","existingRows","newRows","row","existingRow","csvString","totalTime","fileSize","mtime","fileName","fileNameIndex","hashs","size","baseName","data","stat","ComputeHashIndexCache","indexFilePath","chokidar","readdir","PQueue","FolderWatcher","fileProcessor","WATCH_FOLDER_LIST","config","directory","promises","entries","entry","fullPath","promise","countPromises","acc","current","dirname","folderList","chokidarconfig","watcher","error","folders","i","count"],"mappings":"sDAAA,OAAQ,WAAAA,MAAc,UAItB,IAAIC,EAAa,YAAY,QAC7BA,EAAaA,EAAW,QAAQ,MAAO,MAAM,EAC7CA,EAAaA,EAAa,aAC1B,IAAMC,EAAY,IAAI,IAAID,CAAU,EAAE,KACtC,QAAQ,IAAI,eAAgBC,CAAS,EAE9B,IAAMC,EAAN,KAAqB,CAChB,QAER,YAAYC,EAAoB,CAC5B,KAAK,QAAU,IAAIJ,EAAQ,CACvB,WAAY,EAEZ,SAAUI,GAAc,QAAQ,IAAI,YAAcF,CACtD,CAAC,CACL,CAQA,MAAa,YAAYG,EAAkBC,EAAsD,CAC7F,OAAO,KAAK,QAAQ,IAAI,CAAC,SAAAD,EAAU,WAAAC,CAAU,CAAC,CAClD,CACJ,EC3BO,IAAMC,EAAN,KAA8C,CAGjD,YAAoBC,EAAkCJ,EAAoB,CAAtD,gBAAAI,EAChB,KAAK,eAAiB,IAAIL,EAAeC,CAAU,CACvD,CAJQ,eAMR,MAAM,mBAAmBC,EAAkBI,EAAwC,CAE/E,IAAMC,EAAe,KAAK,WAAW,OAAOC,GAAY,CAACF,EAASE,CAAQ,CAAC,EAG3E,GAAID,EAAa,SAAW,EACxB,QAIS,MAAM,KAAK,eAAe,YAAYL,EAAUK,CAAY,GAGpE,QAAQ,CAACE,EAAKC,IAAU,CACzB,IAAMC,EAAWJ,EAAaG,CAAK,EACnCJ,EAASK,CAAQ,EAAIF,CACzB,CAAC,CACL,CACJ,EC5BA,OAAQ,oBAAAG,EAAkB,YAAYC,MAAS,KAC/C,OAAQ,SAAAC,MAAY,YACpB,OAAQ,SAASC,MAAgB,iBACjC,OAAQ,aAAAC,MAAgB,qBCDxB,OAAQ,UAAAC,EAAO,aAAAC,MAAgB,cAE/B,eAAsBC,EAAYjB,EAAoC,CAClE,GAAI,CACA,aAAMe,EAAOf,EAAUgB,EAAU,IAAI,EAC9B,EACX,MAAQ,CACJ,MAAO,EACX,CACJ,CDNA,OAAOE,MAAU,OACjB,OAAQ,iBAAAC,MAAoB,cASrB,IAAMC,EAAgB,CAAC,OAAQ,OAAQ,OAAO,EAExCC,EAAN,KAAuB,CAW1B,YAAYrB,EAA0BG,EAAoC,0BAAqD,EAAG,CAA5F,gBAAAA,EAClCH,EAAWA,EAAS,QAAQ,OAAQ,EAAE,EACtC,QAAWsB,KAAQ,KAAK,WAEpB,GADA,KAAK,UAAUA,CAAI,EAAI,GAAGtB,CAAQ,IAAIsB,CAAI,OACtC,CAAC,KAAK,UAAUA,CAAI,EACpB,MAAM,IAAI,MAAM,+BAA+BA,CAAI,EAAE,CAGjE,CAlBQ,MAAgC,IAAI,IACpC,WACA,aAAuB,IACvB,kBAA4B,EAC5B,cACA,mBAA8B,GAC9B,WAAsB,GACtB,YACA,UAAuD,CAAC,EAYzD,UAAmC,CACtC,OAAO,IAAI,IAAI,KAAK,KAAK,CAC7B,CAMA,MAAa,KAAKC,EAAW,GAAM,CAC/B,OAAK,KAAK,cACN,KAAK,YAAc,IAAI,QAAc,MAAOC,EAASC,IAAW,CAC5D,GAAI,CACA,QAAWH,KAAQ,KAAK,WAAY,CAChC,GAAI,CAAC,KAAK,gBAAgB,KAAK,UAAUA,CAAI,EAAGA,CAAI,EAChD,MAAM,IAAI,MAAM,kCAAkCA,CAAI,EAAE,EAE5D,MAAM,KAAK,UAAUA,CAAI,CAC7B,CACIC,GACA,KAAK,MAAM,EAEfC,EAAQ,CACZ,OAASE,EAAG,CACRD,EAAOC,CAAC,CACZ,CACJ,CAAC,GAEE,KAAK,WAChB,CAGQ,gBAAgBC,EAAoBL,EAAoC,CAC5E,IAAMM,EAAUf,EAAUc,EAAY,CAClC,IAAK,GACL,QAAS,GACT,iBAAkB,EACtB,CAAC,EAED,GAAIC,EAAQ,SAAW,EAEnB,MAAO,GAIX,IAAMC,EAAU,OAAO,KAAKD,EAAQ,CAAC,CAAC,EAOtC,MAHwB,CAAC,GAAGR,EAAeE,CAAI,EAGxB,MAAMQ,GAAUD,EAAQ,SAASC,CAAM,CAAC,CACnE,CAEO,OAAQ,CACX,KAAK,cAAc,KAAK,YAAY,CACxC,CAEO,cAAe,CACd,KAAK,YACLX,EAAc,KAAK,UAAU,CAErC,CAEQ,cAAcY,EAAc,CAChC,KAAK,aAAa,EAClB,KAAK,aAAeA,EACpB,KAAK,WAAa,YAAY,IAAM,KAAK,gBAAgB,EAAGA,CAAI,CACpE,CAEA,MAAa,UAAUT,EAAiD,CACpE,GAAI,MAAML,EAAY,KAAK,UAAUK,CAAI,CAAC,EAAG,CAEzC,IAAMU,EAAQ,MAAMrB,EAAG,KAAK,KAAK,UAAUW,CAAI,CAAC,EAChD,GAAI,KAAK,oBAAsBU,EAAM,KAAM,CAEvC,IAAMJ,EAAuB,MAAM,KAAK,QAAQN,CAAI,EACpD,QAAWW,KAAUL,EAAS,CAC1B,IAAIM,EAAY,KAAK,MAAM,IAAID,EAAO,IAAI,EACrCC,EAIDA,EAAUZ,CAAI,EAAIW,EAAOX,CAAI,EAH7B,KAAK,MAAM,IAAIW,EAAO,KAAMA,CAAM,CAK1C,CACA,YAAK,kBAAoBD,EAAM,KAC/B,KAAK,cAAgBJ,EACdA,CACX,KACI,QAAO,KAAK,aAEpB,CACA,MAAO,CAAC,CACZ,CAEQ,oBAAkC,CACtC,OAAO,MAAM,KAAK,KAAK,MAAM,OAAO,CAAC,CACzC,CAEA,MAAc,QAAQN,EAAiD,CACnE,GAAI,CAAE,MAAML,EAAY,KAAK,UAAUK,CAAI,CAAC,EACxC,MAAO,CAAC,EAEZ,IAAMa,EAAQ,YAAY,IAAI,EAExBC,EAASxB,EAAM,CACjB,QAAS,GACT,iBAAkB,EACtB,CAAC,EAEKgB,EAAuB,CAAC,EAE9B,OAAO,IAAI,QAAQ,CAACJ,EAASC,IAAW,CACpCf,EAAiB,KAAK,UAAUY,CAAI,CAAC,EAChC,KAAKc,CAAM,EACX,GAAG,OAASH,GAAW,CACpBL,EAAQ,KAAKK,CAAM,CACvB,CAAC,EAAE,GAAG,MAAO,IAAM,CACnBT,EAAQI,CAAO,EACf,QAAQ,IAAI,cAAcN,CAAI,SAAS,YAAY,IAAI,EAAIa,CAAK,IAAI,CACxE,CAAC,EAAE,GAAG,QAAUE,GAAQ,CACpBZ,EAAOY,CAAG,CACd,CAAC,CACL,CAAC,CACL,CAEA,MAAc,iBAAiC,CAC3C,GAAI,KAAK,oBAAsB,CAAC,KAAK,WACjC,OAEJ,KAAK,WAAa,GAClB,KAAK,mBAAqB,GAC1B,IAAMF,EAAQ,YAAY,IAAI,EAE9B,QAAWb,KAAQ,KAAK,WAAY,CAChC,IAAIgB,EAA4B,MAAM,KAAK,UAAUhB,CAAI,EACzD,GAAI,KAAK,MAAM,OAAS,EAAG,CAGvB,IAAMiB,EAFyB,KAAK,mBAAmB,EAE7B,OAAOC,GAAO,CAACF,EAAa,KAAKG,GAAeA,EAAY,OAASD,EAAI,IAAI,CAAC,EAExG,GAAID,EAAQ,SAAW,EAAG,CAEtB,IAAMG,EAAY5B,EAAUyB,EAAS,CACjC,OAAQD,EAAa,SAAW,EAChC,QAAS,CACL,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,OAAQ,OAAQ,MAAM,EAC5B,CAAC,IAAK,QAAS,OAAQ,OAAO,EAC9B,CAAC,IAAKhB,EAAM,OAAQA,CAAI,CAC5B,CACJ,CAAC,EAGD,MAAMX,EAAG,WAAW,KAAK,UAAUW,CAAI,EAAGoB,CAAS,CACvD,CACJ,CACJ,CACA,IAAMC,EAAY,YAAY,IAAI,EAAIR,EACtC,QAAQ,IAAI,kBAAkBQ,CAAS,IAAI,EAEvCA,EAAY,GAAK,KAAK,eACtB,KAAK,cAAcA,EAAY,EAAE,EACjC,QAAQ,IAAI,oCAAoCA,EAAY,EAAE,IAAI,GAEtE,KAAK,mBAAqB,EAC9B,CAEO,cAAc3C,EAAkB4C,EAAkBC,EAA0B,CAC/E,IAAMC,EAAW5B,EAAK,SAASlB,CAAQ,EACnC+C,EAAgB,KAAK,MAAM,IAAID,CAAQ,EAC3C,GAAIC,GACA,GAAIA,EAAc,OAEd,GAAIA,EAAc,OAAUH,EAAW,IAAOG,EAAc,QAAUF,EAClE,OAAOE,UAIPA,EAAc,OAAUH,EAAW,GACnC,OAAOG,EAWnB,OAAO,IACX,CAEO,WAAW/C,EAAkB4C,EAAkBC,EAAeG,EAA2D,CAC5H,GAAI,CAAChD,GAAY,CAAC4C,GAAY,CAACC,GAAS,CAACG,EACrC,MAAM,IAAI,MAAM,oBAAoB,EAExC,QAAW1B,KAAQ,KAAK,WACpB,GAAI,CAAC0B,EAAM1B,CAAI,EACX,MAAM,IAAI,MAAM,gBAAgBA,CAAI,EAAE,EAG9C,IAAM2B,EAAOL,EAAW,GAClBM,EAAWhC,EAAK,SAASlB,CAAQ,EACnCkC,EAAY,KAAK,MAAM,IAAIgB,CAAQ,EACvC,GAAKhB,EAKD,QAAWZ,KAAQ,KAAK,WACpBY,EAAUZ,CAAI,EAAI0B,EAAM1B,CAAI,MANpB,CACZ,IAAM6B,EAAO,CAAC,KAAMD,EAAU,KAAMD,EAAM,MAAOJ,EAAO,GAAGG,CAAK,EAChE,KAAK,MAAM,IAAIE,EAAUC,CAAI,CACjC,CAMA,KAAK,WAAa,EACtB,CAEJ,EE9PA,OAAQ,QAAAC,MAAW,cACnB,OAAOlC,MAAU,OAEV,IAAMmC,EAAN,KAAwD,CAI3D,YAAYC,EAA8BnD,EAAoC,0BAAqD,EAAEJ,EAAoB,CAA/G,gBAAAI,EACtC,KAAK,aAAe,IAAID,EAAaC,EAAWJ,CAAU,EAC1D,KAAK,iBAAmB,IAAIsB,EAAiBiC,EAAcnD,CAAU,CACzE,CANA,iBACA,aAOA,MAAM,mBAAmBH,EAAkBI,EAAwC,CAC/E,MAAM,KAAK,iBAAiB,KAAK,EACjC,IAAI4B,EAAQ,MAAMoB,EAAKpD,CAAQ,EAC/B,GAAI,KAAK,iBAAiB,SAAS,EAAE,IAAIkB,EAAK,SAASlB,CAAQ,CAAC,EAAG,CAC/D,IAAMkC,EAAY,KAAK,iBAAiB,cAAclC,EAAUgC,EAAM,KAAMA,EAAM,MAAM,YAAY,CAAC,EACrG,GAAGE,EACC,QAAWZ,KAAQ,KAAK,WAChB,CAAClB,EAASkB,CAAI,GAAKY,EAAUZ,CAAI,IACjClB,EAASkB,CAAI,EAAIY,EAAUZ,CAAI,EAI/C,CACA,MAAM,KAAK,aAAa,mBAAmBtB,EAAUI,CAAQ,EAC7D,KAAK,iBAAiB,WAAWJ,EAAUgC,EAAM,KAAMA,EAAM,MAAM,YAAY,EAAG5B,CAAQ,CAC9F,CACJ,EC/BA,UAAYc,MAAU,OACtB,OAAOqC,MAAc,WACrB,OAAQ,WAAAC,MAAc,cACtB,OAAOC,MAAY,UAGZ,IAAMC,EAAN,KAAoB,CAQvB,YAAYC,EAA6CC,EAAmCC,EAIzF,CAJsD,uBAAAD,EAAmC,YAAAC,EAKxF,KAAK,cAAgBF,CACzB,CAbA,YAAc,GACd,MAAQ,IAAIF,EAAO,CAAC,YAAa,IAAK,UAAW,EAAI,CAAC,EACtD,cACA,UAAY,GACZ,QAAU,EACV,WAAa,IAAI,IAgBjB,MAAc,iBAAiBK,EAAmBC,EAA0B,CACxE,IAAMC,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EAC9D,QAAWG,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAChD,GAAIA,EAAM,YAAY,EAClB,MAAM,KAAK,iBAAiBC,EAAUH,CAAQ,MAC3C,CACH,IAAMI,EAAU,KAAK,YAAYD,CAAQ,EACzCH,EAAS,KAAKI,CAAO,CACzB,CACJ,CACJ,CAEA,MAAc,UAAUL,EAAoC,CACxD,IAAME,EAAU,MAAMR,EAAQM,EAAW,CAAC,cAAe,EAAI,CAAC,EACxDM,EAAgC,CAAC,EAEvC,QAAWH,KAASD,EAAS,CACzB,IAAME,EAAgB,OAAKJ,EAAWG,EAAM,IAAI,EAC5CA,EAAM,YAAY,EAElBG,EAAc,KAAK,KAAK,UAAUF,CAAQ,CAAC,EAG3CE,EAAc,KAAK,KAAK,MAAM,IAC1B,SAAa,MAAM,KAAK,cAAc,eAAeF,CAAQ,EAAK,EAAI,CAC1E,CAAC,CAET,CAIA,OADe,MAAM,QAAQ,IAAIE,CAAa,GAChC,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,CAC3D,CAEA,MAAc,YAAYtE,EAAgC,CACtD,GAAG,MAAM,KAAK,cAAc,eAAeA,CAAQ,GAC5C,CAAC,KAAK,WAAW,IAAIA,CAAQ,EAAG,CAC/B,KAAK,WAAW,IAAIA,CAAQ,EAC5B,IAAMsE,EAAU,EAAE,KAAK,QACvB,MAAM,KAAK,MAAM,IAAI,IAAM,KAAK,cAAc,YAAYA,EAAS,KAAK,UAAWtE,CAAQ,CAAC,EAC5F,KAAK,WAAW,OAAOA,CAAQ,CACnC,CAER,CAOA,MAAc,oBAAoBA,EAAgC,CAC9D,IAAIuE,EAAe,UAAQvE,CAAQ,EACnC,MAAM,KAAK,iBAAiBuE,EAAS,CAAC,CAAC,CAC3C,CAEQ,cAAcC,EAAqB,CACvC,IAAMC,EAAiB,CACnB,cAAe,GACf,WAAY,GACZ,MAAO,IACP,iBAAkB,CACd,mBAAoB,KAAK,OAAO,oBAAsB,IACtD,aAAc,KAAK,OAAO,cAAgB,GAC9C,CACJ,EAEIC,GACC,KAAK,OAAO,UAAY,IAAM,EAC/BA,EAAUnB,EAAS,MAAMiB,EAAYC,CAAc,EAEnDC,EAAUnB,EAAS,MAAMiB,EAAY,CACjC,GAAGC,EACH,WAAY,GACZ,SAAU,KAAK,OAAO,UAAY,CACtC,CAAC,EAGLC,EACK,GAAG,MAAO,MAAO1E,GAAa,CAC3B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS2E,EAAO,CACZ,QAAQ,MAAM,yBAAyB3E,CAAQ,IAAK2E,CAAK,CAC7D,CACJ,CAAC,EACA,GAAG,SAAU,MAAO3E,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS0B,EAAG,CACR,QAAQ,MAAM,yBAAyB1B,CAAQ,IAAK0B,CAAC,CACzD,CACJ,CAAC,EACA,GAAG,SAAU,MAAO1B,GAAa,CAC9B,GAAI,CACA,MAAM,KAAK,cAAc,WAAWA,CAAQ,EAC5C,MAAM,KAAK,oBAAoBA,CAAQ,CAC3C,OAAS0B,EAAG,CACR,QAAQ,MAAM,kCAAmCA,CAAC,CACtD,CACJ,CAAC,EACA,GAAG,QAAUiD,GAAU,QAAQ,MAAM,kBAAkBA,CAAK,EAAE,CAAC,EAC/D,GAAG,QAAS,IAAM,CACf,QAAQ,IAAI,gCAAgCH,CAAU,EAAE,CAC5D,CAAC,CACT,CAEA,MAAM,OAAQ,CACV,IAAII,EAAU,KAAK,kBACnB,GAAI,CAACA,EACD,MAAM,IAAI,MAAM,oBAAoB,EAExC,IAAIJ,EAAaI,EAAQ,MAAM,GAAG,EAElC,KAAK,cAAcJ,CAAU,EAG7B,IAAIJ,EAAgB,CAAC,EACrB,QAASS,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5CT,EAAc,KAAK,KAAK,UAAUI,EAAWK,CAAC,CAAC,CAAC,EAEpD,IAAMC,GAAS,MAAM,QAAQ,IAAIV,CAAa,GAAG,OAAO,CAACC,EAAKC,IAAYD,EAAMC,EAAS,CAAC,EAC1F,QAAQ,IAAI,SAASQ,CAAK,mBAAmB,EAC7C,KAAK,UAAYA,EACjB,IAAIf,EAA2B,CAAC,EAChC,QAASc,EAAI,EAAGA,EAAIL,EAAW,OAAQK,IACnCL,EAAWK,CAAC,EAAS,YAAUL,EAAWK,CAAC,CAAC,EAC5C,MAAM,KAAK,iBAAiBL,EAAWK,CAAC,EAAGd,CAAQ,EAEvD,MAAM,QAAQ,IAAIA,CAAQ,EAC1B,KAAK,YAAc,GAEnB,QAAQ,IAAI,eAAe,CAC/B,CACJ","sourcesContent":["import {Piscina} from \"piscina\";\nimport {CID_ALGORITHM_NAMES} from \"../MultiHashData\";\n\n// Construct the URL for the current module\nlet distFolder = import.meta.dirname;\ndistFolder = distFolder.replace('src', 'dist');\ndistFolder = distFolder + \"/worker.js\";\nconst workerUrl = new URL(distFolder).href;\nconsole.log(\"Worker URL: \", workerUrl);\n\nexport class FileIDComputer {\n private piscina: Piscina;\n\n constructor(workerPath?:string) {\n this.piscina = new Piscina({\n maxThreads: 4,\n //filename: new URL('./ShaComputeWorker.ts', import.meta.url).href\n filename: workerPath || process.env.WORKER_URL || workerUrl\n });\n }\n\n /**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\n public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise {\n return this.piscina.run({filePath, algorithms});\n }\n}\n","import { FileIDComputer } from \"./fileID/FileIDComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\n\nexport class HashComputer implements ComputeInterface{\n private fileIDComputer:FileIDComputer;\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[],workerPath?:string) {\n this.fileIDComputer = new FileIDComputer(workerPath);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);\n\n // Map the computed CIDs back to their respective metadata properties\n cids.forEach((cid, index) => {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n });\n }\n}\n","import {createReadStream, promises as fs} from 'fs';\nimport {parse} from 'csv-parse';\nimport {parse as parseSync} from 'csv-parse/sync';\nimport {stringify} from 'csv-stringify/sync';\nimport {existsAsync} from \"./ExistsAsync\";\nimport path from \"path\";\nimport {clearInterval} from \"node:timers\";\nimport {CID_ALGORITHM_NAMES} from \"./MultiHashData\";\n\ninterface IndexLine extends Partial> {\n path: string;\n size: string;\n mtime: string;\n}\n\nexport const INDEX_HEADERS = ['path', 'size', 'mtime'];\n\nexport class HashIndexManager {\n private cache: Map = new Map();\n private intervalId: NodeJS.Timeout;\n private intervalTime: number = 30000;\n private lastIndexFileSize: number = 0; //size of the index file last time it was read\n private lastCacheFile: IndexLine[]; //state of the file last time it was read\n private indexOpsInProgress: boolean = false;\n private hasChanged: boolean = false;\n private initialLoad: Promise;\n private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};\n\n constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {\n filePath = filePath.replace(\".csv\", \"\");\n for (const hash of this.targetHash) {\n this.filePaths[hash] = `${filePath}-${hash}.csv`;\n if (!this.filePaths[hash]) {\n throw new Error(`Invalid index file path for ${hash}`);\n }\n }\n }\n\n public getCache(): Map {\n return new Map(this.cache);\n }\n\n /**\n * After init consseutively calls to this method will not reload the index\n * @param autosave\n */\n public async init(autosave = true) {\n if (!this.initialLoad) {\n this.initialLoad = new Promise(async (resolve, reject) => {\n try {\n for (const hash of this.targetHash) {\n if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {\n throw new Error(`Invalid index file headers for ${hash}`);\n }\n await this.loadIndex(hash);\n }\n if (autosave) {\n this.start();\n }\n resolve();\n } catch (e) {\n reject(e);\n }\n });\n }\n return this.initialLoad;\n }\n\n // Function to check CSV headers\n private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {\n const records = parseSync(csvContent, {\n bom: true,\n columns: true,\n skip_empty_lines: true,\n });\n\n if (records.length === 0) {\n //we will write the headers\n return true;\n }\n\n // Extract headers\n const headers = Object.keys(records[0]);\n\n // Define required headers\n //from the IndexLine interface\n const requiredHeaders = [...INDEX_HEADERS, hash];\n\n // Check if all required headers are present\n return requiredHeaders.every(header => headers.includes(header));\n };\n\n public start() {\n this.startAutoSave(this.intervalTime);\n }\n\n public stopAutoSave() {\n if (this.intervalId) {\n clearInterval(this.intervalId);\n }\n }\n\n private startAutoSave(time: number) {\n this.stopAutoSave();\n this.intervalTime = time;\n this.intervalId = setInterval(() => this.saveCacheToFile(), time);\n }\n\n public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise {\n if (await existsAsync(this.filePaths[hash])) {\n // check the file size and if it did not change, do not read the file\n const stats = await fs.stat(this.filePaths[hash]);\n if (this.lastIndexFileSize !== stats.size) {\n // Read existing file content and parse it\n const records: IndexLine[] = await this.readCsv(hash);\n for (const record of records) {\n let indexLine = this.cache.get(record.path);\n if (!indexLine) {\n this.cache.set(record.path, record);\n } else {\n //update the cache with the latest data\n indexLine[hash] = record[hash];\n }\n }\n this.lastIndexFileSize = stats.size;\n this.lastCacheFile = records;\n return records;\n } else {\n return this.lastCacheFile;\n }\n }\n return [];\n }\n\n private loadIndexFromCache(): IndexLine[] {\n return Array.from(this.cache.values());\n }\n\n private async readCsv(hash: CID_ALGORITHM_NAMES): Promise {\n if (!(await existsAsync(this.filePaths[hash]))) {\n return [];\n }\n const start = performance.now();\n\n const parser = parse({\n columns: true,\n skip_empty_lines: true,\n });\n\n const records: IndexLine[] = [];\n\n return new Promise((resolve, reject) => {\n createReadStream(this.filePaths[hash])\n .pipe(parser)\n .on('data', (record) => {\n records.push(record);\n }).on('end', () => {\n resolve(records);\n console.log(`Index read ${hash} time ${performance.now() - start}ms`);\n }).on('error', (err) => {\n reject(err);\n });\n });\n }\n\n private async saveCacheToFile(): Promise {\n if (this.indexOpsInProgress || !this.hasChanged) {\n return;\n }\n this.hasChanged = false;\n this.indexOpsInProgress = true;\n const start = performance.now();\n\n for (const hash of this.targetHash) {\n let existingRows: IndexLine[] = await this.loadIndex(hash);\n if (this.cache.size !== 0) {\n const cacheRows: IndexLine[] = this.loadIndexFromCache();\n // Filter out cacheRows that are already in the file\n const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));\n\n if (newRows.length !== 0) {\n // Serialize new cacheRows to CSV string\n const csvString = stringify(newRows, {\n header: existingRows.length === 0, // Only add header if the file was empty\n columns: [\n {key: 'path', header: 'path'},\n {key: 'size', header: 'size'},\n {key: 'mtime', header: 'mtime'},\n {key: hash, header: hash},\n ],\n });\n\n // Append new cacheRows to the file\n await fs.appendFile(this.filePaths[hash], csvString);\n }\n }\n }\n const totalTime = performance.now() - start;\n console.log(`Index saved in ${totalTime}ms`);\n // Check if the time to save the index is greater than the interval time. increase the interval time if needed\n if (totalTime * 10 > this.intervalTime) {\n this.startAutoSave(totalTime * 10);\n console.log(`Index save interval increased to ${totalTime * 10}ms`);\n }\n this.indexOpsInProgress = false;\n }\n\n public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {\n const fileName = path.basename(filePath);\n let fileNameIndex = this.cache.get(fileName);\n if (fileNameIndex) {\n if (fileNameIndex.mtime) {\n //if we have a mtime, we need to check it\n if (fileNameIndex.size === (fileSize + \"\") && fileNameIndex.mtime === mtime) {\n return fileNameIndex;\n }\n } else {\n //mtime is optional\n if (fileNameIndex.size === (fileSize + \"\")) {\n return fileNameIndex;\n }\n }\n }\n\n // 3 - if not found, delete the entry (keeps the index clean)\n /*if (fileNameIndex && fileNameIndex.size !== (fileSize + \"\") && pathIndex && pathIndex.size !== (fileSize + \"\")) {\n this.cache.delete(fileName);\n this.cache.delete(filePath);\n }*/\n\n return null;\n }\n\n public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void {\n if (!filePath || !fileSize || !mtime || !hashs) {\n throw new Error('Invalid parameters');\n }\n for (const hash of this.targetHash) {\n if (!hashs[hash]) {\n throw new Error(`Missing hash ${hash}`);\n }\n }\n const size = fileSize + \"\";\n const baseName = path.basename(filePath);\n let indexLine = this.cache.get(baseName);\n if (!indexLine) {\n const data = {path: baseName, size: size, mtime: mtime, ...hashs};\n this.cache.set(baseName, data);\n } else {\n //update the cache with the latest data\n for (const hash of this.targetHash) {\n indexLine[hash] = hashs[hash];\n }\n }\n this.hasChanged = true;\n }\n\n}","// Example of checking if a file exists asynchronously\n// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.\nimport {access,constants} from \"fs/promises\";\n\nexport async function existsAsync(filePath: string): Promise {\n try {\n await access(filePath, constants.F_OK);\n return true; // File exists\n } catch {\n return false; // File does not exist\n }\n}","import {HashComputer} from \"./HashComputer\";\nimport {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from \"./MultiHashData\";\nimport {HashIndexManager} from \"./HashIndexManager\";\nimport {stat} from \"fs/promises\";\nimport path from \"path\";\n\nexport class ComputeHashIndexCache implements ComputeInterface {\n hashIndexManager: HashIndexManager;\n hashComputer: HashComputer;\n\n constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {\n this.hashComputer = new HashComputer(targetHash,workerPath);\n this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);\n }\n\n async computeMissingHash(filePath: string, metadata: MultiHashData): Promise {\n await this.hashIndexManager.init();\n let stats = await stat(filePath);\n if (this.hashIndexManager.getCache().has(path.basename(filePath))) {\n const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());\n if(indexLine) {\n for (const hash of this.targetHash) {\n if (!metadata[hash] && indexLine[hash]) {\n metadata[hash] = indexLine[hash];\n }\n }\n }\n }\n await this.hashComputer.computeMissingHash(filePath, metadata);\n this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);\n }\n}","import * as path from 'path';\nimport chokidar from 'chokidar';\nimport {readdir} from \"fs/promises\";\nimport PQueue from \"p-queue\";\nimport {FileProcessorInterface} from \"./FileProcessorInterface.js\";\n\nexport class FolderWatcher {\n initialized = false;\n queue = new PQueue({concurrency: 100, autoStart: true});\n fileProcessor: FileProcessorInterface;\n queueSize = -1;\n current = 0;\n processing = new Set();\n\n constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{\n interval?: number,\n stabilityThreshold?: number,\n pollInterval?: number,\n }) {\n this.fileProcessor = fileProcessor;\n }\n\n /**\n * Process the directory and its subdirectories\n * @param directory\n * @param promises\n * @private\n */\n private async processDirectory(directory: string, promises: Promise[]) {\n const entries = await readdir(directory, {withFileTypes: true});\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n await this.processDirectory(fullPath, promises);\n } else {\n const promise = this.processFile(fullPath);\n promises.push(promise);\n }\n }\n }\n\n private async countFile(directory: string): Promise {\n const entries = await readdir(directory, {withFileTypes: true});\n const countPromises: Promise[] = [];\n\n for (const entry of entries) {\n const fullPath = path.join(directory, entry.name);\n if (entry.isDirectory()) {\n // Push the promise without waiting for it here\n countPromises.push(this.countFile(fullPath));\n } else {\n // Use an IIFE (Immediately Invoked Function Expression) to handle asynchronous condition\n countPromises.push(this.queue.add(\n async () => (await this.fileProcessor.canProcessFile(fullPath)) ? 1 : 0\n ));\n }\n }\n\n // Wait for all promises to resolve and then sum up their results\n const counts = await Promise.all(countPromises);\n return counts.reduce((acc, current) => acc + current, 0);\n }\n\n private async processFile(filePath: string):Promise {\n if(await this.fileProcessor.canProcessFile(filePath)) {\n if(!this.processing.has(filePath)) {\n this.processing.add(filePath);//avoid double processing in the queue\n const current = ++this.current;\n await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));\n this.processing.delete(filePath);\n }\n }\n }\n\n /**\n * Process the file and its sibling file included other folder it is a recursive function\n * @param filePath\n * @private\n */\n private async processFileExtended(filePath: string):Promise {\n let dirname = path.dirname(filePath);\n await this.processDirectory(dirname, []);\n }\n\n private chokidarWatch(folderList: string[]){\n const chokidarconfig = {\n ignoreInitial: true,//ignore the initial scan we have our own function\n persistent: true,\n depth: Infinity,\n awaitWriteFinish: {\n stabilityThreshold: this.config.stabilityThreshold || 30000,\n pollInterval: this.config.pollInterval || 5000\n }\n };\n\n let watcher: chokidar.FSWatcher;\n if ((this.config.interval || 0) <= 0) {\n watcher = chokidar.watch(folderList, chokidarconfig);\n } else {\n watcher = chokidar.watch(folderList, {\n ...chokidarconfig,\n usePolling: true,\n interval: this.config.interval || 0\n });\n }\n\n watcher\n .on('add', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (error) {\n console.error(`Error processing file ${filePath}:`, error);\n }\n })\n .on('change', async (filePath) => {\n try {\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file ${filePath}:`, e);\n }\n })\n .on('unlink', async (filePath) => {\n try {\n await this.fileProcessor.deleteFile(filePath);\n await this.processFileExtended(filePath);\n } catch (e) {\n console.error(`Error processing file deletion:`, e);\n }\n })\n .on('error', (error) => console.error(`Watcher error: ${error}`))\n .on('ready', () => {\n console.log(`Watching for file changes on ${folderList}`);\n });\n }\n\n async watch() {\n let folders = this.WATCH_FOLDER_LIST;\n if (!folders) {\n throw new Error(`No folder to watch`);\n }\n let folderList = folders.split(',');\n\n this.chokidarWatch(folderList);\n\n //start the initial scan\n let countPromises = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n countPromises.push(this.countFile(folderList[i]));\n }\n const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);\n console.log(`Found ${count} files to process`);\n this.queueSize = count;\n let promises: Promise[] = [];\n for (let i = 0; i < folderList.length; i++) {\n folderList[i] = path.normalize(folderList[i]);\n await this.processDirectory(folderList[i], promises);\n }\n await Promise.all(promises);\n this.initialized = true;\n\n console.log(`Watcher ready`);\n }\n}\n"]} \ No newline at end of file diff --git a/dist/worker.cjs b/dist/worker.cjs index f81d524..6b29ae1 100644 --- a/dist/worker.cjs +++ b/dist/worker.cjs @@ -1,9 +1,9 @@ 'use strict'; -var chunkDL4KI2PZ_cjs = require('./chunk-DL4KI2PZ.cjs'); +var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs'); -module.exports = chunkDL4KI2PZ_cjs.d; +module.exports = chunkA5PD5ID4_cjs.d; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.cjs.map \ No newline at end of file diff --git a/dist/worker.d.cts b/dist/worker.d.cts index 52fd03e..416bf67 100644 --- a/dist/worker.d.cts +++ b/dist/worker.d.cts @@ -1,2 +1 @@ -import 'crypto'; -export { S as SimpleHash, d as default } from './ShaComputeWorker-DWOF-qaH.cjs'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.cjs'; diff --git a/dist/worker.d.ts b/dist/worker.d.ts index 46935c7..e6a7c0f 100644 --- a/dist/worker.d.ts +++ b/dist/worker.d.ts @@ -1,2 +1 @@ -import 'crypto'; -export { S as SimpleHash, d as default } from './ShaComputeWorker-DWOF-qaH.js'; +export { S as SimpleHash, d as default } from './ShaComputeWorker-DxWHpwX4.js'; diff --git a/dist/worker.js b/dist/worker.js index 9253df7..d0a108d 100644 --- a/dist/worker.js +++ b/dist/worker.js @@ -1,3 +1,3 @@ -export { d as default } from './chunk-VGBIRFOR.js'; +export { d as default } from './chunk-FGATABDV.js'; //# sourceMappingURL=out.js.map //# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/package.json b/package.json index 42a70b0..1904975 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "packageManager": "yarn@4.1.1", "dependencies": { "chokidar": "^3.6.0", + "crc-32": "^1.2.2", "csv-parse": "^5.5.5", "csv-stringify": "^6.4.6", "multiformats": "^13.1.0", diff --git a/src/lib/ComputeHashIndexCache.ts b/src/lib/ComputeHashIndexCache.ts index 76d703f..e512bad 100644 --- a/src/lib/ComputeHashIndexCache.ts +++ b/src/lib/ComputeHashIndexCache.ts @@ -9,8 +9,8 @@ export class ComputeHashIndexCache implements ComputeInterface { hashComputer: HashComputer; constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { - this.hashComputer = new HashComputer(this.targetHash,workerPath); - this.hashIndexManager = new HashIndexManager(indexFilePath); + this.hashComputer = new HashComputer(targetHash,workerPath); + this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash); } async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { diff --git a/src/lib/HashIndexManager.ts b/src/lib/HashIndexManager.ts index 07a9335..9f6f296 100644 --- a/src/lib/HashIndexManager.ts +++ b/src/lib/HashIndexManager.ts @@ -12,6 +12,7 @@ interface IndexLine extends Partial> { size: string; mtime: string; } + export const INDEX_HEADERS = ['path', 'size', 'mtime']; export class HashIndexManager { @@ -22,11 +23,16 @@ export class HashIndexManager { private lastCacheFile: IndexLine[]; //state of the file last time it was read private indexOpsInProgress: boolean = false; private hasChanged: boolean = false; - private initialLoad; + private initialLoad: Promise; + private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {}; - constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) { - if (!filePath) { - throw new Error('Invalid index file path'); + constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) { + filePath = filePath.replace(".csv", ""); + for (const hash of this.targetHash) { + this.filePaths[hash] = `${filePath}-${hash}.csv`; + if (!this.filePaths[hash]) { + throw new Error(`Invalid index file path for ${hash}`); + } } } @@ -38,19 +44,21 @@ export class HashIndexManager { * After init consseutively calls to this method will not reload the index * @param autosave */ - async init(autosave = true) { + public async init(autosave = true) { if (!this.initialLoad) { this.initialLoad = new Promise(async (resolve, reject) => { try { - if (!this.checkCSVHeaders(this.filePath)) { - throw new Error('Invalid index file headers'); + for (const hash of this.targetHash) { + if (!this.checkCSVHeaders(this.filePaths[hash], hash)) { + throw new Error(`Invalid index file headers for ${hash}`); + } + await this.loadIndex(hash); } - await this.loadIndex(); if (autosave) { this.start(); } resolve(); - }catch (e) { + } catch (e) { reject(e); } }); @@ -59,7 +67,7 @@ export class HashIndexManager { } // Function to check CSV headers - checkCSVHeaders (csvContent: string): boolean { + private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean { const records = parseSync(csvContent, { bom: true, columns: true, @@ -76,7 +84,7 @@ export class HashIndexManager { // Define required headers //from the IndexLine interface - const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash]; + const requiredHeaders = [...INDEX_HEADERS, hash]; // Check if all required headers are present return requiredHeaders.every(header => headers.includes(header)); @@ -98,15 +106,21 @@ export class HashIndexManager { this.intervalId = setInterval(() => this.saveCacheToFile(), time); } - public async loadIndex(): Promise { - if (await existsAsync(this.filePath)) { + public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise { + if (await existsAsync(this.filePaths[hash])) { // check the file size and if it did not change, do not read the file - const stats = await fs.stat(this.filePath); + const stats = await fs.stat(this.filePaths[hash]); if (this.lastIndexFileSize !== stats.size) { // Read existing file content and parse it - const records: IndexLine[] = await this.readCsv(); + const records: IndexLine[] = await this.readCsv(hash); for (const record of records) { - this.cache.set(record.path, record); + let indexLine = this.cache.get(record.path); + if (!indexLine) { + this.cache.set(record.path, record); + } else { + //update the cache with the latest data + indexLine[hash] = record[hash]; + } } this.lastIndexFileSize = stats.size; this.lastCacheFile = records; @@ -122,8 +136,8 @@ export class HashIndexManager { return Array.from(this.cache.values()); } - private async readCsv(): Promise { - if (!(await existsAsync(this.filePath))) { + private async readCsv(hash: CID_ALGORITHM_NAMES): Promise { + if (!(await existsAsync(this.filePaths[hash]))) { return []; } const start = performance.now(); @@ -136,13 +150,13 @@ export class HashIndexManager { const records: IndexLine[] = []; return new Promise((resolve, reject) => { - createReadStream(this.filePath) + createReadStream(this.filePaths[hash]) .pipe(parser) .on('data', (record) => { records.push(record); }).on('end', () => { resolve(records); - console.log(`Index read time ${performance.now() - start}ms`); + console.log(`Index read ${hash} time ${performance.now() - start}ms`); }).on('error', (err) => { reject(err); }); @@ -156,27 +170,29 @@ export class HashIndexManager { this.hasChanged = false; this.indexOpsInProgress = true; const start = performance.now(); - let existingRows: IndexLine[] = await this.loadIndex(); - if (this.cache.size !== 0) { - const cacheRows: IndexLine[] = this.loadIndexFromCache(); - // Filter out cacheRows that are already in the file - const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path)); + for (const hash of this.targetHash) { + let existingRows: IndexLine[] = await this.loadIndex(hash); + if (this.cache.size !== 0) { + const cacheRows: IndexLine[] = this.loadIndexFromCache(); + // Filter out cacheRows that are already in the file + const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path)); - if (newRows.length !== 0) { - // Serialize new cacheRows to CSV string - const csvString = stringify(newRows, { - header: existingRows.length === 0, // Only add header if the file was empty - columns: [ - {key: 'path', header: 'path'}, - {key: 'size', header: 'size'}, - {key: 'mtime', header: 'mtime'}, - ...this.targetHash.map(hash => ({key: hash, header: hash})), - ], - }); + if (newRows.length !== 0) { + // Serialize new cacheRows to CSV string + const csvString = stringify(newRows, { + header: existingRows.length === 0, // Only add header if the file was empty + columns: [ + {key: 'path', header: 'path'}, + {key: 'size', header: 'size'}, + {key: 'mtime', header: 'mtime'}, + {key: hash, header: hash}, + ], + }); - // Append new cacheRows to the file - await fs.appendFile(this.filePath, csvString); + // Append new cacheRows to the file + await fs.appendFile(this.filePaths[hash], csvString); + } } } const totalTime = performance.now() - start; @@ -192,13 +208,13 @@ export class HashIndexManager { public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine { const fileName = path.basename(filePath); let fileNameIndex = this.cache.get(fileName); - if(fileNameIndex) { - if(fileNameIndex.mtime) { + if (fileNameIndex) { + if (fileNameIndex.mtime) { //if we have a mtime, we need to check it if (fileNameIndex.size === (fileSize + "") && fileNameIndex.mtime === mtime) { return fileNameIndex; } - }else { + } else { //mtime is optional if (fileNameIndex.size === (fileSize + "")) { return fileNameIndex; @@ -219,10 +235,23 @@ export class HashIndexManager { if (!filePath || !fileSize || !mtime || !hashs) { throw new Error('Invalid parameters'); } + for (const hash of this.targetHash) { + if (!hashs[hash]) { + throw new Error(`Missing hash ${hash}`); + } + } const size = fileSize + ""; const baseName = path.basename(filePath); - const data = {path: baseName, size: size, mtime: mtime, ...hashs}; - this.cache.set(baseName, data); + let indexLine = this.cache.get(baseName); + if (!indexLine) { + const data = {path: baseName, size: size, mtime: mtime, ...hashs}; + this.cache.set(baseName, data); + } else { + //update the cache with the latest data + for (const hash of this.targetHash) { + indexLine[hash] = hashs[hash]; + } + } this.hasChanged = true; } diff --git a/src/lib/MultiHashData.ts b/src/lib/MultiHashData.ts index d4a74d9..bc3248e 100644 --- a/src/lib/MultiHashData.ts +++ b/src/lib/MultiHashData.ts @@ -3,33 +3,40 @@ export interface MultiHashData { cid_sha1?: string; // 0x11 sha1 cid_md5?: string; // 0xd5 md5 "cid_sha3-256"?: string; // 0x16 sha3-256 - //cid_crc32?: string; //0x0132 CRC32 (SFV) TODO + cid_crc32?: string; //0x0132 CRC32 (SFV) } export interface ComputeInterface { computeMissingHash(filePath: string, metadata: MultiHashData): Promise; } +/** + * according to + * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv + */ export enum CID_ALGORITHM_CODES { - sha256 = 0x12, - sha1 = 0x11, + crc32 = 0x0132, md5 = 0xd5, + sha1 = 0x11, + sha256 = 0x12, sha3_256 = 0x16, - //crc32 = 0x0132 + sha3_384 = 0x15, } export enum CID_ALGORITHM_NAMES { - sha256 = 'cid_sha2-256', - sha1 = 'cid_sha1', + crc32 = 'cid_crc32', md5 = 'cid_md5', + sha1 = 'cid_sha1', + sha256 = 'cid_sha2-256', sha3_256 = 'cid_sha3-256', - //crc32 = 'cid_crc32' + sha3_384 = 'cid_sha3-384', } export enum CID_ALGORITHM { - sha256 = 'sha2-256', - sha1 = 'sha1', + crc32 = 'crc32', md5 = 'md5', + sha1 = 'sha1', + sha256 = 'sha2-256', sha3_256 = 'sha3-256', - //crc32 = 'crc32' + sha3_384 = 'sha3-384', } \ No newline at end of file diff --git a/src/lib/fileID/ShaComputeWorker.ts b/src/lib/fileID/ShaComputeWorker.ts index 25c20dc..d9aaa78 100644 --- a/src/lib/fileID/ShaComputeWorker.ts +++ b/src/lib/fileID/ShaComputeWorker.ts @@ -1,14 +1,33 @@ -import { createReadStream } from "fs"; -import {BinaryLike, createHash} from 'crypto'; -import { CID } from 'multiformats/cid'; -import { create } from "multiformats/hashes/digest"; -import {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData"; +import {createReadStream} from "fs"; +import {createHash} from 'crypto'; +import {CID} from 'multiformats/cid'; +import {create} from "multiformats/hashes/digest"; +import crc32 from 'crc-32'; +import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData"; export interface SimpleHash { - update(data: BinaryLike), + update(data: Buffer), + digest(): Buffer; } +class Crc32Hash implements SimpleHash { + private _crc: number = undefined; + + update(data: Buffer): Crc32Hash { + // Update the CRC-32 checksum with the new chunk of data + this._crc = crc32.buf(data, this._crc); + return this; + } + + digest(): Buffer { + console.log((this._crc>>>0).toString(16)) ; + const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits) + buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format + return buffer; + } +} + /** * Compute the CIDs of a file using specific algorithms @@ -16,13 +35,19 @@ export interface SimpleHash { * @param algorithms Array of algorithms ('sha256', 'sha1') * @returns Array of CIDs (in the order of the algorithms) */ -export default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise { +export default async function computeCIDs({filePath, algorithms}: { + filePath: string; + algorithms: CID_ALGORITHM_NAMES[] +}): Promise { const hashFunctions: Record = { - [CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') }, - [CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') }, - [CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') }, - [CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') } + [CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')}, + [CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')}, + [CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')}, + [CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')}, + [CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')}, + [CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()} + }; const hashers = algorithms.map(algo => ({ @@ -33,11 +58,11 @@ export default async function computeCIDs({ filePath, algorithms }: { filePath: const stream = createReadStream(filePath); for await (const chunk of stream) { for (const item of hashers) { - item.hasher.update(chunk); + item.hasher.update(chunk as Buffer); } } - const cids = await Promise.all(hashers.map(async ({ code, hasher }) => { + const cids = await Promise.all(hashers.map(async ({code, hasher}) => { const hashBuffer = hasher.digest(); const digest = create(code, hashBuffer); return CID.createV1(code, digest).toString(); diff --git a/src/tests/main.test.ts b/src/tests/main.test.ts index 1116441..d48b349 100644 --- a/src/tests/main.test.ts +++ b/src/tests/main.test.ts @@ -1,27 +1,42 @@ -import {HashComputer} from "../lib"; -import {CID_ALGORITHM_NAMES} from "../lib"; +import {CID_ALGORITHM_NAMES, HashComputer} from "../lib"; -import {describe, it, expect} from 'vitest'; +import {describe, expect, it} from 'vitest'; import * as process from "process"; process.env.WORKER_URL = "./dist/worker.js"; +/** + * /!\ WARNING /!\ + * Remember to build the worker each time you change the worker code. + */ + describe('add', () => { it('meta data compute', async () => { let hashComputer = new HashComputer([ CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256 + CID_ALGORITHM_NAMES.sha3_256, + CID_ALGORITHM_NAMES.crc32, + CID_ALGORITHM_NAMES.sha3_384 ]); let metadata = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); console.log(metadata); + /** + * test.txt CF20B62A (CRC32) + * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) + * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt + * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt + * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt + */ expect(metadata).toEqual({ - cid_sha1: 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - 'cid_sha2-256': 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - cid_md5: 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - 'cid_sha3-256': 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq' + "cid_crc32": "bagzafmqcathsbnrk", + "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', + "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', + "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', + "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', + "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' }); }); }); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 54f4ca8..91b3ca2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -417,6 +417,7 @@ __metadata: dependencies: "@types/node": "npm:^20.12.2" chokidar: "npm:^3.6.0" + crc-32: "npm:^1.2.2" csv-parse: "npm:^5.5.5" csv-stringify: "npm:^6.4.6" multiformats: "npm:^13.1.0" @@ -962,6 +963,15 @@ __metadata: languageName: node linkType: hard +"crc-32@npm:^1.2.2": + version: 1.2.2 + resolution: "crc-32@npm:1.2.2" + bin: + crc32: bin/crc32.njs + checksum: 10c0/11dcf4a2e77ee793835d49f2c028838eae58b44f50d1ff08394a610bfd817523f105d6ae4d9b5bef0aad45510f633eb23c903e9902e4409bed1ce70cb82b9bf0 + languageName: node + linkType: hard + "create-require@npm:^1.1.0": version: 1.1.1 resolution: "create-require@npm:1.1.1"