ignore the dist
This commit is contained in:
Pierre-Henri
+1
-1
@@ -25,5 +25,5 @@ yarn-error.log*
|
|||||||
/junit.xml
|
/junit.xml
|
||||||
/.nyc_output/
|
/.nyc_output/
|
||||||
/coverage/
|
/coverage/
|
||||||
#/dist/
|
/dist/
|
||||||
.idea
|
.idea
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
|
nmHoistingLimits: dependencies
|
||||||
nodeLinker: node-modules
|
nodeLinker: node-modules
|
||||||
|
|||||||
Vendored
-41
@@ -1,41 +0,0 @@
|
|||||||
interface MultiHashData {
|
|
||||||
cid_crc32?: string;
|
|
||||||
cid_md5?: string;
|
|
||||||
cid_sha1?: string;
|
|
||||||
"cid_sha2-256"?: string;
|
|
||||||
"cid_sha3-256"?: string;
|
|
||||||
"cid_sha3_384"?: string;
|
|
||||||
}
|
|
||||||
interface ComputeInterface {
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* according to
|
|
||||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
|
||||||
*/
|
|
||||||
declare enum CID_ALGORITHM_CODES {
|
|
||||||
crc32 = 306,
|
|
||||||
md5 = 213,
|
|
||||||
sha1 = 17,
|
|
||||||
sha256 = 18,
|
|
||||||
sha3_256 = 22,
|
|
||||||
sha3_384 = 21
|
|
||||||
}
|
|
||||||
declare enum CID_ALGORITHM_NAMES {
|
|
||||||
crc32 = "cid_crc32",
|
|
||||||
md5 = "cid_md5",
|
|
||||||
sha1 = "cid_sha1",
|
|
||||||
sha256 = "cid_sha2-256",
|
|
||||||
sha3_256 = "cid_sha3-256",
|
|
||||||
sha3_384 = "cid_sha3-384"
|
|
||||||
}
|
|
||||||
declare enum CID_ALGORITHM {
|
|
||||||
crc32 = "crc32",
|
|
||||||
md5 = "md5",
|
|
||||||
sha1 = "sha1",
|
|
||||||
sha256 = "sha2-256",
|
|
||||||
sha3_256 = "sha3-256",
|
|
||||||
sha3_384 = "sha3-384"
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c };
|
|
||||||
Vendored
-41
@@ -1,41 +0,0 @@
|
|||||||
interface MultiHashData {
|
|
||||||
cid_crc32?: string;
|
|
||||||
cid_md5?: string;
|
|
||||||
cid_sha1?: string;
|
|
||||||
"cid_sha2-256"?: string;
|
|
||||||
"cid_sha3-256"?: string;
|
|
||||||
"cid_sha3_384"?: string;
|
|
||||||
}
|
|
||||||
interface ComputeInterface {
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* according to
|
|
||||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
|
||||||
*/
|
|
||||||
declare enum CID_ALGORITHM_CODES {
|
|
||||||
crc32 = 306,
|
|
||||||
md5 = 213,
|
|
||||||
sha1 = 17,
|
|
||||||
sha256 = 18,
|
|
||||||
sha3_256 = 22,
|
|
||||||
sha3_384 = 21
|
|
||||||
}
|
|
||||||
declare enum CID_ALGORITHM_NAMES {
|
|
||||||
crc32 = "cid_crc32",
|
|
||||||
md5 = "cid_md5",
|
|
||||||
sha1 = "cid_sha1",
|
|
||||||
sha256 = "cid_sha2-256",
|
|
||||||
sha3_256 = "cid_sha3-256",
|
|
||||||
sha3_384 = "cid_sha3-384"
|
|
||||||
}
|
|
||||||
declare enum CID_ALGORITHM {
|
|
||||||
crc32 = "crc32",
|
|
||||||
md5 = "md5",
|
|
||||||
sha1 = "sha1",
|
|
||||||
sha256 = "sha2-256",
|
|
||||||
sha3_256 = "sha3-256",
|
|
||||||
sha3_384 = "sha3-384"
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES as C, type MultiHashData as M, type ComputeInterface as a, CID_ALGORITHM_CODES as b, CID_ALGORITHM as c };
|
|
||||||
Vendored
-32
@@ -1,32 +0,0 @@
|
|||||||
// src/lib/hash-compute/MultiHashData.ts
|
|
||||||
var CID_ALGORITHM_CODES = /* @__PURE__ */ ((CID_ALGORITHM_CODES2) => {
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["crc32"] = 306] = "crc32";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["md5"] = 213] = "md5";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha1"] = 17] = "sha1";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha256"] = 18] = "sha256";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_256"] = 22] = "sha3_256";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_384"] = 21] = "sha3_384";
|
|
||||||
return CID_ALGORITHM_CODES2;
|
|
||||||
})(CID_ALGORITHM_CODES || {});
|
|
||||||
var CID_ALGORITHM_NAMES = /* @__PURE__ */ ((CID_ALGORITHM_NAMES2) => {
|
|
||||||
CID_ALGORITHM_NAMES2["crc32"] = "cid_crc32";
|
|
||||||
CID_ALGORITHM_NAMES2["md5"] = "cid_md5";
|
|
||||||
CID_ALGORITHM_NAMES2["sha1"] = "cid_sha1";
|
|
||||||
CID_ALGORITHM_NAMES2["sha256"] = "cid_sha2-256";
|
|
||||||
CID_ALGORITHM_NAMES2["sha3_256"] = "cid_sha3-256";
|
|
||||||
CID_ALGORITHM_NAMES2["sha3_384"] = "cid_sha3-384";
|
|
||||||
return CID_ALGORITHM_NAMES2;
|
|
||||||
})(CID_ALGORITHM_NAMES || {});
|
|
||||||
var CID_ALGORITHM = /* @__PURE__ */ ((CID_ALGORITHM2) => {
|
|
||||||
CID_ALGORITHM2["crc32"] = "crc32";
|
|
||||||
CID_ALGORITHM2["md5"] = "md5";
|
|
||||||
CID_ALGORITHM2["sha1"] = "sha1";
|
|
||||||
CID_ALGORITHM2["sha256"] = "sha2-256";
|
|
||||||
CID_ALGORITHM2["sha3_256"] = "sha3-256";
|
|
||||||
CID_ALGORITHM2["sha3_384"] = "sha3-384";
|
|
||||||
return CID_ALGORITHM2;
|
|
||||||
})(CID_ALGORITHM || {});
|
|
||||||
|
|
||||||
export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES };
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-HTZ763NS.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":";AAiBO,IAAK,sBAAL,kBAAKA,yBAAL;AACH,EAAAA,0CAAA,WAAQ,OAAR;AACA,EAAAA,0CAAA,SAAM,OAAN;AACA,EAAAA,0CAAA,UAAO,MAAP;AACA,EAAAA,0CAAA,YAAS,MAAT;AACA,EAAAA,0CAAA,cAAW,MAAX;AACA,EAAAA,0CAAA,cAAW,MAAX;AANQ,SAAAA;AAAA,GAAA;AASL,IAAK,sBAAL,kBAAKC,yBAAL;AACH,EAAAA,qBAAA,WAAQ;AACR,EAAAA,qBAAA,SAAM;AACN,EAAAA,qBAAA,UAAO;AACP,EAAAA,qBAAA,YAAS;AACT,EAAAA,qBAAA,cAAW;AACX,EAAAA,qBAAA,cAAW;AANH,SAAAA;AAAA,GAAA;AASL,IAAK,gBAAL,kBAAKC,mBAAL;AACH,EAAAA,eAAA,WAAQ;AACR,EAAAA,eAAA,SAAM;AACN,EAAAA,eAAA,UAAO;AACP,EAAAA,eAAA,YAAS;AACT,EAAAA,eAAA,cAAW;AACX,EAAAA,eAAA,cAAW;AANH,SAAAA;AAAA,GAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]}
|
|
||||||
Vendored
-4
@@ -1,4 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-KWYLOVWT.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Vendored
-11
@@ -1,11 +0,0 @@
|
|||||||
import { computeCIDs } from './chunk-SV45D2HN.js';
|
|
||||||
import fs from 'fs';
|
|
||||||
|
|
||||||
async function computeCIDs2({ filePath, algorithms }) {
|
|
||||||
const stream = fs.createReadStream(filePath);
|
|
||||||
return computeCIDs({ stream, algorithms });
|
|
||||||
}
|
|
||||||
|
|
||||||
export { computeCIDs2 as computeCIDs };
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-LTWPBXM5.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["computeCIDs"],"mappings":";;;;;AACA,OAAO,QAAQ;AASf,eAAOA,aAAmC,EAAC,UAAU,WAAU,GAGzC;AAClB,QAAM,SAAS,GAAG,iBAAiB,QAAQ;AAC3C,SAAO,YAAoB,EAAC,QAAQ,WAAU,CAAC;AACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise<string[]> {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]}
|
|
||||||
Vendored
-94
@@ -1,94 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs');
|
|
||||||
var stream = require('stream');
|
|
||||||
var digest = require('multiformats/hashes/digest');
|
|
||||||
var cid = require('multiformats/cid');
|
|
||||||
var crypto = require('crypto');
|
|
||||||
var crc32 = require('crc-32');
|
|
||||||
|
|
||||||
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
||||||
|
|
||||||
var crc32__default = /*#__PURE__*/_interopDefault(crc32);
|
|
||||||
|
|
||||||
var Crc32Hash = class {
|
|
||||||
_crc = void 0;
|
|
||||||
update(data) {
|
|
||||||
this._crc = crc32__default.default.buf(data, this._crc);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
digest() {
|
|
||||||
const buffer = Buffer.alloc(4);
|
|
||||||
buffer.writeInt32BE(this._crc, 0);
|
|
||||||
return buffer;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// src/lib/file-id/CreateHasher.ts
|
|
||||||
var codeTable = {
|
|
||||||
["cid_crc32" /* crc32 */]: 306 /* crc32 */,
|
|
||||||
["cid_md5" /* md5 */]: 213 /* md5 */,
|
|
||||||
["cid_sha1" /* sha1 */]: 17 /* sha1 */,
|
|
||||||
["cid_sha2-256" /* sha256 */]: 18 /* sha256 */,
|
|
||||||
["cid_sha3-256" /* sha3_256 */]: 22 /* sha3_256 */,
|
|
||||||
["cid_sha3-384" /* sha3_384 */]: 21 /* sha3_384 */
|
|
||||||
};
|
|
||||||
async function createHasher(algo) {
|
|
||||||
switch (algo) {
|
|
||||||
case "cid_sha2-256" /* sha256 */:
|
|
||||||
return crypto.createHash("sha256");
|
|
||||||
case "cid_sha1" /* sha1 */:
|
|
||||||
return crypto.createHash("sha1");
|
|
||||||
case "cid_md5" /* md5 */:
|
|
||||||
return crypto.createHash("md5");
|
|
||||||
case "cid_sha3-256" /* sha3_256 */:
|
|
||||||
return crypto.createHash("sha3-256");
|
|
||||||
case "cid_sha3-384" /* sha3_384 */:
|
|
||||||
return crypto.createHash("sha3-384");
|
|
||||||
case "cid_crc32" /* crc32 */:
|
|
||||||
return new Crc32Hash();
|
|
||||||
default:
|
|
||||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// src/lib/file-id/ComputeHash.ts
|
|
||||||
async function computeCIDs({ stream: stream$1, algorithms }) {
|
|
||||||
const hashers = await hasherDefiner(algorithms);
|
|
||||||
if (stream$1 instanceof stream.Readable) {
|
|
||||||
for await (const chunk of stream$1) {
|
|
||||||
for (const item of hashers) {
|
|
||||||
await item.hasher.update(chunk);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const reader = stream$1.getReader();
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done)
|
|
||||||
break;
|
|
||||||
for (const item of hashers) {
|
|
||||||
await item.hasher.update(Buffer.from(value));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cidFinalize(hashers);
|
|
||||||
}
|
|
||||||
async function hasherDefiner(algorithms) {
|
|
||||||
const hashers = algorithms.filter((algo) => Object.values(chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES).includes(algo)).map(async (algo) => ({
|
|
||||||
hasher: await createHasher(algo),
|
|
||||||
code: codeTable[algo]
|
|
||||||
}));
|
|
||||||
return await Promise.all(hashers);
|
|
||||||
}
|
|
||||||
async function cidFinalize(hashers) {
|
|
||||||
return await Promise.all(hashers.map(async ({ code, hasher }) => {
|
|
||||||
const hashBuffer = await hasher.digest();
|
|
||||||
const digest$1 = digest.create(code, hashBuffer);
|
|
||||||
return cid.CID.createV1(code, digest$1).toString();
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
exports.computeCIDs = computeCIDs;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-OHUFSPVO.cjs.map
|
|
||||||
Vendored
-1
File diff suppressed because one or more lines are too long
Vendored
-36
@@ -1,36 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
// src/lib/hash-compute/MultiHashData.ts
|
|
||||||
var CID_ALGORITHM_CODES = /* @__PURE__ */ ((CID_ALGORITHM_CODES2) => {
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["crc32"] = 306] = "crc32";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["md5"] = 213] = "md5";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha1"] = 17] = "sha1";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha256"] = 18] = "sha256";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_256"] = 22] = "sha3_256";
|
|
||||||
CID_ALGORITHM_CODES2[CID_ALGORITHM_CODES2["sha3_384"] = 21] = "sha3_384";
|
|
||||||
return CID_ALGORITHM_CODES2;
|
|
||||||
})(CID_ALGORITHM_CODES || {});
|
|
||||||
var CID_ALGORITHM_NAMES = /* @__PURE__ */ ((CID_ALGORITHM_NAMES2) => {
|
|
||||||
CID_ALGORITHM_NAMES2["crc32"] = "cid_crc32";
|
|
||||||
CID_ALGORITHM_NAMES2["md5"] = "cid_md5";
|
|
||||||
CID_ALGORITHM_NAMES2["sha1"] = "cid_sha1";
|
|
||||||
CID_ALGORITHM_NAMES2["sha256"] = "cid_sha2-256";
|
|
||||||
CID_ALGORITHM_NAMES2["sha3_256"] = "cid_sha3-256";
|
|
||||||
CID_ALGORITHM_NAMES2["sha3_384"] = "cid_sha3-384";
|
|
||||||
return CID_ALGORITHM_NAMES2;
|
|
||||||
})(CID_ALGORITHM_NAMES || {});
|
|
||||||
var CID_ALGORITHM = /* @__PURE__ */ ((CID_ALGORITHM2) => {
|
|
||||||
CID_ALGORITHM2["crc32"] = "crc32";
|
|
||||||
CID_ALGORITHM2["md5"] = "md5";
|
|
||||||
CID_ALGORITHM2["sha1"] = "sha1";
|
|
||||||
CID_ALGORITHM2["sha256"] = "sha2-256";
|
|
||||||
CID_ALGORITHM2["sha3_256"] = "sha3-256";
|
|
||||||
CID_ALGORITHM2["sha3_384"] = "sha3-384";
|
|
||||||
return CID_ALGORITHM2;
|
|
||||||
})(CID_ALGORITHM || {});
|
|
||||||
|
|
||||||
exports.CID_ALGORITHM = CID_ALGORITHM;
|
|
||||||
exports.CID_ALGORITHM_CODES = CID_ALGORITHM_CODES;
|
|
||||||
exports.CID_ALGORITHM_NAMES = CID_ALGORITHM_NAMES;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-S7BSPELE.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/hash-compute/MultiHashData.ts"],"names":["CID_ALGORITHM_CODES","CID_ALGORITHM_NAMES","CID_ALGORITHM"],"mappings":";AAiBO,IAAK,sBAAL,kBAAKA,yBAAL;AACH,EAAAA,0CAAA,WAAQ,OAAR;AACA,EAAAA,0CAAA,SAAM,OAAN;AACA,EAAAA,0CAAA,UAAO,MAAP;AACA,EAAAA,0CAAA,YAAS,MAAT;AACA,EAAAA,0CAAA,cAAW,MAAX;AACA,EAAAA,0CAAA,cAAW,MAAX;AANQ,SAAAA;AAAA,GAAA;AASL,IAAK,sBAAL,kBAAKC,yBAAL;AACH,EAAAA,qBAAA,WAAQ;AACR,EAAAA,qBAAA,SAAM;AACN,EAAAA,qBAAA,UAAO;AACP,EAAAA,qBAAA,YAAS;AACT,EAAAA,qBAAA,cAAW;AACX,EAAAA,qBAAA,cAAW;AANH,SAAAA;AAAA,GAAA;AASL,IAAK,gBAAL,kBAAKC,mBAAL;AACH,EAAAA,eAAA,WAAQ;AACR,EAAAA,eAAA,SAAM;AACN,EAAAA,eAAA,UAAO;AACP,EAAAA,eAAA,YAAS;AACT,EAAAA,eAAA,cAAW;AACX,EAAAA,eAAA,cAAW;AANH,SAAAA;AAAA,GAAA","sourcesContent":["export interface MultiHashData {\n cid_crc32?: string; //0x0132 CRC32 (SFV)\n cid_md5?: string; // 0xd5 md5\n cid_sha1?: string; // 0x11 sha1\n \"cid_sha2-256\"?: string;//0x12 sha2-256\n \"cid_sha3-256\"?: string; // 0x16 sha3-256\n \"cid_sha3_384\"?: string; // 0x16 sha3-256\n}\n\nexport interface ComputeInterface {\n computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;\n}\n\n/**\n * according to\n * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv\n */\nexport enum CID_ALGORITHM_CODES {\n crc32 = 0x0132,\n md5 = 0xd5,\n sha1 = 0x11,\n sha256 = 0x12,\n sha3_256 = 0x16,\n sha3_384 = 0x15,\n}\n\nexport enum CID_ALGORITHM_NAMES {\n crc32 = 'cid_crc32',\n md5 = 'cid_md5',\n sha1 = 'cid_sha1',\n sha256 = 'cid_sha2-256',\n sha3_256 = 'cid_sha3-256',\n sha3_384 = 'cid_sha3-384',\n}\n\nexport enum CID_ALGORITHM {\n crc32 = 'crc32',\n md5 = 'md5',\n sha1 = 'sha1',\n sha256 = 'sha2-256',\n sha3_256 = 'sha3-256',\n sha3_384 = 'sha3-384',\n}"]}
|
|
||||||
Vendored
-88
@@ -1,88 +0,0 @@
|
|||||||
import { CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js';
|
|
||||||
import { Readable } from 'stream';
|
|
||||||
import { create } from 'multiformats/hashes/digest';
|
|
||||||
import { CID } from 'multiformats/cid';
|
|
||||||
import { createHash } from 'crypto';
|
|
||||||
import crc32 from 'crc-32';
|
|
||||||
|
|
||||||
var Crc32Hash = class {
|
|
||||||
_crc = void 0;
|
|
||||||
update(data) {
|
|
||||||
this._crc = crc32.buf(data, this._crc);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
digest() {
|
|
||||||
const buffer = Buffer.alloc(4);
|
|
||||||
buffer.writeInt32BE(this._crc, 0);
|
|
||||||
return buffer;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// src/lib/file-id/CreateHasher.ts
|
|
||||||
var codeTable = {
|
|
||||||
["cid_crc32" /* crc32 */]: 306 /* crc32 */,
|
|
||||||
["cid_md5" /* md5 */]: 213 /* md5 */,
|
|
||||||
["cid_sha1" /* sha1 */]: 17 /* sha1 */,
|
|
||||||
["cid_sha2-256" /* sha256 */]: 18 /* sha256 */,
|
|
||||||
["cid_sha3-256" /* sha3_256 */]: 22 /* sha3_256 */,
|
|
||||||
["cid_sha3-384" /* sha3_384 */]: 21 /* sha3_384 */
|
|
||||||
};
|
|
||||||
async function createHasher(algo) {
|
|
||||||
switch (algo) {
|
|
||||||
case "cid_sha2-256" /* sha256 */:
|
|
||||||
return createHash("sha256");
|
|
||||||
case "cid_sha1" /* sha1 */:
|
|
||||||
return createHash("sha1");
|
|
||||||
case "cid_md5" /* md5 */:
|
|
||||||
return createHash("md5");
|
|
||||||
case "cid_sha3-256" /* sha3_256 */:
|
|
||||||
return createHash("sha3-256");
|
|
||||||
case "cid_sha3-384" /* sha3_384 */:
|
|
||||||
return createHash("sha3-384");
|
|
||||||
case "cid_crc32" /* crc32 */:
|
|
||||||
return new Crc32Hash();
|
|
||||||
default:
|
|
||||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// src/lib/file-id/ComputeHash.ts
|
|
||||||
async function computeCIDs({ stream, algorithms }) {
|
|
||||||
const hashers = await hasherDefiner(algorithms);
|
|
||||||
if (stream instanceof Readable) {
|
|
||||||
for await (const chunk of stream) {
|
|
||||||
for (const item of hashers) {
|
|
||||||
await item.hasher.update(chunk);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const reader = stream.getReader();
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done)
|
|
||||||
break;
|
|
||||||
for (const item of hashers) {
|
|
||||||
await item.hasher.update(Buffer.from(value));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cidFinalize(hashers);
|
|
||||||
}
|
|
||||||
async function hasherDefiner(algorithms) {
|
|
||||||
const hashers = algorithms.filter((algo) => Object.values(CID_ALGORITHM_NAMES).includes(algo)).map(async (algo) => ({
|
|
||||||
hasher: await createHasher(algo),
|
|
||||||
code: codeTable[algo]
|
|
||||||
}));
|
|
||||||
return await Promise.all(hashers);
|
|
||||||
}
|
|
||||||
async function cidFinalize(hashers) {
|
|
||||||
return await Promise.all(hashers.map(async ({ code, hasher }) => {
|
|
||||||
const hashBuffer = await hasher.digest();
|
|
||||||
const digest = create(code, hashBuffer);
|
|
||||||
return CID.createV1(code, digest).toString();
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
export { computeCIDs };
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-SV45D2HN.js.map
|
|
||||||
Vendored
-1
File diff suppressed because one or more lines are too long
Vendored
-3
@@ -1,3 +0,0 @@
|
|||||||
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-XCJZALP2.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Vendored
-17
@@ -1,17 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var chunkOHUFSPVO_cjs = require('./chunk-OHUFSPVO.cjs');
|
|
||||||
var fs = require('fs');
|
|
||||||
|
|
||||||
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
||||||
|
|
||||||
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
|
||||||
|
|
||||||
async function computeCIDs2({ filePath, algorithms }) {
|
|
||||||
const stream = fs__default.default.createReadStream(filePath);
|
|
||||||
return chunkOHUFSPVO_cjs.computeCIDs({ stream, algorithms });
|
|
||||||
}
|
|
||||||
|
|
||||||
exports.computeCIDs = computeCIDs2;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=chunk-Y5FIE5OK.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/file-id/ShaComputeWorker.ts"],"names":["computeCIDs"],"mappings":";;;;;AACA,OAAO,QAAQ;AASf,eAAOA,aAAmC,EAAC,UAAU,WAAU,GAGzC;AAClB,QAAM,SAAS,GAAG,iBAAiB,QAAQ;AAC3C,SAAO,YAAoB,EAAC,QAAQ,WAAU,CAAC;AACnD","sourcesContent":["import {CID_ALGORITHM_NAMES} from \"@root/hash-compute/MultiHashData\";\nimport fs from \"fs\";\nimport {computeCIDs as computeCIDsInternal} from \"./ComputeHash\";\n\n/**\n * Compute the CIDs of a file using specific algorithms\n * @param filePath The path to the file\n * @param algorithms Array of algorithms ('sha256', 'sha1')\n * @returns Array of CIDs (in the order of the algorithms)\n */\nexport default async function computeCIDs({filePath, algorithms}: {\n filePath: string;\n algorithms: CID_ALGORITHM_NAMES[]\n}): Promise<string[]> {\n const stream = fs.createReadStream(filePath);\n return computeCIDsInternal({stream, algorithms});\n}\n"]}
|
|
||||||
Vendored
-43
@@ -1,43 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
require('./chunk-KWYLOVWT.cjs');
|
|
||||||
var chunkOHUFSPVO_cjs = require('./chunk-OHUFSPVO.cjs');
|
|
||||||
var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs');
|
|
||||||
|
|
||||||
// src/lib/hash-compute/HashComputer.ts
|
|
||||||
var HashComputer = class {
|
|
||||||
constructor(targetHash) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
}
|
|
||||||
async computeMissingHash(stream, metadata) {
|
|
||||||
const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]);
|
|
||||||
if (neededHashes.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const cids = await chunkOHUFSPVO_cjs.computeCIDs({ stream, algorithms: neededHashes });
|
|
||||||
for (const [index, cid] of cids.entries()) {
|
|
||||||
const hashType = neededHashes[index];
|
|
||||||
metadata[hashType] = cid;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Object.defineProperty(exports, "computeCIDs", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkOHUFSPVO_cjs.computeCIDs; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_CODES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_NAMES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; }
|
|
||||||
});
|
|
||||||
exports.HashComputer = HashComputer;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index-browser.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":[],"mappings":";;;;;;;;;;;AAKO,IAAM,eAAN,MAAkB;AAAA,EAErB,YAAoB,YAAmC;AAAnC;AAAA,EACpB;AAAA,EAEA,MAAM,mBAAmB,QAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,YAAY,EAAC,QAAQ,YAAW,aAAY,CAAC;AAGhE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void> {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]}
|
|
||||||
Vendored
-23
@@ -1,23 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
export { FileIDComputerInterface, FileProcessorInterface } from './index-interface.cjs';
|
|
||||||
import { Readable } from 'stream';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param stream The Readable stream of the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
declare function computeCIDs({ stream, algorithms }: {
|
|
||||||
stream: Readable | ReadableStream;
|
|
||||||
algorithms: CID_ALGORITHM_NAMES[];
|
|
||||||
}): Promise<string[]>;
|
|
||||||
|
|
||||||
declare class HashComputer {
|
|
||||||
private targetHash;
|
|
||||||
constructor(targetHash: CID_ALGORITHM_NAMES[]);
|
|
||||||
computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, HashComputer, MultiHashData, computeCIDs };
|
|
||||||
Vendored
-23
@@ -1,23 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES, M as MultiHashData } from './MultiHashData-B-scpa5G.js';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface } from './MultiHashData-B-scpa5G.js';
|
|
||||||
export { FileIDComputerInterface, FileProcessorInterface } from './index-interface.js';
|
|
||||||
import { Readable } from 'stream';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param stream The Readable stream of the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
declare function computeCIDs({ stream, algorithms }: {
|
|
||||||
stream: Readable | ReadableStream;
|
|
||||||
algorithms: CID_ALGORITHM_NAMES[];
|
|
||||||
}): Promise<string[]>;
|
|
||||||
|
|
||||||
declare class HashComputer {
|
|
||||||
private targetHash;
|
|
||||||
constructor(targetHash: CID_ALGORITHM_NAMES[]);
|
|
||||||
computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, HashComputer, MultiHashData, computeCIDs };
|
|
||||||
Vendored
-26
@@ -1,26 +0,0 @@
|
|||||||
import './chunk-XCJZALP2.js';
|
|
||||||
import { computeCIDs } from './chunk-SV45D2HN.js';
|
|
||||||
export { computeCIDs } from './chunk-SV45D2HN.js';
|
|
||||||
export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js';
|
|
||||||
|
|
||||||
// src/lib/hash-compute/HashComputer.ts
|
|
||||||
var HashComputer = class {
|
|
||||||
constructor(targetHash) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
}
|
|
||||||
async computeMissingHash(stream, metadata) {
|
|
||||||
const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]);
|
|
||||||
if (neededHashes.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const cids = await computeCIDs({ stream, algorithms: neededHashes });
|
|
||||||
for (const [index, cid] of cids.entries()) {
|
|
||||||
const hashType = neededHashes[index];
|
|
||||||
metadata[hashType] = cid;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export { HashComputer };
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index-browser.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":["../src/lib/hash-compute/HashComputer.ts"],"names":[],"mappings":";;;;;;;;;;;AAKO,IAAM,eAAN,MAAkB;AAAA,EAErB,YAAoB,YAAmC;AAAnC;AAAA,EACpB;AAAA,EAEA,MAAM,mBAAmB,QAAkB,UAAwC;AAE/E,UAAM,eAAe,KAAK,WAAW,OAAO,cAAY,CAAC,SAAS,QAAQ,CAAC;AAG3E,QAAI,aAAa,WAAW,GAAG;AAC3B;AAAA,IACJ;AAGA,UAAM,OAAO,MAAM,YAAY,EAAC,QAAQ,YAAW,aAAY,CAAC;AAGhE,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACvC,YAAM,WAAW,aAAa,KAAK;AACnC,eAAS,QAAQ,IAAI;AAAA,IACzB;AAAA,EACJ;AACJ","sourcesContent":["import {CID_ALGORITHM_NAMES, MultiHashData} from \"./MultiHashData\";\nimport {Readable} from \"stream\";\n\nimport {computeCIDs} from \"@root/file-id/ComputeHash\";\n\nexport class HashComputer{\n\n constructor(private targetHash: CID_ALGORITHM_NAMES[]) {\n }\n\n async computeMissingHash(stream: Readable, metadata: MultiHashData): Promise<void> {\n // Dynamically determine which hashes are needed\n const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);\n\n // If all hashes are already computed, skip the processing\n if (neededHashes.length === 0) {\n return;\n }\n\n // Compute only the needed CIDs\n const cids = await computeCIDs({stream, algorithms:neededHashes});\n\n // Map the computed CIDs back to their respective metadata properties\n for (const [index, cid] of cids.entries()) {\n const hashType = neededHashes[index];\n metadata[hashType] = cid;\n }\n }\n}\n"]}
|
|
||||||
Vendored
-21
@@ -1,21 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
require('./chunk-KWYLOVWT.cjs');
|
|
||||||
var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs');
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_CODES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_NAMES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; }
|
|
||||||
});
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index-interface.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Vendored
-15
@@ -1,15 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
import * as Stream from 'stream';
|
|
||||||
|
|
||||||
interface FileProcessorInterface {
|
|
||||||
processFile(current: number, queueSize: number, nfoFilePath: string): Promise<void>;
|
|
||||||
canProcessFile(filePath: string): Promise<boolean>;
|
|
||||||
deleteFile(filePath: string): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FileIDComputerInterface {
|
|
||||||
computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, type FileIDComputerInterface, type FileProcessorInterface };
|
|
||||||
Vendored
-15
@@ -1,15 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.js';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.js';
|
|
||||||
import * as Stream from 'stream';
|
|
||||||
|
|
||||||
interface FileProcessorInterface {
|
|
||||||
processFile(current: number, queueSize: number, nfoFilePath: string): Promise<void>;
|
|
||||||
canProcessFile(filePath: string): Promise<boolean>;
|
|
||||||
deleteFile(filePath: string): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FileIDComputerInterface {
|
|
||||||
computeCIDs(filePath: Stream, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, type FileIDComputerInterface, type FileProcessorInterface };
|
|
||||||
Vendored
-4
@@ -1,4 +0,0 @@
|
|||||||
import './chunk-XCJZALP2.js';
|
|
||||||
export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js';
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index-interface.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Vendored
-496
@@ -1,496 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
require('./chunk-KWYLOVWT.cjs');
|
|
||||||
require('./chunk-Y5FIE5OK.cjs');
|
|
||||||
require('./chunk-OHUFSPVO.cjs');
|
|
||||||
var chunkS7BSPELE_cjs = require('./chunk-S7BSPELE.cjs');
|
|
||||||
var piscina = require('piscina');
|
|
||||||
var fs = require('fs');
|
|
||||||
var csvParse = require('csv-parse');
|
|
||||||
var sync = require('csv-parse/sync');
|
|
||||||
var sync$1 = require('csv-stringify/sync');
|
|
||||||
var promises = require('fs/promises');
|
|
||||||
var path3 = require('path');
|
|
||||||
var timers = require('timers');
|
|
||||||
var chokidar = require('chokidar');
|
|
||||||
var PQueue = require('p-queue');
|
|
||||||
|
|
||||||
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
||||||
|
|
||||||
function _interopNamespace(e) {
|
|
||||||
if (e && e.__esModule) return e;
|
|
||||||
var n = Object.create(null);
|
|
||||||
if (e) {
|
|
||||||
Object.keys(e).forEach(function (k) {
|
|
||||||
if (k !== 'default') {
|
|
||||||
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
||||||
Object.defineProperty(n, k, d.get ? d : {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return e[k]; }
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
n.default = e;
|
|
||||||
return Object.freeze(n);
|
|
||||||
}
|
|
||||||
|
|
||||||
var path3__namespace = /*#__PURE__*/_interopNamespace(path3);
|
|
||||||
var chokidar__default = /*#__PURE__*/_interopDefault(chokidar);
|
|
||||||
var PQueue__default = /*#__PURE__*/_interopDefault(PQueue);
|
|
||||||
|
|
||||||
var distFolder = undefined;
|
|
||||||
distFolder = distFolder.replace("src", "dist");
|
|
||||||
distFolder = distFolder + "/worker.js";
|
|
||||||
var workerUrl = new URL(distFolder).href;
|
|
||||||
console.log("Worker URL: ", workerUrl);
|
|
||||||
var FileIDComputerWorker = class {
|
|
||||||
piscina;
|
|
||||||
constructor(workerPath) {
|
|
||||||
this.piscina = new piscina.Piscina({
|
|
||||||
maxThreads: 4,
|
|
||||||
//filename: new URL('./ShaComputeWorker.ts', import.meta.url).href
|
|
||||||
filename: workerPath || process.env.WORKER_URL || workerUrl
|
|
||||||
});
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
async computeCIDs(filePath, algorithms) {
|
|
||||||
return this.piscina.run({ filePath, algorithms });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// src/lib/hash-compute/HashComputerWorker.ts
|
|
||||||
var HashComputerWorker = class {
|
|
||||||
constructor(targetHash, workerPath) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
|
||||||
}
|
|
||||||
fileIDComputer;
|
|
||||||
async computeMissingHash(filePath, metadata) {
|
|
||||||
const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]);
|
|
||||||
if (neededHashes.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
|
||||||
for (const [index, cid] of cids.entries()) {
|
|
||||||
const hashType = neededHashes[index];
|
|
||||||
metadata[hashType] = cid;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
async function existsAsync(filePath) {
|
|
||||||
try {
|
|
||||||
await promises.access(filePath, promises.constants.F_OK);
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var INDEX_HEADERS = ["path", "size", "mtime"];
|
|
||||||
var HashIndexManager = class {
|
|
||||||
constructor(filePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */]) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
filePath = filePath.replace(".csv", "");
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
this.filePaths[hash] = `${filePath}-${hash}.csv`;
|
|
||||||
if (!this.filePaths[hash]) {
|
|
||||||
throw new Error(`Invalid index file path for ${hash}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cache = /* @__PURE__ */ new Map();
|
|
||||||
intervalId;
|
|
||||||
intervalTime = 3e4;
|
|
||||||
lastIndexFileSize = 0;
|
|
||||||
//size of the index file last time it was read
|
|
||||||
lastCacheFile;
|
|
||||||
//state of the file last time it was read
|
|
||||||
indexOpsInProgress = false;
|
|
||||||
hasChanged = false;
|
|
||||||
initialLoad;
|
|
||||||
filePaths = {};
|
|
||||||
getCache() {
|
|
||||||
return new Map(this.cache);
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* After init consseutively calls to this method will not reload the index
|
|
||||||
* @param autosave
|
|
||||||
*/
|
|
||||||
async init(autosave = true) {
|
|
||||||
if (!this.initialLoad) {
|
|
||||||
this.initialLoad = new Promise(async (resolve, reject) => {
|
|
||||||
try {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {
|
|
||||||
throw new Error(`Invalid index file headers for ${hash}`);
|
|
||||||
}
|
|
||||||
await this.loadIndex(hash);
|
|
||||||
}
|
|
||||||
if (autosave) {
|
|
||||||
this.start();
|
|
||||||
}
|
|
||||||
resolve();
|
|
||||||
} catch (e) {
|
|
||||||
reject(e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return this.initialLoad;
|
|
||||||
}
|
|
||||||
// Function to check CSV headers
|
|
||||||
checkCSVHeaders(csvContent, hash) {
|
|
||||||
const records = sync.parse(csvContent, {
|
|
||||||
bom: true,
|
|
||||||
columns: true,
|
|
||||||
skip_empty_lines: true
|
|
||||||
});
|
|
||||||
if (records.length === 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
const headers = Object.keys(records[0]);
|
|
||||||
const requiredHeaders = [...INDEX_HEADERS, hash];
|
|
||||||
return requiredHeaders.every((header) => headers.includes(header));
|
|
||||||
}
|
|
||||||
start() {
|
|
||||||
this.startAutoSave(this.intervalTime);
|
|
||||||
}
|
|
||||||
stopAutoSave() {
|
|
||||||
if (this.intervalId) {
|
|
||||||
timers.clearInterval(this.intervalId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
startAutoSave(time) {
|
|
||||||
this.stopAutoSave();
|
|
||||||
this.intervalTime = time;
|
|
||||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
|
||||||
}
|
|
||||||
async loadIndex(hash) {
|
|
||||||
if (await existsAsync(this.filePaths[hash])) {
|
|
||||||
const stats = await fs.promises.stat(this.filePaths[hash]);
|
|
||||||
if (this.lastIndexFileSize !== stats.size) {
|
|
||||||
const records = await this.readCsv(hash);
|
|
||||||
for (const record of records) {
|
|
||||||
let indexLine = this.cache.get(record.path);
|
|
||||||
if (!indexLine) {
|
|
||||||
this.cache.set(record.path, record);
|
|
||||||
} else {
|
|
||||||
indexLine[hash] = record[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.lastIndexFileSize = stats.size;
|
|
||||||
this.lastCacheFile = records;
|
|
||||||
return records;
|
|
||||||
} else {
|
|
||||||
return this.lastCacheFile;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
loadIndexFromCache() {
|
|
||||||
return Array.from(this.cache.values());
|
|
||||||
}
|
|
||||||
async readCsv(hash) {
|
|
||||||
if (!await existsAsync(this.filePaths[hash])) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
const start = performance.now();
|
|
||||||
const parser = csvParse.parse({
|
|
||||||
columns: true,
|
|
||||||
skip_empty_lines: true
|
|
||||||
});
|
|
||||||
const records = [];
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
fs.createReadStream(this.filePaths[hash]).pipe(parser).on("data", (record) => {
|
|
||||||
records.push(record);
|
|
||||||
}).on("end", () => {
|
|
||||||
resolve(records);
|
|
||||||
console.log(`Index read ${hash} time ${performance.now() - start}ms`);
|
|
||||||
}).on("error", (err) => {
|
|
||||||
reject(err);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async saveCacheToFile() {
|
|
||||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.hasChanged = false;
|
|
||||||
this.indexOpsInProgress = true;
|
|
||||||
const start = performance.now();
|
|
||||||
let cacheRows;
|
|
||||||
if (this.cache.size !== 0) {
|
|
||||||
cacheRows = this.loadIndexFromCache();
|
|
||||||
}
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
let existingRows = await this.loadIndex(hash);
|
|
||||||
let existingRowsAsMap = new Map(existingRows.map((row) => [row.path, row]));
|
|
||||||
if (this.cache.size !== 0) {
|
|
||||||
const newRows = cacheRows.filter((row) => {
|
|
||||||
const newRow = !existingRowsAsMap.has(row.path);
|
|
||||||
return newRow && !!row[hash];
|
|
||||||
});
|
|
||||||
if (newRows.length !== 0) {
|
|
||||||
const csvString = sync$1.stringify(newRows, {
|
|
||||||
header: existingRows.length === 0,
|
|
||||||
// Only add header if the file was empty
|
|
||||||
columns: [
|
|
||||||
{ key: "path", header: "path" },
|
|
||||||
{ key: "size", header: "size" },
|
|
||||||
{ key: "mtime", header: "mtime" },
|
|
||||||
{ key: hash, header: hash }
|
|
||||||
]
|
|
||||||
});
|
|
||||||
await fs.promises.appendFile(this.filePaths[hash], csvString);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const totalTime = performance.now() - start;
|
|
||||||
console.log(`Index saved in ${totalTime}ms`);
|
|
||||||
if (totalTime * 10 > this.intervalTime) {
|
|
||||||
this.startAutoSave(totalTime * 10);
|
|
||||||
console.log(`Index save interval increased to ${totalTime * 10}ms`);
|
|
||||||
}
|
|
||||||
this.indexOpsInProgress = false;
|
|
||||||
}
|
|
||||||
async getCidForFileAsync(filePath) {
|
|
||||||
const fileName = path3__namespace.default.basename(filePath);
|
|
||||||
const stats = await promises.stat(filePath);
|
|
||||||
return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());
|
|
||||||
}
|
|
||||||
getCidForFile(filePath, fileSize, mtime) {
|
|
||||||
const fileName = path3__namespace.default.basename(filePath);
|
|
||||||
let fileNameIndex = this.cache.get(fileName);
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!fileNameIndex[hash]) {
|
|
||||||
delete fileNameIndex[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fileNameIndex) {
|
|
||||||
if (fileNameIndex.mtime) {
|
|
||||||
if (fileNameIndex.size === fileSize + "" && fileNameIndex.mtime === mtime) {
|
|
||||||
return fileNameIndex;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (fileNameIndex.size === fileSize + "") {
|
|
||||||
return fileNameIndex;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
addFileCid(filePath, fileSize, mtime, hashs) {
|
|
||||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
|
||||||
throw new Error("Invalid parameters");
|
|
||||||
}
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!hashs[hash]) {
|
|
||||||
throw new Error(`Missing hash ${hash}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const size = fileSize + "";
|
|
||||||
const baseName = path3__namespace.default.basename(filePath);
|
|
||||||
let indexLine = this.cache.get(baseName);
|
|
||||||
if (!indexLine) {
|
|
||||||
let filteredHash = {};
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
filteredHash[hash] = hashs[hash];
|
|
||||||
}
|
|
||||||
const data = { path: baseName, size, mtime, ...filteredHash };
|
|
||||||
this.cache.set(baseName, data);
|
|
||||||
} else {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
indexLine[hash] = hashs[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.hasChanged = true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
var HashComputerIndexCache = class {
|
|
||||||
constructor(indexFilePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */], workerPath) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
|
||||||
this.hashIndexManager = new HashIndexManager(indexFilePath, targetHash);
|
|
||||||
}
|
|
||||||
hashIndexManager;
|
|
||||||
hashComputer;
|
|
||||||
async computeMissingHash(filePath, metadata) {
|
|
||||||
await this.hashIndexManager.init();
|
|
||||||
let stats = await promises.stat(filePath);
|
|
||||||
if (this.hashIndexManager.getCache().has(path3__namespace.default.basename(filePath))) {
|
|
||||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
|
||||||
if (indexLine) {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!metadata[hash] && indexLine[hash]) {
|
|
||||||
metadata[hash] = indexLine[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
|
||||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
|
||||||
}
|
|
||||||
async getHashIndexManager() {
|
|
||||||
await this.hashIndexManager.init();
|
|
||||||
return this.hashIndexManager;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
var FolderWatcher = class {
|
|
||||||
constructor(fileProcessor, WATCH_FOLDER_LIST, config) {
|
|
||||||
this.WATCH_FOLDER_LIST = WATCH_FOLDER_LIST;
|
|
||||||
this.config = config;
|
|
||||||
this.fileProcessor = fileProcessor;
|
|
||||||
}
|
|
||||||
initialized = false;
|
|
||||||
queue = new PQueue__default.default({ concurrency: 100, autoStart: true });
|
|
||||||
fileProcessor;
|
|
||||||
queueSize = -1;
|
|
||||||
current = 0;
|
|
||||||
processing = /* @__PURE__ */ new Set();
|
|
||||||
/**
|
|
||||||
* Process the directory and its subdirectories
|
|
||||||
* @param directory
|
|
||||||
* @param promises
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
async processDirectory(directory, promises$1) {
|
|
||||||
const entries = await promises.readdir(directory, { withFileTypes: true });
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path3__namespace.join(directory, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
await this.processDirectory(fullPath, promises$1);
|
|
||||||
} else {
|
|
||||||
const promise = this.processFile(fullPath);
|
|
||||||
promises$1.push(promise);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async countFile(directory) {
|
|
||||||
const entries = await promises.readdir(directory, { withFileTypes: true });
|
|
||||||
const countPromises = [];
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path3__namespace.join(directory, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
countPromises.push(this.countFile(fullPath));
|
|
||||||
} else {
|
|
||||||
countPromises.push(this.queue.add(
|
|
||||||
async () => await this.fileProcessor.canProcessFile(fullPath) ? 1 : 0
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const counts = await Promise.all(countPromises);
|
|
||||||
return counts.reduce((acc, current) => acc + current, 0);
|
|
||||||
}
|
|
||||||
async processFile(filePath) {
|
|
||||||
if (await this.fileProcessor.canProcessFile(filePath)) {
|
|
||||||
if (!this.processing.has(filePath)) {
|
|
||||||
this.processing.add(filePath);
|
|
||||||
const current = ++this.current;
|
|
||||||
await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));
|
|
||||||
this.processing.delete(filePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Process the file and its sibling file included other folder it is a recursive function
|
|
||||||
* @param filePath
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
async processFileExtended(filePath) {
|
|
||||||
let dirname2 = path3__namespace.dirname(filePath);
|
|
||||||
await this.processDirectory(dirname2, []);
|
|
||||||
}
|
|
||||||
chokidarWatch(folderList) {
|
|
||||||
const chokidarconfig = {
|
|
||||||
ignoreInitial: true,
|
|
||||||
//ignore the initial scan we have our own function
|
|
||||||
persistent: true,
|
|
||||||
depth: Infinity,
|
|
||||||
awaitWriteFinish: {
|
|
||||||
stabilityThreshold: this.config.stabilityThreshold || 3e4,
|
|
||||||
pollInterval: this.config.pollInterval || 5e3
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let watcher;
|
|
||||||
if ((this.config.interval || 0) <= 0) {
|
|
||||||
watcher = chokidar__default.default.watch(folderList, chokidarconfig);
|
|
||||||
} else {
|
|
||||||
watcher = chokidar__default.default.watch(folderList, {
|
|
||||||
...chokidarconfig,
|
|
||||||
usePolling: true,
|
|
||||||
interval: this.config.interval || 0
|
|
||||||
});
|
|
||||||
}
|
|
||||||
watcher.on("add", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error processing file ${filePath}:`, error);
|
|
||||||
}
|
|
||||||
}).on("change", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error processing file ${filePath}:`, e);
|
|
||||||
}
|
|
||||||
}).on("unlink", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.fileProcessor.deleteFile(filePath);
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error processing file deletion:`, e);
|
|
||||||
}
|
|
||||||
}).on("error", (error) => console.error(`Watcher error: ${error}`)).on("ready", () => {
|
|
||||||
console.log(`Watching for file changes on ${folderList}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async watch() {
|
|
||||||
let folders = this.WATCH_FOLDER_LIST;
|
|
||||||
if (!folders) {
|
|
||||||
throw new Error(`No folder to watch`);
|
|
||||||
}
|
|
||||||
let folderList = folders.split(",");
|
|
||||||
this.chokidarWatch(folderList);
|
|
||||||
let countPromises = [];
|
|
||||||
for (let i = 0; i < folderList.length; i++) {
|
|
||||||
folderList[i] = path3__namespace.normalize(folderList[i]);
|
|
||||||
countPromises.push(this.countFile(folderList[i]));
|
|
||||||
}
|
|
||||||
const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);
|
|
||||||
console.log(`Found ${count} files to process`);
|
|
||||||
this.queueSize = count;
|
|
||||||
let promises = [];
|
|
||||||
for (let i = 0; i < folderList.length; i++) {
|
|
||||||
folderList[i] = path3__namespace.normalize(folderList[i]);
|
|
||||||
await this.processDirectory(folderList[i], promises);
|
|
||||||
}
|
|
||||||
await Promise.all(promises);
|
|
||||||
this.initialized = true;
|
|
||||||
console.log(`Watcher ready`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_CODES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_CODES; }
|
|
||||||
});
|
|
||||||
Object.defineProperty(exports, "CID_ALGORITHM_NAMES", {
|
|
||||||
enumerable: true,
|
|
||||||
get: function () { return chunkS7BSPELE_cjs.CID_ALGORITHM_NAMES; }
|
|
||||||
});
|
|
||||||
exports.FileIDComputerWorker = FileIDComputerWorker;
|
|
||||||
exports.FolderWatcher = FolderWatcher;
|
|
||||||
exports.HashComputerIndexCache = HashComputerIndexCache;
|
|
||||||
exports.HashComputerWorker = HashComputerWorker;
|
|
||||||
exports.HashIndexManager = HashIndexManager;
|
|
||||||
exports.INDEX_HEADERS = INDEX_HEADERS;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index.cjs.map
|
|
||||||
Vendored
-1
File diff suppressed because one or more lines are too long
Vendored
-108
@@ -1,108 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
import * as PQueue from 'p-queue';
|
|
||||||
import PQueue__default from 'p-queue';
|
|
||||||
import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
|
|
||||||
import { FileProcessorInterface } from './index-interface.cjs';
|
|
||||||
export { FileIDComputerInterface } from './index-interface.cjs';
|
|
||||||
import 'stream';
|
|
||||||
|
|
||||||
declare class FileIDComputerWorker {
|
|
||||||
private piscina;
|
|
||||||
constructor(workerPath?: string);
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
|
||||||
path: string;
|
|
||||||
size: string;
|
|
||||||
mtime: string;
|
|
||||||
}
|
|
||||||
declare const INDEX_HEADERS: string[];
|
|
||||||
declare class HashIndexManager {
|
|
||||||
private targetHash;
|
|
||||||
private cache;
|
|
||||||
private intervalId;
|
|
||||||
private intervalTime;
|
|
||||||
private lastIndexFileSize;
|
|
||||||
private lastCacheFile;
|
|
||||||
private indexOpsInProgress;
|
|
||||||
private hasChanged;
|
|
||||||
private initialLoad;
|
|
||||||
private filePaths;
|
|
||||||
constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]);
|
|
||||||
getCache(): Map<string, IndexLine>;
|
|
||||||
/**
|
|
||||||
* After init consseutively calls to this method will not reload the index
|
|
||||||
* @param autosave
|
|
||||||
*/
|
|
||||||
init(autosave?: boolean): Promise<void>;
|
|
||||||
private checkCSVHeaders;
|
|
||||||
start(): void;
|
|
||||||
stopAutoSave(): void;
|
|
||||||
private startAutoSave;
|
|
||||||
loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]>;
|
|
||||||
private loadIndexFromCache;
|
|
||||||
private readCsv;
|
|
||||||
saveCacheToFile(): Promise<void>;
|
|
||||||
getCidForFileAsync(filePath: string): Promise<IndexLine>;
|
|
||||||
getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine;
|
|
||||||
addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class HashComputerIndexCache implements ComputeInterface {
|
|
||||||
private targetHash;
|
|
||||||
private hashIndexManager;
|
|
||||||
private hashComputer;
|
|
||||||
constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string);
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
getHashIndexManager(): Promise<HashIndexManager>;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class HashComputerWorker implements ComputeInterface {
|
|
||||||
private targetHash;
|
|
||||||
private fileIDComputer;
|
|
||||||
constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class FolderWatcher {
|
|
||||||
private WATCH_FOLDER_LIST;
|
|
||||||
private config;
|
|
||||||
initialized: boolean;
|
|
||||||
queue: PQueue__default<p_queue_dist_priority_queue_js.default, PQueue.QueueAddOptions>;
|
|
||||||
fileProcessor: FileProcessorInterface;
|
|
||||||
queueSize: number;
|
|
||||||
current: number;
|
|
||||||
processing: Set<string>;
|
|
||||||
constructor(fileProcessor: FileProcessorInterface, WATCH_FOLDER_LIST: string, config: {
|
|
||||||
interval?: number;
|
|
||||||
stabilityThreshold?: number;
|
|
||||||
pollInterval?: number;
|
|
||||||
});
|
|
||||||
/**
|
|
||||||
* Process the directory and its subdirectories
|
|
||||||
* @param directory
|
|
||||||
* @param promises
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
private processDirectory;
|
|
||||||
private countFile;
|
|
||||||
private processFile;
|
|
||||||
/**
|
|
||||||
* Process the file and its sibling file included other folder it is a recursive function
|
|
||||||
* @param filePath
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
private processFileExtended;
|
|
||||||
private chokidarWatch;
|
|
||||||
watch(): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, ComputeInterface, FileIDComputerWorker, FileProcessorInterface, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS, MultiHashData };
|
|
||||||
Vendored
-108
@@ -1,108 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './MultiHashData-B-scpa5G.js';
|
|
||||||
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES } from './MultiHashData-B-scpa5G.js';
|
|
||||||
import * as PQueue from 'p-queue';
|
|
||||||
import PQueue__default from 'p-queue';
|
|
||||||
import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
|
|
||||||
import { FileProcessorInterface } from './index-interface.js';
|
|
||||||
export { FileIDComputerInterface } from './index-interface.js';
|
|
||||||
import 'stream';
|
|
||||||
|
|
||||||
declare class FileIDComputerWorker {
|
|
||||||
private piscina;
|
|
||||||
constructor(workerPath?: string);
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
|
||||||
path: string;
|
|
||||||
size: string;
|
|
||||||
mtime: string;
|
|
||||||
}
|
|
||||||
declare const INDEX_HEADERS: string[];
|
|
||||||
declare class HashIndexManager {
|
|
||||||
private targetHash;
|
|
||||||
private cache;
|
|
||||||
private intervalId;
|
|
||||||
private intervalTime;
|
|
||||||
private lastIndexFileSize;
|
|
||||||
private lastCacheFile;
|
|
||||||
private indexOpsInProgress;
|
|
||||||
private hasChanged;
|
|
||||||
private initialLoad;
|
|
||||||
private filePaths;
|
|
||||||
constructor(filePath: string, targetHash?: CID_ALGORITHM_NAMES[]);
|
|
||||||
getCache(): Map<string, IndexLine>;
|
|
||||||
/**
|
|
||||||
* After init consseutively calls to this method will not reload the index
|
|
||||||
* @param autosave
|
|
||||||
*/
|
|
||||||
init(autosave?: boolean): Promise<void>;
|
|
||||||
private checkCSVHeaders;
|
|
||||||
start(): void;
|
|
||||||
stopAutoSave(): void;
|
|
||||||
private startAutoSave;
|
|
||||||
loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]>;
|
|
||||||
private loadIndexFromCache;
|
|
||||||
private readCsv;
|
|
||||||
saveCacheToFile(): Promise<void>;
|
|
||||||
getCidForFileAsync(filePath: string): Promise<IndexLine>;
|
|
||||||
getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine;
|
|
||||||
addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class HashComputerIndexCache implements ComputeInterface {
|
|
||||||
private targetHash;
|
|
||||||
private hashIndexManager;
|
|
||||||
private hashComputer;
|
|
||||||
constructor(indexFilePath: string, targetHash?: CID_ALGORITHM_NAMES[], workerPath?: string);
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
getHashIndexManager(): Promise<HashIndexManager>;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class HashComputerWorker implements ComputeInterface {
|
|
||||||
private targetHash;
|
|
||||||
private fileIDComputer;
|
|
||||||
constructor(targetHash: CID_ALGORITHM_NAMES[], workerPath?: string);
|
|
||||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
declare class FolderWatcher {
|
|
||||||
private WATCH_FOLDER_LIST;
|
|
||||||
private config;
|
|
||||||
initialized: boolean;
|
|
||||||
queue: PQueue__default<p_queue_dist_priority_queue_js.default, PQueue.QueueAddOptions>;
|
|
||||||
fileProcessor: FileProcessorInterface;
|
|
||||||
queueSize: number;
|
|
||||||
current: number;
|
|
||||||
processing: Set<string>;
|
|
||||||
constructor(fileProcessor: FileProcessorInterface, WATCH_FOLDER_LIST: string, config: {
|
|
||||||
interval?: number;
|
|
||||||
stabilityThreshold?: number;
|
|
||||||
pollInterval?: number;
|
|
||||||
});
|
|
||||||
/**
|
|
||||||
* Process the directory and its subdirectories
|
|
||||||
* @param directory
|
|
||||||
* @param promises
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
private processDirectory;
|
|
||||||
private countFile;
|
|
||||||
private processFile;
|
|
||||||
/**
|
|
||||||
* Process the file and its sibling file included other folder it is a recursive function
|
|
||||||
* @param filePath
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
private processFileExtended;
|
|
||||||
private chokidarWatch;
|
|
||||||
watch(): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CID_ALGORITHM_NAMES, ComputeInterface, FileIDComputerWorker, FileProcessorInterface, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS, MultiHashData };
|
|
||||||
Vendored
-454
@@ -1,454 +0,0 @@
|
|||||||
import './chunk-XCJZALP2.js';
|
|
||||||
import './chunk-LTWPBXM5.js';
|
|
||||||
import './chunk-SV45D2HN.js';
|
|
||||||
export { CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES } from './chunk-HTZ763NS.js';
|
|
||||||
import { Piscina } from 'piscina';
|
|
||||||
import { promises, createReadStream } from 'fs';
|
|
||||||
import { parse as parse$1 } from 'csv-parse';
|
|
||||||
import { parse } from 'csv-parse/sync';
|
|
||||||
import { stringify } from 'csv-stringify/sync';
|
|
||||||
import { stat, readdir, access, constants } from 'fs/promises';
|
|
||||||
import * as path3 from 'path';
|
|
||||||
import path3__default from 'path';
|
|
||||||
import { clearInterval } from 'node:timers';
|
|
||||||
import chokidar from 'chokidar';
|
|
||||||
import PQueue from 'p-queue';
|
|
||||||
|
|
||||||
var distFolder = import.meta.dirname;
|
|
||||||
distFolder = distFolder.replace("src", "dist");
|
|
||||||
distFolder = distFolder + "/worker.js";
|
|
||||||
var workerUrl = new URL(distFolder).href;
|
|
||||||
console.log("Worker URL: ", workerUrl);
|
|
||||||
var FileIDComputerWorker = class {
|
|
||||||
piscina;
|
|
||||||
constructor(workerPath) {
|
|
||||||
this.piscina = new Piscina({
|
|
||||||
maxThreads: 4,
|
|
||||||
//filename: new URL('./ShaComputeWorker.ts', import.meta.url).href
|
|
||||||
filename: workerPath || process.env.WORKER_URL || workerUrl
|
|
||||||
});
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
async computeCIDs(filePath, algorithms) {
|
|
||||||
return this.piscina.run({ filePath, algorithms });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// src/lib/hash-compute/HashComputerWorker.ts
|
|
||||||
var HashComputerWorker = class {
|
|
||||||
constructor(targetHash, workerPath) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
|
||||||
}
|
|
||||||
fileIDComputer;
|
|
||||||
async computeMissingHash(filePath, metadata) {
|
|
||||||
const neededHashes = this.targetHash.filter((hashName) => !metadata[hashName]);
|
|
||||||
if (neededHashes.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
|
||||||
for (const [index, cid] of cids.entries()) {
|
|
||||||
const hashType = neededHashes[index];
|
|
||||||
metadata[hashType] = cid;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
async function existsAsync(filePath) {
|
|
||||||
try {
|
|
||||||
await access(filePath, constants.F_OK);
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var INDEX_HEADERS = ["path", "size", "mtime"];
|
|
||||||
var HashIndexManager = class {
|
|
||||||
constructor(filePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */]) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
filePath = filePath.replace(".csv", "");
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
this.filePaths[hash] = `${filePath}-${hash}.csv`;
|
|
||||||
if (!this.filePaths[hash]) {
|
|
||||||
throw new Error(`Invalid index file path for ${hash}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cache = /* @__PURE__ */ new Map();
|
|
||||||
intervalId;
|
|
||||||
intervalTime = 3e4;
|
|
||||||
lastIndexFileSize = 0;
|
|
||||||
//size of the index file last time it was read
|
|
||||||
lastCacheFile;
|
|
||||||
//state of the file last time it was read
|
|
||||||
indexOpsInProgress = false;
|
|
||||||
hasChanged = false;
|
|
||||||
initialLoad;
|
|
||||||
filePaths = {};
|
|
||||||
getCache() {
|
|
||||||
return new Map(this.cache);
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* After init consseutively calls to this method will not reload the index
|
|
||||||
* @param autosave
|
|
||||||
*/
|
|
||||||
async init(autosave = true) {
|
|
||||||
if (!this.initialLoad) {
|
|
||||||
this.initialLoad = new Promise(async (resolve, reject) => {
|
|
||||||
try {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {
|
|
||||||
throw new Error(`Invalid index file headers for ${hash}`);
|
|
||||||
}
|
|
||||||
await this.loadIndex(hash);
|
|
||||||
}
|
|
||||||
if (autosave) {
|
|
||||||
this.start();
|
|
||||||
}
|
|
||||||
resolve();
|
|
||||||
} catch (e) {
|
|
||||||
reject(e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return this.initialLoad;
|
|
||||||
}
|
|
||||||
// Function to check CSV headers
|
|
||||||
checkCSVHeaders(csvContent, hash) {
|
|
||||||
const records = parse(csvContent, {
|
|
||||||
bom: true,
|
|
||||||
columns: true,
|
|
||||||
skip_empty_lines: true
|
|
||||||
});
|
|
||||||
if (records.length === 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
const headers = Object.keys(records[0]);
|
|
||||||
const requiredHeaders = [...INDEX_HEADERS, hash];
|
|
||||||
return requiredHeaders.every((header) => headers.includes(header));
|
|
||||||
}
|
|
||||||
start() {
|
|
||||||
this.startAutoSave(this.intervalTime);
|
|
||||||
}
|
|
||||||
stopAutoSave() {
|
|
||||||
if (this.intervalId) {
|
|
||||||
clearInterval(this.intervalId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
startAutoSave(time) {
|
|
||||||
this.stopAutoSave();
|
|
||||||
this.intervalTime = time;
|
|
||||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
|
||||||
}
|
|
||||||
async loadIndex(hash) {
|
|
||||||
if (await existsAsync(this.filePaths[hash])) {
|
|
||||||
const stats = await promises.stat(this.filePaths[hash]);
|
|
||||||
if (this.lastIndexFileSize !== stats.size) {
|
|
||||||
const records = await this.readCsv(hash);
|
|
||||||
for (const record of records) {
|
|
||||||
let indexLine = this.cache.get(record.path);
|
|
||||||
if (!indexLine) {
|
|
||||||
this.cache.set(record.path, record);
|
|
||||||
} else {
|
|
||||||
indexLine[hash] = record[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.lastIndexFileSize = stats.size;
|
|
||||||
this.lastCacheFile = records;
|
|
||||||
return records;
|
|
||||||
} else {
|
|
||||||
return this.lastCacheFile;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
loadIndexFromCache() {
|
|
||||||
return Array.from(this.cache.values());
|
|
||||||
}
|
|
||||||
async readCsv(hash) {
|
|
||||||
if (!await existsAsync(this.filePaths[hash])) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
const start = performance.now();
|
|
||||||
const parser = parse$1({
|
|
||||||
columns: true,
|
|
||||||
skip_empty_lines: true
|
|
||||||
});
|
|
||||||
const records = [];
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
createReadStream(this.filePaths[hash]).pipe(parser).on("data", (record) => {
|
|
||||||
records.push(record);
|
|
||||||
}).on("end", () => {
|
|
||||||
resolve(records);
|
|
||||||
console.log(`Index read ${hash} time ${performance.now() - start}ms`);
|
|
||||||
}).on("error", (err) => {
|
|
||||||
reject(err);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async saveCacheToFile() {
|
|
||||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.hasChanged = false;
|
|
||||||
this.indexOpsInProgress = true;
|
|
||||||
const start = performance.now();
|
|
||||||
let cacheRows;
|
|
||||||
if (this.cache.size !== 0) {
|
|
||||||
cacheRows = this.loadIndexFromCache();
|
|
||||||
}
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
let existingRows = await this.loadIndex(hash);
|
|
||||||
let existingRowsAsMap = new Map(existingRows.map((row) => [row.path, row]));
|
|
||||||
if (this.cache.size !== 0) {
|
|
||||||
const newRows = cacheRows.filter((row) => {
|
|
||||||
const newRow = !existingRowsAsMap.has(row.path);
|
|
||||||
return newRow && !!row[hash];
|
|
||||||
});
|
|
||||||
if (newRows.length !== 0) {
|
|
||||||
const csvString = stringify(newRows, {
|
|
||||||
header: existingRows.length === 0,
|
|
||||||
// Only add header if the file was empty
|
|
||||||
columns: [
|
|
||||||
{ key: "path", header: "path" },
|
|
||||||
{ key: "size", header: "size" },
|
|
||||||
{ key: "mtime", header: "mtime" },
|
|
||||||
{ key: hash, header: hash }
|
|
||||||
]
|
|
||||||
});
|
|
||||||
await promises.appendFile(this.filePaths[hash], csvString);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const totalTime = performance.now() - start;
|
|
||||||
console.log(`Index saved in ${totalTime}ms`);
|
|
||||||
if (totalTime * 10 > this.intervalTime) {
|
|
||||||
this.startAutoSave(totalTime * 10);
|
|
||||||
console.log(`Index save interval increased to ${totalTime * 10}ms`);
|
|
||||||
}
|
|
||||||
this.indexOpsInProgress = false;
|
|
||||||
}
|
|
||||||
async getCidForFileAsync(filePath) {
|
|
||||||
const fileName = path3__default.basename(filePath);
|
|
||||||
const stats = await stat(filePath);
|
|
||||||
return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());
|
|
||||||
}
|
|
||||||
getCidForFile(filePath, fileSize, mtime) {
|
|
||||||
const fileName = path3__default.basename(filePath);
|
|
||||||
let fileNameIndex = this.cache.get(fileName);
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!fileNameIndex[hash]) {
|
|
||||||
delete fileNameIndex[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fileNameIndex) {
|
|
||||||
if (fileNameIndex.mtime) {
|
|
||||||
if (fileNameIndex.size === fileSize + "" && fileNameIndex.mtime === mtime) {
|
|
||||||
return fileNameIndex;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (fileNameIndex.size === fileSize + "") {
|
|
||||||
return fileNameIndex;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
addFileCid(filePath, fileSize, mtime, hashs) {
|
|
||||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
|
||||||
throw new Error("Invalid parameters");
|
|
||||||
}
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!hashs[hash]) {
|
|
||||||
throw new Error(`Missing hash ${hash}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const size = fileSize + "";
|
|
||||||
const baseName = path3__default.basename(filePath);
|
|
||||||
let indexLine = this.cache.get(baseName);
|
|
||||||
if (!indexLine) {
|
|
||||||
let filteredHash = {};
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
filteredHash[hash] = hashs[hash];
|
|
||||||
}
|
|
||||||
const data = { path: baseName, size, mtime, ...filteredHash };
|
|
||||||
this.cache.set(baseName, data);
|
|
||||||
} else {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
indexLine[hash] = hashs[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.hasChanged = true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
var HashComputerIndexCache = class {
|
|
||||||
constructor(indexFilePath, targetHash = ["cid_sha1" /* sha1 */, "cid_sha2-256" /* sha256 */], workerPath) {
|
|
||||||
this.targetHash = targetHash;
|
|
||||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
|
||||||
this.hashIndexManager = new HashIndexManager(indexFilePath, targetHash);
|
|
||||||
}
|
|
||||||
hashIndexManager;
|
|
||||||
hashComputer;
|
|
||||||
async computeMissingHash(filePath, metadata) {
|
|
||||||
await this.hashIndexManager.init();
|
|
||||||
let stats = await stat(filePath);
|
|
||||||
if (this.hashIndexManager.getCache().has(path3__default.basename(filePath))) {
|
|
||||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
|
||||||
if (indexLine) {
|
|
||||||
for (const hash of this.targetHash) {
|
|
||||||
if (!metadata[hash] && indexLine[hash]) {
|
|
||||||
metadata[hash] = indexLine[hash];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
|
||||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
|
||||||
}
|
|
||||||
async getHashIndexManager() {
|
|
||||||
await this.hashIndexManager.init();
|
|
||||||
return this.hashIndexManager;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
var FolderWatcher = class {
|
|
||||||
constructor(fileProcessor, WATCH_FOLDER_LIST, config) {
|
|
||||||
this.WATCH_FOLDER_LIST = WATCH_FOLDER_LIST;
|
|
||||||
this.config = config;
|
|
||||||
this.fileProcessor = fileProcessor;
|
|
||||||
}
|
|
||||||
initialized = false;
|
|
||||||
queue = new PQueue({ concurrency: 100, autoStart: true });
|
|
||||||
fileProcessor;
|
|
||||||
queueSize = -1;
|
|
||||||
current = 0;
|
|
||||||
processing = /* @__PURE__ */ new Set();
|
|
||||||
/**
|
|
||||||
* Process the directory and its subdirectories
|
|
||||||
* @param directory
|
|
||||||
* @param promises
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
async processDirectory(directory, promises) {
|
|
||||||
const entries = await readdir(directory, { withFileTypes: true });
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path3.join(directory, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
await this.processDirectory(fullPath, promises);
|
|
||||||
} else {
|
|
||||||
const promise = this.processFile(fullPath);
|
|
||||||
promises.push(promise);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async countFile(directory) {
|
|
||||||
const entries = await readdir(directory, { withFileTypes: true });
|
|
||||||
const countPromises = [];
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fullPath = path3.join(directory, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
countPromises.push(this.countFile(fullPath));
|
|
||||||
} else {
|
|
||||||
countPromises.push(this.queue.add(
|
|
||||||
async () => await this.fileProcessor.canProcessFile(fullPath) ? 1 : 0
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const counts = await Promise.all(countPromises);
|
|
||||||
return counts.reduce((acc, current) => acc + current, 0);
|
|
||||||
}
|
|
||||||
async processFile(filePath) {
|
|
||||||
if (await this.fileProcessor.canProcessFile(filePath)) {
|
|
||||||
if (!this.processing.has(filePath)) {
|
|
||||||
this.processing.add(filePath);
|
|
||||||
const current = ++this.current;
|
|
||||||
await this.queue.add(() => this.fileProcessor.processFile(current, this.queueSize, filePath));
|
|
||||||
this.processing.delete(filePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Process the file and its sibling file included other folder it is a recursive function
|
|
||||||
* @param filePath
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
async processFileExtended(filePath) {
|
|
||||||
let dirname2 = path3.dirname(filePath);
|
|
||||||
await this.processDirectory(dirname2, []);
|
|
||||||
}
|
|
||||||
chokidarWatch(folderList) {
|
|
||||||
const chokidarconfig = {
|
|
||||||
ignoreInitial: true,
|
|
||||||
//ignore the initial scan we have our own function
|
|
||||||
persistent: true,
|
|
||||||
depth: Infinity,
|
|
||||||
awaitWriteFinish: {
|
|
||||||
stabilityThreshold: this.config.stabilityThreshold || 3e4,
|
|
||||||
pollInterval: this.config.pollInterval || 5e3
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let watcher;
|
|
||||||
if ((this.config.interval || 0) <= 0) {
|
|
||||||
watcher = chokidar.watch(folderList, chokidarconfig);
|
|
||||||
} else {
|
|
||||||
watcher = chokidar.watch(folderList, {
|
|
||||||
...chokidarconfig,
|
|
||||||
usePolling: true,
|
|
||||||
interval: this.config.interval || 0
|
|
||||||
});
|
|
||||||
}
|
|
||||||
watcher.on("add", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error processing file ${filePath}:`, error);
|
|
||||||
}
|
|
||||||
}).on("change", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error processing file ${filePath}:`, e);
|
|
||||||
}
|
|
||||||
}).on("unlink", async (filePath) => {
|
|
||||||
try {
|
|
||||||
await this.fileProcessor.deleteFile(filePath);
|
|
||||||
await this.processFileExtended(filePath);
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error processing file deletion:`, e);
|
|
||||||
}
|
|
||||||
}).on("error", (error) => console.error(`Watcher error: ${error}`)).on("ready", () => {
|
|
||||||
console.log(`Watching for file changes on ${folderList}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async watch() {
|
|
||||||
let folders = this.WATCH_FOLDER_LIST;
|
|
||||||
if (!folders) {
|
|
||||||
throw new Error(`No folder to watch`);
|
|
||||||
}
|
|
||||||
let folderList = folders.split(",");
|
|
||||||
this.chokidarWatch(folderList);
|
|
||||||
let countPromises = [];
|
|
||||||
for (let i = 0; i < folderList.length; i++) {
|
|
||||||
folderList[i] = path3.normalize(folderList[i]);
|
|
||||||
countPromises.push(this.countFile(folderList[i]));
|
|
||||||
}
|
|
||||||
const count = (await Promise.all(countPromises)).reduce((acc, current) => acc + current, 0);
|
|
||||||
console.log(`Found ${count} files to process`);
|
|
||||||
this.queueSize = count;
|
|
||||||
let promises = [];
|
|
||||||
for (let i = 0; i < folderList.length; i++) {
|
|
||||||
folderList[i] = path3.normalize(folderList[i]);
|
|
||||||
await this.processDirectory(folderList[i], promises);
|
|
||||||
}
|
|
||||||
await Promise.all(promises);
|
|
||||||
this.initialized = true;
|
|
||||||
console.log(`Watcher ready`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export { FileIDComputerWorker, FolderWatcher, HashComputerIndexCache, HashComputerWorker, HashIndexManager, INDEX_HEADERS };
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=index.js.map
|
|
||||||
Vendored
-1
File diff suppressed because one or more lines are too long
Vendored
-11
@@ -1,11 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var chunkY5FIE5OK_cjs = require('./chunk-Y5FIE5OK.cjs');
|
|
||||||
require('./chunk-OHUFSPVO.cjs');
|
|
||||||
require('./chunk-S7BSPELE.cjs');
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
module.exports = chunkY5FIE5OK_cjs.computeCIDs;
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=worker.cjs.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Vendored
-14
@@ -1,14 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.cjs';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
declare function computeCIDs({ filePath, algorithms }: {
|
|
||||||
filePath: string;
|
|
||||||
algorithms: CID_ALGORITHM_NAMES[];
|
|
||||||
}): Promise<string[]>;
|
|
||||||
|
|
||||||
export { computeCIDs as default };
|
|
||||||
Vendored
-14
@@ -1,14 +0,0 @@
|
|||||||
import { C as CID_ALGORITHM_NAMES } from './MultiHashData-B-scpa5G.js';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the CIDs of a file using specific algorithms
|
|
||||||
* @param filePath The path to the file
|
|
||||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
|
||||||
* @returns Array of CIDs (in the order of the algorithms)
|
|
||||||
*/
|
|
||||||
declare function computeCIDs({ filePath, algorithms }: {
|
|
||||||
filePath: string;
|
|
||||||
algorithms: CID_ALGORITHM_NAMES[];
|
|
||||||
}): Promise<string[]>;
|
|
||||||
|
|
||||||
export { computeCIDs as default };
|
|
||||||
Vendored
-5
@@ -1,5 +0,0 @@
|
|||||||
export { computeCIDs as default } from './chunk-LTWPBXM5.js';
|
|
||||||
import './chunk-SV45D2HN.js';
|
|
||||||
import './chunk-HTZ763NS.js';
|
|
||||||
//# sourceMappingURL=out.js.map
|
|
||||||
//# sourceMappingURL=worker.js.map
|
|
||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
{"version":3,"sources":[],"names":[],"mappings":""}
|
|
||||||
Reference in New Issue
Block a user