This commit is contained in:
Metazla
2024-04-06 16:57:21 +02:00
parent 5045120fbb
commit b28ccd5479
24 changed files with 88 additions and 81 deletions
@@ -1,9 +1,10 @@
interface MultiHashData {
"cid_sha2-256"?: string;
cid_sha1?: string;
cid_md5?: string;
"cid_sha3-256"?: string;
cid_crc32?: string;
cid_md5?: string;
cid_sha1?: string;
"cid_sha2-256"?: string;
"cid_sha3-256"?: string;
"cid_sha3_384"?: string;
}
interface ComputeInterface {
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
@@ -1,9 +1,10 @@
interface MultiHashData {
"cid_sha2-256"?: string;
cid_sha1?: string;
cid_md5?: string;
"cid_sha3-256"?: string;
cid_crc32?: string;
cid_md5?: string;
cid_sha1?: string;
"cid_sha2-256"?: string;
"cid_sha3-256"?: string;
"cid_sha3_384"?: string;
}
interface ComputeInterface {
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
+11
View File
@@ -0,0 +1,11 @@
import { createReadStream } from 'fs';
import { createHash } from 'crypto';
import { CID } from 'multiformats/cid';
import { create } from 'multiformats/hashes/digest';
import x from 'crc-32';
var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))}
export { o as a, d as b, u as c, B as d };
//# sourceMappingURL=out.js.map
//# sourceMappingURL=chunk-4EZ47WAR.js.map
+1
View File
File diff suppressed because one or more lines are too long
-20
View File
@@ -1,20 +0,0 @@
'use strict';
var fs = require('fs');
var crypto = require('crypto');
var cid = require('multiformats/cid');
var digest = require('multiformats/hashes/digest');
var x = require('crc-32');
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
var x__default = /*#__PURE__*/_interopDefault(x);
var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=fs.createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))}
exports.a = o;
exports.b = n;
exports.c = u;
exports.d = B;
//# sourceMappingURL=out.js.map
//# sourceMappingURL=chunk-A5PD5ID4.cjs.map
-1
View File
File diff suppressed because one or more lines are too long
+20
View File
@@ -0,0 +1,20 @@
'use strict';
var fs = require('fs');
var crypto = require('crypto');
var cid = require('multiformats/cid');
var digest = require('multiformats/hashes/digest');
var x = require('crc-32');
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
var x__default = /*#__PURE__*/_interopDefault(x);
var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),d=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(d||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x__default.default.buf(a,this._crc),this}digest(){let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:h,algorithms:a}){let c={"cid_sha2-256":{code:18,hasher:crypto.createHash("sha256")},cid_sha1:{code:17,hasher:crypto.createHash("sha1")},cid_md5:{code:213,hasher:crypto.createHash("md5")},"cid_sha3-256":{code:22,hasher:crypto.createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:crypto.createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:c[s].hasher,code:c[s].code})),n=fs.createReadStream(h);for await(let s of n)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=digest.create(s,m);return cid.CID.createV1(s,f).toString()}))}
exports.a = o;
exports.b = d;
exports.c = u;
exports.d = B;
//# sourceMappingURL=out.js.map
//# sourceMappingURL=chunk-ASEKTFOG.cjs.map
+1
View File
File diff suppressed because one or more lines are too long
-11
View File
@@ -1,11 +0,0 @@
import { createReadStream } from 'fs';
import { createHash } from 'crypto';
import { CID } from 'multiformats/cid';
import { create } from 'multiformats/hashes/digest';
import x from 'crc-32';
var o=(s=>(s[s.crc32=306]="crc32",s[s.md5=213]="md5",s[s.sha1=17]="sha1",s[s.sha256=18]="sha256",s[s.sha3_256=22]="sha3_256",s[s.sha3_384=21]="sha3_384",s))(o||{}),n=(s=>(s.crc32="cid_crc32",s.md5="cid_md5",s.sha1="cid_sha1",s.sha256="cid_sha2-256",s.sha3_256="cid_sha3-256",s.sha3_384="cid_sha3-384",s))(n||{}),u=(s=>(s.crc32="crc32",s.md5="md5",s.sha1="sha1",s.sha256="sha2-256",s.sha3_256="sha3-256",s.sha3_384="sha3-384",s))(u||{});var t=class{_crc=void 0;update(a){return this._crc=x.buf(a,this._crc),this}digest(){console.log((this._crc>>>0).toString(16));let a=Buffer.alloc(4);return a.writeInt32BE(this._crc,0),a}};async function B({filePath:c,algorithms:a}){let h={"cid_sha2-256":{code:18,hasher:createHash("sha256")},cid_sha1:{code:17,hasher:createHash("sha1")},cid_md5:{code:213,hasher:createHash("md5")},"cid_sha3-256":{code:22,hasher:createHash("sha3-256")},"cid_sha3-384":{code:21,hasher:createHash("sha3-384")},cid_crc32:{code:306,hasher:new t}},i=a.map(s=>({hasher:h[s].hasher,code:h[s].code})),d=createReadStream(c);for await(let s of d)for(let r of i)r.hasher.update(s);return await Promise.all(i.map(async({code:s,hasher:r})=>{let m=r.digest(),f=create(s,m);return CID.createV1(s,f).toString()}))}
export { o as a, n as b, u as c, B as d };
//# sourceMappingURL=out.js.map
//# sourceMappingURL=chunk-FGATABDV.js.map
-1
View File
File diff suppressed because one or more lines are too long
+12 -12
View File
File diff suppressed because one or more lines are too long
+1 -1
View File
File diff suppressed because one or more lines are too long
+2 -2
View File
@@ -1,5 +1,5 @@
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.cjs';
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.cjs';
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.cjs';
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.cjs';
import * as PQueue from 'p-queue';
import PQueue__default from 'p-queue';
import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
+2 -2
View File
@@ -1,5 +1,5 @@
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-BzBitaFi.js';
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-BzBitaFi.js';
import { C as CID_ALGORITHM_NAMES, a as ComputeInterface, M as MultiHashData } from './ShaComputeWorker-CNdGUbtT.js';
export { c as CID_ALGORITHM, b as CID_ALGORITHM_CODES, S as SimpleHash } from './ShaComputeWorker-CNdGUbtT.js';
import * as PQueue from 'p-queue';
import PQueue__default from 'p-queue';
import * as p_queue_dist_priority_queue_js from 'p-queue/dist/priority-queue.js';
+5 -5
View File
File diff suppressed because one or more lines are too long
+1 -1
View File
File diff suppressed because one or more lines are too long
+2 -2
View File
@@ -1,9 +1,9 @@
'use strict';
var chunkA5PD5ID4_cjs = require('./chunk-A5PD5ID4.cjs');
var chunkASEKTFOG_cjs = require('./chunk-ASEKTFOG.cjs');
module.exports = chunkA5PD5ID4_cjs.d;
module.exports = chunkASEKTFOG_cjs.d;
//# sourceMappingURL=out.js.map
//# sourceMappingURL=worker.cjs.map
+1 -1
View File
@@ -1 +1 @@
export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.cjs';
export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.cjs';
+1 -1
View File
@@ -1 +1 @@
export { S as SimpleHash, d as default } from './ShaComputeWorker-BzBitaFi.js';
export { S as SimpleHash, d as default } from './ShaComputeWorker-CNdGUbtT.js';
+1 -1
View File
@@ -1,3 +1,3 @@
export { d as default } from './chunk-FGATABDV.js';
export { d as default } from './chunk-4EZ47WAR.js';
//# sourceMappingURL=out.js.map
//# sourceMappingURL=worker.js.map
+2 -2
View File
@@ -21,9 +21,9 @@ export class HashComputer implements ComputeInterface{
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
// Map the computed CIDs back to their respective metadata properties
cids.forEach((cid, index) => {
for (const [index, cid] of cids.entries()) {
const hashType = neededHashes[index];
metadata[hashType] = cid;
});
}
}
}
+6 -1
View File
@@ -273,7 +273,12 @@ export class HashIndexManager {
const baseName = path.basename(filePath);
let indexLine = this.cache.get(baseName);
if (!indexLine) {
const data = {path: baseName, size: size, mtime: mtime, ...hashs};
//filter only the hashes we need
let filteredHash = {};
for (const hash of this.targetHash) {
filteredHash[hash] = hashs[hash];
}
const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};
this.cache.set(baseName, data);
} else {
//update the cache with the latest data
+5 -4
View File
@@ -1,9 +1,10 @@
export interface MultiHashData {
"cid_sha2-256"?: string;//0x12 sha2-256
cid_sha1?: string; // 0x11 sha1
cid_md5?: string; // 0xd5 md5
"cid_sha3-256"?: string; // 0x16 sha3-256
cid_crc32?: string; //0x0132 CRC32 (SFV)
cid_md5?: string; // 0xd5 md5
cid_sha1?: string; // 0x11 sha1
"cid_sha2-256"?: string;//0x12 sha2-256
"cid_sha3-256"?: string; // 0x16 sha3-256
"cid_sha3_384"?: string; // 0x16 sha3-256
}
export interface ComputeInterface {
-1
View File
@@ -21,7 +21,6 @@ class Crc32Hash implements SimpleHash {
}
digest(): Buffer {
console.log((this._crc>>>0).toString(16)) ;
const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
return buffer;