metastar/meta-hash
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding suport for CRC32 SFV

Browse Source
This commit is contained in:
Metazla
2024-04-06 11:48:18 +02:00
parent dcc51bf41d
commit e28f924837
28 changed files with 353 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/ComputeHashIndexCache.ts
+2 -2
View File
@@ -9,8 +9,8 @@ export class ComputeHashIndexCache implements ComputeInterface {
hashComputer: HashComputer;
constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
this.hashComputer = new HashComputer(this.targetHash,workerPath);
this.hashIndexManager = new HashIndexManager(indexFilePath);
this.hashComputer = new HashComputer(targetHash,workerPath);
this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash);
}
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
src/lib/HashIndexManager.ts
+72 -43
View File
@@ -12,6 +12,7 @@ interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
size: string;
mtime: string;
}
export const INDEX_HEADERS = ['path', 'size', 'mtime'];
export class HashIndexManager {
@@ -22,11 +23,16 @@ export class HashIndexManager {
private lastCacheFile: IndexLine[]; //state of the file last time it was read
private indexOpsInProgress: boolean = false;
private hasChanged: boolean = false;
private initialLoad;
private initialLoad: Promise<void>;
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
constructor(private filePath: string,private targetHash:CID_ALGORITHM_NAMES[]= [CID_ALGORITHM_NAMES.sha1,CID_ALGORITHM_NAMES.sha256]) {
if (!filePath) {
throw new Error('Invalid index file path');
constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) {
filePath = filePath.replace(".csv", "");
for (const hash of this.targetHash) {
this.filePaths[hash] = `${filePath}-${hash}.csv`;
if (!this.filePaths[hash]) {
throw new Error(`Invalid index file path for ${hash}`);
}
}
}
@@ -38,19 +44,21 @@ export class HashIndexManager {
* After init consseutively calls to this method will not reload the index
* @param autosave
*/
async init(autosave = true) {
public async init(autosave = true) {
if (!this.initialLoad) {
this.initialLoad = new Promise<void>(async (resolve, reject) => {
try {
if (!this.checkCSVHeaders(this.filePath)) {
throw new Error('Invalid index file headers');
for (const hash of this.targetHash) {
if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {
throw new Error(`Invalid index file headers for ${hash}`);
}
await this.loadIndex(hash);
}
await this.loadIndex();
if (autosave) {
this.start();
}
resolve();
}catch (e) {
} catch (e) {
reject(e);
}
});
@@ -59,7 +67,7 @@ export class HashIndexManager {
}
// Function to check CSV headers
checkCSVHeaders (csvContent: string): boolean {
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
const records = parseSync(csvContent, {
bom: true,
columns: true,
@@ -76,7 +84,7 @@ export class HashIndexManager {
// Define required headers
//from the IndexLine interface
const requiredHeaders = ['path', 'size', 'mtime', ...this.targetHash];
const requiredHeaders = [...INDEX_HEADERS, hash];
// Check if all required headers are present
return requiredHeaders.every(header => headers.includes(header));
@@ -98,15 +106,21 @@ export class HashIndexManager {
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
}
public async loadIndex(): Promise<IndexLine[]> {
if (await existsAsync(this.filePath)) {
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
if (await existsAsync(this.filePaths[hash])) {
// check the file size and if it did not change, do not read the file
const stats = await fs.stat(this.filePath);
const stats = await fs.stat(this.filePaths[hash]);
if (this.lastIndexFileSize !== stats.size) {
// Read existing file content and parse it
const records: IndexLine[] = await this.readCsv();
const records: IndexLine[] = await this.readCsv(hash);
for (const record of records) {
this.cache.set(record.path, record);
let indexLine = this.cache.get(record.path);
if (!indexLine) {
this.cache.set(record.path, record);
} else {
//update the cache with the latest data
indexLine[hash] = record[hash];
}
}
this.lastIndexFileSize = stats.size;
this.lastCacheFile = records;
@@ -122,8 +136,8 @@ export class HashIndexManager {
return Array.from(this.cache.values());
}
private async readCsv(): Promise<IndexLine[]> {
if (!(await existsAsync(this.filePath))) {
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
if (!(await existsAsync(this.filePaths[hash]))) {
return [];
}
const start = performance.now();
@@ -136,13 +150,13 @@ export class HashIndexManager {
const records: IndexLine[] = [];
return new Promise((resolve, reject) => {
createReadStream(this.filePath)
createReadStream(this.filePaths[hash])
.pipe(parser)
.on('data', (record) => {
records.push(record);
}).on('end', () => {
resolve(records);
console.log(`Index read time ${performance.now() - start}ms`);
console.log(`Index read ${hash} time ${performance.now() - start}ms`);
}).on('error', (err) => {
reject(err);
});
@@ -156,27 +170,29 @@ export class HashIndexManager {
this.hasChanged = false;
this.indexOpsInProgress = true;
const start = performance.now();
let existingRows: IndexLine[] = await this.loadIndex();
if (this.cache.size !== 0) {
const cacheRows: IndexLine[] = this.loadIndexFromCache();
// Filter out cacheRows that are already in the file
const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));
for (const hash of this.targetHash) {
let existingRows: IndexLine[] = await this.loadIndex(hash);
if (this.cache.size !== 0) {
const cacheRows: IndexLine[] = this.loadIndexFromCache();
// Filter out cacheRows that are already in the file
const newRows = cacheRows.filter(row => !existingRows.find(existingRow => existingRow.path === row.path));
if (newRows.length !== 0) {
// Serialize new cacheRows to CSV string
const csvString = stringify(newRows, {
header: existingRows.length === 0, // Only add header if the file was empty
columns: [
{key: 'path', header: 'path'},
{key: 'size', header: 'size'},
{key: 'mtime', header: 'mtime'},
...this.targetHash.map(hash => ({key: hash, header: hash})),
],
});
if (newRows.length !== 0) {
// Serialize new cacheRows to CSV string
const csvString = stringify(newRows, {
header: existingRows.length === 0, // Only add header if the file was empty
columns: [
{key: 'path', header: 'path'},
{key: 'size', header: 'size'},
{key: 'mtime', header: 'mtime'},
{key: hash, header: hash},
],
});
// Append new cacheRows to the file
await fs.appendFile(this.filePath, csvString);
// Append new cacheRows to the file
await fs.appendFile(this.filePaths[hash], csvString);
}
}
}
const totalTime = performance.now() - start;
@@ -192,13 +208,13 @@ export class HashIndexManager {
public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {
const fileName = path.basename(filePath);
let fileNameIndex = this.cache.get(fileName);
if(fileNameIndex) {
if(fileNameIndex.mtime) {
if (fileNameIndex) {
if (fileNameIndex.mtime) {
//if we have a mtime, we need to check it
if (fileNameIndex.size === (fileSize + "") && fileNameIndex.mtime === mtime) {
return fileNameIndex;
}
}else {
} else {
//mtime is optional
if (fileNameIndex.size === (fileSize + "")) {
return fileNameIndex;
@@ -219,10 +235,23 @@ export class HashIndexManager {
if (!filePath || !fileSize || !mtime || !hashs) {
throw new Error('Invalid parameters');
}
for (const hash of this.targetHash) {
if (!hashs[hash]) {
throw new Error(`Missing hash ${hash}`);
}
}
const size = fileSize + "";
const baseName = path.basename(filePath);
const data = {path: baseName, size: size, mtime: mtime, ...hashs};
this.cache.set(baseName, data);
let indexLine = this.cache.get(baseName);
if (!indexLine) {
const data = {path: baseName, size: size, mtime: mtime, ...hashs};
this.cache.set(baseName, data);
} else {
//update the cache with the latest data
for (const hash of this.targetHash) {
indexLine[hash] = hashs[hash];
}
}
this.hasChanged = true;
}
src/lib/MultiHashData.ts
+17 -10
View File
@@ -3,33 +3,40 @@ export interface MultiHashData {
cid_sha1?: string; // 0x11 sha1
cid_md5?: string; // 0xd5 md5
"cid_sha3-256"?: string; // 0x16 sha3-256
//cid_crc32?: string; //0x0132 CRC32 (SFV) TODO
cid_crc32?: string; //0x0132 CRC32 (SFV)
}
export interface ComputeInterface {
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
}
/**
* according to
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
*/
export enum CID_ALGORITHM_CODES {
sha256 = 0x12,
sha1 = 0x11,
crc32 = 0x0132,
md5 = 0xd5,
sha1 = 0x11,
sha256 = 0x12,
sha3_256 = 0x16,
//crc32 = 0x0132
sha3_384 = 0x15,
}
export enum CID_ALGORITHM_NAMES {
sha256 = 'cid_sha2-256',
sha1 = 'cid_sha1',
crc32 = 'cid_crc32',
md5 = 'cid_md5',
sha1 = 'cid_sha1',
sha256 = 'cid_sha2-256',
sha3_256 = 'cid_sha3-256',
//crc32 = 'cid_crc32'
sha3_384 = 'cid_sha3-384',
}
export enum CID_ALGORITHM {
sha256 = 'sha2-256',
sha1 = 'sha1',
crc32 = 'crc32',
md5 = 'md5',
sha1 = 'sha1',
sha256 = 'sha2-256',
sha3_256 = 'sha3-256',
//crc32 = 'crc32'
sha3_384 = 'sha3-384',
}
src/lib/fileID/ShaComputeWorker.ts
+38 -13
View File
@@ -1,14 +1,33 @@
import { createReadStream } from "fs";
import {BinaryLike, createHash} from 'crypto';
import { CID } from 'multiformats/cid';
import { create } from "multiformats/hashes/digest";
import {CID_ALGORITHM, CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData";
import {createReadStream} from "fs";
import {createHash} from 'crypto';
import {CID} from 'multiformats/cid';
import {create} from "multiformats/hashes/digest";
import crc32 from 'crc-32';
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../MultiHashData";
export interface SimpleHash {
update(data: BinaryLike),
update(data: Buffer),
digest(): Buffer;
}
class Crc32Hash implements SimpleHash {
private _crc: number = undefined;
update(data: Buffer): Crc32Hash {
// Update the CRC-32 checksum with the new chunk of data
this._crc = crc32.buf(data, this._crc);
return this;
}
digest(): Buffer {
console.log((this._crc>>>0).toString(16)) ;
const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
return buffer;
}
}
/**
* Compute the CIDs of a file using specific algorithms
@@ -16,13 +35,19 @@ export interface SimpleHash {
* @param algorithms Array of algorithms ('sha256', 'sha1')
* @returns Array of CIDs (in the order of the algorithms)
*/
export default async function computeCIDs({ filePath, algorithms }: { filePath: string; algorithms: CID_ALGORITHM_NAMES[] }): Promise<string[]> {
export default async function computeCIDs({filePath, algorithms}: {
filePath: string;
algorithms: CID_ALGORITHM_NAMES[]
}): Promise<string[]> {
const hashFunctions: Record<string, { code: number, hasher: SimpleHash }>
= {
[CID_ALGORITHM_NAMES.sha256]: { code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256') },
[CID_ALGORITHM_NAMES.sha1]: { code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1') },
[CID_ALGORITHM_NAMES.md5]: { code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5') },
[CID_ALGORITHM_NAMES.sha3_256]: { code: CID_ALGORITHM_CODES.sha3_256, hasher:createHash('sha3-256') }
[CID_ALGORITHM_NAMES.sha256]: {code: CID_ALGORITHM_CODES.sha256, hasher: createHash('sha256')},
[CID_ALGORITHM_NAMES.sha1]: {code: CID_ALGORITHM_CODES.sha1, hasher: createHash('sha1')},
[CID_ALGORITHM_NAMES.md5]: {code: CID_ALGORITHM_CODES.md5, hasher: createHash('md5')},
[CID_ALGORITHM_NAMES.sha3_256]: {code: CID_ALGORITHM_CODES.sha3_256, hasher: createHash('sha3-256')},
[CID_ALGORITHM_NAMES.sha3_384]: {code: CID_ALGORITHM_CODES.sha3_384, hasher: createHash('sha3-384')},
[CID_ALGORITHM_NAMES.crc32]: {code: CID_ALGORITHM_CODES.crc32, hasher: new Crc32Hash()}
};
const hashers = algorithms.map(algo => ({
@@ -33,11 +58,11 @@ export default async function computeCIDs({ filePath, algorithms }: { filePath:
const stream = createReadStream(filePath);
for await (const chunk of stream) {
for (const item of hashers) {
item.hasher.update(chunk);
item.hasher.update(chunk as Buffer);
}
}
const cids = await Promise.all(hashers.map(async ({ code, hasher }) => {
const cids = await Promise.all(hashers.map(async ({code, hasher}) => {
const hashBuffer = hasher.digest();
const digest = create(code, hashBuffer);
return CID.createV1(code, digest).toString();
src/tests/main.test.ts
+23 -8
View File
@@ -1,27 +1,42 @@
import {HashComputer} from "../lib";
import {CID_ALGORITHM_NAMES} from "../lib";
import {CID_ALGORITHM_NAMES, HashComputer} from "../lib";
import {describe, it, expect} from 'vitest';
import {describe, expect, it} from 'vitest';
import * as process from "process";
process.env.WORKER_URL = "./dist/worker.js";
/**
* /!\ WARNING /!\
* Remember to build the worker each time you change the worker code.
*/
describe('add', () => {
it('meta data compute', async () => {
let hashComputer = new HashComputer([
CID_ALGORITHM_NAMES.sha1,
CID_ALGORITHM_NAMES.sha256,
CID_ALGORITHM_NAMES.md5,
CID_ALGORITHM_NAMES.sha3_256
CID_ALGORITHM_NAMES.sha3_256,
CID_ALGORITHM_NAMES.crc32,
CID_ALGORITHM_NAMES.sha3_384
]);
let metadata = {};
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
console.log(metadata);
/**
* test.txt CF20B62A (CRC32)
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
*/
expect(metadata).toEqual({
cid_sha1: 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
'cid_sha2-256': 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
cid_md5: 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
'cid_sha3-256': 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq'
"cid_crc32": "bagzafmqcathsbnrk",
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
});
});
});