diff --git a/src/lib/file-id/CodeTable.ts b/src/lib/file-id/CodeTable.ts index e0e0885..dd19e03 100644 --- a/src/lib/file-id/CodeTable.ts +++ b/src/lib/file-id/CodeTable.ts @@ -6,5 +6,6 @@ export const codeTable = { [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, - [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384 + [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384, + [CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256 } \ No newline at end of file diff --git a/src/lib/file-id/FastHash.ts b/src/lib/file-id/FastHash.ts new file mode 100644 index 0000000..ef0f74c --- /dev/null +++ b/src/lib/file-id/FastHash.ts @@ -0,0 +1,104 @@ +import { createHash } from 'crypto'; +import { open, stat } from 'fs/promises'; +import { create } from 'multiformats/hashes/digest'; +import { CID } from 'multiformats/cid'; +import { CID_ALGORITHM_CODES } from '../hash-compute/MultiHashData.js'; + +/** + * Computes a midhash256 based on sampling the middle 1MB of a file + * plus the file size as a 64-bit integer prefix. + * + * Algorithm: + * - For files <= 1MB: Hashes entire file content + size prefix + * - For files > 1MB: Hashes middle 1MB + size prefix + * + * The size prefix ensures that files with identical middle content + * but different sizes produce different hashes. + * + * Performance: ~20ms regardless of file size (O(1) complexity) + * + * @param filePath - Path to file + * @returns CID string in proper CID v1 format with multicodec 0x1000 + */ +export async function computeMidHash256(filePath: string): Promise { + const SAMPLE_SIZE = 1024 * 1024; // 1MB + + // Get file size (instant - no file read) + const stats = await stat(filePath); + const fileSize = stats.size; + + // Create size buffer (64-bit big-endian) + const sizeBuffer = Buffer.allocUnsafe(8); + sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); + + // Extract sample data + let sampleData: Buffer; + if (fileSize <= SAMPLE_SIZE) { + // Small file: read entire content + const fd = await open(filePath, 'r'); + sampleData = Buffer.allocUnsafe(fileSize); + await fd.read(sampleData, 0, fileSize, 0); + await fd.close(); + } else { + // Large file: read middle 1MB + const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); + const fd = await open(filePath, 'r'); + sampleData = Buffer.allocUnsafe(SAMPLE_SIZE); + await fd.read(sampleData, 0, SAMPLE_SIZE, middleOffset); + await fd.close(); + } + + // Compute SHA-256 hash of [size + sample] + const hashInput = Buffer.concat([sizeBuffer, sampleData]); + const hashBuffer = createHash('sha256') + .update(hashInput) + .digest(); + + // Wrap in CID format with custom multicodec 0x1000 + const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer); + return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString(); +} + +/** + * Synchronous version of computeMidHash256 for use in worker threads + * Uses synchronous fs operations for compatibility with Piscina workers + */ +export function computeMidHash256Sync(filePath: string): string { + const fs = require('fs'); + const SAMPLE_SIZE = 1024 * 1024; // 1MB + + // Get file size + const stats = fs.statSync(filePath); + const fileSize = stats.size; + + // Create size buffer (64-bit big-endian) + const sizeBuffer = Buffer.allocUnsafe(8); + sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); + + // Extract sample data + let sampleData: Buffer; + if (fileSize <= SAMPLE_SIZE) { + // Small file: read entire content + const fd = fs.openSync(filePath, 'r'); + sampleData = Buffer.allocUnsafe(fileSize); + fs.readSync(fd, sampleData, 0, fileSize, 0); + fs.closeSync(fd); + } else { + // Large file: read middle 1MB + const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); + const fd = fs.openSync(filePath, 'r'); + sampleData = Buffer.allocUnsafe(SAMPLE_SIZE); + fs.readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset); + fs.closeSync(fd); + } + + // Compute SHA-256 hash of [size + sample] + const hashInput = Buffer.concat([sizeBuffer, sampleData]); + const hashBuffer = createHash('sha256') + .update(hashInput) + .digest(); + + // Wrap in CID format with custom multicodec 0x1000 + const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer); + return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString(); +} diff --git a/src/lib/hash-compute/MultiHashData.ts b/src/lib/hash-compute/MultiHashData.ts index e8afe7f..3ceb70b 100644 --- a/src/lib/hash-compute/MultiHashData.ts +++ b/src/lib/hash-compute/MultiHashData.ts @@ -5,6 +5,7 @@ export interface MultiHashData { "cid_sha2-256"?: string;//0x12 sha2-256 "cid_sha3-256"?: string; // 0x16 sha3-256 "cid_sha3_384"?: string; // 0x16 sha3-256 + cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format } export interface ComputeInterface { @@ -22,6 +23,7 @@ export enum CID_ALGORITHM_CODES { sha256 = 0x12, sha3_256 = 0x16, sha3_384 = 0x15, + midhash256 = 0x1000, // Custom code for midhash256 } export enum CID_ALGORITHM_NAMES { @@ -31,6 +33,7 @@ export enum CID_ALGORITHM_NAMES { sha256 = 'cid_sha2-256', sha3_256 = 'cid_sha3-256', sha3_384 = 'cid_sha3-384', + midhash256 = 'cid_midhash256', } export enum CID_ALGORITHM { @@ -40,4 +43,5 @@ export enum CID_ALGORITHM { sha256 = 'sha2-256', sha3_256 = 'sha3-256', sha3_384 = 'sha3-384', + midhash256 = 'midhash256', } \ No newline at end of file diff --git a/src/lib/index.ts b/src/lib/index.ts index 48f8d6a..954d8ba 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,6 +1,7 @@ //fileID export * from './file-id/FileIDComputerWorker.js'; export * from './file-id/ShaComputeWorker.js';//worker +export * from './file-id/FastHash.js'; // Fast-hash computation //hash-compute export * from './hash-compute/HashComputerIndexCache.js';