initial commit
This commit is contained in:
Pierre
@@ -0,0 +1,49 @@
|
||||
// Browser-compatible hash implementation
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
import {createMD5, createSHA1, createSHA256, createSHA3} from "hash-wasm";
|
||||
|
||||
export class BrowserHasher implements SimpleHash {
|
||||
private algorithm: string;
|
||||
private wasmHasher: any;
|
||||
private isWasm: boolean;
|
||||
|
||||
constructor(algorithm: string) {
|
||||
this.algorithm = algorithm;
|
||||
this.isWasm = true;
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (this.isWasm) {
|
||||
switch (this.algorithm) {
|
||||
case 'md5':
|
||||
this.wasmHasher = await createMD5();
|
||||
break;
|
||||
case 'sha-1':
|
||||
this.wasmHasher = await createSHA1();
|
||||
break;
|
||||
case 'sha-256':
|
||||
this.wasmHasher = await createSHA256();
|
||||
break;
|
||||
case 'sha3-256':
|
||||
this.wasmHasher = await createSHA3(256);
|
||||
break;
|
||||
case 'sha3-384':
|
||||
this.wasmHasher = await createSHA3(384);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${this.algorithm}`);
|
||||
}
|
||||
this.wasmHasher.init();
|
||||
}
|
||||
}
|
||||
|
||||
update(data: Uint8Array): SimpleHash {
|
||||
this.wasmHasher.update(data);
|
||||
return this;
|
||||
}
|
||||
|
||||
async digest(): Promise<Uint8Array> {
|
||||
const hash = await this.wasmHasher.digest('binary');
|
||||
return new Uint8Array(hash);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import {createHash} from 'crypto';
|
||||
import {SimpleHash} from './SimpleHash';
|
||||
import {encode} from 'bencodec';
|
||||
import {basename} from 'path';
|
||||
|
||||
/**
|
||||
* BitTorrent v2 Info Hash Hasher (BEP 52)
|
||||
*
|
||||
* Implements BitTorrent v2 file hashing with:
|
||||
* - Fixed 16KB block size
|
||||
* - SHA-256 merkle tree
|
||||
* - Info hash computation
|
||||
*
|
||||
* Architecture:
|
||||
* 1. Streaming: Accumulates file data into 16KB blocks
|
||||
* 2. Block hashing: SHA-256 hash each complete block
|
||||
* 3. Merkle tree: Build balanced tree from block hashes
|
||||
* 4. Info hash: SHA-256(bencode(info_dict))
|
||||
*/
|
||||
export class BtihV2Hasher implements SimpleHash {
|
||||
private static readonly BLOCK_SIZE = 16 * 1024; // 16KB fixed by BEP 52
|
||||
|
||||
private blockHashes: Buffer[] = [];
|
||||
private currentBlock: Buffer[] = [];
|
||||
private currentBlockSize: number = 0;
|
||||
private fileSize: number = 0;
|
||||
private fileName: string = 'unknown';
|
||||
|
||||
/**
|
||||
* Optional: Set filename for info dictionary
|
||||
* If not set, defaults to 'unknown'
|
||||
*/
|
||||
setFileName(fileName: string): void {
|
||||
this.fileName = basename(fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update hasher with new data chunk
|
||||
* Accumulates chunks into 16KB blocks and hashes each complete block
|
||||
*/
|
||||
update(data: Uint8Array): SimpleHash {
|
||||
const buffer = Buffer.from(data);
|
||||
this.fileSize += buffer.length;
|
||||
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const remaining = BtihV2Hasher.BLOCK_SIZE - this.currentBlockSize;
|
||||
const toRead = Math.min(remaining, buffer.length - offset);
|
||||
|
||||
// Accumulate chunk into current block
|
||||
this.currentBlock.push(buffer.subarray(offset, offset + toRead));
|
||||
this.currentBlockSize += toRead;
|
||||
offset += toRead;
|
||||
|
||||
// When block is complete, hash it
|
||||
if (this.currentBlockSize === BtihV2Hasher.BLOCK_SIZE) {
|
||||
this.finalizeBlock();
|
||||
}
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalize current block by hashing it and storing the hash
|
||||
*/
|
||||
private finalizeBlock(): void {
|
||||
if (this.currentBlock.length === 0) return;
|
||||
|
||||
const blockData = Buffer.concat(this.currentBlock);
|
||||
const blockHash = createHash('sha256').update(blockData).digest();
|
||||
this.blockHashes.push(blockHash);
|
||||
|
||||
// Reset current block
|
||||
this.currentBlock = [];
|
||||
this.currentBlockSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute final info hash
|
||||
*
|
||||
* Steps:
|
||||
* 1. Finalize any partial block
|
||||
* 2. Build merkle tree from block hashes
|
||||
* 3. Construct info dictionary (name, length, pieces_root)
|
||||
* 4. Bencode info dict
|
||||
* 5. SHA-256 of bencoded data = info hash
|
||||
*/
|
||||
digest(): Buffer {
|
||||
// Finalize any remaining partial block
|
||||
if (this.currentBlockSize > 0) {
|
||||
this.finalizeBlock();
|
||||
}
|
||||
|
||||
// Handle empty file edge case
|
||||
if (this.blockHashes.length === 0) {
|
||||
// Empty file: hash empty block
|
||||
const emptyHash = createHash('sha256').update(Buffer.alloc(0)).digest();
|
||||
this.blockHashes.push(emptyHash);
|
||||
}
|
||||
|
||||
// Build merkle tree to get pieces root
|
||||
const piecesRoot = this.buildMerkleTree(this.blockHashes);
|
||||
|
||||
// Construct info dictionary (BEP 52 v2 format)
|
||||
const infoDict = {
|
||||
name: this.fileName,
|
||||
length: this.fileSize,
|
||||
'piece length': this.computePieceLength(this.fileSize),
|
||||
'pieces root': piecesRoot,
|
||||
'meta version': 2 // BitTorrent v2
|
||||
};
|
||||
|
||||
// Bencode the info dictionary
|
||||
const bencodedInfo = encode(infoDict);
|
||||
|
||||
// Info hash = SHA-256 of bencoded info dict
|
||||
const infoHash = createHash('sha256').update(bencodedInfo).digest();
|
||||
|
||||
return infoHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build balanced merkle tree from leaf hashes
|
||||
*
|
||||
* Algorithm (from BEP 52 reference implementation):
|
||||
* 1. Pad to power of 2 with zero hashes
|
||||
* 2. Recursively pair adjacent hashes
|
||||
* 3. Hash each pair: SHA-256(left || right)
|
||||
* 4. Repeat until single root hash remains
|
||||
*/
|
||||
private buildMerkleTree(hashes: Buffer[]): Buffer {
|
||||
if (hashes.length === 0) {
|
||||
throw new Error('Cannot build merkle tree from empty hash list');
|
||||
}
|
||||
|
||||
// Pad to next power of 2 with zero hashes
|
||||
let paddedHashes = [...hashes];
|
||||
const targetSize = this.nextPowerOfTwo(paddedHashes.length);
|
||||
const zeroHash = Buffer.alloc(32); // 32 bytes of zeros for SHA-256
|
||||
while (paddedHashes.length < targetSize) {
|
||||
paddedHashes.push(zeroHash);
|
||||
}
|
||||
|
||||
// Build tree by repeatedly pairing and hashing
|
||||
while (paddedHashes.length > 1) {
|
||||
const parentHashes: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < paddedHashes.length; i += 2) {
|
||||
const left = paddedHashes[i];
|
||||
const right = paddedHashes[i + 1];
|
||||
const combined = Buffer.concat([left, right]);
|
||||
const parentHash = createHash('sha256').update(combined).digest();
|
||||
parentHashes.push(parentHash);
|
||||
}
|
||||
|
||||
paddedHashes = parentHashes;
|
||||
}
|
||||
|
||||
return paddedHashes[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute piece length using BEP 52 formula
|
||||
*
|
||||
* Formula: piece_length = 16KB * 2^n
|
||||
* where: file_size / piece_length < 2048
|
||||
*
|
||||
* This ensures each file has between 1024 and 2047 pieces
|
||||
*/
|
||||
private computePieceLength(fileSize: number): number {
|
||||
if (fileSize === 0) {
|
||||
return BtihV2Hasher.BLOCK_SIZE; // 16KB for empty files
|
||||
}
|
||||
|
||||
let pieceLength = BtihV2Hasher.BLOCK_SIZE; // Start at 16KB
|
||||
const maxPieces = 2048;
|
||||
|
||||
// Increase piece length until we have < 2048 pieces
|
||||
while (Math.ceil(fileSize / pieceLength) >= maxPieces) {
|
||||
pieceLength *= 2;
|
||||
}
|
||||
|
||||
return pieceLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find next power of 2 >= n
|
||||
*/
|
||||
private nextPowerOfTwo(n: number): number {
|
||||
if (n === 0) return 1;
|
||||
n--;
|
||||
n |= n >> 1;
|
||||
n |= n >> 2;
|
||||
n |= n >> 4;
|
||||
n |= n >> 8;
|
||||
n |= n >> 16;
|
||||
return n + 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export const codeTable = {
|
||||
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
|
||||
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
|
||||
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
|
||||
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
|
||||
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
|
||||
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
|
||||
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
|
||||
[CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {create} from "multiformats/hashes/digest";
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {codeTable} from "@root/file-id/CodeTable";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param stream The Readable stream of the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
algorithms: CID_ALGORITHM_NAMES[];
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
|
||||
}): Promise<string[]> {
|
||||
const hashers = await hasherDefiner(algorithms,createHasher);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const {done, value} = await reader.read();
|
||||
if (done) break;
|
||||
// Assuming the hasher can handle Uint8Array directly
|
||||
for (const item of hashers) {
|
||||
await item.hasher.update(value);
|
||||
}
|
||||
}
|
||||
return cidFinalize(hashers);
|
||||
}
|
||||
|
||||
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
|
||||
hasher: SimpleHash,
|
||||
code: CID_ALGORITHM_CODES
|
||||
}[]> {
|
||||
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
|
||||
.map(async algo => ({
|
||||
hasher: await createHasher(algo),
|
||||
code: codeTable[algo]
|
||||
}));
|
||||
return await Promise.all(hashers);
|
||||
}
|
||||
|
||||
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
|
||||
return await Promise.all(hashers.map(async ({code, hasher}) => {
|
||||
const hashBuffer = await hasher.digest();
|
||||
const digest = create(code, hashBuffer);
|
||||
return CID.createV1(code, digest).toString();
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import crc32 from "crc-32";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class Crc32Hash implements SimpleHash {
|
||||
private _crc: number = undefined;
|
||||
|
||||
update(data: Buffer): SimpleHash {
|
||||
// Update the CRC-32 checksum with the new chunk of data
|
||||
this._crc = crc32.buf(data, this._crc);
|
||||
return this;
|
||||
}
|
||||
|
||||
digest(): Buffer {
|
||||
const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
|
||||
buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {createHash} from "crypto";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BtihV2Hasher} from "./BtihV2Hasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @param algo
|
||||
*/
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
return createHash("sha256");
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
return createHash("sha1");
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
return createHash("md5");
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
return createHash("sha3-256");
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
return createHash("sha3-384");
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
return new Crc32Hash();
|
||||
case CID_ALGORITHM_NAMES.btih_v2:
|
||||
return new BtihV2Hasher();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BrowserHasher} from "@root/file-id/BrowserHasher";
|
||||
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
const hasher = (() => {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
return new BrowserHasher('sha-256');
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
return new BrowserHasher('sha-1');
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
return new BrowserHasher('md5');
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
return new BrowserHasher('sha3-256');
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
return new BrowserHasher('sha3-384');
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
return new Crc32Hash();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
})();
|
||||
|
||||
if (hasher instanceof BrowserHasher) {
|
||||
await hasher.initialize();
|
||||
}
|
||||
|
||||
return hasher;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { open, stat } from 'fs/promises';
|
||||
import { create } from 'multiformats/hashes/digest';
|
||||
import { CID } from 'multiformats/cid';
|
||||
import { CID_ALGORITHM_CODES } from '../hash-compute/MultiHashData.js';
|
||||
|
||||
/**
|
||||
* Computes a midhash256 based on sampling the middle 1MB of a file
|
||||
* plus the file size as a 64-bit integer prefix.
|
||||
*
|
||||
* Algorithm:
|
||||
* - For files <= 1MB: Hashes entire file content + size prefix
|
||||
* - For files > 1MB: Hashes middle 1MB + size prefix
|
||||
*
|
||||
* The size prefix ensures that files with identical middle content
|
||||
* but different sizes produce different hashes.
|
||||
*
|
||||
* Performance: ~20ms regardless of file size (O(1) complexity)
|
||||
*
|
||||
* @param filePath - Path to file
|
||||
* @returns CID string in proper CID v1 format with multicodec 0x1000
|
||||
*/
|
||||
export async function computeMidHash256(filePath: string): Promise<string> {
|
||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||
|
||||
// Get file size (instant - no file read)
|
||||
const stats = await stat(filePath);
|
||||
const fileSize = stats.size;
|
||||
|
||||
// Create size buffer (64-bit big-endian)
|
||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||
|
||||
// Extract sample data
|
||||
let sampleData: Buffer;
|
||||
if (fileSize <= SAMPLE_SIZE) {
|
||||
// Small file: read entire content
|
||||
const fd = await open(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(fileSize);
|
||||
await fd.read(sampleData, 0, fileSize, 0);
|
||||
await fd.close();
|
||||
} else {
|
||||
// Large file: read middle 1MB
|
||||
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
||||
const fd = await open(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
|
||||
await fd.read(sampleData, 0, SAMPLE_SIZE, middleOffset);
|
||||
await fd.close();
|
||||
}
|
||||
|
||||
// Compute SHA-256 hash of [size + sample]
|
||||
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||
const hashBuffer = createHash('sha256')
|
||||
.update(hashInput)
|
||||
.digest();
|
||||
|
||||
// Wrap in CID format with custom multicodec 0x1000
|
||||
const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer);
|
||||
return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous version of computeMidHash256 for use in worker threads
|
||||
* Uses synchronous fs operations for compatibility with Piscina workers
|
||||
*/
|
||||
export function computeMidHash256Sync(filePath: string): string {
|
||||
const fs = require('fs');
|
||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||
|
||||
// Get file size
|
||||
const stats = fs.statSync(filePath);
|
||||
const fileSize = stats.size;
|
||||
|
||||
// Create size buffer (64-bit big-endian)
|
||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||
|
||||
// Extract sample data
|
||||
let sampleData: Buffer;
|
||||
if (fileSize <= SAMPLE_SIZE) {
|
||||
// Small file: read entire content
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(fileSize);
|
||||
fs.readSync(fd, sampleData, 0, fileSize, 0);
|
||||
fs.closeSync(fd);
|
||||
} else {
|
||||
// Large file: read middle 1MB
|
||||
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
|
||||
fs.readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
|
||||
// Compute SHA-256 hash of [size + sample]
|
||||
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||
const hashBuffer = createHash('sha256')
|
||||
.update(hashInput)
|
||||
.digest();
|
||||
|
||||
// Wrap in CID format with custom multicodec 0x1000
|
||||
const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer);
|
||||
return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString();
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export interface FileIDComputerInterface{
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import {Piscina} from "piscina";
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
|
||||
export class FileIDComputerWorker{
|
||||
private piscina: Piscina;
|
||||
|
||||
constructor(workerPath?:string) {
|
||||
if(!workerPath) {
|
||||
// Construct the URL for the current module
|
||||
let distFolder = import.meta.dirname;
|
||||
distFolder = distFolder.replace('src', 'dist');
|
||||
distFolder = distFolder + "/worker.js";
|
||||
workerPath = new URL(distFolder,"file://").href;
|
||||
}
|
||||
this.piscina = new Piscina({
|
||||
maxThreads: 4,
|
||||
//filename: new URL('./ShaComputeWorker.ts', import.meta.url).href
|
||||
filename: workerPath
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param filePath The path to the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
|
||||
return this.piscina.run({filePath, algorithms});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
import {MultiHashData} from '../hash-compute/MultiHashData';
|
||||
import {CID} from 'multiformats/cid';
|
||||
|
||||
/**
|
||||
* Magnet Link Generator for BitTorrent v2
|
||||
*
|
||||
* Generates BitTorrent v2 magnet links from file metadata.
|
||||
*
|
||||
* Format: magnet:?xt=urn:btmh:1220<info-hash-hex>&dn=<filename>&xl=<filesize>
|
||||
*
|
||||
* Components:
|
||||
* - xt: eXact Topic (torrent identifier)
|
||||
* - btmh: BitTorrent Merkle Hash (v2 URN scheme)
|
||||
* - 1220: Multihash prefix (0x12 = SHA-256, 0x20 = 32 bytes)
|
||||
* - info-hash-hex: 64-character hex string (32 bytes SHA-256)
|
||||
* - dn: Display Name (filename)
|
||||
* - xl: eXact Length (file size in bytes)
|
||||
*
|
||||
* Reference: BEP 52, BEP 9 (magnet URIs)
|
||||
*/
|
||||
export class MagnetLinkGenerator {
|
||||
/**
|
||||
* Generate BitTorrent v2 magnet link from metadata
|
||||
*
|
||||
* @param metadata - File metadata containing cid_btih_v2
|
||||
* @param fileName - Original filename for display name
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if btih_v2 not available
|
||||
*
|
||||
* @example
|
||||
* const metadata = {cid_btih_v2: 'bafyb...'};
|
||||
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
|
||||
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
|
||||
*/
|
||||
static generate(
|
||||
metadata: MultiHashData,
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
const cidString = metadata.cid_btih_v2;
|
||||
if (!cidString) {
|
||||
return null; // BitTorrent v2 hash not computed
|
||||
}
|
||||
|
||||
try {
|
||||
// Decode CID to get raw info hash
|
||||
const cid = CID.parse(cidString);
|
||||
const infoHash = cid.multihash.digest; // Raw 32-byte SHA-256 hash
|
||||
|
||||
// Convert to hex for magnet link
|
||||
const infoHashHex = Buffer.from(infoHash).toString('hex');
|
||||
|
||||
// Build magnet link components
|
||||
const components = [
|
||||
`xt=urn:btmh:1220${infoHashHex}`, // 1220 = multihash prefix (SHA-256, 32 bytes)
|
||||
`dn=${encodeURIComponent(fileName)}`, // Display name (URL encoded)
|
||||
`xl=${fileSize}` // Exact length in bytes
|
||||
];
|
||||
|
||||
return `magnet:?${components.join('&')}`;
|
||||
} catch (error) {
|
||||
console.error('Failed to generate magnet link from CID:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract info hash from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Info hash as hex string, or null if invalid
|
||||
*
|
||||
* @example
|
||||
* const hash = MagnetLinkGenerator.extractInfoHash('magnet:?xt=urn:btmh:1220abc...');
|
||||
* // Returns: 'abc...'
|
||||
*/
|
||||
static extractInfoHash(magnetLink: string): string | null {
|
||||
try {
|
||||
const match = magnetLink.match(/xt=urn:btmh:1220([0-9a-f]{64})/i);
|
||||
return match ? match[1] : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Decoded filename, or null if not present
|
||||
*/
|
||||
static extractFileName(magnetLink: string): string | null {
|
||||
try {
|
||||
const match = magnetLink.match(/dn=([^&]+)/);
|
||||
return match ? decodeURIComponent(match[1]) : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract file size from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns File size in bytes, or null if not present
|
||||
*/
|
||||
static extractFileSize(magnetLink: string): number | null {
|
||||
try {
|
||||
const match = magnetLink.match(/xl=(\d+)/);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate BitTorrent v2 magnet link format
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns True if valid BitTorrent v2 magnet link
|
||||
*/
|
||||
static isValid(magnetLink: string): boolean {
|
||||
if (!magnetLink.startsWith('magnet:?')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Must have btmh (BitTorrent Merkle Hash) identifier
|
||||
if (!magnetLink.includes('xt=urn:btmh:1220')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Must have 64-character hex hash (32 bytes SHA-256)
|
||||
const hashMatch = magnetLink.match(/1220([0-9a-f]{64})/i);
|
||||
if (!hashMatch) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse magnet link into components
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Object with parsed components, or null if invalid
|
||||
*/
|
||||
static parse(magnetLink: string): {
|
||||
infoHash: string;
|
||||
fileName?: string;
|
||||
fileSize?: number;
|
||||
} | null {
|
||||
if (!this.isValid(magnetLink)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const infoHash = this.extractInfoHash(magnetLink);
|
||||
if (!infoHash) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
infoHash,
|
||||
fileName: this.extractFileName(magnetLink) || undefined,
|
||||
fileSize: this.extractFileSize(magnetLink) || undefined
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate magnet link from CID string directly
|
||||
*
|
||||
* @param cidString - CID string (e.g., 'bafyb...')
|
||||
* @param fileName - Original filename
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if CID invalid
|
||||
*/
|
||||
static fromCID(
|
||||
cidString: string,
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import fs from "fs";
|
||||
import {Readable} from 'stream';
|
||||
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param filePath The path to the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
export default async function computeCIDs({filePath, algorithms}: {
|
||||
filePath: string;
|
||||
algorithms: CID_ALGORITHM_NAMES[]
|
||||
}): Promise<string[]> {
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
return computeCIDsInternal({stream, algorithms, createHasher});
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export interface SimpleHash {
|
||||
update(data: Uint8Array): SimpleHash | Promise<SimpleHash>;
|
||||
|
||||
digest(): Uint8Array | Promise<Uint8Array>;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Interface for file processing implementations used by FolderWatcher.
|
||||
*
|
||||
* Provides hooks for different stages of the file processing pipeline:
|
||||
* 1. markPending() - File discovered, waiting in queue
|
||||
* 2. queueFile() - File starts processing (optional pre-processing phase)
|
||||
* 3. processFile() - Main processing phase
|
||||
* 4. finalize() - After all files processed (optional)
|
||||
*/
|
||||
export interface FileProcessorInterface {
|
||||
/** Optional initialization before processing starts */
|
||||
initialize?(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Main file processing method.
|
||||
* @param current - Current file index in the queue
|
||||
* @param queueSize - Total number of files to process
|
||||
* @param nfoFilePath - Path to the file being processed
|
||||
*/
|
||||
processFile(current: number, queueSize: number, nfoFilePath: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Determines if a file should be processed.
|
||||
* @param filePath - Path to the file to check
|
||||
* @returns true if the file should be processed
|
||||
*/
|
||||
canProcessFile(filePath: string): Promise<boolean>;
|
||||
|
||||
/** Optional callback when a file is deleted */
|
||||
deleteFile?(filePath: string): Promise<void>;
|
||||
|
||||
/** Optional callback after all files have been processed */
|
||||
finalize?(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Optional callback when a file is discovered and marked as pending.
|
||||
* Called BEFORE the file enters the processing queue.
|
||||
* Useful for tracking/monitoring pending files.
|
||||
* @param filePath - Path to the file being marked as pending
|
||||
*/
|
||||
markPending?(filePath: string): void;
|
||||
|
||||
/**
|
||||
* Optional callback when a file begins processing.
|
||||
* Called WITHIN the queue (respects concurrency limits).
|
||||
* Can be used for quick/light pre-processing before main processing.
|
||||
* @param filePath - Path to the file being queued
|
||||
*/
|
||||
queueFile?(filePath: string): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import * as path from 'path';
|
||||
import chokidar from 'chokidar';
|
||||
import {readdir} from "fs/promises";
|
||||
|
||||
/**
|
||||
* Generic file discovery utility
|
||||
* Discovers files in directory trees and emits paths as async generator
|
||||
* No filtering logic - just pure file discovery
|
||||
*/
|
||||
export class FolderWatcher {
|
||||
/**
|
||||
* Discover all files in given directories
|
||||
* Returns async generator that yields file paths as they're discovered
|
||||
*
|
||||
* @param directories - Array of directory paths to scan
|
||||
* @returns AsyncGenerator yielding file paths
|
||||
*/
|
||||
async *discoverFiles(directories: string[]): AsyncGenerator<string> {
|
||||
for (const dir of directories) {
|
||||
const normalizedDir = path.normalize(dir);
|
||||
yield* this.walkDirectory(normalizedDir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively walk directory tree
|
||||
* Yields file paths as they're discovered
|
||||
*
|
||||
* @param directory - Directory to walk
|
||||
* @returns AsyncGenerator yielding file paths
|
||||
*/
|
||||
private async *walkDirectory(directory: string): AsyncGenerator<string> {
|
||||
try {
|
||||
const entries = await readdir(directory, {withFileTypes: true});
|
||||
|
||||
// Collect subdirectories for parallel processing
|
||||
const subdirs: string[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(directory, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
subdirs.push(fullPath);
|
||||
} else {
|
||||
// Yield file path immediately
|
||||
yield fullPath;
|
||||
}
|
||||
}
|
||||
|
||||
// Process subdirectories (yields naturally to event loop)
|
||||
for (const subdir of subdirs) {
|
||||
yield* this.walkDirectory(subdir);
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Handle permission errors gracefully
|
||||
if (error.code === 'EACCES' || error.code === 'EPERM') {
|
||||
console.warn(`[FolderWatcher] Permission denied: ${directory} - skipping`);
|
||||
} else if (error.code === 'ENOENT') {
|
||||
console.warn(`[FolderWatcher] Directory not found: ${directory} - skipping`);
|
||||
} else if (error.code === 'ENOTDIR') {
|
||||
console.warn(`[FolderWatcher] Not a directory: ${directory} - skipping`);
|
||||
} else {
|
||||
console.error(`[FolderWatcher] Error reading directory ${directory}:`, error.message);
|
||||
}
|
||||
// Continue processing - don't let one bad directory stop the scan
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch directories for file changes using chokidar
|
||||
*
|
||||
* @param directories - Directories to watch
|
||||
* @param callbacks - Event callbacks
|
||||
* @param options - Chokidar options
|
||||
*/
|
||||
watch(directories: string[], callbacks: {
|
||||
onAdd?: (filePath: string) => void | Promise<void>;
|
||||
onChange?: (filePath: string) => void | Promise<void>;
|
||||
onUnlink?: (filePath: string) => void | Promise<void>;
|
||||
onReady?: () => void | Promise<void>;
|
||||
onError?: (error: Error) => void | Promise<void>;
|
||||
}, options?: {
|
||||
interval?: number;
|
||||
stabilityThreshold?: number;
|
||||
pollInterval?: number;
|
||||
}): chokidar.FSWatcher {
|
||||
const chokidarConfig: chokidar.WatchOptions = {
|
||||
ignoreInitial: true, // Don't emit events for initial scan
|
||||
persistent: true,
|
||||
depth: Infinity,
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: options?.stabilityThreshold || 30000,
|
||||
pollInterval: options?.pollInterval || 5000
|
||||
}
|
||||
};
|
||||
|
||||
let watcher: chokidar.FSWatcher;
|
||||
if ((options?.interval || 0) <= 0) {
|
||||
watcher = chokidar.watch(directories, chokidarConfig);
|
||||
} else {
|
||||
watcher = chokidar.watch(directories, {
|
||||
...chokidarConfig,
|
||||
usePolling: true,
|
||||
interval: options.interval
|
||||
});
|
||||
}
|
||||
|
||||
if (callbacks.onAdd) {
|
||||
watcher.on('add', async (filePath) => {
|
||||
try {
|
||||
await callbacks.onAdd!(filePath);
|
||||
} catch (error) {
|
||||
console.error(`[FolderWatcher] Error in onAdd callback for ${filePath}:`, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (callbacks.onChange) {
|
||||
watcher.on('change', async (filePath) => {
|
||||
try {
|
||||
await callbacks.onChange!(filePath);
|
||||
} catch (error) {
|
||||
console.error(`[FolderWatcher] Error in onChange callback for ${filePath}:`, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (callbacks.onUnlink) {
|
||||
watcher.on('unlink', async (filePath) => {
|
||||
try {
|
||||
await callbacks.onUnlink!(filePath);
|
||||
} catch (error) {
|
||||
console.error(`[FolderWatcher] Error in onUnlink callback for ${filePath}:`, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
watcher.on('error', (error: Error) => {
|
||||
if (callbacks.onError) {
|
||||
callbacks.onError(error);
|
||||
} else {
|
||||
// Default error handling
|
||||
const err = error as any;
|
||||
if (err.code === 'EACCES' || err.code === 'EPERM') {
|
||||
console.warn(`[FolderWatcher] Permission denied for file watcher:`, error.message);
|
||||
} else if (err.code === 'ENOENT') {
|
||||
console.warn(`[FolderWatcher] Watched path no longer exists:`, error.message);
|
||||
} else if (err.code === 'ENOSPC') {
|
||||
console.error(`[FolderWatcher] CRITICAL: No space left on device or inotify watch limit reached:`, error.message);
|
||||
console.error(`[FolderWatcher] Try increasing fs.inotify.max_user_watches: sysctl fs.inotify.max_user_watches=524288`);
|
||||
} else {
|
||||
console.error(`[FolderWatcher] Watcher error:`, error.message || error);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (callbacks.onReady) {
|
||||
watcher.on('ready', async () => {
|
||||
try {
|
||||
console.log(`[FolderWatcher] Watching for file changes on ${directories.join(', ')}`);
|
||||
await callbacks.onReady!();
|
||||
} catch (error) {
|
||||
console.error(`[FolderWatcher] Error in onReady callback:`, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return watcher;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import {readdir} from "fs/promises";
|
||||
import * as path from 'path';
|
||||
import {FileProcessorInterface} from "./FileProcessorInterface.js";
|
||||
|
||||
export interface PollingPathConfig {
|
||||
path: string;
|
||||
intervalMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* PollingWatcher - Manual polling-based file watcher (no chokidar)
|
||||
*
|
||||
* Polls specific directories at configured intervals using simple setInterval + readdir.
|
||||
* Perfect for network drives (SMB/NFS) where filesystem events don't work reliably.
|
||||
*
|
||||
* Usage:
|
||||
* - Configure with array of {path, intervalMs} objects
|
||||
* - Each path can have its own polling interval
|
||||
* - Runs in parallel with FolderWatcher (chokidar)
|
||||
*/
|
||||
export class PollingWatcher {
|
||||
private fileProcessor: FileProcessorInterface;
|
||||
private pollingConfigs: PollingPathConfig[];
|
||||
private intervals: NodeJS.Timeout[] = [];
|
||||
private seenFiles: Set<string> = new Set();
|
||||
private initialized = false;
|
||||
|
||||
constructor(fileProcessor: FileProcessorInterface, pollingConfigs: PollingPathConfig[]) {
|
||||
this.fileProcessor = fileProcessor;
|
||||
this.pollingConfigs = pollingConfigs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively scan directory and find all files
|
||||
*/
|
||||
private async scanDirectory(directory: string, allFiles: string[]): Promise<void> {
|
||||
try {
|
||||
const entries = await readdir(directory, {withFileTypes: true});
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await this.scanDirectory(fullPath, allFiles);
|
||||
} else {
|
||||
allFiles.push(fullPath);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Ignore permission errors and other scan errors
|
||||
if (error.code !== 'EACCES' && error.code !== 'EPERM') {
|
||||
console.error(`Error scanning directory ${directory}:`, error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file (check if new, then queue for processing)
|
||||
*/
|
||||
private async processFile(filePath: string): Promise<void> {
|
||||
// Skip if already seen
|
||||
if (this.seenFiles.has(filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if file type is supported
|
||||
if (!await this.fileProcessor.canProcessFile(filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark as seen
|
||||
this.seenFiles.add(filePath);
|
||||
console.log(`[PollingWatcher] New file detected: ${filePath}`);
|
||||
|
||||
// Queue for processing (same pipeline as chokidar)
|
||||
if (this.fileProcessor.markPending) {
|
||||
this.fileProcessor.markPending(filePath);
|
||||
}
|
||||
|
||||
if (this.fileProcessor.queueFile) {
|
||||
await this.fileProcessor.queueFile(filePath);
|
||||
}
|
||||
|
||||
if (this.fileProcessor.processFile) {
|
||||
this.fileProcessor.processFile(0, 0, filePath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll a specific directory once
|
||||
*/
|
||||
private async pollDirectory(dirPath: string): Promise<void> {
|
||||
console.log(`[PollingWatcher] Polling ${dirPath}...`);
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const allFiles: string[] = [];
|
||||
await this.scanDirectory(dirPath, allFiles);
|
||||
|
||||
console.log(`[PollingWatcher] Found ${allFiles.length} files in ${dirPath} (${Date.now() - startTime}ms)`);
|
||||
|
||||
// Process new files
|
||||
const promises: Promise<void>[] = [];
|
||||
for (const filePath of allFiles) {
|
||||
promises.push(this.processFile(filePath));
|
||||
}
|
||||
await Promise.all(promises);
|
||||
|
||||
console.log(`[PollingWatcher] Poll completed for ${dirPath} (${Date.now() - startTime}ms)`);
|
||||
} catch (error: any) {
|
||||
console.error(`[PollingWatcher] Error polling ${dirPath}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start polling all configured paths
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.pollingConfigs.length === 0) {
|
||||
console.log('[PollingWatcher] No polling paths configured, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[PollingWatcher] Starting polling for ${this.pollingConfigs.length} paths`);
|
||||
|
||||
// Initialize file processor if needed
|
||||
if (!this.initialized && this.fileProcessor.initialize) {
|
||||
await this.fileProcessor.initialize();
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
// Do initial scan for all paths
|
||||
for (const config of this.pollingConfigs) {
|
||||
console.log(`[PollingWatcher] Initial scan: ${config.path} (interval: ${config.intervalMs}ms)`);
|
||||
await this.pollDirectory(config.path);
|
||||
}
|
||||
|
||||
// Start interval polling for each path
|
||||
for (const config of this.pollingConfigs) {
|
||||
const interval = setInterval(async () => {
|
||||
await this.pollDirectory(config.path);
|
||||
}, config.intervalMs);
|
||||
|
||||
this.intervals.push(interval);
|
||||
console.log(`[PollingWatcher] Polling started for ${config.path} every ${config.intervalMs}ms`);
|
||||
}
|
||||
|
||||
console.log('[PollingWatcher] All polling intervals started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop all polling
|
||||
*/
|
||||
stop(): void {
|
||||
console.log('[PollingWatcher] Stopping all polling intervals');
|
||||
for (const interval of this.intervals) {
|
||||
clearInterval(interval);
|
||||
}
|
||||
this.intervals = [];
|
||||
console.log('[PollingWatcher] All polling intervals stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the seen files cache (useful for testing/debugging)
|
||||
*/
|
||||
clearCache(): void {
|
||||
console.log(`[PollingWatcher] Clearing seen files cache (${this.seenFiles.size} files)`);
|
||||
this.seenFiles.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class HashComputer{
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[],
|
||||
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
|
||||
}
|
||||
|
||||
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {Readable} from "stream";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import fs from "fs";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
export class HashComputerFile implements ComputeInterface{
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import {HashComputerWorker} from "./HashComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {HashIndexManager} from "./HashIndexManager";
|
||||
import {stat} from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
export class HashComputerIndexCache implements ComputeInterface {
|
||||
private hashIndexManager: HashIndexManager;
|
||||
private hashComputer: HashComputerWorker;
|
||||
|
||||
constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash,workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
await this.hashIndexManager.init();
|
||||
let stats = await stat(filePath);
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
|
||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!metadata[hash] && indexLine[hash]) {
|
||||
metadata[hash] = indexLine[hash];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
||||
}
|
||||
|
||||
async getHashIndexManager(): Promise<HashIndexManager> {
|
||||
await this.hashIndexManager.init();
|
||||
return this.hashIndexManager;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
|
||||
export class HashComputerWorker implements ComputeInterface {
|
||||
private fileIDComputer: FileIDComputerWorker;
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
|
||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
||||
}
|
||||
|
||||
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
import {createReadStream, promises as fs} from 'fs';
|
||||
import {parse} from 'csv-parse';
|
||||
import {parse as parseSync} from 'csv-parse/sync';
|
||||
import {stringify} from 'csv-stringify/sync';
|
||||
import {existsAsync} from "../utils/ExistsAsync";
|
||||
import path from "path";
|
||||
import {clearInterval} from "node:timers";
|
||||
import {CID_ALGORITHM_NAMES} from "./MultiHashData";
|
||||
import {stat} from "fs/promises";
|
||||
import {Stats} from "node:fs";
|
||||
|
||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
||||
path: string;
|
||||
size: string;
|
||||
mtime: string;
|
||||
}
|
||||
|
||||
export const INDEX_HEADERS = ['path', 'size', 'mtime'];
|
||||
|
||||
export class HashIndexManager {
|
||||
private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
|
||||
private intervalId: any;
|
||||
private intervalTime: number = 30000;
|
||||
private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private indexOpsInProgress: boolean = false;
|
||||
private hasChanged: boolean = false;
|
||||
private initialLoad: Promise<void>;
|
||||
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
|
||||
|
||||
constructor(private indexFolderPath: string,
|
||||
private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
}
|
||||
|
||||
hasFileInCache(filePath: string,stats:Stats): boolean {
|
||||
return this.cache.has(`${path.basename(filePath)}-${stats.size}-${stats.mtime.toISOString()}`);
|
||||
}
|
||||
|
||||
getCacheSize(): number {
|
||||
return this.cache.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* After init consecutively calls to this method will not reload the index
|
||||
* readThe index for the first time and enable autosave
|
||||
* @param autosave
|
||||
*/
|
||||
public async init(autosave = true) {
|
||||
if (!this.initialLoad) {
|
||||
this.initialLoad = new Promise<void>(async (resolve, reject) => {
|
||||
// Create the index folder if it doesn't exist
|
||||
await fs.mkdir(this.indexFolderPath, { recursive: true });
|
||||
const stat = await fs.stat(this.indexFolderPath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error(`Invalid index folder path ${this.indexFolderPath}`);
|
||||
}
|
||||
for (const hash of this.targetHash) {
|
||||
this.filePaths[hash] = path.join(this.indexFolderPath, `index-${hash}.csv`);
|
||||
console.log(`Index file path for ${hash} is ${this.filePaths[hash]}`);
|
||||
if (!this.filePaths[hash]) {
|
||||
throw new Error(`Invalid index file path for ${hash}`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!this.checkCSVHeaders(this.filePaths[hash], hash)) {
|
||||
throw new Error(`Invalid index file headers for ${hash}`);
|
||||
}
|
||||
await this.loadIndex(hash);
|
||||
}
|
||||
if (autosave) {
|
||||
this.startAutoSave()
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
return this.initialLoad;
|
||||
}
|
||||
|
||||
// Function to check CSV headers
|
||||
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
|
||||
const records = parseSync(csvContent, {
|
||||
bom: true,
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
if (records.length === 0) {
|
||||
//we will write the headers
|
||||
return true;
|
||||
}
|
||||
|
||||
// Extract headers
|
||||
const headers = Object.keys(records[0]);
|
||||
|
||||
// Define required headers
|
||||
//from the IndexLine interface
|
||||
const requiredHeaders = [...INDEX_HEADERS, hash];
|
||||
|
||||
// Check if all required headers are present
|
||||
return requiredHeaders.every(header => headers.includes(header));
|
||||
};
|
||||
|
||||
public stopAutoSave() {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
}
|
||||
}
|
||||
|
||||
public startAutoSave(time: number = this.intervalTime) {
|
||||
this.stopAutoSave();
|
||||
this.intervalTime = time;
|
||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
||||
}
|
||||
|
||||
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
if (await existsAsync(this.filePaths[hash])) {
|
||||
// check the file size and if it did not change, do not read the file
|
||||
const stats = await fs.stat(this.filePaths[hash]);
|
||||
if (this.lastIndexFileSize[hash] !== stats.size) {
|
||||
// Read existing file content and parse it
|
||||
const records: IndexLine[] = await this.readCsv(hash);
|
||||
for (const record of records) {
|
||||
const cacheKey = `${record.path}-${record.size}-${record.mtime}`;
|
||||
let indexLine = this.cache.get(cacheKey);
|
||||
if (!indexLine) {
|
||||
this.cache.set(cacheKey, record);
|
||||
} else if(record[hash]){
|
||||
//update the cache with the latest data
|
||||
indexLine[hash] = record[hash];
|
||||
}
|
||||
}
|
||||
this.lastIndexFileSize[hash] = stats.size;
|
||||
this.lastCacheFile[hash] = records;
|
||||
return records;
|
||||
} else {
|
||||
return this.lastCacheFile[hash];
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
private loadIndexFromCache(): IndexLine[] {
|
||||
return Array.from(this.cache.values());
|
||||
}
|
||||
|
||||
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
if (!(await existsAsync(this.filePaths[hash]))) {
|
||||
return [];
|
||||
}
|
||||
const start = performance.now();
|
||||
|
||||
const parser = parse({
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
const records: IndexLine[] = [];
|
||||
|
||||
return new Promise<IndexLine[]>((resolve, reject) => {
|
||||
createReadStream(this.filePaths[hash])
|
||||
.pipe(parser)
|
||||
.on('data', (record) => {
|
||||
records.push(record);
|
||||
})
|
||||
.on('end', () => {
|
||||
resolve(records);
|
||||
console.log(`Index read ${hash} time ${performance.now() - start}ms`);
|
||||
})
|
||||
.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public async saveCacheToFile(): Promise<void> {
|
||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
||||
return;
|
||||
}
|
||||
this.hasChanged = false;
|
||||
this.indexOpsInProgress = true;
|
||||
const start = performance.now();
|
||||
|
||||
let cacheRows: IndexLine[];
|
||||
if (this.cache.size !== 0) {
|
||||
cacheRows = this.loadIndexFromCache();
|
||||
}
|
||||
let didWrite=false;
|
||||
for (const hash of this.targetHash) {
|
||||
let existingRows: IndexLine[] = await this.loadIndex(hash);
|
||||
let existingRowsAsMap: Map<string, IndexLine> = new Map(existingRows.map(row => [row.path + '-' + row.size + '-' + row.mtime, row]));
|
||||
if (this.cache.size !== 0) {
|
||||
// Filter out cacheRows that are already in the file
|
||||
const newRows = cacheRows.filter(row => {
|
||||
//to be added a row must not exist in the file and must exist in the cache (with a hash)
|
||||
const newRow = !existingRowsAsMap.has(row.path + '-' + row.size + '-' + row.mtime);
|
||||
return newRow && !!row[hash];
|
||||
});
|
||||
|
||||
if (newRows.length !== 0) {
|
||||
// Serialize new cacheRows to CSV string
|
||||
const csvString = stringify(newRows, {
|
||||
header: existingRows.length === 0, // Only add header if the file was empty
|
||||
columns: [
|
||||
{key: 'path', header: 'path'},
|
||||
{key: 'size', header: 'size'},
|
||||
{key: 'mtime', header: 'mtime'},
|
||||
{key: hash, header: hash},
|
||||
],
|
||||
});
|
||||
|
||||
// Append new cacheRows to the file
|
||||
await fs.appendFile(this.filePaths[hash], csvString);
|
||||
didWrite = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(didWrite) {
|
||||
const totalTime = performance.now() - start;
|
||||
console.log(`Index saved in ${totalTime}ms`);
|
||||
// Check if the time to save the index is greater than the interval time. increase the interval time if needed
|
||||
if (totalTime * 10 > this.intervalTime) {
|
||||
this.startAutoSave(totalTime * 10);
|
||||
console.log(`Index save interval increased to ${totalTime * 10}ms`);
|
||||
}
|
||||
}
|
||||
this.indexOpsInProgress = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* this.getCidForFile(fileName, stats.size, stats.mtime.toISOString())
|
||||
* @param filePath
|
||||
*/
|
||||
public async getCidForFileAsync(filePath: string): Promise<IndexLine> {
|
||||
const fileName = path.basename(filePath);
|
||||
const stats = await stat(filePath);
|
||||
return this.getCidForFile(fileName, stats.size, stats.mtime.toISOString());
|
||||
}
|
||||
|
||||
public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine {
|
||||
const fileName = path.basename(filePath);
|
||||
let fileNameIndex = this.cache.get(`${fileName}-${fileSize}-${mtime}`);
|
||||
if (fileNameIndex) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!fileNameIndex[hash]) {
|
||||
delete fileNameIndex[hash];
|
||||
}
|
||||
}
|
||||
if (fileNameIndex.mtime) {
|
||||
//if we have a mtime, we need to check it
|
||||
if (fileNameIndex.size === (fileSize + "") && fileNameIndex.mtime === mtime) {
|
||||
return fileNameIndex;
|
||||
}
|
||||
} else {
|
||||
//mtime is optional
|
||||
if (fileNameIndex.size === (fileSize + "")) {
|
||||
return fileNameIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3 - if not found, delete the entry (keeps the index clean)
|
||||
/*if (fileNameIndex && fileNameIndex.size !== (fileSize + "") && pathIndex && pathIndex.size !== (fileSize + "")) {
|
||||
this.cache.delete(fileName);
|
||||
this.cache.delete(filePath);
|
||||
}*/
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
|
||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
||||
throw new Error('Invalid parameters');
|
||||
}
|
||||
for (const hash of this.targetHash) {
|
||||
if (!hashs[hash]) {
|
||||
throw new Error(`Missing hash ${hash}`);
|
||||
}
|
||||
}
|
||||
const size = fileSize + "";
|
||||
const baseName = path.basename(filePath);
|
||||
const cacheKey = `${baseName}-${size}-${mtime}`;
|
||||
let indexLine = this.cache.get(cacheKey);
|
||||
if (!indexLine) {
|
||||
//filter only the hashes we need
|
||||
let filteredHash = {};
|
||||
for (const hash of this.targetHash) {
|
||||
filteredHash[hash] = hashs[hash];
|
||||
}
|
||||
const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};
|
||||
this.cache.set(cacheKey, data);
|
||||
} else {
|
||||
//update the cache with the latest data
|
||||
for (const hash of this.targetHash) {
|
||||
indexLine[hash] = hashs[hash];
|
||||
}
|
||||
}
|
||||
this.hasChanged = true;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
export interface MultiHashData {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
|
||||
cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
|
||||
}
|
||||
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* according to
|
||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
||||
*/
|
||||
export enum CID_ALGORITHM_CODES {
|
||||
crc32 = 0x0132,
|
||||
md5 = 0xd5,
|
||||
sha1 = 0x11,
|
||||
sha256 = 0x12,
|
||||
sha3_256 = 0x16,
|
||||
sha3_384 = 0x15,
|
||||
midhash256 = 0x1000, // Custom code for midhash256
|
||||
btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM_NAMES {
|
||||
crc32 = 'cid_crc32',
|
||||
md5 = 'cid_md5',
|
||||
sha1 = 'cid_sha1',
|
||||
sha256 = 'cid_sha2-256',
|
||||
sha3_256 = 'cid_sha3-256',
|
||||
sha3_384 = 'cid_sha3-384',
|
||||
midhash256 = 'cid_midhash256',
|
||||
btih_v2 = 'cid_btih_v2',
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM {
|
||||
crc32 = 'crc32',
|
||||
md5 = 'md5',
|
||||
sha1 = 'sha1',
|
||||
sha256 = 'sha2-256',
|
||||
sha3_256 = 'sha3-256',
|
||||
sha3_384 = 'sha3-384',
|
||||
midhash256 = 'midhash256',
|
||||
btih_v2 = 'btih-v2',
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
|
||||
export * from './index-interface.js';
|
||||
export * from './file-id/CreateHasherBrowser.js';
|
||||
export * from './hash-compute/HashComputer.js';
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './hash-compute/MultiHashData.js';
|
||||
export * from './folder-watcher/FileProcessorInterface.js';
|
||||
export * from './file-id/FileIDComputerInterface.js';
|
||||
@@ -0,0 +1,20 @@
|
||||
//fileID
|
||||
export * from './file-id/FileIDComputerWorker.js';
|
||||
export * from './file-id/ShaComputeWorker.js';//worker
|
||||
export * from './file-id/FastHash.js'; // Fast-hash computation
|
||||
export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
|
||||
export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
|
||||
|
||||
//hash-compute
|
||||
export * from './hash-compute/HashComputerIndexCache.js';
|
||||
export * from './hash-compute/HashComputerWorker.js';
|
||||
export * from './hash-compute/HashIndexManager.js';
|
||||
export * from './hash-compute/HashComputer.js';
|
||||
export * from './hash-compute/HashComputerFile.js';
|
||||
|
||||
//folder-watcher
|
||||
export * from './folder-watcher/FolderWatcher.js';
|
||||
export * from './folder-watcher/PollingWatcher.js';
|
||||
export * from './index-interface.js';
|
||||
|
||||
export * from './utils/ExistsAsync.js';
|
||||
@@ -0,0 +1,12 @@
|
||||
// Example of checking if a file exists asynchronously
|
||||
// Note: `existsSync` does not have a direct async equivalent, so we use `access` instead.
|
||||
import {access,constants} from "fs/promises";
|
||||
|
||||
export async function existsAsync(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await access(filePath, constants.F_OK);
|
||||
return true; // File exists
|
||||
} catch {
|
||||
return false; // File does not exist
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user