initial commit

This commit is contained in:
Pierre
2026-01-11 17:26:49 +01:00
commit 888790fd11
240 changed files with 48691 additions and 0 deletions
@@ -0,0 +1,49 @@
// Browser-compatible hash implementation
import {SimpleHash} from "@root/file-id/SimpleHash";
import {createMD5, createSHA1, createSHA256, createSHA3} from "hash-wasm";
export class BrowserHasher implements SimpleHash {
private algorithm: string;
private wasmHasher: any;
private isWasm: boolean;
constructor(algorithm: string) {
this.algorithm = algorithm;
this.isWasm = true;
}
async initialize(): Promise<void> {
if (this.isWasm) {
switch (this.algorithm) {
case 'md5':
this.wasmHasher = await createMD5();
break;
case 'sha-1':
this.wasmHasher = await createSHA1();
break;
case 'sha-256':
this.wasmHasher = await createSHA256();
break;
case 'sha3-256':
this.wasmHasher = await createSHA3(256);
break;
case 'sha3-384':
this.wasmHasher = await createSHA3(384);
break;
default:
throw new Error(`Unsupported algorithm: ${this.algorithm}`);
}
this.wasmHasher.init();
}
}
update(data: Uint8Array): SimpleHash {
this.wasmHasher.update(data);
return this;
}
async digest(): Promise<Uint8Array> {
const hash = await this.wasmHasher.digest('binary');
return new Uint8Array(hash);
}
}
@@ -0,0 +1,200 @@
import {createHash} from 'crypto';
import {SimpleHash} from './SimpleHash';
import {encode} from 'bencodec';
import {basename} from 'path';
/**
* BitTorrent v2 Info Hash Hasher (BEP 52)
*
* Implements BitTorrent v2 file hashing with:
* - Fixed 16KB block size
* - SHA-256 merkle tree
* - Info hash computation
*
* Architecture:
* 1. Streaming: Accumulates file data into 16KB blocks
* 2. Block hashing: SHA-256 hash each complete block
* 3. Merkle tree: Build balanced tree from block hashes
* 4. Info hash: SHA-256(bencode(info_dict))
*/
export class BtihV2Hasher implements SimpleHash {
private static readonly BLOCK_SIZE = 16 * 1024; // 16KB fixed by BEP 52
private blockHashes: Buffer[] = [];
private currentBlock: Buffer[] = [];
private currentBlockSize: number = 0;
private fileSize: number = 0;
private fileName: string = 'unknown';
/**
* Optional: Set filename for info dictionary
* If not set, defaults to 'unknown'
*/
setFileName(fileName: string): void {
this.fileName = basename(fileName);
}
/**
* Update hasher with new data chunk
* Accumulates chunks into 16KB blocks and hashes each complete block
*/
update(data: Uint8Array): SimpleHash {
const buffer = Buffer.from(data);
this.fileSize += buffer.length;
let offset = 0;
while (offset < buffer.length) {
const remaining = BtihV2Hasher.BLOCK_SIZE - this.currentBlockSize;
const toRead = Math.min(remaining, buffer.length - offset);
// Accumulate chunk into current block
this.currentBlock.push(buffer.subarray(offset, offset + toRead));
this.currentBlockSize += toRead;
offset += toRead;
// When block is complete, hash it
if (this.currentBlockSize === BtihV2Hasher.BLOCK_SIZE) {
this.finalizeBlock();
}
}
return this;
}
/**
* Finalize current block by hashing it and storing the hash
*/
private finalizeBlock(): void {
if (this.currentBlock.length === 0) return;
const blockData = Buffer.concat(this.currentBlock);
const blockHash = createHash('sha256').update(blockData).digest();
this.blockHashes.push(blockHash);
// Reset current block
this.currentBlock = [];
this.currentBlockSize = 0;
}
/**
* Compute final info hash
*
* Steps:
* 1. Finalize any partial block
* 2. Build merkle tree from block hashes
* 3. Construct info dictionary (name, length, pieces_root)
* 4. Bencode info dict
* 5. SHA-256 of bencoded data = info hash
*/
digest(): Buffer {
// Finalize any remaining partial block
if (this.currentBlockSize > 0) {
this.finalizeBlock();
}
// Handle empty file edge case
if (this.blockHashes.length === 0) {
// Empty file: hash empty block
const emptyHash = createHash('sha256').update(Buffer.alloc(0)).digest();
this.blockHashes.push(emptyHash);
}
// Build merkle tree to get pieces root
const piecesRoot = this.buildMerkleTree(this.blockHashes);
// Construct info dictionary (BEP 52 v2 format)
const infoDict = {
name: this.fileName,
length: this.fileSize,
'piece length': this.computePieceLength(this.fileSize),
'pieces root': piecesRoot,
'meta version': 2 // BitTorrent v2
};
// Bencode the info dictionary
const bencodedInfo = encode(infoDict);
// Info hash = SHA-256 of bencoded info dict
const infoHash = createHash('sha256').update(bencodedInfo).digest();
return infoHash;
}
/**
* Build balanced merkle tree from leaf hashes
*
* Algorithm (from BEP 52 reference implementation):
* 1. Pad to power of 2 with zero hashes
* 2. Recursively pair adjacent hashes
* 3. Hash each pair: SHA-256(left || right)
* 4. Repeat until single root hash remains
*/
private buildMerkleTree(hashes: Buffer[]): Buffer {
if (hashes.length === 0) {
throw new Error('Cannot build merkle tree from empty hash list');
}
// Pad to next power of 2 with zero hashes
let paddedHashes = [...hashes];
const targetSize = this.nextPowerOfTwo(paddedHashes.length);
const zeroHash = Buffer.alloc(32); // 32 bytes of zeros for SHA-256
while (paddedHashes.length < targetSize) {
paddedHashes.push(zeroHash);
}
// Build tree by repeatedly pairing and hashing
while (paddedHashes.length > 1) {
const parentHashes: Buffer[] = [];
for (let i = 0; i < paddedHashes.length; i += 2) {
const left = paddedHashes[i];
const right = paddedHashes[i + 1];
const combined = Buffer.concat([left, right]);
const parentHash = createHash('sha256').update(combined).digest();
parentHashes.push(parentHash);
}
paddedHashes = parentHashes;
}
return paddedHashes[0];
}
/**
* Compute piece length using BEP 52 formula
*
* Formula: piece_length = 16KB * 2^n
* where: file_size / piece_length < 2048
*
* This ensures each file has between 1024 and 2047 pieces
*/
private computePieceLength(fileSize: number): number {
if (fileSize === 0) {
return BtihV2Hasher.BLOCK_SIZE; // 16KB for empty files
}
let pieceLength = BtihV2Hasher.BLOCK_SIZE; // Start at 16KB
const maxPieces = 2048;
// Increase piece length until we have < 2048 pieces
while (Math.ceil(fileSize / pieceLength) >= maxPieces) {
pieceLength *= 2;
}
return pieceLength;
}
/**
* Find next power of 2 >= n
*/
private nextPowerOfTwo(n: number): number {
if (n === 0) return 1;
n--;
n |= n >> 1;
n |= n >> 2;
n |= n >> 4;
n |= n >> 8;
n |= n >> 16;
return n + 1;
}
}
@@ -0,0 +1,12 @@
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
export const codeTable = {
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
[CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
}
@@ -0,0 +1,50 @@
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {create} from "multiformats/hashes/digest";
import {CID} from 'multiformats/cid';
import {SimpleHash} from "./SimpleHash";
import {codeTable} from "@root/file-id/CodeTable";
/**
* Compute the CIDs of a file using specific algorithms
* @param stream The Readable stream of the file
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
* @returns Array of CIDs (in the order of the algorithms)
*/
export async function computeCIDs({stream, algorithms,createHasher}: {
stream: ReadableStream<Uint8Array>;
algorithms: CID_ALGORITHM_NAMES[];
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
}): Promise<string[]> {
const hashers = await hasherDefiner(algorithms,createHasher);
const reader = stream.getReader();
while (true) {
const {done, value} = await reader.read();
if (done) break;
// Assuming the hasher can handle Uint8Array directly
for (const item of hashers) {
await item.hasher.update(value);
}
}
return cidFinalize(hashers);
}
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
hasher: SimpleHash,
code: CID_ALGORITHM_CODES
}[]> {
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
.map(async algo => ({
hasher: await createHasher(algo),
code: codeTable[algo]
}));
return await Promise.all(hashers);
}
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
return await Promise.all(hashers.map(async ({code, hasher}) => {
const hashBuffer = await hasher.digest();
const digest = create(code, hashBuffer);
return CID.createV1(code, digest).toString();
}));
}
@@ -0,0 +1,18 @@
import crc32 from "crc-32";
import {SimpleHash} from "@root/file-id/SimpleHash";
export class Crc32Hash implements SimpleHash {
private _crc: number = undefined;
update(data: Buffer): SimpleHash {
// Update the CRC-32 checksum with the new chunk of data
this._crc = crc32.buf(data, this._crc);
return this;
}
digest(): Buffer {
const buffer = Buffer.alloc(4); // Create a buffer of 4 bytes (32 bits)
buffer.writeInt32BE(this._crc, 0); // Write the unsigned integer to the buffer in big-endian format
return buffer;
}
}
@@ -0,0 +1,31 @@
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {SimpleHash} from "./SimpleHash";
import {createHash} from "crypto";
import {Crc32Hash} from "./Crc32Hash";
import {BtihV2Hasher} from "./BtihV2Hasher";
/**
* Compute the CIDs of a file using specific algorithms
* @returns Array of CIDs (in the order of the algorithms)
* @param algo
*/
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
switch (algo) {
case CID_ALGORITHM_NAMES.sha256:
return createHash("sha256");
case CID_ALGORITHM_NAMES.sha1:
return createHash("sha1");
case CID_ALGORITHM_NAMES.md5:
return createHash("md5");
case CID_ALGORITHM_NAMES.sha3_256:
return createHash("sha3-256");
case CID_ALGORITHM_NAMES.sha3_384:
return createHash("sha3-384");
case CID_ALGORITHM_NAMES.crc32:
return new Crc32Hash();
case CID_ALGORITHM_NAMES.btih_v2:
return new BtihV2Hasher();
default:
throw new Error(`Unsupported algorithm: ${algo}`);
}
}
@@ -0,0 +1,31 @@
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
import {SimpleHash} from "./SimpleHash";
import {Crc32Hash} from "./Crc32Hash";
import {BrowserHasher} from "@root/file-id/BrowserHasher";
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
const hasher = (() => {
switch (algo) {
case CID_ALGORITHM_NAMES.sha256:
return new BrowserHasher('sha-256');
case CID_ALGORITHM_NAMES.sha1:
return new BrowserHasher('sha-1');
case CID_ALGORITHM_NAMES.md5:
return new BrowserHasher('md5');
case CID_ALGORITHM_NAMES.sha3_256:
return new BrowserHasher('sha3-256');
case CID_ALGORITHM_NAMES.sha3_384:
return new BrowserHasher('sha3-384');
case CID_ALGORITHM_NAMES.crc32:
return new Crc32Hash();
default:
throw new Error(`Unsupported algorithm: ${algo}`);
}
})();
if (hasher instanceof BrowserHasher) {
await hasher.initialize();
}
return hasher;
}
@@ -0,0 +1,104 @@
import { createHash } from 'crypto';
import { open, stat } from 'fs/promises';
import { create } from 'multiformats/hashes/digest';
import { CID } from 'multiformats/cid';
import { CID_ALGORITHM_CODES } from '../hash-compute/MultiHashData.js';
/**
* Computes a midhash256 based on sampling the middle 1MB of a file
* plus the file size as a 64-bit integer prefix.
*
* Algorithm:
* - For files <= 1MB: Hashes entire file content + size prefix
* - For files > 1MB: Hashes middle 1MB + size prefix
*
* The size prefix ensures that files with identical middle content
* but different sizes produce different hashes.
*
* Performance: ~20ms regardless of file size (O(1) complexity)
*
* @param filePath - Path to file
* @returns CID string in proper CID v1 format with multicodec 0x1000
*/
export async function computeMidHash256(filePath: string): Promise<string> {
const SAMPLE_SIZE = 1024 * 1024; // 1MB
// Get file size (instant - no file read)
const stats = await stat(filePath);
const fileSize = stats.size;
// Create size buffer (64-bit big-endian)
const sizeBuffer = Buffer.allocUnsafe(8);
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
// Extract sample data
let sampleData: Buffer;
if (fileSize <= SAMPLE_SIZE) {
// Small file: read entire content
const fd = await open(filePath, 'r');
sampleData = Buffer.allocUnsafe(fileSize);
await fd.read(sampleData, 0, fileSize, 0);
await fd.close();
} else {
// Large file: read middle 1MB
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
const fd = await open(filePath, 'r');
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
await fd.read(sampleData, 0, SAMPLE_SIZE, middleOffset);
await fd.close();
}
// Compute SHA-256 hash of [size + sample]
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
const hashBuffer = createHash('sha256')
.update(hashInput)
.digest();
// Wrap in CID format with custom multicodec 0x1000
const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer);
return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString();
}
/**
* Synchronous version of computeMidHash256 for use in worker threads
* Uses synchronous fs operations for compatibility with Piscina workers
*/
export function computeMidHash256Sync(filePath: string): string {
const fs = require('fs');
const SAMPLE_SIZE = 1024 * 1024; // 1MB
// Get file size
const stats = fs.statSync(filePath);
const fileSize = stats.size;
// Create size buffer (64-bit big-endian)
const sizeBuffer = Buffer.allocUnsafe(8);
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
// Extract sample data
let sampleData: Buffer;
if (fileSize <= SAMPLE_SIZE) {
// Small file: read entire content
const fd = fs.openSync(filePath, 'r');
sampleData = Buffer.allocUnsafe(fileSize);
fs.readSync(fd, sampleData, 0, fileSize, 0);
fs.closeSync(fd);
} else {
// Large file: read middle 1MB
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
const fd = fs.openSync(filePath, 'r');
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
fs.readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
fs.closeSync(fd);
}
// Compute SHA-256 hash of [size + sample]
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
const hashBuffer = createHash('sha256')
.update(hashInput)
.digest();
// Wrap in CID format with custom multicodec 0x1000
const digest = create(CID_ALGORITHM_CODES.midhash256, hashBuffer);
return CID.createV1(CID_ALGORITHM_CODES.midhash256, digest).toString();
}
@@ -0,0 +1,5 @@
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
export interface FileIDComputerInterface{
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
}
@@ -0,0 +1,31 @@
import {Piscina} from "piscina";
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
export class FileIDComputerWorker{
private piscina: Piscina;
constructor(workerPath?:string) {
if(!workerPath) {
// Construct the URL for the current module
let distFolder = import.meta.dirname;
distFolder = distFolder.replace('src', 'dist');
distFolder = distFolder + "/worker.js";
workerPath = new URL(distFolder,"file://").href;
}
this.piscina = new Piscina({
maxThreads: 4,
//filename: new URL('./ShaComputeWorker.ts', import.meta.url).href
filename: workerPath
});
}
/**
* Compute the CIDs of a file using specific algorithms
* @param filePath The path to the file
* @param algorithms Array of algorithms ('sha256', 'sha1')
* @returns Array of CIDs (in the order of the algorithms)
*/
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
return this.piscina.run({filePath, algorithms});
}
}
@@ -0,0 +1,183 @@
import {MultiHashData} from '../hash-compute/MultiHashData';
import {CID} from 'multiformats/cid';
/**
* Magnet Link Generator for BitTorrent v2
*
* Generates BitTorrent v2 magnet links from file metadata.
*
* Format: magnet:?xt=urn:btmh:1220<info-hash-hex>&dn=<filename>&xl=<filesize>
*
* Components:
* - xt: eXact Topic (torrent identifier)
* - btmh: BitTorrent Merkle Hash (v2 URN scheme)
* - 1220: Multihash prefix (0x12 = SHA-256, 0x20 = 32 bytes)
* - info-hash-hex: 64-character hex string (32 bytes SHA-256)
* - dn: Display Name (filename)
* - xl: eXact Length (file size in bytes)
*
* Reference: BEP 52, BEP 9 (magnet URIs)
*/
export class MagnetLinkGenerator {
/**
* Generate BitTorrent v2 magnet link from metadata
*
* @param metadata - File metadata containing cid_btih_v2
* @param fileName - Original filename for display name
* @param fileSize - File size in bytes
* @returns Magnet link string, or null if btih_v2 not available
*
* @example
* const metadata = {cid_btih_v2: 'bafyb...'};
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
*/
static generate(
metadata: MultiHashData,
fileName: string,
fileSize: number
): string | null {
const cidString = metadata.cid_btih_v2;
if (!cidString) {
return null; // BitTorrent v2 hash not computed
}
try {
// Decode CID to get raw info hash
const cid = CID.parse(cidString);
const infoHash = cid.multihash.digest; // Raw 32-byte SHA-256 hash
// Convert to hex for magnet link
const infoHashHex = Buffer.from(infoHash).toString('hex');
// Build magnet link components
const components = [
`xt=urn:btmh:1220${infoHashHex}`, // 1220 = multihash prefix (SHA-256, 32 bytes)
`dn=${encodeURIComponent(fileName)}`, // Display name (URL encoded)
`xl=${fileSize}` // Exact length in bytes
];
return `magnet:?${components.join('&')}`;
} catch (error) {
console.error('Failed to generate magnet link from CID:', error);
return null;
}
}
/**
* Extract info hash from magnet link
*
* @param magnetLink - Magnet link string
* @returns Info hash as hex string, or null if invalid
*
* @example
* const hash = MagnetLinkGenerator.extractInfoHash('magnet:?xt=urn:btmh:1220abc...');
* // Returns: 'abc...'
*/
static extractInfoHash(magnetLink: string): string | null {
try {
const match = magnetLink.match(/xt=urn:btmh:1220([0-9a-f]{64})/i);
return match ? match[1] : null;
} catch (error) {
return null;
}
}
/**
* Extract filename from magnet link
*
* @param magnetLink - Magnet link string
* @returns Decoded filename, or null if not present
*/
static extractFileName(magnetLink: string): string | null {
try {
const match = magnetLink.match(/dn=([^&]+)/);
return match ? decodeURIComponent(match[1]) : null;
} catch (error) {
return null;
}
}
/**
* Extract file size from magnet link
*
* @param magnetLink - Magnet link string
* @returns File size in bytes, or null if not present
*/
static extractFileSize(magnetLink: string): number | null {
try {
const match = magnetLink.match(/xl=(\d+)/);
return match ? parseInt(match[1], 10) : null;
} catch (error) {
return null;
}
}
/**
* Validate BitTorrent v2 magnet link format
*
* @param magnetLink - Magnet link string
* @returns True if valid BitTorrent v2 magnet link
*/
static isValid(magnetLink: string): boolean {
if (!magnetLink.startsWith('magnet:?')) {
return false;
}
// Must have btmh (BitTorrent Merkle Hash) identifier
if (!magnetLink.includes('xt=urn:btmh:1220')) {
return false;
}
// Must have 64-character hex hash (32 bytes SHA-256)
const hashMatch = magnetLink.match(/1220([0-9a-f]{64})/i);
if (!hashMatch) {
return false;
}
return true;
}
/**
* Parse magnet link into components
*
* @param magnetLink - Magnet link string
* @returns Object with parsed components, or null if invalid
*/
static parse(magnetLink: string): {
infoHash: string;
fileName?: string;
fileSize?: number;
} | null {
if (!this.isValid(magnetLink)) {
return null;
}
const infoHash = this.extractInfoHash(magnetLink);
if (!infoHash) {
return null;
}
return {
infoHash,
fileName: this.extractFileName(magnetLink) || undefined,
fileSize: this.extractFileSize(magnetLink) || undefined
};
}
/**
* Generate magnet link from CID string directly
*
* @param cidString - CID string (e.g., 'bafyb...')
* @param fileName - Original filename
* @param fileSize - File size in bytes
* @returns Magnet link string, or null if CID invalid
*/
static fromCID(
cidString: string,
fileName: string,
fileSize: number
): string | null {
return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
}
}
@@ -0,0 +1,19 @@
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
import fs from "fs";
import {Readable} from 'stream';
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
import {createHasher} from "@root/file-id/CreateHasher";
/**
* Compute the CIDs of a file using specific algorithms
* @param filePath The path to the file
* @param algorithms Array of algorithms ('sha256', 'sha1')
* @returns Array of CIDs (in the order of the algorithms)
*/
export default async function computeCIDs({filePath, algorithms}: {
filePath: string;
algorithms: CID_ALGORITHM_NAMES[]
}): Promise<string[]> {
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
return computeCIDsInternal({stream, algorithms, createHasher});
}
@@ -0,0 +1,5 @@
export interface SimpleHash {
update(data: Uint8Array): SimpleHash | Promise<SimpleHash>;
digest(): Uint8Array | Promise<Uint8Array>;
}