Compare commits
10
Commits
81d4295427
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
382c21a514 | ||
|
|
ea67a84b2d | ||
|
|
b7ba03c7d9 | ||
|
|
fa8a367d92 | ||
|
|
1eef338f79 | ||
|
|
ef21690b12 | ||
|
|
cd21b9de1e | ||
|
|
1f0412d220 | ||
|
|
08c7479a1a | ||
|
|
a2ac04563e |
@@ -28,3 +28,4 @@ yarn-error.log*
|
||||
/dist/
|
||||
.idea
|
||||
build
|
||||
.pnpm-store/
|
||||
|
||||
@@ -25,7 +25,7 @@ To inspect the details of a CID, including its version, codec, multihash and mor
|
||||
|
||||
## Installation
|
||||
|
||||
This package is part of the Meta Orbit monorepo and uses pnpm workspaces.
|
||||
This package is part of the MetaMesh monorepo and uses pnpm workspaces.
|
||||
|
||||
From the monorepo root:
|
||||
```bash
|
||||
@@ -41,21 +41,21 @@ When using as a dependency in other packages:
|
||||
}
|
||||
```
|
||||
|
||||
## Usage of ComputeHashIndexCache
|
||||
## Usage of HashComputerIndexCache
|
||||
|
||||
To integrate Meta-Hash into your application, follow these steps:
|
||||
|
||||
1. **Import the Library**: Import the necessary components from the library.
|
||||
|
||||
```typescript
|
||||
import { ComputeHashIndexCache, CID_ALGORITHM_NAMES } from "./folder-watch-hasher";
|
||||
import { HashComputerIndexCache, CID_ALGORITHM_NAMES } from "./folder-watch-hasher";
|
||||
```
|
||||
|
||||
2. **Initialize the Watcher**: Create an instance of `ComputeHashIndexCache`, specifying the path to the folder you want to monitor and the hash algorithms you wish to use.
|
||||
2. **Initialize the Watcher**: Create an instance of `HashComputerIndexCache`, specifying the path to the folder you want to monitor and the hash algorithms you wish to use.
|
||||
|
||||
```typescript
|
||||
//note that the csv file will be derived from the path provided into multiple files (one for each algorithm)
|
||||
const computer = new ComputeHashIndexCache("path/to/your/indexfile/index.csv", [CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.sha1]);
|
||||
const computer = new HashComputerIndexCache("path/to/your/indexfile/index.csv", [CID_ALGORITHM_NAMES.sha256, CID_ALGORITHM_NAMES.sha1]);
|
||||
```
|
||||
|
||||
3. **Compute Hashes**: Invoke the `computeMissingHash` method for each file you wish to hash. This method will automatically check the index for existing hashes and compute missing ones.
|
||||
@@ -160,13 +160,6 @@ pnpm test
|
||||
|
||||
Uses Vitest for unit tests.
|
||||
|
||||
## Integration
|
||||
|
||||
Used by:
|
||||
- **meta-mesh**: File content hashing for deduplication
|
||||
- **meta-orbit**: Content addressing for distributed storage
|
||||
- **meta-ui**: Browser-based file verification
|
||||
|
||||
## Performance
|
||||
|
||||
- **Index Cache**: Avoids rehashing unchanged files (checks size + mtime)
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
"vitest": "^1.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"bencodec": "^2.0.1",
|
||||
"chokidar": "^3.6.0",
|
||||
"crc-32": "^1.2.2",
|
||||
"csv-parse": "^5.5.5",
|
||||
|
||||
Generated
+2341
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
allowBuilds:
|
||||
esbuild: set this to true or false
|
||||
@@ -0,0 +1,200 @@
|
||||
import {createHash} from 'crypto';
|
||||
import {SimpleHash} from './SimpleHash';
|
||||
import {encode} from 'bencodec';
|
||||
import {basename} from 'path';
|
||||
|
||||
/**
|
||||
* BitTorrent v2 Info Hash Hasher (BEP 52)
|
||||
*
|
||||
* Implements BitTorrent v2 file hashing with:
|
||||
* - Fixed 16KB block size
|
||||
* - SHA-256 merkle tree
|
||||
* - Info hash computation
|
||||
*
|
||||
* Architecture:
|
||||
* 1. Streaming: Accumulates file data into 16KB blocks
|
||||
* 2. Block hashing: SHA-256 hash each complete block
|
||||
* 3. Merkle tree: Build balanced tree from block hashes
|
||||
* 4. Info hash: SHA-256(bencode(info_dict))
|
||||
*/
|
||||
export class BtihV2Hasher implements SimpleHash {
|
||||
private static readonly BLOCK_SIZE = 16 * 1024; // 16KB fixed by BEP 52
|
||||
|
||||
private blockHashes: Buffer[] = [];
|
||||
private currentBlock: Buffer[] = [];
|
||||
private currentBlockSize: number = 0;
|
||||
private fileSize: number = 0;
|
||||
private fileName: string = 'unknown';
|
||||
|
||||
/**
|
||||
* Optional: Set filename for info dictionary
|
||||
* If not set, defaults to 'unknown'
|
||||
*/
|
||||
setFileName(fileName: string): void {
|
||||
this.fileName = basename(fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update hasher with new data chunk
|
||||
* Accumulates chunks into 16KB blocks and hashes each complete block
|
||||
*/
|
||||
update(data: Uint8Array): SimpleHash {
|
||||
const buffer = Buffer.from(data);
|
||||
this.fileSize += buffer.length;
|
||||
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const remaining = BtihV2Hasher.BLOCK_SIZE - this.currentBlockSize;
|
||||
const toRead = Math.min(remaining, buffer.length - offset);
|
||||
|
||||
// Accumulate chunk into current block
|
||||
this.currentBlock.push(buffer.subarray(offset, offset + toRead));
|
||||
this.currentBlockSize += toRead;
|
||||
offset += toRead;
|
||||
|
||||
// When block is complete, hash it
|
||||
if (this.currentBlockSize === BtihV2Hasher.BLOCK_SIZE) {
|
||||
this.finalizeBlock();
|
||||
}
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalize current block by hashing it and storing the hash
|
||||
*/
|
||||
private finalizeBlock(): void {
|
||||
if (this.currentBlock.length === 0) return;
|
||||
|
||||
const blockData = Buffer.concat(this.currentBlock);
|
||||
const blockHash = createHash('sha256').update(blockData).digest();
|
||||
this.blockHashes.push(blockHash);
|
||||
|
||||
// Reset current block
|
||||
this.currentBlock = [];
|
||||
this.currentBlockSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute final info hash
|
||||
*
|
||||
* Steps:
|
||||
* 1. Finalize any partial block
|
||||
* 2. Build merkle tree from block hashes
|
||||
* 3. Construct info dictionary (name, length, pieces_root)
|
||||
* 4. Bencode info dict
|
||||
* 5. SHA-256 of bencoded data = info hash
|
||||
*/
|
||||
digest(): Buffer {
|
||||
// Finalize any remaining partial block
|
||||
if (this.currentBlockSize > 0) {
|
||||
this.finalizeBlock();
|
||||
}
|
||||
|
||||
// Handle empty file edge case
|
||||
if (this.blockHashes.length === 0) {
|
||||
// Empty file: hash empty block
|
||||
const emptyHash = createHash('sha256').update(Buffer.alloc(0)).digest();
|
||||
this.blockHashes.push(emptyHash);
|
||||
}
|
||||
|
||||
// Build merkle tree to get pieces root
|
||||
const piecesRoot = this.buildMerkleTree(this.blockHashes);
|
||||
|
||||
// Construct info dictionary (BEP 52 v2 format)
|
||||
const infoDict = {
|
||||
name: this.fileName,
|
||||
length: this.fileSize,
|
||||
'piece length': this.computePieceLength(this.fileSize),
|
||||
'pieces root': piecesRoot,
|
||||
'meta version': 2 // BitTorrent v2
|
||||
};
|
||||
|
||||
// Bencode the info dictionary
|
||||
const bencodedInfo = encode(infoDict);
|
||||
|
||||
// Info hash = SHA-256 of bencoded info dict
|
||||
const infoHash = createHash('sha256').update(bencodedInfo).digest();
|
||||
|
||||
return infoHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build balanced merkle tree from leaf hashes
|
||||
*
|
||||
* Algorithm (from BEP 52 reference implementation):
|
||||
* 1. Pad to power of 2 with zero hashes
|
||||
* 2. Recursively pair adjacent hashes
|
||||
* 3. Hash each pair: SHA-256(left || right)
|
||||
* 4. Repeat until single root hash remains
|
||||
*/
|
||||
private buildMerkleTree(hashes: Buffer[]): Buffer {
|
||||
if (hashes.length === 0) {
|
||||
throw new Error('Cannot build merkle tree from empty hash list');
|
||||
}
|
||||
|
||||
// Pad to next power of 2 with zero hashes
|
||||
let paddedHashes = [...hashes];
|
||||
const targetSize = this.nextPowerOfTwo(paddedHashes.length);
|
||||
const zeroHash = Buffer.alloc(32); // 32 bytes of zeros for SHA-256
|
||||
while (paddedHashes.length < targetSize) {
|
||||
paddedHashes.push(zeroHash);
|
||||
}
|
||||
|
||||
// Build tree by repeatedly pairing and hashing
|
||||
while (paddedHashes.length > 1) {
|
||||
const parentHashes: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < paddedHashes.length; i += 2) {
|
||||
const left = paddedHashes[i];
|
||||
const right = paddedHashes[i + 1];
|
||||
const combined = Buffer.concat([left, right]);
|
||||
const parentHash = createHash('sha256').update(combined).digest();
|
||||
parentHashes.push(parentHash);
|
||||
}
|
||||
|
||||
paddedHashes = parentHashes;
|
||||
}
|
||||
|
||||
return paddedHashes[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute piece length using BEP 52 formula
|
||||
*
|
||||
* Formula: piece_length = 16KB * 2^n
|
||||
* where: file_size / piece_length < 2048
|
||||
*
|
||||
* This ensures each file has between 1024 and 2047 pieces
|
||||
*/
|
||||
private computePieceLength(fileSize: number): number {
|
||||
if (fileSize === 0) {
|
||||
return BtihV2Hasher.BLOCK_SIZE; // 16KB for empty files
|
||||
}
|
||||
|
||||
let pieceLength = BtihV2Hasher.BLOCK_SIZE; // Start at 16KB
|
||||
const maxPieces = 2048;
|
||||
|
||||
// Increase piece length until we have < 2048 pieces
|
||||
while (Math.ceil(fileSize / pieceLength) >= maxPieces) {
|
||||
pieceLength *= 2;
|
||||
}
|
||||
|
||||
return pieceLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find next power of 2 >= n
|
||||
*/
|
||||
private nextPowerOfTwo(n: number): number {
|
||||
if (n === 0) return 1;
|
||||
n--;
|
||||
n |= n >> 1;
|
||||
n |= n >> 2;
|
||||
n |= n >> 4;
|
||||
n |= n >> 8;
|
||||
n |= n >> 16;
|
||||
return n + 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
|
||||
|
||||
/**
|
||||
* Decoding a bare CIDv1 back into the algorithm that produced it.
|
||||
*
|
||||
* This is **the** TypeScript CID decoder for the platform. It exists because a
|
||||
* bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
|
||||
* "the sha2-256 of this file" has to ask the multicodec, not a string.
|
||||
*
|
||||
* Before this module, meta-dup carried a private hand-rolled base32+uvarint
|
||||
* decoder, the meta-sort editor carried a second one with a *different* rank
|
||||
* ladder, and meta-fuse had none at all (so it could only read the legacy
|
||||
* `cid_midhash256` field, and silently failed on any record that didn't have
|
||||
* one). One implementation, exported, replaces all three.
|
||||
*
|
||||
* ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
|
||||
* has two encoders that disagree about the codec: meta-hash writes
|
||||
* `codec == algorithm code`, while the fullhash plugin and the gateway wrap
|
||||
* every digest with the raw codec `0x55`. The multihash code is the algorithm in
|
||||
* both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
|
||||
*/
|
||||
|
||||
/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidMultihashCode(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).multihash.code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The content codec of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidCodec(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first member of `cids` whose multihash code is `mhCode`.
|
||||
*
|
||||
* Takes the raw code so callers can ask for digests meta-hash doesn't itself
|
||||
* compute (e.g. the gateway's per-file btih codes).
|
||||
*/
|
||||
export function pickCidByMulticodec(
|
||||
cids: readonly string[] | undefined,
|
||||
mhCode: number
|
||||
): string | undefined {
|
||||
return cids?.find(cid => cidMultihashCode(cid) === mhCode);
|
||||
}
|
||||
|
||||
/** The member of `cids` produced by `algo`, if present. */
|
||||
export function pickCidByAlgorithm(
|
||||
cids: readonly string[] | undefined,
|
||||
algo: CidAlgorithm
|
||||
): string | undefined {
|
||||
return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Has `algo` already been computed for this cid set?
|
||||
*
|
||||
* The "skip if already computed" check. It used to be an O(1) property lookup
|
||||
* (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
|
||||
* That is a handful of microseconds against a hash computation that streams the
|
||||
* whole file — the trade is not close.
|
||||
*/
|
||||
export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
|
||||
return pickCidByAlgorithm(cids, algo) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `cid` to `cids` unless an equal string is already there.
|
||||
*
|
||||
* Dedup is on the exact string. Two encoders can spell the same digest
|
||||
* differently (codec 0x55 vs codec == mh), and both spellings are legitimate
|
||||
* members — meta-core's reverse index maps each to the same record.
|
||||
*/
|
||||
export function addCid(cids: string[], cid: string): void {
|
||||
if (cid && !cids.includes(cid)) {
|
||||
cids.push(cid);
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export const codeTable = {
|
||||
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
|
||||
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
|
||||
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
|
||||
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
|
||||
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
|
||||
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
|
||||
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {create} from "multiformats/hashes/digest";
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {codeTable} from "@root/file-id/CodeTable";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param stream The Readable stream of the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @returns Array of bare CIDv1 strings, in the order of `algorithms`
|
||||
*/
|
||||
export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
export async function computeCIDs({stream, algorithms, createHasher}: {
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
algorithms: CID_ALGORITHM_NAMES[];
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
|
||||
algorithms: CidAlgorithm[];
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
|
||||
}): Promise<string[]> {
|
||||
const hashers = await hasherDefiner(algorithms,createHasher);
|
||||
const hashers = await hasherDefiner(algorithms, createHasher);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const {done, value} = await reader.read();
|
||||
@@ -28,19 +27,27 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
return cidFinalize(hashers);
|
||||
}
|
||||
|
||||
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
|
||||
async function hasherDefiner(algorithms: CidAlgorithm[],
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
|
||||
hasher: SimpleHash,
|
||||
code: CID_ALGORITHM_CODES
|
||||
}[]> {
|
||||
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
|
||||
const hashers = algorithms
|
||||
.filter(algo => algo in CID_ALGORITHM_CODES)
|
||||
.map(async algo => ({
|
||||
hasher: await createHasher(algo),
|
||||
code: codeTable[algo]
|
||||
code: CID_ALGORITHM_CODES[algo]
|
||||
}));
|
||||
return await Promise.all(hashers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
|
||||
* plugin and the gateway instead use the raw codec (0x55) and carry the
|
||||
* algorithm only in the multihash. Both are valid CIDs for the same digest, and
|
||||
* both must decode to the same algorithm — which is why every consumer selects
|
||||
* on the **multihash code**, never the codec. See `CidDecode.ts`.
|
||||
*/
|
||||
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
|
||||
return await Promise.all(hashers.map(async ({code, hasher}) => {
|
||||
const hashBuffer = await hasher.digest();
|
||||
|
||||
@@ -1,27 +1,31 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {createHash} from "crypto";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BtihV2Hasher} from "./BtihV2Hasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @param algo
|
||||
* Build the stream hasher for one algorithm.
|
||||
*
|
||||
* `midhash256` is absent on purpose: it is not a stream digest (it hashes the
|
||||
* file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
|
||||
*/
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return createHash("sha256");
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return createHash("sha1");
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return createHash("md5");
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return createHash("sha3-256");
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return createHash("sha3-384");
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
case 'btih_v2':
|
||||
return new BtihV2Hasher();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BrowserHasher} from "@root/file-id/BrowserHasher";
|
||||
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
/**
|
||||
* Browser build. Note it supports fewer algorithms than the node one: there is
|
||||
* no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
|
||||
* either throws.
|
||||
*/
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
const hasher = (() => {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return new BrowserHasher('sha-256');
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return new BrowserHasher('sha-1');
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return new BrowserHasher('md5');
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return new BrowserHasher('sha3-256');
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return new BrowserHasher('sha3-384');
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export interface FileIDComputerInterface{
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
|
||||
}
|
||||
@@ -1,10 +1,16 @@
|
||||
import {Piscina} from "piscina";
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
|
||||
export class FileIDComputerWorker{
|
||||
private piscina: Piscina;
|
||||
|
||||
constructor(workerPath?:string) {
|
||||
// `WORKER_URL` is the escape hatch the tests have always set — it just
|
||||
// wasn't read. Honour it before falling back to path derivation, which
|
||||
// only works for a caller running from `dist/`.
|
||||
if (!workerPath && process.env.WORKER_URL) {
|
||||
workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
|
||||
}
|
||||
if(!workerPath) {
|
||||
// Construct the URL for the current module
|
||||
let distFolder = import.meta.dirname;
|
||||
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
|
||||
public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
|
||||
return this.piscina.run({filePath, algorithms});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
import {MultiHashData} from '../hash-compute/MultiHashData';
|
||||
import {pickCidByAlgorithm} from './CidDecode';
|
||||
import {CID} from 'multiformats/cid';
|
||||
|
||||
/**
|
||||
* Magnet Link Generator for BitTorrent v2
|
||||
*
|
||||
* Generates BitTorrent v2 magnet links from file metadata.
|
||||
*
|
||||
* Format: magnet:?xt=urn:btmh:1220<info-hash-hex>&dn=<filename>&xl=<filesize>
|
||||
*
|
||||
* Components:
|
||||
* - xt: eXact Topic (torrent identifier)
|
||||
* - btmh: BitTorrent Merkle Hash (v2 URN scheme)
|
||||
* - 1220: Multihash prefix (0x12 = SHA-256, 0x20 = 32 bytes)
|
||||
* - info-hash-hex: 64-character hex string (32 bytes SHA-256)
|
||||
* - dn: Display Name (filename)
|
||||
* - xl: eXact Length (file size in bytes)
|
||||
*
|
||||
* Reference: BEP 52, BEP 9 (magnet URIs)
|
||||
*/
|
||||
export class MagnetLinkGenerator {
|
||||
/**
|
||||
* Generate BitTorrent v2 magnet link from metadata
|
||||
*
|
||||
* @param metadata - File metadata whose `cids` include a btih-v2 CID
|
||||
* @param fileName - Original filename for display name
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if btih_v2 not available
|
||||
*
|
||||
* @example
|
||||
* const metadata = {cids: ['bafyb...']};
|
||||
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
|
||||
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
|
||||
*/
|
||||
static generate(
|
||||
metadata: MultiHashData,
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
// The btih-v2 CID is recovered from the bare list by its multicodec
|
||||
// (0x10B7), not by a field name. See CidDecode.ts.
|
||||
const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
if (!cidString) {
|
||||
return null; // BitTorrent v2 hash not computed
|
||||
}
|
||||
|
||||
try {
|
||||
// Decode CID to get raw info hash
|
||||
const cid = CID.parse(cidString);
|
||||
const infoHash = cid.multihash.digest; // Raw 32-byte SHA-256 hash
|
||||
|
||||
// Convert to hex for magnet link
|
||||
const infoHashHex = Buffer.from(infoHash).toString('hex');
|
||||
|
||||
// Build magnet link components
|
||||
const components = [
|
||||
`xt=urn:btmh:1220${infoHashHex}`, // 1220 = multihash prefix (SHA-256, 32 bytes)
|
||||
`dn=${encodeURIComponent(fileName)}`, // Display name (URL encoded)
|
||||
`xl=${fileSize}` // Exact length in bytes
|
||||
];
|
||||
|
||||
return `magnet:?${components.join('&')}`;
|
||||
} catch (error) {
|
||||
console.error('Failed to generate magnet link from CID:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract info hash from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Info hash as hex string, or null if invalid
|
||||
*
|
||||
* @example
|
||||
* const hash = MagnetLinkGenerator.extractInfoHash('magnet:?xt=urn:btmh:1220abc...');
|
||||
* // Returns: 'abc...'
|
||||
*/
|
||||
static extractInfoHash(magnetLink: string): string | null {
|
||||
try {
|
||||
const match = magnetLink.match(/xt=urn:btmh:1220([0-9a-f]{64})/i);
|
||||
return match ? match[1] : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Decoded filename, or null if not present
|
||||
*/
|
||||
static extractFileName(magnetLink: string): string | null {
|
||||
try {
|
||||
const match = magnetLink.match(/dn=([^&]+)/);
|
||||
return match ? decodeURIComponent(match[1]) : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract file size from magnet link
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns File size in bytes, or null if not present
|
||||
*/
|
||||
static extractFileSize(magnetLink: string): number | null {
|
||||
try {
|
||||
const match = magnetLink.match(/xl=(\d+)/);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate BitTorrent v2 magnet link format
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns True if valid BitTorrent v2 magnet link
|
||||
*/
|
||||
static isValid(magnetLink: string): boolean {
|
||||
if (!magnetLink.startsWith('magnet:?')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Must have btmh (BitTorrent Merkle Hash) identifier
|
||||
if (!magnetLink.includes('xt=urn:btmh:1220')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Must have 64-character hex hash (32 bytes SHA-256)
|
||||
const hashMatch = magnetLink.match(/1220([0-9a-f]{64})/i);
|
||||
if (!hashMatch) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse magnet link into components
|
||||
*
|
||||
* @param magnetLink - Magnet link string
|
||||
* @returns Object with parsed components, or null if invalid
|
||||
*/
|
||||
static parse(magnetLink: string): {
|
||||
infoHash: string;
|
||||
fileName?: string;
|
||||
fileSize?: number;
|
||||
} | null {
|
||||
if (!this.isValid(magnetLink)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const infoHash = this.extractInfoHash(magnetLink);
|
||||
if (!infoHash) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
infoHash,
|
||||
fileName: this.extractFileName(magnetLink) || undefined,
|
||||
fileSize: this.extractFileSize(magnetLink) || undefined
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate magnet link from CID string directly
|
||||
*
|
||||
* @param cidString - CID string (e.g., 'bafyb...')
|
||||
* @param fileName - Original filename
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if CID invalid
|
||||
*/
|
||||
static fromCID(
|
||||
cidString: string,
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
return this.generate({cids: [cidString]}, fileName, fileSize);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
import fs from "fs";
|
||||
import {Readable} from 'stream';
|
||||
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
|
||||
@@ -12,8 +12,8 @@ import {createHasher} from "@root/file-id/CreateHasher";
|
||||
*/
|
||||
export default async function computeCIDs({filePath, algorithms}: {
|
||||
filePath: string;
|
||||
algorithms: CID_ALGORITHM_NAMES[]
|
||||
algorithms: CidAlgorithm[]
|
||||
}): Promise<string[]> {
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath));
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
return computeCIDsInternal({stream, algorithms, createHasher});
|
||||
}
|
||||
@@ -1,30 +1,29 @@
|
||||
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, MultiHashData} from "./MultiHashData";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class HashComputer{
|
||||
export class HashComputer {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[],
|
||||
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
|
||||
constructor(private targetHash: CidAlgorithm[],
|
||||
private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
|
||||
}
|
||||
|
||||
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
// Which digests are still missing? On a bare-CID list this costs a decode
|
||||
// per member rather than a property lookup — negligible against streaming
|
||||
// the whole file through a hasher.
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +1,28 @@
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {Readable} from "stream";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import fs from "fs";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
export class HashComputerFile implements ComputeInterface{
|
||||
export class HashComputerFile implements ComputeInterface {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
constructor(private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath));
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,59 @@
|
||||
import {HashComputerWorker} from "./HashComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {HashIndexManager} from "./HashIndexManager";
|
||||
import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {stat} from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* The translation point between the two shapes.
|
||||
*
|
||||
* `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
|
||||
* persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
|
||||
* **per-algorithm columns**, because its job is the O(1) "already computed?"
|
||||
* check. Neither shape leaks into the other; this class converts.
|
||||
*/
|
||||
export class HashComputerIndexCache implements ComputeInterface {
|
||||
private hashIndexManager: HashIndexManager;
|
||||
private hashComputer: HashComputerWorker;
|
||||
|
||||
constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash,workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
|
||||
constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
await this.hashIndexManager.init();
|
||||
let stats = await stat(filePath);
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
|
||||
const stats = await stat(filePath);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// Cache hit: lift the cached CIDs into the bare list.
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
|
||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!metadata[hash] && indexLine[hash]) {
|
||||
metadata[hash] = indexLine[hash];
|
||||
if (indexLine) {
|
||||
for (const algo of this.targetHash) {
|
||||
const cached = indexLine[algo];
|
||||
if (cached && !hasAlgorithm(cids, algo)) {
|
||||
addCid(cids, cached);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute whatever is still missing (a no-op on a full cache hit).
|
||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
||||
|
||||
// Write back: project the bare list onto the index's per-algorithm columns.
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of this.targetHash) {
|
||||
const cid = pickCidByAlgorithm(cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
}
|
||||
|
||||
async getHashIndexManager(): Promise<HashIndexManager> {
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
|
||||
export class HashComputerWorker implements ComputeInterface {
|
||||
private fileIDComputer: FileIDComputerWorker;
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
|
||||
constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
|
||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
||||
}
|
||||
|
||||
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
|
||||
import {existsAsync} from "../utils/ExistsAsync";
|
||||
import path from "path";
|
||||
import {clearInterval} from "node:timers";
|
||||
import {CID_ALGORITHM_NAMES} from "./MultiHashData";
|
||||
import {CidAlgorithm} from "./MultiHashData";
|
||||
import {stat} from "fs/promises";
|
||||
import {Stats} from "node:fs";
|
||||
|
||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
||||
/**
|
||||
* One row of the on-disk hash-index cache.
|
||||
*
|
||||
* This is the one place a **per-algorithm column** is still the right shape, and
|
||||
* it is not the deprecated one. The index answers "have I already computed sha256
|
||||
* for this file?" on every scan, which a column makes an O(1) lookup and a bare
|
||||
* CID list would make a decode-per-member. It is also strictly local: a wipeable
|
||||
* cache under `/data/cache/hash-index/` (see
|
||||
* `docs/project-architecture/cache-persistence.md`), never a record, never on the
|
||||
* wire.
|
||||
*
|
||||
* What changed is the *naming*: columns are keyed by bare algorithm
|
||||
* (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
|
||||
* so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
|
||||
* shape and the cache shape are now separate concerns that cannot leak into each
|
||||
* other — which is the whole point of the split (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
|
||||
path: string;
|
||||
size: string;
|
||||
mtime: string;
|
||||
@@ -21,15 +38,15 @@ export class HashIndexManager {
|
||||
private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
|
||||
private intervalId: any;
|
||||
private intervalTime: number = 30000;
|
||||
private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private indexOpsInProgress: boolean = false;
|
||||
private hasChanged: boolean = false;
|
||||
private initialLoad: Promise<void>;
|
||||
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
|
||||
private filePaths: { [key in CidAlgorithm]?: string } = {};
|
||||
|
||||
constructor(private indexFolderPath: string,
|
||||
private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
hasFileInCache(filePath: string,stats:Stats): boolean {
|
||||
@@ -48,6 +65,8 @@ export class HashIndexManager {
|
||||
public async init(autosave = true) {
|
||||
if (!this.initialLoad) {
|
||||
this.initialLoad = new Promise<void>(async (resolve, reject) => {
|
||||
// Create the index folder if it doesn't exist
|
||||
await fs.mkdir(this.indexFolderPath, { recursive: true });
|
||||
const stat = await fs.stat(this.indexFolderPath);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error(`Invalid index folder path ${this.indexFolderPath}`);
|
||||
@@ -79,7 +98,7 @@ export class HashIndexManager {
|
||||
}
|
||||
|
||||
// Function to check CSV headers
|
||||
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
|
||||
private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
|
||||
const records = parseSync(csvContent, {
|
||||
bom: true,
|
||||
columns: true,
|
||||
@@ -114,7 +133,7 @@ export class HashIndexManager {
|
||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
||||
}
|
||||
|
||||
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (await existsAsync(this.filePaths[hash])) {
|
||||
// check the file size and if it did not change, do not read the file
|
||||
const stats = await fs.stat(this.filePaths[hash]);
|
||||
@@ -145,7 +164,7 @@ export class HashIndexManager {
|
||||
return Array.from(this.cache.values());
|
||||
}
|
||||
|
||||
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (!(await existsAsync(this.filePaths[hash]))) {
|
||||
return [];
|
||||
}
|
||||
@@ -270,30 +289,30 @@ export class HashIndexManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
|
||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
||||
throw new Error('Invalid parameters');
|
||||
}
|
||||
for (const hash of this.targetHash) {
|
||||
if (!hashs[hash]) {
|
||||
throw new Error(`Missing hash ${hash}`);
|
||||
}
|
||||
// Check that at least one target hash is provided
|
||||
const providedHashes = this.targetHash.filter(hash => hashs[hash]);
|
||||
if (providedHashes.length === 0) {
|
||||
throw new Error('At least one target hash must be provided');
|
||||
}
|
||||
const size = fileSize + "";
|
||||
const baseName = path.basename(filePath);
|
||||
const cacheKey = `${baseName}-${size}-${mtime}`;
|
||||
let indexLine = this.cache.get(cacheKey);
|
||||
if (!indexLine) {
|
||||
//filter only the hashes we need
|
||||
// Filter only the hashes we need (and are provided)
|
||||
let filteredHash = {};
|
||||
for (const hash of this.targetHash) {
|
||||
for (const hash of providedHashes) {
|
||||
filteredHash[hash] = hashs[hash];
|
||||
}
|
||||
const data = {path: baseName, size: size, mtime: mtime, ...filteredHash};
|
||||
this.cache.set(cacheKey, data);
|
||||
} else {
|
||||
//update the cache with the latest data
|
||||
for (const hash of this.targetHash) {
|
||||
// Update the cache with the provided hashes (don't overwrite with undefined)
|
||||
for (const hash of providedHashes) {
|
||||
indexLine[hash] = hashs[hash];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
export interface MultiHashData {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
|
||||
}
|
||||
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* according to
|
||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
||||
* Multicodec / multihash codes for every digest meta-hash can compute.
|
||||
*
|
||||
* Source of truth for these numbers. The same codes appear in the CID utility
|
||||
* RANK, which is a separate thing this file does not implement:
|
||||
* - `packages/meta-core/internal/cid/rank.go`
|
||||
* - the `cid_rank.rs` in each of meta-search and meta-share
|
||||
* - `/cid-rank-vectors.json` (the golden fixture pinning those three together)
|
||||
*
|
||||
* ⚠ This package used to vendor that fixture at `src/tests/cid-rank-vectors.json`
|
||||
* and no test ever loaded it, so every vector added cost a commit here for
|
||||
* nothing. Removed 2026-09-09. If a TypeScript rank implementation is ever
|
||||
* written, re-add the path to `scripts/check-cid-vectors.sh` *and* a test that
|
||||
* asserts it — a vendored fixture nobody reads is not a guard.
|
||||
*
|
||||
* ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
|
||||
* `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
|
||||
* the gateway wrap every digest with the **raw** codec `0x55` and put the
|
||||
* algorithm in the multihash code. The multihash code is the algorithm in both
|
||||
* conventions — so anything selecting a CID by algorithm must read the
|
||||
* **multihash code**, never the content codec. See `CidDecode.ts`.
|
||||
*
|
||||
* https://github.com/multiformats/multicodec/blob/master/table.csv
|
||||
*/
|
||||
export enum CID_ALGORITHM_CODES {
|
||||
crc32 = 0x0132,
|
||||
@@ -23,25 +29,55 @@ export enum CID_ALGORITHM_CODES {
|
||||
sha256 = 0x12,
|
||||
sha3_256 = 0x16,
|
||||
sha3_384 = 0x15,
|
||||
midhash256 = 0x1000, // Custom code for midhash256
|
||||
midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
|
||||
btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM_NAMES {
|
||||
crc32 = 'cid_crc32',
|
||||
md5 = 'cid_md5',
|
||||
sha1 = 'cid_sha1',
|
||||
sha256 = 'cid_sha2-256',
|
||||
sha3_256 = 'cid_sha3-256',
|
||||
sha3_384 = 'cid_sha3-384',
|
||||
midhash256 = 'cid_midhash256',
|
||||
/**
|
||||
* An algorithm *selector*: which digests to compute, and which column of the
|
||||
* local hash-index cache to read.
|
||||
*
|
||||
* This is deliberately **not** a metadata field name. The old
|
||||
* `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
|
||||
* duty — selector, record property name, and CSV column — and the middle duty
|
||||
* was the deprecated one: those `cid_<algo>` fields were stored on records but
|
||||
* never reverse-indexed by meta-core, leaving records unresolvable by their own
|
||||
* CID. meta-core now rejects them at the write boundary (400). See
|
||||
* METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* The triple duty also rotted on its own: the interface declared `cid_sha3_384`
|
||||
* (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
|
||||
* comment on it named the wrong multicodec. A self-describing CID cannot drift
|
||||
* from its own name that way.
|
||||
*/
|
||||
export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
|
||||
|
||||
/** Every algorithm meta-hash knows how to compute. */
|
||||
export const CID_ALGORITHMS: CidAlgorithm[] = [
|
||||
'crc32',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'sha3_256',
|
||||
'sha3_384',
|
||||
'midhash256',
|
||||
'btih_v2',
|
||||
];
|
||||
|
||||
/**
|
||||
* The output of a hash computation: **bare CIDv1 strings, nothing else**.
|
||||
*
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
|
||||
* per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
|
||||
* specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
|
||||
*
|
||||
* This is persisted onto a record as the bare-CID key-set
|
||||
* (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
|
||||
*/
|
||||
export interface MultiHashData {
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM {
|
||||
crc32 = 'crc32',
|
||||
md5 = 'md5',
|
||||
sha1 = 'sha1',
|
||||
sha256 = 'sha2-256',
|
||||
sha3_256 = 'sha3-256',
|
||||
sha3_384 = 'sha3-384',
|
||||
midhash256 = 'midhash256',
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
@@ -2,6 +2,14 @@
|
||||
export * from './file-id/FileIDComputerWorker.js';
|
||||
export * from './file-id/ShaComputeWorker.js';//worker
|
||||
export * from './file-id/FastHash.js'; // Fast-hash computation
|
||||
export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
|
||||
export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
|
||||
export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
|
||||
|
||||
// The CID model: algorithm codes, the algorithm selector, and MultiHashData
|
||||
// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
|
||||
// key-set needs them to recover a specific digest — see CidDecode.
|
||||
export * from './hash-compute/MultiHashData.js';
|
||||
|
||||
//hash-compute
|
||||
export * from './hash-compute/HashComputerIndexCache.js';
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher';
|
||||
import {HashComputerWorker, pickCidByAlgorithm} from "../lib";
|
||||
import {readFile} from 'fs/promises';
|
||||
import {createHash} from 'crypto';
|
||||
import * as process from "process";
|
||||
|
||||
process.env.WORKER_URL = "./dist/worker.js";
|
||||
|
||||
/**
|
||||
* BitTorrent v2 Info Hash Tests (BEP 52)
|
||||
*
|
||||
* Tests verify:
|
||||
* - 16KB block hashing
|
||||
* - Merkle tree construction
|
||||
* - Info hash computation
|
||||
* - CID encoding format
|
||||
* - Edge cases (empty files, partial blocks)
|
||||
*/
|
||||
describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
it('should compute info hash for small file', async () => {
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('test.txt');
|
||||
|
||||
// Read test file and feed to hasher
|
||||
const testData = await readFile('./src/tests/test.txt');
|
||||
hasher.update(testData);
|
||||
|
||||
// Get info hash
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
// Verify hash format
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32); // SHA-256 produces 32 bytes
|
||||
|
||||
// Verify it's non-zero
|
||||
const isNonZero = infoHash.some(byte => byte !== 0);
|
||||
expect(isNonZero).toBe(true);
|
||||
|
||||
console.log('Info hash (hex):', infoHash.toString('hex'));
|
||||
});
|
||||
|
||||
it('should handle empty file', () => {
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('empty.txt');
|
||||
|
||||
// Don't update with any data (empty file)
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
// Verify hash format
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32);
|
||||
|
||||
// Should produce valid hash even for empty file
|
||||
const isNonZero = infoHash.some(byte => byte !== 0);
|
||||
expect(isNonZero).toBe(true);
|
||||
|
||||
console.log('Empty file info hash (hex):', infoHash.toString('hex'));
|
||||
});
|
||||
|
||||
it('should handle file exactly one block (16KB)', () => {
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('exactly-16kb.bin');
|
||||
|
||||
// Create exactly 16KB of data
|
||||
const oneBlock = Buffer.alloc(16 * 1024);
|
||||
for (let i = 0; i < oneBlock.length; i++) {
|
||||
oneBlock[i] = i % 256;
|
||||
}
|
||||
|
||||
hasher.update(oneBlock);
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32);
|
||||
|
||||
console.log('16KB file info hash (hex):', infoHash.toString('hex'));
|
||||
});
|
||||
|
||||
it('should handle streaming updates (partial blocks)', () => {
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('streamed.bin');
|
||||
|
||||
// Simulate streaming: feed data in small chunks
|
||||
const chunkSize = 1024; // 1KB chunks
|
||||
const totalSize = 50 * 1024; // 50KB total
|
||||
const data = Buffer.alloc(totalSize);
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
data[i] = i % 256;
|
||||
}
|
||||
|
||||
// Stream in chunks
|
||||
for (let offset = 0; offset < data.length; offset += chunkSize) {
|
||||
const end = Math.min(offset + chunkSize, data.length);
|
||||
hasher.update(data.subarray(offset, end));
|
||||
}
|
||||
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32);
|
||||
|
||||
console.log('Streamed 50KB info hash (hex):', infoHash.toString('hex'));
|
||||
});
|
||||
|
||||
it('should compute consistent hash for same data', () => {
|
||||
const testData = Buffer.from('Hello BitTorrent v2!');
|
||||
|
||||
// Compute hash twice
|
||||
const hasher1 = new BtihV2Hasher();
|
||||
hasher1.setFileName('hello.txt');
|
||||
hasher1.update(testData);
|
||||
const hash1 = hasher1.digest();
|
||||
|
||||
const hasher2 = new BtihV2Hasher();
|
||||
hasher2.setFileName('hello.txt');
|
||||
hasher2.update(testData);
|
||||
const hash2 = hasher2.digest();
|
||||
|
||||
// Should produce identical hashes
|
||||
expect(hash1.equals(hash2)).toBe(true);
|
||||
});
|
||||
|
||||
it('should produce different hashes for different data', () => {
|
||||
const data1 = Buffer.from('File A content');
|
||||
const data2 = Buffer.from('File B content');
|
||||
|
||||
const hasher1 = new BtihV2Hasher();
|
||||
hasher1.setFileName('fileA.txt');
|
||||
hasher1.update(data1);
|
||||
const hash1 = hasher1.digest();
|
||||
|
||||
const hasher2 = new BtihV2Hasher();
|
||||
hasher2.setFileName('fileB.txt');
|
||||
hasher2.update(data2);
|
||||
const hash2 = hasher2.digest();
|
||||
|
||||
// Should produce different hashes
|
||||
expect(hash1.equals(hash2)).toBe(false);
|
||||
});
|
||||
|
||||
it('should produce different hashes for different filenames', () => {
|
||||
const data = Buffer.from('Same content');
|
||||
|
||||
const hasher1 = new BtihV2Hasher();
|
||||
hasher1.setFileName('nameA.txt');
|
||||
hasher1.update(data);
|
||||
const hash1 = hasher1.digest();
|
||||
|
||||
const hasher2 = new BtihV2Hasher();
|
||||
hasher2.setFileName('nameB.txt');
|
||||
hasher2.update(data);
|
||||
const hash2 = hasher2.digest();
|
||||
|
||||
// Different filenames should produce different info hashes
|
||||
expect(hash1.equals(hash2)).toBe(false);
|
||||
});
|
||||
|
||||
it('should integrate with HashComputerWorker', async () => {
|
||||
const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
// The digests land in a bare CID list; each is recovered by its
|
||||
// multicodec, not by a field name.
|
||||
const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
expect(btih).toBeDefined();
|
||||
expect(typeof btih).toBe('string');
|
||||
expect(btih!.startsWith('b')).toBe(true);
|
||||
|
||||
// Should also have SHA-256 for comparison.
|
||||
expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined();
|
||||
});
|
||||
|
||||
it('should compute piece length correctly', () => {
|
||||
// Test piece length formula: 16KB * 2^n where pieces < 2048
|
||||
const testCases = [
|
||||
{fileSize: 0, expectedMin: 16 * 1024, expectedMax: 16 * 1024}, // Empty file
|
||||
{fileSize: 1024, expectedMin: 16 * 1024, expectedMax: 16 * 1024}, // 1KB
|
||||
{fileSize: 16 * 1024, expectedMin: 16 * 1024, expectedMax: 16 * 1024}, // 16KB
|
||||
{fileSize: 1024 * 1024, expectedMin: 16 * 1024, expectedMax: 16 * 1024}, // 1MB
|
||||
{fileSize: 100 * 1024 * 1024, expectedMin: 64 * 1024, expectedMax: 128 * 1024}, // 100MB
|
||||
];
|
||||
|
||||
for (const {fileSize, expectedMin, expectedMax} of testCases) {
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('size-test.bin');
|
||||
|
||||
// Feed dummy data of specified size
|
||||
const chunkSize = 16 * 1024;
|
||||
let remaining = fileSize;
|
||||
while (remaining > 0) {
|
||||
const toWrite = Math.min(chunkSize, remaining);
|
||||
hasher.update(Buffer.alloc(toWrite));
|
||||
remaining -= toWrite;
|
||||
}
|
||||
|
||||
// Digest triggers piece length computation
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
// We can't directly access piece length from outside,
|
||||
// but we can verify the hash was computed successfully
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32);
|
||||
}
|
||||
});
|
||||
|
||||
it('should produce valid CID format with codec 0x10B7', async () => {
|
||||
const hashComputer = new HashComputerWorker(['btih_v2']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
|
||||
// The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on
|
||||
// multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7"
|
||||
// assertion this test is named for.
|
||||
|
||||
// Verify CID format
|
||||
expect(cid).toBeDefined();
|
||||
expect(typeof cid).toBe('string');
|
||||
|
||||
// CIDv1 base32 starts with 'b'
|
||||
expect(cid.startsWith('b')).toBe(true);
|
||||
|
||||
// Should be longer than minimum CID length
|
||||
expect(cid.length).toBeGreaterThan(10);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {BtihV2Hasher, MagnetLinkGenerator} from '../lib';
|
||||
import {readFile, stat} from 'fs/promises';
|
||||
|
||||
/**
|
||||
* End-to-End Magnet Link Generation Tests
|
||||
*
|
||||
* Tests the complete workflow:
|
||||
* 1. Compute BitTorrent v2 info hash from file
|
||||
* 2. Generate magnet link
|
||||
* 3. Parse and validate magnet link
|
||||
*/
|
||||
describe('Magnet Link Generation (End-to-End)', () => {
|
||||
it('should generate valid magnet link from file', async () => {
|
||||
const filePath = './src/tests/test.txt';
|
||||
const fileName = 'test.txt';
|
||||
|
||||
// Step 1: Read file and compute info hash
|
||||
const fileData = await readFile(filePath);
|
||||
const fileStats = await stat(filePath);
|
||||
const fileSize = fileStats.size;
|
||||
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName(fileName);
|
||||
hasher.update(fileData);
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
// Verify info hash computed
|
||||
expect(infoHash).toBeInstanceOf(Buffer);
|
||||
expect(infoHash.length).toBe(32);
|
||||
|
||||
console.log('File:', fileName);
|
||||
console.log('Size:', fileSize, 'bytes');
|
||||
console.log('Info hash (hex):', infoHash.toString('hex'));
|
||||
|
||||
// Note: We can't test magnet link generation end-to-end without
|
||||
// the CID encoding working properly through HashComputerWorker.
|
||||
// This would require the worker integration to be fixed.
|
||||
});
|
||||
|
||||
it('should generate magnet link from mock CID', () => {
|
||||
// Mock CID format (this is a valid CIDv1 with our custom codec 0x10B7)
|
||||
// In real usage, this would come from HashComputerWorker
|
||||
const fileName = 'test-video.mp4';
|
||||
const fileSize = 1024 * 1024 * 100; // 100MB
|
||||
|
||||
// Create a mock info hash (32 bytes of test data)
|
||||
const mockInfoHash = Buffer.from(
|
||||
'b04c198dd3148719db3e32fecc54ec41ec355d7f866a1c3f71bf980c3cde01bf',
|
||||
'hex'
|
||||
);
|
||||
|
||||
// For this test, we'll construct the magnet link manually
|
||||
// since we can't get a real CID without the worker
|
||||
const infoHashHex = mockInfoHash.toString('hex');
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHashHex}&dn=${encodeURIComponent(fileName)}&xl=${fileSize}`;
|
||||
|
||||
console.log('\nGenerated magnet link:');
|
||||
console.log(magnetLink);
|
||||
|
||||
// Verify magnet link format
|
||||
expect(magnetLink.startsWith('magnet:?')).toBe(true);
|
||||
expect(magnetLink.includes('xt=urn:btmh:1220')).toBe(true);
|
||||
expect(magnetLink.includes(infoHashHex)).toBe(true);
|
||||
expect(magnetLink.includes(`dn=${encodeURIComponent(fileName)}`)).toBe(true);
|
||||
expect(magnetLink.includes(`xl=${fileSize}`)).toBe(true);
|
||||
});
|
||||
|
||||
it('should validate magnet link format', () => {
|
||||
const validLink = 'magnet:?xt=urn:btmh:1220' + 'a'.repeat(64) + '&dn=file.mp4&xl=1024';
|
||||
const invalidLink1 = 'http://example.com';
|
||||
const invalidLink2 = 'magnet:?xt=urn:btih:' + 'a'.repeat(40); // v1 format (SHA-1)
|
||||
const invalidLink3 = 'magnet:?xt=urn:btmh:1220' + 'a'.repeat(32); // Too short
|
||||
|
||||
expect(MagnetLinkGenerator.isValid(validLink)).toBe(true);
|
||||
expect(MagnetLinkGenerator.isValid(invalidLink1)).toBe(false);
|
||||
expect(MagnetLinkGenerator.isValid(invalidLink2)).toBe(false);
|
||||
expect(MagnetLinkGenerator.isValid(invalidLink3)).toBe(false);
|
||||
});
|
||||
|
||||
it('should extract info hash from magnet link', () => {
|
||||
const infoHash = 'b04c198dd3148719db3e32fecc54ec41ec355d7f866a1c3f71bf980c3cde01bf';
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash}&dn=file.mp4&xl=1024`;
|
||||
|
||||
const extracted = MagnetLinkGenerator.extractInfoHash(magnetLink);
|
||||
expect(extracted).toBe(infoHash);
|
||||
});
|
||||
|
||||
it('should extract filename from magnet link', () => {
|
||||
const fileName = 'Big Buck Bunny.mp4';
|
||||
const infoHash = 'a'.repeat(64);
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash}&dn=${encodeURIComponent(fileName)}&xl=1024`;
|
||||
|
||||
const extracted = MagnetLinkGenerator.extractFileName(magnetLink);
|
||||
expect(extracted).toBe(fileName);
|
||||
});
|
||||
|
||||
it('should extract file size from magnet link', () => {
|
||||
const fileSize = 1024000;
|
||||
const infoHash = 'a'.repeat(64);
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash}&dn=file.mp4&xl=${fileSize}`;
|
||||
|
||||
const extracted = MagnetLinkGenerator.extractFileSize(magnetLink);
|
||||
expect(extracted).toBe(fileSize);
|
||||
});
|
||||
|
||||
it('should parse complete magnet link', () => {
|
||||
const infoHash = 'b04c198dd3148719db3e32fecc54ec41ec355d7f866a1c3f71bf980c3cde01bf';
|
||||
const fileName = 'test-video.mp4';
|
||||
const fileSize = 1024000;
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash}&dn=${encodeURIComponent(fileName)}&xl=${fileSize}`;
|
||||
|
||||
const parsed = MagnetLinkGenerator.parse(magnetLink);
|
||||
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed!.infoHash).toBe(infoHash);
|
||||
expect(parsed!.fileName).toBe(fileName);
|
||||
expect(parsed!.fileSize).toBe(fileSize);
|
||||
});
|
||||
|
||||
it('should handle special characters in filename', () => {
|
||||
const fileName = 'My Video [1080p] (2024).mp4';
|
||||
const infoHash = 'a'.repeat(64);
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash}&dn=${encodeURIComponent(fileName)}&xl=1024`;
|
||||
|
||||
const extracted = MagnetLinkGenerator.extractFileName(magnetLink);
|
||||
expect(extracted).toBe(fileName);
|
||||
|
||||
// Verify round-trip
|
||||
const parsed = MagnetLinkGenerator.parse(magnetLink);
|
||||
expect(parsed?.fileName).toBe(fileName);
|
||||
});
|
||||
|
||||
it('should demonstrate complete workflow (conceptual)', async () => {
|
||||
// This test demonstrates the intended workflow once worker integration is fixed
|
||||
const filePath = './src/tests/test.txt';
|
||||
const fileStats = await stat(filePath);
|
||||
|
||||
console.log('\n=== Complete Workflow Demo ===');
|
||||
console.log('1. File:', filePath);
|
||||
console.log('2. Size:', fileStats.size, 'bytes');
|
||||
|
||||
// Step 1: Compute info hash
|
||||
const fileData = await readFile(filePath);
|
||||
const hasher = new BtihV2Hasher();
|
||||
hasher.setFileName('test.txt');
|
||||
hasher.update(fileData);
|
||||
const infoHash = hasher.digest();
|
||||
|
||||
console.log('3. Info hash:', infoHash.toString('hex'));
|
||||
|
||||
// Step 2: In production, this would be the CID from metadata
|
||||
// const cid = metadata.cid_btih_v2;
|
||||
// const magnetLink = MagnetLinkGenerator.generate(metadata, 'test.txt', fileStats.size);
|
||||
|
||||
// Step 3: For demo, construct magnet link manually
|
||||
const magnetLink = `magnet:?xt=urn:btmh:1220${infoHash.toString('hex')}&dn=test.txt&xl=${fileStats.size}`;
|
||||
console.log('4. Magnet link:', magnetLink);
|
||||
|
||||
// Step 4: Verify it's valid
|
||||
const isValid = MagnetLinkGenerator.isValid(magnetLink);
|
||||
console.log('5. Valid:', isValid);
|
||||
|
||||
expect(isValid).toBe(true);
|
||||
|
||||
// Step 5: Parse it back
|
||||
const parsed = MagnetLinkGenerator.parse(magnetLink);
|
||||
console.log('6. Parsed:', parsed);
|
||||
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed!.infoHash).toBe(infoHash.toString('hex'));
|
||||
expect(parsed!.fileName).toBe('test.txt');
|
||||
expect(parsed!.fileSize).toBe(fileStats.size);
|
||||
});
|
||||
});
|
||||
+95
-75
@@ -1,8 +1,13 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
|
||||
import {
|
||||
CidAlgorithm,
|
||||
HashComputerIndexCache,
|
||||
HashComputerWorker,
|
||||
pickCidByAlgorithm,
|
||||
} from "../lib";
|
||||
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import * as process from "process";
|
||||
import { mkdir, rm, rmdir } from "fs/promises";
|
||||
import {mkdir, rm, stat} from "fs/promises";
|
||||
|
||||
process.env.WORKER_URL = "./dist/worker.js";
|
||||
|
||||
@@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js";
|
||||
* Remember to build the worker each time you change the worker code.
|
||||
*/
|
||||
|
||||
const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384'];
|
||||
|
||||
/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */
|
||||
const FIXTURE_BYTES = 490;
|
||||
|
||||
/**
|
||||
* The expected digests of `src/tests/test.txt`, as bare CIDv1 strings.
|
||||
*
|
||||
* Independently reproducible — these are not "whatever the code printed":
|
||||
*
|
||||
* $ sha256sum src/tests/test.txt
|
||||
* be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec
|
||||
* $ md5sum → c8971ace120d1393b658eabb45d08c88
|
||||
* $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7
|
||||
*
|
||||
* and each CID below wraps exactly that digest under its multicodec (meta-hash's
|
||||
* encoder sets `codec == multihash code`).
|
||||
*
|
||||
* ⚠ These values were **re-pinned**. The previous ones described a 499-byte
|
||||
* CRLF copy of the fixture, from before `.gitattributes` normalised it to LF —
|
||||
* so on any LF checkout the digests could never match and this suite was red.
|
||||
* The mismatch was invisible because the worker path was also broken, so the
|
||||
* assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the
|
||||
* fixture's bytes ever shift again, the test says *that*, instead of printing
|
||||
* six inscrutable CID diffs.
|
||||
*/
|
||||
const EXPECTED: Record<string, string> = {
|
||||
crc32: 'bagzafmqcat2hnot3',
|
||||
md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea',
|
||||
sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y',
|
||||
sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q',
|
||||
sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi',
|
||||
sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq',
|
||||
};
|
||||
|
||||
/**
|
||||
* Every expected digest is present in the bare list, and each is recoverable by
|
||||
* its algorithm **through its multicodec** — there is no field name to look up.
|
||||
* That is the whole point of the `cids: string[]` shape: a CIDv1 is
|
||||
* self-describing, so the record never carries a per-algorithm key that could
|
||||
* drift from the CID it names (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
function expectAllDigests(cids: string[] | undefined) {
|
||||
expect(cids).toBeDefined();
|
||||
expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort());
|
||||
|
||||
for (const algo of ALGOS) {
|
||||
expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('add', () => {
|
||||
it('meta data compute', async () => {
|
||||
let hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
it('the fixture has the bytes these CIDs were pinned against', async () => {
|
||||
const {size} = await stat('./src/tests/test.txt');
|
||||
expect(size).toBe(FIXTURE_BYTES);
|
||||
});
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp",{recursive:true});
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
|
||||
it('meta data compute', async () => {
|
||||
const hashComputer = new HashComputerWorker(ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp", {recursive: true, force: true});
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
await (await hashComputer.getHashIndexManager()).saveCacheToFile();
|
||||
});
|
||||
|
||||
it('cache', async () => {
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
|
||||
// The on-disk index is the one place per-algorithm columns survive — it
|
||||
// answers "already computed?" in O(1) and never leaves this machine. Note
|
||||
// the columns are bare algorithm names now (`sha256`), not the removed
|
||||
// record-field names (`cid_sha2-256`).
|
||||
const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
delete indexLine['mtime'];
|
||||
expect(indexLine).toEqual({
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
|
||||
"path": "test.txt",
|
||||
"size": "499",
|
||||
...EXPECTED,
|
||||
path: "test.txt",
|
||||
size: String(FIXTURE_BYTES),
|
||||
});
|
||||
});
|
||||
|
||||
it('a second pass reuses the cache and recomputes nothing', async () => {
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
// Same digests, recovered from the index rather than by re-hashing.
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
});
|
||||
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_crc32
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,bagzafmqcathsbnrk
|
||||
|
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_md5
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa
|
||||
|
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_sha1
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra
|
||||
|
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_sha2-256
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua
|
||||
|
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_sha3-256
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq
|
||||
|
@@ -1,2 +0,0 @@
|
||||
path,size,mtime,cid_sha3-384
|
||||
test.txt,499,2024-04-02T16:35:33.190Z,baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,crc32
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,bagzafmqcat2hnot3
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,md5
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,bahkqdvibcdejogwocigrhe5wldvlwroqrsea
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,sha1
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,sha256
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,sha3_256
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi
|
||||
|
@@ -0,0 +1,2 @@
|
||||
path,size,mtime,sha3_384
|
||||
test.txt,490,2025-12-31T11:56:56.014Z,baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq
|
||||
|
@@ -1,6 +1,12 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
import path from 'path';
|
||||
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
alias: {
|
||||
'@root': path.resolve(__dirname, './src/lib')
|
||||
}
|
||||
},
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
|
||||
Reference in New Issue
Block a user