fix cids
This commit is contained in:
Worph
@@ -0,0 +1,6 @@
|
||||
# FIX CRLF always when developer has not set
|
||||
# Linux/Mac: git config --global core.autocrlf input
|
||||
# Windows: git config --global core.autocrlf true
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
* eol=lf
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"spec": "dist/api/api.test.js",
|
||||
"spec": ["dist/api/api.test.js", "dist/kv/*.test.js"],
|
||||
"timeout": 30000,
|
||||
"exit": true
|
||||
}
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm} from "@metazla/meta-hash";
|
||||
|
||||
// Target hashes for Piscina worker (full-file hash computation)
|
||||
// Note: midhash256 is NOT included here because it's computed in the main thread
|
||||
// during light processing, not by the Piscina worker
|
||||
export const targetHash = [
|
||||
CID_ALGORITHM_NAMES.sha1, // For torrent verification
|
||||
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
|
||||
CID_ALGORITHM_NAMES.md5, // For compatibility
|
||||
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.crc32, // Fast checksum
|
||||
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
|
||||
//
|
||||
// These are algorithm *selectors*, not metadata field names. The old
|
||||
// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
|
||||
// removed `cid_<algo>` record fields kept getting minted in the first place.
|
||||
// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
|
||||
// See METADATA_KEYS.md §2/§14.13.
|
||||
export const targetHash: CidAlgorithm[] = [
|
||||
'sha1', // For torrent verification
|
||||
'sha256', // For IPFS compatibility
|
||||
'md5', // For compatibility
|
||||
'sha3_256', // Additional cryptographic hash
|
||||
'sha3_384', // Additional cryptographic hash
|
||||
'crc32', // Fast checksum
|
||||
'btih_v2', // BitTorrent v2 info hash (BEP 52)
|
||||
]
|
||||
|
||||
// Target hashes for index-cid cache (includes all hashes we want to cache)
|
||||
// This includes midhash256 which is computed separately from the worker hashes
|
||||
export const targetHashForIndex = [
|
||||
export const targetHashForIndex: CidAlgorithm[] = [
|
||||
...targetHash,
|
||||
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
'midhash256', // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
* Reads and stream consumption stay on Redis in PR B; PR C migrates those.
|
||||
*/
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
buildFilePrefix,
|
||||
} from './MetadataUtils.js';
|
||||
|
||||
@@ -48,30 +48,12 @@ export class MetaCoreApiWriter {
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
// Reuse the same flattening logic the Redis path used so the wire
|
||||
// payload matches what handler.MergeMetadataFlat expects (flat map
|
||||
// of "a/b/c" → string value).
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
if (pairs.length === 0) return;
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
// pair.key is "file:<hash>/<field>"; strip the prefix+slash.
|
||||
const field = pair.key.slice(prefix.length + 1);
|
||||
if (!field) continue;
|
||||
// Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
|
||||
// "true"), never as per-algorithm cid_* fields. Rewrite any
|
||||
// legacy cid_* field (e.g. the in-memory cid_midhash256 primary
|
||||
// id) into the key-set form so no deprecated field ever reaches a
|
||||
// record. The CID is self-describing, so the algorithm in the
|
||||
// field name is dropped. See METADATA_KEYS.md §2/§14.13.
|
||||
if (field.startsWith('cid_') && pair.value) {
|
||||
flat[`cids/${pair.value}`] = 'true';
|
||||
} else {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
// `buildRecordFields` is the single write-shape boundary, shared with the
|
||||
// direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
|
||||
// never a per-algorithm `cid_*` field — which meta-core would now reject
|
||||
// with a 400 anyway.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { buildRecordFields } from './MetadataUtils.js';
|
||||
|
||||
/**
|
||||
* `buildRecordFields` is the single boundary where in-memory metadata becomes a
|
||||
* record. It is worth testing precisely because the bug it fixes had no coverage:
|
||||
* meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
|
||||
* fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
|
||||
* The other wrote a literal `cid_midhash256` field, which meta-core stores but
|
||||
* never reverse-indexes — so those records were unresolvable by their own CID,
|
||||
* silently. Both paths now go through this function.
|
||||
*/
|
||||
describe('buildRecordFields', () => {
|
||||
const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
|
||||
const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
|
||||
|
||||
it('turns cids[] into the bare-CID key-set, not array indices', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH, SHA256],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
assert.equal(flat[`cids/${MIDHASH}`], 'true');
|
||||
assert.equal(flat[`cids/${SHA256}`], 'true');
|
||||
|
||||
// The trap: the generic flattener would emit `cids/0`, `cids/1` for an
|
||||
// array. meta-core's reverse-index hook only matches `cids/<cid>`, so
|
||||
// those would be stored and never indexed.
|
||||
assert.equal(flat['cids/0'], undefined);
|
||||
assert.equal(flat['cids/1'], undefined);
|
||||
});
|
||||
|
||||
it('never emits a deprecated cid_* or midhash256 field', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
// meta-core returns 400 for any of these now — a record that carried one
|
||||
// was stored but unindexed.
|
||||
for (const field of Object.keys(flat)) {
|
||||
assert.ok(
|
||||
!field.startsWith('cid_'),
|
||||
`deprecated per-algorithm field reached the record: ${field}`
|
||||
);
|
||||
assert.notEqual(field, 'midhash256');
|
||||
assert.notEqual(field, 'canonical_cid');
|
||||
}
|
||||
});
|
||||
|
||||
it('does not store hashId as a property of its own record', () => {
|
||||
const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
|
||||
// hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
|
||||
assert.equal(flat['hashId'], undefined);
|
||||
});
|
||||
|
||||
it('still flattens ordinary nested metadata', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
|
||||
});
|
||||
|
||||
assert.equal(flat['title'], 'Inception');
|
||||
assert.equal(flat['fileinfo/duration'], '8878.5');
|
||||
assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
|
||||
});
|
||||
|
||||
it('honours excludeFields', () => {
|
||||
const flat = buildRecordFields(
|
||||
{hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
|
||||
['processingStatus']
|
||||
);
|
||||
assert.equal(flat['processingStatus'], undefined);
|
||||
assert.equal(flat['title'], 'x');
|
||||
});
|
||||
|
||||
it('tolerates a record with no cids at all', () => {
|
||||
const flat = buildRecordFields({title: 'x'});
|
||||
assert.equal(flat['title'], 'x');
|
||||
assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
|
||||
* // { key: "/file/abc123/video/codec", value: "h265" }
|
||||
* // ]
|
||||
*/
|
||||
/**
|
||||
* Fields that never go onto a record under their own name.
|
||||
*
|
||||
* `cids` is an **array of bare CIDs** and must become the key-set
|
||||
* `cids/<cid> = "true"`. If it fell through the generic flattener below it would
|
||||
* emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
|
||||
* reverse-index hook would not recognise, silently leaving the record
|
||||
* unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
|
||||
*
|
||||
* `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
|
||||
* storing it as a property inside itself is redundant.
|
||||
*/
|
||||
const STRUCTURAL_FIELDS = ['cids', 'hashId'];
|
||||
|
||||
/**
|
||||
* Turn an in-memory `FileMetadata` into the flat `field → value` map that a
|
||||
* record is made of.
|
||||
*
|
||||
* **This is the single write-shape boundary.** Both writers go through it — the
|
||||
* meta-core HTTP writer and the direct-Redis fallback — because they used not
|
||||
* to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
|
||||
* with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
|
||||
* meta-core never indexed. Records looked fine and could not be found by their
|
||||
* own midhash. One function, one shape, no second path to forget.
|
||||
*
|
||||
* Returns field names relative to the record root (no `/file/{hashId}` prefix).
|
||||
*/
|
||||
export function buildRecordFields(
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Record<string, string> {
|
||||
const prefix = 'r'; // throwaway root; stripped below
|
||||
const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
|
||||
if (field) {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
|
||||
// Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
|
||||
// The midhash is just the member whose multicodec is 0x1000 — there is no
|
||||
// `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
|
||||
const cids: unknown = metadata?.cids;
|
||||
if (Array.isArray(cids)) {
|
||||
for (const cid of cids) {
|
||||
if (typeof cid === 'string' && cid) {
|
||||
flat[`cids/${cid}`] = 'true';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return flat;
|
||||
}
|
||||
|
||||
export function flattenMetadata(
|
||||
metadata: any,
|
||||
prefix: string,
|
||||
|
||||
@@ -16,7 +16,7 @@ import * as IORedis from 'ioredis';
|
||||
import * as os from 'os';
|
||||
import type { IKVClient, KeyValuePair } from './IKVClient.js';
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
reconstructMetadata,
|
||||
buildFilePrefix,
|
||||
buildPropertyKey,
|
||||
@@ -331,24 +331,24 @@ export class RedisKVClient implements IKVClient {
|
||||
return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
|
||||
}
|
||||
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
|
||||
if (pairs.length === 0) return;
|
||||
// Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
|
||||
//
|
||||
// This path used to flatten the metadata itself, which meant it skipped
|
||||
// the `cid_* → cids/` transform that only the HTTP writer applied: it
|
||||
// wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
|
||||
// reverse-index hook only registers aliases for fields named `cids/<cid>`,
|
||||
// so those records were stored but never indexed — unresolvable by their
|
||||
// own midhash, with nothing logged. Both paths now build the record
|
||||
// through the same `buildRecordFields`, so the key-set cannot be skipped.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
// Use pipeline for efficient batch SET operations
|
||||
const pipeline = this.redis.pipeline();
|
||||
|
||||
for (const pair of pairs) {
|
||||
// Extract field name by removing the prefix
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
|
||||
if (field) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
const redisKey = this.buildKey(`file:${hashId}/${field}`);
|
||||
pipeline.set(redisKey, pair.value);
|
||||
}
|
||||
for (const [field, value] of Object.entries(flat)) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
|
||||
}
|
||||
|
||||
await pipeline.exec();
|
||||
|
||||
// Maintain index set for getAllHashIds
|
||||
|
||||
@@ -282,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
);
|
||||
|
||||
let midHash256: string;
|
||||
if (indexLine?.cid_midhash256) {
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
if (indexLine?.midhash256) {
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
const computeStart = performance.now();
|
||||
@@ -297,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
return;
|
||||
}
|
||||
midHash256 = hash;
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
@@ -305,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
filePath,
|
||||
fileStats.size,
|
||||
fileStats.mtime.toISOString(),
|
||||
{ cid_midhash256: midHash256 }
|
||||
{ midhash256: midHash256 }
|
||||
);
|
||||
}
|
||||
|
||||
// STEP 2: Initialize metadata with midhash256
|
||||
// STEP 2: Initialize metadata with midhash256.
|
||||
//
|
||||
// The midhash plays two roles and now says so: it is the record
|
||||
// address (`hashId`, the `/file/{hashId}` key) *and* one member of
|
||||
// the record's bare-CID key-set (`cids`). It used to be a single
|
||||
// `cid_midhash256` property, which conflated the two and leaked a
|
||||
// deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: any = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
filePath,
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
@@ -441,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
const metadata = db.get(filePath);
|
||||
|
||||
// Collect hashId before deletion (for Stremio notification)
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
removedHashIds.push(metadata.cid_midhash256);
|
||||
if (metadata && metadata.hashId) {
|
||||
removedHashIds.push(metadata.hashId);
|
||||
}
|
||||
|
||||
db.delete(filePath);
|
||||
|
||||
@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
let midHash256: string | undefined;
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
|
||||
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -409,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Check cache or compute locally
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 locally (requires file access)
|
||||
@@ -423,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -524,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
throw new Error('Metadata not found - light processing may have failed');
|
||||
}
|
||||
|
||||
const midHash256 = metadata.cid_midhash256;
|
||||
const midHash256 = metadata.hashId;
|
||||
if (!midHash256) {
|
||||
throw new Error('midhash256 not found in metadata');
|
||||
}
|
||||
@@ -622,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Delete from KV if configured (using nested key architecture)
|
||||
if (this.kvClient && metadata) {
|
||||
try {
|
||||
const hashId = metadata.cid_midhash256;
|
||||
const hashId = metadata.hashId;
|
||||
|
||||
// Delete all properties for this file using nested key architecture
|
||||
if (hashId) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
|
||||
import {HashIndexManager} from "@metazla/meta-hash";
|
||||
import {config} from "../../config/EnvConfig.js";
|
||||
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
|
||||
@@ -16,19 +16,34 @@ class SimpleFileToCid{
|
||||
if (!stats || !stats.exists) {
|
||||
return null;
|
||||
}
|
||||
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
return indexLine[CID_ALGORITHM_NAMES.sha256];
|
||||
}else {
|
||||
let hashs = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashs);
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
|
||||
return hashs[CID_ALGORITHM_NAMES.sha256];
|
||||
const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if (indexLine) {
|
||||
// The on-disk index keeps per-algorithm columns, keyed by bare
|
||||
// algorithm name (not the removed `cid_<algo>` record fields).
|
||||
return indexLine['sha256'] ?? null;
|
||||
}
|
||||
|
||||
// The computer returns bare CIDs; the algorithm is recovered from the
|
||||
// multicodec, never from a field name.
|
||||
const hashes: MultiHashData = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashes);
|
||||
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of targetHash) {
|
||||
const cid = pickCidByAlgorithm(hashes.cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
|
||||
return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
|
||||
@@ -166,14 +166,14 @@ export class StreamingPipeline {
|
||||
|
||||
// After light processing, dispatch to plugins and queue background tasks
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
if (metadata && metadata.hashId) {
|
||||
// Notify Stremio addon to add video
|
||||
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
|
||||
this.notifyStremioAdd(metadata.hashId).catch(err => {
|
||||
// Silent fail - Stremio addon may not be running
|
||||
});
|
||||
|
||||
// Dispatch tasks to container plugins (fire and forget)
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
|
||||
|
||||
// Stage 3: Queue for background processing (SHA-256 computation)
|
||||
this.backgroundQueue.add(() => this.processHashPhase(filePath))
|
||||
@@ -246,7 +246,7 @@ export class StreamingPipeline {
|
||||
// Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
|
||||
if (!this.config.containerPluginScheduler) {
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
const hashId = metadata?.cid_midhash256;
|
||||
const hashId = metadata?.hashId;
|
||||
let virtualPath: string | undefined;
|
||||
try {
|
||||
virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;
|
||||
|
||||
@@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta {
|
||||
*/
|
||||
export interface IndexManager {
|
||||
/**
|
||||
* Get cached CID for a file based on path, size, and mtime
|
||||
* Get cached CID for a file based on path, size, and mtime.
|
||||
*
|
||||
* Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
|
||||
* by bare **algorithm** (`midhash256`, `sha256`), not by the removed
|
||||
* `cid_<algo>` record-field names — the local index and the record shape are
|
||||
* deliberately separate concerns now. See METADATA_KEYS.md §14.13.
|
||||
*/
|
||||
getCidForFile(
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string
|
||||
): { cid_midhash256?: string } | undefined;
|
||||
): { midhash256?: string } | undefined;
|
||||
|
||||
/**
|
||||
* Add a file CID to the cache
|
||||
@@ -117,7 +122,7 @@ export interface IndexManager {
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string,
|
||||
cid: { cid_midhash256: string }
|
||||
cid: { midhash256: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,12 +20,26 @@ export interface FileMetadata extends HashMeta {
|
||||
processingStatus?: ProcessingStatus;
|
||||
|
||||
/**
|
||||
* In-memory primary identifier: the file's midhash256 CID, computed in the
|
||||
* light phase and used as the record address (hashId) throughout meta-sort.
|
||||
* This is an in-process working field only — it is persisted to the record
|
||||
* as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
|
||||
* writer's cid_* → cids/ transform), never as a named `cid_midhash256`
|
||||
* field. See METADATA_KEYS.md §1/§14.13.
|
||||
* The record address: the file's midhash256 CID, computed in the light phase
|
||||
* and used as the `hashId` throughout meta-sort and as the key of the
|
||||
* `/file/{hashId}` record.
|
||||
*
|
||||
* This replaces the old `cid_midhash256` field. The rename is the point: that
|
||||
* name was a *metadata property* name, and it leaked — a `cid_*` field written
|
||||
* to a record is stored but never reverse-indexed by meta-core, leaving the
|
||||
* record unresolvable by its own CID. meta-core now rejects such fields with a
|
||||
* 400. See METADATA_KEYS.md §1/§14.13.
|
||||
*/
|
||||
cid_midhash256?: string;
|
||||
hashId?: string;
|
||||
|
||||
/**
|
||||
* Every digest known for this file, as **bare CIDv1 strings** — the midhash
|
||||
* included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
|
||||
* shape meta-core indexes.
|
||||
*
|
||||
* A CIDv1 is self-describing, so a specific digest is recovered by decoding
|
||||
* the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
|
||||
* field name.
|
||||
*/
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
@@ -375,16 +375,32 @@ const CID_LABELS: Record<number, string> = {
|
||||
0xd5: 'MD5',
|
||||
0x132: 'CRC32',
|
||||
0x1000: 'MidHash256',
|
||||
0x10b7: 'BitTorrent v2',
|
||||
0xb702: 'BT Pieces Root',
|
||||
0x70: 'IPFS (dag-pb)',
|
||||
0x10b7: 'BitTorrent v2',
|
||||
0x1001: 'BitTorrent v1 (file)',
|
||||
0x1002: 'BitTorrent v2 (file)',
|
||||
0x1005: 'Usenet release',
|
||||
0x1006: 'URL',
|
||||
};
|
||||
|
||||
/**
|
||||
* Display order for a record's CIDs. This mirrors the canonical rank ladder
|
||||
* (`packages/meta-core/internal/cid/rank.go`, the two `cid_rank.rs`, and the
|
||||
* `/cid-rank-vectors.json` golden fixture) — same ordering, scaled down.
|
||||
*
|
||||
* It used to be its own invented ladder (5/4/3/2) with no dag-pb entry at all,
|
||||
* which meant an IPFS root fell through to 0 and sorted *below* MD5. Display-only,
|
||||
* so it never broke anything — but it was a seventh, silently divergent copy of a
|
||||
* rule that already has too many. Keep it in step: a code with no entry sorts last.
|
||||
*/
|
||||
const RANK_BY_LABEL: Record<string, number> = {
|
||||
'IPFS (dag-pb)': 6,
|
||||
'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5,
|
||||
'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4,
|
||||
MidHash256: 3, 'SHA-1': 2, MD5: 2, CRC32: 2,
|
||||
MidHash256: 3,
|
||||
'Usenet release': 2, URL: 2,
|
||||
'SHA-1': 1, MD5: 1, CRC32: 1,
|
||||
};
|
||||
|
||||
const B32 = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
|
||||
|
||||
/**
|
||||
* Decoding a bare CIDv1 back into the algorithm that produced it.
|
||||
*
|
||||
* This is **the** TypeScript CID decoder for the platform. It exists because a
|
||||
* bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
|
||||
* "the sha2-256 of this file" has to ask the multicodec, not a string.
|
||||
*
|
||||
* Before this module, meta-dup carried a private hand-rolled base32+uvarint
|
||||
* decoder, the meta-sort editor carried a second one with a *different* rank
|
||||
* ladder, and meta-fuse had none at all (so it could only read the legacy
|
||||
* `cid_midhash256` field, and silently failed on any record that didn't have
|
||||
* one). One implementation, exported, replaces all three.
|
||||
*
|
||||
* ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
|
||||
* has two encoders that disagree about the codec: meta-hash writes
|
||||
* `codec == algorithm code`, while the fullhash plugin and the gateway wrap
|
||||
* every digest with the raw codec `0x55`. The multihash code is the algorithm in
|
||||
* both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
|
||||
*/
|
||||
|
||||
/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidMultihashCode(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).multihash.code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The content codec of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidCodec(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first member of `cids` whose multihash code is `mhCode`.
|
||||
*
|
||||
* Takes the raw code so callers can ask for digests meta-hash doesn't itself
|
||||
* compute (e.g. the gateway's per-file btih codes).
|
||||
*/
|
||||
export function pickCidByMulticodec(
|
||||
cids: readonly string[] | undefined,
|
||||
mhCode: number
|
||||
): string | undefined {
|
||||
return cids?.find(cid => cidMultihashCode(cid) === mhCode);
|
||||
}
|
||||
|
||||
/** The member of `cids` produced by `algo`, if present. */
|
||||
export function pickCidByAlgorithm(
|
||||
cids: readonly string[] | undefined,
|
||||
algo: CidAlgorithm
|
||||
): string | undefined {
|
||||
return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Has `algo` already been computed for this cid set?
|
||||
*
|
||||
* The "skip if already computed" check. It used to be an O(1) property lookup
|
||||
* (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
|
||||
* That is a handful of microseconds against a hash computation that streams the
|
||||
* whole file — the trade is not close.
|
||||
*/
|
||||
export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
|
||||
return pickCidByAlgorithm(cids, algo) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `cid` to `cids` unless an equal string is already there.
|
||||
*
|
||||
* Dedup is on the exact string. Two encoders can spell the same digest
|
||||
* differently (codec 0x55 vs codec == mh), and both spellings are legitimate
|
||||
* members — meta-core's reverse index maps each to the same record.
|
||||
*/
|
||||
export function addCid(cids: string[], cid: string): void {
|
||||
if (cid && !cids.includes(cid)) {
|
||||
cids.push(cid);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export const codeTable = {
|
||||
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
|
||||
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
|
||||
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
|
||||
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
|
||||
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
|
||||
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
|
||||
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
|
||||
[CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {create} from "multiformats/hashes/digest";
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {codeTable} from "@root/file-id/CodeTable";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param stream The Readable stream of the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @returns Array of bare CIDv1 strings, in the order of `algorithms`
|
||||
*/
|
||||
export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
export async function computeCIDs({stream, algorithms, createHasher}: {
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
algorithms: CID_ALGORITHM_NAMES[];
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
|
||||
algorithms: CidAlgorithm[];
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
|
||||
}): Promise<string[]> {
|
||||
const hashers = await hasherDefiner(algorithms,createHasher);
|
||||
const hashers = await hasherDefiner(algorithms, createHasher);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const {done, value} = await reader.read();
|
||||
@@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
return cidFinalize(hashers);
|
||||
}
|
||||
|
||||
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
|
||||
async function hasherDefiner(algorithms: CidAlgorithm[],
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
|
||||
hasher: SimpleHash,
|
||||
code: CID_ALGORITHM_CODES
|
||||
}[]> {
|
||||
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
|
||||
const hashers = algorithms
|
||||
.filter(algo => algo in CID_ALGORITHM_CODES)
|
||||
.map(async algo => ({
|
||||
hasher: await createHasher(algo),
|
||||
code: codeTable[algo]
|
||||
code: CID_ALGORITHM_CODES[algo]
|
||||
}));
|
||||
return await Promise.all(hashers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
|
||||
* plugin and the gateway instead use the raw codec (0x55) and carry the
|
||||
* algorithm only in the multihash. Both are valid CIDs for the same digest, and
|
||||
* both must decode to the same algorithm — which is why every consumer selects
|
||||
* on the **multihash code**, never the codec. See `CidDecode.ts`.
|
||||
*/
|
||||
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
|
||||
return await Promise.all(hashers.map(async ({code, hasher}) => {
|
||||
const hashBuffer = await hasher.digest();
|
||||
const digest = create(code, hashBuffer);
|
||||
return CID.createV1(code, digest).toString();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +1,32 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {createHash} from "crypto";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BtihV2Hasher} from "./BtihV2Hasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @param algo
|
||||
* Build the stream hasher for one algorithm.
|
||||
*
|
||||
* `midhash256` is absent on purpose: it is not a stream digest (it hashes the
|
||||
* file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
|
||||
*/
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return createHash("sha256");
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return createHash("sha1");
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return createHash("md5");
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return createHash("sha3-256");
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return createHash("sha3-384");
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
case CID_ALGORITHM_NAMES.btih_v2:
|
||||
case 'btih_v2':
|
||||
return new BtihV2Hasher();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BrowserHasher} from "@root/file-id/BrowserHasher";
|
||||
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
/**
|
||||
* Browser build. Note it supports fewer algorithms than the node one: there is
|
||||
* no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
|
||||
* either throws.
|
||||
*/
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
const hasher = (() => {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return new BrowserHasher('sha-256');
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return new BrowserHasher('sha-1');
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return new BrowserHasher('md5');
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return new BrowserHasher('sha3-256');
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return new BrowserHasher('sha3-384');
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
@@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHas
|
||||
}
|
||||
|
||||
return hasher;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export interface FileIDComputerInterface{
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
|
||||
}
|
||||
@@ -1,10 +1,16 @@
|
||||
import {Piscina} from "piscina";
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
|
||||
export class FileIDComputerWorker{
|
||||
private piscina: Piscina;
|
||||
|
||||
constructor(workerPath?:string) {
|
||||
// `WORKER_URL` is the escape hatch the tests have always set — it just
|
||||
// wasn't read. Honour it before falling back to path derivation, which
|
||||
// only works for a caller running from `dist/`.
|
||||
if (!workerPath && process.env.WORKER_URL) {
|
||||
workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
|
||||
}
|
||||
if(!workerPath) {
|
||||
// Construct the URL for the current module
|
||||
let distFolder = import.meta.dirname;
|
||||
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
|
||||
public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
|
||||
return this.piscina.run({filePath, algorithms});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {MultiHashData} from '../hash-compute/MultiHashData';
|
||||
import {pickCidByAlgorithm} from './CidDecode';
|
||||
import {CID} from 'multiformats/cid';
|
||||
|
||||
/**
|
||||
@@ -22,13 +23,13 @@ export class MagnetLinkGenerator {
|
||||
/**
|
||||
* Generate BitTorrent v2 magnet link from metadata
|
||||
*
|
||||
* @param metadata - File metadata containing cid_btih_v2
|
||||
* @param metadata - File metadata whose `cids` include a btih-v2 CID
|
||||
* @param fileName - Original filename for display name
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if btih_v2 not available
|
||||
*
|
||||
* @example
|
||||
* const metadata = {cid_btih_v2: 'bafyb...'};
|
||||
* const metadata = {cids: ['bafyb...']};
|
||||
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
|
||||
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
|
||||
*/
|
||||
@@ -37,7 +38,9 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
const cidString = metadata.cid_btih_v2;
|
||||
// The btih-v2 CID is recovered from the bare list by its multicodec
|
||||
// (0x10B7), not by a field name. See CidDecode.ts.
|
||||
const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
if (!cidString) {
|
||||
return null; // BitTorrent v2 hash not computed
|
||||
}
|
||||
@@ -178,6 +181,6 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
|
||||
return this.generate({cids: [cidString]}, fileName, fileSize);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
import fs from "fs";
|
||||
import {Readable} from 'stream';
|
||||
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
|
||||
@@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher";
|
||||
*/
|
||||
export default async function computeCIDs({filePath, algorithms}: {
|
||||
filePath: string;
|
||||
algorithms: CID_ALGORITHM_NAMES[]
|
||||
algorithms: CidAlgorithm[]
|
||||
}): Promise<string[]> {
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
return computeCIDsInternal({stream, algorithms, createHasher});
|
||||
|
||||
@@ -1,30 +1,29 @@
|
||||
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, MultiHashData} from "./MultiHashData";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class HashComputer{
|
||||
export class HashComputer {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[],
|
||||
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
|
||||
constructor(private targetHash: CidAlgorithm[],
|
||||
private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
|
||||
}
|
||||
|
||||
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
// Which digests are still missing? On a bare-CID list this costs a decode
|
||||
// per member rather than a property lookup — negligible against streaming
|
||||
// the whole file through a hasher.
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +1,28 @@
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {Readable} from "stream";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import fs from "fs";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
export class HashComputerFile implements ComputeInterface{
|
||||
export class HashComputerFile implements ComputeInterface {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
constructor(private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,59 @@
|
||||
import {HashComputerWorker} from "./HashComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {HashIndexManager} from "./HashIndexManager";
|
||||
import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {stat} from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* The translation point between the two shapes.
|
||||
*
|
||||
* `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
|
||||
* persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
|
||||
* **per-algorithm columns**, because its job is the O(1) "already computed?"
|
||||
* check. Neither shape leaks into the other; this class converts.
|
||||
*/
|
||||
export class HashComputerIndexCache implements ComputeInterface {
|
||||
private hashIndexManager: HashIndexManager;
|
||||
private hashComputer: HashComputerWorker;
|
||||
|
||||
constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash,workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
|
||||
constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
await this.hashIndexManager.init();
|
||||
let stats = await stat(filePath);
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
|
||||
const stats = await stat(filePath);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// Cache hit: lift the cached CIDs into the bare list.
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
|
||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!metadata[hash] && indexLine[hash]) {
|
||||
metadata[hash] = indexLine[hash];
|
||||
if (indexLine) {
|
||||
for (const algo of this.targetHash) {
|
||||
const cached = indexLine[algo];
|
||||
if (cached && !hasAlgorithm(cids, algo)) {
|
||||
addCid(cids, cached);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute whatever is still missing (a no-op on a full cache hit).
|
||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
||||
|
||||
// Write back: project the bare list onto the index's per-algorithm columns.
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of this.targetHash) {
|
||||
const cid = pickCidByAlgorithm(cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
}
|
||||
|
||||
async getHashIndexManager(): Promise<HashIndexManager> {
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
|
||||
export class HashComputerWorker implements ComputeInterface {
|
||||
private fileIDComputer: FileIDComputerWorker;
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
|
||||
constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
|
||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
||||
}
|
||||
|
||||
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
|
||||
import {existsAsync} from "../utils/ExistsAsync";
|
||||
import path from "path";
|
||||
import {clearInterval} from "node:timers";
|
||||
import {CID_ALGORITHM_NAMES} from "./MultiHashData";
|
||||
import {CidAlgorithm} from "./MultiHashData";
|
||||
import {stat} from "fs/promises";
|
||||
import {Stats} from "node:fs";
|
||||
|
||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
||||
/**
|
||||
* One row of the on-disk hash-index cache.
|
||||
*
|
||||
* This is the one place a **per-algorithm column** is still the right shape, and
|
||||
* it is not the deprecated one. The index answers "have I already computed sha256
|
||||
* for this file?" on every scan, which a column makes an O(1) lookup and a bare
|
||||
* CID list would make a decode-per-member. It is also strictly local: a wipeable
|
||||
* cache under `/data/cache/hash-index/` (see
|
||||
* `docs/project-architecture/cache-persistence.md`), never a record, never on the
|
||||
* wire.
|
||||
*
|
||||
* What changed is the *naming*: columns are keyed by bare algorithm
|
||||
* (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
|
||||
* so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
|
||||
* shape and the cache shape are now separate concerns that cannot leak into each
|
||||
* other — which is the whole point of the split (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
|
||||
path: string;
|
||||
size: string;
|
||||
mtime: string;
|
||||
@@ -21,15 +38,15 @@ export class HashIndexManager {
|
||||
private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
|
||||
private intervalId: any;
|
||||
private intervalTime: number = 30000;
|
||||
private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private indexOpsInProgress: boolean = false;
|
||||
private hasChanged: boolean = false;
|
||||
private initialLoad: Promise<void>;
|
||||
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
|
||||
private filePaths: { [key in CidAlgorithm]?: string } = {};
|
||||
|
||||
constructor(private indexFolderPath: string,
|
||||
private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
hasFileInCache(filePath: string,stats:Stats): boolean {
|
||||
@@ -81,7 +98,7 @@ export class HashIndexManager {
|
||||
}
|
||||
|
||||
// Function to check CSV headers
|
||||
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
|
||||
private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
|
||||
const records = parseSync(csvContent, {
|
||||
bom: true,
|
||||
columns: true,
|
||||
@@ -116,7 +133,7 @@ export class HashIndexManager {
|
||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
||||
}
|
||||
|
||||
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (await existsAsync(this.filePaths[hash])) {
|
||||
// check the file size and if it did not change, do not read the file
|
||||
const stats = await fs.stat(this.filePaths[hash]);
|
||||
@@ -147,7 +164,7 @@ export class HashIndexManager {
|
||||
return Array.from(this.cache.values());
|
||||
}
|
||||
|
||||
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (!(await existsAsync(this.filePaths[hash]))) {
|
||||
return [];
|
||||
}
|
||||
@@ -178,10 +195,8 @@ export class HashIndexManager {
|
||||
|
||||
public async saveCacheToFile(): Promise<void> {
|
||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`);
|
||||
return;
|
||||
}
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`);
|
||||
this.hasChanged = false;
|
||||
this.indexOpsInProgress = true;
|
||||
const start = performance.now();
|
||||
@@ -274,17 +289,15 @@ export class HashIndexManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
|
||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
||||
throw new Error('Invalid parameters');
|
||||
}
|
||||
// Check that at least one target hash is provided
|
||||
const providedHashes = this.targetHash.filter(hash => hashs[hash]);
|
||||
if (providedHashes.length === 0) {
|
||||
console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`);
|
||||
throw new Error('At least one target hash must be provided');
|
||||
}
|
||||
console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`);
|
||||
const size = fileSize + "";
|
||||
const baseName = path.basename(filePath);
|
||||
const cacheKey = `${baseName}-${size}-${mtime}`;
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
export interface MultiHashData {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
|
||||
cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
|
||||
}
|
||||
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* according to
|
||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
||||
* Multicodec / multihash codes for every digest meta-hash can compute.
|
||||
*
|
||||
* Source of truth for these numbers, mirrored in:
|
||||
* - `packages/meta-core/internal/cid/rank.go`
|
||||
* - the `cid_rank.rs` in each of meta-search and meta-share
|
||||
* - `/cid-rank-vectors.json` (the golden fixture that pins them all together)
|
||||
*
|
||||
* ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
|
||||
* `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
|
||||
* the gateway wrap every digest with the **raw** codec `0x55` and put the
|
||||
* algorithm in the multihash code. The multihash code is the algorithm in both
|
||||
* conventions — so anything selecting a CID by algorithm must read the
|
||||
* **multihash code**, never the content codec. See `CidDecode.ts`.
|
||||
*
|
||||
* https://github.com/multiformats/multicodec/blob/master/table.csv
|
||||
*/
|
||||
export enum CID_ALGORITHM_CODES {
|
||||
crc32 = 0x0132,
|
||||
@@ -24,28 +22,55 @@ export enum CID_ALGORITHM_CODES {
|
||||
sha256 = 0x12,
|
||||
sha3_256 = 0x16,
|
||||
sha3_384 = 0x15,
|
||||
midhash256 = 0x1000, // Custom code for midhash256
|
||||
btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
|
||||
midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
|
||||
btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM_NAMES {
|
||||
crc32 = 'cid_crc32',
|
||||
md5 = 'cid_md5',
|
||||
sha1 = 'cid_sha1',
|
||||
sha256 = 'cid_sha2-256',
|
||||
sha3_256 = 'cid_sha3-256',
|
||||
sha3_384 = 'cid_sha3-384',
|
||||
midhash256 = 'cid_midhash256',
|
||||
btih_v2 = 'cid_btih_v2',
|
||||
/**
|
||||
* An algorithm *selector*: which digests to compute, and which column of the
|
||||
* local hash-index cache to read.
|
||||
*
|
||||
* This is deliberately **not** a metadata field name. The old
|
||||
* `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
|
||||
* duty — selector, record property name, and CSV column — and the middle duty
|
||||
* was the deprecated one: those `cid_<algo>` fields were stored on records but
|
||||
* never reverse-indexed by meta-core, leaving records unresolvable by their own
|
||||
* CID. meta-core now rejects them at the write boundary (400). See
|
||||
* METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* The triple duty also rotted on its own: the interface declared `cid_sha3_384`
|
||||
* (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
|
||||
* comment on it named the wrong multicodec. A self-describing CID cannot drift
|
||||
* from its own name that way.
|
||||
*/
|
||||
export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
|
||||
|
||||
/** Every algorithm meta-hash knows how to compute. */
|
||||
export const CID_ALGORITHMS: CidAlgorithm[] = [
|
||||
'crc32',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'sha3_256',
|
||||
'sha3_384',
|
||||
'midhash256',
|
||||
'btih_v2',
|
||||
];
|
||||
|
||||
/**
|
||||
* The output of a hash computation: **bare CIDv1 strings, nothing else**.
|
||||
*
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
|
||||
* per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
|
||||
* specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
|
||||
*
|
||||
* This is persisted onto a record as the bare-CID key-set
|
||||
* (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
|
||||
*/
|
||||
export interface MultiHashData {
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM {
|
||||
crc32 = 'crc32',
|
||||
md5 = 'md5',
|
||||
sha1 = 'sha1',
|
||||
sha256 = 'sha2-256',
|
||||
sha3_256 = 'sha3-256',
|
||||
sha3_384 = 'sha3-384',
|
||||
midhash256 = 'midhash256',
|
||||
btih_v2 = 'btih-v2',
|
||||
}
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker
|
||||
export * from './file-id/FastHash.js'; // Fast-hash computation
|
||||
export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
|
||||
export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
|
||||
export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
|
||||
|
||||
// The CID model: algorithm codes, the algorithm selector, and MultiHashData
|
||||
// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
|
||||
// key-set needs them to recover a specific digest — see CidDecode.
|
||||
export * from './hash-compute/MultiHashData.js';
|
||||
|
||||
//hash-compute
|
||||
export * from './hash-compute/HashComputerIndexCache.js';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher';
|
||||
import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib";
|
||||
import {HashComputerWorker, pickCidByAlgorithm} from "../lib";
|
||||
import {readFile} from 'fs/promises';
|
||||
import {createHash} from 'crypto';
|
||||
import * as process from "process";
|
||||
@@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should integrate with HashComputerWorker', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2,
|
||||
CID_ALGORITHM_NAMES.sha256
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
console.log('Computed metadata:', metadata);
|
||||
// The digests land in a bare CID list; each is recovered by its
|
||||
// multicodec, not by a field name.
|
||||
const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
expect(btih).toBeDefined();
|
||||
expect(typeof btih).toBe('string');
|
||||
expect(btih!.startsWith('b')).toBe(true);
|
||||
|
||||
// Verify btih_v2 CID was computed
|
||||
expect(metadata).toHaveProperty('cid_btih_v2');
|
||||
expect(typeof metadata.cid_btih_v2).toBe('string');
|
||||
expect(metadata.cid_btih_v2.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify CID format (should start with 'b' for base32 CIDv1)
|
||||
expect(metadata.cid_btih_v2.startsWith('b')).toBe(true);
|
||||
|
||||
// Should also have SHA-256 for comparison
|
||||
expect(metadata).toHaveProperty('cid_sha2-256');
|
||||
// Should also have SHA-256 for comparison.
|
||||
expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined();
|
||||
});
|
||||
|
||||
it('should compute piece length correctly', () => {
|
||||
@@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should produce valid CID format with codec 0x10B7', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
const cid = metadata.cid_btih_v2;
|
||||
console.log('BitTorrent v2 CID:', cid);
|
||||
const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
|
||||
// The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on
|
||||
// multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7"
|
||||
// assertion this test is named for.
|
||||
|
||||
// Verify CID format
|
||||
expect(cid).toBeDefined();
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"$schema": "cid-rank-vectors/v1",
|
||||
"$comment": [
|
||||
"CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
|
||||
"The rank decides which of a record's several CIDs is elected canonical - the cid a",
|
||||
"record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
|
||||
"they cannot share code (independent submodules, no cargo path-dep). These vectors are",
|
||||
"the only thing holding them in sync: every implementation's test suite loads this file.",
|
||||
"",
|
||||
"This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
|
||||
"own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
|
||||
"",
|
||||
"Ranking is by UTILITY for external retrieval, not cryptographic strength.",
|
||||
"Ties break on the lexicographically smaller cid, so the election is deterministic",
|
||||
"across peers and does not churn.",
|
||||
"",
|
||||
"See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
|
||||
],
|
||||
"ladder": {
|
||||
"40": "dag-pb (IPFS block)",
|
||||
"30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
|
||||
"20": "BitTorrent infohashes - swarm interop",
|
||||
"10": "midhash256 - the record address",
|
||||
"5": "opaque locators (nzb-release, url) - never outrank a real digest",
|
||||
"0": "weak/unknown digests, unparseable input"
|
||||
},
|
||||
"tieBreak": "lexicographically smaller cid wins",
|
||||
"vectors": [
|
||||
{
|
||||
"name": "ipfs-dagpb",
|
||||
"cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 40,
|
||||
"note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
|
||||
},
|
||||
{
|
||||
"name": "sha2-256-raw-codec",
|
||||
"cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 30,
|
||||
"note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
|
||||
},
|
||||
{
|
||||
"name": "sha2-256-codec-eq-mh",
|
||||
"cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture; meta-hash sets codec == mh"
|
||||
},
|
||||
{
|
||||
"name": "sha3-256",
|
||||
"cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture"
|
||||
},
|
||||
{
|
||||
"name": "sha3-384",
|
||||
"cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-codec-eq-mh",
|
||||
"cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "meta-hash encoder convention"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-raw-codec",
|
||||
"cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "fullhash encoder convention - SAME tier, both conventions must match"
|
||||
},
|
||||
{
|
||||
"name": "btih-v1-file",
|
||||
"cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "gateway per-file BitTorrent v1"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-file",
|
||||
"cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "gateway per-file BitTorrent v2"
|
||||
},
|
||||
{
|
||||
"name": "midhash256-codec-eq-mh",
|
||||
"cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 10,
|
||||
"note": "the record address; internally meaningful, externally invisible"
|
||||
},
|
||||
{
|
||||
"name": "midhash256-raw-codec",
|
||||
"cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 10
|
||||
},
|
||||
{
|
||||
"name": "nzb-release-locator",
|
||||
"cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
|
||||
"rank": 5,
|
||||
"note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
|
||||
},
|
||||
{
|
||||
"name": "url-locator",
|
||||
"cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
|
||||
"rank": 5,
|
||||
"note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
|
||||
},
|
||||
{
|
||||
"name": "sha1",
|
||||
"cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "md5",
|
||||
"cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "crc32",
|
||||
"cid": "bagzafmqcathsbnrk",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "bt-pieces-root",
|
||||
"cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 0,
|
||||
"note": "torrent-internal merkle root, NOT a file-identity cid"
|
||||
},
|
||||
{
|
||||
"name": "cidv0-rejected",
|
||||
"cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
|
||||
"rank": 0,
|
||||
"note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
|
||||
},
|
||||
{
|
||||
"name": "token-form-rejected",
|
||||
"cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 0,
|
||||
"note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
|
||||
},
|
||||
{
|
||||
"name": "garbage",
|
||||
"cid": "not-a-cid",
|
||||
"rank": 0
|
||||
},
|
||||
{
|
||||
"name": "empty",
|
||||
"cid": "",
|
||||
"rank": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
|
||||
import {
|
||||
CidAlgorithm,
|
||||
HashComputerIndexCache,
|
||||
HashComputerWorker,
|
||||
pickCidByAlgorithm,
|
||||
} from "../lib";
|
||||
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import * as process from "process";
|
||||
import { mkdir, rm, rmdir } from "fs/promises";
|
||||
import {mkdir, rm, stat} from "fs/promises";
|
||||
|
||||
process.env.WORKER_URL = "./dist/worker.js";
|
||||
|
||||
@@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js";
|
||||
* Remember to build the worker each time you change the worker code.
|
||||
*/
|
||||
|
||||
const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384'];
|
||||
|
||||
/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */
|
||||
const FIXTURE_BYTES = 490;
|
||||
|
||||
/**
|
||||
* The expected digests of `src/tests/test.txt`, as bare CIDv1 strings.
|
||||
*
|
||||
* Independently reproducible — these are not "whatever the code printed":
|
||||
*
|
||||
* $ sha256sum src/tests/test.txt
|
||||
* be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec
|
||||
* $ md5sum → c8971ace120d1393b658eabb45d08c88
|
||||
* $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7
|
||||
*
|
||||
* and each CID below wraps exactly that digest under its multicodec (meta-hash's
|
||||
* encoder sets `codec == multihash code`).
|
||||
*
|
||||
* ⚠ These values were **re-pinned**. The previous ones described a 499-byte
|
||||
* CRLF copy of the fixture, from before `.gitattributes` normalised it to LF —
|
||||
* so on any LF checkout the digests could never match and this suite was red.
|
||||
* The mismatch was invisible because the worker path was also broken, so the
|
||||
* assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the
|
||||
* fixture's bytes ever shift again, the test says *that*, instead of printing
|
||||
* six inscrutable CID diffs.
|
||||
*/
|
||||
const EXPECTED: Record<string, string> = {
|
||||
crc32: 'bagzafmqcat2hnot3',
|
||||
md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea',
|
||||
sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y',
|
||||
sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q',
|
||||
sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi',
|
||||
sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq',
|
||||
};
|
||||
|
||||
/**
|
||||
* Every expected digest is present in the bare list, and each is recoverable by
|
||||
* its algorithm **through its multicodec** — there is no field name to look up.
|
||||
* That is the whole point of the `cids: string[]` shape: a CIDv1 is
|
||||
* self-describing, so the record never carries a per-algorithm key that could
|
||||
* drift from the CID it names (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
function expectAllDigests(cids: string[] | undefined) {
|
||||
expect(cids).toBeDefined();
|
||||
expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort());
|
||||
|
||||
for (const algo of ALGOS) {
|
||||
expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('add', () => {
|
||||
it('meta data compute', async () => {
|
||||
let hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
it('the fixture has the bytes these CIDs were pinned against', async () => {
|
||||
const {size} = await stat('./src/tests/test.txt');
|
||||
expect(size).toBe(FIXTURE_BYTES);
|
||||
});
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp",{recursive:true});
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
|
||||
it('meta data compute', async () => {
|
||||
const hashComputer = new HashComputerWorker(ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp", {recursive: true, force: true});
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
await (await hashComputer.getHashIndexManager()).saveCacheToFile();
|
||||
});
|
||||
|
||||
it('cache', async () => {
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
|
||||
// The on-disk index is the one place per-algorithm columns survive — it
|
||||
// answers "already computed?" in O(1) and never leaves this machine. Note
|
||||
// the columns are bare algorithm names now (`sha256`), not the removed
|
||||
// record-field names (`cid_sha2-256`).
|
||||
const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
delete indexLine['mtime'];
|
||||
expect(indexLine).toEqual({
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
|
||||
"path": "test.txt",
|
||||
"size": "499",
|
||||
...EXPECTED,
|
||||
path: "test.txt",
|
||||
size: String(FIXTURE_BYTES),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('a second pass reuses the cache and recomputes nothing', async () => {
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
// Same digests, recovered from the index rather than by re-hashing.
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user