From 13fa36ffc709360db5a831d109b4ef93c7df787a Mon Sep 17 00:00:00 2001 From: Worph Date: Tue, 14 Jul 2026 16:08:20 +0200 Subject: [PATCH] fix cids --- .gitattributes | 6 + packages/meta-sort-core/.mocharc.json | 2 +- .../meta-sort-core/src/config/TargetHash.ts | 30 +-- .../src/kv/MetaCoreApiWriter.ts | 32 +--- .../src/kv/MetadataUtils.test.ts | 86 +++++++++ .../meta-sort-core/src/kv/MetadataUtils.ts | 57 ++++++ packages/meta-sort-core/src/kv/RedisClient.ts | 30 +-- .../src/logic/WatchedFileProcessor.ts | 23 ++- .../fileProcessor/FileProcessorPiscina.ts | 40 ++-- .../logic/fileProcessor/SimpleFileToCid.ts | 35 +++- .../src/logic/pipeline/StreamingPipeline.ts | 8 +- .../src/types/ExtendedInterfaces.ts | 11 +- .../meta-sort-core/src/types/FileMetadata.ts | 28 ++- .../src/components/MetadataEditor.tsx | 20 +- .../meta-hash/src/lib/file-id/CidDecode.ts | 86 +++++++++ .../meta-hash/src/lib/file-id/CodeTable.ts | 12 -- .../meta-hash/src/lib/file-id/ComputeHash.ts | 31 ++-- .../meta-hash/src/lib/file-id/CreateHasher.ts | 27 +-- .../src/lib/file-id/CreateHasherBrowser.ts | 23 ++- .../lib/file-id/FileIDComputerInterface.ts | 4 +- .../src/lib/file-id/FileIDComputerWorker.ts | 10 +- .../src/lib/file-id/MagnetLinkGenerator.ts | 11 +- .../src/lib/file-id/ShaComputeWorker.ts | 4 +- .../src/lib/hash-compute/HashComputer.ts | 27 ++- .../src/lib/hash-compute/HashComputerFile.ts | 22 +-- .../hash-compute/HashComputerIndexCache.ts | 48 +++-- .../lib/hash-compute/HashComputerWorker.ts | 20 +- .../src/lib/hash-compute/HashIndexManager.ts | 41 +++-- .../src/lib/hash-compute/MultiHashData.ts | 101 ++++++---- packages/shared/meta-hash/src/lib/index.ts | 6 + .../shared/meta-hash/src/tests/btihv2.test.ts | 37 ++-- .../meta-hash/src/tests/cid-rank-vectors.json | 153 ++++++++++++++++ .../shared/meta-hash/src/tests/main.test.ts | 172 ++++++++++-------- 33 files changed, 887 insertions(+), 356 deletions(-) create mode 100644 .gitattributes create mode 100644 packages/meta-sort-core/src/kv/MetadataUtils.test.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/CidDecode.ts delete mode 100644 packages/shared/meta-hash/src/lib/file-id/CodeTable.ts create mode 100644 packages/shared/meta-hash/src/tests/cid-rank-vectors.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/packages/meta-sort-core/.mocharc.json b/packages/meta-sort-core/.mocharc.json index 22072d8..f5eefc2 100644 --- a/packages/meta-sort-core/.mocharc.json +++ b/packages/meta-sort-core/.mocharc.json @@ -1,5 +1,5 @@ { - "spec": "dist/api/api.test.js", + "spec": ["dist/api/api.test.js", "dist/kv/*.test.js"], "timeout": 30000, "exit": true } diff --git a/packages/meta-sort-core/src/config/TargetHash.ts b/packages/meta-sort-core/src/config/TargetHash.ts index bc3785c..0679059 100644 --- a/packages/meta-sort-core/src/config/TargetHash.ts +++ b/packages/meta-sort-core/src/config/TargetHash.ts @@ -1,21 +1,27 @@ -import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash"; +import {CidAlgorithm} from "@metazla/meta-hash"; // Target hashes for Piscina worker (full-file hash computation) // Note: midhash256 is NOT included here because it's computed in the main thread // during light processing, not by the Piscina worker -export const targetHash = [ - CID_ALGORITHM_NAMES.sha1, // For torrent verification - CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility - CID_ALGORITHM_NAMES.md5, // For compatibility - CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash - CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash - CID_ALGORITHM_NAMES.crc32, // Fast checksum - CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52) +// +// These are algorithm *selectors*, not metadata field names. The old +// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the +// removed `cid_` record fields kept getting minted in the first place. +// Digests are now persisted only as bare-CID key-set members (`cids/`). +// See METADATA_KEYS.md §2/§14.13. +export const targetHash: CidAlgorithm[] = [ + 'sha1', // For torrent verification + 'sha256', // For IPFS compatibility + 'md5', // For compatibility + 'sha3_256', // Additional cryptographic hash + 'sha3_384', // Additional cryptographic hash + 'crc32', // Fast checksum + 'btih_v2', // BitTorrent v2 info hash (BEP 52) ] // Target hashes for index-cid cache (includes all hashes we want to cache) // This includes midhash256 which is computed separately from the worker hashes -export const targetHashForIndex = [ +export const targetHashForIndex: CidAlgorithm[] = [ ...targetHash, - CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing) -] \ No newline at end of file + 'midhash256', // Fast hash for file identification (computed in light processing) +] diff --git a/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts b/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts index 5616095..ae47e2e 100644 --- a/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts +++ b/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts @@ -15,7 +15,7 @@ * Reads and stream consumption stay on Redis in PR B; PR C migrates those. */ import { - flattenMetadata, + buildRecordFields, buildFilePrefix, } from './MetadataUtils.js'; @@ -48,30 +48,12 @@ export class MetaCoreApiWriter { metadata: any, excludeFields: string[] = [] ): Promise { - // Reuse the same flattening logic the Redis path used so the wire - // payload matches what handler.MergeMetadataFlat expects (flat map - // of "a/b/c" → string value). - const prefix = buildFilePrefix(hashId); - const pairs = flattenMetadata(metadata, prefix, excludeFields); - if (pairs.length === 0) return; - - const flat: Record = {}; - for (const pair of pairs) { - // pair.key is "file:/"; strip the prefix+slash. - const field = pair.key.slice(prefix.length + 1); - if (!field) continue; - // Sibling CIDs are stored as a bare-CID key-set (cids/ = - // "true"), never as per-algorithm cid_* fields. Rewrite any - // legacy cid_* field (e.g. the in-memory cid_midhash256 primary - // id) into the key-set form so no deprecated field ever reaches a - // record. The CID is self-describing, so the algorithm in the - // field name is dropped. See METADATA_KEYS.md §2/§14.13. - if (field.startsWith('cid_') && pair.value) { - flat[`cids/${pair.value}`] = 'true'; - } else { - flat[field] = pair.value; - } - } + // `buildRecordFields` is the single write-shape boundary, shared with the + // direct-Redis path in RedisClient. It emits the `cids/` key-set and + // never a per-algorithm `cid_*` field — which meta-core would now reject + // with a 400 anyway. + const flat = buildRecordFields(metadata, excludeFields); + if (Object.keys(flat).length === 0) return; await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat); } diff --git a/packages/meta-sort-core/src/kv/MetadataUtils.test.ts b/packages/meta-sort-core/src/kv/MetadataUtils.test.ts new file mode 100644 index 0000000..9284fd2 --- /dev/null +++ b/packages/meta-sort-core/src/kv/MetadataUtils.test.ts @@ -0,0 +1,86 @@ +import { strict as assert } from 'assert'; +import { buildRecordFields } from './MetadataUtils.js'; + +/** + * `buildRecordFields` is the single boundary where in-memory metadata becomes a + * record. It is worth testing precisely because the bug it fixes had no coverage: + * meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis + * fallback), only one of which turned sibling CIDs into the `cids/` key-set. + * The other wrote a literal `cid_midhash256` field, which meta-core stores but + * never reverse-indexes — so those records were unresolvable by their own CID, + * silently. Both paths now go through this function. + */ +describe('buildRecordFields', () => { + const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4'; + const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua'; + + it('turns cids[] into the bare-CID key-set, not array indices', () => { + const flat = buildRecordFields({ + hashId: MIDHASH, + cids: [MIDHASH, SHA256], + title: 'Inception', + }); + + assert.equal(flat[`cids/${MIDHASH}`], 'true'); + assert.equal(flat[`cids/${SHA256}`], 'true'); + + // The trap: the generic flattener would emit `cids/0`, `cids/1` for an + // array. meta-core's reverse-index hook only matches `cids/`, so + // those would be stored and never indexed. + assert.equal(flat['cids/0'], undefined); + assert.equal(flat['cids/1'], undefined); + }); + + it('never emits a deprecated cid_* or midhash256 field', () => { + const flat = buildRecordFields({ + hashId: MIDHASH, + cids: [MIDHASH], + title: 'Inception', + }); + + // meta-core returns 400 for any of these now — a record that carried one + // was stored but unindexed. + for (const field of Object.keys(flat)) { + assert.ok( + !field.startsWith('cid_'), + `deprecated per-algorithm field reached the record: ${field}` + ); + assert.notEqual(field, 'midhash256'); + assert.notEqual(field, 'canonical_cid'); + } + }); + + it('does not store hashId as a property of its own record', () => { + const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'}); + // hashId IS the record key (`/file/{hashId}`); repeating it inside is noise. + assert.equal(flat['hashId'], undefined); + }); + + it('still flattens ordinary nested metadata', () => { + const flat = buildRecordFields({ + hashId: MIDHASH, + cids: [MIDHASH], + title: 'Inception', + fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}}, + }); + + assert.equal(flat['title'], 'Inception'); + assert.equal(flat['fileinfo/duration'], '8878.5'); + assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264'); + }); + + it('honours excludeFields', () => { + const flat = buildRecordFields( + {hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'}, + ['processingStatus'] + ); + assert.equal(flat['processingStatus'], undefined); + assert.equal(flat['title'], 'x'); + }); + + it('tolerates a record with no cids at all', () => { + const flat = buildRecordFields({title: 'x'}); + assert.equal(flat['title'], 'x'); + assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/'))); + }); +}); diff --git a/packages/meta-sort-core/src/kv/MetadataUtils.ts b/packages/meta-sort-core/src/kv/MetadataUtils.ts index 24cf136..9a90a39 100644 --- a/packages/meta-sort-core/src/kv/MetadataUtils.ts +++ b/packages/meta-sort-core/src/kv/MetadataUtils.ts @@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js'; * // { key: "/file/abc123/video/codec", value: "h265" } * // ] */ +/** + * Fields that never go onto a record under their own name. + * + * `cids` is an **array of bare CIDs** and must become the key-set + * `cids/ = "true"`. If it fell through the generic flattener below it would + * emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's + * reverse-index hook would not recognise, silently leaving the record + * unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly. + * + * `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so + * storing it as a property inside itself is redundant. + */ +const STRUCTURAL_FIELDS = ['cids', 'hashId']; + +/** + * Turn an in-memory `FileMetadata` into the flat `field → value` map that a + * record is made of. + * + * **This is the single write-shape boundary.** Both writers go through it — the + * meta-core HTTP writer and the direct-Redis fallback — because they used not + * to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured + * with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that + * meta-core never indexed. Records looked fine and could not be found by their + * own midhash. One function, one shape, no second path to forget. + * + * Returns field names relative to the record root (no `/file/{hashId}` prefix). + */ +export function buildRecordFields( + metadata: any, + excludeFields: string[] = [] +): Record { + const prefix = 'r'; // throwaway root; stripped below + const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]); + + const flat: Record = {}; + for (const pair of pairs) { + const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/' + if (field) { + flat[field] = pair.value; + } + } + + // Sibling CIDs are a bare-CID key-set, never per-algorithm named fields. + // The midhash is just the member whose multicodec is 0x1000 — there is no + // `cid_midhash256`. See METADATA_KEYS.md §2/§14.13. + const cids: unknown = metadata?.cids; + if (Array.isArray(cids)) { + for (const cid of cids) { + if (typeof cid === 'string' && cid) { + flat[`cids/${cid}`] = 'true'; + } + } + } + + return flat; +} + export function flattenMetadata( metadata: any, prefix: string, diff --git a/packages/meta-sort-core/src/kv/RedisClient.ts b/packages/meta-sort-core/src/kv/RedisClient.ts index ec55555..94209b4 100644 --- a/packages/meta-sort-core/src/kv/RedisClient.ts +++ b/packages/meta-sort-core/src/kv/RedisClient.ts @@ -16,7 +16,7 @@ import * as IORedis from 'ioredis'; import * as os from 'os'; import type { IKVClient, KeyValuePair } from './IKVClient.js'; import { - flattenMetadata, + buildRecordFields, reconstructMetadata, buildFilePrefix, buildPropertyKey, @@ -331,24 +331,24 @@ export class RedisKVClient implements IKVClient { return this.writer.setMetadataFlat(hashId, metadata, excludeFields); } - const prefix = buildFilePrefix(hashId); - const pairs = flattenMetadata(metadata, prefix, excludeFields); - - if (pairs.length === 0) return; + // Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager). + // + // This path used to flatten the metadata itself, which meant it skipped + // the `cid_* → cids/` transform that only the HTTP writer applied: it + // wrote a literal `file:/cid_midhash256` key. meta-core's + // reverse-index hook only registers aliases for fields named `cids/`, + // so those records were stored but never indexed — unresolvable by their + // own midhash, with nothing logged. Both paths now build the record + // through the same `buildRecordFields`, so the key-set cannot be skipped. + const flat = buildRecordFields(metadata, excludeFields); + if (Object.keys(flat).length === 0) return; // Use pipeline for efficient batch SET operations const pipeline = this.redis.pipeline(); - - for (const pair of pairs) { - // Extract field name by removing the prefix - const field = pair.key.slice(prefix.length + 1); // +1 for trailing / - if (field) { - // Key format: file:{hashId}/{property} - const redisKey = this.buildKey(`file:${hashId}/${field}`); - pipeline.set(redisKey, pair.value); - } + for (const [field, value] of Object.entries(flat)) { + // Key format: file:{hashId}/{property} + pipeline.set(this.buildKey(`file:${hashId}/${field}`), value); } - await pipeline.exec(); // Maintain index set for getAllHashIds diff --git a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts index bde27b2..0cb3f4d 100644 --- a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts +++ b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts @@ -282,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface { ); let midHash256: string; - if (indexLine?.cid_midhash256) { - midHash256 = indexLine.cid_midhash256; + if (indexLine?.midhash256) { + midHash256 = indexLine.midhash256; performanceMetrics.recordCacheHit('midhash256'); } else { const computeStart = performance.now(); @@ -297,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface { return; } midHash256 = hash; - performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart)); + performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart)); performanceMetrics.recordCacheMiss('midhash256'); // Store in cache for future lookups @@ -305,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface { filePath, fileStats.size, fileStats.mtime.toISOString(), - { cid_midhash256: midHash256 } + { midhash256: midHash256 } ); } - // STEP 2: Initialize metadata with midhash256 + // STEP 2: Initialize metadata with midhash256. + // + // The midhash plays two roles and now says so: it is the record + // address (`hashId`, the `/file/{hashId}` key) *and* one member of + // the record's bare-CID key-set (`cids`). It used to be a single + // `cid_midhash256` property, which conflated the two and leaked a + // deprecated field name onto the record. See METADATA_KEYS.md §14.13. const metadata: any = { - cid_midhash256: midHash256, + hashId: midHash256, + cids: [midHash256], filePath, processingStatus: 'processing' }; @@ -441,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface { const metadata = db.get(filePath); // Collect hashId before deletion (for Stremio notification) - if (metadata && metadata.cid_midhash256) { - removedHashIds.push(metadata.cid_midhash256); + if (metadata && metadata.hashId) { + removedHashIds.push(metadata.hashId); } db.delete(filePath); diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts index 1644a35..2562d12 100644 --- a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts @@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ let midHash256: string | undefined; const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); - if (indexLine && indexLine.cid_midhash256) { + if (indexLine && indexLine.midhash256) { // Cache hit! Use cached midhash256 (no network I/O needed) - midHash256 = indexLine.cid_midhash256; + midHash256 = indexLine.midhash256; performanceMetrics.recordCacheHit('midhash256'); } else { // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives) @@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ const computeTime = Math.ceil(performance.now() - computeStart); // Record midhash256 computation time (excludes cache hits) - performanceMetrics.recordHashComputation('cid_midhash256', computeTime); + performanceMetrics.recordHashComputation('midhash256', computeTime); performanceMetrics.recordCacheMiss('midhash256'); // Store in cache for future lookups - this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 }); + this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 }); } - // Initialize metadata with midhash256 + // Initialize metadata with midhash256. + // + // Two roles, now named separately: the record address (`hashId`, the + // `/file/{hashId}` key) and a member of the record's bare-CID key-set + // (`cids`). The old single `cid_midhash256` property conflated them and + // leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13. const metadata: FileMetadata = { - cid_midhash256: midHash256, + hashId: midHash256, + cids: [midHash256], processingStatus: 'processing' }; @@ -409,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ // Check cache or compute locally const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); - if (indexLine && indexLine.cid_midhash256) { + if (indexLine && indexLine.midhash256) { // Cache hit! Use cached midhash256 (no network I/O needed) - midHash256 = indexLine.cid_midhash256; + midHash256 = indexLine.midhash256; performanceMetrics.recordCacheHit('midhash256'); } else { // Cache miss - compute midhash256 locally (requires file access) @@ -423,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ const computeTime = Math.ceil(performance.now() - computeStart); // Record midhash256 computation time (excludes cache hits) - performanceMetrics.recordHashComputation('cid_midhash256', computeTime); + performanceMetrics.recordHashComputation('midhash256', computeTime); performanceMetrics.recordCacheMiss('midhash256'); // Store in cache for future lookups - this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 }); + this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 }); } } - // Initialize metadata with midhash256 + // Initialize metadata with midhash256. + // + // Two roles, now named separately: the record address (`hashId`, the + // `/file/{hashId}` key) and a member of the record's bare-CID key-set + // (`cids`). The old single `cid_midhash256` property conflated them and + // leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13. const metadata: FileMetadata = { - cid_midhash256: midHash256, + hashId: midHash256, + cids: [midHash256], processingStatus: 'processing' }; @@ -524,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ throw new Error('Metadata not found - light processing may have failed'); } - const midHash256 = metadata.cid_midhash256; + const midHash256 = metadata.hashId; if (!midHash256) { throw new Error('midhash256 not found in metadata'); } @@ -622,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ // Delete from KV if configured (using nested key architecture) if (this.kvClient && metadata) { try { - const hashId = metadata.cid_midhash256; + const hashId = metadata.hashId; // Delete all properties for this file using nested key architecture if (hashId) { diff --git a/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts b/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts index 93d2a6e..2a6d9cd 100644 --- a/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts +++ b/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts @@ -1,4 +1,4 @@ -import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash"; +import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash"; import {HashIndexManager} from "@metazla/meta-hash"; import {config} from "../../config/EnvConfig.js"; import {targetHash, targetHashForIndex} from "../../config/TargetHash.js"; @@ -16,19 +16,34 @@ class SimpleFileToCid{ if (!stats || !stats.exists) { return null; } - let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); - if(indexLine) { - return indexLine[CID_ALGORITHM_NAMES.sha256]; - }else { - let hashs = {}; - await globalHashComputer.computeMissingHash(filePath, hashs); - globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs); - return hashs[CID_ALGORITHM_NAMES.sha256]; + const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + if (indexLine) { + // The on-disk index keeps per-algorithm columns, keyed by bare + // algorithm name (not the removed `cid_` record fields). + return indexLine['sha256'] ?? null; } + + // The computer returns bare CIDs; the algorithm is recovered from the + // multicodec, never from a field name. + const hashes: MultiHashData = {}; + await globalHashComputer.computeMissingHash(filePath, hashes); + + const columns: Partial> = {}; + for (const algo of targetHash) { + const cid = pickCidByAlgorithm(hashes.cids, algo); + if (cid) { + columns[algo] = cid; + } + } + if (Object.keys(columns).length > 0) { + globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns); + } + + return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null; } catch (e) { return null; } } } -export const simpleFileToCid = new SimpleFileToCid(); \ No newline at end of file +export const simpleFileToCid = new SimpleFileToCid(); diff --git a/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts index 9328bdc..bab29aa 100644 --- a/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts +++ b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts @@ -166,14 +166,14 @@ export class StreamingPipeline { // After light processing, dispatch to plugins and queue background tasks const metadata = this.config.fileProcessor.getDatabase().get(filePath); - if (metadata && metadata.cid_midhash256) { + if (metadata && metadata.hashId) { // Notify Stremio addon to add video - this.notifyStremioAdd(metadata.cid_midhash256).catch(err => { + this.notifyStremioAdd(metadata.hashId).catch(err => { // Silent fail - Stremio addon may not be running }); // Dispatch tasks to container plugins (fire and forget) - this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata); + this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata); // Stage 3: Queue for background processing (SHA-256 computation) this.backgroundQueue.add(() => this.processHashPhase(filePath)) @@ -246,7 +246,7 @@ export class StreamingPipeline { // Otherwise, wait for 'file:complete' event from ContainerPluginScheduler. if (!this.config.containerPluginScheduler) { const metadata = this.config.fileProcessor.getDatabase().get(filePath); - const hashId = metadata?.cid_midhash256; + const hashId = metadata?.hashId; let virtualPath: string | undefined; try { virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined; diff --git a/packages/meta-sort-core/src/types/ExtendedInterfaces.ts b/packages/meta-sort-core/src/types/ExtendedInterfaces.ts index 0a77ad8..60c5bf6 100644 --- a/packages/meta-sort-core/src/types/ExtendedInterfaces.ts +++ b/packages/meta-sort-core/src/types/ExtendedInterfaces.ts @@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta { */ export interface IndexManager { /** - * Get cached CID for a file based on path, size, and mtime + * Get cached CID for a file based on path, size, and mtime. + * + * Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed + * by bare **algorithm** (`midhash256`, `sha256`), not by the removed + * `cid_` record-field names — the local index and the record shape are + * deliberately separate concerns now. See METADATA_KEYS.md §14.13. */ getCidForFile( path: string, size: number, mtime: string - ): { cid_midhash256?: string } | undefined; + ): { midhash256?: string } | undefined; /** * Add a file CID to the cache @@ -117,7 +122,7 @@ export interface IndexManager { path: string, size: number, mtime: string, - cid: { cid_midhash256: string } + cid: { midhash256: string } ): void; } diff --git a/packages/meta-sort-core/src/types/FileMetadata.ts b/packages/meta-sort-core/src/types/FileMetadata.ts index 5f2734f..1a5f5dd 100644 --- a/packages/meta-sort-core/src/types/FileMetadata.ts +++ b/packages/meta-sort-core/src/types/FileMetadata.ts @@ -20,12 +20,26 @@ export interface FileMetadata extends HashMeta { processingStatus?: ProcessingStatus; /** - * In-memory primary identifier: the file's midhash256 CID, computed in the - * light phase and used as the record address (hashId) throughout meta-sort. - * This is an in-process working field only — it is persisted to the record - * as a `cids/` key-set member (by the meta-core watcher and the KV - * writer's cid_* → cids/ transform), never as a named `cid_midhash256` - * field. See METADATA_KEYS.md §1/§14.13. + * The record address: the file's midhash256 CID, computed in the light phase + * and used as the `hashId` throughout meta-sort and as the key of the + * `/file/{hashId}` record. + * + * This replaces the old `cid_midhash256` field. The rename is the point: that + * name was a *metadata property* name, and it leaked — a `cid_*` field written + * to a record is stored but never reverse-indexed by meta-core, leaving the + * record unresolvable by its own CID. meta-core now rejects such fields with a + * 400. See METADATA_KEYS.md §1/§14.13. */ - cid_midhash256?: string; + hashId?: string; + + /** + * Every digest known for this file, as **bare CIDv1 strings** — the midhash + * included. Persisted as the key-set `cids/ = "true"`, which is the only + * shape meta-core indexes. + * + * A CIDv1 is self-describing, so a specific digest is recovered by decoding + * the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a + * field name. + */ + cids?: string[]; } diff --git a/packages/meta-sort-editor/src/components/MetadataEditor.tsx b/packages/meta-sort-editor/src/components/MetadataEditor.tsx index 494dc89..212e552 100644 --- a/packages/meta-sort-editor/src/components/MetadataEditor.tsx +++ b/packages/meta-sort-editor/src/components/MetadataEditor.tsx @@ -375,16 +375,32 @@ const CID_LABELS: Record = { 0xd5: 'MD5', 0x132: 'CRC32', 0x1000: 'MidHash256', - 0x10b7: 'BitTorrent v2', 0xb702: 'BT Pieces Root', + 0x70: 'IPFS (dag-pb)', + 0x10b7: 'BitTorrent v2', 0x1001: 'BitTorrent v1 (file)', 0x1002: 'BitTorrent v2 (file)', + 0x1005: 'Usenet release', + 0x1006: 'URL', }; +/** + * Display order for a record's CIDs. This mirrors the canonical rank ladder + * (`packages/meta-core/internal/cid/rank.go`, the two `cid_rank.rs`, and the + * `/cid-rank-vectors.json` golden fixture) — same ordering, scaled down. + * + * It used to be its own invented ladder (5/4/3/2) with no dag-pb entry at all, + * which meant an IPFS root fell through to 0 and sorted *below* MD5. Display-only, + * so it never broke anything — but it was a seventh, silently divergent copy of a + * rule that already has too many. Keep it in step: a code with no entry sorts last. + */ const RANK_BY_LABEL: Record = { + 'IPFS (dag-pb)': 6, 'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5, 'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4, - MidHash256: 3, 'SHA-1': 2, MD5: 2, CRC32: 2, + MidHash256: 3, + 'Usenet release': 2, URL: 2, + 'SHA-1': 1, MD5: 1, CRC32: 1, }; const B32 = 'abcdefghijklmnopqrstuvwxyz234567'; diff --git a/packages/shared/meta-hash/src/lib/file-id/CidDecode.ts b/packages/shared/meta-hash/src/lib/file-id/CidDecode.ts new file mode 100644 index 0000000..6cfb0d8 --- /dev/null +++ b/packages/shared/meta-hash/src/lib/file-id/CidDecode.ts @@ -0,0 +1,86 @@ +import {CID} from 'multiformats/cid'; +import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData'; + +/** + * Decoding a bare CIDv1 back into the algorithm that produced it. + * + * This is **the** TypeScript CID decoder for the platform. It exists because a + * bare-CID key-set (`cids/`) carries no field names — a consumer that wants + * "the sha2-256 of this file" has to ask the multicodec, not a string. + * + * Before this module, meta-dup carried a private hand-rolled base32+uvarint + * decoder, the meta-sort editor carried a second one with a *different* rank + * ladder, and meta-fuse had none at all (so it could only read the legacy + * `cid_midhash256` field, and silently failed on any record that didn't have + * one). One implementation, exported, replaces all three. + * + * ⚠ **Always match on the multihash code, never the content codec.** MetaMesh + * has two encoders that disagree about the codec: meta-hash writes + * `codec == algorithm code`, while the fullhash plugin and the gateway wrap + * every digest with the raw codec `0x55`. The multihash code is the algorithm in + * both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.) + */ + +/** The multihash function code of a bare CID, or `undefined` if it won't parse. */ +export function cidMultihashCode(cid: string): number | undefined { + try { + return CID.parse(cid).multihash.code; + } catch { + return undefined; + } +} + +/** The content codec of a bare CID, or `undefined` if it won't parse. */ +export function cidCodec(cid: string): number | undefined { + try { + return CID.parse(cid).code; + } catch { + return undefined; + } +} + +/** + * The first member of `cids` whose multihash code is `mhCode`. + * + * Takes the raw code so callers can ask for digests meta-hash doesn't itself + * compute (e.g. the gateway's per-file btih codes). + */ +export function pickCidByMulticodec( + cids: readonly string[] | undefined, + mhCode: number +): string | undefined { + return cids?.find(cid => cidMultihashCode(cid) === mhCode); +} + +/** The member of `cids` produced by `algo`, if present. */ +export function pickCidByAlgorithm( + cids: readonly string[] | undefined, + algo: CidAlgorithm +): string | undefined { + return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]); +} + +/** + * Has `algo` already been computed for this cid set? + * + * The "skip if already computed" check. It used to be an O(1) property lookup + * (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member. + * That is a handful of microseconds against a hash computation that streams the + * whole file — the trade is not close. + */ +export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean { + return pickCidByAlgorithm(cids, algo) !== undefined; +} + +/** + * Add `cid` to `cids` unless an equal string is already there. + * + * Dedup is on the exact string. Two encoders can spell the same digest + * differently (codec 0x55 vs codec == mh), and both spellings are legitimate + * members — meta-core's reverse index maps each to the same record. + */ +export function addCid(cids: string[], cid: string): void { + if (cid && !cids.includes(cid)) { + cids.push(cid); + } +} diff --git a/packages/shared/meta-hash/src/lib/file-id/CodeTable.ts b/packages/shared/meta-hash/src/lib/file-id/CodeTable.ts deleted file mode 100644 index 13ac685..0000000 --- a/packages/shared/meta-hash/src/lib/file-id/CodeTable.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; - -export const codeTable = { - [CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32, - [CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5, - [CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1, - [CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256, - [CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256, - [CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384, - [CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256, - [CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2 -} \ No newline at end of file diff --git a/packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts b/packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts index beaaa74..41a0868 100644 --- a/packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts +++ b/packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts @@ -1,21 +1,20 @@ -import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData"; import {create} from "multiformats/hashes/digest"; import {CID} from 'multiformats/cid'; import {SimpleHash} from "./SimpleHash"; -import {codeTable} from "@root/file-id/CodeTable"; /** * Compute the CIDs of a file using specific algorithms * @param stream The Readable stream of the file * @param algorithms Array of algorithms ('sha256', 'sha1', etc.) - * @returns Array of CIDs (in the order of the algorithms) + * @returns Array of bare CIDv1 strings, in the order of `algorithms` */ -export async function computeCIDs({stream, algorithms,createHasher}: { +export async function computeCIDs({stream, algorithms, createHasher}: { stream: ReadableStream; - algorithms: CID_ALGORITHM_NAMES[]; - createHasher : (algo: CID_ALGORITHM_NAMES) => Promise + algorithms: CidAlgorithm[]; + createHasher: (algo: CidAlgorithm) => Promise }): Promise { - const hashers = await hasherDefiner(algorithms,createHasher); + const hashers = await hasherDefiner(algorithms, createHasher); const reader = stream.getReader(); while (true) { const {done, value} = await reader.read(); @@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: { return cidFinalize(hashers); } -async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[], - createHasher : (algo: CID_ALGORITHM_NAMES) => Promise): Promise<{ +async function hasherDefiner(algorithms: CidAlgorithm[], + createHasher: (algo: CidAlgorithm) => Promise): Promise<{ hasher: SimpleHash, code: CID_ALGORITHM_CODES }[]> { - const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo)) + const hashers = algorithms + .filter(algo => algo in CID_ALGORITHM_CODES) .map(async algo => ({ hasher: await createHasher(algo), - code: codeTable[algo] + code: CID_ALGORITHM_CODES[algo] })); return await Promise.all(hashers); } +/** + * Encoder convention: meta-hash sets `codec == multihash code`. The fullhash + * plugin and the gateway instead use the raw codec (0x55) and carry the + * algorithm only in the multihash. Both are valid CIDs for the same digest, and + * both must decode to the same algorithm — which is why every consumer selects + * on the **multihash code**, never the codec. See `CidDecode.ts`. + */ async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise { return await Promise.all(hashers.map(async ({code, hasher}) => { const hashBuffer = await hasher.digest(); const digest = create(code, hashBuffer); return CID.createV1(code, digest).toString(); })); -} \ No newline at end of file +} diff --git a/packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts b/packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts index 978dae8..f379817 100644 --- a/packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts +++ b/packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts @@ -1,31 +1,32 @@ -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; import {SimpleHash} from "./SimpleHash"; import {createHash} from "crypto"; import {Crc32Hash} from "./Crc32Hash"; import {BtihV2Hasher} from "./BtihV2Hasher"; /** - * Compute the CIDs of a file using specific algorithms - * @returns Array of CIDs (in the order of the algorithms) - * @param algo + * Build the stream hasher for one algorithm. + * + * `midhash256` is absent on purpose: it is not a stream digest (it hashes the + * file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead. */ -export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { +export async function createHasher(algo: CidAlgorithm): Promise { switch (algo) { - case CID_ALGORITHM_NAMES.sha256: + case 'sha256': return createHash("sha256"); - case CID_ALGORITHM_NAMES.sha1: + case 'sha1': return createHash("sha1"); - case CID_ALGORITHM_NAMES.md5: + case 'md5': return createHash("md5"); - case CID_ALGORITHM_NAMES.sha3_256: + case 'sha3_256': return createHash("sha3-256"); - case CID_ALGORITHM_NAMES.sha3_384: + case 'sha3_384': return createHash("sha3-384"); - case CID_ALGORITHM_NAMES.crc32: + case 'crc32': return new Crc32Hash(); - case CID_ALGORITHM_NAMES.btih_v2: + case 'btih_v2': return new BtihV2Hasher(); default: throw new Error(`Unsupported algorithm: ${algo}`); } -} \ No newline at end of file +} diff --git a/packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts b/packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts index 4ea320b..1a3ca85 100644 --- a/packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts +++ b/packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts @@ -1,22 +1,27 @@ -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; import {SimpleHash} from "./SimpleHash"; import {Crc32Hash} from "./Crc32Hash"; import {BrowserHasher} from "@root/file-id/BrowserHasher"; -export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise { +/** + * Browser build. Note it supports fewer algorithms than the node one: there is + * no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for + * either throws. + */ +export async function createHasher(algo: CidAlgorithm): Promise { const hasher = (() => { switch (algo) { - case CID_ALGORITHM_NAMES.sha256: + case 'sha256': return new BrowserHasher('sha-256'); - case CID_ALGORITHM_NAMES.sha1: + case 'sha1': return new BrowserHasher('sha-1'); - case CID_ALGORITHM_NAMES.md5: + case 'md5': return new BrowserHasher('md5'); - case CID_ALGORITHM_NAMES.sha3_256: + case 'sha3_256': return new BrowserHasher('sha3-256'); - case CID_ALGORITHM_NAMES.sha3_384: + case 'sha3_384': return new BrowserHasher('sha3-384'); - case CID_ALGORITHM_NAMES.crc32: + case 'crc32': return new Crc32Hash(); default: throw new Error(`Unsupported algorithm: ${algo}`); @@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise, algorithms: CID_ALGORITHM_NAMES[]): Promise; + computeCIDs(filePath: ReadableStream, algorithms: CidAlgorithm[]): Promise; } \ No newline at end of file diff --git a/packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts b/packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts index 0ec6ce4..360e439 100644 --- a/packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts +++ b/packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts @@ -1,10 +1,16 @@ import {Piscina} from "piscina"; -import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData"; +import {CidAlgorithm} from "../hash-compute/MultiHashData"; export class FileIDComputerWorker{ private piscina: Piscina; constructor(workerPath?:string) { + // `WORKER_URL` is the escape hatch the tests have always set — it just + // wasn't read. Honour it before falling back to path derivation, which + // only works for a caller running from `dist/`. + if (!workerPath && process.env.WORKER_URL) { + workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href; + } if(!workerPath) { // Construct the URL for the current module let distFolder = import.meta.dirname; @@ -25,7 +31,7 @@ export class FileIDComputerWorker{ * @param algorithms Array of algorithms ('sha256', 'sha1') * @returns Array of CIDs (in the order of the algorithms) */ - public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise { + public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise { return this.piscina.run({filePath, algorithms}); } } diff --git a/packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts b/packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts index f03a1c4..5466626 100644 --- a/packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts +++ b/packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts @@ -1,4 +1,5 @@ import {MultiHashData} from '../hash-compute/MultiHashData'; +import {pickCidByAlgorithm} from './CidDecode'; import {CID} from 'multiformats/cid'; /** @@ -22,13 +23,13 @@ export class MagnetLinkGenerator { /** * Generate BitTorrent v2 magnet link from metadata * - * @param metadata - File metadata containing cid_btih_v2 + * @param metadata - File metadata whose `cids` include a btih-v2 CID * @param fileName - Original filename for display name * @param fileSize - File size in bytes * @returns Magnet link string, or null if btih_v2 not available * * @example - * const metadata = {cid_btih_v2: 'bafyb...'}; + * const metadata = {cids: ['bafyb...']}; * const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000); * // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000 */ @@ -37,7 +38,9 @@ export class MagnetLinkGenerator { fileName: string, fileSize: number ): string | null { - const cidString = metadata.cid_btih_v2; + // The btih-v2 CID is recovered from the bare list by its multicodec + // (0x10B7), not by a field name. See CidDecode.ts. + const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2'); if (!cidString) { return null; // BitTorrent v2 hash not computed } @@ -178,6 +181,6 @@ export class MagnetLinkGenerator { fileName: string, fileSize: number ): string | null { - return this.generate({cid_btih_v2: cidString}, fileName, fileSize); + return this.generate({cids: [cidString]}, fileName, fileSize); } } diff --git a/packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts b/packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts index 3e05a49..a4188f1 100644 --- a/packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts +++ b/packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts @@ -1,4 +1,4 @@ -import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData"; +import {CidAlgorithm} from "@root/hash-compute/MultiHashData"; import fs from "fs"; import {Readable} from 'stream'; import {computeCIDs as computeCIDsInternal} from "./ComputeHash"; @@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher"; */ export default async function computeCIDs({filePath, algorithms}: { filePath: string; - algorithms: CID_ALGORITHM_NAMES[] + algorithms: CidAlgorithm[] }): Promise { const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream; return computeCIDsInternal({stream, algorithms, createHasher}); diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts index 256a216..456ac74 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts @@ -1,30 +1,29 @@ -import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, MultiHashData} from "./MultiHashData"; import {computeCIDs} from "@root/file-id/ComputeHash"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; import {SimpleHash} from "@root/file-id/SimpleHash"; -export class HashComputer{ +export class HashComputer { - constructor(private targetHash: CID_ALGORITHM_NAMES[], - private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise) { + constructor(private targetHash: CidAlgorithm[], + private createHasher: (algo: CidAlgorithm) => Promise) { } async computeMissingHash(stream: ReadableStream, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + // Which digests are still missing? On a bare-CID list this costs a decode + // per member rather than a property lookup — negligible against streaming + // the whole file through a hasher. + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs - const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher}); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher}); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts index 676b7e3..78cca1d 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts @@ -1,32 +1,28 @@ -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; import {Readable} from "stream"; import {computeCIDs} from "@root/file-id/ComputeHash"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; import fs from "fs"; import {createHasher} from "@root/file-id/CreateHasher"; -export class HashComputerFile implements ComputeInterface{ +export class HashComputerFile implements ComputeInterface { - constructor(private targetHash: CID_ALGORITHM_NAMES[]) { + constructor(private targetHash: CidAlgorithm[]) { } public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream; - const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher}); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher}); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts index 00547a7..82b5dca 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts @@ -1,33 +1,59 @@ import {HashComputerWorker} from "./HashComputerWorker"; -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; import {HashIndexManager} from "./HashIndexManager"; +import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode"; import {stat} from "fs/promises"; import path from "path"; +/** + * The translation point between the two shapes. + * + * `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets + * persisted onto a record as the `cids/` key-set. The on-disk index is + * **per-algorithm columns**, because its job is the O(1) "already computed?" + * check. Neither shape leaks into the other; this class converts. + */ export class HashComputerIndexCache implements ComputeInterface { private hashIndexManager: HashIndexManager; private hashComputer: HashComputerWorker; - constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { - this.hashComputer = new HashComputerWorker(targetHash,workerPath); - this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash); + constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) { + this.hashComputer = new HashComputerWorker(targetHash, workerPath); + this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash); } public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { await this.hashIndexManager.init(); - let stats = await stat(filePath); - if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) { + const stats = await stat(filePath); + const cids = (metadata.cids ??= []); + + // Cache hit: lift the cached CIDs into the bare list. + if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) { const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); - if(indexLine) { - for (const hash of this.targetHash) { - if (!metadata[hash] && indexLine[hash]) { - metadata[hash] = indexLine[hash]; + if (indexLine) { + for (const algo of this.targetHash) { + const cached = indexLine[algo]; + if (cached && !hasAlgorithm(cids, algo)) { + addCid(cids, cached); } } } } + + // Compute whatever is still missing (a no-op on a full cache hit). await this.hashComputer.computeMissingHash(filePath, metadata); - this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata); + + // Write back: project the bare list onto the index's per-algorithm columns. + const columns: Partial> = {}; + for (const algo of this.targetHash) { + const cid = pickCidByAlgorithm(cids, algo); + if (cid) { + columns[algo] = cid; + } + } + if (Object.keys(columns).length > 0) { + this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns); + } } async getHashIndexManager(): Promise { diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts index 659658a..399f5c9 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts @@ -1,29 +1,25 @@ import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker"; -import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {addCid, hasAlgorithm} from "@root/file-id/CidDecode"; export class HashComputerWorker implements ComputeInterface { private fileIDComputer: FileIDComputerWorker; - constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) { + constructor(private targetHash: CidAlgorithm[], workerPath?: string) { this.fileIDComputer = new FileIDComputerWorker(workerPath); } async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { - // Dynamically determine which hashes are needed - const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + const cids = (metadata.cids ??= []); - // If all hashes are already computed, skip the processing + const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo)); if (neededHashes.length === 0) { return; } - // Compute only the needed CIDs - const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes); - - // Map the computed CIDs back to their respective metadata properties - for (const [index, cid] of cids.entries()) { - const hashType = neededHashes[index]; - metadata[hashType] = cid; + const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes); + for (const cid of computed) { + addCid(cids, cid); } } } diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts index 98cef38..0dcb3d4 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts @@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync'; import {existsAsync} from "../utils/ExistsAsync"; import path from "path"; import {clearInterval} from "node:timers"; -import {CID_ALGORITHM_NAMES} from "./MultiHashData"; +import {CidAlgorithm} from "./MultiHashData"; import {stat} from "fs/promises"; import {Stats} from "node:fs"; -interface IndexLine extends Partial> { +/** + * One row of the on-disk hash-index cache. + * + * This is the one place a **per-algorithm column** is still the right shape, and + * it is not the deprecated one. The index answers "have I already computed sha256 + * for this file?" on every scan, which a column makes an O(1) lookup and a bare + * CID list would make a decode-per-member. It is also strictly local: a wipeable + * cache under `/data/cache/hash-index/` (see + * `docs/project-architecture/cache-persistence.md`), never a record, never on the + * wire. + * + * What changed is the *naming*: columns are keyed by bare algorithm + * (`sha256`, `midhash256`) instead of by the old `cid_` record-field names, + * so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record + * shape and the cache shape are now separate concerns that cannot leak into each + * other — which is the whole point of the split (METADATA_KEYS.md §14.13). + */ +interface IndexLine extends Partial> { path: string; size: string; mtime: string; @@ -21,15 +38,15 @@ export class HashIndexManager { private cache: Map = new Map();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}` private intervalId: any; private intervalTime: number = 30000; - private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read - private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read + private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read + private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read private indexOpsInProgress: boolean = false; private hasChanged: boolean = false; private initialLoad: Promise; - private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {}; + private filePaths: { [key in CidAlgorithm]?: string } = {}; constructor(private indexFolderPath: string, - private targetHash: CID_ALGORITHM_NAMES[]) { + private targetHash: CidAlgorithm[]) { } hasFileInCache(filePath: string,stats:Stats): boolean { @@ -81,7 +98,7 @@ export class HashIndexManager { } // Function to check CSV headers - private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean { + private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean { const records = parseSync(csvContent, { bom: true, columns: true, @@ -116,7 +133,7 @@ export class HashIndexManager { this.intervalId = setInterval(() => this.saveCacheToFile(), time); } - public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise { + public async loadIndex(hash: CidAlgorithm): Promise { if (await existsAsync(this.filePaths[hash])) { // check the file size and if it did not change, do not read the file const stats = await fs.stat(this.filePaths[hash]); @@ -147,7 +164,7 @@ export class HashIndexManager { return Array.from(this.cache.values()); } - private async readCsv(hash: CID_ALGORITHM_NAMES): Promise { + private async readCsv(hash: CidAlgorithm): Promise { if (!(await existsAsync(this.filePaths[hash]))) { return []; } @@ -178,10 +195,8 @@ export class HashIndexManager { public async saveCacheToFile(): Promise { if (this.indexOpsInProgress || !this.hasChanged) { - console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`); return; } - console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`); this.hasChanged = false; this.indexOpsInProgress = true; const start = performance.now(); @@ -274,17 +289,15 @@ export class HashIndexManager { return null; } - public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void { + public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial>): void { if (!filePath || !fileSize || !mtime || !hashs) { throw new Error('Invalid parameters'); } // Check that at least one target hash is provided const providedHashes = this.targetHash.filter(hash => hashs[hash]); if (providedHashes.length === 0) { - console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`); throw new Error('At least one target hash must be provided'); } - console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`); const size = fileSize + ""; const baseName = path.basename(filePath); const cacheKey = `${baseName}-${size}-${mtime}`; diff --git a/packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts b/packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts index 8f40b78..986fe20 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts @@ -1,21 +1,19 @@ -export interface MultiHashData { - cid_crc32?: string; //0x0132 CRC32 (SFV) - cid_md5?: string; // 0xd5 md5 - cid_sha1?: string; // 0x11 sha1 - "cid_sha2-256"?: string;//0x12 sha2-256 - "cid_sha3-256"?: string; // 0x16 sha3-256 - "cid_sha3_384"?: string; // 0x16 sha3-256 - cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format - cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format -} - -export interface ComputeInterface { - computeMissingHash(filePath: string, metadata: MultiHashData): Promise; -} - /** - * according to - * https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv + * Multicodec / multihash codes for every digest meta-hash can compute. + * + * Source of truth for these numbers, mirrored in: + * - `packages/meta-core/internal/cid/rank.go` + * - the `cid_rank.rs` in each of meta-search and meta-share + * - `/cid-rank-vectors.json` (the golden fixture that pins them all together) + * + * ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`: + * `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and + * the gateway wrap every digest with the **raw** codec `0x55` and put the + * algorithm in the multihash code. The multihash code is the algorithm in both + * conventions — so anything selecting a CID by algorithm must read the + * **multihash code**, never the content codec. See `CidDecode.ts`. + * + * https://github.com/multiformats/multicodec/blob/master/table.csv */ export enum CID_ALGORITHM_CODES { crc32 = 0x0132, @@ -24,28 +22,55 @@ export enum CID_ALGORITHM_CODES { sha256 = 0x12, sha3_256 = 0x16, sha3_384 = 0x15, - midhash256 = 0x1000, // Custom code for midhash256 - btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash + midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB) + btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52) } -export enum CID_ALGORITHM_NAMES { - crc32 = 'cid_crc32', - md5 = 'cid_md5', - sha1 = 'cid_sha1', - sha256 = 'cid_sha2-256', - sha3_256 = 'cid_sha3-256', - sha3_384 = 'cid_sha3-384', - midhash256 = 'cid_midhash256', - btih_v2 = 'cid_btih_v2', +/** + * An algorithm *selector*: which digests to compute, and which column of the + * local hash-index cache to read. + * + * This is deliberately **not** a metadata field name. The old + * `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple + * duty — selector, record property name, and CSV column — and the middle duty + * was the deprecated one: those `cid_` fields were stored on records but + * never reverse-indexed by meta-core, leaving records unresolvable by their own + * CID. meta-core now rejects them at the write boundary (400). See + * METADATA_KEYS.md §2/§14.13. + * + * The triple duty also rotted on its own: the interface declared `cid_sha3_384` + * (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc + * comment on it named the wrong multicodec. A self-describing CID cannot drift + * from its own name that way. + */ +export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES; + +/** Every algorithm meta-hash knows how to compute. */ +export const CID_ALGORITHMS: CidAlgorithm[] = [ + 'crc32', + 'md5', + 'sha1', + 'sha256', + 'sha3_256', + 'sha3_384', + 'midhash256', + 'btih_v2', +]; + +/** + * The output of a hash computation: **bare CIDv1 strings, nothing else**. + * + * A CIDv1 is self-describing — its algorithm is the multicodec — so there is no + * per-algorithm field name to keep in sync, and no `:` prefix. To find a + * specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`). + * + * This is persisted onto a record as the bare-CID key-set + * (`cids/ = "true"`), which is the only shape meta-core indexes. + */ +export interface MultiHashData { + cids?: string[]; } -export enum CID_ALGORITHM { - crc32 = 'crc32', - md5 = 'md5', - sha1 = 'sha1', - sha256 = 'sha2-256', - sha3_256 = 'sha3-256', - sha3_384 = 'sha3-384', - midhash256 = 'midhash256', - btih_v2 = 'btih-v2', -} \ No newline at end of file +export interface ComputeInterface { + computeMissingHash(filePath: string, metadata: MultiHashData): Promise; +} diff --git a/packages/shared/meta-hash/src/lib/index.ts b/packages/shared/meta-hash/src/lib/index.ts index 425eebc..1661c71 100644 --- a/packages/shared/meta-hash/src/lib/index.ts +++ b/packages/shared/meta-hash/src/lib/index.ts @@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker export * from './file-id/FastHash.js'; // Fast-hash computation export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation +export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder) + +// The CID model: algorithm codes, the algorithm selector, and MultiHashData +// (bare `cids: string[]`). Exported because every consumer of a `cids/` +// key-set needs them to recover a specific digest — see CidDecode. +export * from './hash-compute/MultiHashData.js'; //hash-compute export * from './hash-compute/HashComputerIndexCache.js'; diff --git a/packages/shared/meta-hash/src/tests/btihv2.test.ts b/packages/shared/meta-hash/src/tests/btihv2.test.ts index 8a32716..9eed921 100644 --- a/packages/shared/meta-hash/src/tests/btihv2.test.ts +++ b/packages/shared/meta-hash/src/tests/btihv2.test.ts @@ -1,6 +1,6 @@ import {describe, expect, it} from 'vitest'; import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher'; -import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib"; +import {HashComputerWorker, pickCidByAlgorithm} from "../lib"; import {readFile} from 'fs/promises'; import {createHash} from 'crypto'; import * as process from "process"; @@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => { }); it('should integrate with HashComputerWorker', async () => { - const hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.btih_v2, - CID_ALGORITHM_NAMES.sha256 - ]); + const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']); const metadata: any = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log('Computed metadata:', metadata); + // The digests land in a bare CID list; each is recovered by its + // multicodec, not by a field name. + const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2'); + expect(btih).toBeDefined(); + expect(typeof btih).toBe('string'); + expect(btih!.startsWith('b')).toBe(true); - // Verify btih_v2 CID was computed - expect(metadata).toHaveProperty('cid_btih_v2'); - expect(typeof metadata.cid_btih_v2).toBe('string'); - expect(metadata.cid_btih_v2.length).toBeGreaterThan(0); - - // Verify CID format (should start with 'b' for base32 CIDv1) - expect(metadata.cid_btih_v2.startsWith('b')).toBe(true); - - // Should also have SHA-256 for comparison - expect(metadata).toHaveProperty('cid_sha2-256'); + // Should also have SHA-256 for comparison. + expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined(); }); it('should compute piece length correctly', () => { @@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => { }); it('should produce valid CID format with codec 0x10B7', async () => { - const hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.btih_v2 - ]); + const hashComputer = new HashComputerWorker(['btih_v2']); const metadata: any = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - const cid = metadata.cid_btih_v2; - console.log('BitTorrent v2 CID:', cid); + const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2'); + + // The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on + // multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7" + // assertion this test is named for. // Verify CID format expect(cid).toBeDefined(); diff --git a/packages/shared/meta-hash/src/tests/cid-rank-vectors.json b/packages/shared/meta-hash/src/tests/cid-rank-vectors.json new file mode 100644 index 0000000..a772ee7 --- /dev/null +++ b/packages/shared/meta-hash/src/tests/cid-rank-vectors.json @@ -0,0 +1,153 @@ +{ + "$schema": "cid-rank-vectors/v1", + "$comment": [ + "CANONICAL golden vectors for the CID utility rank. THE single source of truth.", + "The rank decides which of a record's several CIDs is elected canonical - the cid a", + "record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;", + "they cannot share code (independent submodules, no cargo path-dep). These vectors are", + "the only thing holding them in sync: every implementation's test suite loads this file.", + "", + "This file is VENDORED into each consuming submodule (submodule CI cannot read above its", + "own tree). scripts/check-cid-vectors.sh diffs every copy against this one.", + "", + "Ranking is by UTILITY for external retrieval, not cryptographic strength.", + "Ties break on the lexicographically smaller cid, so the election is deterministic", + "across peers and does not churn.", + "", + "See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md." + ], + "ladder": { + "40": "dag-pb (IPFS block)", + "30": "sha2-256 / sha3-256 / sha3-384 - real content digests", + "20": "BitTorrent infohashes - swarm interop", + "10": "midhash256 - the record address", + "5": "opaque locators (nzb-release, url) - never outrank a real digest", + "0": "weak/unknown digests, unparseable input" + }, + "tieBreak": "lexicographically smaller cid wins", + "vectors": [ + { + "name": "ipfs-dagpb", + "cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 40, + "note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach" + }, + { + "name": "sha2-256-raw-codec", + "cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 30, + "note": "fullhash/gateway convention: raw codec 0x55, mh names the algo" + }, + { + "name": "sha2-256-codec-eq-mh", + "cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua", + "rank": 30, + "note": "REAL meta-hash fixture; meta-hash sets codec == mh" + }, + { + "name": "sha3-256", + "cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq", + "rank": 30, + "note": "REAL meta-hash fixture" + }, + { + "name": "sha3-384", + "cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga", + "rank": 30, + "note": "REAL meta-hash fixture" + }, + { + "name": "btih-v2-codec-eq-mh", + "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "meta-hash encoder convention" + }, + { + "name": "btih-v2-raw-codec", + "cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "fullhash encoder convention - SAME tier, both conventions must match" + }, + { + "name": "btih-v1-file", + "cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "gateway per-file BitTorrent v1" + }, + { + "name": "btih-v2-file", + "cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 20, + "note": "gateway per-file BitTorrent v2" + }, + { + "name": "midhash256-codec-eq-mh", + "cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 10, + "note": "the record address; internally meaningful, externally invisible" + }, + { + "name": "midhash256-raw-codec", + "cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 10 + }, + { + "name": "nzb-release-locator", + "cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab", + "rank": 5, + "note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid." + }, + { + "name": "url-locator", + "cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg", + "rank": 5, + "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release." + }, + { + "name": "sha1", + "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "md5", + "cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "crc32", + "cid": "bagzafmqcathsbnrk", + "rank": 0, + "note": "REAL meta-hash fixture; weak digest" + }, + { + "name": "bt-pieces-root", + "cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 0, + "note": "torrent-internal merkle root, NOT a file-identity cid" + }, + { + "name": "cidv0-rejected", + "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", + "rank": 0, + "note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix." + }, + { + "name": "token-form-rejected", + "cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rank": 0, + "note": "TRAP: the deprecated : token form. Rust used to strip the prefix and rank it 10. It must now be rejected." + }, + { + "name": "garbage", + "cid": "not-a-cid", + "rank": 0 + }, + { + "name": "empty", + "cid": "", + "rank": 0 + } + ] +} diff --git a/packages/shared/meta-hash/src/tests/main.test.ts b/packages/shared/meta-hash/src/tests/main.test.ts index 944e785..c4e29be 100644 --- a/packages/shared/meta-hash/src/tests/main.test.ts +++ b/packages/shared/meta-hash/src/tests/main.test.ts @@ -1,8 +1,13 @@ -import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib"; +import { + CidAlgorithm, + HashComputerIndexCache, + HashComputerWorker, + pickCidByAlgorithm, +} from "../lib"; import {describe, expect, it} from 'vitest'; import * as process from "process"; -import { mkdir, rm, rmdir } from "fs/promises"; +import {mkdir, rm, stat} from "fs/promises"; process.env.WORKER_URL = "./dist/worker.js"; @@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js"; * Remember to build the worker each time you change the worker code. */ +const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384']; + +/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */ +const FIXTURE_BYTES = 490; + +/** + * The expected digests of `src/tests/test.txt`, as bare CIDv1 strings. + * + * Independently reproducible — these are not "whatever the code printed": + * + * $ sha256sum src/tests/test.txt + * be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec + * $ md5sum → c8971ace120d1393b658eabb45d08c88 + * $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7 + * + * and each CID below wraps exactly that digest under its multicodec (meta-hash's + * encoder sets `codec == multihash code`). + * + * ⚠ These values were **re-pinned**. The previous ones described a 499-byte + * CRLF copy of the fixture, from before `.gitattributes` normalised it to LF — + * so on any LF checkout the digests could never match and this suite was red. + * The mismatch was invisible because the worker path was also broken, so the + * assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the + * fixture's bytes ever shift again, the test says *that*, instead of printing + * six inscrutable CID diffs. + */ +const EXPECTED: Record = { + crc32: 'bagzafmqcat2hnot3', + md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea', + sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y', + sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q', + sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi', + sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq', +}; + +/** + * Every expected digest is present in the bare list, and each is recoverable by + * its algorithm **through its multicodec** — there is no field name to look up. + * That is the whole point of the `cids: string[]` shape: a CIDv1 is + * self-describing, so the record never carries a per-algorithm key that could + * drift from the CID it names (METADATA_KEYS.md §14.13). + */ +function expectAllDigests(cids: string[] | undefined) { + expect(cids).toBeDefined(); + expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort()); + + for (const algo of ALGOS) { + expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]); + } +} + describe('add', () => { - it('meta data compute', async () => { - let hashComputer = new HashComputerWorker([ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let metadata = {}; - await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log(metadata); - /** - * test.txt CF20B62A (CRC32) - * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) - * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt - * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt - * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt - */ - expect(metadata).toEqual({ - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' - }); + it('the fixture has the bytes these CIDs were pinned against', async () => { + const {size} = await stat('./src/tests/test.txt'); + expect(size).toBe(FIXTURE_BYTES); }); - it('meta data compute with index', async () => { - await rm("./tmp",{recursive:true}); - await mkdir("./tmp",{recursive:true}); - let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let metadata = {}; + + it('meta data compute', async () => { + const hashComputer = new HashComputerWorker(ALGOS); + const metadata = {}; await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); - console.log(metadata); - /** - * test.txt CF20B62A (CRC32) - * 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5) - * bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt - * 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt - * 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt - */ - expect(metadata).toEqual({ - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga' - }); + expectAllDigests((metadata as { cids?: string[] }).cids); + }); + + it('meta data compute with index', async () => { + await rm("./tmp", {recursive: true, force: true}); + await mkdir("./tmp", {recursive: true}); + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + const metadata = {}; + await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); + expectAllDigests((metadata as { cids?: string[] }).cids); await (await hashComputer.getHashIndexManager()).saveCacheToFile(); }); + it('cache', async () => { - await mkdir("./tmp",{recursive:true}); - let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[ - CID_ALGORITHM_NAMES.sha1, - CID_ALGORITHM_NAMES.sha256, - CID_ALGORITHM_NAMES.md5, - CID_ALGORITHM_NAMES.sha3_256, - CID_ALGORITHM_NAMES.crc32, - CID_ALGORITHM_NAMES.sha3_384 - ]); - let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt'); + await mkdir("./tmp", {recursive: true}); + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + + // The on-disk index is the one place per-algorithm columns survive — it + // answers "already computed?" in O(1) and never leaves this machine. Note + // the columns are bare algorithm names now (`sha256`), not the removed + // record-field names (`cid_sha2-256`). + const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt'); delete indexLine['mtime']; expect(indexLine).toEqual({ - "cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra', - "cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua', - "cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa', - "cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq', - "cid_crc32": "bagzafmqcathsbnrk", - "cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga', - "path": "test.txt", - "size": "499", + ...EXPECTED, + path: "test.txt", + size: String(FIXTURE_BYTES), }); }); -}); \ No newline at end of file + + it('a second pass reuses the cache and recomputes nothing', async () => { + const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS); + const metadata = {}; + await hashComputer.computeMissingHash('./src/tests/test.txt', metadata); + // Same digests, recovered from the index rather than by re-hashing. + expectAllDigests((metadata as { cids?: string[] }).cids); + }); +});