diff --git a/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts b/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts index c2057f3..5616095 100644 --- a/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts +++ b/packages/meta-sort-core/src/kv/MetaCoreApiWriter.ts @@ -59,7 +59,16 @@ export class MetaCoreApiWriter { for (const pair of pairs) { // pair.key is "file:/"; strip the prefix+slash. const field = pair.key.slice(prefix.length + 1); - if (field) { + if (!field) continue; + // Sibling CIDs are stored as a bare-CID key-set (cids/ = + // "true"), never as per-algorithm cid_* fields. Rewrite any + // legacy cid_* field (e.g. the in-memory cid_midhash256 primary + // id) into the key-set form so no deprecated field ever reaches a + // record. The CID is self-describing, so the algorithm in the + // field name is dropped. See METADATA_KEYS.md §2/§14.13. + if (field.startsWith('cid_') && pair.value) { + flat[`cids/${pair.value}`] = 'true'; + } else { flat[field] = pair.value; } } diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts index 5b62dd1..1644a35 100644 --- a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts @@ -296,19 +296,16 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ metadata.processingStatus = 'complete'; this.database.set(filePath, metadata); - // STEP 5: Update metadata in KV with full hashes + // STEP 5: Update metadata in KV. + // Full hashes (sha2-256, sha1, …) are computed by the full-hash + // container plugin and written to meta-core directly as cids/ + // members — they are not held in this in-memory metadata. The + // midhash256 is the primary id; on a (rare) midhash collision the + // file simply shares the record address until the plugin's + // distinct cids/* members disambiguate it. if (this.kvClient) { try { - // Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256 - let hashId: string; - if (collisionDetected && metadata['cid_sha2-256']) { - // Collision: Use full SHA-256 as primary ID - hashId = metadata['cid_sha2-256']; - console.log(` Using SHA-256 as primary ID for collision: ${hashId}`); - } else { - // Normal: Use midhash256 as primary ID - hashId = midHash256; - } + const hashId = midHash256; if (hashId) { // Prepare metadata with filePath added @@ -625,7 +622,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ // Delete from KV if configured (using nested key architecture) if (this.kvClient && metadata) { try { - const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5; + const hashId = metadata.cid_midhash256; // Delete all properties for this file using nested key architecture if (hashId) { diff --git a/packages/meta-sort-core/src/types/FileMetadata.ts b/packages/meta-sort-core/src/types/FileMetadata.ts index d75d800..5f2734f 100644 --- a/packages/meta-sort-core/src/types/FileMetadata.ts +++ b/packages/meta-sort-core/src/types/FileMetadata.ts @@ -18,4 +18,14 @@ export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error'; export interface FileMetadata extends HashMeta { /** Current status of file processing */ processingStatus?: ProcessingStatus; + + /** + * In-memory primary identifier: the file's midhash256 CID, computed in the + * light phase and used as the record address (hashId) throughout meta-sort. + * This is an in-process working field only — it is persisted to the record + * as a `cids/` key-set member (by the meta-core watcher and the KV + * writer's cid_* → cids/ transform), never as a named `cid_midhash256` + * field. See METADATA_KEYS.md §1/§14.13. + */ + cid_midhash256?: string; } diff --git a/packages/meta-sort-editor/src/components/MetadataEditor.tsx b/packages/meta-sort-editor/src/components/MetadataEditor.tsx index 7cb7ea1..494dc89 100644 --- a/packages/meta-sort-editor/src/components/MetadataEditor.tsx +++ b/packages/meta-sort-editor/src/components/MetadataEditor.tsx @@ -365,27 +365,99 @@ function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): b return false; } +// Human label per multihash code / CID codec. Source of truth for codes: +// meta-hash MultiHashData + METADATA_KEYS.md §2. +const CID_LABELS: Record = { + 0x12: 'SHA-256', + 0x16: 'SHA3-256', + 0x15: 'SHA3-384', + 0x11: 'SHA-1', + 0xd5: 'MD5', + 0x132: 'CRC32', + 0x1000: 'MidHash256', + 0x10b7: 'BitTorrent v2', + 0xb702: 'BT Pieces Root', + 0x1001: 'BitTorrent v1 (file)', + 0x1002: 'BitTorrent v2 (file)', +}; + +const RANK_BY_LABEL: Record = { + 'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5, + 'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4, + MidHash256: 3, 'SHA-1': 2, MD5: 2, CRC32: 2, +}; + +const B32 = 'abcdefghijklmnopqrstuvwxyz234567'; + +function decodeBase32(s: string): Uint8Array | null { + let bits = 0, value = 0; + const out: number[] = []; + for (const ch of s) { + const idx = B32.indexOf(ch); + if (idx < 0) return null; + value = (value << 5) | idx; + bits += 5; + if (bits >= 8) { + bits -= 8; + out.push((value >> bits) & 0xff); + } + } + return new Uint8Array(out); +} + +function readUvarint(buf: Uint8Array, pos: number): [number, number] { + let result = 0, shift = 0, p = pos; + while (p < buf.length) { + const b = buf[p++]; + result |= (b & 0x7f) << shift; + if ((b & 0x80) === 0) return [result >>> 0, p]; + shift += 7; + } + return [0, pos]; +} + /** - * Extract all hash values (cid_*) from metadata + * Decode a bare multibase-base32 CIDv1 into [codec, multihashCode], or null. + * The algorithm is recovered from these — sibling CIDs are stored as a + * key-set (cids/) with no per-algorithm field name. + */ +function decodeCid(cid: string): { codec: number; mh: number } | null { + if (!cid.startsWith('b')) return null; + const bytes = decodeBase32(cid.slice(1)); + if (!bytes || bytes.length < 3) return null; + let v: number, pos = 0; + [v, pos] = readUvarint(bytes, pos); + if (v !== 1) return null; + let codec: number; + [codec, pos] = readUvarint(bytes, pos); + let mh: number; + [mh, pos] = readUvarint(bytes, pos); + return { codec, mh }; +} + +function cidLabel(cid: string): string { + const d = decodeCid(cid); + if (!d) return 'CID'; + // Per-file BitTorrent codecs are carried on the CID codec, not the mh code. + return CID_LABELS[d.codec] ?? CID_LABELS[d.mh] ?? 'CID'; +} + +/** + * Extract all sibling CIDs from the record's bare-CID key-set (cids/), + * labelling each by its multicodec. Replaces the old per-algorithm cid_* + * fields. See METADATA_KEYS.md §2/§14.13. */ function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] { - const hashOrder = [ - { key: 'cid_sha2-256', label: 'SHA-256' }, - { key: 'cid_sha3-256', label: 'SHA3-256' }, - { key: 'cid_sha3-384', label: 'SHA3-384' }, - { key: 'cid_sha1', label: 'SHA-1' }, - { key: 'cid_md5', label: 'MD5' }, - { key: 'cid_crc32', label: 'CRC32' }, - { key: 'cid_btih_v2', label: 'BitTorrent v2' }, - ]; - - return hashOrder - .filter(({ key }) => (metadata as any)[key]) - .map(({ key, label }) => ({ - key, - label, - value: (metadata as any)[key], - })); + const out: { key: string; label: string; value: string }[] = []; + for (const key of Object.keys(metadata as any)) { + if (!key.startsWith('cids/')) continue; + const cid = key.slice('cids/'.length); + if (!cid) continue; + out.push({ key, label: cidLabel(cid), value: cid }); + } + // Most useful first (sha/ipfs > btih > midhash > weak digests), then by label. + out.sort((a, b) => (RANK_BY_LABEL[b.label] ?? 0) - (RANK_BY_LABEL[a.label] ?? 0) || a.label.localeCompare(b.label)); + return out; } /** diff --git a/packages/meta-sort-editor/src/types/index.ts b/packages/meta-sort-editor/src/types/index.ts index d5ad97b..e9aa260 100644 --- a/packages/meta-sort-editor/src/types/index.ts +++ b/packages/meta-sort-editor/src/types/index.ts @@ -41,13 +41,10 @@ export interface FileMetadata { collectionnumber?: number; criticrating?: number; - // Hashes - cid_crc32?: string; - cid_md5?: string; - cid_sha1?: string; - 'cid_sha2-256'?: string; - 'cid_sha3-256'?: string; - cid_sha3_384?: string; + // Sibling CIDs are the bare-CID key-set `cids/ = "true"` (dynamic + // keys), not per-algorithm fields. Accessed via prefix-scan; the algorithm + // is recovered from each CID's multicodec. See METADATA_KEYS.md §2/§14.13. + [cidsKey: `cids/${string}`]: string | undefined; // Video info video?: { diff --git a/packages/shared/meta-interface/src/lib/MetaDataName.ts b/packages/shared/meta-interface/src/lib/MetaDataName.ts index 84b3c9e..e149d94 100644 --- a/packages/shared/meta-interface/src/lib/MetaDataName.ts +++ b/packages/shared/meta-interface/src/lib/MetaDataName.ts @@ -4,13 +4,10 @@ export const FileMetaDataName = [ "byteSize", ] +// Sibling CIDs are stored as the bare-CID key-set `cids/` (see HashMeta), +// not as per-algorithm cid_* fields. The single member prefix is `cids`. export const hashMetaDataName = [ - "cid_crc32", - "cid_md5", - "cid_sha1", - "cid_sha2-256", - "cid_sha3-256", - "cid_sha3_384", + "cids", ]; export const AnimeMetaData = [ diff --git a/packages/shared/meta-interface/src/lib/metadata-type/HashMeta.ts b/packages/shared/meta-interface/src/lib/metadata-type/HashMeta.ts index 86f3a3c..d64d171 100644 --- a/packages/shared/meta-interface/src/lib/metadata-type/HashMeta.ts +++ b/packages/shared/meta-interface/src/lib/metadata-type/HashMeta.ts @@ -1,32 +1,23 @@ /** - * HashMeta - Content identification hashes + * HashMeta - Content identification hashes. * - * Primary Global ID: cid_midhash256 (SHA-256 hash of middle 1MB + file size) - * - Format: CID v1 with custom multicodec 0x1000 - * - Example: bafkr4ih5kapbjzqvmj7jxr... (base32 CID encoding) + * Sibling CIDs are NOT stored as per-algorithm fields anymore. Every digest + * of the file (sha2-256, sha1, md5, crc32, sha3-*, midhash256, btih-v2, …) + * lives in a single bare-CID key-set on the record: * - * Legacy IDs: cid_sha2-256, cid_sha1 (computed optionally for compatibility) + * cids/ = "true" * - * Migration Note: cid_midhash256 is the new primary identifier replacing full SHA-256. - * Old hashes remain for backward compatibility and external system integration. - * All CIDs now use proper CID v1 format (IPFS-compatible structure). + * A CIDv1 is self-describing — its algorithm is the multicodec — so there is + * no per-algorithm field name and no `:` prefix. The canonical / + * network CID is derived by rank from this set on read, never stored. The + * midhash stays the file's address (hashId). See METADATA_KEYS.md §2/§14.13. + * + * Because the members are dynamic keys (`cids/`), they can't be typed as + * named fields here; consumers read them by prefix-scanning the flat record + * and recover the algorithm from each CID's multicodec. */ export interface HashMeta { - cid_crc32?: string; //0x0132 CRC32 (SFV) - cid_md5?: string; // 0xd5 md5 - cid_sha1?: string; // 0x11 sha1 - "cid_sha2-256"?: string;//0x12 sha2-256 - "cid_sha3-256"?: string; // 0x16 sha3-256 - "cid_sha3_384"?: string; // 0x16 sha3-256 - cid_midhash256?: string; // 0x1000 midhash256 (CID v1 format with custom multicodec) -} - -export const HashMetaFields = [ - "cid_crc32", - "cid_md5", - "cid_sha1", - "cid_sha2-256", - "cid_sha3-256", - "cid_sha3_384", - "cid_midhash256", -]; \ No newline at end of file + // Intentionally empty: sibling CIDs are the `cids/` key-set (above), + // not typed fields. Kept as a named type so existing `extends HashMeta` + // composition sites stay valid. +} \ No newline at end of file