change how cid are handled

2026-05-31 11:40:37 +02:00
parent 31bb1245da
commit 44b103c97e
7 changed files with 143 additions and 70 deletions
@@ -59,7 +59,16 @@ export class MetaCoreApiWriter {
        for (const pair of pairs) {
            // pair.key is "file:<hash>/<field>"; strip the prefix+slash.
            const field = pair.key.slice(prefix.length + 1);
-            if (field) {
+            if (!field) continue;
+            // Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
+            // "true"), never as per-algorithm cid_* fields. Rewrite any
+            // legacy cid_* field (e.g. the in-memory cid_midhash256 primary
+            // id) into the key-set form so no deprecated field ever reaches a
+            // record. The CID is self-describing, so the algorithm in the
+            // field name is dropped. See METADATA_KEYS.md §2/§14.13.
+            if (field.startsWith('cid_') && pair.value) {
+                flat[`cids/${pair.value}`] = 'true';
+            } else {
                flat[field] = pair.value;
            }
        }
@@ -296,19 +296,16 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
            metadata.processingStatus = 'complete';
            this.database.set(filePath, metadata);

-            // STEP 5: Update metadata in KV with full hashes
+            // STEP 5: Update metadata in KV.
+            // Full hashes (sha2-256, sha1, …) are computed by the full-hash
+            // container plugin and written to meta-core directly as cids/<cid>
+            // members — they are not held in this in-memory metadata. The
+            // midhash256 is the primary id; on a (rare) midhash collision the
+            // file simply shares the record address until the plugin's
+            // distinct cids/* members disambiguate it.
            if (this.kvClient) {
                try {
-                    // Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
-                    let hashId: string;
-                    if (collisionDetected && metadata['cid_sha2-256']) {
-                        // Collision: Use full SHA-256 as primary ID
-                        hashId = metadata['cid_sha2-256'];
-                        console.log(`   Using SHA-256 as primary ID for collision: ${hashId}`);
-                    } else {
-                        // Normal: Use midhash256 as primary ID
-                        hashId = midHash256;
-                    }
+                    const hashId = midHash256;

                    if (hashId) {
                        // Prepare metadata with filePath added
@@ -625,7 +622,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
        // Delete from KV if configured (using nested key architecture)
        if (this.kvClient && metadata) {
            try {
-                const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
+                const hashId = metadata.cid_midhash256;

                // Delete all properties for this file using nested key architecture
                if (hashId) {
@@ -18,4 +18,14 @@ export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
 export interface FileMetadata extends HashMeta {
  /** Current status of file processing */
  processingStatus?: ProcessingStatus;
+
+  /**
+   * In-memory primary identifier: the file's midhash256 CID, computed in the
+   * light phase and used as the record address (hashId) throughout meta-sort.
+   * This is an in-process working field only — it is persisted to the record
+   * as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
+   * writer's cid_* → cids/ transform), never as a named `cid_midhash256`
+   * field. See METADATA_KEYS.md §1/§14.13.
+   */
+  cid_midhash256?: string;
 }
@@ -365,27 +365,99 @@ function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): b
  return false;
 }

+// Human label per multihash code / CID codec. Source of truth for codes:
+// meta-hash MultiHashData + METADATA_KEYS.md §2.
+const CID_LABELS: Record<number, string> = {
+  0x12: 'SHA-256',
+  0x16: 'SHA3-256',
+  0x15: 'SHA3-384',
+  0x11: 'SHA-1',
+  0xd5: 'MD5',
+  0x132: 'CRC32',
+  0x1000: 'MidHash256',
+  0x10b7: 'BitTorrent v2',
+  0xb702: 'BT Pieces Root',
+  0x1001: 'BitTorrent v1 (file)',
+  0x1002: 'BitTorrent v2 (file)',
+};
+
+const RANK_BY_LABEL: Record<string, number> = {
+  'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5,
+  'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4,
+  MidHash256: 3, 'SHA-1': 2, MD5: 2, CRC32: 2,
+};
+
+const B32 = 'abcdefghijklmnopqrstuvwxyz234567';
+
+function decodeBase32(s: string): Uint8Array | null {
+  let bits = 0, value = 0;
+  const out: number[] = [];
+  for (const ch of s) {
+    const idx = B32.indexOf(ch);
+    if (idx < 0) return null;
+    value = (value << 5) | idx;
+    bits += 5;
+    if (bits >= 8) {
+      bits -= 8;
+      out.push((value >> bits) & 0xff);
+    }
+  }
+  return new Uint8Array(out);
+}
+
+function readUvarint(buf: Uint8Array, pos: number): [number, number] {
+  let result = 0, shift = 0, p = pos;
+  while (p < buf.length) {
+    const b = buf[p++];
+    result |= (b & 0x7f) << shift;
+    if ((b & 0x80) === 0) return [result >>> 0, p];
+    shift += 7;
+  }
+  return [0, pos];
+}
+
 /**
- * Extract all hash values (cid_*) from metadata
+ * Decode a bare multibase-base32 CIDv1 into [codec, multihashCode], or null.
+ * The algorithm is recovered from these — sibling CIDs are stored as a
+ * key-set (cids/<cid>) with no per-algorithm field name.
+ */
+function decodeCid(cid: string): { codec: number; mh: number } | null {
+  if (!cid.startsWith('b')) return null;
+  const bytes = decodeBase32(cid.slice(1));
+  if (!bytes || bytes.length < 3) return null;
+  let v: number, pos = 0;
+  [v, pos] = readUvarint(bytes, pos);
+  if (v !== 1) return null;
+  let codec: number;
+  [codec, pos] = readUvarint(bytes, pos);
+  let mh: number;
+  [mh, pos] = readUvarint(bytes, pos);
+  return { codec, mh };
+}
+
+function cidLabel(cid: string): string {
+  const d = decodeCid(cid);
+  if (!d) return 'CID';
+  // Per-file BitTorrent codecs are carried on the CID codec, not the mh code.
+  return CID_LABELS[d.codec] ?? CID_LABELS[d.mh] ?? 'CID';
+}
+
+/**
+ * Extract all sibling CIDs from the record's bare-CID key-set (cids/<cid>),
+ * labelling each by its multicodec. Replaces the old per-algorithm cid_*
+ * fields. See METADATA_KEYS.md §2/§14.13.
 */
 function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
-  const hashOrder = [
-    { key: 'cid_sha2-256', label: 'SHA-256' },
-    { key: 'cid_sha3-256', label: 'SHA3-256' },
-    { key: 'cid_sha3-384', label: 'SHA3-384' },
-    { key: 'cid_sha1', label: 'SHA-1' },
-    { key: 'cid_md5', label: 'MD5' },
-    { key: 'cid_crc32', label: 'CRC32' },
-    { key: 'cid_btih_v2', label: 'BitTorrent v2' },
-  ];
-
-  return hashOrder
-    .filter(({ key }) => (metadata as any)[key])
-    .map(({ key, label }) => ({
-      key,
-      label,
-      value: (metadata as any)[key],
-    }));
+  const out: { key: string; label: string; value: string }[] = [];
+  for (const key of Object.keys(metadata as any)) {
+    if (!key.startsWith('cids/')) continue;
+    const cid = key.slice('cids/'.length);
+    if (!cid) continue;
+    out.push({ key, label: cidLabel(cid), value: cid });
+  }
+  // Most useful first (sha/ipfs > btih > midhash > weak digests), then by label.
+  out.sort((a, b) => (RANK_BY_LABEL[b.label] ?? 0) - (RANK_BY_LABEL[a.label] ?? 0) || a.label.localeCompare(b.label));
+  return out;
 }

 /**
@@ -41,13 +41,10 @@ export interface FileMetadata {
  collectionnumber?: number;
  criticrating?: number;

-  // Hashes
-  cid_crc32?: string;
-  cid_md5?: string;
-  cid_sha1?: string;
-  'cid_sha2-256'?: string;
-  'cid_sha3-256'?: string;
-  cid_sha3_384?: string;
+  // Sibling CIDs are the bare-CID key-set `cids/<cid> = "true"` (dynamic
+  // keys), not per-algorithm fields. Accessed via prefix-scan; the algorithm
+  // is recovered from each CID's multicodec. See METADATA_KEYS.md §2/§14.13.
+  [cidsKey: `cids/${string}`]: string | undefined;

  // Video info
  video?: {
@@ -4,13 +4,10 @@ export const FileMetaDataName = [
    "byteSize",
 ]

+// Sibling CIDs are stored as the bare-CID key-set `cids/<cid>` (see HashMeta),
+// not as per-algorithm cid_* fields. The single member prefix is `cids`.
 export const hashMetaDataName = [
-    "cid_crc32",
-    "cid_md5",
-    "cid_sha1",
-    "cid_sha2-256",
-    "cid_sha3-256",
-    "cid_sha3_384",
+    "cids",
 ];

 export const AnimeMetaData = [
@@ -1,32 +1,23 @@
 /**
- * HashMeta - Content identification hashes
+ * HashMeta - Content identification hashes.
 *
- * Primary Global ID: cid_midhash256 (SHA-256 hash of middle 1MB + file size)
- * - Format: CID v1 with custom multicodec 0x1000
- * - Example: bafkr4ih5kapbjzqvmj7jxr... (base32 CID encoding)
+ * Sibling CIDs are NOT stored as per-algorithm fields anymore. Every digest
+ * of the file (sha2-256, sha1, md5, crc32, sha3-*, midhash256, btih-v2, …)
+ * lives in a single bare-CID key-set on the record:
 *
- * Legacy IDs: cid_sha2-256, cid_sha1 (computed optionally for compatibility)
+ *     cids/<cidV1> = "true"
 *
- * Migration Note: cid_midhash256 is the new primary identifier replacing full SHA-256.
- * Old hashes remain for backward compatibility and external system integration.
- * All CIDs now use proper CID v1 format (IPFS-compatible structure).
+ * A CIDv1 is self-describing — its algorithm is the multicodec — so there is
+ * no per-algorithm field name and no `<algo>:` prefix. The canonical /
+ * network CID is derived by rank from this set on read, never stored. The
+ * midhash stays the file's address (hashId). See METADATA_KEYS.md §2/§14.13.
+ *
+ * Because the members are dynamic keys (`cids/<cid>`), they can't be typed as
+ * named fields here; consumers read them by prefix-scanning the flat record
+ * and recover the algorithm from each CID's multicodec.
 */
 export interface HashMeta {
-    cid_crc32?: string; //0x0132 CRC32 (SFV)
-    cid_md5?: string; // 0xd5 md5
-    cid_sha1?: string; // 0x11 sha1
-    "cid_sha2-256"?: string;//0x12 sha2-256
-    "cid_sha3-256"?: string; // 0x16 sha3-256
-    "cid_sha3_384"?: string; // 0x16 sha3-256
-    cid_midhash256?: string; // 0x1000 midhash256 (CID v1 format with custom multicodec)
-}
-
-export const HashMetaFields = [
-    "cid_crc32",
-    "cid_md5",
-    "cid_sha1",
-    "cid_sha2-256",
-    "cid_sha3-256",
-    "cid_sha3_384",
-    "cid_midhash256",
-];
+    // Intentionally empty: sibling CIDs are the `cids/<cid>` key-set (above),
+    // not typed fields. Kept as a named type so existing `extends HashMeta`
+    // composition sites stay valid.
+}