change how cid are handled

This commit is contained in:
Pierre
2026-05-31 11:40:37 +02:00
parent 31bb1245da
commit 44b103c97e
7 changed files with 143 additions and 70 deletions
@@ -59,7 +59,16 @@ export class MetaCoreApiWriter {
for (const pair of pairs) {
// pair.key is "file:<hash>/<field>"; strip the prefix+slash.
const field = pair.key.slice(prefix.length + 1);
if (field) {
if (!field) continue;
// Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
// "true"), never as per-algorithm cid_* fields. Rewrite any
// legacy cid_* field (e.g. the in-memory cid_midhash256 primary
// id) into the key-set form so no deprecated field ever reaches a
// record. The CID is self-describing, so the algorithm in the
// field name is dropped. See METADATA_KEYS.md §2/§14.13.
if (field.startsWith('cid_') && pair.value) {
flat[`cids/${pair.value}`] = 'true';
} else {
flat[field] = pair.value;
}
}
@@ -296,19 +296,16 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
metadata.processingStatus = 'complete';
this.database.set(filePath, metadata);
// STEP 5: Update metadata in KV with full hashes
// STEP 5: Update metadata in KV.
// Full hashes (sha2-256, sha1, …) are computed by the full-hash
// container plugin and written to meta-core directly as cids/<cid>
// members — they are not held in this in-memory metadata. The
// midhash256 is the primary id; on a (rare) midhash collision the
// file simply shares the record address until the plugin's
// distinct cids/* members disambiguate it.
if (this.kvClient) {
try {
// Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
let hashId: string;
if (collisionDetected && metadata['cid_sha2-256']) {
// Collision: Use full SHA-256 as primary ID
hashId = metadata['cid_sha2-256'];
console.log(` Using SHA-256 as primary ID for collision: ${hashId}`);
} else {
// Normal: Use midhash256 as primary ID
hashId = midHash256;
}
const hashId = midHash256;
if (hashId) {
// Prepare metadata with filePath added
@@ -625,7 +622,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
// Delete from KV if configured (using nested key architecture)
if (this.kvClient && metadata) {
try {
const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
const hashId = metadata.cid_midhash256;
// Delete all properties for this file using nested key architecture
if (hashId) {
@@ -18,4 +18,14 @@ export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
export interface FileMetadata extends HashMeta {
/** Current status of file processing */
processingStatus?: ProcessingStatus;
/**
* In-memory primary identifier: the file's midhash256 CID, computed in the
* light phase and used as the record address (hashId) throughout meta-sort.
* This is an in-process working field only — it is persisted to the record
* as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
* writer's cid_* → cids/ transform), never as a named `cid_midhash256`
* field. See METADATA_KEYS.md §1/§14.13.
*/
cid_midhash256?: string;
}