fix cids
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"spec": "dist/api/api.test.js",
|
||||
"spec": ["dist/api/api.test.js", "dist/kv/*.test.js"],
|
||||
"timeout": 30000,
|
||||
"exit": true
|
||||
}
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm} from "@metazla/meta-hash";
|
||||
|
||||
// Target hashes for Piscina worker (full-file hash computation)
|
||||
// Note: midhash256 is NOT included here because it's computed in the main thread
|
||||
// during light processing, not by the Piscina worker
|
||||
export const targetHash = [
|
||||
CID_ALGORITHM_NAMES.sha1, // For torrent verification
|
||||
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
|
||||
CID_ALGORITHM_NAMES.md5, // For compatibility
|
||||
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.crc32, // Fast checksum
|
||||
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
|
||||
//
|
||||
// These are algorithm *selectors*, not metadata field names. The old
|
||||
// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
|
||||
// removed `cid_<algo>` record fields kept getting minted in the first place.
|
||||
// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
|
||||
// See METADATA_KEYS.md §2/§14.13.
|
||||
export const targetHash: CidAlgorithm[] = [
|
||||
'sha1', // For torrent verification
|
||||
'sha256', // For IPFS compatibility
|
||||
'md5', // For compatibility
|
||||
'sha3_256', // Additional cryptographic hash
|
||||
'sha3_384', // Additional cryptographic hash
|
||||
'crc32', // Fast checksum
|
||||
'btih_v2', // BitTorrent v2 info hash (BEP 52)
|
||||
]
|
||||
|
||||
// Target hashes for index-cid cache (includes all hashes we want to cache)
|
||||
// This includes midhash256 which is computed separately from the worker hashes
|
||||
export const targetHashForIndex = [
|
||||
export const targetHashForIndex: CidAlgorithm[] = [
|
||||
...targetHash,
|
||||
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
'midhash256', // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
* Reads and stream consumption stay on Redis in PR B; PR C migrates those.
|
||||
*/
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
buildFilePrefix,
|
||||
} from './MetadataUtils.js';
|
||||
|
||||
@@ -48,30 +48,12 @@ export class MetaCoreApiWriter {
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
// Reuse the same flattening logic the Redis path used so the wire
|
||||
// payload matches what handler.MergeMetadataFlat expects (flat map
|
||||
// of "a/b/c" → string value).
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
if (pairs.length === 0) return;
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
// pair.key is "file:<hash>/<field>"; strip the prefix+slash.
|
||||
const field = pair.key.slice(prefix.length + 1);
|
||||
if (!field) continue;
|
||||
// Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
|
||||
// "true"), never as per-algorithm cid_* fields. Rewrite any
|
||||
// legacy cid_* field (e.g. the in-memory cid_midhash256 primary
|
||||
// id) into the key-set form so no deprecated field ever reaches a
|
||||
// record. The CID is self-describing, so the algorithm in the
|
||||
// field name is dropped. See METADATA_KEYS.md §2/§14.13.
|
||||
if (field.startsWith('cid_') && pair.value) {
|
||||
flat[`cids/${pair.value}`] = 'true';
|
||||
} else {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
// `buildRecordFields` is the single write-shape boundary, shared with the
|
||||
// direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
|
||||
// never a per-algorithm `cid_*` field — which meta-core would now reject
|
||||
// with a 400 anyway.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { buildRecordFields } from './MetadataUtils.js';
|
||||
|
||||
/**
|
||||
* `buildRecordFields` is the single boundary where in-memory metadata becomes a
|
||||
* record. It is worth testing precisely because the bug it fixes had no coverage:
|
||||
* meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
|
||||
* fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
|
||||
* The other wrote a literal `cid_midhash256` field, which meta-core stores but
|
||||
* never reverse-indexes — so those records were unresolvable by their own CID,
|
||||
* silently. Both paths now go through this function.
|
||||
*/
|
||||
describe('buildRecordFields', () => {
|
||||
const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
|
||||
const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
|
||||
|
||||
it('turns cids[] into the bare-CID key-set, not array indices', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH, SHA256],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
assert.equal(flat[`cids/${MIDHASH}`], 'true');
|
||||
assert.equal(flat[`cids/${SHA256}`], 'true');
|
||||
|
||||
// The trap: the generic flattener would emit `cids/0`, `cids/1` for an
|
||||
// array. meta-core's reverse-index hook only matches `cids/<cid>`, so
|
||||
// those would be stored and never indexed.
|
||||
assert.equal(flat['cids/0'], undefined);
|
||||
assert.equal(flat['cids/1'], undefined);
|
||||
});
|
||||
|
||||
it('never emits a deprecated cid_* or midhash256 field', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
// meta-core returns 400 for any of these now — a record that carried one
|
||||
// was stored but unindexed.
|
||||
for (const field of Object.keys(flat)) {
|
||||
assert.ok(
|
||||
!field.startsWith('cid_'),
|
||||
`deprecated per-algorithm field reached the record: ${field}`
|
||||
);
|
||||
assert.notEqual(field, 'midhash256');
|
||||
assert.notEqual(field, 'canonical_cid');
|
||||
}
|
||||
});
|
||||
|
||||
it('does not store hashId as a property of its own record', () => {
|
||||
const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
|
||||
// hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
|
||||
assert.equal(flat['hashId'], undefined);
|
||||
});
|
||||
|
||||
it('still flattens ordinary nested metadata', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
|
||||
});
|
||||
|
||||
assert.equal(flat['title'], 'Inception');
|
||||
assert.equal(flat['fileinfo/duration'], '8878.5');
|
||||
assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
|
||||
});
|
||||
|
||||
it('honours excludeFields', () => {
|
||||
const flat = buildRecordFields(
|
||||
{hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
|
||||
['processingStatus']
|
||||
);
|
||||
assert.equal(flat['processingStatus'], undefined);
|
||||
assert.equal(flat['title'], 'x');
|
||||
});
|
||||
|
||||
it('tolerates a record with no cids at all', () => {
|
||||
const flat = buildRecordFields({title: 'x'});
|
||||
assert.equal(flat['title'], 'x');
|
||||
assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
|
||||
* // { key: "/file/abc123/video/codec", value: "h265" }
|
||||
* // ]
|
||||
*/
|
||||
/**
|
||||
* Fields that never go onto a record under their own name.
|
||||
*
|
||||
* `cids` is an **array of bare CIDs** and must become the key-set
|
||||
* `cids/<cid> = "true"`. If it fell through the generic flattener below it would
|
||||
* emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
|
||||
* reverse-index hook would not recognise, silently leaving the record
|
||||
* unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
|
||||
*
|
||||
* `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
|
||||
* storing it as a property inside itself is redundant.
|
||||
*/
|
||||
const STRUCTURAL_FIELDS = ['cids', 'hashId'];
|
||||
|
||||
/**
|
||||
* Turn an in-memory `FileMetadata` into the flat `field → value` map that a
|
||||
* record is made of.
|
||||
*
|
||||
* **This is the single write-shape boundary.** Both writers go through it — the
|
||||
* meta-core HTTP writer and the direct-Redis fallback — because they used not
|
||||
* to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
|
||||
* with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
|
||||
* meta-core never indexed. Records looked fine and could not be found by their
|
||||
* own midhash. One function, one shape, no second path to forget.
|
||||
*
|
||||
* Returns field names relative to the record root (no `/file/{hashId}` prefix).
|
||||
*/
|
||||
export function buildRecordFields(
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Record<string, string> {
|
||||
const prefix = 'r'; // throwaway root; stripped below
|
||||
const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
|
||||
if (field) {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
|
||||
// Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
|
||||
// The midhash is just the member whose multicodec is 0x1000 — there is no
|
||||
// `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
|
||||
const cids: unknown = metadata?.cids;
|
||||
if (Array.isArray(cids)) {
|
||||
for (const cid of cids) {
|
||||
if (typeof cid === 'string' && cid) {
|
||||
flat[`cids/${cid}`] = 'true';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return flat;
|
||||
}
|
||||
|
||||
export function flattenMetadata(
|
||||
metadata: any,
|
||||
prefix: string,
|
||||
|
||||
@@ -16,7 +16,7 @@ import * as IORedis from 'ioredis';
|
||||
import * as os from 'os';
|
||||
import type { IKVClient, KeyValuePair } from './IKVClient.js';
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
reconstructMetadata,
|
||||
buildFilePrefix,
|
||||
buildPropertyKey,
|
||||
@@ -331,24 +331,24 @@ export class RedisKVClient implements IKVClient {
|
||||
return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
|
||||
}
|
||||
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
|
||||
if (pairs.length === 0) return;
|
||||
// Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
|
||||
//
|
||||
// This path used to flatten the metadata itself, which meant it skipped
|
||||
// the `cid_* → cids/` transform that only the HTTP writer applied: it
|
||||
// wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
|
||||
// reverse-index hook only registers aliases for fields named `cids/<cid>`,
|
||||
// so those records were stored but never indexed — unresolvable by their
|
||||
// own midhash, with nothing logged. Both paths now build the record
|
||||
// through the same `buildRecordFields`, so the key-set cannot be skipped.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
// Use pipeline for efficient batch SET operations
|
||||
const pipeline = this.redis.pipeline();
|
||||
|
||||
for (const pair of pairs) {
|
||||
// Extract field name by removing the prefix
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
|
||||
if (field) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
const redisKey = this.buildKey(`file:${hashId}/${field}`);
|
||||
pipeline.set(redisKey, pair.value);
|
||||
}
|
||||
for (const [field, value] of Object.entries(flat)) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
|
||||
}
|
||||
|
||||
await pipeline.exec();
|
||||
|
||||
// Maintain index set for getAllHashIds
|
||||
|
||||
@@ -282,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
);
|
||||
|
||||
let midHash256: string;
|
||||
if (indexLine?.cid_midhash256) {
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
if (indexLine?.midhash256) {
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
const computeStart = performance.now();
|
||||
@@ -297,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
return;
|
||||
}
|
||||
midHash256 = hash;
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
@@ -305,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
filePath,
|
||||
fileStats.size,
|
||||
fileStats.mtime.toISOString(),
|
||||
{ cid_midhash256: midHash256 }
|
||||
{ midhash256: midHash256 }
|
||||
);
|
||||
}
|
||||
|
||||
// STEP 2: Initialize metadata with midhash256
|
||||
// STEP 2: Initialize metadata with midhash256.
|
||||
//
|
||||
// The midhash plays two roles and now says so: it is the record
|
||||
// address (`hashId`, the `/file/{hashId}` key) *and* one member of
|
||||
// the record's bare-CID key-set (`cids`). It used to be a single
|
||||
// `cid_midhash256` property, which conflated the two and leaked a
|
||||
// deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: any = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
filePath,
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
@@ -441,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
const metadata = db.get(filePath);
|
||||
|
||||
// Collect hashId before deletion (for Stremio notification)
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
removedHashIds.push(metadata.cid_midhash256);
|
||||
if (metadata && metadata.hashId) {
|
||||
removedHashIds.push(metadata.hashId);
|
||||
}
|
||||
|
||||
db.delete(filePath);
|
||||
|
||||
@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
let midHash256: string | undefined;
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
|
||||
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -409,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Check cache or compute locally
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 locally (requires file access)
|
||||
@@ -423,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -524,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
throw new Error('Metadata not found - light processing may have failed');
|
||||
}
|
||||
|
||||
const midHash256 = metadata.cid_midhash256;
|
||||
const midHash256 = metadata.hashId;
|
||||
if (!midHash256) {
|
||||
throw new Error('midhash256 not found in metadata');
|
||||
}
|
||||
@@ -622,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Delete from KV if configured (using nested key architecture)
|
||||
if (this.kvClient && metadata) {
|
||||
try {
|
||||
const hashId = metadata.cid_midhash256;
|
||||
const hashId = metadata.hashId;
|
||||
|
||||
// Delete all properties for this file using nested key architecture
|
||||
if (hashId) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
|
||||
import {HashIndexManager} from "@metazla/meta-hash";
|
||||
import {config} from "../../config/EnvConfig.js";
|
||||
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
|
||||
@@ -16,19 +16,34 @@ class SimpleFileToCid{
|
||||
if (!stats || !stats.exists) {
|
||||
return null;
|
||||
}
|
||||
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
return indexLine[CID_ALGORITHM_NAMES.sha256];
|
||||
}else {
|
||||
let hashs = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashs);
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
|
||||
return hashs[CID_ALGORITHM_NAMES.sha256];
|
||||
const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if (indexLine) {
|
||||
// The on-disk index keeps per-algorithm columns, keyed by bare
|
||||
// algorithm name (not the removed `cid_<algo>` record fields).
|
||||
return indexLine['sha256'] ?? null;
|
||||
}
|
||||
|
||||
// The computer returns bare CIDs; the algorithm is recovered from the
|
||||
// multicodec, never from a field name.
|
||||
const hashes: MultiHashData = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashes);
|
||||
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of targetHash) {
|
||||
const cid = pickCidByAlgorithm(hashes.cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
|
||||
return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
|
||||
@@ -166,14 +166,14 @@ export class StreamingPipeline {
|
||||
|
||||
// After light processing, dispatch to plugins and queue background tasks
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
if (metadata && metadata.hashId) {
|
||||
// Notify Stremio addon to add video
|
||||
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
|
||||
this.notifyStremioAdd(metadata.hashId).catch(err => {
|
||||
// Silent fail - Stremio addon may not be running
|
||||
});
|
||||
|
||||
// Dispatch tasks to container plugins (fire and forget)
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
|
||||
|
||||
// Stage 3: Queue for background processing (SHA-256 computation)
|
||||
this.backgroundQueue.add(() => this.processHashPhase(filePath))
|
||||
@@ -246,7 +246,7 @@ export class StreamingPipeline {
|
||||
// Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
|
||||
if (!this.config.containerPluginScheduler) {
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
const hashId = metadata?.cid_midhash256;
|
||||
const hashId = metadata?.hashId;
|
||||
let virtualPath: string | undefined;
|
||||
try {
|
||||
virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;
|
||||
|
||||
@@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta {
|
||||
*/
|
||||
export interface IndexManager {
|
||||
/**
|
||||
* Get cached CID for a file based on path, size, and mtime
|
||||
* Get cached CID for a file based on path, size, and mtime.
|
||||
*
|
||||
* Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
|
||||
* by bare **algorithm** (`midhash256`, `sha256`), not by the removed
|
||||
* `cid_<algo>` record-field names — the local index and the record shape are
|
||||
* deliberately separate concerns now. See METADATA_KEYS.md §14.13.
|
||||
*/
|
||||
getCidForFile(
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string
|
||||
): { cid_midhash256?: string } | undefined;
|
||||
): { midhash256?: string } | undefined;
|
||||
|
||||
/**
|
||||
* Add a file CID to the cache
|
||||
@@ -117,7 +122,7 @@ export interface IndexManager {
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string,
|
||||
cid: { cid_midhash256: string }
|
||||
cid: { midhash256: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,12 +20,26 @@ export interface FileMetadata extends HashMeta {
|
||||
processingStatus?: ProcessingStatus;
|
||||
|
||||
/**
|
||||
* In-memory primary identifier: the file's midhash256 CID, computed in the
|
||||
* light phase and used as the record address (hashId) throughout meta-sort.
|
||||
* This is an in-process working field only — it is persisted to the record
|
||||
* as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
|
||||
* writer's cid_* → cids/ transform), never as a named `cid_midhash256`
|
||||
* field. See METADATA_KEYS.md §1/§14.13.
|
||||
* The record address: the file's midhash256 CID, computed in the light phase
|
||||
* and used as the `hashId` throughout meta-sort and as the key of the
|
||||
* `/file/{hashId}` record.
|
||||
*
|
||||
* This replaces the old `cid_midhash256` field. The rename is the point: that
|
||||
* name was a *metadata property* name, and it leaked — a `cid_*` field written
|
||||
* to a record is stored but never reverse-indexed by meta-core, leaving the
|
||||
* record unresolvable by its own CID. meta-core now rejects such fields with a
|
||||
* 400. See METADATA_KEYS.md §1/§14.13.
|
||||
*/
|
||||
cid_midhash256?: string;
|
||||
hashId?: string;
|
||||
|
||||
/**
|
||||
* Every digest known for this file, as **bare CIDv1 strings** — the midhash
|
||||
* included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
|
||||
* shape meta-core indexes.
|
||||
*
|
||||
* A CIDv1 is self-describing, so a specific digest is recovered by decoding
|
||||
* the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
|
||||
* field name.
|
||||
*/
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user