This commit is contained in:
Worph
2026-07-14 16:08:20 +02:00
parent 44b103c97e
commit 13fa36ffc7
33 changed files with 887 additions and 356 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
{
"spec": "dist/api/api.test.js",
"spec": ["dist/api/api.test.js", "dist/kv/*.test.js"],
"timeout": 30000,
"exit": true
}
@@ -1,21 +1,27 @@
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
import {CidAlgorithm} from "@metazla/meta-hash";
// Target hashes for Piscina worker (full-file hash computation)
// Note: midhash256 is NOT included here because it's computed in the main thread
// during light processing, not by the Piscina worker
export const targetHash = [
CID_ALGORITHM_NAMES.sha1, // For torrent verification
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
CID_ALGORITHM_NAMES.md5, // For compatibility
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
CID_ALGORITHM_NAMES.crc32, // Fast checksum
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
//
// These are algorithm *selectors*, not metadata field names. The old
// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
// removed `cid_<algo>` record fields kept getting minted in the first place.
// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
// See METADATA_KEYS.md §2/§14.13.
export const targetHash: CidAlgorithm[] = [
'sha1', // For torrent verification
'sha256', // For IPFS compatibility
'md5', // For compatibility
'sha3_256', // Additional cryptographic hash
'sha3_384', // Additional cryptographic hash
'crc32', // Fast checksum
'btih_v2', // BitTorrent v2 info hash (BEP 52)
]
// Target hashes for index-cid cache (includes all hashes we want to cache)
// This includes midhash256 which is computed separately from the worker hashes
export const targetHashForIndex = [
export const targetHashForIndex: CidAlgorithm[] = [
...targetHash,
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
]
'midhash256', // Fast hash for file identification (computed in light processing)
]
@@ -15,7 +15,7 @@
* Reads and stream consumption stay on Redis in PR B; PR C migrates those.
*/
import {
flattenMetadata,
buildRecordFields,
buildFilePrefix,
} from './MetadataUtils.js';
@@ -48,30 +48,12 @@ export class MetaCoreApiWriter {
metadata: any,
excludeFields: string[] = []
): Promise<void> {
// Reuse the same flattening logic the Redis path used so the wire
// payload matches what handler.MergeMetadataFlat expects (flat map
// of "a/b/c" → string value).
const prefix = buildFilePrefix(hashId);
const pairs = flattenMetadata(metadata, prefix, excludeFields);
if (pairs.length === 0) return;
const flat: Record<string, string> = {};
for (const pair of pairs) {
// pair.key is "file:<hash>/<field>"; strip the prefix+slash.
const field = pair.key.slice(prefix.length + 1);
if (!field) continue;
// Sibling CIDs are stored as a bare-CID key-set (cids/<cid> =
// "true"), never as per-algorithm cid_* fields. Rewrite any
// legacy cid_* field (e.g. the in-memory cid_midhash256 primary
// id) into the key-set form so no deprecated field ever reaches a
// record. The CID is self-describing, so the algorithm in the
// field name is dropped. See METADATA_KEYS.md §2/§14.13.
if (field.startsWith('cid_') && pair.value) {
flat[`cids/${pair.value}`] = 'true';
} else {
flat[field] = pair.value;
}
}
// `buildRecordFields` is the single write-shape boundary, shared with the
// direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
// never a per-algorithm `cid_*` field — which meta-core would now reject
// with a 400 anyway.
const flat = buildRecordFields(metadata, excludeFields);
if (Object.keys(flat).length === 0) return;
await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
}
@@ -0,0 +1,86 @@
import { strict as assert } from 'assert';
import { buildRecordFields } from './MetadataUtils.js';
/**
* `buildRecordFields` is the single boundary where in-memory metadata becomes a
* record. It is worth testing precisely because the bug it fixes had no coverage:
* meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
* fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
* The other wrote a literal `cid_midhash256` field, which meta-core stores but
* never reverse-indexes — so those records were unresolvable by their own CID,
* silently. Both paths now go through this function.
*/
describe('buildRecordFields', () => {
const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
it('turns cids[] into the bare-CID key-set, not array indices', () => {
const flat = buildRecordFields({
hashId: MIDHASH,
cids: [MIDHASH, SHA256],
title: 'Inception',
});
assert.equal(flat[`cids/${MIDHASH}`], 'true');
assert.equal(flat[`cids/${SHA256}`], 'true');
// The trap: the generic flattener would emit `cids/0`, `cids/1` for an
// array. meta-core's reverse-index hook only matches `cids/<cid>`, so
// those would be stored and never indexed.
assert.equal(flat['cids/0'], undefined);
assert.equal(flat['cids/1'], undefined);
});
it('never emits a deprecated cid_* or midhash256 field', () => {
const flat = buildRecordFields({
hashId: MIDHASH,
cids: [MIDHASH],
title: 'Inception',
});
// meta-core returns 400 for any of these now — a record that carried one
// was stored but unindexed.
for (const field of Object.keys(flat)) {
assert.ok(
!field.startsWith('cid_'),
`deprecated per-algorithm field reached the record: ${field}`
);
assert.notEqual(field, 'midhash256');
assert.notEqual(field, 'canonical_cid');
}
});
it('does not store hashId as a property of its own record', () => {
const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
// hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
assert.equal(flat['hashId'], undefined);
});
it('still flattens ordinary nested metadata', () => {
const flat = buildRecordFields({
hashId: MIDHASH,
cids: [MIDHASH],
title: 'Inception',
fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
});
assert.equal(flat['title'], 'Inception');
assert.equal(flat['fileinfo/duration'], '8878.5');
assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
});
it('honours excludeFields', () => {
const flat = buildRecordFields(
{hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
['processingStatus']
);
assert.equal(flat['processingStatus'], undefined);
assert.equal(flat['title'], 'x');
});
it('tolerates a record with no cids at all', () => {
const flat = buildRecordFields({title: 'x'});
assert.equal(flat['title'], 'x');
assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
});
});
@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
* // { key: "/file/abc123/video/codec", value: "h265" }
* // ]
*/
/**
* Fields that never go onto a record under their own name.
*
* `cids` is an **array of bare CIDs** and must become the key-set
* `cids/<cid> = "true"`. If it fell through the generic flattener below it would
* emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
* reverse-index hook would not recognise, silently leaving the record
* unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
*
* `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
* storing it as a property inside itself is redundant.
*/
const STRUCTURAL_FIELDS = ['cids', 'hashId'];
/**
* Turn an in-memory `FileMetadata` into the flat `field → value` map that a
* record is made of.
*
* **This is the single write-shape boundary.** Both writers go through it — the
* meta-core HTTP writer and the direct-Redis fallback — because they used not
* to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
* with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
* meta-core never indexed. Records looked fine and could not be found by their
* own midhash. One function, one shape, no second path to forget.
*
* Returns field names relative to the record root (no `/file/{hashId}` prefix).
*/
export function buildRecordFields(
metadata: any,
excludeFields: string[] = []
): Record<string, string> {
const prefix = 'r'; // throwaway root; stripped below
const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
const flat: Record<string, string> = {};
for (const pair of pairs) {
const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
if (field) {
flat[field] = pair.value;
}
}
// Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
// The midhash is just the member whose multicodec is 0x1000 — there is no
// `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
const cids: unknown = metadata?.cids;
if (Array.isArray(cids)) {
for (const cid of cids) {
if (typeof cid === 'string' && cid) {
flat[`cids/${cid}`] = 'true';
}
}
}
return flat;
}
export function flattenMetadata(
metadata: any,
prefix: string,
+15 -15
View File
@@ -16,7 +16,7 @@ import * as IORedis from 'ioredis';
import * as os from 'os';
import type { IKVClient, KeyValuePair } from './IKVClient.js';
import {
flattenMetadata,
buildRecordFields,
reconstructMetadata,
buildFilePrefix,
buildPropertyKey,
@@ -331,24 +331,24 @@ export class RedisKVClient implements IKVClient {
return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
}
const prefix = buildFilePrefix(hashId);
const pairs = flattenMetadata(metadata, prefix, excludeFields);
if (pairs.length === 0) return;
// Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
//
// This path used to flatten the metadata itself, which meant it skipped
// the `cid_* → cids/` transform that only the HTTP writer applied: it
// wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
// reverse-index hook only registers aliases for fields named `cids/<cid>`,
// so those records were stored but never indexed — unresolvable by their
// own midhash, with nothing logged. Both paths now build the record
// through the same `buildRecordFields`, so the key-set cannot be skipped.
const flat = buildRecordFields(metadata, excludeFields);
if (Object.keys(flat).length === 0) return;
// Use pipeline for efficient batch SET operations
const pipeline = this.redis.pipeline();
for (const pair of pairs) {
// Extract field name by removing the prefix
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
if (field) {
// Key format: file:{hashId}/{property}
const redisKey = this.buildKey(`file:${hashId}/${field}`);
pipeline.set(redisKey, pair.value);
}
for (const [field, value] of Object.entries(flat)) {
// Key format: file:{hashId}/{property}
pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
}
await pipeline.exec();
// Maintain index set for getAllHashIds
@@ -282,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
);
let midHash256: string;
if (indexLine?.cid_midhash256) {
midHash256 = indexLine.cid_midhash256;
if (indexLine?.midhash256) {
midHash256 = indexLine.midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
const computeStart = performance.now();
@@ -297,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
return;
}
midHash256 = hash;
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
performanceMetrics.recordCacheMiss('midhash256');
// Store in cache for future lookups
@@ -305,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
filePath,
fileStats.size,
fileStats.mtime.toISOString(),
{ cid_midhash256: midHash256 }
{ midhash256: midHash256 }
);
}
// STEP 2: Initialize metadata with midhash256
// STEP 2: Initialize metadata with midhash256.
//
// The midhash plays two roles and now says so: it is the record
// address (`hashId`, the `/file/{hashId}` key) *and* one member of
// the record's bare-CID key-set (`cids`). It used to be a single
// `cid_midhash256` property, which conflated the two and leaked a
// deprecated field name onto the record. See METADATA_KEYS.md §14.13.
const metadata: any = {
cid_midhash256: midHash256,
hashId: midHash256,
cids: [midHash256],
filePath,
processingStatus: 'processing'
};
@@ -441,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
const metadata = db.get(filePath);
// Collect hashId before deletion (for Stremio notification)
if (metadata && metadata.cid_midhash256) {
removedHashIds.push(metadata.cid_midhash256);
if (metadata && metadata.hashId) {
removedHashIds.push(metadata.hashId);
}
db.delete(filePath);
@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
let midHash256: string | undefined;
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if (indexLine && indexLine.cid_midhash256) {
if (indexLine && indexLine.midhash256) {
// Cache hit! Use cached midhash256 (no network I/O needed)
midHash256 = indexLine.cid_midhash256;
midHash256 = indexLine.midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
const computeTime = Math.ceil(performance.now() - computeStart);
// Record midhash256 computation time (excludes cache hits)
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
performanceMetrics.recordHashComputation('midhash256', computeTime);
performanceMetrics.recordCacheMiss('midhash256');
// Store in cache for future lookups
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
}
// Initialize metadata with midhash256
// Initialize metadata with midhash256.
//
// Two roles, now named separately: the record address (`hashId`, the
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
// (`cids`). The old single `cid_midhash256` property conflated them and
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
const metadata: FileMetadata = {
cid_midhash256: midHash256,
hashId: midHash256,
cids: [midHash256],
processingStatus: 'processing'
};
@@ -409,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
// Check cache or compute locally
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if (indexLine && indexLine.cid_midhash256) {
if (indexLine && indexLine.midhash256) {
// Cache hit! Use cached midhash256 (no network I/O needed)
midHash256 = indexLine.cid_midhash256;
midHash256 = indexLine.midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
// Cache miss - compute midhash256 locally (requires file access)
@@ -423,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
const computeTime = Math.ceil(performance.now() - computeStart);
// Record midhash256 computation time (excludes cache hits)
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
performanceMetrics.recordHashComputation('midhash256', computeTime);
performanceMetrics.recordCacheMiss('midhash256');
// Store in cache for future lookups
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
}
}
// Initialize metadata with midhash256
// Initialize metadata with midhash256.
//
// Two roles, now named separately: the record address (`hashId`, the
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
// (`cids`). The old single `cid_midhash256` property conflated them and
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
const metadata: FileMetadata = {
cid_midhash256: midHash256,
hashId: midHash256,
cids: [midHash256],
processingStatus: 'processing'
};
@@ -524,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
throw new Error('Metadata not found - light processing may have failed');
}
const midHash256 = metadata.cid_midhash256;
const midHash256 = metadata.hashId;
if (!midHash256) {
throw new Error('midhash256 not found in metadata');
}
@@ -622,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
// Delete from KV if configured (using nested key architecture)
if (this.kvClient && metadata) {
try {
const hashId = metadata.cid_midhash256;
const hashId = metadata.hashId;
// Delete all properties for this file using nested key architecture
if (hashId) {
@@ -1,4 +1,4 @@
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
import {HashIndexManager} from "@metazla/meta-hash";
import {config} from "../../config/EnvConfig.js";
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
@@ -16,19 +16,34 @@ class SimpleFileToCid{
if (!stats || !stats.exists) {
return null;
}
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if(indexLine) {
return indexLine[CID_ALGORITHM_NAMES.sha256];
}else {
let hashs = {};
await globalHashComputer.computeMissingHash(filePath, hashs);
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
return hashs[CID_ALGORITHM_NAMES.sha256];
const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if (indexLine) {
// The on-disk index keeps per-algorithm columns, keyed by bare
// algorithm name (not the removed `cid_<algo>` record fields).
return indexLine['sha256'] ?? null;
}
// The computer returns bare CIDs; the algorithm is recovered from the
// multicodec, never from a field name.
const hashes: MultiHashData = {};
await globalHashComputer.computeMissingHash(filePath, hashes);
const columns: Partial<Record<CidAlgorithm, string>> = {};
for (const algo of targetHash) {
const cid = pickCidByAlgorithm(hashes.cids, algo);
if (cid) {
columns[algo] = cid;
}
}
if (Object.keys(columns).length > 0) {
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
}
return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
} catch (e) {
return null;
}
}
}
export const simpleFileToCid = new SimpleFileToCid();
export const simpleFileToCid = new SimpleFileToCid();
@@ -166,14 +166,14 @@ export class StreamingPipeline {
// After light processing, dispatch to plugins and queue background tasks
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
if (metadata && metadata.cid_midhash256) {
if (metadata && metadata.hashId) {
// Notify Stremio addon to add video
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
this.notifyStremioAdd(metadata.hashId).catch(err => {
// Silent fail - Stremio addon may not be running
});
// Dispatch tasks to container plugins (fire and forget)
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
// Stage 3: Queue for background processing (SHA-256 computation)
this.backgroundQueue.add(() => this.processHashPhase(filePath))
@@ -246,7 +246,7 @@ export class StreamingPipeline {
// Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
if (!this.config.containerPluginScheduler) {
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
const hashId = metadata?.cid_midhash256;
const hashId = metadata?.hashId;
let virtualPath: string | undefined;
try {
virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;
@@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta {
*/
export interface IndexManager {
/**
* Get cached CID for a file based on path, size, and mtime
* Get cached CID for a file based on path, size, and mtime.
*
* Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
* by bare **algorithm** (`midhash256`, `sha256`), not by the removed
* `cid_<algo>` record-field names — the local index and the record shape are
* deliberately separate concerns now. See METADATA_KEYS.md §14.13.
*/
getCidForFile(
path: string,
size: number,
mtime: string
): { cid_midhash256?: string } | undefined;
): { midhash256?: string } | undefined;
/**
* Add a file CID to the cache
@@ -117,7 +122,7 @@ export interface IndexManager {
path: string,
size: number,
mtime: string,
cid: { cid_midhash256: string }
cid: { midhash256: string }
): void;
}
@@ -20,12 +20,26 @@ export interface FileMetadata extends HashMeta {
processingStatus?: ProcessingStatus;
/**
* In-memory primary identifier: the file's midhash256 CID, computed in the
* light phase and used as the record address (hashId) throughout meta-sort.
* This is an in-process working field only — it is persisted to the record
* as a `cids/<cid>` key-set member (by the meta-core watcher and the KV
* writer's cid_* → cids/ transform), never as a named `cid_midhash256`
* field. See METADATA_KEYS.md §1/§14.13.
* The record address: the file's midhash256 CID, computed in the light phase
* and used as the `hashId` throughout meta-sort and as the key of the
* `/file/{hashId}` record.
*
* This replaces the old `cid_midhash256` field. The rename is the point: that
* name was a *metadata property* name, and it leaked — a `cid_*` field written
* to a record is stored but never reverse-indexed by meta-core, leaving the
* record unresolvable by its own CID. meta-core now rejects such fields with a
* 400. See METADATA_KEYS.md §1/§14.13.
*/
cid_midhash256?: string;
hashId?: string;
/**
* Every digest known for this file, as **bare CIDv1 strings** — the midhash
* included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
* shape meta-core indexes.
*
* A CIDv1 is self-describing, so a specific digest is recovered by decoding
* the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
* field name.
*/
cids?: string[];
}