improved version

This commit is contained in:
Pierre
2026-03-13 22:10:17 +01:00
parent a75615c7ad
commit 2856f3264d
3 changed files with 138 additions and 78 deletions
+6 -8
View File
@@ -2,17 +2,15 @@
* MetaMesh Plugin: subtitle-extractor * MetaMesh Plugin: subtitle-extractor
* *
* ============================================================================ * ============================================================================
* PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION * PLUGIN FILE ACCESS ARCHITECTURE - WebDAV
* ============================================================================ * ============================================================================
* *
* Each plugin container has exactly 3 mounts: * File access via WebDAV (WEBDAV_URL environment variable):
* - Read media files: GET /webdav/watch/... or /webdav/test/...
* - Write output: PUT /webdav/plugin/subtitle-extractor/...
* - Cache: Local /cache mount (for temporary files during extraction)
* *
* 1. /files (READ-ONLY) - Shared media files, read access only * SECURITY: Plugins use WebDAV for all file access. No /files mount needed.
* 2. /cache (READ-WRITE) - Plugin-specific cache folder
* 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles
*
* SECURITY: Plugins must NEVER write to /files directly.
* All write operations go to /cache or /output only.
* *
* ============================================================================ * ============================================================================
*/ */
+103 -63
View File
@@ -5,40 +5,49 @@
* Republishes extracted subtitles in the pipeline (similar to TMDB plugin with posters). * Republishes extracted subtitles in the pipeline (similar to TMDB plugin with posters).
* *
* ============================================================================ * ============================================================================
* PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION * PLUGIN FILE ACCESS ARCHITECTURE - WebDAV
* ============================================================================ * ============================================================================
* *
* Each plugin container has exactly 3 mounts: * File access via WebDAV (WEBDAV_URL environment variable):
* - Read media files: GET /webdav/watch/... or /webdav/test/...
* - Write output: PUT /webdav/plugin/subtitle-extractor/...
* - Cache: Local /cache mount (for temporary files during extraction)
* *
* 1. /files (READ-ONLY) - Shared media files, read access only * Benefits:
* 2. /cache (READ-WRITE) - Plugin-specific cache folder * - No need for /files or /output volume mounts
* 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles * - Works across multiple hosts
* * - Plugins are fully isolated
* SECURITY: Plugins must NEVER write to /files directly.
* - Use /cache for temporary/cache data
* - Use /output for output files (extracted subtitles)
* *
* ============================================================================ * ============================================================================
*/ */
import { spawn } from 'child_process'; import { spawn } from 'child_process';
import { existsSync, mkdirSync, statSync, openSync, readSync, closeSync, unlinkSync } from 'fs'; import { existsSync, mkdirSync, statSync, unlinkSync, readFileSync } from 'fs';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import * as path from 'path'; import * as path from 'path';
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
import { MetaCoreClient } from './meta-core-client.js'; import { MetaCoreClient } from './meta-core-client.js';
import { createWebDAVClient, WebDAVClient } from './webdav-client.js'; import { createWebDAVClient } from './webdav-client.js';
// Initialize WebDAV client if WEBDAV_URL is set // Initialize WebDAV client if WEBDAV_URL is set
const webdavClient = createWebDAVClient(); const webdavClient = createWebDAVClient();
if (webdavClient) { if (webdavClient) {
console.log('[subtitle-extractor] Using WebDAV for file access'); console.log('[subtitle-extractor] WebDAV client initialized for file access');
} else { } else {
console.log('[subtitle-extractor] Using direct filesystem access'); console.warn('[subtitle-extractor] WARNING: WEBDAV_URL not set - plugin will not be able to write output files');
} }
// Output path for extracted subtitles /**
const PLUGIN_OUTPUT_PATH = '/output'; * ============================================================================
* PLUGIN OUTPUT PATH - Written via WebDAV
* ============================================================================
* Output files (subtitles) are written via WebDAV PUT requests.
* Path: /files/plugin/subtitle-extractor/<filename> (accessible at WEBDAV_URL/plugin/subtitle-extractor/)
* TEMP_PATH (/cache/temp) - Local temp folder for ffmpeg extraction before WebDAV upload
* ============================================================================
*/
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/subtitle-extractor';
const TEMP_PATH = '/cache/temp';
// Image-based subtitle codecs that cannot be converted to text // Image-based subtitle codecs that cannot be converted to text
const UNSUPPORTED_SUBTITLE_CODECS = new Set([ const UNSUPPORTED_SUBTITLE_CODECS = new Set([
@@ -102,30 +111,24 @@ export function configure(config: Record<string, unknown>): void {
} }
/** /**
* Compute midhash256 CID for a file (matches meta-hash algorithm) * Compute midhash256 CID from a Buffer (matches meta-hash algorithm)
* Used for computing CID before/after WebDAV upload
*/ */
function computeMidHash256Sync(filePath: string): string { function computeMidHash256FromBuffer(data: Buffer): string {
const SAMPLE_SIZE = 1024 * 1024; // 1MB const SAMPLE_SIZE = 1024 * 1024; // 1MB
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]); const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
const stats = statSync(filePath); const fileSize = data.length;
const fileSize = stats.size;
const sizeBuffer = Buffer.allocUnsafe(8); const sizeBuffer = Buffer.allocUnsafe(8);
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
let sampleData: Buffer; let sampleData: Buffer;
if (fileSize <= SAMPLE_SIZE) { if (fileSize <= SAMPLE_SIZE) {
const fd = openSync(filePath, 'r'); sampleData = data;
sampleData = Buffer.allocUnsafe(fileSize);
readSync(fd, sampleData, 0, fileSize, 0);
closeSync(fd);
} else { } else {
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
const fd = openSync(filePath, 'r'); sampleData = data.subarray(middleOffset, middleOffset + SAMPLE_SIZE);
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
closeSync(fd);
} }
const hashInput = Buffer.concat([sizeBuffer, sampleData]); const hashInput = Buffer.concat([sizeBuffer, sampleData]);
@@ -181,35 +184,46 @@ interface SubtitleStream {
function parseSubtitleStreams(existingMeta: Record<string, string>): SubtitleStream[] { function parseSubtitleStreams(existingMeta: Record<string, string>): SubtitleStream[] {
const streams: SubtitleStream[] = []; const streams: SubtitleStream[] = [];
// ffmpeg plugin stores streams as JSON in 'streams' field // ffmpeg plugin stores streams as 'stream' field - an array of JSON strings
// or as individual fields like 'subtitle_0_codec', 'subtitle_0_language', etc. // Each element is a stringified JSON object like:
const streamsJson = existingMeta['streams']; // '{"type":"subtitle","codec":"ass","index":2,"language":"eng",...}'
if (streamsJson) { const streamData = existingMeta['stream'];
if (streamData) {
try { try {
const allStreams = JSON.parse(streamsJson) as Array<{ // Parse the outer array (it's already an array from meta-core)
codec_type?: string; const allStreams = Array.isArray(streamData)
codec_name?: string; ? streamData
index?: number; : JSON.parse(streamData) as string[];
tags?: { language?: string; title?: string };
}>;
let subtitleIndex = 0; let subtitleIndex = 0;
for (const stream of allStreams) { for (const streamStr of allStreams) {
if (stream.codec_type === 'subtitle') { // Each stream is a JSON string
const stream = typeof streamStr === 'string'
? JSON.parse(streamStr) as {
type?: string;
codec?: string;
index?: number;
language?: string;
title?: string;
}
: streamStr;
if (stream.type === 'subtitle') {
streams.push({ streams.push({
index: subtitleIndex, index: subtitleIndex,
codec: stream.codec_name || 'unknown', codec: stream.codec || 'unknown',
language: stream.tags?.language, language: stream.language,
title: stream.tags?.title, title: stream.title,
}); });
subtitleIndex++; subtitleIndex++;
} }
} }
} catch { } catch (e) {
// Fall through to individual field parsing console.error('[subtitle-extractor] Failed to parse stream metadata:', e);
} }
} }
// Fallback: parse individual fields // Fallback: parse individual fields (legacy format)
if (streams.length === 0) { if (streams.length === 0) {
for (let i = 0; i < 20; i++) { for (let i = 0; i < 20; i++) {
const codec = existingMeta[`subtitle_${i}_codec`]; const codec = existingMeta[`subtitle_${i}_codec`];
@@ -357,16 +371,25 @@ export async function process(
console.log(`[subtitle-extractor] Found ${textSubtitles.length} text-based subtitle(s) in ${filePath}`); console.log(`[subtitle-extractor] Found ${textSubtitles.length} text-based subtitle(s) in ${filePath}`);
// Get input path (WebDAV URL or filesystem path) // Require WebDAV client for writing output
let inputPath = filePath; if (!webdavClient) {
if (webdavClient) { console.error('[subtitle-extractor] WebDAV client not available, cannot write output files');
inputPath = webdavClient.toWebDAVUrl(filePath); await sendCallback({
console.log(`[subtitle-extractor] Using WebDAV input: ${inputPath}`); taskId: request.taskId,
status: 'failed',
duration: Date.now() - startTime,
error: 'WebDAV client not configured (WEBDAV_URL not set)',
});
return;
} }
// Ensure output directory exists // Get input path (WebDAV URL for reading)
if (!existsSync(PLUGIN_OUTPUT_PATH)) { let inputPath = webdavClient.toWebDAVUrl(filePath);
mkdirSync(PLUGIN_OUTPUT_PATH, { recursive: true }); console.log(`[subtitle-extractor] Using WebDAV input: ${inputPath}`);
// Ensure temp directory exists for ffmpeg extraction
if (!existsSync(TEMP_PATH)) {
mkdirSync(TEMP_PATH, { recursive: true });
} }
// Get video title for output filename // Get video title for output filename
@@ -380,38 +403,50 @@ export async function process(
// Extract each subtitle // Extract each subtitle
for (const sub of textSubtitles) { for (const sub of textSubtitles) {
const langCode = sub.language || 'und';
const langSuffix = sub.language ? `.${sub.language}` : `.${sub.index}`; const langSuffix = sub.language ? `.${sub.language}` : `.${sub.index}`;
const ext = outputFormat; const ext = outputFormat;
// Build output filename: Title (Year)[videoCID]_subtitle.lang.srt // Build output filename: Title (Year)[videoCID]_subtitle.lang.srt
const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle${langSuffix}.${ext}`; const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle${langSuffix}.${ext}`;
const outputPath = path.join(PLUGIN_OUTPUT_PATH, outputFilename); const webdavOutputPath = `${PLUGIN_OUTPUT_WEBDAV_PATH}/${outputFilename}`;
const tempOutputPath = path.join(TEMP_PATH, outputFilename);
// Check if already extracted // Check if already extracted via WebDAV
if (existsSync(outputPath) && !forceRecompute) { if (!forceRecompute) {
const exists = await webdavClient.exists(webdavOutputPath);
if (exists) {
console.log(`[subtitle-extractor] Subtitle already exists: ${outputFilename}`); console.log(`[subtitle-extractor] Subtitle already exists: ${outputFilename}`);
// Read existing file to get CID
try { try {
const subtitleCid = computeMidHash256Sync(outputPath); const existingData = await webdavClient.readFile(webdavOutputPath);
const subtitleCid = computeMidHash256FromBuffer(existingData);
extractedCids.push(subtitleCid); extractedCids.push(subtitleCid);
if (sub.language && !extractedLanguages.includes(sub.language)) { if (sub.language && !extractedLanguages.includes(sub.language)) {
extractedLanguages.push(sub.language); extractedLanguages.push(sub.language);
} }
} catch (e) { } catch (e) {
console.error(`[subtitle-extractor] Failed to compute CID: ${e}`); console.error(`[subtitle-extractor] Failed to read existing subtitle: ${e}`);
} }
continue; continue;
} }
}
// Extract subtitle // Extract subtitle to temp file
const extracted = await extractSubtitle(inputPath, outputPath, sub.index, sub.codec); const extracted = await extractSubtitle(inputPath, tempOutputPath, sub.index, sub.codec);
if (extracted) { if (extracted) {
try { try {
// Compute CID for the extracted subtitle file // Read extracted subtitle file
const subtitleCid = computeMidHash256Sync(outputPath); const subtitleData = readFileSync(tempOutputPath);
// Compute CID from buffer
const subtitleCid = computeMidHash256FromBuffer(subtitleData);
console.log(`[subtitle-extractor] Subtitle CID: ${subtitleCid}`); console.log(`[subtitle-extractor] Subtitle CID: ${subtitleCid}`);
// Upload to WebDAV
await webdavClient.writeFile(webdavOutputPath, subtitleData);
console.log(`[subtitle-extractor] Uploaded subtitle to WebDAV: ${webdavOutputPath}`);
extractedCids.push(subtitleCid); extractedCids.push(subtitleCid);
if (sub.language && !extractedLanguages.includes(sub.language)) { if (sub.language && !extractedLanguages.includes(sub.language)) {
extractedLanguages.push(sub.language); extractedLanguages.push(sub.language);
@@ -424,8 +459,13 @@ export async function process(
if (sub.language) { if (sub.language) {
await metaCore.addToSet(cid, 'subtitleLanguages', sub.language); await metaCore.addToSet(cid, 'subtitleLanguages', sub.language);
} }
// Clean up temp file
try { unlinkSync(tempOutputPath); } catch {}
} catch (e) { } catch (e) {
console.error(`[subtitle-extractor] Failed to process extracted subtitle: ${e}`); console.error(`[subtitle-extractor] Failed to process extracted subtitle: ${e}`);
// Clean up temp file on error
try { unlinkSync(tempOutputPath); } catch {}
} }
} }
} }
+22
View File
@@ -100,6 +100,28 @@ export class WebDAVClient {
return Buffer.from(arrayBuffer); return Buffer.from(arrayBuffer);
} }
/**
* Write file via HTTP PUT
*/
async writeFile(filePath: string, data: Buffer | string): Promise<void> {
const url = this.toWebDAVUrl(filePath);
// Convert Buffer to Uint8Array for fetch compatibility
const body = typeof data === 'string' ? data : new Uint8Array(data);
const response = await fetch(url, {
method: 'PUT',
body,
headers: {
'Content-Type': 'application/octet-stream',
},
});
if (!response.ok) {
throw new Error(`WebDAV PUT failed for ${filePath}: ${response.status} ${response.statusText}`);
}
}
/** /**
* Check if file exists via HTTP HEAD * Check if file exists via HTTP HEAD
*/ */