commit 678af9f9bfd7621163d61657f6b1b1322e69c8b9 Author: Pierre Date: Sun Jan 18 00:24:14 2026 +0100 initial commit diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..8cdc3e7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,56 @@ +name: Build and Push Docker Image + +on: + push: + branches: [main, master] + tags: ['v*'] + pull_request: + branches: [main, master] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd8fe26 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +*.log +.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c738421 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM node:20-slim AS builder +WORKDIR /app +COPY package.json tsconfig.json ./ +RUN npm install +COPY src/ ./src/ +RUN npm run build + +FROM node:20-slim +# Install ffmpeg for subtitle extraction +RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY package.json ./ +RUN npm install --omit=dev +COPY --from=builder /app/dist ./dist +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))" +CMD ["node", "dist/index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..3be413c --- /dev/null +++ b/README.md @@ -0,0 +1,158 @@ +# MetaMesh Plugin: Subtitle Extractor + +Extracts embedded subtitles from video files and saves them as standalone text files. The extracted subtitles are republished in the MetaMesh pipeline and linked to the original video via CID. + +## Features + +- **Automatic extraction**: Extracts all text-based subtitle tracks from video files +- **Multiple formats**: Supports SRT, ASS, SSA, WebVTT, and MOV_TEXT codecs +- **Smart filtering**: Automatically skips image-based subtitles (PGS, DVD, DVB) that cannot be converted to text +- **Language preservation**: Maintains language metadata from video streams +- **Pipeline integration**: Extracted files are saved to `/output` and automatically picked up by meta-sort +- **CID linking**: Stores subtitle CIDs as metadata on the source video + +## Dependencies + +This plugin requires the following plugins to run first: +- `file-info` - Determines file type +- `ffmpeg` - Provides subtitle stream information + +## Installation + +### Build from source + +```bash +cd packages/plugins/metamesh-plugin-subtitle-extractor +npm install +npm run build +``` + +### Build Docker image + +```bash +docker build -t metamesh-plugin-subtitle-extractor:main . +``` + +### Configure in plugins.yml + +Add to `dev/plugins.yml`: + +```yaml +plugins: + subtitle-extractor: + enabled: true + image: metamesh-plugin-subtitle-extractor:main + instances: 1 + resources: + memory: 256m + cpus: 0.5 +``` + +## Configuration + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `forceRecompute` | boolean | `false` | Re-extract subtitles even if already processed | +| `outputFormat` | select | `srt` | Output format: `srt`, `vtt`, or `ass` | + +## Output + +### Extracted Files + +Subtitle files are saved to `/output` with the naming pattern: + +``` +{Title} ({Year})[{VideoCID}]_subtitle.{lang}.{ext} +``` + +Example: `Sintel (2010)[bafk...abc]_subtitle.eng.srt` + +### Metadata + +The plugin stores the following metadata on the source video: + +| Field | Type | Description | +|-------|------|-------------| +| `extractedSubtitles` | array | CIDs of extracted subtitle files | +| `subtitleLanguages` | array | Language codes of extracted subtitles | + +## Supported Codecs + +### Text-based (supported) +- `subrip` / `srt` - SubRip +- `ass` - Advanced SubStation Alpha +- `ssa` - SubStation Alpha +- `webvtt` - WebVTT +- `mov_text` - QuickTime text + +### Image-based (skipped) +- `hdmv_pgs_subtitle` - Blu-ray PGS +- `dvd_subtitle` - DVD VOB +- `dvb_subtitle` - DVB +- `xsub` - DivX XSUB + +## Architecture + +``` +Video File (MKV/MP4/AVI) + │ + ▼ +┌───────────────────┐ +│ ffmpeg plugin │ ← Provides stream info +└───────────────────┘ + │ + ▼ +┌───────────────────┐ +│ subtitle-extractor│ +│ │ +│ 1. Parse streams │ +│ 2. Filter codecs │ +│ 3. Extract via │ +│ ffmpeg │ +│ 4. Save to │ +│ /output │ +│ 5. Compute CID │ +│ 6. Store in │ +│ Redis │ +└───────────────────┘ + │ + ▼ +┌───────────────────┐ +│ /output folder │ ← Watched by meta-sort +│ *.srt files │ +└───────────────────┘ + │ + ▼ +┌───────────────────┐ +│ meta-sort │ ← Processes extracted files +│ file watcher │ like any other media +└───────────────────┘ +``` + +## Container Mounts + +| Mount | Access | Purpose | +|-------|--------|---------| +| `/files` | READ-ONLY | Source video files | +| `/cache` | READ-WRITE | Plugin cache | +| `/output` | READ-WRITE | Extracted subtitle output | + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `PORT` | HTTP server port (default: 8080) | +| `WEBDAV_URL` | WebDAV base URL for file access | + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/health` | GET | Health check | +| `/manifest` | GET | Plugin manifest | +| `/configure` | POST | Update configuration | +| `/process` | POST | Process a video file | + +## License + +MIT diff --git a/package.json b/package.json new file mode 100644 index 0000000..26904cd --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "@metazla/plugin-subtitle-extractor", + "version": "1.0.0", + "description": "MetaMesh plugin: Extracts embedded subtitles from video files", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js" + }, + "dependencies": { + "fastify": "^5.0.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.3.0" + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..4a0f9a3 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,82 @@ +/** + * MetaMesh Plugin: subtitle-extractor + * + * ============================================================================ + * PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION + * ============================================================================ + * + * Each plugin container has exactly 3 mounts: + * + * 1. /files (READ-ONLY) - Shared media files, read access only + * 2. /cache (READ-WRITE) - Plugin-specific cache folder + * 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles + * + * SECURITY: Plugins must NEVER write to /files directly. + * All write operations go to /cache or /output only. + * + * ============================================================================ + */ + +import Fastify from 'fastify'; +import type { HealthResponse, ProcessRequest, ProcessResponse, CallbackPayload, ConfigureRequest, ConfigureResponse } from './types.js'; +import { manifest, process as processFile, configure } from './plugin.js'; + +const app = Fastify({ logger: true }); +let ready = false; + +app.get('/health', async (): Promise => ({ + status: 'healthy', + ready, + version: manifest.version +})); + +app.get('/manifest', async () => manifest); + +app.post<{ Body: ConfigureRequest }>('/configure', async (request): Promise => { + try { + configure(request.body.config || {}); + console.log(`[${manifest.id}] Configuration updated`); + return { status: 'ok' }; + } catch (error) { + console.error(`[${manifest.id}] Configuration error:`, error); + return { status: 'error', error: error instanceof Error ? error.message : String(error) }; + } +}); + +app.post<{ Body: ProcessRequest }>('/process', async (request, reply) => { + const { taskId, cid, filePath, callbackUrl, metaCoreUrl } = request.body; + + if (!taskId || !cid || !filePath || !callbackUrl || !metaCoreUrl) { + return reply.send({ status: 'rejected', error: 'Missing required fields' } as ProcessResponse); + } + + // Process asynchronously and send callback when done + processFile(request.body, async (payload: CallbackPayload) => { + try { + await fetch(callbackUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + } catch (error) { + console.error(`[${manifest.id}] Callback error:`, error); + } + }).catch((error) => { + console.error(`[${manifest.id}] Process error:`, error); + }); + + return reply.send({ status: 'accepted' } as ProcessResponse); +}); + +const port = parseInt(process.env.PORT || '8080', 10); + +app.listen({ port, host: '0.0.0.0' }).then(() => { + ready = true; + console.log(`[${manifest.id}] Listening on port ${port}`); +}); + +process.on('SIGTERM', async () => { + ready = false; + await app.close(); + process.exit(0); +}); diff --git a/src/meta-core-client.ts b/src/meta-core-client.ts new file mode 100644 index 0000000..6bdbad8 --- /dev/null +++ b/src/meta-core-client.ts @@ -0,0 +1,93 @@ +/** + * meta-core API Client for writing metadata + * Fails gracefully if meta-core is unavailable (logs warning, returns without throwing) + */ + +export class MetaCoreClient { + constructor(private baseUrl: string) {} + + private async safeFetch(url: string, options: RequestInit): Promise { + try { + return await fetch(url, { ...options, signal: AbortSignal.timeout(5000) }); + } catch (error) { + console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}`); + return null; + } + } + + async setProperty(hashId: string, key: string, value: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ value }), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to set property: ${response.status}`); + } + } + + async getProperty(hashId: string, key: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'GET' }); + if (!response || response.status === 404) return null; + if (!response.ok) return null; + const data = await response.json() as { value?: string }; + return data.value ?? null; + } + + async mergeMetadata(hashId: string, metadata: Record): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(metadata), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to merge metadata: ${response.status}`); + } + } + + async deleteProperty(hashId: string, key: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'DELETE' }); + if (response && !response.ok && response.status !== 404) { + console.warn(`[MetaCoreClient] Failed to delete property: ${response.status}`); + } + } + + async addToSet(hashId: string, key: string, value: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/_add/${key}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ value }), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to add to set: ${response.status}`); + } + } + + async getMetadata(hashId: string): Promise> { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' }); + if (!response || response.status === 404) return {}; + if (!response.ok) return {}; + const data = await response.json() as { metadata?: Record }; + return data.metadata ?? {}; + } + + /** + * Compute the CID (Content Identifier) for a file + * Uses the meta-core /file/cid API endpoint + */ + async computeFileCID(filePath: string): Promise { + try { + const response = await this.safeFetch(`${this.baseUrl}/file/cid`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path: filePath }), + }); + if (!response || !response.ok) return null; + const data = await response.json() as { cid?: string }; + return data.cid ?? null; + } catch (error) { + console.debug(`[MetaCoreClient] Failed to compute CID for ${filePath}:`, error); + return null; + } + } +} diff --git a/src/plugin.ts b/src/plugin.ts new file mode 100644 index 0000000..06c59e0 --- /dev/null +++ b/src/plugin.ts @@ -0,0 +1,453 @@ +/** + * Subtitle Extractor Plugin + * + * Extracts embedded subtitles from video files and saves them as text files. + * Republishes extracted subtitles in the pipeline (similar to TMDB plugin with posters). + * + * ============================================================================ + * PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION + * ============================================================================ + * + * Each plugin container has exactly 3 mounts: + * + * 1. /files (READ-ONLY) - Shared media files, read access only + * 2. /cache (READ-WRITE) - Plugin-specific cache folder + * 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles + * + * SECURITY: Plugins must NEVER write to /files directly. + * - Use /cache for temporary/cache data + * - Use /output for output files (extracted subtitles) + * + * ============================================================================ + */ + +import { spawn } from 'child_process'; +import { existsSync, mkdirSync, statSync, openSync, readSync, closeSync, unlinkSync } from 'fs'; +import { createHash } from 'crypto'; +import * as path from 'path'; +import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; +import { MetaCoreClient } from './meta-core-client.js'; +import { createWebDAVClient, WebDAVClient } from './webdav-client.js'; + +// Initialize WebDAV client if WEBDAV_URL is set +const webdavClient = createWebDAVClient(); +if (webdavClient) { + console.log('[subtitle-extractor] Using WebDAV for file access'); +} else { + console.log('[subtitle-extractor] Using direct filesystem access'); +} + +// Output path for extracted subtitles +const PLUGIN_OUTPUT_PATH = '/output'; + +// Image-based subtitle codecs that cannot be converted to text +const UNSUPPORTED_SUBTITLE_CODECS = new Set([ + 'hdmv_pgs_subtitle', + 'pgssub', + 'dvd_subtitle', + 'dvdsub', + 'dvb_subtitle', + 'dvbsub', + 'xsub', +]); + +// Supported text-based subtitle codecs +const CODEC_EXTENSION_MAP: Record = { + 'subrip': 'srt', + 'srt': 'srt', + 'ass': 'ass', + 'ssa': 'ssa', + 'webvtt': 'vtt', + 'mov_text': 'srt', + 'text': 'srt', +}; + +export const manifest: PluginManifest = { + id: 'subtitle-extractor', + name: 'Subtitle Extractor', + version: '1.0.0', + description: 'Extracts embedded subtitles from video files and saves them as text files', + author: 'MetaMesh', + dependencies: ['file-info', 'ffmpeg'], + priority: 50, + color: '#FF5722', + defaultQueue: 'background', + timeout: 300000, // 5 minutes for large files + schema: { + extractedSubtitles: { label: 'Extracted Subtitles', type: 'array', readonly: true }, + subtitleLanguages: { label: 'Subtitle Languages', type: 'array', readonly: true }, + }, + config: { + forceRecompute: { + type: 'boolean', + label: 'Force Recompute', + default: false, + }, + outputFormat: { + type: 'select', + label: 'Output Format', + default: 'srt', + }, + }, +}; + +// Configuration +let forceRecompute = false; +let outputFormat = 'srt'; + +export function configure(config: Record): void { + forceRecompute = config.forceRecompute === true; + outputFormat = (config.outputFormat as string) || 'srt'; + console.log(`[subtitle-extractor] Config: forceRecompute=${forceRecompute}, outputFormat=${outputFormat}`); +} + +/** + * Compute midhash256 CID for a file (matches meta-hash algorithm) + */ +function computeMidHash256Sync(filePath: string): string { + const SAMPLE_SIZE = 1024 * 1024; // 1MB + const MIDHASH_VARINT = Buffer.from([0x80, 0x20]); + + const stats = statSync(filePath); + const fileSize = stats.size; + + const sizeBuffer = Buffer.allocUnsafe(8); + sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); + + let sampleData: Buffer; + if (fileSize <= SAMPLE_SIZE) { + const fd = openSync(filePath, 'r'); + sampleData = Buffer.allocUnsafe(fileSize); + readSync(fd, sampleData, 0, fileSize, 0); + closeSync(fd); + } else { + const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); + const fd = openSync(filePath, 'r'); + sampleData = Buffer.allocUnsafe(SAMPLE_SIZE); + readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset); + closeSync(fd); + } + + const hashInput = Buffer.concat([sizeBuffer, sampleData]); + const hashBuffer = createHash('sha256').update(hashInput).digest(); + + const cidBytes = Buffer.concat([ + Buffer.from([0x01]), + MIDHASH_VARINT, + MIDHASH_VARINT, + Buffer.from([0x20]), + hashBuffer + ]); + + const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; + let cid = 'b'; + let bits = 0; + let value = 0; + for (const byte of cidBytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + cid += base32Chars[(value >> bits) & 0x1f]; + } + } + if (bits > 0) { + cid += base32Chars[(value << (5 - bits)) & 0x1f]; + } + + return cid; +} + +/** + * Sanitize filename by removing invalid characters + */ +function sanitizeFilename(name: string): string { + return name + .replace(/[<>:"/\\|?*]/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Parse subtitle streams from ffmpeg plugin metadata + */ +interface SubtitleStream { + index: number; + codec: string; + language?: string; + title?: string; +} + +function parseSubtitleStreams(existingMeta: Record): SubtitleStream[] { + const streams: SubtitleStream[] = []; + + // ffmpeg plugin stores streams as JSON in 'streams' field + // or as individual fields like 'subtitle_0_codec', 'subtitle_0_language', etc. + const streamsJson = existingMeta['streams']; + if (streamsJson) { + try { + const allStreams = JSON.parse(streamsJson) as Array<{ + codec_type?: string; + codec_name?: string; + index?: number; + tags?: { language?: string; title?: string }; + }>; + let subtitleIndex = 0; + for (const stream of allStreams) { + if (stream.codec_type === 'subtitle') { + streams.push({ + index: subtitleIndex, + codec: stream.codec_name || 'unknown', + language: stream.tags?.language, + title: stream.tags?.title, + }); + subtitleIndex++; + } + } + } catch { + // Fall through to individual field parsing + } + } + + // Fallback: parse individual fields + if (streams.length === 0) { + for (let i = 0; i < 20; i++) { + const codec = existingMeta[`subtitle_${i}_codec`]; + if (codec) { + streams.push({ + index: i, + codec, + language: existingMeta[`subtitle_${i}_language`], + title: existingMeta[`subtitle_${i}_title`], + }); + } + } + } + + return streams; +} + +/** + * Extract a subtitle track using ffmpeg + */ +async function extractSubtitle( + inputPath: string, + outputPath: string, + subtitleIndex: number, + codec: string +): Promise { + return new Promise((resolve) => { + // Determine output codec based on format + let outputCodec = 'srt'; + if (outputFormat === 'vtt') { + outputCodec = 'webvtt'; + } else if (outputFormat === 'ass') { + outputCodec = 'ass'; + } + + const args = [ + '-y', + '-hide_banner', + '-loglevel', 'error', + '-probesize', '1M', + '-analyzeduration', '1M', + '-i', inputPath, + '-map', `0:s:${subtitleIndex}`, + '-c:s', outputCodec, + outputPath + ]; + + console.log(`[subtitle-extractor] Running ffmpeg for subtitle ${subtitleIndex}`); + + const ffmpeg = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] }); + + let stderr = ''; + ffmpeg.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('close', (code) => { + if (code === 0 && existsSync(outputPath)) { + const size = statSync(outputPath).size; + if (size > 10) { + console.log(`[subtitle-extractor] Extracted subtitle ${subtitleIndex} (${size} bytes)`); + resolve(true); + } else { + console.log(`[subtitle-extractor] Subtitle ${subtitleIndex} extraction produced empty file`); + try { unlinkSync(outputPath); } catch {} + resolve(false); + } + } else { + console.log(`[subtitle-extractor] Failed to extract subtitle ${subtitleIndex}: ${stderr.slice(0, 200)}`); + resolve(false); + } + }); + + ffmpeg.on('error', (err) => { + console.log(`[subtitle-extractor] ffmpeg error: ${err.message}`); + resolve(false); + }); + + // Timeout after 2 minutes per subtitle + setTimeout(() => { + ffmpeg.kill('SIGKILL'); + resolve(false); + }, 120000); + }); +} + +export async function process( + request: ProcessRequest, + sendCallback: (payload: CallbackPayload) => Promise +): Promise { + const startTime = Date.now(); + const metaCore = new MetaCoreClient(request.metaCoreUrl); + + try { + const { cid, filePath, existingMeta } = request; + + // Only process video files + if (existingMeta?.fileType !== 'video') { + await sendCallback({ + taskId: request.taskId, + status: 'skipped', + duration: Date.now() - startTime, + reason: 'Not a video file', + }); + return; + } + + // Skip if already extracted (unless forceRecompute) + if (existingMeta?.extractedSubtitles && !forceRecompute) { + await sendCallback({ + taskId: request.taskId, + status: 'skipped', + duration: Date.now() - startTime, + reason: 'Subtitles already extracted', + }); + return; + } + + // Parse subtitle streams from ffmpeg metadata + const subtitleStreams = parseSubtitleStreams(existingMeta || {}); + + if (subtitleStreams.length === 0) { + await sendCallback({ + taskId: request.taskId, + status: 'skipped', + duration: Date.now() - startTime, + reason: 'No subtitle streams found', + }); + return; + } + + // Filter out unsupported (image-based) codecs + const textSubtitles = subtitleStreams.filter(s => !UNSUPPORTED_SUBTITLE_CODECS.has(s.codec)); + + if (textSubtitles.length === 0) { + console.log(`[subtitle-extractor] All ${subtitleStreams.length} subtitles are image-based, skipping`); + await sendCallback({ + taskId: request.taskId, + status: 'skipped', + duration: Date.now() - startTime, + reason: 'Only image-based subtitles (cannot convert to text)', + }); + return; + } + + console.log(`[subtitle-extractor] Found ${textSubtitles.length} text-based subtitle(s) in ${filePath}`); + + // Get input path (WebDAV URL or filesystem path) + let inputPath = filePath; + if (webdavClient) { + inputPath = webdavClient.toWebDAVUrl(filePath); + console.log(`[subtitle-extractor] Using WebDAV input: ${inputPath}`); + } + + // Ensure output directory exists + if (!existsSync(PLUGIN_OUTPUT_PATH)) { + mkdirSync(PLUGIN_OUTPUT_PATH, { recursive: true }); + } + + // Get video title for output filename + const title = existingMeta?.originalTitle || existingMeta?.title || existingMeta?.fileName || 'video'; + const safeTitle = sanitizeFilename(title); + const year = existingMeta?.movieYear; + const yearStr = year ? ` (${year})` : ''; + + const extractedCids: string[] = []; + const extractedLanguages: string[] = []; + + // Extract each subtitle + for (const sub of textSubtitles) { + const langCode = sub.language || 'und'; + const langSuffix = sub.language ? `.${sub.language}` : `.${sub.index}`; + const ext = outputFormat; + + // Build output filename: Title (Year)[videoCID]_subtitle.lang.srt + const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle${langSuffix}.${ext}`; + const outputPath = path.join(PLUGIN_OUTPUT_PATH, outputFilename); + + // Check if already extracted + if (existsSync(outputPath) && !forceRecompute) { + console.log(`[subtitle-extractor] Subtitle already exists: ${outputFilename}`); + try { + const subtitleCid = computeMidHash256Sync(outputPath); + extractedCids.push(subtitleCid); + if (sub.language && !extractedLanguages.includes(sub.language)) { + extractedLanguages.push(sub.language); + } + } catch (e) { + console.error(`[subtitle-extractor] Failed to compute CID: ${e}`); + } + continue; + } + + // Extract subtitle + const extracted = await extractSubtitle(inputPath, outputPath, sub.index, sub.codec); + + if (extracted) { + try { + // Compute CID for the extracted subtitle file + const subtitleCid = computeMidHash256Sync(outputPath); + console.log(`[subtitle-extractor] Subtitle CID: ${subtitleCid}`); + + extractedCids.push(subtitleCid); + if (sub.language && !extractedLanguages.includes(sub.language)) { + extractedLanguages.push(sub.language); + } + + // Store subtitle CID as metadata on the video + await metaCore.addToSet(cid, 'extractedSubtitles', subtitleCid); + + // Store language if available + if (sub.language) { + await metaCore.addToSet(cid, 'subtitleLanguages', sub.language); + } + } catch (e) { + console.error(`[subtitle-extractor] Failed to process extracted subtitle: ${e}`); + } + } + } + + if (extractedCids.length > 0) { + console.log(`[subtitle-extractor] Extracted ${extractedCids.length} subtitle(s) from ${filePath}`); + } else { + console.log(`[subtitle-extractor] No subtitles could be extracted from ${filePath}`); + } + + await sendCallback({ + taskId: request.taskId, + status: 'completed', + duration: Date.now() - startTime, + }); + } catch (error) { + console.error(`[subtitle-extractor] Error:`, error); + await sendCallback({ + taskId: request.taskId, + status: 'failed', + duration: Date.now() - startTime, + error: error instanceof Error ? error.message : String(error), + }); + } +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..4dbbfe2 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,70 @@ +/** + * MetaMesh Plugin Types + */ + +export interface PluginManifest { + id: string; + name: string; + version: string; + description?: string; + author?: string; + dependencies?: string[]; + schema?: Record; + config?: Record; + priority?: number; + color?: string; + defaultQueue?: 'fast' | 'background'; + timeout?: number; +} + +export interface SchemaField { + label: string; + type?: 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid'; + readonly?: boolean; + hint?: string; +} + +export interface ConfigField { + type: 'string' | 'number' | 'boolean' | 'select'; + label?: string; + required?: boolean; + default?: unknown; + secret?: boolean; +} + +export interface HealthResponse { + status: 'healthy' | 'unhealthy'; + ready: boolean; + version: string; +} + +export interface ProcessRequest { + taskId: string; + cid: string; + filePath: string; + callbackUrl: string; + metaCoreUrl: string; + existingMeta?: Record; +} + +export interface ProcessResponse { + status: 'accepted' | 'rejected'; + error?: string; +} + +export interface CallbackPayload { + taskId: string; + status: 'completed' | 'failed' | 'skipped'; + duration: number; + error?: string; + reason?: string; +} + +export interface ConfigureRequest { + config: Record; +} + +export interface ConfigureResponse { + status: 'ok' | 'error'; + error?: string; +} diff --git a/src/webdav-client.ts b/src/webdav-client.ts new file mode 100644 index 0000000..45c4ebc --- /dev/null +++ b/src/webdav-client.ts @@ -0,0 +1,152 @@ +/** + * WebDAV Client for Plugin File Access + * + * Provides HTTP-based file access to meta-sort's WebDAV endpoint. + * Replaces direct filesystem access for containerized plugins. + */ + +export interface FileStats { + size: number; + mtime?: Date; +} + +export class WebDAVClient { + private baseUrl: string; + + constructor(baseUrl: string) { + // Remove trailing slash if present + this.baseUrl = baseUrl.replace(/\/$/, ''); + } + + /** + * Convert absolute file path to WebDAV URL + * Strips /files prefix and appends to base URL + */ + toWebDAVUrl(filePath: string): string { + // filePath is like "/files/watch/movie.mkv" + // We need to convert to "http://meta-sort-dev/webdav/watch/movie.mkv" + let relativePath = filePath; + + // Strip /files prefix if present + if (relativePath.startsWith('/files/')) { + relativePath = relativePath.substring(6); // Remove "/files" + } else if (relativePath.startsWith('/files')) { + relativePath = relativePath.substring(6); + } + + // Ensure path starts with / + if (!relativePath.startsWith('/')) { + relativePath = '/' + relativePath; + } + + return this.baseUrl + relativePath; + } + + /** + * Get file stats (size, mtime) via HTTP HEAD request + */ + async stat(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'HEAD' }); + + if (!response.ok) { + throw new Error(`WebDAV HEAD failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const contentLength = response.headers.get('content-length'); + const lastModified = response.headers.get('last-modified'); + + return { + size: contentLength ? parseInt(contentLength, 10) : 0, + mtime: lastModified ? new Date(lastModified) : undefined, + }; + } + + /** + * Read bytes from a file via HTTP Range request + */ + async readBytes(filePath: string, start: number, end: number): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { + method: 'GET', + headers: { + 'Range': `bytes=${start}-${end}`, + }, + }); + + if (!response.ok && response.status !== 206) { + throw new Error(`WebDAV Range GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Read entire file as buffer (for small files) + */ + async readFile(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'GET' }); + + if (!response.ok) { + throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Check if file exists via HTTP HEAD + */ + async exists(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + try { + const response = await fetch(url, { method: 'HEAD' }); + return response.ok; + } catch { + return false; + } + } + + /** + * List directory contents via HTTP GET (returns JSON from nginx autoindex) + */ + async readdir(dirPath: string): Promise { + const url = this.toWebDAVUrl(dirPath); + const finalUrl = url.endsWith('/') ? url : url + '/'; + + const response = await fetch(finalUrl, { + method: 'GET', + headers: { + 'Accept': 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`WebDAV GET (dir) failed for ${dirPath}: ${response.status} ${response.statusText}`); + } + + const entries = await response.json() as Array<{ name: string; type: string }>; + return entries.map(e => e.name); + } +} + +/** + * Create a WebDAV client from environment variables + */ +export function createWebDAVClient(): WebDAVClient | null { + const webdavUrl = process.env.WEBDAV_URL; + + if (!webdavUrl) { + console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable'); + return null; + } + + return new WebDAVClient(webdavUrl); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0017704 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}