initial commit
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -0,0 +1,4 @@
|
||||
node_modules/
|
||||
dist/
|
||||
*.log
|
||||
.env
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
FROM node:20-slim AS builder
|
||||
WORKDIR /app
|
||||
COPY package.json tsconfig.json ./
|
||||
RUN npm install
|
||||
COPY src/ ./src/
|
||||
RUN npm run build
|
||||
|
||||
FROM node:20-slim
|
||||
# Install ffmpeg for subtitle extraction
|
||||
RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY package.json ./
|
||||
RUN npm install --omit=dev
|
||||
COPY --from=builder /app/dist ./dist
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))"
|
||||
CMD ["node", "dist/index.js"]
|
||||
@@ -0,0 +1,158 @@
|
||||
# MetaMesh Plugin: Subtitle Extractor
|
||||
|
||||
Extracts embedded subtitles from video files and saves them as standalone text files. The extracted subtitles are republished in the MetaMesh pipeline and linked to the original video via CID.
|
||||
|
||||
## Features
|
||||
|
||||
- **Automatic extraction**: Extracts all text-based subtitle tracks from video files
|
||||
- **Multiple formats**: Supports SRT, ASS, SSA, WebVTT, and MOV_TEXT codecs
|
||||
- **Smart filtering**: Automatically skips image-based subtitles (PGS, DVD, DVB) that cannot be converted to text
|
||||
- **Language preservation**: Maintains language metadata from video streams
|
||||
- **Pipeline integration**: Extracted files are saved to `/output` and automatically picked up by meta-sort
|
||||
- **CID linking**: Stores subtitle CIDs as metadata on the source video
|
||||
|
||||
## Dependencies
|
||||
|
||||
This plugin requires the following plugins to run first:
|
||||
- `file-info` - Determines file type
|
||||
- `ffmpeg` - Provides subtitle stream information
|
||||
|
||||
## Installation
|
||||
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
cd packages/plugins/metamesh-plugin-subtitle-extractor
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
### Build Docker image
|
||||
|
||||
```bash
|
||||
docker build -t metamesh-plugin-subtitle-extractor:main .
|
||||
```
|
||||
|
||||
### Configure in plugins.yml
|
||||
|
||||
Add to `dev/plugins.yml`:
|
||||
|
||||
```yaml
|
||||
plugins:
|
||||
subtitle-extractor:
|
||||
enabled: true
|
||||
image: metamesh-plugin-subtitle-extractor:main
|
||||
instances: 1
|
||||
resources:
|
||||
memory: 256m
|
||||
cpus: 0.5
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
| Option | Type | Default | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| `forceRecompute` | boolean | `false` | Re-extract subtitles even if already processed |
|
||||
| `outputFormat` | select | `srt` | Output format: `srt`, `vtt`, or `ass` |
|
||||
|
||||
## Output
|
||||
|
||||
### Extracted Files
|
||||
|
||||
Subtitle files are saved to `/output` with the naming pattern:
|
||||
|
||||
```
|
||||
{Title} ({Year})[{VideoCID}]_subtitle.{lang}.{ext}
|
||||
```
|
||||
|
||||
Example: `Sintel (2010)[bafk...abc]_subtitle.eng.srt`
|
||||
|
||||
### Metadata
|
||||
|
||||
The plugin stores the following metadata on the source video:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `extractedSubtitles` | array | CIDs of extracted subtitle files |
|
||||
| `subtitleLanguages` | array | Language codes of extracted subtitles |
|
||||
|
||||
## Supported Codecs
|
||||
|
||||
### Text-based (supported)
|
||||
- `subrip` / `srt` - SubRip
|
||||
- `ass` - Advanced SubStation Alpha
|
||||
- `ssa` - SubStation Alpha
|
||||
- `webvtt` - WebVTT
|
||||
- `mov_text` - QuickTime text
|
||||
|
||||
### Image-based (skipped)
|
||||
- `hdmv_pgs_subtitle` - Blu-ray PGS
|
||||
- `dvd_subtitle` - DVD VOB
|
||||
- `dvb_subtitle` - DVB
|
||||
- `xsub` - DivX XSUB
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Video File (MKV/MP4/AVI)
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ ffmpeg plugin │ ← Provides stream info
|
||||
└───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ subtitle-extractor│
|
||||
│ │
|
||||
│ 1. Parse streams │
|
||||
│ 2. Filter codecs │
|
||||
│ 3. Extract via │
|
||||
│ ffmpeg │
|
||||
│ 4. Save to │
|
||||
│ /output │
|
||||
│ 5. Compute CID │
|
||||
│ 6. Store in │
|
||||
│ Redis │
|
||||
└───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ /output folder │ ← Watched by meta-sort
|
||||
│ *.srt files │
|
||||
└───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ meta-sort │ ← Processes extracted files
|
||||
│ file watcher │ like any other media
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## Container Mounts
|
||||
|
||||
| Mount | Access | Purpose |
|
||||
|-------|--------|---------|
|
||||
| `/files` | READ-ONLY | Source video files |
|
||||
| `/cache` | READ-WRITE | Plugin cache |
|
||||
| `/output` | READ-WRITE | Extracted subtitle output |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `PORT` | HTTP server port (default: 8080) |
|
||||
| `WEBDAV_URL` | WebDAV base URL for file access |
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Endpoint | Method | Description |
|
||||
|----------|--------|-------------|
|
||||
| `/health` | GET | Health check |
|
||||
| `/manifest` | GET | Plugin manifest |
|
||||
| `/configure` | POST | Update configuration |
|
||||
| `/process` | POST | Process a video file |
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "@metazla/plugin-subtitle-extractor",
|
||||
"version": "1.0.0",
|
||||
"description": "MetaMesh plugin: Extracts embedded subtitles from video files",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"fastify": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"typescript": "^5.3.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* MetaMesh Plugin: subtitle-extractor
|
||||
*
|
||||
* ============================================================================
|
||||
* PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION
|
||||
* ============================================================================
|
||||
*
|
||||
* Each plugin container has exactly 3 mounts:
|
||||
*
|
||||
* 1. /files (READ-ONLY) - Shared media files, read access only
|
||||
* 2. /cache (READ-WRITE) - Plugin-specific cache folder
|
||||
* 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles
|
||||
*
|
||||
* SECURITY: Plugins must NEVER write to /files directly.
|
||||
* All write operations go to /cache or /output only.
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
import Fastify from 'fastify';
|
||||
import type { HealthResponse, ProcessRequest, ProcessResponse, CallbackPayload, ConfigureRequest, ConfigureResponse } from './types.js';
|
||||
import { manifest, process as processFile, configure } from './plugin.js';
|
||||
|
||||
const app = Fastify({ logger: true });
|
||||
let ready = false;
|
||||
|
||||
app.get('/health', async (): Promise<HealthResponse> => ({
|
||||
status: 'healthy',
|
||||
ready,
|
||||
version: manifest.version
|
||||
}));
|
||||
|
||||
app.get('/manifest', async () => manifest);
|
||||
|
||||
app.post<{ Body: ConfigureRequest }>('/configure', async (request): Promise<ConfigureResponse> => {
|
||||
try {
|
||||
configure(request.body.config || {});
|
||||
console.log(`[${manifest.id}] Configuration updated`);
|
||||
return { status: 'ok' };
|
||||
} catch (error) {
|
||||
console.error(`[${manifest.id}] Configuration error:`, error);
|
||||
return { status: 'error', error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{ Body: ProcessRequest }>('/process', async (request, reply) => {
|
||||
const { taskId, cid, filePath, callbackUrl, metaCoreUrl } = request.body;
|
||||
|
||||
if (!taskId || !cid || !filePath || !callbackUrl || !metaCoreUrl) {
|
||||
return reply.send({ status: 'rejected', error: 'Missing required fields' } as ProcessResponse);
|
||||
}
|
||||
|
||||
// Process asynchronously and send callback when done
|
||||
processFile(request.body, async (payload: CallbackPayload) => {
|
||||
try {
|
||||
await fetch(callbackUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`[${manifest.id}] Callback error:`, error);
|
||||
}
|
||||
}).catch((error) => {
|
||||
console.error(`[${manifest.id}] Process error:`, error);
|
||||
});
|
||||
|
||||
return reply.send({ status: 'accepted' } as ProcessResponse);
|
||||
});
|
||||
|
||||
const port = parseInt(process.env.PORT || '8080', 10);
|
||||
|
||||
app.listen({ port, host: '0.0.0.0' }).then(() => {
|
||||
ready = true;
|
||||
console.log(`[${manifest.id}] Listening on port ${port}`);
|
||||
});
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
ready = false;
|
||||
await app.close();
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* meta-core API Client for writing metadata
|
||||
* Fails gracefully if meta-core is unavailable (logs warning, returns without throwing)
|
||||
*/
|
||||
|
||||
export class MetaCoreClient {
|
||||
constructor(private baseUrl: string) {}
|
||||
|
||||
private async safeFetch(url: string, options: RequestInit): Promise<Response | null> {
|
||||
try {
|
||||
return await fetch(url, { ...options, signal: AbortSignal.timeout(5000) });
|
||||
} catch (error) {
|
||||
console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async setProperty(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to set property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getProperty(hashId: string, key: string): Promise<string | null> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return null;
|
||||
if (!response.ok) return null;
|
||||
const data = await response.json() as { value?: string };
|
||||
return data.value ?? null;
|
||||
}
|
||||
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(metadata),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to merge metadata: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteProperty(hashId: string, key: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'DELETE' });
|
||||
if (response && !response.ok && response.status !== 404) {
|
||||
console.warn(`[MetaCoreClient] Failed to delete property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async addToSet(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/_add/${key}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to add to set: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(hashId: string): Promise<Record<string, string>> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return {};
|
||||
if (!response.ok) return {};
|
||||
const data = await response.json() as { metadata?: Record<string, string> };
|
||||
return data.metadata ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CID (Content Identifier) for a file
|
||||
* Uses the meta-core /file/cid API endpoint
|
||||
*/
|
||||
async computeFileCID(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/file/cid`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ path: filePath }),
|
||||
});
|
||||
if (!response || !response.ok) return null;
|
||||
const data = await response.json() as { cid?: string };
|
||||
return data.cid ?? null;
|
||||
} catch (error) {
|
||||
console.debug(`[MetaCoreClient] Failed to compute CID for ${filePath}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Subtitle Extractor Plugin
|
||||
*
|
||||
* Extracts embedded subtitles from video files and saves them as text files.
|
||||
* Republishes extracted subtitles in the pipeline (similar to TMDB plugin with posters).
|
||||
*
|
||||
* ============================================================================
|
||||
* PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION
|
||||
* ============================================================================
|
||||
*
|
||||
* Each plugin container has exactly 3 mounts:
|
||||
*
|
||||
* 1. /files (READ-ONLY) - Shared media files, read access only
|
||||
* 2. /cache (READ-WRITE) - Plugin-specific cache folder
|
||||
* 3. /output (READ-WRITE) - Plugin output folder for extracted subtitles
|
||||
*
|
||||
* SECURITY: Plugins must NEVER write to /files directly.
|
||||
* - Use /cache for temporary/cache data
|
||||
* - Use /output for output files (extracted subtitles)
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync, mkdirSync, statSync, openSync, readSync, closeSync, unlinkSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import * as path from 'path';
|
||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||
import { MetaCoreClient } from './meta-core-client.js';
|
||||
import { createWebDAVClient, WebDAVClient } from './webdav-client.js';
|
||||
|
||||
// Initialize WebDAV client if WEBDAV_URL is set
|
||||
const webdavClient = createWebDAVClient();
|
||||
if (webdavClient) {
|
||||
console.log('[subtitle-extractor] Using WebDAV for file access');
|
||||
} else {
|
||||
console.log('[subtitle-extractor] Using direct filesystem access');
|
||||
}
|
||||
|
||||
// Output path for extracted subtitles
|
||||
const PLUGIN_OUTPUT_PATH = '/output';
|
||||
|
||||
// Image-based subtitle codecs that cannot be converted to text
|
||||
const UNSUPPORTED_SUBTITLE_CODECS = new Set([
|
||||
'hdmv_pgs_subtitle',
|
||||
'pgssub',
|
||||
'dvd_subtitle',
|
||||
'dvdsub',
|
||||
'dvb_subtitle',
|
||||
'dvbsub',
|
||||
'xsub',
|
||||
]);
|
||||
|
||||
// Supported text-based subtitle codecs
|
||||
const CODEC_EXTENSION_MAP: Record<string, string> = {
|
||||
'subrip': 'srt',
|
||||
'srt': 'srt',
|
||||
'ass': 'ass',
|
||||
'ssa': 'ssa',
|
||||
'webvtt': 'vtt',
|
||||
'mov_text': 'srt',
|
||||
'text': 'srt',
|
||||
};
|
||||
|
||||
export const manifest: PluginManifest = {
|
||||
id: 'subtitle-extractor',
|
||||
name: 'Subtitle Extractor',
|
||||
version: '1.0.0',
|
||||
description: 'Extracts embedded subtitles from video files and saves them as text files',
|
||||
author: 'MetaMesh',
|
||||
dependencies: ['file-info', 'ffmpeg'],
|
||||
priority: 50,
|
||||
color: '#FF5722',
|
||||
defaultQueue: 'background',
|
||||
timeout: 300000, // 5 minutes for large files
|
||||
schema: {
|
||||
extractedSubtitles: { label: 'Extracted Subtitles', type: 'array', readonly: true },
|
||||
subtitleLanguages: { label: 'Subtitle Languages', type: 'array', readonly: true },
|
||||
},
|
||||
config: {
|
||||
forceRecompute: {
|
||||
type: 'boolean',
|
||||
label: 'Force Recompute',
|
||||
default: false,
|
||||
},
|
||||
outputFormat: {
|
||||
type: 'select',
|
||||
label: 'Output Format',
|
||||
default: 'srt',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Configuration
|
||||
let forceRecompute = false;
|
||||
let outputFormat = 'srt';
|
||||
|
||||
export function configure(config: Record<string, unknown>): void {
|
||||
forceRecompute = config.forceRecompute === true;
|
||||
outputFormat = (config.outputFormat as string) || 'srt';
|
||||
console.log(`[subtitle-extractor] Config: forceRecompute=${forceRecompute}, outputFormat=${outputFormat}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute midhash256 CID for a file (matches meta-hash algorithm)
|
||||
*/
|
||||
function computeMidHash256Sync(filePath: string): string {
|
||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
||||
|
||||
const stats = statSync(filePath);
|
||||
const fileSize = stats.size;
|
||||
|
||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||
|
||||
let sampleData: Buffer;
|
||||
if (fileSize <= SAMPLE_SIZE) {
|
||||
const fd = openSync(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(fileSize);
|
||||
readSync(fd, sampleData, 0, fileSize, 0);
|
||||
closeSync(fd);
|
||||
} else {
|
||||
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
||||
const fd = openSync(filePath, 'r');
|
||||
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
|
||||
readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
|
||||
closeSync(fd);
|
||||
}
|
||||
|
||||
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||
const hashBuffer = createHash('sha256').update(hashInput).digest();
|
||||
|
||||
const cidBytes = Buffer.concat([
|
||||
Buffer.from([0x01]),
|
||||
MIDHASH_VARINT,
|
||||
MIDHASH_VARINT,
|
||||
Buffer.from([0x20]),
|
||||
hashBuffer
|
||||
]);
|
||||
|
||||
const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
let cid = 'b';
|
||||
let bits = 0;
|
||||
let value = 0;
|
||||
for (const byte of cidBytes) {
|
||||
value = (value << 8) | byte;
|
||||
bits += 8;
|
||||
while (bits >= 5) {
|
||||
bits -= 5;
|
||||
cid += base32Chars[(value >> bits) & 0x1f];
|
||||
}
|
||||
}
|
||||
if (bits > 0) {
|
||||
cid += base32Chars[(value << (5 - bits)) & 0x1f];
|
||||
}
|
||||
|
||||
return cid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize filename by removing invalid characters
|
||||
*/
|
||||
function sanitizeFilename(name: string): string {
|
||||
return name
|
||||
.replace(/[<>:"/\\|?*]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse subtitle streams from ffmpeg plugin metadata
|
||||
*/
|
||||
interface SubtitleStream {
|
||||
index: number;
|
||||
codec: string;
|
||||
language?: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
function parseSubtitleStreams(existingMeta: Record<string, string>): SubtitleStream[] {
|
||||
const streams: SubtitleStream[] = [];
|
||||
|
||||
// ffmpeg plugin stores streams as JSON in 'streams' field
|
||||
// or as individual fields like 'subtitle_0_codec', 'subtitle_0_language', etc.
|
||||
const streamsJson = existingMeta['streams'];
|
||||
if (streamsJson) {
|
||||
try {
|
||||
const allStreams = JSON.parse(streamsJson) as Array<{
|
||||
codec_type?: string;
|
||||
codec_name?: string;
|
||||
index?: number;
|
||||
tags?: { language?: string; title?: string };
|
||||
}>;
|
||||
let subtitleIndex = 0;
|
||||
for (const stream of allStreams) {
|
||||
if (stream.codec_type === 'subtitle') {
|
||||
streams.push({
|
||||
index: subtitleIndex,
|
||||
codec: stream.codec_name || 'unknown',
|
||||
language: stream.tags?.language,
|
||||
title: stream.tags?.title,
|
||||
});
|
||||
subtitleIndex++;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fall through to individual field parsing
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: parse individual fields
|
||||
if (streams.length === 0) {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const codec = existingMeta[`subtitle_${i}_codec`];
|
||||
if (codec) {
|
||||
streams.push({
|
||||
index: i,
|
||||
codec,
|
||||
language: existingMeta[`subtitle_${i}_language`],
|
||||
title: existingMeta[`subtitle_${i}_title`],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return streams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a subtitle track using ffmpeg
|
||||
*/
|
||||
async function extractSubtitle(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
subtitleIndex: number,
|
||||
codec: string
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
// Determine output codec based on format
|
||||
let outputCodec = 'srt';
|
||||
if (outputFormat === 'vtt') {
|
||||
outputCodec = 'webvtt';
|
||||
} else if (outputFormat === 'ass') {
|
||||
outputCodec = 'ass';
|
||||
}
|
||||
|
||||
const args = [
|
||||
'-y',
|
||||
'-hide_banner',
|
||||
'-loglevel', 'error',
|
||||
'-probesize', '1M',
|
||||
'-analyzeduration', '1M',
|
||||
'-i', inputPath,
|
||||
'-map', `0:s:${subtitleIndex}`,
|
||||
'-c:s', outputCodec,
|
||||
outputPath
|
||||
];
|
||||
|
||||
console.log(`[subtitle-extractor] Running ffmpeg for subtitle ${subtitleIndex}`);
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
|
||||
let stderr = '';
|
||||
ffmpeg.stderr?.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0 && existsSync(outputPath)) {
|
||||
const size = statSync(outputPath).size;
|
||||
if (size > 10) {
|
||||
console.log(`[subtitle-extractor] Extracted subtitle ${subtitleIndex} (${size} bytes)`);
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] Subtitle ${subtitleIndex} extraction produced empty file`);
|
||||
try { unlinkSync(outputPath); } catch {}
|
||||
resolve(false);
|
||||
}
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] Failed to extract subtitle ${subtitleIndex}: ${stderr.slice(0, 200)}`);
|
||||
resolve(false);
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
console.log(`[subtitle-extractor] ffmpeg error: ${err.message}`);
|
||||
resolve(false);
|
||||
});
|
||||
|
||||
// Timeout after 2 minutes per subtitle
|
||||
setTimeout(() => {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
resolve(false);
|
||||
}, 120000);
|
||||
});
|
||||
}
|
||||
|
||||
export async function process(
|
||||
request: ProcessRequest,
|
||||
sendCallback: (payload: CallbackPayload) => Promise<void>
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||
|
||||
try {
|
||||
const { cid, filePath, existingMeta } = request;
|
||||
|
||||
// Only process video files
|
||||
if (existingMeta?.fileType !== 'video') {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Not a video file',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip if already extracted (unless forceRecompute)
|
||||
if (existingMeta?.extractedSubtitles && !forceRecompute) {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Subtitles already extracted',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse subtitle streams from ffmpeg metadata
|
||||
const subtitleStreams = parseSubtitleStreams(existingMeta || {});
|
||||
|
||||
if (subtitleStreams.length === 0) {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'No subtitle streams found',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter out unsupported (image-based) codecs
|
||||
const textSubtitles = subtitleStreams.filter(s => !UNSUPPORTED_SUBTITLE_CODECS.has(s.codec));
|
||||
|
||||
if (textSubtitles.length === 0) {
|
||||
console.log(`[subtitle-extractor] All ${subtitleStreams.length} subtitles are image-based, skipping`);
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Only image-based subtitles (cannot convert to text)',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[subtitle-extractor] Found ${textSubtitles.length} text-based subtitle(s) in ${filePath}`);
|
||||
|
||||
// Get input path (WebDAV URL or filesystem path)
|
||||
let inputPath = filePath;
|
||||
if (webdavClient) {
|
||||
inputPath = webdavClient.toWebDAVUrl(filePath);
|
||||
console.log(`[subtitle-extractor] Using WebDAV input: ${inputPath}`);
|
||||
}
|
||||
|
||||
// Ensure output directory exists
|
||||
if (!existsSync(PLUGIN_OUTPUT_PATH)) {
|
||||
mkdirSync(PLUGIN_OUTPUT_PATH, { recursive: true });
|
||||
}
|
||||
|
||||
// Get video title for output filename
|
||||
const title = existingMeta?.originalTitle || existingMeta?.title || existingMeta?.fileName || 'video';
|
||||
const safeTitle = sanitizeFilename(title);
|
||||
const year = existingMeta?.movieYear;
|
||||
const yearStr = year ? ` (${year})` : '';
|
||||
|
||||
const extractedCids: string[] = [];
|
||||
const extractedLanguages: string[] = [];
|
||||
|
||||
// Extract each subtitle
|
||||
for (const sub of textSubtitles) {
|
||||
const langCode = sub.language || 'und';
|
||||
const langSuffix = sub.language ? `.${sub.language}` : `.${sub.index}`;
|
||||
const ext = outputFormat;
|
||||
|
||||
// Build output filename: Title (Year)[videoCID]_subtitle.lang.srt
|
||||
const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle${langSuffix}.${ext}`;
|
||||
const outputPath = path.join(PLUGIN_OUTPUT_PATH, outputFilename);
|
||||
|
||||
// Check if already extracted
|
||||
if (existsSync(outputPath) && !forceRecompute) {
|
||||
console.log(`[subtitle-extractor] Subtitle already exists: ${outputFilename}`);
|
||||
try {
|
||||
const subtitleCid = computeMidHash256Sync(outputPath);
|
||||
extractedCids.push(subtitleCid);
|
||||
if (sub.language && !extractedLanguages.includes(sub.language)) {
|
||||
extractedLanguages.push(sub.language);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[subtitle-extractor] Failed to compute CID: ${e}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract subtitle
|
||||
const extracted = await extractSubtitle(inputPath, outputPath, sub.index, sub.codec);
|
||||
|
||||
if (extracted) {
|
||||
try {
|
||||
// Compute CID for the extracted subtitle file
|
||||
const subtitleCid = computeMidHash256Sync(outputPath);
|
||||
console.log(`[subtitle-extractor] Subtitle CID: ${subtitleCid}`);
|
||||
|
||||
extractedCids.push(subtitleCid);
|
||||
if (sub.language && !extractedLanguages.includes(sub.language)) {
|
||||
extractedLanguages.push(sub.language);
|
||||
}
|
||||
|
||||
// Store subtitle CID as metadata on the video
|
||||
await metaCore.addToSet(cid, 'extractedSubtitles', subtitleCid);
|
||||
|
||||
// Store language if available
|
||||
if (sub.language) {
|
||||
await metaCore.addToSet(cid, 'subtitleLanguages', sub.language);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[subtitle-extractor] Failed to process extracted subtitle: ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (extractedCids.length > 0) {
|
||||
console.log(`[subtitle-extractor] Extracted ${extractedCids.length} subtitle(s) from ${filePath}`);
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] No subtitles could be extracted from ${filePath}`);
|
||||
}
|
||||
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'completed',
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`[subtitle-extractor] Error:`, error);
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'failed',
|
||||
duration: Date.now() - startTime,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* MetaMesh Plugin Types
|
||||
*/
|
||||
|
||||
export interface PluginManifest {
|
||||
id: string;
|
||||
name: string;
|
||||
version: string;
|
||||
description?: string;
|
||||
author?: string;
|
||||
dependencies?: string[];
|
||||
schema?: Record<string, SchemaField>;
|
||||
config?: Record<string, ConfigField>;
|
||||
priority?: number;
|
||||
color?: string;
|
||||
defaultQueue?: 'fast' | 'background';
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface SchemaField {
|
||||
label: string;
|
||||
type?: 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid';
|
||||
readonly?: boolean;
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
export interface ConfigField {
|
||||
type: 'string' | 'number' | 'boolean' | 'select';
|
||||
label?: string;
|
||||
required?: boolean;
|
||||
default?: unknown;
|
||||
secret?: boolean;
|
||||
}
|
||||
|
||||
export interface HealthResponse {
|
||||
status: 'healthy' | 'unhealthy';
|
||||
ready: boolean;
|
||||
version: string;
|
||||
}
|
||||
|
||||
export interface ProcessRequest {
|
||||
taskId: string;
|
||||
cid: string;
|
||||
filePath: string;
|
||||
callbackUrl: string;
|
||||
metaCoreUrl: string;
|
||||
existingMeta?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ProcessResponse {
|
||||
status: 'accepted' | 'rejected';
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface CallbackPayload {
|
||||
taskId: string;
|
||||
status: 'completed' | 'failed' | 'skipped';
|
||||
duration: number;
|
||||
error?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface ConfigureRequest {
|
||||
config: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ConfigureResponse {
|
||||
status: 'ok' | 'error';
|
||||
error?: string;
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* WebDAV Client for Plugin File Access
|
||||
*
|
||||
* Provides HTTP-based file access to meta-sort's WebDAV endpoint.
|
||||
* Replaces direct filesystem access for containerized plugins.
|
||||
*/
|
||||
|
||||
export interface FileStats {
|
||||
size: number;
|
||||
mtime?: Date;
|
||||
}
|
||||
|
||||
export class WebDAVClient {
|
||||
private baseUrl: string;
|
||||
|
||||
constructor(baseUrl: string) {
|
||||
// Remove trailing slash if present
|
||||
this.baseUrl = baseUrl.replace(/\/$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert absolute file path to WebDAV URL
|
||||
* Strips /files prefix and appends to base URL
|
||||
*/
|
||||
toWebDAVUrl(filePath: string): string {
|
||||
// filePath is like "/files/watch/movie.mkv"
|
||||
// We need to convert to "http://meta-sort-dev/webdav/watch/movie.mkv"
|
||||
let relativePath = filePath;
|
||||
|
||||
// Strip /files prefix if present
|
||||
if (relativePath.startsWith('/files/')) {
|
||||
relativePath = relativePath.substring(6); // Remove "/files"
|
||||
} else if (relativePath.startsWith('/files')) {
|
||||
relativePath = relativePath.substring(6);
|
||||
}
|
||||
|
||||
// Ensure path starts with /
|
||||
if (!relativePath.startsWith('/')) {
|
||||
relativePath = '/' + relativePath;
|
||||
}
|
||||
|
||||
return this.baseUrl + relativePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file stats (size, mtime) via HTTP HEAD request
|
||||
*/
|
||||
async stat(filePath: string): Promise<FileStats> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
const response = await fetch(url, { method: 'HEAD' });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV HEAD failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const lastModified = response.headers.get('last-modified');
|
||||
|
||||
return {
|
||||
size: contentLength ? parseInt(contentLength, 10) : 0,
|
||||
mtime: lastModified ? new Date(lastModified) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read bytes from a file via HTTP Range request
|
||||
*/
|
||||
async readBytes(filePath: string, start: number, end: number): Promise<Buffer> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Range': `bytes=${start}-${end}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok && response.status !== 206) {
|
||||
throw new Error(`WebDAV Range GET failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(arrayBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read entire file as buffer (for small files)
|
||||
*/
|
||||
async readFile(filePath: string): Promise<Buffer> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
const response = await fetch(url, { method: 'GET' });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(arrayBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file exists via HTTP HEAD
|
||||
*/
|
||||
async exists(filePath: string): Promise<boolean> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, { method: 'HEAD' });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List directory contents via HTTP GET (returns JSON from nginx autoindex)
|
||||
*/
|
||||
async readdir(dirPath: string): Promise<string[]> {
|
||||
const url = this.toWebDAVUrl(dirPath);
|
||||
const finalUrl = url.endsWith('/') ? url : url + '/';
|
||||
|
||||
const response = await fetch(finalUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV GET (dir) failed for ${dirPath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const entries = await response.json() as Array<{ name: string; type: string }>;
|
||||
return entries.map(e => e.name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a WebDAV client from environment variables
|
||||
*/
|
||||
export function createWebDAVClient(): WebDAVClient | null {
|
||||
const webdavUrl = process.env.WEBDAV_URL;
|
||||
|
||||
if (!webdavUrl) {
|
||||
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable');
|
||||
return null;
|
||||
}
|
||||
|
||||
return new WebDAVClient(webdavUrl);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"declaration": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user