This commit is contained in:
Worph
+170
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Language code normalisation — the `lang3` vocabulary meta-watch reads.
|
||||
*
|
||||
* Port of meta-watch's `MW_LANG.toLang3` (packages/meta-watch/ui/js/lang-codes.js),
|
||||
* the same normaliser the `language` plugin copied. Every spelling folds to
|
||||
* **ISO 639-2/B** — `fre` not `fra`, `ger` not `deu`, `chi` not `zho` — because
|
||||
* that is what meta-watch writes (`subtitles/fre/<cid>`, routes/subtitles.rs)
|
||||
* and what its readers compare against:
|
||||
*
|
||||
* ISO 639-1 "fr" -> fre
|
||||
* ISO 639-2/B "fre" -> fre
|
||||
* ISO 639-2/T "fra" -> fre (MP4 `mdhd` speaks T, Matroska speaks B)
|
||||
* BCP-47 "pt-BR" -> por (primary subtag only)
|
||||
*
|
||||
* One deliberate difference: "no language" is `und`, not "" — the code becomes a
|
||||
* key segment (`subtitles/und/<cid>`, METADATA_KEYS.md §8) and `und` is the
|
||||
* registry's spelling for undetermined.
|
||||
*
|
||||
* Copied, not imported: plugins are independent submodules. This file is
|
||||
* byte-identical in metamesh-plugin-subtitle and metamesh-plugin-subtitle-extractor.
|
||||
*/
|
||||
|
||||
// Canonical table keyed by 639-2/B: display name (lowercased for filename
|
||||
// matching) and the 639-1 code that folds into it.
|
||||
const LANGS: Record<string, { name: string; a1?: string }> = {
|
||||
eng: { name: 'english', a1: 'en' },
|
||||
fre: { name: 'french', a1: 'fr' },
|
||||
ger: { name: 'german', a1: 'de' },
|
||||
spa: { name: 'spanish', a1: 'es' },
|
||||
ita: { name: 'italian', a1: 'it' },
|
||||
por: { name: 'portuguese', a1: 'pt' },
|
||||
dut: { name: 'dutch', a1: 'nl' },
|
||||
swe: { name: 'swedish', a1: 'sv' },
|
||||
nor: { name: 'norwegian', a1: 'no' },
|
||||
dan: { name: 'danish', a1: 'da' },
|
||||
fin: { name: 'finnish', a1: 'fi' },
|
||||
ice: { name: 'icelandic', a1: 'is' },
|
||||
pol: { name: 'polish', a1: 'pl' },
|
||||
cze: { name: 'czech', a1: 'cs' },
|
||||
slo: { name: 'slovak', a1: 'sk' },
|
||||
slv: { name: 'slovenian', a1: 'sl' },
|
||||
hun: { name: 'hungarian', a1: 'hu' },
|
||||
rum: { name: 'romanian', a1: 'ro' },
|
||||
bul: { name: 'bulgarian', a1: 'bg' },
|
||||
rus: { name: 'russian', a1: 'ru' },
|
||||
ukr: { name: 'ukrainian', a1: 'uk' },
|
||||
bel: { name: 'belarusian', a1: 'be' },
|
||||
srp: { name: 'serbian', a1: 'sr' },
|
||||
hrv: { name: 'croatian', a1: 'hr' },
|
||||
bos: { name: 'bosnian', a1: 'bs' },
|
||||
mac: { name: 'macedonian', a1: 'mk' },
|
||||
alb: { name: 'albanian', a1: 'sq' },
|
||||
gre: { name: 'greek', a1: 'el' },
|
||||
tur: { name: 'turkish', a1: 'tr' },
|
||||
heb: { name: 'hebrew', a1: 'he' },
|
||||
ara: { name: 'arabic', a1: 'ar' },
|
||||
per: { name: 'persian', a1: 'fa' },
|
||||
urd: { name: 'urdu', a1: 'ur' },
|
||||
hin: { name: 'hindi', a1: 'hi' },
|
||||
ben: { name: 'bengali', a1: 'bn' },
|
||||
tam: { name: 'tamil', a1: 'ta' },
|
||||
tel: { name: 'telugu', a1: 'te' },
|
||||
mal: { name: 'malayalam', a1: 'ml' },
|
||||
kan: { name: 'kannada', a1: 'kn' },
|
||||
mar: { name: 'marathi', a1: 'mr' },
|
||||
guj: { name: 'gujarati', a1: 'gu' },
|
||||
pan: { name: 'punjabi', a1: 'pa' },
|
||||
nep: { name: 'nepali', a1: 'ne' },
|
||||
sin: { name: 'sinhala', a1: 'si' },
|
||||
tha: { name: 'thai', a1: 'th' },
|
||||
lao: { name: 'lao', a1: 'lo' },
|
||||
khm: { name: 'khmer', a1: 'km' },
|
||||
bur: { name: 'burmese', a1: 'my' },
|
||||
vie: { name: 'vietnamese', a1: 'vi' },
|
||||
ind: { name: 'indonesian', a1: 'id' },
|
||||
may: { name: 'malay', a1: 'ms' },
|
||||
tgl: { name: 'tagalog', a1: 'tl' },
|
||||
fil: { name: 'filipino' },
|
||||
jpn: { name: 'japanese', a1: 'ja' },
|
||||
kor: { name: 'korean', a1: 'ko' },
|
||||
chi: { name: 'chinese', a1: 'zh' },
|
||||
mon: { name: 'mongolian', a1: 'mn' },
|
||||
kaz: { name: 'kazakh', a1: 'kk' },
|
||||
uzb: { name: 'uzbek', a1: 'uz' },
|
||||
aze: { name: 'azerbaijani', a1: 'az' },
|
||||
geo: { name: 'georgian', a1: 'ka' },
|
||||
arm: { name: 'armenian', a1: 'hy' },
|
||||
est: { name: 'estonian', a1: 'et' },
|
||||
lav: { name: 'latvian', a1: 'lv' },
|
||||
lit: { name: 'lithuanian', a1: 'lt' },
|
||||
cat: { name: 'catalan', a1: 'ca' },
|
||||
glg: { name: 'galician', a1: 'gl' },
|
||||
baq: { name: 'basque', a1: 'eu' },
|
||||
wel: { name: 'welsh', a1: 'cy' },
|
||||
gle: { name: 'irish', a1: 'ga' },
|
||||
afr: { name: 'afrikaans', a1: 'af' },
|
||||
swa: { name: 'swahili', a1: 'sw' },
|
||||
amh: { name: 'amharic', a1: 'am' },
|
||||
zul: { name: 'zulu', a1: 'zu' },
|
||||
lat: { name: 'latin', a1: 'la' },
|
||||
epo: { name: 'esperanto', a1: 'eo' },
|
||||
yid: { name: 'yiddish', a1: 'yi' },
|
||||
jav: { name: 'javanese', a1: 'jv' },
|
||||
mul: { name: 'multiple' },
|
||||
};
|
||||
|
||||
// ISO 639-2/T -> 639-2/B: the only codes where the two standards disagree.
|
||||
const T2B: Record<string, string> = {
|
||||
sqi: 'alb', hye: 'arm', eus: 'baq', mya: 'bur', ces: 'cze',
|
||||
zho: 'chi', cym: 'wel', deu: 'ger', nld: 'dut', ell: 'gre', fas: 'per',
|
||||
fra: 'fre', kat: 'geo', isl: 'ice', mkd: 'mac', msa: 'may',
|
||||
ron: 'rum', slk: 'slo',
|
||||
};
|
||||
|
||||
// "This track has no language" spellings.
|
||||
const NONE = new Set(['und', 'zxx', 'mis', 'unknown', 'none']);
|
||||
|
||||
// 639-1 -> 639-2/B, from the table plus legacy two-letter codes.
|
||||
const A1: Record<string, string> = { nb: 'nor', nn: 'nor', iw: 'heb', in: 'ind', ji: 'yid', jw: 'jav' };
|
||||
const NAMES: Record<string, string> = {};
|
||||
for (const [b, entry] of Object.entries(LANGS)) {
|
||||
if (entry.a1) A1[entry.a1] = b;
|
||||
NAMES[entry.name] = b;
|
||||
}
|
||||
|
||||
/**
|
||||
* franc-min answers in ISO 639-3 individual-language codes; a few of them are
|
||||
* not 639-2 codes at all (Mandarin `cmn` is 639-2 `chi`). Folded before toLang3.
|
||||
*/
|
||||
const FRANC_TO_639_2: Record<string, string> = {
|
||||
cmn: 'chi', arb: 'ara', pes: 'per', swh: 'swa', uzn: 'uzb', zlm: 'may',
|
||||
npi: 'nep', azj: 'aze', plt: 'mlg', pbu: 'pus', ckb: 'kur', fuv: 'ful', qug: 'que',
|
||||
};
|
||||
|
||||
/** Canonical lang3 (ISO 639-2/B) for any code; `und` when there is none. */
|
||||
export function toLang3(code: unknown): string {
|
||||
const s = String(code ?? '').trim().toLowerCase();
|
||||
if (!s) return 'und';
|
||||
if (NAMES[s]) return NAMES[s];
|
||||
const p = s.split(/[-_]/)[0];
|
||||
if (NONE.has(p)) return 'und';
|
||||
if (p.length === 2) return A1[p] ?? 'und';
|
||||
if (p.length === 3 && /^[a-z]{3}$/.test(p)) return T2B[p] ?? p;
|
||||
return 'und';
|
||||
}
|
||||
|
||||
/** lang3 for a franc-min result (`und` stays `und`). */
|
||||
export function francToLang3(code: string): string {
|
||||
return toLang3(FRANC_TO_639_2[code] ?? code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict variant for filename tokens (`video.fr.srt`, `video.English.srt`):
|
||||
* only a token that is recognisably a language resolves, so `S01`, `x264` or
|
||||
* `Extended` never read as one. Returns null for anything else.
|
||||
*/
|
||||
export function languageToken(token: string): string | null {
|
||||
const t = token.trim().toLowerCase();
|
||||
if (!t) return null;
|
||||
if (NAMES[t]) return NAMES[t];
|
||||
if (t === 'und') return 'und';
|
||||
const p = t.split(/[-_]/)[0];
|
||||
if (p !== t && !/^[a-z]{2,3}[-_][a-z0-9]{2,8}$/.test(t)) return null;
|
||||
if (p.length === 2) return A1[p] ?? null;
|
||||
if (p.length === 3) {
|
||||
if (LANGS[p]) return p;
|
||||
if (T2B[p]) return T2B[p];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
+39
-64
@@ -1,6 +1,10 @@
|
||||
/**
|
||||
* meta-core API Client for writing metadata
|
||||
* Fails gracefully if meta-core is unavailable (logs warning, returns without throwing)
|
||||
* meta-core API client for writing metadata.
|
||||
*
|
||||
* Writes report success so a failed write fails the task instead of being
|
||||
* swallowed. Key-sets are written as individual `prefix/member = "true"`
|
||||
* fields through PATCH — never via `_add`, which stores one comma-joined
|
||||
* string and can never produce the key-set shape.
|
||||
*/
|
||||
|
||||
export class MetaCoreClient {
|
||||
@@ -8,86 +12,57 @@ export class MetaCoreClient {
|
||||
|
||||
private async safeFetch(url: string, options: RequestInit): Promise<Response | null> {
|
||||
try {
|
||||
return await fetch(url, { ...options, signal: AbortSignal.timeout(5000) });
|
||||
return await fetch(url, { ...options, signal: AbortSignal.timeout(10000) });
|
||||
} catch (error) {
|
||||
console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}`);
|
||||
console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async setProperty(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to set property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getProperty(hashId: string, key: string): Promise<string | null> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return null;
|
||||
if (!response.ok) return null;
|
||||
const data = await response.json() as { value?: string };
|
||||
return data.value ?? null;
|
||||
if (!response || !response.ok) return null;
|
||||
const data = await response.json() as { value?: unknown };
|
||||
return data.value === undefined || data.value === null ? null : String(data.value);
|
||||
}
|
||||
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<void> {
|
||||
/**
|
||||
* Whole record, in whatever form the endpoint serves (meta-sort's /meta is
|
||||
* nested, meta-core's is flat) — callers flatten. `null` when the call
|
||||
* itself failed, `{}` when the record does not exist.
|
||||
*/
|
||||
async getMetadata(hashId: string): Promise<Record<string, unknown> | null> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' });
|
||||
if (!response) return null;
|
||||
if (response.status === 404) return {};
|
||||
if (!response.ok) return null;
|
||||
const data = await response.json() as { metadata?: Record<string, unknown> };
|
||||
return data.metadata ?? {};
|
||||
}
|
||||
|
||||
/** PATCH — merge `metadata` into the record. True on success. */
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<boolean> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(metadata),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to merge metadata: ${response.status}`);
|
||||
if (!response) return false;
|
||||
if (!response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to merge metadata into ${hashId}: ${response.status}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteProperty(hashId: string, key: string): Promise<void> {
|
||||
/** DELETE one field. A missing field counts as deleted. */
|
||||
async deleteProperty(hashId: string, key: string): Promise<boolean> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'DELETE' });
|
||||
if (response && !response.ok && response.status !== 404) {
|
||||
console.warn(`[MetaCoreClient] Failed to delete property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async addToSet(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/_add/${key}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to add to set: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(hashId: string): Promise<Record<string, string>> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return {};
|
||||
if (!response.ok) return {};
|
||||
const data = await response.json() as { metadata?: Record<string, string> };
|
||||
return data.metadata ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CID (Content Identifier) for a file
|
||||
* Uses the meta-core /file/cid API endpoint
|
||||
*/
|
||||
async computeFileCID(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/file/cid`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ path: filePath }),
|
||||
});
|
||||
if (!response || !response.ok) return null;
|
||||
const data = await response.json() as { cid?: string };
|
||||
return data.cid ?? null;
|
||||
} catch (error) {
|
||||
console.debug(`[MetaCoreClient] Failed to compute CID for ${filePath}:`, error);
|
||||
return null;
|
||||
if (!response) return false;
|
||||
if (!response.ok && response.status !== 404) {
|
||||
console.warn(`[MetaCoreClient] Failed to delete ${hashId}/${key}: ${response.status}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Reading a record regardless of the shape it arrives in, and writing only what
|
||||
* is new.
|
||||
*
|
||||
* meta-sort's scheduler fills `existingMeta` from meta-sort's own GET /meta/{hash},
|
||||
* which returns the NESTED document (`fileinfo: {duration: 1433.089}`,
|
||||
* `stream: [...]`, a key-set as `subtitles: {fre: {<cid>: true}}`). meta-core's
|
||||
* backend serves the FLAT form (`fileinfo/duration`, `stream/0`,
|
||||
* `subtitles/fre/<cid>`). A plugin that reads `existingMeta['x/y']` silently sees
|
||||
* nothing in production — so everything here goes through `flattenMeta`.
|
||||
*
|
||||
* Byte-identical in metamesh-plugin-subtitle and metamesh-plugin-subtitle-extractor.
|
||||
*/
|
||||
|
||||
/** Flatten either form into `a/b/c -> string` (meta-sort's flattenMetadata, plus array indices). */
|
||||
export function flattenMeta(meta: Record<string, unknown> | undefined | null): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const walk = (value: unknown, key: string) => {
|
||||
if (value === null || value === undefined) return;
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((v, i) => walk(v, key ? `${key}/${i}` : String(i)));
|
||||
} else if (typeof value === 'object') {
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
||||
walk(v, key ? `${key}/${k}` : k);
|
||||
}
|
||||
} else if (key) {
|
||||
out[key] = String(value);
|
||||
}
|
||||
};
|
||||
walk(meta ?? {}, '');
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The subset of `wanted` the record does not already hold with that exact value. */
|
||||
export function onlyNewKeys(existingFlat: Record<string, string>, wanted: Record<string, string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(wanted)) {
|
||||
if (existingFlat[k] !== v) out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Members of a LEGACY scalar csv-set (`subtitles = "cidA,cidB"`, written by the
|
||||
* old `_add` path), or null when the field is absent or already a key-set.
|
||||
*
|
||||
* Such a scalar must be deleted before any `field/...` key is written: meta-sort
|
||||
* rebuilds its nested document by walking the `/`-segments, and a string sitting
|
||||
* where an object is needed either throws or silently hides the key-set.
|
||||
*/
|
||||
export function legacyCsvMembers(meta: Record<string, unknown> | undefined | null, field: string): string[] | null {
|
||||
const v = meta?.[field];
|
||||
if (typeof v !== 'string' && typeof v !== 'number') return null;
|
||||
return String(v).split(/[,|]/).map((s) => s.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
/** Every `<prefix>/<lang>/<cid>` leaf for `cid` whose language is not `lang3`. */
|
||||
export function staleLanguageLeaves(existingFlat: Record<string, string>, prefix: string, cid: string, lang3: string): string[] {
|
||||
const stale: string[] = [];
|
||||
for (const key of Object.keys(existingFlat)) {
|
||||
const parts = key.split('/');
|
||||
if (parts.length === 3 && parts[0] === prefix && parts[2] === cid && parts[1] !== lang3) stale.push(key);
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
+508
-343
@@ -1,125 +1,155 @@
|
||||
/**
|
||||
* Subtitle Extractor Plugin
|
||||
*
|
||||
* Extracts embedded subtitles from video files and saves them as text files.
|
||||
* Republishes extracted subtitles in the pipeline (similar to TMDB plugin with posters).
|
||||
* Extracts a video's embedded TEXT subtitle streams into standalone files under
|
||||
* /files/plugin/subtitle-extractor/ and links them to the video so meta-watch can
|
||||
* offer them.
|
||||
*
|
||||
* ============================================================================
|
||||
* PLUGIN FILE ACCESS ARCHITECTURE - WebDAV
|
||||
* WHAT IT WRITES (METADATA_KEYS.md §8 / §9)
|
||||
* ============================================================================
|
||||
* On the video record, per extracted file `<subCid>` (midhash256 of its bytes)
|
||||
* in language `<lang3>` (ISO 639-2/B, `und` when the stream declares none):
|
||||
* subtitles/<lang3>/<subCid> = "true" what meta-watch lists
|
||||
* extractedSubtitles/<lang3>/<subCid> = "true" provenance: came out of the container
|
||||
* subtitleLanguages/<lang3> = "true" (not for `und`)
|
||||
* languages/<lang3> = "true" (not for `und`)
|
||||
* On the subtitle file's own record:
|
||||
* videos/<videoCid> = "true"
|
||||
* subtitleLanguage = <lang3> (not for `und`; the `subtitle`
|
||||
* plugin's text sniff fills it)
|
||||
* Writes are one PATCH carrying only keys the record does not already hold.
|
||||
* Legacy scalar csv-sets from the old `_add` path (`extractedSubtitles`,
|
||||
* `subtitleLanguages`, `subtitles`) are deleted first; see meta-shape.ts.
|
||||
*
|
||||
* File access via WebDAV (WEBDAV_URL environment variable):
|
||||
* - Read media files: GET /webdav/watch/... or /webdav/test/...
|
||||
* - Write output: PUT /webdav/plugin/subtitle-extractor/...
|
||||
* - Cache: Local /cache mount (for temporary files during extraction)
|
||||
*
|
||||
* Benefits:
|
||||
* - No need for /files or /output volume mounts
|
||||
* - Works across multiple hosts
|
||||
* - Plugins are fully isolated
|
||||
* Image codecs (PGS, VobSub, DVB) are skipped: turning them into text is OCR.
|
||||
*
|
||||
* ============================================================================
|
||||
* READ COST
|
||||
* ============================================================================
|
||||
* Subtitle packets are interleaved through the whole container, so extracting
|
||||
* even one track means demuxing the file end to end — over WebDAV, one full
|
||||
* sequential read of the video. All text tracks therefore come out of ONE ffmpeg
|
||||
* pass with one output per track (one read, not N). A per-track retry happens
|
||||
* only when that pass fails outright (one broken track aborts every output).
|
||||
*
|
||||
* ============================================================================
|
||||
* FILE ACCESS - WebDAV
|
||||
* ============================================================================
|
||||
* The WebDAV endpoint is resolved per request from the meta-core that drove the
|
||||
* /process call (its /urls -> webdavUrlInternal; WEBDAV_URL overrides), so the
|
||||
* bytes land in the same core the CIDs are written to.
|
||||
* - Read media: GET /webdav/watch/...
|
||||
* - Write output: PUT /webdav/plugin/subtitle-extractor/...
|
||||
* - Temp: $CACHE_PATH/temp (default /cache/temp)
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync, mkdirSync, statSync, unlinkSync, readFileSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { existsSync, mkdirSync, readFileSync, unlinkSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||
import { MetaCoreClient } from './meta-core-client.js';
|
||||
import { createWebDAVClient } from './webdav-client.js';
|
||||
import { getWebDAVClient } from './webdav-client.js';
|
||||
import { toLang3 } from './langs.js';
|
||||
import { flattenMeta, onlyNewKeys, legacyCsvMembers, staleLanguageLeaves } from './meta-shape.js';
|
||||
|
||||
// Initialize WebDAV client if WEBDAV_URL is set
|
||||
const webdavClient = createWebDAVClient();
|
||||
if (webdavClient) {
|
||||
console.log('[subtitle-extractor] WebDAV client initialized for file access');
|
||||
} else {
|
||||
console.warn('[subtitle-extractor] WARNING: WEBDAV_URL not set - plugin will not be able to write output files');
|
||||
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/subtitle-extractor';
|
||||
|
||||
function tempDir(): string {
|
||||
return path.join(globalThis.process.env.CACHE_PATH || '/cache', 'temp');
|
||||
}
|
||||
|
||||
/**
|
||||
* ============================================================================
|
||||
* PLUGIN OUTPUT PATH - Written via WebDAV
|
||||
* ============================================================================
|
||||
* Output files (subtitles) are written via WebDAV PUT requests.
|
||||
* Path: /files/plugin/subtitle-extractor/<filename> (accessible at WEBDAV_URL/plugin/subtitle-extractor/)
|
||||
* TEMP_PATH (/cache/temp) - Local temp folder for ffmpeg extraction before WebDAV upload
|
||||
* ============================================================================
|
||||
*/
|
||||
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/subtitle-extractor';
|
||||
const TEMP_PATH = '/cache/temp';
|
||||
|
||||
// Image-based subtitle codecs that cannot be converted to text
|
||||
const UNSUPPORTED_SUBTITLE_CODECS = new Set([
|
||||
/** Bitmap subtitle codecs: no text to extract without OCR. */
|
||||
export const IMAGE_SUBTITLE_CODECS = new Set([
|
||||
'hdmv_pgs_subtitle',
|
||||
'pgssub',
|
||||
'dvd_subtitle',
|
||||
'dvdsub',
|
||||
'dvb_subtitle',
|
||||
'dvbsub',
|
||||
'dvb_teletext',
|
||||
'xsub',
|
||||
]);
|
||||
|
||||
// Supported text-based subtitle codecs
|
||||
const CODEC_EXTENSION_MAP: Record<string, string> = {
|
||||
'subrip': 'srt',
|
||||
'srt': 'srt',
|
||||
'ass': 'ass',
|
||||
'ssa': 'ssa',
|
||||
'webvtt': 'vtt',
|
||||
'mov_text': 'srt',
|
||||
'text': 'srt',
|
||||
type SubtitleExt = 'srt' | 'ass' | 'ssa' | 'vtt';
|
||||
|
||||
/** Text codecs we extract, and the file type each one naturally becomes. */
|
||||
const TEXT_SUBTITLE_CODECS: Record<string, SubtitleExt> = {
|
||||
subrip: 'srt',
|
||||
srt: 'srt',
|
||||
ass: 'ass',
|
||||
ssa: 'ssa',
|
||||
webvtt: 'vtt',
|
||||
mov_text: 'srt',
|
||||
text: 'srt',
|
||||
};
|
||||
|
||||
/** Codecs whose packets can be stream-copied into their natural file type. */
|
||||
const COPYABLE_CODECS = new Set(['subrip', 'srt', 'ass', 'ssa', 'webvtt']);
|
||||
|
||||
const ENCODER_FOR_EXT: Record<SubtitleExt, string> = { srt: 'srt', vtt: 'webvtt', ass: 'ass', ssa: 'ass' };
|
||||
|
||||
export type OutputFormat = 'native' | 'srt' | 'vtt' | 'ass';
|
||||
const OUTPUT_FORMATS = new Set<OutputFormat>(['native', 'srt', 'vtt', 'ass']);
|
||||
|
||||
export const manifest: PluginManifest = {
|
||||
id: 'subtitle-extractor',
|
||||
name: 'Subtitle Extractor',
|
||||
version: '1.0.0',
|
||||
description: 'Extracts embedded subtitles from video files and saves them as text files',
|
||||
version: '1.1.0',
|
||||
description: 'Extracts embedded text subtitles from video files and links them to the video',
|
||||
author: 'MetaMesh',
|
||||
// file-info gives `fileType`; ffmpeg gives the `stream` table.
|
||||
dependencies: ['file-info', 'ffmpeg'],
|
||||
priority: 50,
|
||||
color: '#FF5722',
|
||||
// One full read of the video per run. Never the fast queue.
|
||||
defaultQueue: 'background',
|
||||
timeout: 300000, // 5 minutes for large files
|
||||
timeout: 300000,
|
||||
schema: {
|
||||
extractedSubtitles: { label: 'Extracted Subtitles', type: 'array', readonly: true },
|
||||
subtitleLanguages: { label: 'Subtitle Languages', type: 'array', readonly: true },
|
||||
subtitles: { label: 'Subtitles', type: 'json', readonly: true, hint: 'subtitles/<lang3>/<cid>' },
|
||||
extractedSubtitles: { label: 'Extracted Subtitles', type: 'json', readonly: true, hint: 'extractedSubtitles/<lang3>/<cid>' },
|
||||
subtitleLanguages: { label: 'Subtitle Languages', type: 'json', readonly: true, hint: 'subtitleLanguages/<lang3>' },
|
||||
},
|
||||
config: {
|
||||
forceRecompute: {
|
||||
type: 'boolean',
|
||||
label: 'Force Recompute',
|
||||
default: false,
|
||||
},
|
||||
forceRecompute: { type: 'boolean', label: 'Force Recompute', default: false },
|
||||
outputFormat: {
|
||||
type: 'select',
|
||||
label: 'Output Format',
|
||||
default: 'srt',
|
||||
label: 'Output Format (native keeps ASS styling; srt/vtt/ass convert)',
|
||||
default: 'native',
|
||||
},
|
||||
extractionTimeoutMs: {
|
||||
type: 'number',
|
||||
label: 'Extraction timeout (ms) for the single ffmpeg pass',
|
||||
default: 270000,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Configuration
|
||||
let forceRecompute = false;
|
||||
let outputFormat = 'srt';
|
||||
let outputFormat: OutputFormat = 'native';
|
||||
let extractionTimeoutMs = 270000;
|
||||
|
||||
export function configure(config: Record<string, unknown>): void {
|
||||
forceRecompute = config.forceRecompute === true;
|
||||
outputFormat = (config.outputFormat as string) || 'srt';
|
||||
console.log(`[subtitle-extractor] Config: forceRecompute=${forceRecompute}, outputFormat=${outputFormat}`);
|
||||
const format = String(config.outputFormat ?? 'native') as OutputFormat;
|
||||
outputFormat = OUTPUT_FORMATS.has(format) ? format : 'native';
|
||||
const timeout = config.extractionTimeoutMs === null || config.extractionTimeoutMs === undefined
|
||||
? NaN
|
||||
: Number(config.extractionTimeoutMs);
|
||||
extractionTimeoutMs = Number.isFinite(timeout) && timeout > 0 ? timeout : 270000;
|
||||
console.log(`[subtitle-extractor] Config: forceRecompute=${forceRecompute}, outputFormat=${outputFormat}, extractionTimeoutMs=${extractionTimeoutMs}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute midhash256 CID from a Buffer (matches meta-hash algorithm)
|
||||
* Used for computing CID before/after WebDAV upload
|
||||
* midhash256 CID of a buffer (matches meta-hash's computeMidHash256Sync):
|
||||
* sha256(size_u64_be ‖ middle 1 MiB), wrapped as CIDv1 multibase base32.
|
||||
*/
|
||||
function computeMidHash256FromBuffer(data: Buffer): string {
|
||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||
export function computeMidHash256FromBuffer(data: Buffer): string {
|
||||
const SAMPLE_SIZE = 1024 * 1024;
|
||||
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
||||
|
||||
const fileSize = data.length;
|
||||
|
||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||
|
||||
@@ -131,16 +161,8 @@ function computeMidHash256FromBuffer(data: Buffer): string {
|
||||
sampleData = data.subarray(middleOffset, middleOffset + SAMPLE_SIZE);
|
||||
}
|
||||
|
||||
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||
const hashBuffer = createHash('sha256').update(hashInput).digest();
|
||||
|
||||
const cidBytes = Buffer.concat([
|
||||
Buffer.from([0x01]),
|
||||
MIDHASH_VARINT,
|
||||
MIDHASH_VARINT,
|
||||
Buffer.from([0x20]),
|
||||
hashBuffer
|
||||
]);
|
||||
const hashBuffer = createHash('sha256').update(Buffer.concat([sizeBuffer, sampleData])).digest();
|
||||
const cidBytes = Buffer.concat([Buffer.from([0x01]), MIDHASH_VARINT, MIDHASH_VARINT, Buffer.from([0x20]), hashBuffer]);
|
||||
|
||||
const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
let cid = 'b';
|
||||
@@ -157,337 +179,480 @@ function computeMidHash256FromBuffer(data: Buffer): string {
|
||||
if (bits > 0) {
|
||||
cid += base32Chars[(value << (5 - bits)) & 0x1f];
|
||||
}
|
||||
|
||||
return cid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize filename by removing invalid characters
|
||||
*/
|
||||
function sanitizeFilename(name: string): string {
|
||||
/** Strip characters that are hostile to paths or to the WebDAV URL. */
|
||||
export function sanitizeFilename(name: string): string {
|
||||
return name
|
||||
.replace(/[<>:"/\\|?*]/g, '')
|
||||
.replace(/[<>:"/\\|?*#%\x00-\x1f]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse subtitle streams from ffmpeg plugin metadata
|
||||
*/
|
||||
interface SubtitleStream {
|
||||
index: number;
|
||||
codec: string;
|
||||
language?: string;
|
||||
title?: string;
|
||||
// ----------------------------------------------------------------------------
|
||||
// Stream table
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
function numericKeyOrder(a: string, b: string): number {
|
||||
return Number(a) - Number(b);
|
||||
}
|
||||
|
||||
function parseSubtitleStreams(existingMeta: Record<string, string>): SubtitleStream[] {
|
||||
const streams: SubtitleStream[] = [];
|
||||
/**
|
||||
* The ffmpeg plugin's per-stream objects, in stream order, from any of the shapes
|
||||
* a record can arrive in:
|
||||
* - nested array of JSON strings stream: ["{...}", "{...}"] (meta-sort /meta today)
|
||||
* - nested array of objects stream: [{...}, {...}]
|
||||
* - nested object keyed by index stream: {"0": "{...}", "1": ...}
|
||||
* - JSON string of an array stream: "[...]"
|
||||
* - flat meta-core keys "stream/0": "{...}", "stream/10": ... (ordered numerically)
|
||||
*/
|
||||
export function readStreamEntries(existingMeta: Record<string, unknown> | undefined): Record<string, unknown>[] {
|
||||
if (!existingMeta) return [];
|
||||
const raw: unknown[] = [];
|
||||
|
||||
// ffmpeg plugin stores streams as 'stream' field - an array of JSON strings
|
||||
// Each element is a stringified JSON object like:
|
||||
// '{"type":"subtitle","codec":"ass","index":2,"language":"eng",...}'
|
||||
const streamData = existingMeta['stream'];
|
||||
if (streamData) {
|
||||
const nested = existingMeta['stream'];
|
||||
if (nested !== undefined && nested !== null && nested !== '') {
|
||||
try {
|
||||
// Parse the outer array (it's already an array from meta-core)
|
||||
const allStreams = Array.isArray(streamData)
|
||||
? streamData
|
||||
: JSON.parse(streamData) as string[];
|
||||
|
||||
let subtitleIndex = 0;
|
||||
for (const streamStr of allStreams) {
|
||||
// Each stream is a JSON string
|
||||
const stream = typeof streamStr === 'string'
|
||||
? JSON.parse(streamStr) as {
|
||||
type?: string;
|
||||
codec?: string;
|
||||
index?: number;
|
||||
language?: string;
|
||||
title?: string;
|
||||
}
|
||||
: streamStr;
|
||||
|
||||
if (stream.type === 'subtitle') {
|
||||
streams.push({
|
||||
index: subtitleIndex,
|
||||
codec: stream.codec || 'unknown',
|
||||
language: stream.language,
|
||||
title: stream.title,
|
||||
});
|
||||
subtitleIndex++;
|
||||
}
|
||||
const parsed = typeof nested === 'string' ? JSON.parse(nested) : nested;
|
||||
if (Array.isArray(parsed)) {
|
||||
raw.push(...parsed);
|
||||
} else if (parsed && typeof parsed === 'object') {
|
||||
const obj = parsed as Record<string, unknown>;
|
||||
raw.push(...Object.keys(obj).sort(numericKeyOrder).map((k) => obj[k]));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[subtitle-extractor] Failed to parse stream metadata:', e);
|
||||
} catch {
|
||||
// fall through to the flat form
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: parse individual fields (legacy format)
|
||||
if (streams.length === 0) {
|
||||
if (raw.length === 0) {
|
||||
const flat = Object.keys(existingMeta)
|
||||
.filter((k) => /^stream\/\d+$/.test(k))
|
||||
.sort((a, b) => numericKeyOrder(a.slice(7), b.slice(7)));
|
||||
raw.push(...flat.map((k) => existingMeta[k]));
|
||||
}
|
||||
|
||||
const entries: Record<string, unknown>[] = [];
|
||||
for (const item of raw) {
|
||||
try {
|
||||
const obj = typeof item === 'string' ? JSON.parse(item) : item;
|
||||
if (obj && typeof obj === 'object' && !Array.isArray(obj)) entries.push(obj as Record<string, unknown>);
|
||||
} catch {
|
||||
// skip an unparseable entry, keep the rest
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
export interface SubtitleStream {
|
||||
/** Position among subtitle streams (ffmpeg `0:s:<ordinal>`). */
|
||||
ordinal: number;
|
||||
/** Global ffmpeg stream index (`0:<streamIndex>`), when the record has it. */
|
||||
streamIndex?: number;
|
||||
codec: string;
|
||||
/** ISO 639-2/B, `und` when undeclared. */
|
||||
lang3: string;
|
||||
title?: string;
|
||||
forced: boolean;
|
||||
}
|
||||
|
||||
const isTrue = (v: unknown) => v === true || v === 'true';
|
||||
|
||||
/** Subtitle streams of a record, from the stream table or the legacy `subtitle_{i}_*` fields. */
|
||||
export function parseSubtitleStreams(existingMeta: Record<string, unknown> | undefined): SubtitleStream[] {
|
||||
const streams: SubtitleStream[] = [];
|
||||
let ordinal = 0;
|
||||
for (const entry of readStreamEntries(existingMeta)) {
|
||||
if (entry.type !== 'subtitle') continue;
|
||||
const index = entry.index === undefined || entry.index === null ? NaN : Number(entry.index);
|
||||
streams.push({
|
||||
ordinal: ordinal++,
|
||||
streamIndex: Number.isInteger(index) && index >= 0 ? index : undefined,
|
||||
codec: String(entry.codec ?? 'unknown').toLowerCase(),
|
||||
lang3: toLang3(entry.language),
|
||||
title: typeof entry.title === 'string' ? entry.title : undefined,
|
||||
forced: isTrue(entry.forced),
|
||||
});
|
||||
}
|
||||
|
||||
if (streams.length === 0 && existingMeta) {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const codec = existingMeta[`subtitle_${i}_codec`];
|
||||
if (codec) {
|
||||
streams.push({
|
||||
index: i,
|
||||
codec,
|
||||
language: existingMeta[`subtitle_${i}_language`],
|
||||
title: existingMeta[`subtitle_${i}_title`],
|
||||
});
|
||||
}
|
||||
if (!codec) continue;
|
||||
const title = existingMeta[`subtitle_${i}_title`];
|
||||
streams.push({
|
||||
ordinal: i,
|
||||
codec: String(codec).toLowerCase(),
|
||||
lang3: toLang3(existingMeta[`subtitle_${i}_language`]),
|
||||
title: typeof title === 'string' ? title : undefined,
|
||||
forced: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return streams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a subtitle track using ffmpeg
|
||||
*/
|
||||
async function extractSubtitle(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
subtitleIndex: number,
|
||||
codec: string
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
// Determine output codec based on format
|
||||
let outputCodec = 'srt';
|
||||
if (outputFormat === 'vtt') {
|
||||
outputCodec = 'webvtt';
|
||||
} else if (outputFormat === 'ass') {
|
||||
outputCodec = 'ass';
|
||||
/** File type and ffmpeg encoder for a codec under an output format; null when not a text codec. */
|
||||
export function outputFor(codec: string, format: OutputFormat = outputFormat): { ext: SubtitleExt; encoder: string } | null {
|
||||
const nativeExt = TEXT_SUBTITLE_CODECS[codec];
|
||||
if (!nativeExt) return null;
|
||||
const ext: SubtitleExt = format === 'native' ? nativeExt : format;
|
||||
const encoder = ext === nativeExt && COPYABLE_CODECS.has(codec) ? 'copy' : ENCODER_FOR_EXT[ext];
|
||||
return { ext, encoder };
|
||||
}
|
||||
|
||||
export type TextSubtitleStream = SubtitleStream & { ext: SubtitleExt; encoder: string };
|
||||
|
||||
export function classifySubtitleStreams(streams: SubtitleStream[], format: OutputFormat = outputFormat): {
|
||||
text: TextSubtitleStream[];
|
||||
image: SubtitleStream[];
|
||||
unsupported: SubtitleStream[];
|
||||
} {
|
||||
const text: TextSubtitleStream[] = [];
|
||||
const image: SubtitleStream[] = [];
|
||||
const unsupported: SubtitleStream[] = [];
|
||||
for (const s of streams) {
|
||||
if (IMAGE_SUBTITLE_CODECS.has(s.codec)) {
|
||||
image.push(s);
|
||||
continue;
|
||||
}
|
||||
const out = outputFor(s.codec, format);
|
||||
if (out) text.push({ ...s, ...out });
|
||||
else unsupported.push(s);
|
||||
}
|
||||
return { text, image, unsupported };
|
||||
}
|
||||
|
||||
const args = [
|
||||
'-y',
|
||||
'-hide_banner',
|
||||
'-loglevel', 'error',
|
||||
'-probesize', '1M',
|
||||
'-analyzeduration', '1M',
|
||||
'-i', inputPath,
|
||||
'-map', `0:s:${subtitleIndex}`,
|
||||
'-c:s', outputCodec,
|
||||
outputPath
|
||||
];
|
||||
const scalar = (v: unknown): string => (typeof v === 'string' || typeof v === 'number' ? String(v) : '');
|
||||
|
||||
console.log(`[subtitle-extractor] Running ffmpeg for subtitle ${subtitleIndex}`);
|
||||
/**
|
||||
* `Title (Year)[videoCid]_subtitle.s<streamIndex>.<lang3>[.forced].<ext>`.
|
||||
* The stream index keeps two tracks in one language (full + signs, es-ES +
|
||||
* es-419) from overwriting each other; the trailing `<lang3>` is what the
|
||||
* `subtitle` plugin reads back as the file's language.
|
||||
*/
|
||||
export function subtitleFilename(existingMeta: Record<string, unknown>, videoCid: string, stream: SubtitleStream, ext: string): string {
|
||||
const fileName = scalar(existingMeta.fileName);
|
||||
const rawTitle = scalar(existingMeta.originalTitle) || scalar(existingMeta.title) || (fileName ? path.parse(fileName).name : '');
|
||||
const safeTitle = sanitizeFilename(rawTitle).slice(0, 100) || 'video';
|
||||
const year = scalar(existingMeta.movieYear);
|
||||
const track = stream.streamIndex !== undefined ? `s${stream.streamIndex}` : `t${stream.ordinal}`;
|
||||
return `${safeTitle}${year ? ` (${year})` : ''}[${videoCid}]_subtitle.${track}.${stream.lang3}${stream.forced ? '.forced' : ''}.${ext}`;
|
||||
}
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
// ----------------------------------------------------------------------------
|
||||
// Extraction
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
export interface ExtractJob {
|
||||
stream: TextSubtitleStream;
|
||||
tempFile: string;
|
||||
}
|
||||
|
||||
/** ONE ffmpeg invocation writing every job's track to its own output (one read of the input). */
|
||||
export function buildExtractionArgs(input: string, jobs: ExtractJob[]): string[] {
|
||||
const args = ['-y', '-hide_banner', '-nostdin', '-loglevel', 'error', '-i', input];
|
||||
for (const job of jobs) {
|
||||
const map = job.stream.streamIndex !== undefined ? `0:${job.stream.streamIndex}` : `0:s:${job.stream.ordinal}`;
|
||||
args.push('-map', map, '-c:s', job.stream.encoder, job.tempFile);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
/** Number of ffmpeg processes started — each one is a full read of the input. */
|
||||
export const extractionStats = { ffmpegPasses: 0 };
|
||||
|
||||
function runFfmpeg(args: string[], timeoutMs: number): Promise<{ code: number | null; stderr: string; timedOut: boolean }> {
|
||||
extractionStats.ffmpegPasses++;
|
||||
return new Promise((resolve) => {
|
||||
const ffmpeg = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] });
|
||||
let stderr = '';
|
||||
ffmpeg.stderr?.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0 && existsSync(outputPath)) {
|
||||
const size = statSync(outputPath).size;
|
||||
if (size > 10) {
|
||||
console.log(`[subtitle-extractor] Extracted subtitle ${subtitleIndex} (${size} bytes)`);
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] Subtitle ${subtitleIndex} extraction produced empty file`);
|
||||
try { unlinkSync(outputPath); } catch {}
|
||||
resolve(false);
|
||||
}
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] Failed to extract subtitle ${subtitleIndex}: ${stderr.slice(0, 200)}`);
|
||||
resolve(false);
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
console.log(`[subtitle-extractor] ffmpeg error: ${err.message}`);
|
||||
resolve(false);
|
||||
});
|
||||
|
||||
// Timeout after 2 minutes per subtitle
|
||||
setTimeout(() => {
|
||||
let timedOut = false;
|
||||
let settled = false;
|
||||
const finish = (code: number | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve({ code, stderr, timedOut });
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
ffmpeg.kill('SIGKILL');
|
||||
resolve(false);
|
||||
}, 120000);
|
||||
}, timeoutMs);
|
||||
ffmpeg.stderr?.on('data', (d) => { if (stderr.length < 4000) stderr += d.toString(); });
|
||||
ffmpeg.on('close', (code) => finish(code));
|
||||
ffmpeg.on('error', (err) => { stderr += err.message; finish(-1); });
|
||||
});
|
||||
}
|
||||
|
||||
/** Does an extracted file carry at least one cue (an ASS header alone does not count)? */
|
||||
export function hasCues(data: Buffer, ext: string): boolean {
|
||||
const text = data.toString('utf8');
|
||||
if (ext === 'ass' || ext === 'ssa') return /^Dialogue:/m.test(text);
|
||||
return text.includes('-->');
|
||||
}
|
||||
|
||||
function takeOutput(job: ExtractJob): Buffer | null {
|
||||
if (!existsSync(job.tempFile)) return null;
|
||||
const data = readFileSync(job.tempFile);
|
||||
try { unlinkSync(job.tempFile); } catch { /* ignore */ }
|
||||
return hasCues(data, job.stream.ext) ? data : null;
|
||||
}
|
||||
|
||||
function discard(job: ExtractJob): void {
|
||||
try { if (existsSync(job.tempFile)) unlinkSync(job.tempFile); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export interface ExtractResult {
|
||||
outputs: Array<{ job: ExtractJob; data: Buffer | null }>;
|
||||
passes: number;
|
||||
timedOut: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export async function extractTracks(input: string, jobs: ExtractJob[], timeoutMs: number = extractionTimeoutMs): Promise<ExtractResult> {
|
||||
const first = await runFfmpeg(buildExtractionArgs(input, jobs), timeoutMs);
|
||||
if (first.code === 0) {
|
||||
return { outputs: jobs.map((job) => ({ job, data: takeOutput(job) })), passes: 1, timedOut: false };
|
||||
}
|
||||
jobs.forEach(discard);
|
||||
const error = first.timedOut ? `ffmpeg timed out after ${timeoutMs} ms` : first.stderr.slice(0, 300) || `ffmpeg exited ${first.code}`;
|
||||
if (first.timedOut || jobs.length === 1) {
|
||||
return { outputs: jobs.map((job) => ({ job, data: null })), passes: 1, timedOut: first.timedOut, error };
|
||||
}
|
||||
|
||||
// One broken track aborts every output of the shared pass. Salvage the rest
|
||||
// one track at a time — at the price of one more full read per track.
|
||||
console.warn(`[subtitle-extractor] single-pass extraction failed (${error}); retrying ${jobs.length} track(s) one by one`);
|
||||
const outputs: ExtractResult['outputs'] = [];
|
||||
let passes = 1;
|
||||
let timedOut = false;
|
||||
for (const job of jobs) {
|
||||
if (timedOut) {
|
||||
outputs.push({ job, data: null });
|
||||
continue;
|
||||
}
|
||||
const r = await runFfmpeg(buildExtractionArgs(input, [job]), timeoutMs);
|
||||
passes++;
|
||||
timedOut = r.timedOut;
|
||||
if (r.code === 0) {
|
||||
outputs.push({ job, data: takeOutput(job) });
|
||||
} else {
|
||||
discard(job);
|
||||
outputs.push({ job, data: null });
|
||||
}
|
||||
}
|
||||
return { outputs, passes, timedOut, error };
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Record keys
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
export interface ProducedSubtitle {
|
||||
subCid: string;
|
||||
lang3: string;
|
||||
}
|
||||
|
||||
/** True when this record already carries the `extractedSubtitles` key-set (nested or flat). */
|
||||
export function hasExtractedKeySet(existingMeta: Record<string, unknown> | undefined): boolean {
|
||||
if (!existingMeta) return false;
|
||||
const v = existingMeta['extractedSubtitles'];
|
||||
if (v && typeof v === 'object') return true;
|
||||
return Object.keys(existingMeta).some((k) => k.startsWith('extractedSubtitles/'));
|
||||
}
|
||||
|
||||
/** Keys on the video record for a set of extracted files (METADATA_KEYS.md §8, §9). */
|
||||
export function videoSubtitleKeys(produced: ProducedSubtitle[]): Record<string, string> {
|
||||
const keys: Record<string, string> = {};
|
||||
for (const { subCid, lang3 } of produced) {
|
||||
keys[`subtitles/${lang3}/${subCid}`] = 'true';
|
||||
keys[`extractedSubtitles/${lang3}/${subCid}`] = 'true';
|
||||
if (lang3 !== 'und') {
|
||||
keys[`subtitleLanguages/${lang3}`] = 'true';
|
||||
keys[`languages/${lang3}`] = 'true';
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** Keys on the subtitle file's own record. */
|
||||
export function subtitleRecordKeys(videoCid: string, lang3: string): Record<string, string> {
|
||||
const keys: Record<string, string> = { [`videos/${videoCid}`]: 'true' };
|
||||
if (lang3 !== 'und') keys.subtitleLanguage = lang3;
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy scalar csv-sets the old version of this plugin wrote (and the old
|
||||
* `subtitle` plugin's `subtitles`), turned into deletes plus the key-set leaves
|
||||
* that preserve what they said. `extractedSubtitles` members are dropped: this
|
||||
* run re-derives them with their language.
|
||||
*/
|
||||
export function legacySubtitleMigration(existingMeta: Record<string, unknown> | undefined): { deletes: string[]; sets: Record<string, string> } {
|
||||
const deletes: string[] = [];
|
||||
const sets: Record<string, string> = {};
|
||||
|
||||
if (legacyCsvMembers(existingMeta, 'extractedSubtitles')) deletes.push('extractedSubtitles');
|
||||
|
||||
const langs = legacyCsvMembers(existingMeta, 'subtitleLanguages');
|
||||
if (langs) {
|
||||
deletes.push('subtitleLanguages');
|
||||
for (const l of langs.map(toLang3)) if (l !== 'und') sets[`subtitleLanguages/${l}`] = 'true';
|
||||
}
|
||||
|
||||
const subs = legacyCsvMembers(existingMeta, 'subtitles');
|
||||
if (subs) {
|
||||
deletes.push('subtitles');
|
||||
for (const cid of subs) sets[`subtitles/und/${cid}`] = 'true';
|
||||
}
|
||||
return { deletes, sets };
|
||||
}
|
||||
|
||||
/** Write the links for `produced` onto the video and each subtitle record. False on any failed write. */
|
||||
export async function linkSubtitles(
|
||||
metaCore: MetaCoreClient,
|
||||
videoCid: string,
|
||||
existingMeta: Record<string, unknown>,
|
||||
produced: ProducedSubtitle[],
|
||||
): Promise<boolean> {
|
||||
const { deletes, sets } = legacySubtitleMigration(existingMeta);
|
||||
const flat = flattenMeta(existingMeta);
|
||||
for (const key of deletes) delete flat[key];
|
||||
|
||||
for (const { subCid, lang3 } of produced) {
|
||||
deletes.push(
|
||||
...staleLanguageLeaves(flat, 'subtitles', subCid, lang3),
|
||||
...staleLanguageLeaves(flat, 'extractedSubtitles', subCid, lang3),
|
||||
);
|
||||
}
|
||||
for (const key of deletes) {
|
||||
if (!(await metaCore.deleteProperty(videoCid, key))) return false;
|
||||
delete flat[key];
|
||||
}
|
||||
|
||||
const fresh = onlyNewKeys(flat, { ...sets, ...videoSubtitleKeys(produced) });
|
||||
if (Object.keys(fresh).length > 0 && !(await metaCore.mergeMetadata(videoCid, fresh))) return false;
|
||||
|
||||
for (const { subCid, lang3 } of produced) {
|
||||
const current = await metaCore.getMetadata(subCid);
|
||||
const wanted = onlyNewKeys(flattenMeta(current), subtitleRecordKeys(videoCid, lang3));
|
||||
if (Object.keys(wanted).length > 0 && !(await metaCore.mergeMetadata(subCid, wanted))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Task
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
export async function process(
|
||||
request: ProcessRequest,
|
||||
sendCallback: (payload: CallbackPayload) => Promise<void>
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const { taskId, cid, filePath } = request;
|
||||
const existingMeta = (request.existingMeta ?? {}) as Record<string, unknown>;
|
||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||
|
||||
const finish = (status: CallbackPayload['status'], extra: { reason?: string; error?: string } = {}) =>
|
||||
sendCallback({ taskId, status, duration: Date.now() - startTime, ...extra });
|
||||
const skip = (reason: string, quiet = false) => {
|
||||
if (!quiet) console.log(`[subtitle-extractor] ${filePath}: skipped — ${reason}`);
|
||||
return finish('skipped', { reason });
|
||||
};
|
||||
const fail = (error: string) => {
|
||||
console.error(`[subtitle-extractor] ${filePath}: failed — ${error}`);
|
||||
return finish('failed', { error });
|
||||
};
|
||||
|
||||
try {
|
||||
const { cid, filePath, existingMeta } = request;
|
||||
|
||||
// Only process video files
|
||||
if (existingMeta?.fileType !== 'video') {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Not a video file',
|
||||
});
|
||||
if (existingMeta.fileType !== 'video') {
|
||||
await skip('Not a video file', true);
|
||||
return;
|
||||
}
|
||||
if (!forceRecompute && hasExtractedKeySet(existingMeta)) {
|
||||
await skip('Subtitles already extracted', true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip if already extracted (unless forceRecompute)
|
||||
if (existingMeta?.extractedSubtitles && !forceRecompute) {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Subtitles already extracted',
|
||||
});
|
||||
const streams = parseSubtitleStreams(existingMeta);
|
||||
if (streams.length === 0) {
|
||||
await skip('No subtitle streams found', true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse subtitle streams from ffmpeg metadata
|
||||
const subtitleStreams = parseSubtitleStreams(existingMeta || {});
|
||||
|
||||
if (subtitleStreams.length === 0) {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'No subtitle streams found',
|
||||
});
|
||||
const { text, image, unsupported } = classifySubtitleStreams(streams);
|
||||
if (image.length > 0) {
|
||||
console.log(`[subtitle-extractor] ${filePath}: skipping ${image.length} image-based subtitle stream(s) (${[...new Set(image.map((s) => s.codec))].join(', ')})`);
|
||||
}
|
||||
if (unsupported.length > 0) {
|
||||
console.log(`[subtitle-extractor] ${filePath}: skipping ${unsupported.length} unsupported subtitle stream(s) (${[...new Set(unsupported.map((s) => s.codec))].join(', ')})`);
|
||||
}
|
||||
if (text.length === 0) {
|
||||
await skip(image.length > 0 ? 'Only image-based subtitles (cannot convert to text)' : 'No supported text subtitle streams');
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter out unsupported (image-based) codecs
|
||||
const textSubtitles = subtitleStreams.filter(s => !UNSUPPORTED_SUBTITLE_CODECS.has(s.codec));
|
||||
|
||||
if (textSubtitles.length === 0) {
|
||||
console.log(`[subtitle-extractor] All ${subtitleStreams.length} subtitles are image-based, skipping`);
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'skipped',
|
||||
duration: Date.now() - startTime,
|
||||
reason: 'Only image-based subtitles (cannot convert to text)',
|
||||
});
|
||||
const webdav = await getWebDAVClient(request.metaCoreUrl);
|
||||
if (!webdav) {
|
||||
await fail('No WebDAV endpoint available');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[subtitle-extractor] Found ${textSubtitles.length} text-based subtitle(s) in ${filePath}`);
|
||||
const produced: ProducedSubtitle[] = [];
|
||||
const pending: Array<ExtractJob & { webdavPath: string }> = [];
|
||||
const tmp = tempDir();
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
const safeTask = taskId.replace(/[^A-Za-z0-9_-]/g, '_');
|
||||
|
||||
// Require WebDAV client for writing output
|
||||
if (!webdavClient) {
|
||||
console.error('[subtitle-extractor] WebDAV client not available, cannot write output files');
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'failed',
|
||||
duration: Date.now() - startTime,
|
||||
error: 'WebDAV client not configured (WEBDAV_URL not set)',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Get input path (WebDAV URL for reading)
|
||||
let inputPath = webdavClient.toWebDAVUrl(filePath);
|
||||
console.log(`[subtitle-extractor] Using WebDAV input: ${inputPath}`);
|
||||
|
||||
// Ensure temp directory exists for ffmpeg extraction
|
||||
if (!existsSync(TEMP_PATH)) {
|
||||
mkdirSync(TEMP_PATH, { recursive: true });
|
||||
}
|
||||
|
||||
// Get video title for output filename
|
||||
const title = existingMeta?.originalTitle || existingMeta?.title || existingMeta?.fileName || 'video';
|
||||
const safeTitle = sanitizeFilename(title);
|
||||
const year = existingMeta?.movieYear;
|
||||
const yearStr = year ? ` (${year})` : '';
|
||||
|
||||
const extractedCids: string[] = [];
|
||||
const extractedLanguages: string[] = [];
|
||||
|
||||
// Extract each subtitle
|
||||
for (const sub of textSubtitles) {
|
||||
const langSuffix = sub.language ? `.${sub.language}` : `.${sub.index}`;
|
||||
const ext = outputFormat;
|
||||
|
||||
// Build output filename: Title (Year)[videoCID]_subtitle.lang.srt
|
||||
const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle${langSuffix}.${ext}`;
|
||||
const webdavOutputPath = `${PLUGIN_OUTPUT_WEBDAV_PATH}/${outputFilename}`;
|
||||
const tempOutputPath = path.join(TEMP_PATH, outputFilename);
|
||||
|
||||
// Check if already extracted via WebDAV
|
||||
if (!forceRecompute) {
|
||||
const exists = await webdavClient.exists(webdavOutputPath);
|
||||
if (exists) {
|
||||
console.log(`[subtitle-extractor] Subtitle already exists: ${outputFilename}`);
|
||||
// Read existing file to get CID
|
||||
try {
|
||||
const existingData = await webdavClient.readFile(webdavOutputPath);
|
||||
const subtitleCid = computeMidHash256FromBuffer(existingData);
|
||||
extractedCids.push(subtitleCid);
|
||||
if (sub.language && !extractedLanguages.includes(sub.language)) {
|
||||
extractedLanguages.push(sub.language);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[subtitle-extractor] Failed to read existing subtitle: ${e}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract subtitle to temp file
|
||||
const extracted = await extractSubtitle(inputPath, tempOutputPath, sub.index, sub.codec);
|
||||
|
||||
if (extracted) {
|
||||
for (const stream of text) {
|
||||
const webdavPath = `${PLUGIN_OUTPUT_WEBDAV_PATH}/${subtitleFilename(existingMeta, cid, stream, stream.ext)}`;
|
||||
if (!forceRecompute && await webdav.exists(webdavPath)) {
|
||||
try {
|
||||
// Read extracted subtitle file
|
||||
const subtitleData = readFileSync(tempOutputPath);
|
||||
|
||||
// Compute CID from buffer
|
||||
const subtitleCid = computeMidHash256FromBuffer(subtitleData);
|
||||
console.log(`[subtitle-extractor] Subtitle CID: ${subtitleCid}`);
|
||||
|
||||
// Upload to WebDAV
|
||||
await webdavClient.writeFile(webdavOutputPath, subtitleData);
|
||||
console.log(`[subtitle-extractor] Uploaded subtitle to WebDAV: ${webdavOutputPath}`);
|
||||
|
||||
extractedCids.push(subtitleCid);
|
||||
if (sub.language && !extractedLanguages.includes(sub.language)) {
|
||||
extractedLanguages.push(sub.language);
|
||||
}
|
||||
|
||||
// Store subtitle CID as metadata on the video
|
||||
await metaCore.addToSet(cid, 'extractedSubtitles', subtitleCid);
|
||||
|
||||
// Store language if available
|
||||
if (sub.language) {
|
||||
await metaCore.addToSet(cid, 'subtitleLanguages', sub.language);
|
||||
}
|
||||
|
||||
// Clean up temp file
|
||||
try { unlinkSync(tempOutputPath); } catch {}
|
||||
// A previous run already wrote this track — reuse it, no read of the video.
|
||||
produced.push({ subCid: computeMidHash256FromBuffer(await webdav.readFile(webdavPath)), lang3: stream.lang3 });
|
||||
continue;
|
||||
} catch (e) {
|
||||
console.error(`[subtitle-extractor] Failed to process extracted subtitle: ${e}`);
|
||||
// Clean up temp file on error
|
||||
try { unlinkSync(tempOutputPath); } catch {}
|
||||
console.warn(`[subtitle-extractor] could not reuse ${webdavPath}: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
}
|
||||
pending.push({ stream, webdavPath, tempFile: path.join(tmp, `${safeTask}-${stream.ordinal}.${stream.ext}`) });
|
||||
}
|
||||
|
||||
let extraction: ExtractResult | undefined;
|
||||
if (pending.length > 0) {
|
||||
console.log(`[subtitle-extractor] ${filePath}: extracting ${pending.length} text track(s) in one pass`);
|
||||
extraction = await extractTracks(webdav.toWebDAVUrl(filePath), pending);
|
||||
for (const { job, data } of extraction.outputs) {
|
||||
if (!data) continue;
|
||||
const target = (job as ExtractJob & { webdavPath: string }).webdavPath;
|
||||
await webdav.writeFile(target, data);
|
||||
produced.push({ subCid: computeMidHash256FromBuffer(data), lang3: job.stream.lang3 });
|
||||
}
|
||||
}
|
||||
|
||||
if (extractedCids.length > 0) {
|
||||
console.log(`[subtitle-extractor] Extracted ${extractedCids.length} subtitle(s) from ${filePath}`);
|
||||
} else {
|
||||
console.log(`[subtitle-extractor] No subtitles could be extracted from ${filePath}`);
|
||||
if (produced.length === 0) {
|
||||
if (extraction?.error) {
|
||||
await fail(extraction.error);
|
||||
} else {
|
||||
await skip('No text subtitle track produced any cues');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'completed',
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
if (!(await linkSubtitles(metaCore, cid, existingMeta, produced))) {
|
||||
await fail('meta-core write failed');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[subtitle-extractor] ${filePath}: linked ${produced.length} subtitle(s) ` +
|
||||
`[${produced.map((p) => p.lang3).join(', ')}] in ${extraction?.passes ?? 0} ffmpeg pass(es)`
|
||||
);
|
||||
await finish('completed');
|
||||
} catch (error) {
|
||||
console.error(`[subtitle-extractor] Error:`, error);
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'failed',
|
||||
duration: Date.now() - startTime,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
await fail(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@ export interface ProcessRequest {
|
||||
filePath: string;
|
||||
callbackUrl: string;
|
||||
metaCoreUrl: string;
|
||||
existingMeta?: Record<string, string>;
|
||||
// meta-sort sends its NESTED /meta document (`fileinfo: {...}`, `stream: [...]`);
|
||||
// meta-core serves the flat `a/b` form. Top-level scalars match in both.
|
||||
existingMeta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ProcessResponse {
|
||||
|
||||
+60
-35
@@ -64,7 +64,8 @@ export class WebDAVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read bytes from a file via HTTP Range request
|
||||
* Read first N bytes of a file (for magic byte detection)
|
||||
* Uses HTTP Range request
|
||||
*/
|
||||
async readBytes(filePath: string, start: number, end: number): Promise<Buffer> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
@@ -85,7 +86,7 @@ export class WebDAVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read entire file as buffer (for small files)
|
||||
* Read entire file as buffer (for small files only)
|
||||
*/
|
||||
async readFile(filePath: string): Promise<Buffer> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
@@ -106,12 +107,9 @@ export class WebDAVClient {
|
||||
async writeFile(filePath: string, data: Buffer | string): Promise<void> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
// Convert Buffer to Uint8Array for fetch compatibility
|
||||
const body = typeof data === 'string' ? data : new Uint8Array(data);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'PUT',
|
||||
body,
|
||||
body: data,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
@@ -135,40 +133,67 @@ export class WebDAVClient {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List directory contents via HTTP GET (returns JSON from nginx autoindex)
|
||||
*/
|
||||
async readdir(dirPath: string): Promise<string[]> {
|
||||
const url = this.toWebDAVUrl(dirPath);
|
||||
const finalUrl = url.endsWith('/') ? url : url + '/';
|
||||
|
||||
const response = await fetch(finalUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV GET (dir) failed for ${dirPath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const entries = await response.json() as Array<{ name: string; type: string }>;
|
||||
return entries.map(e => e.name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a WebDAV client from environment variables
|
||||
* WebDAV client for the meta-core that drove a /process call.
|
||||
*
|
||||
* The endpoint is read from that core's own /urls (`webdavUrlInternal`),
|
||||
* so blobs always land in the /files of the core we write metadata to. A feeder
|
||||
* whose plugins run beside it but enrich a remote hub core needs no per-stack
|
||||
* wiring, and it is impossible to write posters into one core while pointing
|
||||
* CIDs at another.
|
||||
*
|
||||
* WEBDAV_URL still wins when set — an explicit override for tests and for
|
||||
* topologies where the core's advertised internal URL isn't reachable from here.
|
||||
*
|
||||
* Resolutions are cached per meta-core URL; failures are not, so a core that is
|
||||
* still starting up is retried on the next task.
|
||||
*/
|
||||
export function createWebDAVClient(): WebDAVClient | null {
|
||||
const webdavUrl = process.env.WEBDAV_URL;
|
||||
const clientCache = new Map<string, Promise<WebDAVClient | null>>();
|
||||
|
||||
if (!webdavUrl) {
|
||||
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable');
|
||||
return null;
|
||||
async function resolveWebDAVUrl(metaCoreUrl: string): Promise<string | null> {
|
||||
// /urls, not /api/urls — metaCoreUrl points at the meta-core backend, whose
|
||||
// discovery routes are unprefixed (same base the /meta/* writes use).
|
||||
const base = metaCoreUrl.replace(/\/$/, '');
|
||||
const response = await fetch(`${base}/urls`, { signal: AbortSignal.timeout(5000) });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`GET ${base}/urls failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return new WebDAVClient(webdavUrl);
|
||||
const urls = await response.json() as { webdavUrlInternal?: string; webdavUrl?: string };
|
||||
return urls.webdavUrlInternal ?? urls.webdavUrl ?? null;
|
||||
}
|
||||
|
||||
export async function getWebDAVClient(metaCoreUrl: string): Promise<WebDAVClient | null> {
|
||||
const override = process.env.WEBDAV_URL;
|
||||
if (override) {
|
||||
return new WebDAVClient(override);
|
||||
}
|
||||
|
||||
let pending = clientCache.get(metaCoreUrl);
|
||||
if (!pending) {
|
||||
pending = resolveWebDAVUrl(metaCoreUrl).then((webdavUrl) => {
|
||||
if (!webdavUrl) {
|
||||
console.warn(`[webdav-client] meta-core ${metaCoreUrl} advertises no WebDAV URL`);
|
||||
return null;
|
||||
}
|
||||
console.log(`[webdav-client] Using WebDAV endpoint ${webdavUrl} (from ${metaCoreUrl})`);
|
||||
return new WebDAVClient(webdavUrl);
|
||||
});
|
||||
clientCache.set(metaCoreUrl, pending);
|
||||
}
|
||||
|
||||
try {
|
||||
const client = await pending;
|
||||
if (!client) {
|
||||
clientCache.delete(metaCoreUrl);
|
||||
}
|
||||
return client;
|
||||
} catch (error) {
|
||||
clientCache.delete(metaCoreUrl);
|
||||
console.warn(`[webdav-client] Failed to resolve WebDAV URL from ${metaCoreUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user