This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
node_modules
|
||||
dist
|
||||
.git
|
||||
.pnpm-store
|
||||
@@ -0,0 +1,30 @@
|
||||
# MetaMesh Plugin: subtitle - Test Image
|
||||
# Builds the plugin and runs its tests (no ffmpeg needed: WebDAV and the meta
|
||||
# API are faked in-process)
|
||||
|
||||
FROM node:20-slim
|
||||
|
||||
# git for the GitHub dependency (filename-tools)
|
||||
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PNPM_HOME="/pnpm"
|
||||
ENV PATH="$PNPM_HOME:$PATH"
|
||||
RUN npm install -g corepack@latest && corepack enable
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json pnpm-lock.yaml ./
|
||||
RUN corepack install
|
||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
||||
|
||||
# Build the git dependency (filename-tools), as the production Dockerfile does
|
||||
RUN cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build
|
||||
|
||||
COPY tsconfig.json vitest.config.ts ./
|
||||
COPY src/ ./src/
|
||||
COPY test/ ./test/
|
||||
RUN pnpm run build
|
||||
|
||||
ENV NODE_ENV=test
|
||||
|
||||
CMD ["pnpm", "test"]
|
||||
@@ -1,65 +1,38 @@
|
||||
# MetaMesh Plugin: Subtitle
|
||||
|
||||
A MetaMesh plugin that processes subtitle files and links them to videos.
|
||||
Links subtitle **sidecar** files to the video they sit next to, in both directions, and determines each sidecar's language.
|
||||
|
||||
## Description
|
||||
## What counts as a sidecar
|
||||
|
||||
This plugin handles subtitle file processing:
|
||||
`<video basename>.<ext>` or `<video basename>.<qualifiers>.<ext>` (case-insensitive), `<ext>` in `srt`, `ass`, `ssa`, `vtt`, where every qualifier is a language (`en`, `eng`, `fre`, `pt-BR`, `English`) or a flag (`forced`, `sdh`, `cc`, `hi`, `default`, `full`, `signs`, `songs`):
|
||||
|
||||
**For subtitle files (`.srt`, `.ass`, `.ssa`, `.vtt`):**
|
||||
- Detects subtitle language using content analysis
|
||||
- Finds sibling video files with matching names
|
||||
- `Movie.srt`, `Movie.eng.srt`, `Movie.en.srt`, `Movie.fr.forced.srt` → sidecars of `Movie.mkv`
|
||||
- `Movie.Extended.eng.srt` → not a sidecar of `Movie.mkv`
|
||||
|
||||
**For video files:**
|
||||
- Discovers associated subtitle files in the same directory
|
||||
Language: the qualifier when present, else a `franc-min` sniff of the subtitle text (timecodes and ASS markup stripped), else `und`. Codes are ISO 639-2/B (`fre`, `ger`, `chi`), normalised the same way meta-watch does. `hi` means hearing-impaired, not Hindi (use `hin`).
|
||||
|
||||
## Metadata Fields
|
||||
## What it writes (METADATA_KEYS.md §8)
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `subtitleLanguage` | Detected language of subtitle content |
|
||||
| `linkedVideoName` | Name of associated video file |
|
||||
| `subtitleFile/{ext}` | Associated subtitle filename by extension |
|
||||
| Record | Key | Value |
|
||||
|--------|-----|-------|
|
||||
| video | `subtitles/<lang3>/<subCid>` | `"true"` |
|
||||
| video | `subtitleLanguages/<lang3>` | `"true"` (not for `und`) |
|
||||
| subtitle | `videos/<videoCid>` | `"true"` |
|
||||
| subtitle | `subtitleLanguage` | `<lang3>` (not for `und`) |
|
||||
|
||||
## Dependencies
|
||||
Each run writes its own record in full and the other record only if it already exists, so the result is the same whichever file the watcher processes first. Writes are one `PATCH` of only-new keys; a failed write fails the task. Legacy comma-joined `subtitles` / `videos` scalars are migrated to the key-set.
|
||||
|
||||
- Requires `file-info` plugin to run first
|
||||
## File access
|
||||
|
||||
## Configuration
|
||||
WebDAV only (the container has no `/files` mount): directory listing, size and bytes come from the meta-core that drove the task (`/urls` → `webdavUrlInternal`; `WEBDAV_URL` overrides). CIDs are midhash256 computed from one ≤1 MiB Range read.
|
||||
|
||||
No configuration required.
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Endpoint | Method | Description |
|
||||
|----------|--------|-------------|
|
||||
| `/health` | GET | Health check |
|
||||
| `/manifest` | GET | Plugin manifest |
|
||||
| `/configure` | POST | Update configuration |
|
||||
| `/process` | POST | Process a file |
|
||||
|
||||
## Running Locally
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
npm start
|
||||
pnpm install && (cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build) && pnpm test
|
||||
./test.sh # in Docker
|
||||
```
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
docker build -t metamesh-plugin-subtitle .
|
||||
docker run -p 8080:8080 metamesh-plugin-subtitle
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `PORT` | `8080` | HTTP server port |
|
||||
| `HOST` | `0.0.0.0` | HTTP server host |
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
+15
-6
@@ -5,11 +5,20 @@
|
||||
"description": "MetaMesh plugin: Processes subtitle files and links",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": { "build": "tsc", "start": "node dist/index.js" },
|
||||
"dependencies": {
|
||||
"fastify": "^5.0.0",
|
||||
"franc-min": "^6.2.0",
|
||||
"@metazla/filename-tools": "github:worph/filename-tool"
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"devDependencies": { "@types/node": "^20.0.0", "typescript": "^5.3.0" }
|
||||
"dependencies": {
|
||||
"@metazla/filename-tools": "github:worph/filename-tool",
|
||||
"fastify": "^5.0.0",
|
||||
"franc-min": "^6.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"typescript": "^5.3.0",
|
||||
"vitest": "^1.6.1"
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+1088
File diff suppressed because it is too large
Load Diff
+170
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Language code normalisation — the `lang3` vocabulary meta-watch reads.
|
||||
*
|
||||
* Port of meta-watch's `MW_LANG.toLang3` (packages/meta-watch/ui/js/lang-codes.js),
|
||||
* the same normaliser the `language` plugin copied. Every spelling folds to
|
||||
* **ISO 639-2/B** — `fre` not `fra`, `ger` not `deu`, `chi` not `zho` — because
|
||||
* that is what meta-watch writes (`subtitles/fre/<cid>`, routes/subtitles.rs)
|
||||
* and what its readers compare against:
|
||||
*
|
||||
* ISO 639-1 "fr" -> fre
|
||||
* ISO 639-2/B "fre" -> fre
|
||||
* ISO 639-2/T "fra" -> fre (MP4 `mdhd` speaks T, Matroska speaks B)
|
||||
* BCP-47 "pt-BR" -> por (primary subtag only)
|
||||
*
|
||||
* One deliberate difference: "no language" is `und`, not "" — the code becomes a
|
||||
* key segment (`subtitles/und/<cid>`, METADATA_KEYS.md §8) and `und` is the
|
||||
* registry's spelling for undetermined.
|
||||
*
|
||||
* Copied, not imported: plugins are independent submodules. This file is
|
||||
* byte-identical in metamesh-plugin-subtitle and metamesh-plugin-subtitle-extractor.
|
||||
*/
|
||||
|
||||
// Canonical table keyed by 639-2/B: display name (lowercased for filename
|
||||
// matching) and the 639-1 code that folds into it.
|
||||
const LANGS: Record<string, { name: string; a1?: string }> = {
|
||||
eng: { name: 'english', a1: 'en' },
|
||||
fre: { name: 'french', a1: 'fr' },
|
||||
ger: { name: 'german', a1: 'de' },
|
||||
spa: { name: 'spanish', a1: 'es' },
|
||||
ita: { name: 'italian', a1: 'it' },
|
||||
por: { name: 'portuguese', a1: 'pt' },
|
||||
dut: { name: 'dutch', a1: 'nl' },
|
||||
swe: { name: 'swedish', a1: 'sv' },
|
||||
nor: { name: 'norwegian', a1: 'no' },
|
||||
dan: { name: 'danish', a1: 'da' },
|
||||
fin: { name: 'finnish', a1: 'fi' },
|
||||
ice: { name: 'icelandic', a1: 'is' },
|
||||
pol: { name: 'polish', a1: 'pl' },
|
||||
cze: { name: 'czech', a1: 'cs' },
|
||||
slo: { name: 'slovak', a1: 'sk' },
|
||||
slv: { name: 'slovenian', a1: 'sl' },
|
||||
hun: { name: 'hungarian', a1: 'hu' },
|
||||
rum: { name: 'romanian', a1: 'ro' },
|
||||
bul: { name: 'bulgarian', a1: 'bg' },
|
||||
rus: { name: 'russian', a1: 'ru' },
|
||||
ukr: { name: 'ukrainian', a1: 'uk' },
|
||||
bel: { name: 'belarusian', a1: 'be' },
|
||||
srp: { name: 'serbian', a1: 'sr' },
|
||||
hrv: { name: 'croatian', a1: 'hr' },
|
||||
bos: { name: 'bosnian', a1: 'bs' },
|
||||
mac: { name: 'macedonian', a1: 'mk' },
|
||||
alb: { name: 'albanian', a1: 'sq' },
|
||||
gre: { name: 'greek', a1: 'el' },
|
||||
tur: { name: 'turkish', a1: 'tr' },
|
||||
heb: { name: 'hebrew', a1: 'he' },
|
||||
ara: { name: 'arabic', a1: 'ar' },
|
||||
per: { name: 'persian', a1: 'fa' },
|
||||
urd: { name: 'urdu', a1: 'ur' },
|
||||
hin: { name: 'hindi', a1: 'hi' },
|
||||
ben: { name: 'bengali', a1: 'bn' },
|
||||
tam: { name: 'tamil', a1: 'ta' },
|
||||
tel: { name: 'telugu', a1: 'te' },
|
||||
mal: { name: 'malayalam', a1: 'ml' },
|
||||
kan: { name: 'kannada', a1: 'kn' },
|
||||
mar: { name: 'marathi', a1: 'mr' },
|
||||
guj: { name: 'gujarati', a1: 'gu' },
|
||||
pan: { name: 'punjabi', a1: 'pa' },
|
||||
nep: { name: 'nepali', a1: 'ne' },
|
||||
sin: { name: 'sinhala', a1: 'si' },
|
||||
tha: { name: 'thai', a1: 'th' },
|
||||
lao: { name: 'lao', a1: 'lo' },
|
||||
khm: { name: 'khmer', a1: 'km' },
|
||||
bur: { name: 'burmese', a1: 'my' },
|
||||
vie: { name: 'vietnamese', a1: 'vi' },
|
||||
ind: { name: 'indonesian', a1: 'id' },
|
||||
may: { name: 'malay', a1: 'ms' },
|
||||
tgl: { name: 'tagalog', a1: 'tl' },
|
||||
fil: { name: 'filipino' },
|
||||
jpn: { name: 'japanese', a1: 'ja' },
|
||||
kor: { name: 'korean', a1: 'ko' },
|
||||
chi: { name: 'chinese', a1: 'zh' },
|
||||
mon: { name: 'mongolian', a1: 'mn' },
|
||||
kaz: { name: 'kazakh', a1: 'kk' },
|
||||
uzb: { name: 'uzbek', a1: 'uz' },
|
||||
aze: { name: 'azerbaijani', a1: 'az' },
|
||||
geo: { name: 'georgian', a1: 'ka' },
|
||||
arm: { name: 'armenian', a1: 'hy' },
|
||||
est: { name: 'estonian', a1: 'et' },
|
||||
lav: { name: 'latvian', a1: 'lv' },
|
||||
lit: { name: 'lithuanian', a1: 'lt' },
|
||||
cat: { name: 'catalan', a1: 'ca' },
|
||||
glg: { name: 'galician', a1: 'gl' },
|
||||
baq: { name: 'basque', a1: 'eu' },
|
||||
wel: { name: 'welsh', a1: 'cy' },
|
||||
gle: { name: 'irish', a1: 'ga' },
|
||||
afr: { name: 'afrikaans', a1: 'af' },
|
||||
swa: { name: 'swahili', a1: 'sw' },
|
||||
amh: { name: 'amharic', a1: 'am' },
|
||||
zul: { name: 'zulu', a1: 'zu' },
|
||||
lat: { name: 'latin', a1: 'la' },
|
||||
epo: { name: 'esperanto', a1: 'eo' },
|
||||
yid: { name: 'yiddish', a1: 'yi' },
|
||||
jav: { name: 'javanese', a1: 'jv' },
|
||||
mul: { name: 'multiple' },
|
||||
};
|
||||
|
||||
// ISO 639-2/T -> 639-2/B: the only codes where the two standards disagree.
|
||||
const T2B: Record<string, string> = {
|
||||
sqi: 'alb', hye: 'arm', eus: 'baq', mya: 'bur', ces: 'cze',
|
||||
zho: 'chi', cym: 'wel', deu: 'ger', nld: 'dut', ell: 'gre', fas: 'per',
|
||||
fra: 'fre', kat: 'geo', isl: 'ice', mkd: 'mac', msa: 'may',
|
||||
ron: 'rum', slk: 'slo',
|
||||
};
|
||||
|
||||
// "This track has no language" spellings.
|
||||
const NONE = new Set(['und', 'zxx', 'mis', 'unknown', 'none']);
|
||||
|
||||
// 639-1 -> 639-2/B, from the table plus legacy two-letter codes.
|
||||
const A1: Record<string, string> = { nb: 'nor', nn: 'nor', iw: 'heb', in: 'ind', ji: 'yid', jw: 'jav' };
|
||||
const NAMES: Record<string, string> = {};
|
||||
for (const [b, entry] of Object.entries(LANGS)) {
|
||||
if (entry.a1) A1[entry.a1] = b;
|
||||
NAMES[entry.name] = b;
|
||||
}
|
||||
|
||||
/**
|
||||
* franc-min answers in ISO 639-3 individual-language codes; a few of them are
|
||||
* not 639-2 codes at all (Mandarin `cmn` is 639-2 `chi`). Folded before toLang3.
|
||||
*/
|
||||
const FRANC_TO_639_2: Record<string, string> = {
|
||||
cmn: 'chi', arb: 'ara', pes: 'per', swh: 'swa', uzn: 'uzb', zlm: 'may',
|
||||
npi: 'nep', azj: 'aze', plt: 'mlg', pbu: 'pus', ckb: 'kur', fuv: 'ful', qug: 'que',
|
||||
};
|
||||
|
||||
/** Canonical lang3 (ISO 639-2/B) for any code; `und` when there is none. */
|
||||
export function toLang3(code: unknown): string {
|
||||
const s = String(code ?? '').trim().toLowerCase();
|
||||
if (!s) return 'und';
|
||||
if (NAMES[s]) return NAMES[s];
|
||||
const p = s.split(/[-_]/)[0];
|
||||
if (NONE.has(p)) return 'und';
|
||||
if (p.length === 2) return A1[p] ?? 'und';
|
||||
if (p.length === 3 && /^[a-z]{3}$/.test(p)) return T2B[p] ?? p;
|
||||
return 'und';
|
||||
}
|
||||
|
||||
/** lang3 for a franc-min result (`und` stays `und`). */
|
||||
export function francToLang3(code: string): string {
|
||||
return toLang3(FRANC_TO_639_2[code] ?? code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict variant for filename tokens (`video.fr.srt`, `video.English.srt`):
|
||||
* only a token that is recognisably a language resolves, so `S01`, `x264` or
|
||||
* `Extended` never read as one. Returns null for anything else.
|
||||
*/
|
||||
export function languageToken(token: string): string | null {
|
||||
const t = token.trim().toLowerCase();
|
||||
if (!t) return null;
|
||||
if (NAMES[t]) return NAMES[t];
|
||||
if (t === 'und') return 'und';
|
||||
const p = t.split(/[-_]/)[0];
|
||||
if (p !== t && !/^[a-z]{2,3}[-_][a-z0-9]{2,8}$/.test(t)) return null;
|
||||
if (p.length === 2) return A1[p] ?? null;
|
||||
if (p.length === 3) {
|
||||
if (LANGS[p]) return p;
|
||||
if (T2B[p]) return T2B[p];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
+39
-64
@@ -1,6 +1,10 @@
|
||||
/**
|
||||
* meta-core API Client for writing metadata
|
||||
* Fails gracefully if meta-core is unavailable (logs warning, returns without throwing)
|
||||
* meta-core API client for writing metadata.
|
||||
*
|
||||
* Writes report success so a failed write fails the task instead of being
|
||||
* swallowed. Key-sets are written as individual `prefix/member = "true"`
|
||||
* fields through PATCH — never via `_add`, which stores one comma-joined
|
||||
* string and can never produce the key-set shape.
|
||||
*/
|
||||
|
||||
export class MetaCoreClient {
|
||||
@@ -8,86 +12,57 @@ export class MetaCoreClient {
|
||||
|
||||
private async safeFetch(url: string, options: RequestInit): Promise<Response | null> {
|
||||
try {
|
||||
return await fetch(url, { ...options, signal: AbortSignal.timeout(5000) });
|
||||
return await fetch(url, { ...options, signal: AbortSignal.timeout(10000) });
|
||||
} catch (error) {
|
||||
console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}`);
|
||||
console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async setProperty(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to set property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getProperty(hashId: string, key: string): Promise<string | null> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return null;
|
||||
if (!response.ok) return null;
|
||||
const data = await response.json() as { value?: string };
|
||||
return data.value ?? null;
|
||||
if (!response || !response.ok) return null;
|
||||
const data = await response.json() as { value?: unknown };
|
||||
return data.value === undefined || data.value === null ? null : String(data.value);
|
||||
}
|
||||
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<void> {
|
||||
/**
|
||||
* Whole record, in whatever form the endpoint serves (meta-sort's /meta is
|
||||
* nested, meta-core's is flat) — callers flatten. `null` when the call
|
||||
* itself failed, `{}` when the record does not exist.
|
||||
*/
|
||||
async getMetadata(hashId: string): Promise<Record<string, unknown> | null> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' });
|
||||
if (!response) return null;
|
||||
if (response.status === 404) return {};
|
||||
if (!response.ok) return null;
|
||||
const data = await response.json() as { metadata?: Record<string, unknown> };
|
||||
return data.metadata ?? {};
|
||||
}
|
||||
|
||||
/** PATCH — merge `metadata` into the record. True on success. */
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<boolean> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(metadata),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to merge metadata: ${response.status}`);
|
||||
if (!response) return false;
|
||||
if (!response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to merge metadata into ${hashId}: ${response.status}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteProperty(hashId: string, key: string): Promise<void> {
|
||||
/** DELETE one field. A missing field counts as deleted. */
|
||||
async deleteProperty(hashId: string, key: string): Promise<boolean> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'DELETE' });
|
||||
if (response && !response.ok && response.status !== 404) {
|
||||
console.warn(`[MetaCoreClient] Failed to delete property: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async addToSet(hashId: string, key: string, value: string): Promise<void> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/_add/${key}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ value }),
|
||||
});
|
||||
if (response && !response.ok) {
|
||||
console.warn(`[MetaCoreClient] Failed to add to set: ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(hashId: string): Promise<Record<string, string>> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' });
|
||||
if (!response || response.status === 404) return {};
|
||||
if (!response.ok) return {};
|
||||
const data = await response.json() as { metadata?: Record<string, string> };
|
||||
return data.metadata ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CID (Content Identifier) for a file
|
||||
* Uses the meta-core /file/cid API endpoint
|
||||
*/
|
||||
async computeFileCID(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/file/cid`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ path: filePath }),
|
||||
});
|
||||
if (!response || !response.ok) return null;
|
||||
const data = await response.json() as { cid?: string };
|
||||
return data.cid ?? null;
|
||||
} catch (error) {
|
||||
console.debug(`[MetaCoreClient] Failed to compute CID for ${filePath}:`, error);
|
||||
return null;
|
||||
if (!response) return false;
|
||||
if (!response.ok && response.status !== 404) {
|
||||
console.warn(`[MetaCoreClient] Failed to delete ${hashId}/${key}: ${response.status}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Reading a record regardless of the shape it arrives in, and writing only what
|
||||
* is new.
|
||||
*
|
||||
* meta-sort's scheduler fills `existingMeta` from meta-sort's own GET /meta/{hash},
|
||||
* which returns the NESTED document (`fileinfo: {duration: 1433.089}`,
|
||||
* `stream: [...]`, a key-set as `subtitles: {fre: {<cid>: true}}`). meta-core's
|
||||
* backend serves the FLAT form (`fileinfo/duration`, `stream/0`,
|
||||
* `subtitles/fre/<cid>`). A plugin that reads `existingMeta['x/y']` silently sees
|
||||
* nothing in production — so everything here goes through `flattenMeta`.
|
||||
*
|
||||
* Byte-identical in metamesh-plugin-subtitle and metamesh-plugin-subtitle-extractor.
|
||||
*/
|
||||
|
||||
/** Flatten either form into `a/b/c -> string` (meta-sort's flattenMetadata, plus array indices). */
|
||||
export function flattenMeta(meta: Record<string, unknown> | undefined | null): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const walk = (value: unknown, key: string) => {
|
||||
if (value === null || value === undefined) return;
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((v, i) => walk(v, key ? `${key}/${i}` : String(i)));
|
||||
} else if (typeof value === 'object') {
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
||||
walk(v, key ? `${key}/${k}` : k);
|
||||
}
|
||||
} else if (key) {
|
||||
out[key] = String(value);
|
||||
}
|
||||
};
|
||||
walk(meta ?? {}, '');
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The subset of `wanted` the record does not already hold with that exact value. */
|
||||
export function onlyNewKeys(existingFlat: Record<string, string>, wanted: Record<string, string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(wanted)) {
|
||||
if (existingFlat[k] !== v) out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Members of a LEGACY scalar csv-set (`subtitles = "cidA,cidB"`, written by the
|
||||
* old `_add` path), or null when the field is absent or already a key-set.
|
||||
*
|
||||
* Such a scalar must be deleted before any `field/...` key is written: meta-sort
|
||||
* rebuilds its nested document by walking the `/`-segments, and a string sitting
|
||||
* where an object is needed either throws or silently hides the key-set.
|
||||
*/
|
||||
export function legacyCsvMembers(meta: Record<string, unknown> | undefined | null, field: string): string[] | null {
|
||||
const v = meta?.[field];
|
||||
if (typeof v !== 'string' && typeof v !== 'number') return null;
|
||||
return String(v).split(/[,|]/).map((s) => s.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
/** Every `<prefix>/<lang>/<cid>` leaf for `cid` whose language is not `lang3`. */
|
||||
export function staleLanguageLeaves(existingFlat: Record<string, string>, prefix: string, cid: string, lang3: string): string[] {
|
||||
const stale: string[] = [];
|
||||
for (const key of Object.keys(existingFlat)) {
|
||||
const parts = key.split('/');
|
||||
if (parts.length === 3 && parts[0] === prefix && parts[2] === cid && parts[1] !== lang3) stale.push(key);
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
+433
-135
@@ -1,37 +1,78 @@
|
||||
/**
|
||||
* Subtitle Plugin
|
||||
*
|
||||
* For subtitle files: finds sibling video files and detects language.
|
||||
* For video files: finds sibling subtitle files.
|
||||
* Links subtitle SIDECAR files to the video they sit next to, in both
|
||||
* directions, and determines each sidecar's language.
|
||||
*
|
||||
* Matches old SubtitleProcessor output:
|
||||
* - For subtitle files: videos (add CID), subtitleLanguage
|
||||
* - For video files: subtitles (add CID)
|
||||
* ============================================================================
|
||||
* WHAT IT WRITES (METADATA_KEYS.md §8)
|
||||
* ============================================================================
|
||||
* On the video record, per sidecar `<subCid>` (midhash256) in `<lang3>`
|
||||
* (ISO 639-2/B, `und` when undetermined):
|
||||
* subtitles/<lang3>/<subCid> = "true" what meta-watch lists
|
||||
* subtitleLanguages/<lang3> = "true" (not for `und`)
|
||||
* On the sidecar's own record:
|
||||
* videos/<videoCid> = "true"
|
||||
* subtitleLanguage = <lang3> (not for `und`)
|
||||
*
|
||||
* Whichever of the two records is processed first, the result is the same: each
|
||||
* run writes its own record in full, and the other record only when it already
|
||||
* exists (no stub records carrying nothing but subtitle keys). Writes are one
|
||||
* PATCH of only-new keys; a failed write fails the task.
|
||||
*
|
||||
* ============================================================================
|
||||
* WHAT COUNTS AS A SIDECAR
|
||||
* ============================================================================
|
||||
* `<video basename>.<ext>` or `<video basename>.<qualifiers>.<ext>`, compared
|
||||
* case-insensitively, `<ext>` in srt/ass/ssa/vtt, where every qualifier token is
|
||||
* a language (`en`, `eng`, `fre`, `pt-BR`, `English`) or a flag (`forced`, `sdh`,
|
||||
* `cc`, `hi`, `default`, `full`, `signs`, `songs`). So `Movie.fr.forced.srt`
|
||||
* belongs to `Movie.mkv`, but `Movie.Extended.eng.srt` does not.
|
||||
* Language: the qualifier token when there is one, else a franc-min sniff of the
|
||||
* subtitle text (timecodes and ASS markup stripped), else `und`. `hi` is read as
|
||||
* the hearing-impaired flag, not Hindi (use `hin`).
|
||||
*
|
||||
* ============================================================================
|
||||
* FILE ACCESS - WebDAV only
|
||||
* ============================================================================
|
||||
* The container has no /files mount. Directory listings, size and bytes all come
|
||||
* over WebDAV from the meta-core that drove the /process call (its /urls ->
|
||||
* webdavUrlInternal; WEBDAV_URL overrides).
|
||||
*/
|
||||
|
||||
import { readFile, access, open } from 'fs/promises';
|
||||
import { dirname, extname } from 'path';
|
||||
import { posix } from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import { franc } from 'franc-min';
|
||||
import { FileType, getSiblingFiles } from '@metazla/filename-tools';
|
||||
import { FileType } from '@metazla/filename-tools';
|
||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||
import { MetaCoreClient } from './meta-core-client.js';
|
||||
import { createWebDAVClient, WebDAVClient } from './webdav-client.js';
|
||||
import { getWebDAVClient, WebDAVClient, DirectoryEntry } from './webdav-client.js';
|
||||
import { languageToken, francToLang3 } from './langs.js';
|
||||
import { flattenMeta, onlyNewKeys, legacyCsvMembers, staleLanguageLeaves } from './meta-shape.js';
|
||||
|
||||
// Initialize WebDAV client if WEBDAV_URL is set
|
||||
const webdavClient = createWebDAVClient();
|
||||
if (webdavClient) {
|
||||
console.log('[subtitle] Using WebDAV for file access');
|
||||
} else {
|
||||
console.log('[subtitle] Using direct filesystem access');
|
||||
}
|
||||
const fileTypeTool = new FileType();
|
||||
|
||||
const fileType = new FileType();
|
||||
export const SIDECAR_EXTENSIONS = new Set(['srt', 'ass', 'ssa', 'vtt']);
|
||||
|
||||
/** Bytes of subtitle text read for language detection (ASS style headers can run several KB). */
|
||||
const DETECT_BYTES = 64 * 1024;
|
||||
|
||||
const FLAG_TOKENS: Record<string, 'forced' | 'hearingImpaired' | 'other'> = {
|
||||
forced: 'forced',
|
||||
sdh: 'hearingImpaired',
|
||||
cc: 'hearingImpaired',
|
||||
hi: 'hearingImpaired',
|
||||
default: 'other',
|
||||
full: 'other',
|
||||
signs: 'other',
|
||||
songs: 'other',
|
||||
};
|
||||
|
||||
export const manifest: PluginManifest = {
|
||||
id: 'subtitle',
|
||||
name: 'Subtitle Processor',
|
||||
version: '1.0.0',
|
||||
description: 'Processes subtitle files and links them to videos',
|
||||
version: '1.1.0',
|
||||
description: 'Links subtitle sidecar files to their videos and detects their language',
|
||||
author: 'MetaMesh',
|
||||
dependencies: ['file-info'],
|
||||
priority: 45,
|
||||
@@ -40,40 +81,362 @@ export const manifest: PluginManifest = {
|
||||
timeout: 30000,
|
||||
schema: {
|
||||
subtitleLanguage: { label: 'Subtitle Language', type: 'string', readonly: true },
|
||||
videos: { label: 'Linked Videos', type: 'array', readonly: true },
|
||||
subtitles: { label: 'Linked Subtitles', type: 'array', readonly: true },
|
||||
videos: { label: 'Linked Videos', type: 'json', readonly: true, hint: 'videos/<cid>' },
|
||||
subtitles: { label: 'Linked Subtitles', type: 'json', readonly: true, hint: 'subtitles/<lang3>/<cid>' },
|
||||
},
|
||||
config: {},
|
||||
};
|
||||
|
||||
async function fileExists(path: string): Promise<boolean> {
|
||||
if (webdavClient) {
|
||||
return webdavClient.exists(path);
|
||||
// ----------------------------------------------------------------------------
|
||||
// Names
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
export function splitExtension(fileName: string): { base: string; ext: string } {
|
||||
const dot = fileName.lastIndexOf('.');
|
||||
if (dot <= 0) return { base: fileName, ext: '' };
|
||||
return { base: fileName.slice(0, dot), ext: fileName.slice(dot + 1).toLowerCase() };
|
||||
}
|
||||
|
||||
export interface SubtitleQualifiers {
|
||||
lang3?: string;
|
||||
forced: boolean;
|
||||
hearingImpaired: boolean;
|
||||
}
|
||||
|
||||
/** Qualifier tokens (`fr`, `forced`, …); null as soon as one is neither a language nor a flag. */
|
||||
export function parseQualifierTokens(tokens: string[]): SubtitleQualifiers | null {
|
||||
const q: SubtitleQualifiers = { forced: false, hearingImpaired: false };
|
||||
for (const raw of tokens) {
|
||||
const token = raw.trim().toLowerCase();
|
||||
if (!token) continue;
|
||||
const flag = FLAG_TOKENS[token];
|
||||
if (flag === 'forced') q.forced = true;
|
||||
else if (flag === 'hearingImpaired') q.hearingImpaired = true;
|
||||
else if (flag !== 'other') {
|
||||
const lang = languageToken(token);
|
||||
if (!lang) return null;
|
||||
q.lang3 ??= lang;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await access(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
return q;
|
||||
}
|
||||
|
||||
/** Is `candidate` a sidecar of `videoFileName`, and what do its qualifiers say? */
|
||||
export function parseSidecarName(videoFileName: string, candidate: string): (SubtitleQualifiers & { ext: string }) | null {
|
||||
const { ext } = splitExtension(candidate);
|
||||
if (!SIDECAR_EXTENSIONS.has(ext)) return null;
|
||||
const videoBase = splitExtension(videoFileName).base.toLowerCase();
|
||||
const base = splitExtension(candidate).base;
|
||||
if (!videoBase) return null;
|
||||
if (base.toLowerCase() === videoBase) return { ext, forced: false, hearingImpaired: false };
|
||||
if (!base.toLowerCase().startsWith(`${videoBase}.`)) return null;
|
||||
const q = parseQualifierTokens(base.slice(videoBase.length + 1).split('.'));
|
||||
return q ? { ...q, ext } : null;
|
||||
}
|
||||
|
||||
/** Qualifiers read off the end of a subtitle's own name, when no sibling video anchors it. */
|
||||
export function trailingQualifiers(fileName: string): SubtitleQualifiers {
|
||||
const tokens = splitExtension(fileName).base.split('.');
|
||||
const q: SubtitleQualifiers = { forced: false, hearingImpaired: false };
|
||||
// Keep at least one token as the name proper.
|
||||
for (let i = tokens.length - 1; i >= 1; i--) {
|
||||
const one = parseQualifierTokens([tokens[i]]);
|
||||
if (!one) break;
|
||||
q.forced ||= one.forced;
|
||||
q.hearingImpaired ||= one.hearingImpaired;
|
||||
// Walking backwards: the token nearest the name wins, as in parseQualifierTokens.
|
||||
if (one.lang3) q.lang3 = one.lang3;
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Language sniffing
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
/** The spoken text of a subtitle file: no counters, timecodes, headers or markup. */
|
||||
export function cleanSubtitleText(raw: string, ext: string): string {
|
||||
const lines = raw.replace(/^\uFEFF/, '').split(/\r?\n/);
|
||||
let text: string[];
|
||||
if (ext === 'ass' || ext === 'ssa') {
|
||||
text = lines
|
||||
.filter((l) => l.startsWith('Dialogue:'))
|
||||
.map((l) => l.split(',').slice(9).join(','))
|
||||
.map((l) => l.replace(/\{[^}]*\}/g, '').replace(/\\[Nnh]/g, ' '));
|
||||
} else {
|
||||
text = lines.filter((l) => {
|
||||
const t = l.trim();
|
||||
return t && !/^\d+$/.test(t) && !t.includes('-->') && !/^WEBVTT/.test(t) && !/^(NOTE|STYLE|REGION)\b/.test(t);
|
||||
});
|
||||
}
|
||||
return text
|
||||
.join(' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\{[^}]*\}/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** lang3 from subtitle text, `und` when there is too little to tell. */
|
||||
export function detectLanguageFromText(text: string): string {
|
||||
if (text.length < 20) return 'und';
|
||||
const code = franc(text);
|
||||
return code === 'und' ? 'und' : francToLang3(code);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// CIDs
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
const MIDHASH_SAMPLE_SIZE = 1024 * 1024;
|
||||
|
||||
/** midhash256 CID from a file's size and its middle sample (matches meta-hash). */
|
||||
export function computeMidHash256(fileSize: number, sampleData: Buffer): string {
|
||||
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||
const hashBuffer = createHash('sha256').update(Buffer.concat([sizeBuffer, sampleData])).digest();
|
||||
const cidBytes = Buffer.concat([Buffer.from([0x01]), MIDHASH_VARINT, MIDHASH_VARINT, Buffer.from([0x20]), hashBuffer]);
|
||||
|
||||
const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
let cid = 'b';
|
||||
let bits = 0;
|
||||
let value = 0;
|
||||
for (const byte of cidBytes) {
|
||||
value = (value << 8) | byte;
|
||||
bits += 8;
|
||||
while (bits >= 5) {
|
||||
bits -= 5;
|
||||
cid += base32Chars[(value >> bits) & 0x1f];
|
||||
}
|
||||
}
|
||||
if (bits > 0) {
|
||||
cid += base32Chars[(value << (5 - bits)) & 0x1f];
|
||||
}
|
||||
return cid;
|
||||
}
|
||||
|
||||
export function midhashSampleRange(fileSize: number): { start: number; end: number } {
|
||||
if (fileSize <= MIDHASH_SAMPLE_SIZE) return { start: 0, end: fileSize - 1 };
|
||||
const start = Math.floor((fileSize - MIDHASH_SAMPLE_SIZE) / 2);
|
||||
return { start, end: start + MIDHASH_SAMPLE_SIZE - 1 };
|
||||
}
|
||||
|
||||
export function computeMidHash256FromBuffer(data: Buffer): string {
|
||||
const { start, end } = midhashSampleRange(data.length);
|
||||
return computeMidHash256(data.length, data.subarray(start, end + 1));
|
||||
}
|
||||
|
||||
/** CID of a file over WebDAV: one Range read of at most 1 MiB, however large the file. */
|
||||
export async function midhashOverWebDAV(client: WebDAVClient, filePath: string, size?: number): Promise<string> {
|
||||
const fileSize = size ?? (await client.stat(filePath)).size;
|
||||
const { start, end } = midhashSampleRange(fileSize);
|
||||
const sample = fileSize === 0 ? Buffer.alloc(0) : await client.readBytes(filePath, start, end);
|
||||
if (sample.length !== end - start + 1) {
|
||||
throw new Error(`short read for ${filePath}: wanted ${end - start + 1} bytes, got ${sample.length}`);
|
||||
}
|
||||
return computeMidHash256(fileSize, sample);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Record keys
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
export function videoSubtitleKeys(subCid: string, lang3: string): Record<string, string> {
|
||||
const keys: Record<string, string> = { [`subtitles/${lang3}/${subCid}`]: 'true' };
|
||||
if (lang3 !== 'und') keys[`subtitleLanguages/${lang3}`] = 'true';
|
||||
return keys;
|
||||
}
|
||||
|
||||
export function subtitleRecordKeys(lang3: string, videoCids: string[]): Record<string, string> {
|
||||
const keys: Record<string, string> = {};
|
||||
for (const v of videoCids) keys[`videos/${v}`] = 'true';
|
||||
if (lang3 !== 'und') keys.subtitleLanguage = lang3;
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function detectFromFile(client: WebDAVClient, filePath: string, ext: string, size?: number): Promise<string> {
|
||||
if (size === 0) return 'und';
|
||||
const end = size !== undefined ? Math.min(size, DETECT_BYTES) - 1 : DETECT_BYTES - 1;
|
||||
const bytes = await client.readBytes(filePath, 0, end);
|
||||
return detectLanguageFromText(cleanSubtitleText(bytes.toString('utf8'), ext));
|
||||
}
|
||||
|
||||
/**
|
||||
* Read first N bytes of a file
|
||||
* Link `subCid`/`lang3` onto a video record that is not the one being processed:
|
||||
* only if it exists, only new keys, and with any other-language leaf for the
|
||||
* same CID removed (one file, one language).
|
||||
*/
|
||||
async function readFirstData(filePath: string, encoding: BufferEncoding = 'utf8', size = 1024): Promise<string> {
|
||||
if (webdavClient) {
|
||||
const buffer = await webdavClient.readBytes(filePath, 0, size - 1);
|
||||
return buffer.toString(encoding);
|
||||
async function linkOntoVideo(metaCore: MetaCoreClient, videoCid: string, subCid: string, lang3: string): Promise<boolean> {
|
||||
const current = await metaCore.getMetadata(videoCid);
|
||||
if (current === null) return false;
|
||||
const flat = flattenMeta(current);
|
||||
if (!flat.fileType) return true; // not indexed yet — its own run links it
|
||||
return writeVideoLinks(metaCore, videoCid, current, flat, [{ subCid, lang3 }]);
|
||||
}
|
||||
|
||||
async function writeVideoLinks(
|
||||
metaCore: MetaCoreClient,
|
||||
videoCid: string,
|
||||
meta: Record<string, unknown>,
|
||||
flat: Record<string, string>,
|
||||
links: Array<{ subCid: string; lang3: string }>,
|
||||
): Promise<boolean> {
|
||||
const deletes: string[] = [];
|
||||
const sets: Record<string, string> = {};
|
||||
|
||||
// Legacy scalar csv-sets from the `_add` era. Members still found on disk are
|
||||
// re-linked with their language below; the rest are kept as `und` leaves.
|
||||
const legacySubs = legacyCsvMembers(meta, 'subtitles');
|
||||
if (legacySubs) {
|
||||
deletes.push('subtitles');
|
||||
for (const cid of legacySubs) {
|
||||
if (!links.some((l) => l.subCid === cid)) sets[`subtitles/und/${cid}`] = 'true';
|
||||
}
|
||||
}
|
||||
const handle = await open(filePath, 'r');
|
||||
try {
|
||||
const buffer = Buffer.alloc(size);
|
||||
await handle.read(buffer, 0, size, 0);
|
||||
return buffer.toString(encoding);
|
||||
} finally {
|
||||
await handle.close();
|
||||
const legacyLangs = legacyCsvMembers(meta, 'subtitleLanguages');
|
||||
if (legacyLangs) {
|
||||
deletes.push('subtitleLanguages');
|
||||
for (const code of legacyLangs) {
|
||||
const l = languageToken(code);
|
||||
if (l && l !== 'und') sets[`subtitleLanguages/${l}`] = 'true';
|
||||
}
|
||||
}
|
||||
for (const { subCid, lang3 } of links) deletes.push(...staleLanguageLeaves(flat, 'subtitles', subCid, lang3));
|
||||
|
||||
for (const key of deletes) {
|
||||
if (!(await metaCore.deleteProperty(videoCid, key))) return false;
|
||||
delete flat[key];
|
||||
}
|
||||
|
||||
let wanted: Record<string, string> = { ...sets };
|
||||
for (const { subCid, lang3 } of links) wanted = { ...wanted, ...videoSubtitleKeys(subCid, lang3) };
|
||||
const fresh = onlyNewKeys(flat, wanted);
|
||||
return Object.keys(fresh).length === 0 || metaCore.mergeMetadata(videoCid, fresh);
|
||||
}
|
||||
|
||||
async function writeSubtitleRecord(
|
||||
metaCore: MetaCoreClient,
|
||||
subCid: string,
|
||||
meta: Record<string, unknown>,
|
||||
lang3: string,
|
||||
videoCids: string[],
|
||||
): Promise<boolean> {
|
||||
const flat = flattenMeta(meta);
|
||||
const sets: Record<string, string> = {};
|
||||
const legacyVideos = legacyCsvMembers(meta, 'videos');
|
||||
if (legacyVideos) {
|
||||
if (!(await metaCore.deleteProperty(subCid, 'videos'))) return false;
|
||||
delete flat.videos;
|
||||
for (const v of legacyVideos) sets[`videos/${v}`] = 'true';
|
||||
}
|
||||
const fresh = onlyNewKeys(flat, { ...sets, ...subtitleRecordKeys(lang3, videoCids) });
|
||||
return Object.keys(fresh).length === 0 || metaCore.mergeMetadata(subCid, fresh);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Task
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
type Outcome = { ok: true; summary: string } | { ok: false; error: string };
|
||||
|
||||
/** Video record: find its sidecars, link them. */
|
||||
export async function processVideoFile(
|
||||
client: WebDAVClient,
|
||||
metaCore: MetaCoreClient,
|
||||
videoCid: string,
|
||||
filePath: string,
|
||||
meta: Record<string, unknown>,
|
||||
): Promise<Outcome> {
|
||||
const dir = posix.dirname(filePath);
|
||||
const videoName = posix.basename(filePath);
|
||||
const entries = await client.list(dir);
|
||||
|
||||
const links: Array<{ subCid: string; lang3: string; entry: DirectoryEntry }> = [];
|
||||
for (const entry of entries) {
|
||||
if (entry.type !== 'file' || entry.size === 0) continue;
|
||||
const q = parseSidecarName(videoName, entry.name);
|
||||
if (!q) continue;
|
||||
const subPath = `${dir}/${entry.name}`;
|
||||
const subCid = await midhashOverWebDAV(client, subPath, entry.size);
|
||||
const lang3 = q.lang3 ?? (await detectFromFile(client, subPath, q.ext, entry.size));
|
||||
links.push({ subCid, lang3, entry });
|
||||
}
|
||||
|
||||
const flat = flattenMeta(meta);
|
||||
if (!(await writeVideoLinks(metaCore, videoCid, meta, flat, links))) {
|
||||
return { ok: false, error: 'meta-core write failed (video record)' };
|
||||
}
|
||||
|
||||
for (const { subCid, lang3 } of links) {
|
||||
const subMeta = await metaCore.getMetadata(subCid);
|
||||
if (subMeta === null) return { ok: false, error: `meta-core read failed (${subCid})` };
|
||||
if (!flattenMeta(subMeta).fileType) continue; // not indexed yet — its own run links it
|
||||
if (!(await writeSubtitleRecord(metaCore, subCid, subMeta, lang3, [videoCid]))) {
|
||||
return { ok: false, error: `meta-core write failed (${subCid})` };
|
||||
}
|
||||
}
|
||||
return { ok: true, summary: `${links.length} sidecar(s) [${links.map((l) => l.lang3).join(', ')}]` };
|
||||
}
|
||||
|
||||
/** Subtitle record: find the video(s) it belongs to, determine its language, link. */
|
||||
export async function processSubtitleFile(
|
||||
client: WebDAVClient,
|
||||
metaCore: MetaCoreClient,
|
||||
subCid: string,
|
||||
filePath: string,
|
||||
meta: Record<string, unknown>,
|
||||
): Promise<Outcome> {
|
||||
const dir = posix.dirname(filePath);
|
||||
const subName = posix.basename(filePath);
|
||||
const ext = splitExtension(subName).ext;
|
||||
|
||||
// Plugin output folders (subtitle-extractor, opensubtitles, meta-watch uploads)
|
||||
// hold no videos; their names already end in `.<lang3>.<ext>`.
|
||||
const videos: DirectoryEntry[] = [];
|
||||
if (!dir.startsWith('/files/plugin/') && SIDECAR_EXTENSIONS.has(ext)) {
|
||||
for (const entry of await client.list(dir)) {
|
||||
if (entry.type !== 'file') continue;
|
||||
if (fileTypeTool.getFileTypeFromExtension(entry.name) !== 'video') continue;
|
||||
if (parseSidecarName(entry.name, subName)) videos.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Same qualifiers the video-side run derives, so both runs agree on the language.
|
||||
const q = videos.length > 0 ? parseSidecarName(videos[0].name, subName)! : trailingQualifiers(subName);
|
||||
const size = meta.sizeByte !== undefined && Number.isFinite(Number(meta.sizeByte)) ? Number(meta.sizeByte) : undefined;
|
||||
const lang3 = q.lang3 ?? (await detectFromFile(client, filePath, ext, size));
|
||||
|
||||
const videoCids: string[] = [];
|
||||
for (const v of videos) videoCids.push(await midhashOverWebDAV(client, `${dir}/${v.name}`, v.size));
|
||||
|
||||
if (!(await writeSubtitleRecord(metaCore, subCid, meta, lang3, videoCids))) {
|
||||
return { ok: false, error: 'meta-core write failed (subtitle record)' };
|
||||
}
|
||||
for (const videoCid of videoCids) {
|
||||
if (!(await linkOntoVideo(metaCore, videoCid, subCid, lang3))) {
|
||||
return { ok: false, error: `meta-core write failed (${videoCid})` };
|
||||
}
|
||||
}
|
||||
return { ok: true, summary: `lang=${lang3}, ${videoCids.length} video(s)` };
|
||||
}
|
||||
|
||||
/**
|
||||
* A subtitle file the byte classifier could not place. file-info returns
|
||||
* `undefined` for a `text/plain` sniff (METADATA_KEYS.md `fileType`: "could be a
|
||||
* document, a subtitle or anything") and has been seen writing `other` for
|
||||
* extracted `.ass` files — either way the `fileType === 'subtitle'` gate alone
|
||||
* would never let this plugin see the file. The extension decides then.
|
||||
*/
|
||||
export function isUnclassifiedSubtitle(meta: Record<string, unknown>, filePath: string): boolean {
|
||||
const fileType = meta.fileType;
|
||||
if (fileType !== undefined && fileType !== null && fileType !== '' && fileType !== 'other' && fileType !== 'undefined') {
|
||||
return false;
|
||||
}
|
||||
const ext = typeof meta.extension === 'string' && meta.extension
|
||||
? meta.extension.toLowerCase()
|
||||
: splitExtension(posix.basename(filePath)).ext;
|
||||
return SIDECAR_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
export async function process(
|
||||
@@ -81,104 +444,39 @@ export async function process(
|
||||
sendCallback: (payload: CallbackPayload) => Promise<void>
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||
const { taskId, cid, filePath } = request;
|
||||
const meta = (request.existingMeta ?? {}) as Record<string, unknown>;
|
||||
const finish = (status: CallbackPayload['status'], extra: { reason?: string; error?: string } = {}) =>
|
||||
sendCallback({ taskId, status, duration: Date.now() - startTime, ...extra });
|
||||
|
||||
try {
|
||||
const { cid, filePath, existingMeta } = request;
|
||||
const currentFileType = existingMeta?.fileType;
|
||||
|
||||
if (currentFileType === 'subtitle') {
|
||||
await processSubtitleFile(metaCore, cid, filePath);
|
||||
} else if (currentFileType === 'video') {
|
||||
await processVideoFile(metaCore, cid, filePath, existingMeta);
|
||||
const fileType = meta.fileType === 'subtitle' || isUnclassifiedSubtitle(meta, filePath) ? 'subtitle' : meta.fileType;
|
||||
if (fileType !== 'video' && fileType !== 'subtitle') {
|
||||
await finish('skipped', { reason: 'Not a video or subtitle file' });
|
||||
return;
|
||||
}
|
||||
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'completed',
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
const client = await getWebDAVClient(request.metaCoreUrl);
|
||||
if (!client) {
|
||||
console.error(`[subtitle] ${filePath}: no WebDAV endpoint available`);
|
||||
await finish('failed', { error: 'No WebDAV endpoint available' });
|
||||
return;
|
||||
}
|
||||
|
||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||
const outcome = fileType === 'subtitle'
|
||||
? await processSubtitleFile(client, metaCore, cid, filePath, meta)
|
||||
: await processVideoFile(client, metaCore, cid, filePath, meta);
|
||||
|
||||
if (!outcome.ok) {
|
||||
console.error(`[subtitle] ${filePath}: ${outcome.error}`);
|
||||
await finish('failed', { error: outcome.error });
|
||||
return;
|
||||
}
|
||||
console.log(`[subtitle] ${filePath}: ${outcome.summary}`);
|
||||
await finish('completed');
|
||||
} catch (error) {
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
status: 'failed',
|
||||
duration: Date.now() - startTime,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process subtitle file: find sibling videos and detect language
|
||||
*/
|
||||
async function processSubtitleFile(
|
||||
metaCore: MetaCoreClient,
|
||||
cid: string,
|
||||
filePath: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Find sibling video files
|
||||
const siblings = await getSiblingFiles(filePath);
|
||||
|
||||
for (const siblingPath of siblings) {
|
||||
const siblingType = fileType.getFileTypeFromExtension(siblingPath);
|
||||
if (siblingType === 'video') {
|
||||
try {
|
||||
const videoCid = await metaCore.computeFileCID(siblingPath);
|
||||
if (videoCid) {
|
||||
await metaCore.addToSet(cid, 'videos', videoCid);
|
||||
}
|
||||
} catch (e) {
|
||||
console.debug(`[subtitle] Could not get CID for video: ${siblingPath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Detect subtitle language by reading first 4KB
|
||||
try {
|
||||
const content = await readFirstData(filePath, 'utf8', 4096);
|
||||
const detectedLang = franc(content);
|
||||
if (detectedLang && detectedLang !== 'und') {
|
||||
await metaCore.setProperty(cid, 'subtitleLanguage', detectedLang);
|
||||
}
|
||||
} catch (e) {
|
||||
console.debug(`[subtitle] Could not detect language for: ${filePath}`);
|
||||
}
|
||||
|
||||
const mode = webdavClient ? 'WebDAV' : 'filesystem';
|
||||
console.log(`[subtitle] Processed subtitle file: ${filePath} (${mode})`);
|
||||
} catch (error) {
|
||||
console.error(`[subtitle] Error processing subtitle file ${filePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process video file: find sibling subtitle files
|
||||
*/
|
||||
async function processVideoFile(
|
||||
metaCore: MetaCoreClient,
|
||||
cid: string,
|
||||
filePath: string,
|
||||
existingMeta?: Record<string, string>
|
||||
): Promise<void> {
|
||||
try {
|
||||
const extension = existingMeta?.extension || '';
|
||||
const srtPath = filePath.replace(new RegExp(`\\.${extension}$`, 'i'), '.srt');
|
||||
|
||||
// Check for .srt subtitle
|
||||
if (await fileExists(srtPath)) {
|
||||
try {
|
||||
const subtitleCid = await metaCore.computeFileCID(srtPath);
|
||||
if (subtitleCid) {
|
||||
await metaCore.addToSet(cid, 'subtitles', subtitleCid);
|
||||
}
|
||||
} catch (e) {
|
||||
console.debug(`[subtitle] Could not get CID for subtitle: ${srtPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[subtitle] Checked subtitles for video: ${filePath}`);
|
||||
} catch (error) {
|
||||
console.error(`[subtitle] Error finding subtitles for ${filePath}:`, error);
|
||||
console.error(`[subtitle] ${filePath}:`, error);
|
||||
await finish('failed', { error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@ export interface ProcessRequest {
|
||||
filePath: string;
|
||||
callbackUrl: string;
|
||||
metaCoreUrl: string;
|
||||
existingMeta?: Record<string, string>;
|
||||
// meta-sort sends its NESTED /meta document (`fileinfo: {...}`, `stream: [...]`);
|
||||
// meta-core serves the flat `a/b` form. Top-level scalars match in both.
|
||||
existingMeta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ProcessResponse {
|
||||
|
||||
+97
-20
@@ -64,7 +64,7 @@ export class WebDAVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read first N bytes of a file (for language detection)
|
||||
* Read first N bytes of a file (for magic byte detection)
|
||||
* Uses HTTP Range request
|
||||
*/
|
||||
async readBytes(filePath: string, start: number, end: number): Promise<Buffer> {
|
||||
@@ -86,7 +86,7 @@ export class WebDAVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read entire file as buffer (for small files)
|
||||
* Read entire file as buffer (for small files only)
|
||||
*/
|
||||
async readFile(filePath: string): Promise<Buffer> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
@@ -101,6 +101,25 @@ export class WebDAVClient {
|
||||
return Buffer.from(arrayBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write file via HTTP PUT
|
||||
*/
|
||||
async writeFile(filePath: string, data: Buffer | string): Promise<void> {
|
||||
const url = this.toWebDAVUrl(filePath);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'PUT',
|
||||
body: data,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV PUT failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file exists via HTTP HEAD
|
||||
*/
|
||||
@@ -116,38 +135,96 @@ export class WebDAVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* List directory contents via HTTP GET (returns JSON from nginx autoindex)
|
||||
* List a directory: GET <dir>/ with Accept: application/json, which both
|
||||
* meta-core's WebDAV and meta-sort's nginx autoindex answer with
|
||||
* [{name, type: "file"|"directory", size, mtime}].
|
||||
*
|
||||
* (Addition to the tmdb/still-extractor copy of this file: the subtitle
|
||||
* plugin has no /files mount, so sibling discovery must go over WebDAV.)
|
||||
*/
|
||||
async readdir(dirPath: string): Promise<string[]> {
|
||||
async list(dirPath: string): Promise<DirectoryEntry[]> {
|
||||
const url = this.toWebDAVUrl(dirPath);
|
||||
const finalUrl = url.endsWith('/') ? url : url + '/';
|
||||
|
||||
const response = await fetch(finalUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
const response = await fetch(finalUrl, { method: 'GET', headers: { 'Accept': 'application/json' } });
|
||||
if (!response.ok) {
|
||||
throw new Error(`WebDAV GET (dir) failed for ${dirPath}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const entries = await response.json() as Array<{ name: string; type: string }>;
|
||||
return entries.map(e => e.name);
|
||||
const entries = await response.json() as Array<{ name: string; type: string; size?: number }>;
|
||||
return entries.map((e) => ({
|
||||
name: e.name,
|
||||
type: e.type === 'directory' ? 'directory' : 'file',
|
||||
size: typeof e.size === 'number' ? e.size : undefined,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
export interface DirectoryEntry {
|
||||
name: string;
|
||||
type: 'file' | 'directory';
|
||||
size?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a WebDAV client from environment variables
|
||||
* WebDAV client for the meta-core that drove a /process call.
|
||||
*
|
||||
* The endpoint is read from that core's own /urls (`webdavUrlInternal`),
|
||||
* so blobs always land in the /files of the core we write metadata to. A feeder
|
||||
* whose plugins run beside it but enrich a remote hub core needs no per-stack
|
||||
* wiring, and it is impossible to write posters into one core while pointing
|
||||
* CIDs at another.
|
||||
*
|
||||
* WEBDAV_URL still wins when set — an explicit override for tests and for
|
||||
* topologies where the core's advertised internal URL isn't reachable from here.
|
||||
*
|
||||
* Resolutions are cached per meta-core URL; failures are not, so a core that is
|
||||
* still starting up is retried on the next task.
|
||||
*/
|
||||
export function createWebDAVClient(): WebDAVClient | null {
|
||||
const webdavUrl = process.env.WEBDAV_URL;
|
||||
const clientCache = new Map<string, Promise<WebDAVClient | null>>();
|
||||
|
||||
if (!webdavUrl) {
|
||||
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable');
|
||||
return null;
|
||||
async function resolveWebDAVUrl(metaCoreUrl: string): Promise<string | null> {
|
||||
// /urls, not /api/urls — metaCoreUrl points at the meta-core backend, whose
|
||||
// discovery routes are unprefixed (same base the /meta/* writes use).
|
||||
const base = metaCoreUrl.replace(/\/$/, '');
|
||||
const response = await fetch(`${base}/urls`, { signal: AbortSignal.timeout(5000) });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`GET ${base}/urls failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return new WebDAVClient(webdavUrl);
|
||||
const urls = await response.json() as { webdavUrlInternal?: string; webdavUrl?: string };
|
||||
return urls.webdavUrlInternal ?? urls.webdavUrl ?? null;
|
||||
}
|
||||
|
||||
export async function getWebDAVClient(metaCoreUrl: string): Promise<WebDAVClient | null> {
|
||||
const override = process.env.WEBDAV_URL;
|
||||
if (override) {
|
||||
return new WebDAVClient(override);
|
||||
}
|
||||
|
||||
let pending = clientCache.get(metaCoreUrl);
|
||||
if (!pending) {
|
||||
pending = resolveWebDAVUrl(metaCoreUrl).then((webdavUrl) => {
|
||||
if (!webdavUrl) {
|
||||
console.warn(`[webdav-client] meta-core ${metaCoreUrl} advertises no WebDAV URL`);
|
||||
return null;
|
||||
}
|
||||
console.log(`[webdav-client] Using WebDAV endpoint ${webdavUrl} (from ${metaCoreUrl})`);
|
||||
return new WebDAVClient(webdavUrl);
|
||||
});
|
||||
clientCache.set(metaCoreUrl, pending);
|
||||
}
|
||||
|
||||
try {
|
||||
const client = await pending;
|
||||
if (!client) {
|
||||
clientCache.delete(metaCoreUrl);
|
||||
}
|
||||
return client;
|
||||
} catch (error) {
|
||||
clientCache.delete(metaCoreUrl);
|
||||
console.warn(`[webdav-client] Failed to resolve WebDAV URL from ${metaCoreUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Subtitle Plugin Test Runner
|
||||
#
|
||||
# Runs integration tests inside a Docker container.
|
||||
# Usage:
|
||||
# ./test.sh # Build and run tests
|
||||
# ./test.sh --no-cache # Rebuild without cache
|
||||
# ./test.sh --shell # Start shell in test container for debugging
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
IMAGE_NAME="metamesh-plugin-subtitle-test"
|
||||
DOCKERFILE="Dockerfile.test"
|
||||
|
||||
# Parse arguments
|
||||
NO_CACHE=""
|
||||
SHELL_MODE=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--no-cache)
|
||||
NO_CACHE="--no-cache"
|
||||
;;
|
||||
--shell)
|
||||
SHELL_MODE=1
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --no-cache Rebuild Docker image without cache"
|
||||
echo " --shell Start interactive shell in test container"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=============================================="
|
||||
echo " Subtitle Plugin Test Runner"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
# Check if Docker is available
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo "ERROR: Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build the test image
|
||||
echo "[1/2] Building test image..."
|
||||
docker build $NO_CACHE -f "$DOCKERFILE" -t "$IMAGE_NAME" .
|
||||
|
||||
if [ -n "$SHELL_MODE" ]; then
|
||||
echo ""
|
||||
echo "[2/2] Starting interactive shell..."
|
||||
echo " Run 'npm test' to execute tests"
|
||||
echo ""
|
||||
docker run --rm -it \
|
||||
-v "$SCRIPT_DIR/test:/app/test:ro" \
|
||||
-v "$SCRIPT_DIR/src:/app/src:ro" \
|
||||
"$IMAGE_NAME" \
|
||||
/bin/bash
|
||||
else
|
||||
echo ""
|
||||
echo "[2/2] Running tests..."
|
||||
echo ""
|
||||
|
||||
# Run tests and capture exit code
|
||||
docker run --rm \
|
||||
-e CI=true \
|
||||
"$IMAGE_NAME"
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo "=============================================="
|
||||
echo " All tests passed!"
|
||||
echo "=============================================="
|
||||
else
|
||||
echo "=============================================="
|
||||
echo " Tests failed (exit code: $EXIT_CODE)"
|
||||
echo "=============================================="
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* In-process stand-in for the two HTTP surfaces a plugin talks to:
|
||||
* - WebDAV at /webdav/... (JSON directory listing, HEAD, Range GET, PUT)
|
||||
* - meta API at /meta/... (GET record / property, PATCH merge, DELETE property)
|
||||
* Records are stored FLAT (`subtitles/fre/<cid>`), the way meta-core keeps them.
|
||||
*/
|
||||
|
||||
import http from 'http';
|
||||
import type { AddressInfo } from 'net';
|
||||
|
||||
export interface FakeCall {
|
||||
method: string;
|
||||
path: string;
|
||||
body?: unknown;
|
||||
}
|
||||
|
||||
export interface FakeCore {
|
||||
url: string;
|
||||
webdavUrl: string;
|
||||
/** Keyed by path under /files, e.g. `/watch/show/a.mkv`. */
|
||||
files: Map<string, Buffer>;
|
||||
records: Map<string, Record<string, string>>;
|
||||
calls: FakeCall[];
|
||||
/** GET requests per file path (each ffmpeg pass opens the file at least once). */
|
||||
fileGets: Map<string, number>;
|
||||
failPatch: boolean;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
function readBody(req: http.IncomingMessage): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (c) => chunks.push(c));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
export async function startFakeCore(): Promise<FakeCore> {
|
||||
const core = {
|
||||
files: new Map<string, Buffer>(),
|
||||
records: new Map<string, Record<string, string>>(),
|
||||
calls: [] as FakeCall[],
|
||||
fileGets: new Map<string, number>(),
|
||||
failPatch: false,
|
||||
};
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? '/', 'http://x');
|
||||
const pathname = decodeURIComponent(url.pathname);
|
||||
const send = (status: number, body?: unknown, headers: Record<string, string> = {}) => {
|
||||
if (body === undefined) {
|
||||
res.writeHead(status, headers);
|
||||
res.end();
|
||||
} else if (Buffer.isBuffer(body)) {
|
||||
res.writeHead(status, headers);
|
||||
res.end(body);
|
||||
} else {
|
||||
res.writeHead(status, { 'Content-Type': 'application/json', ...headers });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
};
|
||||
|
||||
if (pathname.startsWith('/webdav')) {
|
||||
const p = pathname.slice('/webdav'.length) || '/';
|
||||
if (req.method === 'PUT') {
|
||||
core.files.set(p, await readBody(req));
|
||||
return send(201);
|
||||
}
|
||||
if (p.endsWith('/')) {
|
||||
const dir = p.replace(/\/$/, '');
|
||||
const entries = new Map<string, { name: string; type: string; size: number }>();
|
||||
for (const [fp, data] of core.files) {
|
||||
if (!fp.startsWith(`${dir}/`)) continue;
|
||||
const rest = fp.slice(dir.length + 1);
|
||||
const slash = rest.indexOf('/');
|
||||
if (slash === -1) entries.set(rest, { name: rest, type: 'file', size: data.length });
|
||||
else entries.set(rest.slice(0, slash), { name: rest.slice(0, slash), type: 'directory', size: 512 });
|
||||
}
|
||||
return send(200, [...entries.values()]);
|
||||
}
|
||||
const data = core.files.get(p);
|
||||
if (!data) return send(404);
|
||||
if (req.method === 'HEAD') {
|
||||
return send(200, undefined, { 'Content-Length': String(data.length), 'Accept-Ranges': 'bytes' });
|
||||
}
|
||||
core.fileGets.set(p, (core.fileGets.get(p) ?? 0) + 1);
|
||||
const range = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range ?? '');
|
||||
if (range) {
|
||||
const start = Number(range[1]);
|
||||
const end = range[2] ? Math.min(Number(range[2]), data.length - 1) : data.length - 1;
|
||||
if (start >= data.length) return send(416, undefined, { 'Content-Range': `bytes */${data.length}` });
|
||||
return send(206, data.subarray(start, end + 1), {
|
||||
'Content-Range': `bytes ${start}-${end}/${data.length}`,
|
||||
'Content-Length': String(end - start + 1),
|
||||
'Accept-Ranges': 'bytes',
|
||||
});
|
||||
}
|
||||
return send(200, data, { 'Content-Length': String(data.length), 'Accept-Ranges': 'bytes' });
|
||||
}
|
||||
|
||||
const m = /^\/meta\/([^/]+)(?:\/(.+))?$/.exec(pathname);
|
||||
if (m) {
|
||||
const [, hash, key] = m;
|
||||
const raw = await readBody(req);
|
||||
const body = raw.length ? JSON.parse(raw.toString('utf8')) : undefined;
|
||||
if (req.method !== 'GET') core.calls.push({ method: req.method ?? '', path: pathname, body });
|
||||
const record = core.records.get(hash);
|
||||
|
||||
if (req.method === 'GET' && !key) return record ? send(200, { metadata: record }) : send(404, { error: 'not found' });
|
||||
if (req.method === 'GET' && key) {
|
||||
return record && key in record ? send(200, { value: record[key] }) : send(404, { error: 'Property not found' });
|
||||
}
|
||||
if (req.method === 'PATCH' && !key) {
|
||||
if (core.failPatch) return send(500, { error: 'boom' });
|
||||
core.records.set(hash, { ...(record ?? {}), ...(body as Record<string, string>) });
|
||||
return send(200, { status: 'ok' });
|
||||
}
|
||||
if (req.method === 'DELETE' && key) {
|
||||
if (record) delete record[key];
|
||||
return send(200, { status: 'ok' });
|
||||
}
|
||||
}
|
||||
send(404, { error: 'no route' });
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
|
||||
return {
|
||||
...core,
|
||||
get failPatch() { return core.failPatch; },
|
||||
set failPatch(v: boolean) { core.failPatch = v; },
|
||||
url,
|
||||
webdavUrl: `${url}/webdav`,
|
||||
close: () => new Promise<void>((resolve) => {
|
||||
server.closeAllConnections?.();
|
||||
server.close(() => resolve());
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Mutating calls (PATCH / DELETE) against one record. */
|
||||
export function writesTo(core: FakeCore, hash: string): FakeCall[] {
|
||||
return core.calls.filter((c) => c.path === `/meta/${hash}` || c.path.startsWith(`/meta/${hash}/`));
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/**
|
||||
* Subtitle Plugin Tests — pure helpers plus both processing directions against
|
||||
* an in-process fake WebDAV + meta API (no Docker, no ffmpeg).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
|
||||
import {
|
||||
manifest,
|
||||
process as processFile,
|
||||
splitExtension,
|
||||
parseSidecarName,
|
||||
trailingQualifiers,
|
||||
cleanSubtitleText,
|
||||
detectLanguageFromText,
|
||||
computeMidHash256,
|
||||
computeMidHash256FromBuffer,
|
||||
midhashSampleRange,
|
||||
videoSubtitleKeys,
|
||||
subtitleRecordKeys,
|
||||
isUnclassifiedSubtitle,
|
||||
} from '../src/plugin.js';
|
||||
import { toLang3, languageToken, francToLang3 } from '../src/langs.js';
|
||||
import { flattenMeta, onlyNewKeys, legacyCsvMembers, staleLanguageLeaves } from '../src/meta-shape.js';
|
||||
import type { CallbackPayload } from '../src/types.js';
|
||||
import { startFakeCore, writesTo, type FakeCore } from './fake-core.js';
|
||||
|
||||
const ENGLISH_SRT = `1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
<i>Where were you last night?</i>
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:08,000
|
||||
I was at home, reading the letters my father left me.
|
||||
|
||||
3
|
||||
00:00:08,500 --> 00:00:12,000
|
||||
You should have called. We were all worried about you.
|
||||
|
||||
4
|
||||
00:00:12,500 --> 00:00:16,000
|
||||
I know, and I am sorry. It will not happen again.
|
||||
`;
|
||||
|
||||
const FRENCH_SRT = `1
|
||||
00:00:01,000 --> 00:00:04,000
|
||||
Où étais-tu hier soir ?
|
||||
|
||||
2
|
||||
00:00:04,500 --> 00:00:08,000
|
||||
J'étais à la maison, en train de lire les lettres que mon père m'a laissées.
|
||||
|
||||
3
|
||||
00:00:08,500 --> 00:00:12,000
|
||||
Tu aurais dû appeler. Nous étions tous inquiets pour toi.
|
||||
|
||||
4
|
||||
00:00:12,500 --> 00:00:16,000
|
||||
Je sais, et je suis désolé. Cela ne se reproduira plus.
|
||||
`;
|
||||
|
||||
const ENGLISH_ASS = `[Script Info]
|
||||
Title: test
|
||||
ScriptType: v4.00+
|
||||
|
||||
[V4+ Styles]
|
||||
Format: Name, Fontname, Fontsize, PrimaryColour
|
||||
Style: Default,Arial,20,&H00FFFFFF
|
||||
|
||||
[Events]
|
||||
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
||||
Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,{\\i1}Where were you last night?{\\i0}
|
||||
Dialogue: 0,0:00:04.50,0:00:08.00,Default,,0,0,0,,I was at home,\\Nreading the letters my father left me.
|
||||
Dialogue: 0,0:00:08.50,0:00:12.00,Default,,0,0,0,,You should have called. We were all worried about you.
|
||||
`;
|
||||
|
||||
/** Deterministic pseudo-video bigger than the 1 MiB midhash sample. */
|
||||
function fakeVideo(bytes: number, seed: number): Buffer {
|
||||
const b = Buffer.alloc(bytes);
|
||||
let x = seed;
|
||||
for (let i = 0; i < bytes; i++) {
|
||||
x = (x * 1103515245 + 12345) & 0x7fffffff;
|
||||
b[i] = x & 0xff;
|
||||
}
|
||||
return b;
|
||||
}
|
||||
|
||||
describe('manifest', () => {
|
||||
it('runs after file-info on the fast queue', () => {
|
||||
expect(manifest.id).toBe('subtitle');
|
||||
expect(manifest.dependencies).toEqual(['file-info']);
|
||||
expect(manifest.defaultQueue).toBe('fast');
|
||||
});
|
||||
});
|
||||
|
||||
describe('language normalisation (ISO 639-2/B, same as meta-watch)', () => {
|
||||
it('folds 639-1, 639-2/T, BCP-47 and names onto 639-2/B', () => {
|
||||
expect(toLang3('fr')).toBe('fre');
|
||||
expect(toLang3('fra')).toBe('fre');
|
||||
expect(toLang3('ger')).toBe('ger');
|
||||
expect(toLang3('deu')).toBe('ger');
|
||||
expect(toLang3('zh')).toBe('chi');
|
||||
expect(toLang3('pt-BR')).toBe('por');
|
||||
expect(toLang3('French')).toBe('fre');
|
||||
});
|
||||
|
||||
it('uses und for no language and junk', () => {
|
||||
for (const v of ['', 'und', 'zxx', 'qq', undefined]) expect(toLang3(v)).toBe('und');
|
||||
});
|
||||
|
||||
it('folds franc-min results, including its non-639-2 codes', () => {
|
||||
expect(francToLang3('fra')).toBe('fre');
|
||||
expect(francToLang3('deu')).toBe('ger');
|
||||
expect(francToLang3('cmn')).toBe('chi');
|
||||
expect(francToLang3('eng')).toBe('eng');
|
||||
});
|
||||
|
||||
it('filename tokens resolve only when they are plainly a language', () => {
|
||||
expect(languageToken('en')).toBe('eng');
|
||||
expect(languageToken('ENG')).toBe('eng');
|
||||
expect(languageToken('pt-BR')).toBe('por');
|
||||
expect(languageToken('English')).toBe('eng');
|
||||
expect(languageToken('und')).toBe('und');
|
||||
for (const t of ['S01E01', 'x264', '1080p', 'Extended', 'WEB', 'tlh']) expect(languageToken(t)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSidecarName', () => {
|
||||
const video = 'Show - S01E01 (1080p).mkv';
|
||||
const cases: Array<[string, { lang3?: string; forced?: boolean; hearingImpaired?: boolean; ext?: string } | null]> = [
|
||||
['Show - S01E01 (1080p).srt', { lang3: undefined, ext: 'srt' }],
|
||||
['Show - S01E01 (1080p).eng.srt', { lang3: 'eng' }],
|
||||
['Show - S01E01 (1080p).en.srt', { lang3: 'eng' }],
|
||||
['Show - S01E01 (1080p).fr.forced.srt', { lang3: 'fre', forced: true }],
|
||||
['Show - S01E01 (1080p).fra.srt', { lang3: 'fre' }],
|
||||
['Show - S01E01 (1080p).pt-BR.ass', { lang3: 'por', ext: 'ass' }],
|
||||
['Show - S01E01 (1080p).English.sdh.vtt', { lang3: 'eng', hearingImpaired: true, ext: 'vtt' }],
|
||||
['show - s01e01 (1080P).DE.ssa', { lang3: 'ger', ext: 'ssa' }],
|
||||
['Show - S01E01 (1080p).hi.srt', { lang3: undefined, hearingImpaired: true }],
|
||||
['Show - S01E01 (1080p).forced.srt', { lang3: undefined, forced: true }],
|
||||
['Show - S01E01 (1080p).Extended.eng.srt', null],
|
||||
['Show - S01E01 (1080p).sub', null],
|
||||
['Show - S01E01 (1080p).idx', null],
|
||||
['Show - S01E02 (1080p).eng.srt', null],
|
||||
['Show - S01E01 (1080p) extra.srt', null],
|
||||
];
|
||||
|
||||
it.each(cases)('%s', (candidate, expected) => {
|
||||
const got = parseSidecarName(video, candidate);
|
||||
if (expected === null) {
|
||||
expect(got).toBeNull();
|
||||
} else {
|
||||
expect(got).not.toBeNull();
|
||||
expect(got!.lang3).toBe(expected.lang3);
|
||||
expect(got!.forced).toBe(expected.forced ?? false);
|
||||
expect(got!.hearingImpaired).toBe(expected.hearingImpaired ?? false);
|
||||
expect(got!.ext).toBe(expected.ext ?? 'srt');
|
||||
}
|
||||
});
|
||||
|
||||
it('splitExtension lowercases the extension only', () => {
|
||||
expect(splitExtension('A.B.SRT')).toEqual({ base: 'A.B', ext: 'srt' });
|
||||
expect(splitExtension('.hidden')).toEqual({ base: '.hidden', ext: '' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('trailingQualifiers (subtitle files without a sibling video)', () => {
|
||||
it('reads the subtitle-extractor naming', () => {
|
||||
expect(trailingQualifiers('Show (2024)[bagvid]_subtitle.s4.eng.forced.ass')).toEqual({ lang3: 'eng', forced: true, hearingImpaired: false });
|
||||
});
|
||||
|
||||
it('reads meta-watch upload and opensubtitles naming', () => {
|
||||
expect(trailingQualifiers('0123456789abcdef.fre.srt').lang3).toBe('fre');
|
||||
expect(trailingQualifiers('Show[bagvid]_subtitle.ger.srt').lang3).toBe('ger');
|
||||
});
|
||||
|
||||
it('never takes the whole name as a qualifier', () => {
|
||||
expect(trailingQualifiers('english.srt').lang3).toBeUndefined();
|
||||
expect(trailingQualifiers('Movie.2019.srt').lang3).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('language sniffing', () => {
|
||||
it('strips counters, timecodes and tags from SRT', () => {
|
||||
const text = cleanSubtitleText(ENGLISH_SRT, 'srt');
|
||||
expect(text).not.toMatch(/-->|<i>|^1 /);
|
||||
expect(text.startsWith('Where were you last night?')).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps only dialogue text from ASS', () => {
|
||||
const text = cleanSubtitleText(ENGLISH_ASS, 'ass');
|
||||
expect(text).not.toMatch(/Script Info|Style:|\{\\i1\}|\\N/);
|
||||
expect(text).toContain('I was at home, reading the letters');
|
||||
});
|
||||
|
||||
it('detects English and French as lang3', () => {
|
||||
expect(detectLanguageFromText(cleanSubtitleText(ENGLISH_SRT, 'srt'))).toBe('eng');
|
||||
expect(detectLanguageFromText(cleanSubtitleText(FRENCH_SRT, 'srt'))).toBe('fre');
|
||||
expect(detectLanguageFromText(cleanSubtitleText(ENGLISH_ASS, 'ass'))).toBe('eng');
|
||||
});
|
||||
|
||||
it('answers und when there is too little text', () => {
|
||||
expect(detectLanguageFromText('ok')).toBe('und');
|
||||
expect(detectLanguageFromText(cleanSubtitleText('1\n00:00:00,000 --> 00:00:01,000\n♪\n', 'srt'))).toBe('und');
|
||||
});
|
||||
});
|
||||
|
||||
describe('midhash256', () => {
|
||||
it('matches meta-hash for a known buffer', () => {
|
||||
expect(computeMidHash256FromBuffer(Buffer.from('metamesh-still-extractor-cid-vector')))
|
||||
.toBe('bagacbabaecybg7wcyxl7su3dvjleuwgwil5tgeoybrwc35jasqtywb6bnjzk4');
|
||||
});
|
||||
|
||||
it('needs only the middle 1 MiB of a large file', () => {
|
||||
const big = fakeVideo(3 * 1024 * 1024 + 7, 1);
|
||||
const { start, end } = midhashSampleRange(big.length);
|
||||
expect(end - start + 1).toBe(1024 * 1024);
|
||||
expect(computeMidHash256(big.length, big.subarray(start, end + 1))).toBe(computeMidHash256FromBuffer(big));
|
||||
});
|
||||
});
|
||||
|
||||
describe('record keys (METADATA_KEYS.md §8)', () => {
|
||||
it('video: language-nested leaf, plus the facet for a known language', () => {
|
||||
expect(videoSubtitleKeys('bagsub', 'fre')).toEqual({ 'subtitles/fre/bagsub': 'true', 'subtitleLanguages/fre': 'true' });
|
||||
expect(videoSubtitleKeys('bagsub', 'und')).toEqual({ 'subtitles/und/bagsub': 'true' });
|
||||
});
|
||||
|
||||
it('subtitle: back-pointer key-set and the language when determined', () => {
|
||||
expect(subtitleRecordKeys('eng', ['bagv1', 'bagv2'])).toEqual({ 'videos/bagv1': 'true', 'videos/bagv2': 'true', subtitleLanguage: 'eng' });
|
||||
expect(subtitleRecordKeys('und', ['bagv1'])).toEqual({ 'videos/bagv1': 'true' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('meta-shape helpers', () => {
|
||||
it('the nested and flat payloads flatten to the same keys', () => {
|
||||
const nested = { fileType: 'video', subtitles: { fre: { bagsub: true } }, fileinfo: { duration: 12.5 } };
|
||||
const flat = { fileType: 'video', 'subtitles/fre/bagsub': 'true', 'fileinfo/duration': '12.5' };
|
||||
expect(flattenMeta(nested)).toEqual(flat);
|
||||
expect(flattenMeta(flat)).toEqual(flat);
|
||||
});
|
||||
|
||||
it('legacyCsvMembers only fires on a scalar', () => {
|
||||
expect(legacyCsvMembers({ subtitles: 'a,b' }, 'subtitles')).toEqual(['a', 'b']);
|
||||
expect(legacyCsvMembers({ subtitles: { fre: { a: true } } }, 'subtitles')).toBeNull();
|
||||
expect(legacyCsvMembers({}, 'subtitles')).toBeNull();
|
||||
});
|
||||
|
||||
it('onlyNewKeys / staleLanguageLeaves', () => {
|
||||
expect(onlyNewKeys({ 'subtitles/fre/a': 'true' }, { 'subtitles/fre/a': 'true', 'subtitles/eng/b': 'true' }))
|
||||
.toEqual({ 'subtitles/eng/b': 'true' });
|
||||
expect(staleLanguageLeaves({ 'subtitles/und/a': 'true', 'subtitles/fre/a': 'true', 'subtitles/und/b': 'true' }, 'subtitles', 'a', 'fre'))
|
||||
.toEqual(['subtitles/und/a']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('process against a fake core', () => {
|
||||
let core: FakeCore;
|
||||
const dir = '/watch/Show';
|
||||
const videoName = 'Show - S01E01.mkv';
|
||||
const video = fakeVideo(3 * 1024 * 1024, 7);
|
||||
const videoCid = computeMidHash256FromBuffer(video);
|
||||
const sidecars: Record<string, Buffer> = {
|
||||
'Show - S01E01.srt': Buffer.from(ENGLISH_SRT),
|
||||
'Show - S01E01.fr.forced.srt': Buffer.from(FRENCH_SRT),
|
||||
'Show - S01E01.eng.ass': Buffer.from(ENGLISH_ASS),
|
||||
'Show - S01E01.Extended.eng.srt': Buffer.from(ENGLISH_SRT + '\n'),
|
||||
'Show - S01E02.srt': Buffer.from(ENGLISH_SRT + '\n\n'),
|
||||
};
|
||||
const cid = (name: string) => computeMidHash256FromBuffer(sidecars[name]);
|
||||
|
||||
const run = async (fileCid: string, filePath: string, existingMeta: Record<string, unknown>, metaCoreUrl = core.url) => {
|
||||
const seen: CallbackPayload[] = [];
|
||||
await processFile({ taskId: 't', cid: fileCid, filePath, callbackUrl: 'http://unused', metaCoreUrl, existingMeta }, async (p) => { seen.push(p); });
|
||||
return seen[0];
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
core = await startFakeCore();
|
||||
core.files.set(`${dir}/${videoName}`, video);
|
||||
core.files.set(`${dir}/Show - S01E02.mkv`, fakeVideo(2048, 9));
|
||||
for (const [name, data] of Object.entries(sidecars)) core.files.set(`${dir}/${name}`, data);
|
||||
process.env.WEBDAV_URL = core.webdavUrl;
|
||||
|
||||
// The .srt and .fr.forced.srt are indexed already; the .ass is not.
|
||||
core.records.set(cid('Show - S01E01.srt'), { fileType: 'subtitle', extension: 'srt' });
|
||||
core.records.set(cid('Show - S01E01.fr.forced.srt'), { fileType: 'subtitle', extension: 'srt', videos: videoCid });
|
||||
core.records.set(videoCid, { fileType: 'video', extension: 'mkv', subtitles: `bagacbabaelegacy,${cid('Show - S01E01.srt')}` });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
delete process.env.WEBDAV_URL;
|
||||
await core.close();
|
||||
});
|
||||
|
||||
it('video run: links every sidecar under its language and migrates the legacy csv', async () => {
|
||||
// Nested form, as meta-sort's /meta serves it.
|
||||
const result = await run(videoCid, `/files${dir}/${videoName}`, {
|
||||
fileType: 'video',
|
||||
extension: 'mkv',
|
||||
subtitles: `bagacbabaelegacy,${cid('Show - S01E01.srt')}`,
|
||||
});
|
||||
expect(result.status).toBe('completed');
|
||||
|
||||
const v = core.records.get(videoCid)!;
|
||||
expect(v.subtitles).toBeUndefined();
|
||||
expect(Object.keys(v).filter((k) => k.startsWith('subtitle')).sort()).toEqual([
|
||||
`subtitleLanguages/eng`,
|
||||
`subtitleLanguages/fre`,
|
||||
`subtitles/eng/${cid('Show - S01E01.eng.ass')}`,
|
||||
`subtitles/eng/${cid('Show - S01E01.srt')}`,
|
||||
`subtitles/fre/${cid('Show - S01E01.fr.forced.srt')}`,
|
||||
'subtitles/und/bagacbabaelegacy',
|
||||
].sort());
|
||||
|
||||
expect(core.records.get(cid('Show - S01E01.srt'))).toEqual({
|
||||
fileType: 'subtitle', extension: 'srt', [`videos/${videoCid}`]: 'true', subtitleLanguage: 'eng',
|
||||
});
|
||||
// Legacy `videos` scalar converted to the key-set.
|
||||
expect(core.records.get(cid('Show - S01E01.fr.forced.srt'))).toEqual({
|
||||
fileType: 'subtitle', extension: 'srt', [`videos/${videoCid}`]: 'true', subtitleLanguage: 'fre',
|
||||
});
|
||||
// Not indexed yet: no stub record.
|
||||
expect(core.records.has(cid('Show - S01E01.eng.ass'))).toBe(false);
|
||||
// Not sidecars of this video.
|
||||
expect(JSON.stringify(v)).not.toContain(cid('Show - S01E01.Extended.eng.srt'));
|
||||
expect(JSON.stringify(v)).not.toContain(cid('Show - S01E02.srt'));
|
||||
});
|
||||
|
||||
it('video run again over the updated record writes nothing to it', async () => {
|
||||
const before = writesTo(core, videoCid).length;
|
||||
const result = await run(videoCid, `/files${dir}/${videoName}`, core.records.get(videoCid)!);
|
||||
expect(result.status).toBe('completed');
|
||||
expect(writesTo(core, videoCid).length).toBe(before);
|
||||
});
|
||||
|
||||
it('subtitle run: links a late-indexed sidecar onto the video from its side', async () => {
|
||||
const assCid = cid('Show - S01E01.eng.ass');
|
||||
core.records.set(assCid, { fileType: 'subtitle', extension: 'ass' });
|
||||
const result = await run(assCid, `/files${dir}/Show - S01E01.eng.ass`, { fileType: 'subtitle', extension: 'ass' });
|
||||
expect(result.status).toBe('completed');
|
||||
expect(core.records.get(assCid)).toMatchObject({ [`videos/${videoCid}`]: 'true', subtitleLanguage: 'eng' });
|
||||
expect(core.records.get(videoCid)![`subtitles/eng/${assCid}`]).toBe('true');
|
||||
});
|
||||
|
||||
it('subtitle run: an unsuffixed sidecar gets its language from its text', async () => {
|
||||
const srtCid = cid('Show - S01E01.srt');
|
||||
core.records.set(srtCid, { fileType: 'subtitle', extension: 'srt' });
|
||||
const result = await run(srtCid, `/files${dir}/Show - S01E01.srt`, { fileType: 'subtitle', extension: 'srt' });
|
||||
expect(result.status).toBe('completed');
|
||||
expect(core.records.get(srtCid)).toMatchObject({ subtitleLanguage: 'eng', [`videos/${videoCid}`]: 'true' });
|
||||
});
|
||||
|
||||
it('subtitle run: one file, one language — a stale leaf for the same cid is removed', async () => {
|
||||
const frCid = cid('Show - S01E01.fr.forced.srt');
|
||||
core.records.get(videoCid)![`subtitles/und/${frCid}`] = 'true';
|
||||
const result = await run(frCid, `/files${dir}/Show - S01E01.fr.forced.srt`, core.records.get(frCid)!);
|
||||
expect(result.status).toBe('completed');
|
||||
expect(core.records.get(videoCid)![`subtitles/und/${frCid}`]).toBeUndefined();
|
||||
expect(core.records.get(videoCid)![`subtitles/fre/${frCid}`]).toBe('true');
|
||||
});
|
||||
|
||||
it('a subtitle in a plugin output folder takes its language from its name, no sibling search', async () => {
|
||||
const data = Buffer.from(FRENCH_SRT + '\n\n\n\n'); // distinct bytes from the watch-folder sidecar
|
||||
const subCid = computeMidHash256FromBuffer(data);
|
||||
const p = '/plugin/subtitle-extractor/Show[bagvid]_subtitle.s5.ger.srt'; // name wins over text
|
||||
core.files.set(p, data);
|
||||
const result = await run(subCid, `/files${p}`, { fileType: 'subtitle' });
|
||||
expect(result.status).toBe('completed');
|
||||
expect(core.records.get(subCid)).toEqual({ subtitleLanguage: 'ger' });
|
||||
});
|
||||
|
||||
it('still runs on a subtitle file the classifier left as other/undefined', async () => {
|
||||
const data = Buffer.from(ENGLISH_ASS + '\n');
|
||||
const subCid = computeMidHash256FromBuffer(data);
|
||||
const p = '/plugin/subtitle-extractor/Show[bagvid]_subtitle.s9.ita.ass';
|
||||
core.files.set(p, data);
|
||||
const other = await run(subCid, `/files${p}`, { fileType: 'other', extension: 'ass' });
|
||||
expect(other.status).toBe('completed');
|
||||
expect(core.records.get(subCid)).toEqual({ subtitleLanguage: 'ita' });
|
||||
|
||||
const untyped = await run('bagtxt', '/files/watch/Show/notes.srt', { fileType: 'undefined' });
|
||||
expect(untyped.status).not.toBe('skipped');
|
||||
});
|
||||
|
||||
it('isUnclassifiedSubtitle only rescues subtitle extensions', () => {
|
||||
expect(isUnclassifiedSubtitle({ fileType: 'other', extension: 'ASS' }, '/files/x.ass')).toBe(true);
|
||||
expect(isUnclassifiedSubtitle({ fileType: 'undefined' }, '/files/x.srt')).toBe(true);
|
||||
expect(isUnclassifiedSubtitle({}, '/files/x.vtt')).toBe(true);
|
||||
expect(isUnclassifiedSubtitle({ fileType: 'other', extension: 'txt' }, '/files/x.txt')).toBe(false);
|
||||
expect(isUnclassifiedSubtitle({ fileType: 'document', extension: 'srt' }, '/files/x.srt')).toBe(false);
|
||||
});
|
||||
|
||||
it('skips files that are neither video nor subtitle', async () => {
|
||||
const result = await run('bagimg', '/files/watch/a.jpg', { fileType: 'image' });
|
||||
expect(result).toMatchObject({ status: 'skipped', reason: 'Not a video or subtitle file' });
|
||||
});
|
||||
|
||||
it('fails the task when meta-core rejects a write', async () => {
|
||||
const fresh = fakeVideo(4096, 11);
|
||||
const freshCid = computeMidHash256FromBuffer(fresh);
|
||||
core.files.set('/watch/Other/Film.mkv', fresh);
|
||||
core.files.set('/watch/Other/Film.en.srt', Buffer.from(ENGLISH_SRT));
|
||||
core.failPatch = true;
|
||||
try {
|
||||
const result = await run(freshCid, '/files/watch/Other/Film.mkv', { fileType: 'video' });
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.error).toContain('meta-core write failed');
|
||||
} finally {
|
||||
core.failPatch = false;
|
||||
}
|
||||
});
|
||||
|
||||
it('fails when no WebDAV endpoint can be resolved', async () => {
|
||||
const saved = process.env.WEBDAV_URL;
|
||||
delete process.env.WEBDAV_URL;
|
||||
try {
|
||||
const result = await run(videoCid, `/files${dir}/${videoName}`, { fileType: 'video' }, 'http://127.0.0.1:9');
|
||||
expect(result).toMatchObject({ status: 'failed', error: 'No WebDAV endpoint available' });
|
||||
} finally {
|
||||
process.env.WEBDAV_URL = saved;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['test/**/*.test.ts'],
|
||||
testTimeout: 30000, // 30s timeout for FFprobe tests
|
||||
hookTimeout: 60000, // 60s for setup/teardown
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'json', 'html'],
|
||||
include: ['src/**/*.ts'],
|
||||
exclude: ['src/types.ts'],
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user