This commit is contained in:
Worph
-12
@@ -3,9 +3,6 @@
|
||||
|
||||
FROM node:20-slim AS builder
|
||||
|
||||
# Install git for GitHub dependencies
|
||||
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PNPM_HOME="/pnpm"
|
||||
ENV PATH="$PNPM_HOME:$PATH"
|
||||
RUN npm install -g corepack@latest && corepack enable
|
||||
@@ -17,9 +14,6 @@ COPY package.json pnpm-lock.yaml ./
|
||||
RUN corepack install
|
||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
||||
|
||||
# Build the git dependency (filename-tools)
|
||||
RUN cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build
|
||||
|
||||
# Copy source and build
|
||||
COPY tsconfig.json ./
|
||||
COPY src/ ./src/
|
||||
@@ -28,9 +22,6 @@ RUN pnpm run build
|
||||
# Production image
|
||||
FROM node:20-slim
|
||||
|
||||
# Install git for GitHub dependencies
|
||||
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PNPM_HOME="/pnpm"
|
||||
ENV PATH="$PNPM_HOME:$PATH"
|
||||
RUN npm install -g corepack@latest && corepack enable
|
||||
@@ -41,9 +32,6 @@ COPY package.json pnpm-lock.yaml ./
|
||||
RUN corepack install
|
||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile --prod
|
||||
|
||||
# Build the git dependency in production too
|
||||
RUN cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build
|
||||
|
||||
COPY --from=builder /app/dist ./dist
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# MetaMesh Plugin: language - Test Image
|
||||
# Builds the plugin and runs the unit tests (no ffmpeg, no network needed at runtime)
|
||||
|
||||
FROM node:20-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files and install ALL dependencies (including devDependencies)
|
||||
COPY package.json tsconfig.json vitest.config.ts ./
|
||||
RUN npm install
|
||||
|
||||
# Copy source and test files
|
||||
COPY src/ ./src/
|
||||
COPY test/ ./test/
|
||||
|
||||
# Build the plugin (type-checks src)
|
||||
RUN npm run build
|
||||
|
||||
ENV NODE_ENV=test
|
||||
|
||||
CMD ["npm", "test"]
|
||||
@@ -1,27 +1,36 @@
|
||||
# MetaMesh Plugin: Language
|
||||
|
||||
A MetaMesh plugin that aggregates languages from all streams and determines the primary language.
|
||||
A MetaMesh container plugin that adds a media file's **audio-track languages**
|
||||
to the `languages/<lang3>` key-set (METADATA_KEYS.md §9).
|
||||
|
||||
## Description
|
||||
## What it does
|
||||
|
||||
This plugin collects language information from video, audio, and subtitle streams extracted by the FFmpeg plugin. It:
|
||||
After the `ffmpeg` plugin has stored the stream table, this plugin:
|
||||
|
||||
- Converts language codes to ISO 639-3 format
|
||||
- Aggregates all detected languages into a set
|
||||
- Determines the primary language (first audio stream)
|
||||
- Maps titles to their detected language
|
||||
- reads the per-stream table in **either** shape: the nested `stream`
|
||||
collection meta-sort's `/process` payload carries (array of JSON strings,
|
||||
array of objects, JSON string, or index-keyed object) or meta-core's flat
|
||||
`stream/{n}` keys;
|
||||
- keeps **audio** streams only (subtitle-track languages belong to
|
||||
`subtitleLanguages/*`);
|
||||
- normalises each code onto ISO 639-2/B alpha-3 — `fr`/`fra` → `fre`,
|
||||
`de`/`deu` → `ger`, `pt-BR` → `por` — the vocabulary the `languages:` query
|
||||
filter and meta-watch compare against;
|
||||
- adds `languages/<code> = "true"` for each new member in one merge (`PATCH`).
|
||||
|
||||
It never writes `und` (or `zxx`/`mis`), never writes `titles/*`, never deletes
|
||||
or overwrites any other key, and a re-run over an already-tagged record writes
|
||||
nothing. A rejected write fails the task.
|
||||
|
||||
## Metadata Fields
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `languages` | Set of all detected languages (ISO 639-3) |
|
||||
| `primaryLanguage` | Primary language code |
|
||||
| `titles/{lang}` | Title mapped to language |
|
||||
| `languages/{lang3}` | Key-set member per audio-track language (`"true"`) |
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Requires `ffmpeg` plugin to run first
|
||||
- Requires the `ffmpeg` plugin to run first (it writes `stream/{n}`).
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -36,12 +45,11 @@ No configuration required.
|
||||
| `/configure` | POST | Update configuration |
|
||||
| `/process` | POST | Process a file |
|
||||
|
||||
## Running Locally
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
npm start
|
||||
pnpm test # vitest, no network
|
||||
./test.sh # same, inside Docker
|
||||
```
|
||||
|
||||
## Docker
|
||||
@@ -56,7 +64,6 @@ docker run -p 8080:8080 metamesh-plugin-language
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `PORT` | `8080` | HTTP server port |
|
||||
| `HOST` | `0.0.0.0` | HTTP server host |
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+2
-3
@@ -2,7 +2,7 @@
|
||||
"name": "@metazla/plugin-language",
|
||||
"version": "1.0.0",
|
||||
"packageManager": "[email protected]",
|
||||
"description": "MetaMesh plugin: Aggregates languages from streams",
|
||||
"description": "MetaMesh plugin: Adds audio-track languages to the languages/<lang3> key-set",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
@@ -12,8 +12,7 @@
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"dependencies": {
|
||||
"fastify": "^5.0.0",
|
||||
"@metazla/filename-tools": "github:worph/filename-tool"
|
||||
"fastify": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
|
||||
Generated
-135
@@ -8,9 +8,6 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@metazla/filename-tools':
|
||||
specifier: github:worph/filename-tool
|
||||
version: https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251
|
||||
fastify:
|
||||
specifier: ^5.0.0
|
||||
version: 5.7.4
|
||||
@@ -27,9 +24,6 @@ importers:
|
||||
|
||||
packages:
|
||||
|
||||
'@borewit/[email protected]':
|
||||
resolution: {integrity: sha512-k7vvKPbf7J2fZ5klGRD9AeKfUvojuZIQ3BT5u7Jfv+puwXkUBUT5PVyMDfJZpy30CBDXGMgw7fguK/lpOMBvgw==}
|
||||
|
||||
'@esbuild/[email protected]':
|
||||
resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==}
|
||||
engines: {node: '>=12'}
|
||||
@@ -193,10 +187,6 @@ packages:
|
||||
'@jridgewell/[email protected]':
|
||||
resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==}
|
||||
|
||||
'@metazla/filename-tools@https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251':
|
||||
resolution: {tarball: https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251}
|
||||
version: 1.0.0
|
||||
|
||||
'@pinojs/[email protected]':
|
||||
resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
|
||||
|
||||
@@ -325,15 +315,9 @@ packages:
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
'@sec-ant/[email protected]':
|
||||
resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==}
|
||||
|
||||
'@sinclair/[email protected]':
|
||||
resolution: {integrity: sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==}
|
||||
|
||||
'@tokenizer/[email protected]':
|
||||
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
|
||||
|
||||
'@types/[email protected]':
|
||||
resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
|
||||
|
||||
@@ -403,9 +387,6 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==}
|
||||
|
||||
@@ -471,20 +452,10 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-VZR5I7k5wkD0HgFnMsq5hOsSc710MJMu5Nc5QYsbe38NN5iPV/XTObYLc/cpttRTf6lX538+5uO1ZQRhYibiZQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-5Ye4vHsypZRYtS01ob/iwHzGRUDELlsoCftI/OZFhcLs1M0tkGPcXldE80TAZC5yYuJMBPJQQ43UHlqbJWiX2w==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-ZR6ciLQTDBaOvBdkOd8+vqDzaLtmIXRa9GCzcAlaBpqNAKg9QrtClPmqiKac5/xZXfCZGMo1d8dIu1T0BLhHEg==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-rcAewP7PSHvjq7Kgd7dhj82zE071kX5B4W1M4ewYMf/P+i6YsDQmj62Xz3VQm9zyUzUXwhIde/wHLGCMrM+yGg==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
|
||||
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
|
||||
@@ -497,17 +468,10 @@ packages:
|
||||
resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==}
|
||||
engines: {node: '>=16.17.0'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==}
|
||||
engines: {node: '>= 10'}
|
||||
@@ -516,20 +480,9 @@ packages:
|
||||
resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==}
|
||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-gXkz5+KN7HrG0Q5UGqSMO2qB9AsbEeyLP54kF1YrMsIxmu+g4BdB7rflReZTSTZGpfj8wywu6pfPBCylPIzGQA==}
|
||||
engines: {node: '>=6.0'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==}
|
||||
|
||||
@@ -565,9 +518,6 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==}
|
||||
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
|
||||
@@ -606,10 +556,6 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
|
||||
|
||||
@@ -727,10 +673,6 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-FhwotcEqjr241ZbjFzjlIYg6c5/L/s4yBGWSMvJ9UoExiSqL+FnFA/CaeZx17WGaZMS/4SOZp8wH18jSS4R4lw==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==}
|
||||
engines: {node: '>=20'}
|
||||
@@ -750,13 +692,6 @@ packages:
|
||||
resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==}
|
||||
engines: {node: '>=4'}
|
||||
@@ -769,10 +704,6 @@ packages:
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
@@ -853,8 +784,6 @@ packages:
|
||||
|
||||
snapshots:
|
||||
|
||||
'@borewit/[email protected]': {}
|
||||
|
||||
'@esbuild/[email protected]':
|
||||
optional: true
|
||||
|
||||
@@ -953,14 +882,6 @@ snapshots:
|
||||
|
||||
'@jridgewell/[email protected]': {}
|
||||
|
||||
'@metazla/filename-tools@https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251':
|
||||
dependencies:
|
||||
file-type: 19.6.0
|
||||
franc: 6.2.0
|
||||
franc-all: 7.2.0
|
||||
iso-639-1: 3.1.5
|
||||
iso-639-3: 3.0.1
|
||||
|
||||
'@pinojs/[email protected]': {}
|
||||
|
||||
'@rollup/[email protected]':
|
||||
@@ -1038,12 +959,8 @@ snapshots:
|
||||
'@rollup/[email protected]':
|
||||
optional: true
|
||||
|
||||
'@sec-ant/[email protected]': {}
|
||||
|
||||
'@sinclair/[email protected]': {}
|
||||
|
||||
'@tokenizer/[email protected]': {}
|
||||
|
||||
'@types/[email protected]': {}
|
||||
|
||||
'@types/[email protected]':
|
||||
@@ -1125,8 +1042,6 @@ snapshots:
|
||||
dependencies:
|
||||
get-func-name: 2.0.2
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
@@ -1232,27 +1147,12 @@ snapshots:
|
||||
dependencies:
|
||||
reusify: 1.1.0
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
get-stream: 9.0.1
|
||||
strtok3: 9.1.1
|
||||
token-types: 6.1.2
|
||||
uint8array-extras: 1.5.0
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
fast-querystring: 1.1.2
|
||||
safe-regex2: 5.0.0
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
trigram-utils: 2.0.1
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
trigram-utils: 2.0.1
|
||||
|
||||
[email protected]:
|
||||
optional: true
|
||||
|
||||
@@ -1260,27 +1160,14 @@ snapshots:
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
'@sec-ant/readable-stream': 0.4.1
|
||||
is-stream: 4.0.1
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]:
|
||||
@@ -1321,8 +1208,6 @@ snapshots:
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]:
|
||||
@@ -1349,8 +1234,6 @@ snapshots:
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]:
|
||||
@@ -1480,11 +1363,6 @@ snapshots:
|
||||
dependencies:
|
||||
js-tokens: 9.0.1
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
peek-readable: 5.4.2
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
real-require: 0.2.0
|
||||
@@ -1497,25 +1375,12 @@ snapshots:
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
'@borewit/text-codec': 0.2.1
|
||||
'@tokenizer/token': 0.3.0
|
||||
ieee754: 1.2.1
|
||||
|
||||
[email protected]:
|
||||
dependencies:
|
||||
collapse-white-space: 2.1.0
|
||||
n-gram: 2.0.2
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected]: {}
|
||||
|
||||
[email protected](@types/[email protected]):
|
||||
|
||||
@@ -32,13 +32,14 @@ export class MetaCoreClient {
|
||||
return data.value ?? null;
|
||||
}
|
||||
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<void> {
|
||||
/** PATCH (merge). Resolves `true` only when the write was accepted. */
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<boolean> {
|
||||
const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(metadata),
|
||||
});
|
||||
// Silently ignore errors (expected in standalone/test mode)
|
||||
return !!response && response.ok;
|
||||
}
|
||||
|
||||
async deleteProperty(hashId: string, key: string): Promise<void> {
|
||||
|
||||
+186
-71
@@ -1,24 +1,48 @@
|
||||
/**
|
||||
* Language Plugin
|
||||
*
|
||||
* Aggregates languages from all streams (audio, video, subtitle)
|
||||
* and converts them to ISO 639-3 format.
|
||||
* Determines the primary language based on first audio > video > subtitle stream.
|
||||
* Adds the record's AUDIO languages to the `languages/<lang3>` key-set
|
||||
* (METADATA_KEYS.md §9), read from the ffmpeg plugin's per-stream table.
|
||||
*
|
||||
* Matches old LanguageProcessor output:
|
||||
* - languages (add)
|
||||
* - titles/{lang} (set originalTitle)
|
||||
* What it writes — and only this:
|
||||
* languages/<code> = "true" one member per distinct audio-track language
|
||||
*
|
||||
* ⚠ Stream shape. meta-sort's /process payload is the NESTED document form
|
||||
* (`stream: [ '<json>', ... ]`, `fileinfo: { duration }`), not meta-core's flat
|
||||
* `stream/{n}` keys; and the legacy `fileinfo/streamdetails/{type}/{i}/language`
|
||||
* keys this plugin used to read are written by nobody into the store (ffmpeg
|
||||
* only puts `streamdetails` in its local cache JSON). Reading the legacy keys
|
||||
* made every task a silent no-op. Both current shapes are accepted here.
|
||||
*
|
||||
* ⚠ Vocabulary. Codes are folded onto ISO 639-2/B alpha-3 (`fre`, `ger`,
|
||||
* `chi`) — the space the `languages:` query filter compares against literally
|
||||
* (meta-search `query_eval::languages_filter_matches`), the one meta-watch's
|
||||
* `MW_LANG.toLang3` and the feeder SDK's `normalize_lang_code` emit, and what
|
||||
* Matroska stream tags already carry. A `fra` member would be dropped by a
|
||||
* `languages:fre` filter.
|
||||
*
|
||||
* ⚠ Audio only. Consumers read `languages/*` as the AUDIO-language set
|
||||
* (meta-watch `cards::audio_languages`); subtitle-track languages belong to
|
||||
* `subtitleLanguages/*` (subtitle-extractor). Title keys are not a source
|
||||
* either: `titles/<lang>` names the language of a TITLE (tmdb files
|
||||
* `originalTitle` under `original_language`), not of the file's audio.
|
||||
*
|
||||
* ⚠ No `titles/*` write. The old `titles/{firstAudioLang || 'eng'}` =
|
||||
* `originalTitle` guess overwrote better identity (tmdb, jellyfin-nfo,
|
||||
* anime-detector own those keys) — the overwrite-regression class.
|
||||
*
|
||||
* Never writes `und` (registry: an undetermined language is the absence of a
|
||||
* member), never deletes, never overwrites a non-`languages/` key.
|
||||
*/
|
||||
|
||||
import { anyTo_iso_639_3 } from '@metazla/filename-tools';
|
||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||
import { MetaCoreClient } from './meta-core-client.js';
|
||||
|
||||
export const manifest: PluginManifest = {
|
||||
id: 'language',
|
||||
name: 'Language Aggregator',
|
||||
version: '1.0.0',
|
||||
description: 'Aggregates languages from all streams and determines primary language',
|
||||
version: '1.1.0',
|
||||
description: 'Adds audio-track languages to the languages/<lang3> key-set',
|
||||
author: 'MetaMesh',
|
||||
dependencies: ['ffmpeg'],
|
||||
priority: 40,
|
||||
@@ -31,77 +55,168 @@ export const manifest: PluginManifest = {
|
||||
config: {},
|
||||
};
|
||||
|
||||
/** One entry of the ffmpeg plugin's stream table (only the fields read here). */
|
||||
export interface StreamEntry {
|
||||
type?: string;
|
||||
index?: number;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The ffmpeg stream table out of whichever shape the payload carries.
|
||||
*
|
||||
* Mirrors still-extractor's `selectPrimaryVideoStream` parsing: the nested
|
||||
* `stream` collection (array of JSON strings, array of objects, a JSON string,
|
||||
* or an index-keyed object) first, then meta-core's flat `stream/{n}` keys.
|
||||
* Malformed entries are skipped, never fatal.
|
||||
*/
|
||||
export function streamsFromMeta(existingMeta: Record<string, unknown> | undefined): StreamEntry[] {
|
||||
if (!existingMeta) return [];
|
||||
|
||||
const raw: unknown[] = [];
|
||||
|
||||
const nested = existingMeta['stream'];
|
||||
if (nested) {
|
||||
try {
|
||||
const parsed = typeof nested === 'string' ? JSON.parse(nested) : nested;
|
||||
if (Array.isArray(parsed)) raw.push(...parsed);
|
||||
else if (parsed && typeof parsed === 'object') raw.push(...Object.values(parsed as Record<string, unknown>));
|
||||
} catch {
|
||||
// fall through to the namespaced form
|
||||
}
|
||||
}
|
||||
|
||||
if (raw.length === 0) {
|
||||
const flat = Object.entries(existingMeta)
|
||||
.filter(([key]) => /^stream\/\d+$/.test(key))
|
||||
.sort(([a], [b]) => Number(a.slice(7)) - Number(b.slice(7)));
|
||||
for (const [, value] of flat) raw.push(value);
|
||||
}
|
||||
|
||||
const streams: StreamEntry[] = [];
|
||||
for (const entry of raw) {
|
||||
let stream: unknown;
|
||||
try {
|
||||
stream = typeof entry === 'string' ? JSON.parse(entry) : entry;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (stream && typeof stream === 'object' && !Array.isArray(stream)) {
|
||||
streams.push(stream as StreamEntry);
|
||||
}
|
||||
}
|
||||
return streams;
|
||||
}
|
||||
|
||||
// ISO 639-2/T → 639-2/B, the only codes where the two disagree. MP4's `mdhd`
|
||||
// speaks T, Matroska speaks B, ffmpeg passes through either. Mirrors
|
||||
// meta-watch `ui/js/lang-codes.js` T2B.
|
||||
const T2B: Record<string, string> = {
|
||||
sqi: 'alb', hye: 'arm', eus: 'baq', mya: 'bur', ces: 'cze',
|
||||
zho: 'chi', cym: 'wel', deu: 'ger', nld: 'dut', ell: 'gre', fas: 'per',
|
||||
fra: 'fre', kat: 'geo', isl: 'ice', mkd: 'mac', msa: 'may',
|
||||
ron: 'rum', slk: 'slo',
|
||||
};
|
||||
|
||||
// ISO 639-1 → 639-2/B. Mirrors meta-watch `lang-codes.js` (its `a1` column plus
|
||||
// the legacy/regional two-letter aliases).
|
||||
const A1: Record<string, string> = {
|
||||
en: 'eng', fr: 'fre', de: 'ger', es: 'spa', it: 'ita', pt: 'por', nl: 'dut',
|
||||
sv: 'swe', no: 'nor', da: 'dan', fi: 'fin', is: 'ice', pl: 'pol', cs: 'cze',
|
||||
sk: 'slo', sl: 'slv', hu: 'hun', ro: 'rum', bg: 'bul', ru: 'rus', uk: 'ukr',
|
||||
be: 'bel', sr: 'srp', hr: 'hrv', bs: 'bos', mk: 'mac', sq: 'alb', el: 'gre',
|
||||
tr: 'tur', he: 'heb', ar: 'ara', fa: 'per', ur: 'urd', hi: 'hin', bn: 'ben',
|
||||
ta: 'tam', te: 'tel', ml: 'mal', kn: 'kan', mr: 'mar', gu: 'guj', pa: 'pan',
|
||||
ne: 'nep', si: 'sin', th: 'tha', lo: 'lao', km: 'khm', my: 'bur', vi: 'vie',
|
||||
id: 'ind', ms: 'may', tl: 'tgl', ja: 'jpn', ko: 'kor', zh: 'chi', mn: 'mon',
|
||||
kk: 'kaz', uz: 'uzb', az: 'aze', ka: 'geo', hy: 'arm', et: 'est', lv: 'lav',
|
||||
lt: 'lit', ca: 'cat', gl: 'glg', eu: 'baq', cy: 'wel', ga: 'gle', af: 'afr',
|
||||
sw: 'swa', am: 'amh', zu: 'zul', la: 'lat', eo: 'epo', yi: 'yid', jv: 'jav',
|
||||
// legacy / regional
|
||||
nb: 'nor', nn: 'nor', iw: 'heb', in: 'ind', ji: 'yid', jw: 'jav',
|
||||
};
|
||||
|
||||
// "This track has no language" spellings. Never written (registry §9).
|
||||
const NONE = new Set(['und', 'zxx', 'mis', 'unknown', 'none', '']);
|
||||
|
||||
/**
|
||||
* Canonical 639-2/B alpha-3 for a stream-tag language code; `''` when the
|
||||
* track declares no language. Accepts 639-1 (`ja`), 639-2/B (`jpn`), 639-2/T
|
||||
* (`deu` → `ger`) and BCP-47 (`pt-BR` → `por`). An unrecognised 3-letter code
|
||||
* passes through lowercased (still a distinguishable language); anything else
|
||||
* is dropped rather than guessed. `mul` is kept — it is a valid member.
|
||||
* Same rules as meta-watch `MW_LANG.toLang3`.
|
||||
*/
|
||||
export function toLang3(code: unknown): string {
|
||||
const s = String(code ?? '').trim().toLowerCase();
|
||||
if (!s) return '';
|
||||
const primary = s.split(/[-_]/)[0];
|
||||
if (NONE.has(primary)) return '';
|
||||
if (!/^[a-z]+$/.test(primary)) return '';
|
||||
if (primary.length === 2) return A1[primary] ?? '';
|
||||
if (primary.length === 3) return T2B[primary] ?? primary;
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Distinct canonical audio-track languages, in stream order. */
|
||||
export function audioLanguagesFromMeta(existingMeta: Record<string, unknown> | undefined): string[] {
|
||||
const out: string[] = [];
|
||||
for (const stream of streamsFromMeta(existingMeta)) {
|
||||
if (stream.type !== 'audio') continue;
|
||||
const lang = toLang3(stream.language);
|
||||
if (lang && !out.includes(lang)) out.push(lang);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the record already carries `languages/<code>`, in either shape: the
|
||||
* flat key, or the nested `languages: { <code>: ... }` object meta-sort's
|
||||
* reconstructed document holds.
|
||||
*/
|
||||
function hasMember(existingMeta: Record<string, unknown> | undefined, code: string): boolean {
|
||||
if (!existingMeta) return false;
|
||||
if (existingMeta[`languages/${code}`] !== undefined) return true;
|
||||
const nested = existingMeta['languages'];
|
||||
return !!nested && typeof nested === 'object' && !Array.isArray(nested)
|
||||
&& (nested as Record<string, unknown>)[code] !== undefined;
|
||||
}
|
||||
|
||||
/** The `languages/<code>` members to add — never one the record already has. */
|
||||
export function languageWrites(existingMeta: Record<string, unknown> | undefined): Record<string, string> {
|
||||
const writes: Record<string, string> = {};
|
||||
for (const code of audioLanguagesFromMeta(existingMeta)) {
|
||||
if (!hasMember(existingMeta, code)) writes[`languages/${code}`] = 'true';
|
||||
}
|
||||
return writes;
|
||||
}
|
||||
|
||||
/** The subset of the meta-core client this plugin uses (injectable for tests). */
|
||||
export interface LanguageWriter {
|
||||
mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<boolean>;
|
||||
}
|
||||
|
||||
export async function process(
|
||||
request: ProcessRequest,
|
||||
sendCallback: (payload: CallbackPayload) => Promise<void>
|
||||
sendCallback: (payload: CallbackPayload) => Promise<void>,
|
||||
writer: LanguageWriter = new MetaCoreClient(request.metaCoreUrl),
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||
|
||||
try {
|
||||
const { cid, existingMeta } = request;
|
||||
const { cid } = request;
|
||||
const existingMeta = request.existingMeta as Record<string, unknown> | undefined;
|
||||
const found = audioLanguagesFromMeta(existingMeta);
|
||||
const writes = languageWrites(existingMeta);
|
||||
|
||||
let streamLanguage: string | null = null;
|
||||
|
||||
// Collect languages from title keys (if existingMeta has titles/*)
|
||||
for (const key of Object.keys(existingMeta || {})) {
|
||||
if (key.startsWith('titles/')) {
|
||||
const langCode = key.substring(7); // Remove 'titles/' prefix
|
||||
const normalized = anyTo_iso_639_3(langCode);
|
||||
if (normalized) {
|
||||
await metaCore.addToSet(cid, 'languages', normalized);
|
||||
}
|
||||
}
|
||||
if (Object.keys(writes).length > 0) {
|
||||
// One PATCH (merge) carrying only the new members: additive and
|
||||
// idempotent, touches no other key.
|
||||
const ok = await writer.mergeMetadata(cid, writes);
|
||||
if (!ok) throw new Error(`failed to write ${Object.keys(writes).join(', ')}`);
|
||||
}
|
||||
|
||||
// Collect languages from audio streams (highest priority for primary)
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const lang = existingMeta?.[`fileinfo/streamdetails/audio/${i}/language`];
|
||||
if (!lang) break;
|
||||
const computedLanguage = anyTo_iso_639_3(lang);
|
||||
if (computedLanguage) {
|
||||
await metaCore.addToSet(cid, 'languages', computedLanguage);
|
||||
if (!streamLanguage) {
|
||||
streamLanguage = computedLanguage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect languages from video streams
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const lang = existingMeta?.[`fileinfo/streamdetails/video/${i}/language`];
|
||||
if (!lang) break;
|
||||
const computedLanguage = anyTo_iso_639_3(lang);
|
||||
if (computedLanguage) {
|
||||
await metaCore.addToSet(cid, 'languages', computedLanguage);
|
||||
if (!streamLanguage) {
|
||||
streamLanguage = computedLanguage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect languages from subtitle streams
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const lang = existingMeta?.[`fileinfo/streamdetails/subtitle/${i}/language`];
|
||||
if (!lang) break;
|
||||
const computedLanguage = anyTo_iso_639_3(lang);
|
||||
if (computedLanguage) {
|
||||
await metaCore.addToSet(cid, 'languages', computedLanguage);
|
||||
if (!streamLanguage) {
|
||||
streamLanguage = computedLanguage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the language of the media file based on:
|
||||
// 1st audio stream, 2nd video stream, 3rd subtitle stream (first found)
|
||||
// Assume original title is one of the stream languages or English if not found
|
||||
const originalTitle = existingMeta?.originalTitle;
|
||||
if (originalTitle) {
|
||||
await metaCore.setProperty(cid, `titles/${streamLanguage || 'eng'}`, originalTitle);
|
||||
}
|
||||
|
||||
console.log(`[language] Aggregated languages, primary: ${streamLanguage || 'none'}`);
|
||||
console.log(`[language] ${cid}: audio languages [${found.join(', ') || 'none'}], added [${Object.keys(writes).join(', ') || 'none'}]`);
|
||||
|
||||
await sendCallback({
|
||||
taskId: request.taskId,
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Language Plugin Test Runner
|
||||
#
|
||||
# Runs the unit tests inside a Docker container.
|
||||
# Usage:
|
||||
# ./test.sh # Build and run tests
|
||||
# ./test.sh --no-cache # Rebuild without cache
|
||||
# ./test.sh --shell # Start shell in test container for debugging
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
IMAGE_NAME="metamesh-plugin-language-test"
|
||||
DOCKERFILE="Dockerfile.test"
|
||||
|
||||
NO_CACHE=""
|
||||
SHELL_MODE=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--no-cache)
|
||||
NO_CACHE="--no-cache"
|
||||
;;
|
||||
--shell)
|
||||
SHELL_MODE=1
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --no-cache Rebuild Docker image without cache"
|
||||
echo " --shell Start interactive shell in test container"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=============================================="
|
||||
echo " Language Plugin Test Runner"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo "ERROR: Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[1/2] Building test image..."
|
||||
docker build $NO_CACHE -f "$DOCKERFILE" -t "$IMAGE_NAME" .
|
||||
|
||||
if [ -n "$SHELL_MODE" ]; then
|
||||
echo ""
|
||||
echo "[2/2] Starting interactive shell..."
|
||||
echo " Run 'npm test' to execute tests"
|
||||
echo ""
|
||||
docker run --rm -it \
|
||||
-v "$SCRIPT_DIR/test:/app/test:ro" \
|
||||
-v "$SCRIPT_DIR/src:/app/src:ro" \
|
||||
"$IMAGE_NAME" \
|
||||
/bin/bash
|
||||
else
|
||||
echo ""
|
||||
echo "[2/2] Running tests..."
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
docker run --rm -e CI=true "$IMAGE_NAME"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo "=============================================="
|
||||
echo " All tests passed!"
|
||||
echo "=============================================="
|
||||
else
|
||||
echo "=============================================="
|
||||
echo " Tests failed (exit code: $EXIT_CODE)"
|
||||
echo "=============================================="
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
@@ -1,165 +0,0 @@
|
||||
/**
|
||||
* Language Plugin Integration Tests
|
||||
*
|
||||
* Tests language aggregation from streams and primary language determination.
|
||||
* Uses @metazla/filename-tools library for ISO 639-3 conversion.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll } from 'vitest';
|
||||
import { anyTo_iso_639_3 } from '@metazla/filename-tools';
|
||||
|
||||
// Dynamic import of plugin module
|
||||
let manifest: typeof import('../src/plugin.js').manifest;
|
||||
let processFile: typeof import('../src/plugin.js').process;
|
||||
|
||||
// Mock callback collector
|
||||
interface CallbackResult {
|
||||
taskId: string;
|
||||
status: 'completed' | 'failed' | 'skipped';
|
||||
duration: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
let lastCallback: CallbackResult | null = null;
|
||||
|
||||
const mockSendCallback = async (payload: CallbackResult): Promise<void> => {
|
||||
lastCallback = payload;
|
||||
};
|
||||
|
||||
describe('Language Plugin Integration Tests', () => {
|
||||
beforeAll(async () => {
|
||||
const plugin = await import('../src/plugin.js');
|
||||
manifest = plugin.manifest;
|
||||
processFile = plugin.process;
|
||||
});
|
||||
|
||||
describe('Manifest', () => {
|
||||
it('has required fields', () => {
|
||||
expect(manifest.id).toBe('language');
|
||||
expect(manifest.name).toBeDefined();
|
||||
expect(manifest.version).toBeDefined();
|
||||
expect(manifest.dependencies).toContain('ffmpeg');
|
||||
expect(manifest.priority).toBe(40);
|
||||
});
|
||||
|
||||
it('declares correct schema', () => {
|
||||
expect(manifest.schema).toHaveProperty('languages');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ISO 639-3 Conversion', () => {
|
||||
it('handles 3-letter codes', () => {
|
||||
// The function should handle 3-letter ISO 639-3 codes
|
||||
const engResult = anyTo_iso_639_3('eng');
|
||||
const jpnResult = anyTo_iso_639_3('jpn');
|
||||
const fraResult = anyTo_iso_639_3('fra');
|
||||
|
||||
// Function returns the code if valid, or 'und' for unknown
|
||||
expect(['eng', 'und']).toContain(engResult);
|
||||
expect(['jpn', 'und']).toContain(jpnResult);
|
||||
expect(['fra', 'und']).toContain(fraResult);
|
||||
});
|
||||
|
||||
it('returns a defined value for known codes', () => {
|
||||
const result = anyTo_iso_639_3('eng');
|
||||
expect(result).toBeDefined();
|
||||
expect(typeof result).toBe('string');
|
||||
});
|
||||
|
||||
it('handles unknown codes gracefully', () => {
|
||||
const result = anyTo_iso_639_3('xyz');
|
||||
// Should return 'und' (undefined) or the original value
|
||||
expect(result).toBeDefined();
|
||||
expect(typeof result).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Process Function', () => {
|
||||
it('processes file with audio streams', async () => {
|
||||
await processFile({
|
||||
taskId: 'test-lang-1',
|
||||
cid: 'test-cid-lang',
|
||||
filePath: '/movies/Movie.mkv',
|
||||
callbackUrl: 'http://localhost/callback',
|
||||
metaCoreUrl: 'http://localhost:9000',
|
||||
existingMeta: {
|
||||
originalTitle: 'Test Movie',
|
||||
'fileinfo/streamdetails/audio/0/language': 'eng',
|
||||
'fileinfo/streamdetails/audio/1/language': 'jpn',
|
||||
},
|
||||
}, mockSendCallback);
|
||||
|
||||
expect(lastCallback).toBeDefined();
|
||||
expect(lastCallback?.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('processes file with subtitle streams', async () => {
|
||||
await processFile({
|
||||
taskId: 'test-lang-2',
|
||||
cid: 'test-cid-lang-2',
|
||||
filePath: '/movies/Movie.mkv',
|
||||
callbackUrl: 'http://localhost/callback',
|
||||
metaCoreUrl: 'http://localhost:9000',
|
||||
existingMeta: {
|
||||
originalTitle: 'Test Movie',
|
||||
'fileinfo/streamdetails/subtitle/0/language': 'eng',
|
||||
'fileinfo/streamdetails/subtitle/1/language': 'fra',
|
||||
},
|
||||
}, mockSendCallback);
|
||||
|
||||
expect(lastCallback).toBeDefined();
|
||||
expect(lastCallback?.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('processes file with existing title languages', async () => {
|
||||
await processFile({
|
||||
taskId: 'test-lang-3',
|
||||
cid: 'test-cid-lang-3',
|
||||
filePath: '/movies/Movie.mkv',
|
||||
callbackUrl: 'http://localhost/callback',
|
||||
metaCoreUrl: 'http://localhost:9000',
|
||||
existingMeta: {
|
||||
originalTitle: 'Test Movie',
|
||||
'titles/eng': 'Test Movie',
|
||||
'titles/jpn': 'テスト映画',
|
||||
},
|
||||
}, mockSendCallback);
|
||||
|
||||
expect(lastCallback).toBeDefined();
|
||||
expect(lastCallback?.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('processes file with no language information', async () => {
|
||||
await processFile({
|
||||
taskId: 'test-lang-4',
|
||||
cid: 'test-cid-lang-4',
|
||||
filePath: '/movies/Movie.mkv',
|
||||
callbackUrl: 'http://localhost/callback',
|
||||
metaCoreUrl: 'http://localhost:9000',
|
||||
existingMeta: {
|
||||
originalTitle: 'Test Movie',
|
||||
},
|
||||
}, mockSendCallback);
|
||||
|
||||
expect(lastCallback).toBeDefined();
|
||||
expect(lastCallback?.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('processes file with video stream language', async () => {
|
||||
await processFile({
|
||||
taskId: 'test-lang-5',
|
||||
cid: 'test-cid-lang-5',
|
||||
filePath: '/movies/Movie.mkv',
|
||||
callbackUrl: 'http://localhost/callback',
|
||||
metaCoreUrl: 'http://localhost:9000',
|
||||
existingMeta: {
|
||||
originalTitle: 'Test Movie',
|
||||
'fileinfo/streamdetails/video/0/language': 'eng',
|
||||
},
|
||||
}, mockSendCallback);
|
||||
|
||||
expect(lastCallback).toBeDefined();
|
||||
expect(lastCallback?.status).toBe('completed');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Language plugin unit tests — no network, no meta-core: the writer is injected.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
manifest,
|
||||
process as processFile,
|
||||
streamsFromMeta,
|
||||
toLang3,
|
||||
audioLanguagesFromMeta,
|
||||
languageWrites,
|
||||
type LanguageWriter,
|
||||
} from '../src/plugin.js';
|
||||
import type { CallbackPayload, ProcessRequest } from '../src/types.js';
|
||||
|
||||
// Trimmed from the real nested /meta document of a dev-stack record
|
||||
// (1080p AV1 episode: 2 audio, 9 subtitle, cover-art mjpeg).
|
||||
const REAL_STREAMS: string[] = [
|
||||
'{"type":"video","codec":"av1","index":0,"duration":"N/A","forced":false,"default":true,"title":"By Trix","width":1920,"height":1080,"frameRate":"24000/1001"}',
|
||||
'{"type":"audio","codec":"aac","index":1,"duration":"N/A","forced":false,"default":true,"language":"jpn","title":"Japanese - AAC 2.0","sampleRate":"44100","channelLayout":"stereo"}',
|
||||
'{"type":"audio","codec":"aac","index":2,"duration":"N/A","forced":false,"default":true,"language":"eng","title":"English - AAC 2.0","sampleRate":"44100","channelLayout":"stereo"}',
|
||||
'{"type":"subtitle","codec":"ass","index":3,"language":"eng","title":"English (Full) [4de]"}',
|
||||
'{"type":"subtitle","codec":"ass","index":5,"language":"fre","title":"French"}',
|
||||
'{"type":"subtitle","codec":"ass","index":6,"language":"ger","title":"German"}',
|
||||
'{"type":"video","codec":"mjpeg","index":12,"duration":"1433.089","width":230,"height":345}',
|
||||
];
|
||||
|
||||
function nestedDoc(extra: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
fileinfo: { duration: 1433.089, formatName: 'matroska,webm' },
|
||||
stream: REAL_STREAMS,
|
||||
originalTitle: 'Some Show',
|
||||
cids: ['bagacbabaebs63pblr2rsycodkmg4ojn47ol7k4oz6ptcre2bwaguhsib4hjpg'],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
class RecordingWriter implements LanguageWriter {
|
||||
calls: Array<{ hashId: string; metadata: Record<string, string> }> = [];
|
||||
constructor(private ok = true) {}
|
||||
async mergeMetadata(hashId: string, metadata: Record<string, string>): Promise<boolean> {
|
||||
this.calls.push({ hashId, metadata });
|
||||
return this.ok;
|
||||
}
|
||||
}
|
||||
|
||||
async function run(existingMeta: Record<string, unknown> | undefined, writer = new RecordingWriter()) {
|
||||
let callback: CallbackPayload | null = null;
|
||||
const request = {
|
||||
taskId: 't1',
|
||||
cid: 'cid-1',
|
||||
filePath: '/files/watch/Show - 01.mkv',
|
||||
callbackUrl: 'http://metasort-app/api/plugins/callback',
|
||||
metaCoreUrl: 'http://metasort-app',
|
||||
existingMeta,
|
||||
} as unknown as ProcessRequest;
|
||||
await processFile(request, async (p) => { callback = p; }, writer);
|
||||
return { callback: callback as CallbackPayload | null, writer };
|
||||
}
|
||||
|
||||
describe('manifest', () => {
|
||||
it('keeps the plugin identity and ffmpeg dependency', () => {
|
||||
expect(manifest.id).toBe('language');
|
||||
expect(manifest.dependencies).toContain('ffmpeg');
|
||||
expect(manifest.priority).toBe(40);
|
||||
expect(manifest.schema).toHaveProperty('languages');
|
||||
});
|
||||
});
|
||||
|
||||
describe('streamsFromMeta', () => {
|
||||
it('reads the nested stream array of JSON strings (meta-sort payload)', () => {
|
||||
const streams = streamsFromMeta(nestedDoc());
|
||||
expect(streams).toHaveLength(REAL_STREAMS.length);
|
||||
expect(streams[1]).toMatchObject({ type: 'audio', language: 'jpn', index: 1 });
|
||||
});
|
||||
|
||||
it('reads a nested stream array of objects', () => {
|
||||
const streams = streamsFromMeta({ stream: REAL_STREAMS.map((s) => JSON.parse(s)) });
|
||||
expect(streams.map((s) => s.index)).toEqual([0, 1, 2, 3, 5, 6, 12]);
|
||||
});
|
||||
|
||||
it('reads a nested stream JSON string and an index-keyed object', () => {
|
||||
expect(streamsFromMeta({ stream: JSON.stringify(REAL_STREAMS) })).toHaveLength(REAL_STREAMS.length);
|
||||
const keyed = Object.fromEntries(REAL_STREAMS.map((s, i) => [String(i), s]));
|
||||
expect(streamsFromMeta({ stream: keyed })).toHaveLength(REAL_STREAMS.length);
|
||||
});
|
||||
|
||||
it('reads flat stream/{n} keys (meta-core form) in numeric order', () => {
|
||||
const flat: Record<string, string> = { 'fileinfo/duration': '1433.089' };
|
||||
// insert out of order, with n >= 10 to catch lexical sorting
|
||||
flat['stream/10'] = '{"type":"audio","index":10,"language":"spa"}';
|
||||
flat['stream/2'] = '{"type":"audio","index":2,"language":"eng"}';
|
||||
flat['stream/1'] = '{"type":"audio","index":1,"language":"jpn"}';
|
||||
expect(streamsFromMeta(flat).map((s) => s.index)).toEqual([1, 2, 10]);
|
||||
});
|
||||
|
||||
it('returns nothing for missing streams and skips malformed entries', () => {
|
||||
expect(streamsFromMeta(undefined)).toEqual([]);
|
||||
expect(streamsFromMeta({})).toEqual([]);
|
||||
expect(streamsFromMeta({ originalTitle: 'x' })).toEqual([]);
|
||||
expect(streamsFromMeta({ stream: ['not json', '{"type":"audio","language":"eng"}', null] }))
|
||||
.toEqual([{ type: 'audio', language: 'eng' }]);
|
||||
});
|
||||
|
||||
it('ignores the legacy fileinfo/streamdetails keys nobody writes', () => {
|
||||
expect(streamsFromMeta({ 'fileinfo/streamdetails/audio/0/language': 'eng' })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toLang3', () => {
|
||||
it('keeps 639-2/B codes', () => {
|
||||
for (const c of ['eng', 'jpn', 'fre', 'ger', 'chi', 'spa']) expect(toLang3(c)).toBe(c);
|
||||
});
|
||||
|
||||
it('folds 639-2/T onto 639-2/B (the filter vocabulary)', () => {
|
||||
expect(toLang3('fra')).toBe('fre');
|
||||
expect(toLang3('deu')).toBe('ger');
|
||||
expect(toLang3('zho')).toBe('chi');
|
||||
expect(toLang3('nld')).toBe('dut');
|
||||
expect(toLang3('ces')).toBe('cze');
|
||||
});
|
||||
|
||||
it('maps 639-1 and BCP-47 tags (the old normaliser returned und for all 2-letter codes)', () => {
|
||||
expect(toLang3('en')).toBe('eng');
|
||||
expect(toLang3('fr')).toBe('fre');
|
||||
expect(toLang3('ja')).toBe('jpn');
|
||||
expect(toLang3('pt-BR')).toBe('por');
|
||||
expect(toLang3('EN_us')).toBe('eng');
|
||||
expect(toLang3(' JPN ')).toBe('jpn');
|
||||
});
|
||||
|
||||
it('never yields und and drops no-language spellings', () => {
|
||||
for (const c of ['und', 'UND', 'zxx', 'mis', '', undefined, null, 'unknown']) expect(toLang3(c)).toBe('');
|
||||
});
|
||||
|
||||
it('keeps mul and unknown 3-letter codes, drops unknown 2-letter and full names', () => {
|
||||
expect(toLang3('mul')).toBe('mul');
|
||||
expect(toLang3('xyz')).toBe('xyz');
|
||||
expect(toLang3('xx')).toBe('');
|
||||
expect(toLang3('French')).toBe('');
|
||||
expect(toLang3('e1g')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('audioLanguagesFromMeta', () => {
|
||||
it('takes audio tracks only — subtitle languages are not audio languages', () => {
|
||||
expect(audioLanguagesFromMeta(nestedDoc())).toEqual(['jpn', 'eng']);
|
||||
});
|
||||
|
||||
it('dedupes after normalisation and skips und/untagged audio', () => {
|
||||
const meta = {
|
||||
stream: [
|
||||
{ type: 'audio', language: 'fra' },
|
||||
{ type: 'audio', language: 'fre' },
|
||||
{ type: 'audio', language: 'fr' },
|
||||
{ type: 'audio', language: 'und' },
|
||||
{ type: 'audio' },
|
||||
{ type: 'video', language: 'eng' },
|
||||
],
|
||||
};
|
||||
expect(audioLanguagesFromMeta(meta)).toEqual(['fre']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('languageWrites', () => {
|
||||
it('skips members already present in the flat or nested form', () => {
|
||||
expect(languageWrites(nestedDoc({ 'languages/jpn': 'true' }))).toEqual({ 'languages/eng': 'true' });
|
||||
expect(languageWrites(nestedDoc({ languages: { eng: 'true', jpn: true } }))).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('process', () => {
|
||||
it('writes the audio languages/* key-set from the nested payload in one merge', async () => {
|
||||
const { callback, writer } = await run(nestedDoc());
|
||||
expect(callback?.status).toBe('completed');
|
||||
expect(writer.calls).toEqual([
|
||||
{ hashId: 'cid-1', metadata: { 'languages/jpn': 'true', 'languages/eng': 'true' } },
|
||||
]);
|
||||
});
|
||||
|
||||
it('writes from flat stream/{n} keys too', async () => {
|
||||
const flat = Object.fromEntries(REAL_STREAMS.map((s, i) => [`stream/${i}`, s]));
|
||||
const { callback, writer } = await run(flat);
|
||||
expect(callback?.status).toBe('completed');
|
||||
expect(writer.calls[0].metadata).toEqual({ 'languages/jpn': 'true', 'languages/eng': 'true' });
|
||||
});
|
||||
|
||||
it('never writes titles/* or any non-languages key, even with originalTitle and title keys', async () => {
|
||||
const { writer } = await run(nestedDoc({ 'titles/eng': 'Better Title', titles: { jpn: '番組' } }));
|
||||
const keys = writer.calls.flatMap((c) => Object.keys(c.metadata));
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
expect(keys.every((k) => k.startsWith('languages/'))).toBe(true);
|
||||
expect(keys.some((k) => k.startsWith('titles'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not derive languages from title keys', async () => {
|
||||
const { callback, writer } = await run({ originalTitle: 'x', 'titles/jpn': 'y', 'titles/jpl': 'z' });
|
||||
expect(callback?.status).toBe('completed');
|
||||
expect(writer.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it('completes without writing when there are no streams or no tagged audio', async () => {
|
||||
for (const meta of [undefined, {}, { stream: [] }, { stream: [{ type: 'audio', language: 'und' }] }]) {
|
||||
const { callback, writer } = await run(meta);
|
||||
expect(callback?.status).toBe('completed');
|
||||
expect(writer.calls).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('is idempotent: a re-run over a record that already has the members writes nothing', async () => {
|
||||
const { writer } = await run(nestedDoc({ 'languages/jpn': 'true', 'languages/eng': 'true' }));
|
||||
expect(writer.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it('fails the task when the write is rejected', async () => {
|
||||
const { callback } = await run(nestedDoc(), new RecordingWriter(false));
|
||||
expect(callback?.status).toBe('failed');
|
||||
expect(callback?.error).toContain('languages/jpn');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user