diff --git a/Dockerfile b/Dockerfile index 5be5745..e1c184b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,6 @@ FROM node:20-slim AS builder -# Install git for GitHub dependencies -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* - ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN npm install -g corepack@latest && corepack enable @@ -17,9 +14,6 @@ COPY package.json pnpm-lock.yaml ./ RUN corepack install RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -# Build the git dependency (filename-tools) -RUN cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build - # Copy source and build COPY tsconfig.json ./ COPY src/ ./src/ @@ -28,9 +22,6 @@ RUN pnpm run build # Production image FROM node:20-slim -# Install git for GitHub dependencies -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* - ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN npm install -g corepack@latest && corepack enable @@ -41,9 +32,6 @@ COPY package.json pnpm-lock.yaml ./ RUN corepack install RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile --prod -# Build the git dependency in production too -RUN cd node_modules/@metazla/filename-tools && pnpm install && pnpm run build - COPY --from=builder /app/dist ./dist EXPOSE 8080 diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..433103e --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,21 @@ +# MetaMesh Plugin: language - Test Image +# Builds the plugin and runs the unit tests (no ffmpeg, no network needed at runtime) + +FROM node:20-slim + +WORKDIR /app + +# Copy package files and install ALL dependencies (including devDependencies) +COPY package.json tsconfig.json vitest.config.ts ./ +RUN npm install + +# Copy source and test files +COPY src/ ./src/ +COPY test/ ./test/ + +# Build the plugin (type-checks src) +RUN npm run build + +ENV NODE_ENV=test + +CMD ["npm", "test"] diff --git a/README.md b/README.md index 676454d..8fdde23 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,36 @@ # MetaMesh Plugin: Language -A MetaMesh plugin that aggregates languages from all streams and determines the primary language. +A MetaMesh container plugin that adds a media file's **audio-track languages** +to the `languages/` key-set (METADATA_KEYS.md §9). -## Description +## What it does -This plugin collects language information from video, audio, and subtitle streams extracted by the FFmpeg plugin. It: +After the `ffmpeg` plugin has stored the stream table, this plugin: -- Converts language codes to ISO 639-3 format -- Aggregates all detected languages into a set -- Determines the primary language (first audio stream) -- Maps titles to their detected language +- reads the per-stream table in **either** shape: the nested `stream` + collection meta-sort's `/process` payload carries (array of JSON strings, + array of objects, JSON string, or index-keyed object) or meta-core's flat + `stream/{n}` keys; +- keeps **audio** streams only (subtitle-track languages belong to + `subtitleLanguages/*`); +- normalises each code onto ISO 639-2/B alpha-3 — `fr`/`fra` → `fre`, + `de`/`deu` → `ger`, `pt-BR` → `por` — the vocabulary the `languages:` query + filter and meta-watch compare against; +- adds `languages/ = "true"` for each new member in one merge (`PATCH`). + +It never writes `und` (or `zxx`/`mis`), never writes `titles/*`, never deletes +or overwrites any other key, and a re-run over an already-tagged record writes +nothing. A rejected write fails the task. ## Metadata Fields | Field | Description | |-------|-------------| -| `languages` | Set of all detected languages (ISO 639-3) | -| `primaryLanguage` | Primary language code | -| `titles/{lang}` | Title mapped to language | +| `languages/{lang3}` | Key-set member per audio-track language (`"true"`) | ## Dependencies -- Requires `ffmpeg` plugin to run first +- Requires the `ffmpeg` plugin to run first (it writes `stream/{n}`). ## Configuration @@ -36,12 +45,11 @@ No configuration required. | `/configure` | POST | Update configuration | | `/process` | POST | Process a file | -## Running Locally +## Tests ```bash -npm install -npm run build -npm start +pnpm test # vitest, no network +./test.sh # same, inside Docker ``` ## Docker @@ -56,7 +64,6 @@ docker run -p 8080:8080 metamesh-plugin-language | Variable | Default | Description | |----------|---------|-------------| | `PORT` | `8080` | HTTP server port | -| `HOST` | `0.0.0.0` | HTTP server host | ## License diff --git a/package.json b/package.json index c63aa9c..a07f921 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@metazla/plugin-language", "version": "1.0.0", "packageManager": "pnpm@10.28.0", - "description": "MetaMesh plugin: Aggregates languages from streams", + "description": "MetaMesh plugin: Adds audio-track languages to the languages/ key-set", "type": "module", "main": "dist/index.js", "scripts": { @@ -12,8 +12,7 @@ "test:watch": "vitest" }, "dependencies": { - "fastify": "^5.0.0", - "@metazla/filename-tools": "github:worph/filename-tool" + "fastify": "^5.0.0" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63b2946..ddad657 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,9 +8,6 @@ importers: .: dependencies: - '@metazla/filename-tools': - specifier: github:worph/filename-tool - version: https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251 fastify: specifier: ^5.0.0 version: 5.7.4 @@ -27,9 +24,6 @@ importers: packages: - '@borewit/text-codec@0.2.1': - resolution: {integrity: sha512-k7vvKPbf7J2fZ5klGRD9AeKfUvojuZIQ3BT5u7Jfv+puwXkUBUT5PVyMDfJZpy30CBDXGMgw7fguK/lpOMBvgw==} - '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} engines: {node: '>=12'} @@ -193,10 +187,6 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - '@metazla/filename-tools@https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251': - resolution: {tarball: https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251} - version: 1.0.0 - '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -325,15 +315,9 @@ packages: cpu: [x64] os: [win32] - '@sec-ant/readable-stream@0.4.1': - resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - '@sinclair/typebox@0.27.10': resolution: {integrity: sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==} - '@tokenizer/token@0.3.0': - resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} - '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -403,9 +387,6 @@ packages: check-error@1.0.3: resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} - collapse-white-space@2.1.0: - resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} - confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -471,20 +452,10 @@ packages: fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} - file-type@19.6.0: - resolution: {integrity: sha512-VZR5I7k5wkD0HgFnMsq5hOsSc710MJMu5Nc5QYsbe38NN5iPV/XTObYLc/cpttRTf6lX538+5uO1ZQRhYibiZQ==} - engines: {node: '>=18'} - find-my-way@9.4.0: resolution: {integrity: sha512-5Ye4vHsypZRYtS01ob/iwHzGRUDELlsoCftI/OZFhcLs1M0tkGPcXldE80TAZC5yYuJMBPJQQ43UHlqbJWiX2w==} engines: {node: '>=20'} - franc-all@7.2.0: - resolution: {integrity: sha512-ZR6ciLQTDBaOvBdkOd8+vqDzaLtmIXRa9GCzcAlaBpqNAKg9QrtClPmqiKac5/xZXfCZGMo1d8dIu1T0BLhHEg==} - - franc@6.2.0: - resolution: {integrity: sha512-rcAewP7PSHvjq7Kgd7dhj82zE071kX5B4W1M4ewYMf/P+i6YsDQmj62Xz3VQm9zyUzUXwhIde/wHLGCMrM+yGg==} - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -497,17 +468,10 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} - get-stream@9.0.1: - resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} - engines: {node: '>=18'} - human-signals@5.0.0: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - ipaddr.js@2.3.0: resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==} engines: {node: '>= 10'} @@ -516,20 +480,9 @@ packages: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - is-stream@4.0.1: - resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} - engines: {node: '>=18'} - isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - iso-639-1@3.1.5: - resolution: {integrity: sha512-gXkz5+KN7HrG0Q5UGqSMO2qB9AsbEeyLP54kF1YrMsIxmu+g4BdB7rflReZTSTZGpfj8wywu6pfPBCylPIzGQA==} - engines: {node: '>=6.0'} - - iso-639-3@3.0.1: - resolution: {integrity: sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==} - js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} @@ -565,9 +518,6 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - n-gram@2.0.2: - resolution: {integrity: sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ==} - nanoid@3.3.11: resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -606,10 +556,6 @@ packages: pathval@1.1.1: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} - peek-readable@5.4.2: - resolution: {integrity: sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==} - engines: {node: '>=14.16'} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -727,10 +673,6 @@ packages: strip-literal@2.1.1: resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==} - strtok3@9.1.1: - resolution: {integrity: sha512-FhwotcEqjr241ZbjFzjlIYg6c5/L/s4yBGWSMvJ9UoExiSqL+FnFA/CaeZx17WGaZMS/4SOZp8wH18jSS4R4lw==} - engines: {node: '>=16'} - thread-stream@4.0.0: resolution: {integrity: sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==} engines: {node: '>=20'} @@ -750,13 +692,6 @@ packages: resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} engines: {node: '>=12'} - token-types@6.1.2: - resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==} - engines: {node: '>=14.16'} - - trigram-utils@2.0.1: - resolution: {integrity: sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ==} - type-detect@4.1.0: resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} engines: {node: '>=4'} @@ -769,10 +704,6 @@ packages: ufo@1.6.3: resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} - uint8array-extras@1.5.0: - resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==} - engines: {node: '>=18'} - undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -853,8 +784,6 @@ packages: snapshots: - '@borewit/text-codec@0.2.1': {} - '@esbuild/aix-ppc64@0.21.5': optional: true @@ -953,14 +882,6 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} - '@metazla/filename-tools@https://codeload.github.com/worph/filename-tool/tar.gz/8a9c357c8af20bf7af040985993f5f78d1aa5251': - dependencies: - file-type: 19.6.0 - franc: 6.2.0 - franc-all: 7.2.0 - iso-639-1: 3.1.5 - iso-639-3: 3.0.1 - '@pinojs/redact@0.4.0': {} '@rollup/rollup-android-arm-eabi@4.57.1': @@ -1038,12 +959,8 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.57.1': optional: true - '@sec-ant/readable-stream@0.4.1': {} - '@sinclair/typebox@0.27.10': {} - '@tokenizer/token@0.3.0': {} - '@types/estree@1.0.8': {} '@types/node@20.19.33': @@ -1125,8 +1042,6 @@ snapshots: dependencies: get-func-name: 2.0.2 - collapse-white-space@2.1.0: {} - confbox@0.1.8: {} cookie@1.1.1: {} @@ -1232,27 +1147,12 @@ snapshots: dependencies: reusify: 1.1.0 - file-type@19.6.0: - dependencies: - get-stream: 9.0.1 - strtok3: 9.1.1 - token-types: 6.1.2 - uint8array-extras: 1.5.0 - find-my-way@9.4.0: dependencies: fast-deep-equal: 3.1.3 fast-querystring: 1.1.2 safe-regex2: 5.0.0 - franc-all@7.2.0: - dependencies: - trigram-utils: 2.0.1 - - franc@6.2.0: - dependencies: - trigram-utils: 2.0.1 - fsevents@2.3.3: optional: true @@ -1260,27 +1160,14 @@ snapshots: get-stream@8.0.1: {} - get-stream@9.0.1: - dependencies: - '@sec-ant/readable-stream': 0.4.1 - is-stream: 4.0.1 - human-signals@5.0.0: {} - ieee754@1.2.1: {} - ipaddr.js@2.3.0: {} is-stream@3.0.0: {} - is-stream@4.0.1: {} - isexe@2.0.0: {} - iso-639-1@3.1.5: {} - - iso-639-3@3.0.1: {} - js-tokens@9.0.1: {} json-schema-ref-resolver@3.0.0: @@ -1321,8 +1208,6 @@ snapshots: ms@2.1.3: {} - n-gram@2.0.2: {} - nanoid@3.3.11: {} npm-run-path@5.3.0: @@ -1349,8 +1234,6 @@ snapshots: pathval@1.1.1: {} - peek-readable@5.4.2: {} - picocolors@1.1.1: {} pino-abstract-transport@3.0.0: @@ -1480,11 +1363,6 @@ snapshots: dependencies: js-tokens: 9.0.1 - strtok3@9.1.1: - dependencies: - '@tokenizer/token': 0.3.0 - peek-readable: 5.4.2 - thread-stream@4.0.0: dependencies: real-require: 0.2.0 @@ -1497,25 +1375,12 @@ snapshots: toad-cache@3.7.0: {} - token-types@6.1.2: - dependencies: - '@borewit/text-codec': 0.2.1 - '@tokenizer/token': 0.3.0 - ieee754: 1.2.1 - - trigram-utils@2.0.1: - dependencies: - collapse-white-space: 2.1.0 - n-gram: 2.0.2 - type-detect@4.1.0: {} typescript@5.9.3: {} ufo@1.6.3: {} - uint8array-extras@1.5.0: {} - undici-types@6.21.0: {} vite-node@1.6.1(@types/node@20.19.33): diff --git a/src/meta-core-client.ts b/src/meta-core-client.ts index ff111ca..6f8e50d 100644 --- a/src/meta-core-client.ts +++ b/src/meta-core-client.ts @@ -32,13 +32,14 @@ export class MetaCoreClient { return data.value ?? null; } - async mergeMetadata(hashId: string, metadata: Record): Promise { + /** PATCH (merge). Resolves `true` only when the write was accepted. */ + async mergeMetadata(hashId: string, metadata: Record): Promise { const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(metadata), }); - // Silently ignore errors (expected in standalone/test mode) + return !!response && response.ok; } async deleteProperty(hashId: string, key: string): Promise { diff --git a/src/plugin.ts b/src/plugin.ts index 90d3526..57378a0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1,24 +1,48 @@ /** * Language Plugin * - * Aggregates languages from all streams (audio, video, subtitle) - * and converts them to ISO 639-3 format. - * Determines the primary language based on first audio > video > subtitle stream. + * Adds the record's AUDIO languages to the `languages/` key-set + * (METADATA_KEYS.md §9), read from the ffmpeg plugin's per-stream table. * - * Matches old LanguageProcessor output: - * - languages (add) - * - titles/{lang} (set originalTitle) + * What it writes — and only this: + * languages/ = "true" one member per distinct audio-track language + * + * ⚠ Stream shape. meta-sort's /process payload is the NESTED document form + * (`stream: [ '', ... ]`, `fileinfo: { duration }`), not meta-core's flat + * `stream/{n}` keys; and the legacy `fileinfo/streamdetails/{type}/{i}/language` + * keys this plugin used to read are written by nobody into the store (ffmpeg + * only puts `streamdetails` in its local cache JSON). Reading the legacy keys + * made every task a silent no-op. Both current shapes are accepted here. + * + * ⚠ Vocabulary. Codes are folded onto ISO 639-2/B alpha-3 (`fre`, `ger`, + * `chi`) — the space the `languages:` query filter compares against literally + * (meta-search `query_eval::languages_filter_matches`), the one meta-watch's + * `MW_LANG.toLang3` and the feeder SDK's `normalize_lang_code` emit, and what + * Matroska stream tags already carry. A `fra` member would be dropped by a + * `languages:fre` filter. + * + * ⚠ Audio only. Consumers read `languages/*` as the AUDIO-language set + * (meta-watch `cards::audio_languages`); subtitle-track languages belong to + * `subtitleLanguages/*` (subtitle-extractor). Title keys are not a source + * either: `titles/` names the language of a TITLE (tmdb files + * `originalTitle` under `original_language`), not of the file's audio. + * + * ⚠ No `titles/*` write. The old `titles/{firstAudioLang || 'eng'}` = + * `originalTitle` guess overwrote better identity (tmdb, jellyfin-nfo, + * anime-detector own those keys) — the overwrite-regression class. + * + * Never writes `und` (registry: an undetermined language is the absence of a + * member), never deletes, never overwrites a non-`languages/` key. */ -import { anyTo_iso_639_3 } from '@metazla/filename-tools'; import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; import { MetaCoreClient } from './meta-core-client.js'; export const manifest: PluginManifest = { id: 'language', name: 'Language Aggregator', - version: '1.0.0', - description: 'Aggregates languages from all streams and determines primary language', + version: '1.1.0', + description: 'Adds audio-track languages to the languages/ key-set', author: 'MetaMesh', dependencies: ['ffmpeg'], priority: 40, @@ -31,77 +55,168 @@ export const manifest: PluginManifest = { config: {}, }; +/** One entry of the ffmpeg plugin's stream table (only the fields read here). */ +export interface StreamEntry { + type?: string; + index?: number; + language?: string; +} + +/** + * The ffmpeg stream table out of whichever shape the payload carries. + * + * Mirrors still-extractor's `selectPrimaryVideoStream` parsing: the nested + * `stream` collection (array of JSON strings, array of objects, a JSON string, + * or an index-keyed object) first, then meta-core's flat `stream/{n}` keys. + * Malformed entries are skipped, never fatal. + */ +export function streamsFromMeta(existingMeta: Record | undefined): StreamEntry[] { + if (!existingMeta) return []; + + const raw: unknown[] = []; + + const nested = existingMeta['stream']; + if (nested) { + try { + const parsed = typeof nested === 'string' ? JSON.parse(nested) : nested; + if (Array.isArray(parsed)) raw.push(...parsed); + else if (parsed && typeof parsed === 'object') raw.push(...Object.values(parsed as Record)); + } catch { + // fall through to the namespaced form + } + } + + if (raw.length === 0) { + const flat = Object.entries(existingMeta) + .filter(([key]) => /^stream\/\d+$/.test(key)) + .sort(([a], [b]) => Number(a.slice(7)) - Number(b.slice(7))); + for (const [, value] of flat) raw.push(value); + } + + const streams: StreamEntry[] = []; + for (const entry of raw) { + let stream: unknown; + try { + stream = typeof entry === 'string' ? JSON.parse(entry) : entry; + } catch { + continue; + } + if (stream && typeof stream === 'object' && !Array.isArray(stream)) { + streams.push(stream as StreamEntry); + } + } + return streams; +} + +// ISO 639-2/T → 639-2/B, the only codes where the two disagree. MP4's `mdhd` +// speaks T, Matroska speaks B, ffmpeg passes through either. Mirrors +// meta-watch `ui/js/lang-codes.js` T2B. +const T2B: Record = { + sqi: 'alb', hye: 'arm', eus: 'baq', mya: 'bur', ces: 'cze', + zho: 'chi', cym: 'wel', deu: 'ger', nld: 'dut', ell: 'gre', fas: 'per', + fra: 'fre', kat: 'geo', isl: 'ice', mkd: 'mac', msa: 'may', + ron: 'rum', slk: 'slo', +}; + +// ISO 639-1 → 639-2/B. Mirrors meta-watch `lang-codes.js` (its `a1` column plus +// the legacy/regional two-letter aliases). +const A1: Record = { + en: 'eng', fr: 'fre', de: 'ger', es: 'spa', it: 'ita', pt: 'por', nl: 'dut', + sv: 'swe', no: 'nor', da: 'dan', fi: 'fin', is: 'ice', pl: 'pol', cs: 'cze', + sk: 'slo', sl: 'slv', hu: 'hun', ro: 'rum', bg: 'bul', ru: 'rus', uk: 'ukr', + be: 'bel', sr: 'srp', hr: 'hrv', bs: 'bos', mk: 'mac', sq: 'alb', el: 'gre', + tr: 'tur', he: 'heb', ar: 'ara', fa: 'per', ur: 'urd', hi: 'hin', bn: 'ben', + ta: 'tam', te: 'tel', ml: 'mal', kn: 'kan', mr: 'mar', gu: 'guj', pa: 'pan', + ne: 'nep', si: 'sin', th: 'tha', lo: 'lao', km: 'khm', my: 'bur', vi: 'vie', + id: 'ind', ms: 'may', tl: 'tgl', ja: 'jpn', ko: 'kor', zh: 'chi', mn: 'mon', + kk: 'kaz', uz: 'uzb', az: 'aze', ka: 'geo', hy: 'arm', et: 'est', lv: 'lav', + lt: 'lit', ca: 'cat', gl: 'glg', eu: 'baq', cy: 'wel', ga: 'gle', af: 'afr', + sw: 'swa', am: 'amh', zu: 'zul', la: 'lat', eo: 'epo', yi: 'yid', jv: 'jav', + // legacy / regional + nb: 'nor', nn: 'nor', iw: 'heb', in: 'ind', ji: 'yid', jw: 'jav', +}; + +// "This track has no language" spellings. Never written (registry §9). +const NONE = new Set(['und', 'zxx', 'mis', 'unknown', 'none', '']); + +/** + * Canonical 639-2/B alpha-3 for a stream-tag language code; `''` when the + * track declares no language. Accepts 639-1 (`ja`), 639-2/B (`jpn`), 639-2/T + * (`deu` → `ger`) and BCP-47 (`pt-BR` → `por`). An unrecognised 3-letter code + * passes through lowercased (still a distinguishable language); anything else + * is dropped rather than guessed. `mul` is kept — it is a valid member. + * Same rules as meta-watch `MW_LANG.toLang3`. + */ +export function toLang3(code: unknown): string { + const s = String(code ?? '').trim().toLowerCase(); + if (!s) return ''; + const primary = s.split(/[-_]/)[0]; + if (NONE.has(primary)) return ''; + if (!/^[a-z]+$/.test(primary)) return ''; + if (primary.length === 2) return A1[primary] ?? ''; + if (primary.length === 3) return T2B[primary] ?? primary; + return ''; +} + +/** Distinct canonical audio-track languages, in stream order. */ +export function audioLanguagesFromMeta(existingMeta: Record | undefined): string[] { + const out: string[] = []; + for (const stream of streamsFromMeta(existingMeta)) { + if (stream.type !== 'audio') continue; + const lang = toLang3(stream.language); + if (lang && !out.includes(lang)) out.push(lang); + } + return out; +} + +/** + * Whether the record already carries `languages/`, in either shape: the + * flat key, or the nested `languages: { : ... }` object meta-sort's + * reconstructed document holds. + */ +function hasMember(existingMeta: Record | undefined, code: string): boolean { + if (!existingMeta) return false; + if (existingMeta[`languages/${code}`] !== undefined) return true; + const nested = existingMeta['languages']; + return !!nested && typeof nested === 'object' && !Array.isArray(nested) + && (nested as Record)[code] !== undefined; +} + +/** The `languages/` members to add — never one the record already has. */ +export function languageWrites(existingMeta: Record | undefined): Record { + const writes: Record = {}; + for (const code of audioLanguagesFromMeta(existingMeta)) { + if (!hasMember(existingMeta, code)) writes[`languages/${code}`] = 'true'; + } + return writes; +} + +/** The subset of the meta-core client this plugin uses (injectable for tests). */ +export interface LanguageWriter { + mergeMetadata(hashId: string, metadata: Record): Promise; +} + export async function process( request: ProcessRequest, - sendCallback: (payload: CallbackPayload) => Promise + sendCallback: (payload: CallbackPayload) => Promise, + writer: LanguageWriter = new MetaCoreClient(request.metaCoreUrl), ): Promise { const startTime = Date.now(); - const metaCore = new MetaCoreClient(request.metaCoreUrl); try { - const { cid, existingMeta } = request; + const { cid } = request; + const existingMeta = request.existingMeta as Record | undefined; + const found = audioLanguagesFromMeta(existingMeta); + const writes = languageWrites(existingMeta); - let streamLanguage: string | null = null; - - // Collect languages from title keys (if existingMeta has titles/*) - for (const key of Object.keys(existingMeta || {})) { - if (key.startsWith('titles/')) { - const langCode = key.substring(7); // Remove 'titles/' prefix - const normalized = anyTo_iso_639_3(langCode); - if (normalized) { - await metaCore.addToSet(cid, 'languages', normalized); - } - } + if (Object.keys(writes).length > 0) { + // One PATCH (merge) carrying only the new members: additive and + // idempotent, touches no other key. + const ok = await writer.mergeMetadata(cid, writes); + if (!ok) throw new Error(`failed to write ${Object.keys(writes).join(', ')}`); } - // Collect languages from audio streams (highest priority for primary) - for (let i = 0; i < 20; i++) { - const lang = existingMeta?.[`fileinfo/streamdetails/audio/${i}/language`]; - if (!lang) break; - const computedLanguage = anyTo_iso_639_3(lang); - if (computedLanguage) { - await metaCore.addToSet(cid, 'languages', computedLanguage); - if (!streamLanguage) { - streamLanguage = computedLanguage; - } - } - } - - // Collect languages from video streams - for (let i = 0; i < 20; i++) { - const lang = existingMeta?.[`fileinfo/streamdetails/video/${i}/language`]; - if (!lang) break; - const computedLanguage = anyTo_iso_639_3(lang); - if (computedLanguage) { - await metaCore.addToSet(cid, 'languages', computedLanguage); - if (!streamLanguage) { - streamLanguage = computedLanguage; - } - } - } - - // Collect languages from subtitle streams - for (let i = 0; i < 20; i++) { - const lang = existingMeta?.[`fileinfo/streamdetails/subtitle/${i}/language`]; - if (!lang) break; - const computedLanguage = anyTo_iso_639_3(lang); - if (computedLanguage) { - await metaCore.addToSet(cid, 'languages', computedLanguage); - if (!streamLanguage) { - streamLanguage = computedLanguage; - } - } - } - - // Determine the language of the media file based on: - // 1st audio stream, 2nd video stream, 3rd subtitle stream (first found) - // Assume original title is one of the stream languages or English if not found - const originalTitle = existingMeta?.originalTitle; - if (originalTitle) { - await metaCore.setProperty(cid, `titles/${streamLanguage || 'eng'}`, originalTitle); - } - - console.log(`[language] Aggregated languages, primary: ${streamLanguage || 'none'}`); + console.log(`[language] ${cid}: audio languages [${found.join(', ') || 'none'}], added [${Object.keys(writes).join(', ') || 'none'}]`); await sendCallback({ taskId: request.taskId, diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..52af0b6 --- /dev/null +++ b/test.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# ============================================================================= +# Language Plugin Test Runner +# +# Runs the unit tests inside a Docker container. +# Usage: +# ./test.sh # Build and run tests +# ./test.sh --no-cache # Rebuild without cache +# ./test.sh --shell # Start shell in test container for debugging +# ============================================================================= + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +IMAGE_NAME="metamesh-plugin-language-test" +DOCKERFILE="Dockerfile.test" + +NO_CACHE="" +SHELL_MODE="" + +for arg in "$@"; do + case $arg in + --no-cache) + NO_CACHE="--no-cache" + ;; + --shell) + SHELL_MODE=1 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --no-cache Rebuild Docker image without cache" + echo " --shell Start interactive shell in test container" + echo " --help Show this help message" + exit 0 + ;; + esac +done + +echo "==============================================" +echo " Language Plugin Test Runner" +echo "==============================================" +echo "" + +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed or not in PATH" + exit 1 +fi + +echo "[1/2] Building test image..." +docker build $NO_CACHE -f "$DOCKERFILE" -t "$IMAGE_NAME" . + +if [ -n "$SHELL_MODE" ]; then + echo "" + echo "[2/2] Starting interactive shell..." + echo " Run 'npm test' to execute tests" + echo "" + docker run --rm -it \ + -v "$SCRIPT_DIR/test:/app/test:ro" \ + -v "$SCRIPT_DIR/src:/app/src:ro" \ + "$IMAGE_NAME" \ + /bin/bash +else + echo "" + echo "[2/2] Running tests..." + echo "" + + set +e + docker run --rm -e CI=true "$IMAGE_NAME" + EXIT_CODE=$? + set -e + + echo "" + if [ $EXIT_CODE -eq 0 ]; then + echo "==============================================" + echo " All tests passed!" + echo "==============================================" + else + echo "==============================================" + echo " Tests failed (exit code: $EXIT_CODE)" + echo "==============================================" + fi + + exit $EXIT_CODE +fi diff --git a/test/integration.test.ts b/test/integration.test.ts deleted file mode 100644 index 1e0deca..0000000 --- a/test/integration.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Language Plugin Integration Tests - * - * Tests language aggregation from streams and primary language determination. - * Uses @metazla/filename-tools library for ISO 639-3 conversion. - */ - -import { describe, it, expect, beforeAll } from 'vitest'; -import { anyTo_iso_639_3 } from '@metazla/filename-tools'; - -// Dynamic import of plugin module -let manifest: typeof import('../src/plugin.js').manifest; -let processFile: typeof import('../src/plugin.js').process; - -// Mock callback collector -interface CallbackResult { - taskId: string; - status: 'completed' | 'failed' | 'skipped'; - duration: number; - error?: string; -} - -let lastCallback: CallbackResult | null = null; - -const mockSendCallback = async (payload: CallbackResult): Promise => { - lastCallback = payload; -}; - -describe('Language Plugin Integration Tests', () => { - beforeAll(async () => { - const plugin = await import('../src/plugin.js'); - manifest = plugin.manifest; - processFile = plugin.process; - }); - - describe('Manifest', () => { - it('has required fields', () => { - expect(manifest.id).toBe('language'); - expect(manifest.name).toBeDefined(); - expect(manifest.version).toBeDefined(); - expect(manifest.dependencies).toContain('ffmpeg'); - expect(manifest.priority).toBe(40); - }); - - it('declares correct schema', () => { - expect(manifest.schema).toHaveProperty('languages'); - }); - }); - - describe('ISO 639-3 Conversion', () => { - it('handles 3-letter codes', () => { - // The function should handle 3-letter ISO 639-3 codes - const engResult = anyTo_iso_639_3('eng'); - const jpnResult = anyTo_iso_639_3('jpn'); - const fraResult = anyTo_iso_639_3('fra'); - - // Function returns the code if valid, or 'und' for unknown - expect(['eng', 'und']).toContain(engResult); - expect(['jpn', 'und']).toContain(jpnResult); - expect(['fra', 'und']).toContain(fraResult); - }); - - it('returns a defined value for known codes', () => { - const result = anyTo_iso_639_3('eng'); - expect(result).toBeDefined(); - expect(typeof result).toBe('string'); - }); - - it('handles unknown codes gracefully', () => { - const result = anyTo_iso_639_3('xyz'); - // Should return 'und' (undefined) or the original value - expect(result).toBeDefined(); - expect(typeof result).toBe('string'); - }); - }); - - describe('Process Function', () => { - it('processes file with audio streams', async () => { - await processFile({ - taskId: 'test-lang-1', - cid: 'test-cid-lang', - filePath: '/movies/Movie.mkv', - callbackUrl: 'http://localhost/callback', - metaCoreUrl: 'http://localhost:9000', - existingMeta: { - originalTitle: 'Test Movie', - 'fileinfo/streamdetails/audio/0/language': 'eng', - 'fileinfo/streamdetails/audio/1/language': 'jpn', - }, - }, mockSendCallback); - - expect(lastCallback).toBeDefined(); - expect(lastCallback?.status).toBe('completed'); - }); - - it('processes file with subtitle streams', async () => { - await processFile({ - taskId: 'test-lang-2', - cid: 'test-cid-lang-2', - filePath: '/movies/Movie.mkv', - callbackUrl: 'http://localhost/callback', - metaCoreUrl: 'http://localhost:9000', - existingMeta: { - originalTitle: 'Test Movie', - 'fileinfo/streamdetails/subtitle/0/language': 'eng', - 'fileinfo/streamdetails/subtitle/1/language': 'fra', - }, - }, mockSendCallback); - - expect(lastCallback).toBeDefined(); - expect(lastCallback?.status).toBe('completed'); - }); - - it('processes file with existing title languages', async () => { - await processFile({ - taskId: 'test-lang-3', - cid: 'test-cid-lang-3', - filePath: '/movies/Movie.mkv', - callbackUrl: 'http://localhost/callback', - metaCoreUrl: 'http://localhost:9000', - existingMeta: { - originalTitle: 'Test Movie', - 'titles/eng': 'Test Movie', - 'titles/jpn': 'テスト映画', - }, - }, mockSendCallback); - - expect(lastCallback).toBeDefined(); - expect(lastCallback?.status).toBe('completed'); - }); - - it('processes file with no language information', async () => { - await processFile({ - taskId: 'test-lang-4', - cid: 'test-cid-lang-4', - filePath: '/movies/Movie.mkv', - callbackUrl: 'http://localhost/callback', - metaCoreUrl: 'http://localhost:9000', - existingMeta: { - originalTitle: 'Test Movie', - }, - }, mockSendCallback); - - expect(lastCallback).toBeDefined(); - expect(lastCallback?.status).toBe('completed'); - }); - - it('processes file with video stream language', async () => { - await processFile({ - taskId: 'test-lang-5', - cid: 'test-cid-lang-5', - filePath: '/movies/Movie.mkv', - callbackUrl: 'http://localhost/callback', - metaCoreUrl: 'http://localhost:9000', - existingMeta: { - originalTitle: 'Test Movie', - 'fileinfo/streamdetails/video/0/language': 'eng', - }, - }, mockSendCallback); - - expect(lastCallback).toBeDefined(); - expect(lastCallback?.status).toBe('completed'); - }); - }); -}); diff --git a/test/language.test.ts b/test/language.test.ts new file mode 100644 index 0000000..7dd16cc --- /dev/null +++ b/test/language.test.ts @@ -0,0 +1,221 @@ +/** + * Language plugin unit tests — no network, no meta-core: the writer is injected. + */ + +import { describe, it, expect } from 'vitest'; +import { + manifest, + process as processFile, + streamsFromMeta, + toLang3, + audioLanguagesFromMeta, + languageWrites, + type LanguageWriter, +} from '../src/plugin.js'; +import type { CallbackPayload, ProcessRequest } from '../src/types.js'; + +// Trimmed from the real nested /meta document of a dev-stack record +// (1080p AV1 episode: 2 audio, 9 subtitle, cover-art mjpeg). +const REAL_STREAMS: string[] = [ + '{"type":"video","codec":"av1","index":0,"duration":"N/A","forced":false,"default":true,"title":"By Trix","width":1920,"height":1080,"frameRate":"24000/1001"}', + '{"type":"audio","codec":"aac","index":1,"duration":"N/A","forced":false,"default":true,"language":"jpn","title":"Japanese - AAC 2.0","sampleRate":"44100","channelLayout":"stereo"}', + '{"type":"audio","codec":"aac","index":2,"duration":"N/A","forced":false,"default":true,"language":"eng","title":"English - AAC 2.0","sampleRate":"44100","channelLayout":"stereo"}', + '{"type":"subtitle","codec":"ass","index":3,"language":"eng","title":"English (Full) [4de]"}', + '{"type":"subtitle","codec":"ass","index":5,"language":"fre","title":"French"}', + '{"type":"subtitle","codec":"ass","index":6,"language":"ger","title":"German"}', + '{"type":"video","codec":"mjpeg","index":12,"duration":"1433.089","width":230,"height":345}', +]; + +function nestedDoc(extra: Record = {}): Record { + return { + fileinfo: { duration: 1433.089, formatName: 'matroska,webm' }, + stream: REAL_STREAMS, + originalTitle: 'Some Show', + cids: ['bagacbabaebs63pblr2rsycodkmg4ojn47ol7k4oz6ptcre2bwaguhsib4hjpg'], + ...extra, + }; +} + +class RecordingWriter implements LanguageWriter { + calls: Array<{ hashId: string; metadata: Record }> = []; + constructor(private ok = true) {} + async mergeMetadata(hashId: string, metadata: Record): Promise { + this.calls.push({ hashId, metadata }); + return this.ok; + } +} + +async function run(existingMeta: Record | undefined, writer = new RecordingWriter()) { + let callback: CallbackPayload | null = null; + const request = { + taskId: 't1', + cid: 'cid-1', + filePath: '/files/watch/Show - 01.mkv', + callbackUrl: 'http://metasort-app/api/plugins/callback', + metaCoreUrl: 'http://metasort-app', + existingMeta, + } as unknown as ProcessRequest; + await processFile(request, async (p) => { callback = p; }, writer); + return { callback: callback as CallbackPayload | null, writer }; +} + +describe('manifest', () => { + it('keeps the plugin identity and ffmpeg dependency', () => { + expect(manifest.id).toBe('language'); + expect(manifest.dependencies).toContain('ffmpeg'); + expect(manifest.priority).toBe(40); + expect(manifest.schema).toHaveProperty('languages'); + }); +}); + +describe('streamsFromMeta', () => { + it('reads the nested stream array of JSON strings (meta-sort payload)', () => { + const streams = streamsFromMeta(nestedDoc()); + expect(streams).toHaveLength(REAL_STREAMS.length); + expect(streams[1]).toMatchObject({ type: 'audio', language: 'jpn', index: 1 }); + }); + + it('reads a nested stream array of objects', () => { + const streams = streamsFromMeta({ stream: REAL_STREAMS.map((s) => JSON.parse(s)) }); + expect(streams.map((s) => s.index)).toEqual([0, 1, 2, 3, 5, 6, 12]); + }); + + it('reads a nested stream JSON string and an index-keyed object', () => { + expect(streamsFromMeta({ stream: JSON.stringify(REAL_STREAMS) })).toHaveLength(REAL_STREAMS.length); + const keyed = Object.fromEntries(REAL_STREAMS.map((s, i) => [String(i), s])); + expect(streamsFromMeta({ stream: keyed })).toHaveLength(REAL_STREAMS.length); + }); + + it('reads flat stream/{n} keys (meta-core form) in numeric order', () => { + const flat: Record = { 'fileinfo/duration': '1433.089' }; + // insert out of order, with n >= 10 to catch lexical sorting + flat['stream/10'] = '{"type":"audio","index":10,"language":"spa"}'; + flat['stream/2'] = '{"type":"audio","index":2,"language":"eng"}'; + flat['stream/1'] = '{"type":"audio","index":1,"language":"jpn"}'; + expect(streamsFromMeta(flat).map((s) => s.index)).toEqual([1, 2, 10]); + }); + + it('returns nothing for missing streams and skips malformed entries', () => { + expect(streamsFromMeta(undefined)).toEqual([]); + expect(streamsFromMeta({})).toEqual([]); + expect(streamsFromMeta({ originalTitle: 'x' })).toEqual([]); + expect(streamsFromMeta({ stream: ['not json', '{"type":"audio","language":"eng"}', null] })) + .toEqual([{ type: 'audio', language: 'eng' }]); + }); + + it('ignores the legacy fileinfo/streamdetails keys nobody writes', () => { + expect(streamsFromMeta({ 'fileinfo/streamdetails/audio/0/language': 'eng' })).toEqual([]); + }); +}); + +describe('toLang3', () => { + it('keeps 639-2/B codes', () => { + for (const c of ['eng', 'jpn', 'fre', 'ger', 'chi', 'spa']) expect(toLang3(c)).toBe(c); + }); + + it('folds 639-2/T onto 639-2/B (the filter vocabulary)', () => { + expect(toLang3('fra')).toBe('fre'); + expect(toLang3('deu')).toBe('ger'); + expect(toLang3('zho')).toBe('chi'); + expect(toLang3('nld')).toBe('dut'); + expect(toLang3('ces')).toBe('cze'); + }); + + it('maps 639-1 and BCP-47 tags (the old normaliser returned und for all 2-letter codes)', () => { + expect(toLang3('en')).toBe('eng'); + expect(toLang3('fr')).toBe('fre'); + expect(toLang3('ja')).toBe('jpn'); + expect(toLang3('pt-BR')).toBe('por'); + expect(toLang3('EN_us')).toBe('eng'); + expect(toLang3(' JPN ')).toBe('jpn'); + }); + + it('never yields und and drops no-language spellings', () => { + for (const c of ['und', 'UND', 'zxx', 'mis', '', undefined, null, 'unknown']) expect(toLang3(c)).toBe(''); + }); + + it('keeps mul and unknown 3-letter codes, drops unknown 2-letter and full names', () => { + expect(toLang3('mul')).toBe('mul'); + expect(toLang3('xyz')).toBe('xyz'); + expect(toLang3('xx')).toBe(''); + expect(toLang3('French')).toBe(''); + expect(toLang3('e1g')).toBe(''); + }); +}); + +describe('audioLanguagesFromMeta', () => { + it('takes audio tracks only — subtitle languages are not audio languages', () => { + expect(audioLanguagesFromMeta(nestedDoc())).toEqual(['jpn', 'eng']); + }); + + it('dedupes after normalisation and skips und/untagged audio', () => { + const meta = { + stream: [ + { type: 'audio', language: 'fra' }, + { type: 'audio', language: 'fre' }, + { type: 'audio', language: 'fr' }, + { type: 'audio', language: 'und' }, + { type: 'audio' }, + { type: 'video', language: 'eng' }, + ], + }; + expect(audioLanguagesFromMeta(meta)).toEqual(['fre']); + }); +}); + +describe('languageWrites', () => { + it('skips members already present in the flat or nested form', () => { + expect(languageWrites(nestedDoc({ 'languages/jpn': 'true' }))).toEqual({ 'languages/eng': 'true' }); + expect(languageWrites(nestedDoc({ languages: { eng: 'true', jpn: true } }))).toEqual({}); + }); +}); + +describe('process', () => { + it('writes the audio languages/* key-set from the nested payload in one merge', async () => { + const { callback, writer } = await run(nestedDoc()); + expect(callback?.status).toBe('completed'); + expect(writer.calls).toEqual([ + { hashId: 'cid-1', metadata: { 'languages/jpn': 'true', 'languages/eng': 'true' } }, + ]); + }); + + it('writes from flat stream/{n} keys too', async () => { + const flat = Object.fromEntries(REAL_STREAMS.map((s, i) => [`stream/${i}`, s])); + const { callback, writer } = await run(flat); + expect(callback?.status).toBe('completed'); + expect(writer.calls[0].metadata).toEqual({ 'languages/jpn': 'true', 'languages/eng': 'true' }); + }); + + it('never writes titles/* or any non-languages key, even with originalTitle and title keys', async () => { + const { writer } = await run(nestedDoc({ 'titles/eng': 'Better Title', titles: { jpn: '番組' } })); + const keys = writer.calls.flatMap((c) => Object.keys(c.metadata)); + expect(keys.length).toBeGreaterThan(0); + expect(keys.every((k) => k.startsWith('languages/'))).toBe(true); + expect(keys.some((k) => k.startsWith('titles'))).toBe(false); + }); + + it('does not derive languages from title keys', async () => { + const { callback, writer } = await run({ originalTitle: 'x', 'titles/jpn': 'y', 'titles/jpl': 'z' }); + expect(callback?.status).toBe('completed'); + expect(writer.calls).toEqual([]); + }); + + it('completes without writing when there are no streams or no tagged audio', async () => { + for (const meta of [undefined, {}, { stream: [] }, { stream: [{ type: 'audio', language: 'und' }] }]) { + const { callback, writer } = await run(meta); + expect(callback?.status).toBe('completed'); + expect(writer.calls).toEqual([]); + } + }); + + it('is idempotent: a re-run over a record that already has the members writes nothing', async () => { + const { writer } = await run(nestedDoc({ 'languages/jpn': 'true', 'languages/eng': 'true' })); + expect(writer.calls).toEqual([]); + }); + + it('fails the task when the write is rejected', async () => { + const { callback } = await run(nestedDoc(), new RecordingWriter(false)); + expect(callback?.status).toBe('failed'); + expect(callback?.error).toContain('languages/jpn'); + }); +});