commit 62a2b796e2389fb98005be8183f959364d9a2062 Author: Pierre Date: Sat Jun 6 15:20:36 2026 +0200 initial commit diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..8cdc3e7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,56 @@ +name: Build and Push Docker Image + +on: + push: + branches: [main, master] + tags: ['v*'] + pull_request: + branches: [main, master] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..faec21b --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +dist/ +dist-test/ +*.log +.env +.DS_Store +.pnpm-store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dbd82f2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# MetaMesh Plugin: opensubtitles +# Fetches subtitle sidecars from OpenSubtitles + +FROM node:20-slim AS builder + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN npm install -g corepack@latest && corepack enable + +WORKDIR /app + +# Copy package files first for layer caching +COPY package.json pnpm-lock.yaml ./ +RUN corepack install +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile + +# Copy source and build +COPY tsconfig.json ./ +COPY src/ ./src/ +RUN pnpm run build + +# Production image +FROM node:20-slim + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN npm install -g corepack@latest && corepack enable + +WORKDIR /app + +COPY package.json pnpm-lock.yaml ./ +RUN corepack install +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/dist ./dist + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))" + +CMD ["node", "dist/index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..f33fba9 --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# MetaMesh Plugin: OpenSubtitles + +A MetaMesh plugin that **fetches** subtitle sidecars from +[OpenSubtitles](https://www.opensubtitles.com) (REST API v1) and links them to +videos. This is the network-download counterpart to `subtitle-extractor`, which +only pulls subtitles already embedded in the media file. + +## Description + +For each video, the plugin: + +1. Computes the OpenSubtitles **moviehash** (size + first/last 64 KiB, read over + WebDAV) for exact-match accuracy. +2. Searches `/subtitles` by `moviehash` + `imdb_id`/`tmdb_id` (+ season/episode + for TV), falling back to a title/year query only when no structured + identifier is available. +3. Picks the single best subtitle per wanted language (exact moviehash match + first, then popularity). +4. Logs in (JWT) and downloads each chosen file, writes it via WebDAV to + `/files/plugin/opensubtitles/`, and registers the resulting CID on the video. + +**Note**: Runs on the **background** queue (external API calls, download +quota). Depends on `tmdb` so `imdbid`/`tmdbid` are populated before it runs. + +## Metadata Fields (written on the video record) + +| Field | Format | Description | +|-------|--------|-------------| +| `subtitles` | csv-set of `cid` | Fetched subtitle file CIDs (external/sidecar provenance, per METADATA_KEYS.md §8) | +| `subtitleLanguages` | csv-set of `lang3` | Denormalized union of available subtitle languages | +| `languages/` | key-set | Adds the subtitle language to the file's language set (§9) | + +Each downloaded subtitle is itself indexed as a first-class record under its +own `midhash256` CID — only the CID is stored on the video, never a path. + +> **Migration note:** `subtitles` / `subtitleLanguages` are written as legacy +> `csv-set` via `addToSet` to stay compatible with current consumers. They are +> slated to migrate to `key-set` (METADATA_KEYS.md §14.12) in lockstep with the +> other subtitle writers (`subtitle`, `subtitle-extractor`). + +## Dependencies + +- Requires `file-info`, `filename-parser`, and `tmdb` (for `imdbid`/`tmdbid`). + +## Configuration + +Set at runtime via the meta-sort REST API (never in `plugins.yml`): + +```bash +curl -k -X PUT https://metasort-dev.localhost:8180/api/plugins/opensubtitles/config \ + -H "Content-Type: application/json" \ + -d '{"apiKey":"YOUR_API_KEY","username":"YOUR_USER","password":"YOUR_PASS","languages":"en,fr"}' +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `apiKey` | string | Yes | OpenSubtitles API key (Consumer key from your account) | +| `username` | string | For downloads | OpenSubtitles account username | +| `password` | string | For downloads | OpenSubtitles account password | +| `languages` | string | No | CSV of ISO 639-1 codes (default `en`) | +| `userAgent` | string | No | User-Agent registered with OpenSubtitles | +| `forceRecompute` | boolean | No | Re-fetch even if subs already present | + +> The API key alone permits **search**; **downloading** requires a logged-in +> account (username + password → JWT). The free tier has a small daily download +> quota — the plugin stops early and reports when quota is exhausted. + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/health` | GET | Health check | +| `/manifest` | GET | Plugin manifest | +| `/configure` | POST | Update configuration | +| `/process` | POST | Process a file | + +## Docker + +```bash +# From the plugin directory +docker build -t metamesh-plugin-opensubtitles:main . +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | `8080` | HTTP server port | +| `WEBDAV_URL` | — | meta-sort WebDAV base (provided by the orchestrator) | +| `CACHE_DIR` | `/cache` | Plugin cache mount | + +## License + +MIT diff --git a/package.json b/package.json new file mode 100644 index 0000000..fda89ab --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "@metazla/plugin-opensubtitles", + "version": "1.0.0", + "packageManager": "pnpm@10.28.0", + "description": "MetaMesh plugin: Fetches subtitle sidecars from OpenSubtitles", + "type": "module", + "main": "dist/index.js", + "scripts": { "build": "tsc", "start": "node dist/index.js", "test": "tsc -p tsconfig.test.json && node --test \"dist-test/*.test.js\"" }, + "dependencies": { "fastify": "^5.0.0", "axios": "^1.6.0" }, + "devDependencies": { "@types/node": "^20.0.0", "typescript": "^5.3.0" } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..29e6d95 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,568 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + axios: + specifier: ^1.6.0 + version: 1.13.5 + fastify: + specifier: ^5.0.0 + version: 5.7.4 + devDependencies: + '@types/node': + specifier: ^20.0.0 + version: 20.19.33 + typescript: + specifier: ^5.3.0 + version: 5.9.3 + +packages: + + '@fastify/ajv-compiler@4.0.5': + resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} + + '@fastify/error@4.2.0': + resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} + + '@fastify/fast-json-stringify-compiler@5.0.3': + resolution: {integrity: sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==} + + '@fastify/forwarded@3.0.1': + resolution: {integrity: sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==} + + '@fastify/merge-json-schemas@0.2.1': + resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} + + '@fastify/proxy-addr@5.1.0': + resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + + '@types/node@20.19.33': + resolution: {integrity: sha512-Rs1bVAIdBs5gbTIKza/tgpMuG1k3U/UMJLWecIMxNdJFDMzcM5LOiLVRYh3PilWEYDIeUDv7bpiHPLPsbydGcw==} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@9.1.0: + resolution: {integrity: sha512-fYASnYi600CsH/j9EQov7lECAniYiBFiiAtBNuZYLA2leLe9qOvZzqYHFjtIj6gD2VMoMLP14834LFWvr4IfDw==} + + axios@1.13.5: + resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@6.3.0: + resolution: {integrity: sha512-oRCntNDY/329HJPlmdNLIdogNtt6Vyjb1WuT01Soss3slIdyUp8kAcDU3saQTOquEK8KFVfwIIF7FebxUAu+yA==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + + fastify@5.7.4: + resolution: {integrity: sha512-e6l5NsRdaEP8rdD8VR0ErJASeyaRbzXYpmkrpr2SuvuMq6Si3lvsaVy5C+7gLanEkvjpMDzBXWE5HPeb/hgTxA==} + + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + + find-my-way@9.4.0: + resolution: {integrity: sha512-5Ye4vHsypZRYtS01ob/iwHzGRUDELlsoCftI/OZFhcLs1M0tkGPcXldE80TAZC5yYuJMBPJQQ43UHlqbJWiX2w==} + engines: {node: '>=20'} + + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + ipaddr.js@2.3.0: + resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==} + engines: {node: '>= 10'} + + json-schema-ref-resolver@3.0.0: + resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + light-my-request@6.6.0: + resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@10.3.0: + resolution: {integrity: sha512-0GNPNzHXBKw6U/InGe79A3Crzyk9bcSyObF9/Gfo9DLEf5qj5RF50RSjsu0W1rZ6ZqRGdzDFCRBQvi9/rSGPtA==} + hasBin: true + + process-warning@4.0.1: + resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} + + process-warning@5.0.0: + resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + ret@0.5.0: + resolution: {integrity: sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==} + engines: {node: '>=10'} + + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + + safe-regex2@5.0.0: + resolution: {integrity: sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==} + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + + sonic-boom@4.2.0: + resolution: {integrity: sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + thread-stream@4.0.0: + resolution: {integrity: sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==} + engines: {node: '>=20'} + + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + +snapshots: + + '@fastify/ajv-compiler@4.0.5': + dependencies: + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) + fast-uri: 3.1.0 + + '@fastify/error@4.2.0': {} + + '@fastify/fast-json-stringify-compiler@5.0.3': + dependencies: + fast-json-stringify: 6.3.0 + + '@fastify/forwarded@3.0.1': {} + + '@fastify/merge-json-schemas@0.2.1': + dependencies: + dequal: 2.0.3 + + '@fastify/proxy-addr@5.1.0': + dependencies: + '@fastify/forwarded': 3.0.1 + ipaddr.js: 2.3.0 + + '@pinojs/redact@0.4.0': {} + + '@types/node@20.19.33': + dependencies: + undici-types: 6.21.0 + + abstract-logging@2.0.1: {} + + ajv-formats@3.0.1(ajv@8.17.1): + optionalDependencies: + ajv: 8.17.1 + + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + asynckit@0.4.0: {} + + atomic-sleep@1.0.0: {} + + avvio@9.1.0: + dependencies: + '@fastify/error': 4.2.0 + fastq: 1.20.1 + + axios@1.13.5: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + cookie@1.1.1: {} + + delayed-stream@1.0.0: {} + + dequal@2.0.3: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@6.3.0: + dependencies: + '@fastify/merge-json-schemas': 0.2.1 + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) + fast-uri: 3.1.0 + json-schema-ref-resolver: 3.0.0 + rfdc: 1.4.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-uri@3.1.0: {} + + fastify@5.7.4: + dependencies: + '@fastify/ajv-compiler': 4.0.5 + '@fastify/error': 4.2.0 + '@fastify/fast-json-stringify-compiler': 5.0.3 + '@fastify/proxy-addr': 5.1.0 + abstract-logging: 2.0.1 + avvio: 9.1.0 + fast-json-stringify: 6.3.0 + find-my-way: 9.4.0 + light-my-request: 6.6.0 + pino: 10.3.0 + process-warning: 5.0.0 + rfdc: 1.4.1 + secure-json-parse: 4.1.0 + semver: 7.7.4 + toad-cache: 3.7.0 + + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + + find-my-way@9.4.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 5.0.0 + + follow-redirects@1.15.11: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + gopd@1.2.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + ipaddr.js@2.3.0: {} + + json-schema-ref-resolver@3.0.0: + dependencies: + dequal: 2.0.3 + + json-schema-traverse@1.0.0: {} + + light-my-request@6.6.0: + dependencies: + cookie: 1.1.1 + process-warning: 4.0.1 + set-cookie-parser: 2.7.2 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + on-exit-leak-free@2.1.2: {} + + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@7.1.0: {} + + pino@10.3.0: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.0 + thread-stream: 4.0.0 + + process-warning@4.0.1: {} + + process-warning@5.0.0: {} + + proxy-from-env@1.1.0: {} + + quick-format-unescaped@4.0.4: {} + + real-require@0.2.0: {} + + require-from-string@2.0.2: {} + + ret@0.5.0: {} + + reusify@1.1.0: {} + + rfdc@1.4.1: {} + + safe-regex2@5.0.0: + dependencies: + ret: 0.5.0 + + safe-stable-stringify@2.5.0: {} + + secure-json-parse@4.1.0: {} + + semver@7.7.4: {} + + set-cookie-parser@2.7.2: {} + + sonic-boom@4.2.0: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + thread-stream@4.0.0: + dependencies: + real-require: 0.2.0 + + toad-cache@3.7.0: {} + + typescript@5.9.3: {} + + undici-types@6.21.0: {} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..45e5918 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,59 @@ +/** + * MetaMesh Plugin: OpenSubtitles + * + * ============================================================================ + * PLUGIN MOUNT ARCHITECTURE - DO NOT MODIFY WITHOUT AUTHORIZATION + * ============================================================================ + * + * Each plugin container has exactly 3 mounts: + * + * 1. /files (READ-ONLY) - Shared media files, read access only + * 2. /cache (READ-WRITE) - Plugin-specific cache folder + * 3. /files/plugin/ (READ-WRITE for this plugin, READ-ONLY for others) + * - Plugin output folder for generated files + * + * SECURITY: Plugins must NEVER write to /files directly. + * - Use /cache for temporary/cache data + * - Use /files/plugin/ for output files (e.g., posters, generated assets) + * + * This architecture ensures: + * - Plugins cannot corrupt user's media files + * - Plugins can share generated assets with other plugins (read-only) + * - Each plugin has isolated cache storage + * + * ============================================================================ + */ + +import Fastify from 'fastify'; +import type { HealthResponse, ProcessRequest, ProcessResponse, CallbackPayload, ConfigureRequest, ConfigureResponse } from './types.js'; +import { manifest, process as processFile, configure } from './plugin.js'; + +const app = Fastify({ logger: true }); +let ready = false; + +app.get('/health', async (): Promise => ({ status: 'healthy', ready, version: manifest.version })); +app.get('/manifest', async () => manifest); +app.post<{ Body: ConfigureRequest }>('/configure', async (request): Promise => { + try { + configure(request.body.config || {}); + console.log(`[${manifest.id}] Configuration updated`); + return { status: 'ok' }; + } catch (error) { + return { status: 'error', error: error instanceof Error ? error.message : String(error) }; + } +}); +app.post<{ Body: ProcessRequest }>('/process', async (request, reply) => { + const { taskId, cid, filePath, callbackUrl, metaCoreUrl } = request.body; + if (!taskId || !cid || !filePath || !callbackUrl || !metaCoreUrl) { + return reply.send({ status: 'rejected', error: 'Missing required fields' } as ProcessResponse); + } + processFile(request.body, async (payload: CallbackPayload) => { + try { await fetch(callbackUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); } + catch (error) { console.error(`[${manifest.id}] Callback error:`, error); } + }).catch(console.error); + return reply.send({ status: 'accepted' } as ProcessResponse); +}); + +const port = parseInt(process.env.PORT || '8080', 10); +app.listen({ port, host: '0.0.0.0' }).then(() => { ready = true; console.log(`[${manifest.id}] Listening on port ${port}`); }); +process.on('SIGTERM', async () => { ready = false; await app.close(); process.exit(0); }); diff --git a/src/langs.ts b/src/langs.ts new file mode 100644 index 0000000..70ceaed --- /dev/null +++ b/src/langs.ts @@ -0,0 +1,72 @@ +/** + * Language code normalization. + * + * OpenSubtitles speaks ISO 639-1 (2-letter, e.g. "en", "pt-br") while the + * MetaMesh metadata store uses ISO 639-3 (`lang3`, e.g. "eng") everywhere + * (see METADATA_KEYS.md §"Value formats" and §9). This maps the common + * subtitle languages OpenSubtitles serves to their 639-3 equivalents. + * + * Special case mirrored from the rest of the store: `jpl` is reserved for + * romaji transliterations of Japanese — OpenSubtitles has no such code, so + * it never appears here. + */ + +// ISO 639-1 → ISO 639-3 for the languages OpenSubtitles commonly serves. +const ISO1_TO_ISO3: Record = { + ab: 'abk', af: 'afr', am: 'amh', ar: 'ara', as: 'asm', az: 'aze', + be: 'bel', bg: 'bul', bn: 'ben', br: 'bre', bs: 'bos', + ca: 'cat', cs: 'ces', cy: 'cym', da: 'dan', de: 'deu', + el: 'ell', en: 'eng', eo: 'epo', es: 'spa', et: 'est', eu: 'eus', + fa: 'fas', fi: 'fin', fr: 'fra', ga: 'gle', gl: 'glg', + he: 'heb', hi: 'hin', hr: 'hrv', hu: 'hun', hy: 'hye', + id: 'ind', is: 'isl', it: 'ita', ja: 'jpn', ka: 'kat', kk: 'kaz', + km: 'khm', kn: 'kan', ko: 'kor', ku: 'kur', lb: 'ltz', lt: 'lit', + lv: 'lav', mk: 'mkd', ml: 'mal', mn: 'mon', mr: 'mar', ms: 'msa', + my: 'mya', nb: 'nob', ne: 'nep', nl: 'nld', nn: 'nno', no: 'nor', + oc: 'oci', pa: 'pan', pl: 'pol', pt: 'por', ro: 'ron', ru: 'rus', + si: 'sin', sk: 'slk', sl: 'slv', sq: 'sqi', sr: 'srp', sv: 'swe', + sw: 'swa', ta: 'tam', te: 'tel', th: 'tha', tl: 'tgl', tr: 'tur', + uk: 'ukr', ur: 'urd', uz: 'uzb', vi: 'vie', zh: 'zho', +}; + +// ISO 639-2/B (bibliographic) → ISO 639-3 (terminological) aliases. These show +// up in OpenSubtitles responses and release filenames; canonicalize them so the +// store never holds two codes for one language (e.g. `fre` and `fra`). Ported +// from the gateway's torznab subtitles.rs::iso3_canonical. +const ISO3_ALIASES: Record = { + fre: 'fra', ger: 'deu', dut: 'nld', chi: 'zho', cze: 'ces', + gre: 'ell', rum: 'ron', baq: 'eus', wel: 'cym', arm: 'hye', + geo: 'kat', ice: 'isl', mac: 'mkd', may: 'msa', bur: 'mya', + per: 'fas', tib: 'bod', slo: 'slk', alb: 'sqi', +}; + +/** + * Normalize an OpenSubtitles language code to ISO 639-3 (`lang3`). + * + * Handles region suffixes ("pt-BR" → "por", "zh-CN" → "zho") by taking the + * primary subtag, and canonicalizes 639-2/B bibliographic codes ("fre" → "fra"). + * Already-canonical 3-letter codes pass through lowercased. Returns "und" + * (undetermined) for anything unrecognized so the value still satisfies the + * `lang3` contract. + */ +export function toLang3(code: string | undefined | null): string { + if (!code) return 'und'; + const primary = code.toLowerCase().split(/[-_]/)[0]; + if (primary.length === 3) return ISO3_ALIASES[primary] ?? primary; + return ISO1_TO_ISO3[primary] ?? 'und'; +} + +/** + * Parse a user-supplied CSV of language codes (e.g. "en,fr,pt-BR") into the + * deduped lowercase 2-letter (or region-tagged) codes OpenSubtitles expects + * in its `languages` query parameter. + */ +export function parseWantedLanguages(csv: string | undefined): string[] { + if (!csv) return ['en']; + const seen = new Set(); + for (const raw of csv.split(',')) { + const code = raw.trim().toLowerCase(); + if (code) seen.add(code); + } + return seen.size > 0 ? [...seen] : ['en']; +} diff --git a/src/meta-core-client.ts b/src/meta-core-client.ts new file mode 100644 index 0000000..dc54232 --- /dev/null +++ b/src/meta-core-client.ts @@ -0,0 +1,89 @@ +/** + * meta-core API Client for writing metadata + * Fails gracefully if meta-core is unavailable (logs warning, returns without throwing) + */ + +export class MetaCoreClient { + constructor(private baseUrl: string) {} + + private async safeFetch(url: string, options: RequestInit): Promise { + try { + return await fetch(url, { ...options, signal: AbortSignal.timeout(5000) }); + } catch (error) { + console.warn(`[MetaCoreClient] Warning: meta-core unavailable at ${this.baseUrl}`); + return null; + } + } + + async setProperty(hashId: string, key: string, value: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ value }), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to set property: ${response.status}`); + } + } + + async getProperty(hashId: string, key: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'GET' }); + if (!response || response.status === 404) return null; + if (!response.ok) return null; + const data = await response.json() as { value?: string }; + return data.value ?? null; + } + + async mergeMetadata(hashId: string, metadata: Record): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(metadata), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to merge metadata: ${response.status}`); + } + } + + async deleteProperty(hashId: string, key: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/${key}`, { method: 'DELETE' }); + if (response && !response.ok && response.status !== 404) { + console.warn(`[MetaCoreClient] Failed to delete property: ${response.status}`); + } + } + + async addToSet(hashId: string, key: string, value: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}/_add/${key}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ value }), + }); + if (response && !response.ok) { + console.warn(`[MetaCoreClient] Failed to add to set: ${response.status}`); + } + } + + async getMetadata(hashId: string): Promise> { + const response = await this.safeFetch(`${this.baseUrl}/meta/${hashId}`, { method: 'GET' }); + if (!response || response.status === 404) return {}; + if (!response.ok) return {}; + const data = await response.json() as { metadata?: Record }; + return data.metadata ?? {}; + } + + /** + * Compute CID for a file + * @param path - Path relative to FILES_PATH (e.g., "poster/Movie (2024)[tmdb12345]_poster.jpg") + * @returns CID string or null if failed + */ + async computeFileCID(path: string): Promise { + const response = await this.safeFetch(`${this.baseUrl}/file/cid`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path }), + }); + if (!response || !response.ok) return null; + const data = await response.json() as { cid?: string }; + return data.cid ?? null; + } +} diff --git a/src/opensubtitles.ts b/src/opensubtitles.ts new file mode 100644 index 0000000..051fcf0 --- /dev/null +++ b/src/opensubtitles.ts @@ -0,0 +1,222 @@ +/** + * OpenSubtitles REST API client (api.opensubtitles.com / API v1). + * + * Auth is two-tier: + * - The `Api-Key` header + a registered `User-Agent` are enough to call + * `/subtitles` (search). + * - The `/download` endpoint additionally needs a JWT obtained from + * `POST /login` (username + password). The token is cached and reused + * until the API rejects it, then re-fetched once. + * + * Docs: https://opensubtitles.stoplight.io/docs/opensubtitles-api + */ + +import axios, { AxiosInstance, isAxiosError } from 'axios'; + +const DEFAULT_BASE_URL = 'https://api.opensubtitles.com/api/v1'; + +export interface OpenSubtitlesConfig { + apiKey: string; + username?: string; + password?: string; + userAgent: string; +} + +export interface SubtitleFile { + fileId: number; + fileName?: string; +} + +export interface SubtitleResult { + /** ISO 639-1 (or region-tagged) language code as OpenSubtitles reports it. */ + language: string; + moviehashMatch: boolean; + /** Transient popularity signal — used only to rank candidates, never stored. */ + downloadCount: number; + files: SubtitleFile[]; + release?: string; +} + +export interface SearchParams { + imdbId?: string; // numeric, no "tt" prefix + tmdbId?: string; + query?: string; + year?: string; + seasonNumber?: string; + episodeNumber?: string; + moviehash?: string; + /** CSV of 2-letter codes, e.g. "en,fr". */ + languages: string; +} + +export interface DownloadResult { + link: string; + fileName?: string; + /** Downloads remaining in the current quota window (per OpenSubtitles). */ + remaining: number; +} + +/** + * Compute the OpenSubtitles "moviehash" — a 64-bit hash of the file size plus + * the 64-bit little-endian words across the first and last 64 KiB. Returned + * as a zero-padded 16-char lowercase hex string. Caller supplies the size and + * the two 64 KiB chunks (read over WebDAV Range requests). + * + * Returns null if the file is smaller than 128 KiB (the algorithm is undefined + * there and OpenSubtitles ignores undersized hashes anyway). + */ +export function computeMovieHash(size: number, head: Buffer, tail: Buffer): string | null { + const CHUNK = 65536; + if (size < CHUNK * 2 || head.length < CHUNK || tail.length < CHUNK) return null; + + const mask = (1n << 64n) - 1n; + let hash = BigInt(size) & mask; + + const addWords = (buf: Buffer) => { + for (let i = 0; i + 8 <= buf.length; i += 8) { + hash = (hash + buf.readBigUInt64LE(i)) & mask; + } + }; + addWords(head.subarray(0, CHUNK)); + addWords(tail.subarray(tail.length - CHUNK)); + + return hash.toString(16).padStart(16, '0'); +} + +export class OpenSubtitlesClient { + private http: AxiosInstance; + private token: string | null = null; + + constructor(private cfg: OpenSubtitlesConfig) { + this.http = axios.create({ + baseURL: DEFAULT_BASE_URL, + timeout: 30000, + headers: { + 'Api-Key': cfg.apiKey, + 'User-Agent': cfg.userAgent, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + }); + } + + /** Lazily log in and cache the JWT. No-op if already authenticated. */ + private async ensureToken(): Promise { + if (this.token) return this.token; + if (!this.cfg.username || !this.cfg.password) { + throw new Error('OpenSubtitles username/password required for downloads'); + } + const resp = await this.http.post('/login', { + username: this.cfg.username, + password: this.cfg.password, + }); + const token = resp.data?.token as string | undefined; + if (!token) throw new Error('OpenSubtitles login returned no token'); + this.token = token; + return token; + } + + /** + * Search for subtitles. Best-effort: returns [] on any API error so a + * single bad lookup never fails the whole pipeline task. + */ + async search(params: SearchParams): Promise { + const query: Record = { languages: params.languages }; + if (params.imdbId) query.imdb_id = params.imdbId; + if (params.tmdbId) query.tmdb_id = params.tmdbId; + if (params.moviehash) query.moviehash = params.moviehash; + if (params.query) query.query = params.query; + if (params.year) query.year = params.year; + if (params.seasonNumber) query.season_number = params.seasonNumber; + if (params.episodeNumber) query.episode_number = params.episodeNumber; + + try { + const resp = await this.http.get('/subtitles', { params: query }); + const data = (resp.data?.data ?? []) as any[]; + return data.map((item) => { + const attr = item.attributes ?? {}; + return { + language: attr.language ?? '', + moviehashMatch: attr.moviehash_match === true, + downloadCount: Number(attr.download_count ?? 0), + release: attr.release, + files: (attr.files ?? []) + .map((f: any) => ({ fileId: Number(f.file_id), fileName: f.file_name })) + .filter((f: SubtitleFile) => Number.isFinite(f.fileId) && f.fileId > 0), + } as SubtitleResult; + }).filter((r) => r.files.length > 0); + } catch (err) { + console.warn(`[opensubtitles] search failed: ${describeError(err)}`); + return []; + } + } + + /** + * Request a download link for a subtitle file. Retries once on 401/403 by + * dropping the cached token and re-authenticating. Throws on quota + * exhaustion (406) so the caller can stop early. + */ + async requestDownload(fileId: number): Promise { + const doRequest = async (token: string) => { + const resp = await this.http.post( + '/download', + { file_id: fileId }, + { headers: { Authorization: `Bearer ${token}` } }, + ); + return resp.data; + }; + + let token = await this.ensureToken(); + let data: any; + try { + data = await doRequest(token); + } catch (err) { + if (isAxiosError(err) && (err.response?.status === 401 || err.response?.status === 403)) { + // Stale token — re-auth once. + this.token = null; + token = await this.ensureToken(); + data = await doRequest(token); + } else if (isAxiosError(err) && err.response?.status === 406) { + throw new QuotaExceededError(describeError(err)); + } else { + throw err; + } + } + + if (!data?.link) throw new Error('OpenSubtitles download returned no link'); + return { + link: data.link as string, + fileName: data.file_name as string | undefined, + remaining: Number(data.remaining ?? -1), + }; + } + + /** Fetch the actual subtitle bytes from a (short-lived) download link. */ + async fetchSubtitleBytes(link: string): Promise { + const resp = await axios.get(link, { + responseType: 'arraybuffer', + timeout: 30000, + headers: { 'User-Agent': this.cfg.userAgent }, + }); + return Buffer.from(resp.data); + } +} + +export class QuotaExceededError extends Error { + constructor(message: string) { + super(message); + this.name = 'QuotaExceededError'; + } +} + +function describeError(err: unknown): string { + if (isAxiosError(err)) { + const status = err.response?.status; + const body = err.response?.data; + const detail = typeof body === 'object' && body + ? (body as any).message ?? JSON.stringify(body) + : String(body ?? ''); + return `HTTP ${status ?? '?'}${detail ? ` — ${detail}` : ''}`; + } + return err instanceof Error ? err.message : String(err); +} diff --git a/src/plugin.test.ts b/src/plugin.test.ts new file mode 100644 index 0000000..49a84a4 --- /dev/null +++ b/src/plugin.test.ts @@ -0,0 +1,63 @@ +/** + * Unit tests for the pure helpers, ported from the gateway's torznab + * subtitles.rs tests (lang mapping, region-stripping) plus the moviehash. + * Run with `pnpm test` (uses Node's built-in test runner — no extra deps). + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { toLang3, parseWantedLanguages } from './langs.js'; +import { computeMovieHash } from './opensubtitles.js'; + +test('toLang3 maps ISO 639-1 to 639-3', () => { + assert.equal(toLang3('en'), 'eng'); + assert.equal(toLang3('fr'), 'fra'); + assert.equal(toLang3('pt'), 'por'); +}); + +test('toLang3 strips region suffixes', () => { + assert.equal(toLang3('pt-BR'), 'por'); + assert.equal(toLang3('zh-CN'), 'zho'); + assert.equal(toLang3('en_US'), 'eng'); +}); + +test('toLang3 canonicalizes 639-2/B bibliographic codes', () => { + assert.equal(toLang3('fre'), 'fra'); + assert.equal(toLang3('ger'), 'deu'); + assert.equal(toLang3('chi'), 'zho'); + assert.equal(toLang3('fra'), 'fra'); // already canonical +}); + +test('toLang3 falls back to und', () => { + assert.equal(toLang3('xx'), 'und'); + assert.equal(toLang3(undefined), 'und'); + assert.equal(toLang3(''), 'und'); +}); + +test('parseWantedLanguages dedupes and lowercases, defaults to en', () => { + assert.deepEqual(parseWantedLanguages('en,fr'), ['en', 'fr']); + assert.deepEqual(parseWantedLanguages(' EN , Fr , en '), ['en', 'fr']); + assert.deepEqual(parseWantedLanguages(undefined), ['en']); + assert.deepEqual(parseWantedLanguages(''), ['en']); +}); + +test('computeMovieHash returns null for files smaller than 128 KiB', () => { + const small = Buffer.alloc(1000); + assert.equal(computeMovieHash(1000, small, small), null); +}); + +test('computeMovieHash sums size + 64KiB head/tail words (zero chunks => size)', () => { + // All-zero head/tail → hash == size, as 16-char zero-padded lowercase hex. + const zeros = Buffer.alloc(65536); + const size = 0x20000; // 131072 = 2 * 64 KiB + assert.equal(computeMovieHash(size, zeros, zeros), '0000000000020000'); +}); + +test('computeMovieHash folds in non-zero words', () => { + const head = Buffer.alloc(65536); + head.writeBigUInt64LE(1n, 0); // one word = 1 + const tail = Buffer.alloc(65536); + const size = 0x20000; + // hash = size + 1 + assert.equal(computeMovieHash(size, head, tail), '0000000000020001'); +}); diff --git a/src/plugin.ts b/src/plugin.ts new file mode 100644 index 0000000..b9e73d5 --- /dev/null +++ b/src/plugin.ts @@ -0,0 +1,470 @@ +/** + * OpenSubtitles Plugin + * + * Fetches subtitle sidecars from OpenSubtitles (api.opensubtitles.com) for a + * video and registers them as first-class subtitle records, the same way + * `subtitle-extractor` registers subs pulled out of embedded streams — except + * the bytes come from the network instead of ffmpeg. + * + * ============================================================================ + * PLUGIN FILE ACCESS ARCHITECTURE - WebDAV + * ============================================================================ + * + * File access via WebDAV (WEBDAV_URL environment variable): + * - Read media files (for moviehash): GET (Range) /webdav/watch/... + * - Write output: PUT /webdav/plugin/opensubtitles/... + * - Cache: Local /cache mount (unused for now) + * + * Output subtitle files become their own records under their midhash256 CID; + * we only stamp the CID onto the video's `subtitles` set — no path is stored + * (the watcher indexes /files/plugin/opensubtitles/ as a first-class root). + * + * ============================================================================ + * METADATA (see METADATA_KEYS.md §8/§9) + * ============================================================================ + * Fetched subs are external/sidecar provenance, which §8 maps to `subtitles` + * (NOT `extractedSubtitles`, which is embedded-stream-only). We write the + * language-nested **key-set** target shape (matching the gateway's torznab + * enricher, which this plugin supersedes): + * - companion record on the subtitle CID: fileType=subtitle, subtitleLanguage, + * videos/=true (written explicitly — no watcher dependency) + * - on the video: subtitles//=true, subtitleLanguages/=true, + * languages/=true (all via PATCH/merge, path-safe like tmdb plot/eng) + */ + +import { createHash } from 'crypto'; +import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; +import { MetaCoreClient } from './meta-core-client.js'; +import { WebDAVClient } from './webdav-client.js'; +import { + OpenSubtitlesClient, + QuotaExceededError, + computeMovieHash, + type SearchParams, + type SubtitleResult, +} from './opensubtitles.js'; +import { toLang3, parseWantedLanguages } from './langs.js'; + +const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/opensubtitles'; +const HASH_CHUNK = 65536; // 64 KiB head/tail for the OpenSubtitles moviehash + +/** + * Resolve the WebDAV target from the meta-core URL handed to us in the request, + * rather than a static WEBDAV_URL env. This keeps the subtitle BYTES in the same + * meta-core whose METADATA we update — critical when the gateway drives this + * plugin against its own dedicated meta-core (otherwise we'd split-brain: bytes + * to meta-sort's volume, metadata to the gateway's). Falls back to the + * WEBDAV_URL env when the urls endpoint is unreachable. Cached per meta-core. + */ +const webdavCache = new Map(); +async function resolveWebdav(metaCoreUrl: string): Promise { + const cached = webdavCache.get(metaCoreUrl); + if (cached !== undefined) return cached; + + let client: WebDAVClient | null = null; + const base = metaCoreUrl.replace(/\/$/, ''); + for (const path of ['/urls', '/api/urls']) { + try { + const resp = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(5000) }); + if (!resp.ok) continue; + const data = (await resp.json()) as { webdavUrlInternal?: string; webdavUrl?: string }; + const url = data.webdavUrlInternal || data.webdavUrl; + if (url) { + client = new WebDAVClient(url); + console.log(`[opensubtitles] WebDAV resolved from ${base}${path}: ${url}`); + break; + } + } catch { + /* try next path / fall back to env */ + } + } + if (!client) { + // NB: this module exports a function named `process`, which shadows the + // Node global at module scope — reach the real one via globalThis. + const env = globalThis.process.env.WEBDAV_URL; + if (env) { + client = new WebDAVClient(env); + console.log(`[opensubtitles] WebDAV from WEBDAV_URL env: ${env}`); + } else { + console.warn(`[opensubtitles] WARNING: could not resolve WebDAV from ${base} and WEBDAV_URL unset`); + } + } + webdavCache.set(metaCoreUrl, client); + return client; +} + +export const manifest: PluginManifest = { + id: 'opensubtitles', + name: 'OpenSubtitles Fetcher', + version: '1.0.0', + description: 'Fetches subtitle sidecars from OpenSubtitles and links them to videos', + author: 'MetaMesh', + dependencies: ['file-info', 'filename-parser', 'tmdb'], + priority: 55, // after tmdb (30) so imdbid/tmdbid are populated + color: '#1B9335', + defaultQueue: 'background', // network-bound, quota-limited + timeout: 120000, + schema: { + subtitles: { label: 'Subtitles', type: 'array', readonly: true }, + subtitleLanguages: { label: 'Subtitle Languages', type: 'array', readonly: true }, + }, + config: { + apiKey: { + type: 'string', + label: 'OpenSubtitles API Key', + required: true, + secret: true, + }, + username: { + type: 'string', + label: 'OpenSubtitles Username (required for downloads)', + }, + password: { + type: 'string', + label: 'OpenSubtitles Password', + secret: true, + }, + languages: { + type: 'string', + label: 'Wanted Languages (CSV of ISO 639-1 codes, e.g. "en,fr")', + default: 'en', + }, + userAgent: { + type: 'string', + label: 'User-Agent (registered with OpenSubtitles)', + default: 'MetaMesh-OpenSubtitles-Plugin/1.0', + }, + forceRecompute: { + type: 'boolean', + label: 'Force Recompute', + default: false, + }, + }, +}; + +let apiKey: string | null = null; +let username: string | undefined; +let password: string | undefined; +let wantedLanguages: string[] = ['en']; +let userAgent = 'MetaMesh-OpenSubtitles-Plugin/1.0'; +let forceRecompute = false; + +export function configure(config: Record): void { + console.log(`[opensubtitles] Received config keys: ${Object.keys(config).join(', ')}`); + apiKey = (config.apiKey as string) || null; + username = (config.username as string) || undefined; + password = (config.password as string) || undefined; + wantedLanguages = parseWantedLanguages(config.languages as string | undefined); + userAgent = (config.userAgent as string) || 'MetaMesh-OpenSubtitles-Plugin/1.0'; + forceRecompute = config.forceRecompute === true; + console.log( + `[opensubtitles] Configured: apiKey=${apiKey ? 'set' : 'missing'}, ` + + `login=${username ? 'set' : 'missing'}, languages=${wantedLanguages.join(',')}`, + ); +} + +/** + * Compute the midhash256 CID of a buffer (matches the meta-hash algorithm; + * identical to subtitle-extractor's helper). + */ +function computeMidHash256FromBuffer(data: Buffer): string { + const SAMPLE_SIZE = 1024 * 1024; + const MIDHASH_VARINT = Buffer.from([0x80, 0x20]); + const fileSize = data.length; + + const sizeBuffer = Buffer.allocUnsafe(8); + sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); + + let sampleData: Buffer; + if (fileSize <= SAMPLE_SIZE) { + sampleData = data; + } else { + const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); + sampleData = data.subarray(middleOffset, middleOffset + SAMPLE_SIZE); + } + + const hashInput = Buffer.concat([sizeBuffer, sampleData]); + const hashBuffer = createHash('sha256').update(hashInput).digest(); + + const cidBytes = Buffer.concat([ + Buffer.from([0x01]), + MIDHASH_VARINT, + MIDHASH_VARINT, + Buffer.from([0x20]), + hashBuffer, + ]); + + const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; + let cid = 'b'; + let bits = 0; + let value = 0; + for (const byte of cidBytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + cid += base32Chars[(value >> bits) & 0x1f]; + } + } + if (bits > 0) { + cid += base32Chars[(value << (5 - bits)) & 0x1f]; + } + return cid; +} + +function sanitizeFilename(name: string): string { + return name.replace(/[<>:"/\\|?*]/g, '').replace(/\s+/g, ' ').trim(); +} + +/** Derive a subtitle file extension from a remote filename, defaulting to srt. */ +function subtitleExt(fileName: string | undefined): string { + if (!fileName) return 'srt'; + const m = fileName.toLowerCase().match(/\.(srt|ass|ssa|vtt|sub|smi)$/); + return m ? m[1] : 'srt'; +} + +/** Best-effort OpenSubtitles moviehash for a video, read over WebDAV. */ +async function computeVideoMovieHash(client: WebDAVClient, filePath: string): Promise { + try { + const stats = await client.stat(filePath); + const size = stats.size; + if (!size || size < HASH_CHUNK * 2) return null; + const head = await client.readBytes(filePath, 0, HASH_CHUNK - 1); + const tail = await client.readBytes(filePath, size - HASH_CHUNK, size - 1); + return computeMovieHash(size, head, tail); + } catch (e) { + console.warn(`[opensubtitles] moviehash unavailable: ${e instanceof Error ? e.message : String(e)}`); + return null; + } +} + +/** + * Negative cache of search misses, keyed by the search signature. Ported from + * the gateway's redb `(tmdb_id, lang)` negative cache (opensubtitles.rs §quota): + * a title OpenSubtitles has no subs for must never re-burn the (rate-limited) + * search budget on a rescan. In-memory per process — good enough for the + * plugin's lifetime; cleared on restart. + */ +const searchMissCache = new Set(); + +/** True once OpenSubtitles reports its download quota spent — short-circuits the + * rest of this process's downloads until restart (mirrors the gateway's + * "skip OS for the rest of this run" hard stop). */ +let downloadQuotaSpent = false; + +/** Rank: prefer an exact moviehash match, then higher (transient) popularity. */ +function rankResults(a: SubtitleResult, b: SubtitleResult): number { + if (a.moviehashMatch !== b.moviehashMatch) return a.moviehashMatch ? -1 : 1; + return b.downloadCount - a.downloadCount; +} + +export async function process( + request: ProcessRequest, + sendCallback: (payload: CallbackPayload) => Promise, +): Promise { + const startTime = Date.now(); + const metaCore = new MetaCoreClient(request.metaCoreUrl); + const done = (status: CallbackPayload['status'], extra: Partial = {}) => + sendCallback({ taskId: request.taskId, status, duration: Date.now() - startTime, ...extra }); + + try { + const { cid, filePath, existingMeta } = request; + + if (!apiKey) return void (await done('skipped', { reason: 'No API key configured' })); + if (existingMeta?.fileType !== 'video') return void (await done('skipped', { reason: 'Not a video file' })); + if (downloadQuotaSpent && !forceRecompute) { + return void (await done('skipped', { reason: 'OpenSubtitles download quota spent for this run' })); + } + + // Resolve WebDAV from the meta-core we were handed (not a static env) so + // bytes and metadata land in the same store. Required for gateway use. + const webdavClient = await resolveWebdav(request.metaCoreUrl); + if (!webdavClient) return void (await done('failed', { error: 'Could not resolve a WebDAV target' })); + + // Which lang3 codes do we still need? Read both the legacy csv-set + // (`subtitleLanguages`) and the key-set (`subtitleLanguages/`). + const wantedLang3 = new Map(); // lang3 -> original 2-letter code + for (const code of wantedLanguages) wantedLang3.set(toLang3(code), code); + + const haveLang3 = new Set(); + for (const l of (existingMeta?.subtitleLanguages ?? '').split(',').map((s) => s.trim()).filter(Boolean)) { + haveLang3.add(l); + } + for (const key of Object.keys(existingMeta ?? {})) { + if (key.startsWith('subtitleLanguages/')) { + const l = key.slice('subtitleLanguages/'.length); + if (l) haveLang3.add(l); + } + } + if (!forceRecompute) { + for (const lang3 of [...wantedLang3.keys()]) { + if (haveLang3.has(lang3)) wantedLang3.delete(lang3); + } + } + if (wantedLang3.size === 0) { + return void (await done('skipped', { reason: 'Subtitles already present for all wanted languages' })); + } + + // Build the search. Prefer structured IDs + moviehash for accuracy. + const moviehash = await computeVideoMovieHash(webdavClient, filePath); + const imdbRaw = existingMeta?.imdbid; + const imdbId = imdbRaw ? imdbRaw.replace(/^tt/i, '').replace(/^0+/, '') : undefined; + const tmdbId = existingMeta?.tmdbid; + const isEpisode = !!(existingMeta?.season || existingMeta?.episode) + || existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'; + + let title = existingMeta?.originalTitle || existingMeta?.title || existingMeta?.fileName; + const year = existingMeta?.movieYear; + if (title && year) { + title = title.replace(new RegExp(`\\s*[\\(\\[]?${year}[\\)\\]]?\\s*$`), '').trim(); + } + + const search: SearchParams = { languages: [...wantedLang3.values()].join(',') }; + if (moviehash) search.moviehash = moviehash; + if (imdbId) search.imdbId = imdbId; + if (tmdbId) search.tmdbId = tmdbId; + if (isEpisode && existingMeta?.season) search.seasonNumber = existingMeta.season; + if (isEpisode && existingMeta?.episode) search.episodeNumber = existingMeta.episode; + // Title query only as a fallback when we have no structured identifier. + if (!imdbId && !tmdbId && !moviehash && title) { + search.query = title; + if (year) search.year = year; + } + + // Negative cache: skip a search we already know returns nothing (no + // moviehash, since that's video-specific). Saves the search budget. + const missKey = [ + moviehash ? '' : (tmdbId ?? ''), + moviehash ? '' : (imdbId ?? ''), + moviehash ? '' : (search.query ?? ''), + search.seasonNumber ?? '', + search.episodeNumber ?? '', + [...wantedLang3.keys()].sort().join('+'), + ].join('|'); + if (!moviehash && !forceRecompute && searchMissCache.has(missKey)) { + return void (await done('skipped', { reason: 'Known OpenSubtitles miss (negative cache)' })); + } + + const client = new OpenSubtitlesClient({ apiKey, username, password, userAgent }); + const results = await client.search(search); + if (results.length === 0) { + if (!moviehash) searchMissCache.add(missKey); + return void (await done('completed', { reason: 'No OpenSubtitles match' })); + } + + // Pick the single best subtitle per still-wanted language. + const bestByLang3 = new Map(); + for (const r of results) { + const lang3 = toLang3(r.language); + if (!wantedLang3.has(lang3)) continue; + const existing = bestByLang3.get(lang3); + if (!existing || rankResults(r, existing) < 0) bestByLang3.set(lang3, r); + } + if (bestByLang3.size === 0) { + if (!moviehash) searchMissCache.add(missKey); + return void (await done('completed', { reason: 'No results in wanted languages' })); + } + + const displayTitle = existingMeta?.title || existingMeta?.originalTitle || title || undefined; + const safeTitle = sanitizeFilename(title || existingMeta?.fileName || 'video'); + const yearStr = year ? ` (${year})` : ''; + + let fetched = 0; + let quotaHit = false; + + for (const [lang3, result] of bestByLang3) { + const file = result.files[0]; + if (!file) continue; + + try { + const download = await client.requestDownload(file.fileId); + const ext = subtitleExt(download.fileName || file.fileName); + const outputFilename = `${safeTitle}${yearStr}[${cid}]_subtitle.${lang3}.${ext}`; + const webdavOutputPath = `${PLUGIN_OUTPUT_WEBDAV_PATH}/${outputFilename}`; + const opts = { extension: ext, fileName: download.fileName || file.fileName, parentTitle: displayTitle }; + + // Reuse an already-downloaded sidecar unless forcing. + if (!forceRecompute && (await webdavClient.exists(webdavOutputPath))) { + const existingData = await webdavClient.readFile(webdavOutputPath); + await registerSubtitle(metaCore, cid, computeMidHash256FromBuffer(existingData), lang3, opts); + fetched++; + continue; + } + + const bytes = await client.fetchSubtitleBytes(download.link); + if (bytes.length < 10) { + console.warn(`[opensubtitles] empty subtitle for ${lang3}, skipping`); + continue; + } + const subtitleCid = computeMidHash256FromBuffer(bytes); + await webdavClient.writeFile(webdavOutputPath, bytes); + await registerSubtitle(metaCore, cid, subtitleCid, lang3, opts); + fetched++; + console.log(`[opensubtitles] fetched ${lang3} subtitle -> ${subtitleCid} (${webdavOutputPath})`); + + if (download.remaining === 0) { + console.warn('[opensubtitles] download quota exhausted, stopping'); + downloadQuotaSpent = true; + quotaHit = true; + break; + } + } catch (e) { + if (e instanceof QuotaExceededError) { + console.warn(`[opensubtitles] quota exceeded: ${e.message}`); + downloadQuotaSpent = true; + quotaHit = true; + break; + } + console.error(`[opensubtitles] failed to fetch ${lang3}: ${e instanceof Error ? e.message : String(e)}`); + } + } + + await done('completed', { + reason: quotaHit + ? `Fetched ${fetched} subtitle(s); stopped on quota` + : `Fetched ${fetched} subtitle(s)`, + }); + } catch (error) { + await done('failed', { error: error instanceof Error ? error.message : String(error) }); + } +} + +/** Companion metadata for a fetched subtitle's own record. */ +interface SubtitleOpts { + extension: string; + fileName?: string; + parentTitle?: string; +} + +/** + * Register a fetched subtitle. Mirrors the gateway's torznab subtitle enricher + * (subtitles.rs): writes the subtitle's own companion record (keyed by its CID) + * AND stamps the language-nested key-set onto the video. See METADATA_KEYS.md + * §8/§9. We write the companion record explicitly (not via the watcher) so this + * works when the gateway drives the plugin against a meta-core with no watcher. + */ +async function registerSubtitle( + metaCore: MetaCoreClient, + videoCid: string, + subtitleCid: string, + lang3: string, + opts: SubtitleOpts, +): Promise { + // 1. Companion subtitle-file record (fileType=subtitle, lang, video backref). + const companion: Record = { + fileType: 'subtitle', + extension: opts.extension, + subtitleLanguage: lang3, + [`videos/${videoCid}`]: 'true', + }; + if (opts.fileName) companion.fileName = opts.fileName; + if (opts.parentTitle) companion.title = `${opts.parentTitle} (${lang3} subtitles)`; + await metaCore.mergeMetadata(subtitleCid, companion); + + // 2. Language-nested key-set on the video (target shape, METADATA_KEYS.md §8). + await metaCore.mergeMetadata(videoCid, { + [`subtitles/${lang3}/${subtitleCid}`]: 'true', + [`subtitleLanguages/${lang3}`]: 'true', + [`languages/${lang3}`]: 'true', + }); +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..4dbbfe2 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,70 @@ +/** + * MetaMesh Plugin Types + */ + +export interface PluginManifest { + id: string; + name: string; + version: string; + description?: string; + author?: string; + dependencies?: string[]; + schema?: Record; + config?: Record; + priority?: number; + color?: string; + defaultQueue?: 'fast' | 'background'; + timeout?: number; +} + +export interface SchemaField { + label: string; + type?: 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid'; + readonly?: boolean; + hint?: string; +} + +export interface ConfigField { + type: 'string' | 'number' | 'boolean' | 'select'; + label?: string; + required?: boolean; + default?: unknown; + secret?: boolean; +} + +export interface HealthResponse { + status: 'healthy' | 'unhealthy'; + ready: boolean; + version: string; +} + +export interface ProcessRequest { + taskId: string; + cid: string; + filePath: string; + callbackUrl: string; + metaCoreUrl: string; + existingMeta?: Record; +} + +export interface ProcessResponse { + status: 'accepted' | 'rejected'; + error?: string; +} + +export interface CallbackPayload { + taskId: string; + status: 'completed' | 'failed' | 'skipped'; + duration: number; + error?: string; + reason?: string; +} + +export interface ConfigureRequest { + config: Record; +} + +export interface ConfigureResponse { + status: 'ok' | 'error'; + error?: string; +} diff --git a/src/webdav-client.ts b/src/webdav-client.ts new file mode 100644 index 0000000..4b859d7 --- /dev/null +++ b/src/webdav-client.ts @@ -0,0 +1,150 @@ +/** + * WebDAV Client for Plugin File Access + * + * Provides HTTP-based file access to meta-sort's WebDAV endpoint. + * Replaces direct filesystem access for containerized plugins. + */ + +export interface FileStats { + size: number; + mtime?: Date; +} + +export class WebDAVClient { + private baseUrl: string; + + constructor(baseUrl: string) { + // Remove trailing slash if present + this.baseUrl = baseUrl.replace(/\/$/, ''); + } + + /** + * Convert absolute file path to WebDAV URL + * Strips /files prefix and appends to base URL + */ + toWebDAVUrl(filePath: string): string { + // filePath is like "/files/watch/movie.mkv" + // We need to convert to "http://meta-sort-dev/webdav/watch/movie.mkv" + let relativePath = filePath; + + // Strip /files prefix if present + if (relativePath.startsWith('/files/')) { + relativePath = relativePath.substring(6); // Remove "/files" + } else if (relativePath.startsWith('/files')) { + relativePath = relativePath.substring(6); + } + + // Ensure path starts with / + if (!relativePath.startsWith('/')) { + relativePath = '/' + relativePath; + } + + return this.baseUrl + relativePath; + } + + /** + * Get file stats (size, mtime) via HTTP HEAD request + */ + async stat(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'HEAD' }); + + if (!response.ok) { + throw new Error(`WebDAV HEAD failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const contentLength = response.headers.get('content-length'); + const lastModified = response.headers.get('last-modified'); + + return { + size: contentLength ? parseInt(contentLength, 10) : 0, + mtime: lastModified ? new Date(lastModified) : undefined, + }; + } + + /** + * Read first N bytes of a file (for magic byte detection) + * Uses HTTP Range request + */ + async readBytes(filePath: string, start: number, end: number): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { + method: 'GET', + headers: { + 'Range': `bytes=${start}-${end}`, + }, + }); + + if (!response.ok && response.status !== 206) { + throw new Error(`WebDAV Range GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Read entire file as buffer (for small files only) + */ + async readFile(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'GET' }); + + if (!response.ok) { + throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Write file via HTTP PUT + */ + async writeFile(filePath: string, data: Buffer | string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { + method: 'PUT', + body: data, + headers: { + 'Content-Type': 'application/octet-stream', + }, + }); + + if (!response.ok) { + throw new Error(`WebDAV PUT failed for ${filePath}: ${response.status} ${response.statusText}`); + } + } + + /** + * Check if file exists via HTTP HEAD + */ + async exists(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + try { + const response = await fetch(url, { method: 'HEAD' }); + return response.ok; + } catch { + return false; + } + } +} + +/** + * Create a WebDAV client from environment variables + */ +export function createWebDAVClient(): WebDAVClient | null { + const webdavUrl = process.env.WEBDAV_URL; + + if (!webdavUrl) { + console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable'); + return null; + } + + return new WebDAVClient(webdavUrl); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..e60556a --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] +} diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..52decf6 --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-test", + "sourceMap": false, + "declaration": false + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "dist-test"] +}