fix webdav dependencies
Build and Push Docker Image / build (push) Failing after 14s

This commit is contained in:
Worph
2026-07-12 19:20:15 +02:00
parent 0d57dbeee3
commit bee5734251
2 changed files with 79 additions and 79 deletions
+23 -72
View File
@@ -9,7 +9,8 @@
* PLUGIN FILE ACCESS ARCHITECTURE * PLUGIN FILE ACCESS ARCHITECTURE
* ============================================================================ * ============================================================================
* *
* File access via WebDAV (WEBDAV_URL environment variable): * File access via WebDAV, on the meta-core named by each /process request
* (its /urls -> webdavUrlInternal; WEBDAV_URL overrides):
* - Read media files: GET /webdav/watch/... or /webdav/test/... * - Read media files: GET /webdav/watch/... or /webdav/test/...
* - Write output: PUT /webdav/plugin/tmdb/... * - Write output: PUT /webdav/plugin/tmdb/...
* - Cache: Local /cache mount (for JSON cache files) * - Cache: Local /cache mount (for JSON cache files)
@@ -18,6 +19,7 @@
* - No output mount needed on plugin containers * - No output mount needed on plugin containers
* - Consistent read/write architecture via HTTP * - Consistent read/write architecture via HTTP
* - Works in any orchestration environment * - Works in any orchestration environment
* - Posters land in the same core the CIDs are written to, always
* *
* ============================================================================ * ============================================================================
* *
@@ -30,21 +32,12 @@
*/ */
import axios from 'axios'; import axios from 'axios';
import { statSync, openSync, readSync, closeSync } from 'fs';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import * as path from 'path'; import * as path from 'path';
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
import { MetaCoreClient } from './meta-core-client.js'; import { MetaCoreClient } from './meta-core-client.js';
import { readJson, writeJson } from './cache.js'; import { readJson, writeJson } from './cache.js';
import { createWebDAVClient, WebDAVClient } from './webdav-client.js'; import { getWebDAVClient, WebDAVClient } from './webdav-client.js';
// Initialize WebDAV client - required for plugin operation
const webdavClient = createWebDAVClient();
if (webdavClient) {
console.log('[tmdb] WebDAV client initialized for file access');
} else {
console.warn('[tmdb] WARNING: WEBDAV_URL not set - plugin will not be able to write output files');
}
const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original'; const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
@@ -52,54 +45,14 @@ const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
* ============================================================================ * ============================================================================
* PLUGIN OUTPUT PATH - Written via WebDAV * PLUGIN OUTPUT PATH - Written via WebDAV
* ============================================================================ * ============================================================================
* Output files (posters, backdrops) are written via WebDAV PUT requests. * Output files (posters, backdrops) are written via WebDAV PUT requests, to the
* Path: /files/plugin/tmdb/<filename> (accessible at WEBDAV_URL/plugin/tmdb/) * WebDAV of the meta-core that issued the /process call.
* Path: /files/plugin/tmdb/<filename>
* CACHE_PATH (/cache) - Local mount for plugin cache (handled by cache.ts) * CACHE_PATH (/cache) - Local mount for plugin cache (handled by cache.ts)
* ============================================================================ * ============================================================================
*/ */
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/tmdb'; const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/tmdb';
/**
* Compute midhash256 CID for a file (matches meta-hash algorithm)
*
* Algorithm:
* - For files <= 1MB: Hashes entire file content + 8-byte size prefix
* - For files > 1MB: Hashes middle 1MB + 8-byte size prefix
* - Returns CIDv1 with custom codec 0x1000
*/
function computeMidHash256Sync(filePath: string): string {
const SAMPLE_SIZE = 1024 * 1024; // 1MB
// varint encoding of 0x1000 (4096) for both codec and hash function code
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
// Get file size
const stats = statSync(filePath);
const fileSize = stats.size;
// Create size buffer (64-bit big-endian)
const sizeBuffer = Buffer.allocUnsafe(8);
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
// Extract sample data
let sampleData: Buffer;
if (fileSize <= SAMPLE_SIZE) {
// Small file: read entire content
const fd = openSync(filePath, 'r');
sampleData = Buffer.allocUnsafe(fileSize);
readSync(fd, sampleData, 0, fileSize, 0);
closeSync(fd);
} else {
// Large file: read middle 1MB
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
const fd = openSync(filePath, 'r');
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
closeSync(fd);
}
return computeMidHash256FromData(fileSize, sampleData);
}
/** /**
* Compute midhash256 CID via WebDAV * Compute midhash256 CID via WebDAV
*/ */
@@ -170,16 +123,6 @@ function computeMidHash256FromData(fileSize: number, sampleData: Buffer): string
return cid; return cid;
} }
/**
* Compute midhash256 CID for a file (auto-selects WebDAV or filesystem)
*/
async function computeMidHash256(filePath: string): Promise<string> {
if (webdavClient) {
return computeMidHash256WebDAV(webdavClient, filePath);
}
return computeMidHash256Sync(filePath);
}
export const manifest: PluginManifest = { export const manifest: PluginManifest = {
id: 'tmdb', id: 'tmdb',
name: 'TMDB Metadata', name: 'TMDB Metadata',
@@ -352,6 +295,7 @@ async function downloadImageToWebDAV(
* no need to store any path in the parent file's metadata. * no need to store any path in the parent file's metadata.
*/ */
async function downloadAndHashImage( async function downloadAndHashImage(
webdavClient: WebDAVClient | null,
imagePath: string, imagePath: string,
imageType: string, imageType: string,
title: string, title: string,
@@ -409,6 +353,9 @@ export async function process(
): Promise<void> { ): Promise<void> {
const startTime = Date.now(); const startTime = Date.now();
const metaCore = new MetaCoreClient(request.metaCoreUrl); const metaCore = new MetaCoreClient(request.metaCoreUrl);
// WebDAV of the very core we're enriching, so poster blobs and the poster
// CIDs we write can never point at different cores. Cached per core URL.
const webdavClient = await getWebDAVClient(request.metaCoreUrl);
try { try {
const { cid, existingMeta } = request; const { cid, existingMeta } = request;
@@ -465,6 +412,7 @@ export async function process(
const isEntry = cached.data !== undefined; const isEntry = cached.data !== undefined;
await applyTmdbData( await applyTmdbData(
metaCore, metaCore,
webdavClient,
cid, cid,
isEntry ? cached.data : cached, isEntry ? cached.data : cached,
// Legacy raw payloads carry no image CIDs → let applyTmdbData recompute. // Legacy raw payloads carry no image CIDs → let applyTmdbData recompute.
@@ -490,7 +438,7 @@ export async function process(
existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv' existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'
? 'tv' ? 'tv'
: 'movie'; : 'movie';
entry = await resolveEntryById(knownTmdbId, mediaType); entry = await resolveEntryById(webdavClient, knownTmdbId, mediaType);
} }
// Fallback: no usable id → resolve by IMDB id or fuzzy title, then cache // Fallback: no usable id → resolve by IMDB id or fuzzy title, then cache
@@ -521,7 +469,7 @@ export async function process(
} }
if (tmdbData) { if (tmdbData) {
const images = await computeImageCids(tmdbData); const images = await computeImageCids(webdavClient, tmdbData);
entry = { data: tmdbData, ...images }; entry = { data: tmdbData, ...images };
if (tmdbData.id) { if (tmdbData.id) {
await writeJson(tmdbCacheKey(String(tmdbData.id), mediaTypeOf(tmdbData)), entry); await writeJson(tmdbCacheKey(String(tmdbData.id), mediaTypeOf(tmdbData)), entry);
@@ -530,7 +478,7 @@ export async function process(
} }
if (entry) { if (entry) {
await applyTmdbData(metaCore, cid, entry.data, { await applyTmdbData(metaCore, webdavClient, cid, entry.data, {
posterCid: entry.posterCid, posterCid: entry.posterCid,
backdropCid: entry.backdropCid, backdropCid: entry.backdropCid,
}); });
@@ -594,6 +542,7 @@ function mediaTypeOf(data: any): string {
// (the image filename is keyed by tmdbId, so the blob dedups across records), so // (the image filename is keyed by tmdbId, so the blob dedups across records), so
// the CIDs can be computed once per show and reused for every release. // the CIDs can be computed once per show and reused for every release.
async function computeImageCids( async function computeImageCids(
webdavClient: WebDAVClient | null,
data: any data: any
): Promise<{ posterCid: string | null; backdropCid: string | null }> { ): Promise<{ posterCid: string | null; backdropCid: string | null }> {
const tmdbId = data?.id ? String(data.id) : ''; const tmdbId = data?.id ? String(data.id) : '';
@@ -603,11 +552,11 @@ async function computeImageCids(
const year = releaseDate ? releaseDate.split('-')[0] : undefined; const year = releaseDate ? releaseDate.split('-')[0] : undefined;
const posterCid = const posterCid =
data?.poster_path && tmdbId data?.poster_path && tmdbId
? await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId) ? await downloadAndHashImage(webdavClient, data.poster_path, 'poster', displayTitle, year, tmdbId)
: null; : null;
const backdropCid = const backdropCid =
data?.backdrop_path && tmdbId data?.backdrop_path && tmdbId
? await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId) ? await downloadAndHashImage(webdavClient, data.backdrop_path, 'backdrop', displayTitle, year, tmdbId)
: null; : null;
return { posterCid, backdropCid }; return { posterCid, backdropCid };
} }
@@ -617,6 +566,7 @@ async function computeImageCids(
// This is the hot path for gateway records (torrent + usenet), which always // This is the hot path for gateway records (torrent + usenet), which always
// carry a `tmdbid` anchor — so N releases of one show share one TMDB fetch. // carry a `tmdbid` anchor — so N releases of one show share one TMDB fetch.
async function resolveEntryById( async function resolveEntryById(
webdavClient: WebDAVClient | null,
tmdbId: string, tmdbId: string,
mediaType: string mediaType: string
): Promise<TmdbCacheEntry | null> { ): Promise<TmdbCacheEntry | null> {
@@ -637,7 +587,7 @@ async function resolveEntryById(
if (!data) { if (!data) {
return null; return null;
} }
const images = await computeImageCids(data); const images = await computeImageCids(webdavClient, data);
const entry: TmdbCacheEntry = { data, ...images }; const entry: TmdbCacheEntry = { data, ...images };
await writeJson(key, entry); await writeJson(key, entry);
return entry; return entry;
@@ -655,6 +605,7 @@ async function resolveEntryById(
*/ */
async function applyTmdbData( async function applyTmdbData(
metaCore: MetaCoreClient, metaCore: MetaCoreClient,
webdavClient: WebDAVClient | null,
cid: string, cid: string,
data: any, data: any,
images?: { posterCid: string | null; backdropCid: string | null } images?: { posterCid: string | null; backdropCid: string | null }
@@ -750,7 +701,7 @@ async function applyTmdbData(
// up as first-class file entries, so storing the CID is all that's needed. // up as first-class file entries, so storing the CID is all that's needed.
let posterCid = images ? images.posterCid : null; let posterCid = images ? images.posterCid : null;
if (!images && data.poster_path && tmdbId) { if (!images && data.poster_path && tmdbId) {
posterCid = await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId); posterCid = await downloadAndHashImage(webdavClient, data.poster_path, 'poster', displayTitle, year, tmdbId);
} }
if (posterCid) { if (posterCid) {
await metaCore.setProperty(cid, 'poster', posterCid); await metaCore.setProperty(cid, 'poster', posterCid);
@@ -759,7 +710,7 @@ async function applyTmdbData(
let backdropCid = images ? images.backdropCid : null; let backdropCid = images ? images.backdropCid : null;
if (!images && data.backdrop_path && tmdbId) { if (!images && data.backdrop_path && tmdbId) {
backdropCid = await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId); backdropCid = await downloadAndHashImage(webdavClient, data.backdrop_path, 'backdrop', displayTitle, year, tmdbId);
} }
if (backdropCid) { if (backdropCid) {
await metaCore.setProperty(cid, 'backdrop', backdropCid); await metaCore.setProperty(cid, 'backdrop', backdropCid);
+54 -5
View File
@@ -136,15 +136,64 @@ export class WebDAVClient {
} }
/** /**
* Create a WebDAV client from environment variables * WebDAV client for the meta-core that drove a /process call.
*
* The endpoint is read from that core's own /urls (`webdavUrlInternal`),
* so blobs always land in the /files of the core we write metadata to. A feeder
* whose plugins run beside it but enrich a remote hub core needs no per-stack
* wiring, and it is impossible to write posters into one core while pointing
* CIDs at another.
*
* WEBDAV_URL still wins when set — an explicit override for tests and for
* topologies where the core's advertised internal URL isn't reachable from here.
*
* Resolutions are cached per meta-core URL; failures are not, so a core that is
* still starting up is retried on the next task.
*/ */
export function createWebDAVClient(): WebDAVClient | null { const clientCache = new Map<string, Promise<WebDAVClient | null>>();
const webdavUrl = process.env.WEBDAV_URL;
async function resolveWebDAVUrl(metaCoreUrl: string): Promise<string | null> {
// /urls, not /api/urls — metaCoreUrl points at the meta-core backend, whose
// discovery routes are unprefixed (same base the /meta/* writes use).
const base = metaCoreUrl.replace(/\/$/, '');
const response = await fetch(`${base}/urls`, { signal: AbortSignal.timeout(5000) });
if (!response.ok) {
throw new Error(`GET ${base}/urls failed: ${response.status} ${response.statusText}`);
}
const urls = await response.json() as { webdavUrlInternal?: string; webdavUrl?: string };
return urls.webdavUrlInternal ?? urls.webdavUrl ?? null;
}
export async function getWebDAVClient(metaCoreUrl: string): Promise<WebDAVClient | null> {
const override = process.env.WEBDAV_URL;
if (override) {
return new WebDAVClient(override);
}
let pending = clientCache.get(metaCoreUrl);
if (!pending) {
pending = resolveWebDAVUrl(metaCoreUrl).then((webdavUrl) => {
if (!webdavUrl) { if (!webdavUrl) {
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable'); console.warn(`[webdav-client] meta-core ${metaCoreUrl} advertises no WebDAV URL`);
return null; return null;
} }
console.log(`[webdav-client] Using WebDAV endpoint ${webdavUrl} (from ${metaCoreUrl})`);
return new WebDAVClient(webdavUrl); return new WebDAVClient(webdavUrl);
});
clientCache.set(metaCoreUrl, pending);
}
try {
const client = await pending;
if (!client) {
clientCache.delete(metaCoreUrl);
}
return client;
} catch (error) {
clientCache.delete(metaCoreUrl);
console.warn(`[webdav-client] Failed to resolve WebDAV URL from ${metaCoreUrl}: ${error instanceof Error ? error.message : String(error)}`);
return null;
}
} }