fix webdav dependencies
This commit is contained in:
+23
-72
@@ -9,7 +9,8 @@
|
|||||||
* PLUGIN FILE ACCESS ARCHITECTURE
|
* PLUGIN FILE ACCESS ARCHITECTURE
|
||||||
* ============================================================================
|
* ============================================================================
|
||||||
*
|
*
|
||||||
* File access via WebDAV (WEBDAV_URL environment variable):
|
* File access via WebDAV, on the meta-core named by each /process request
|
||||||
|
* (its /urls -> webdavUrlInternal; WEBDAV_URL overrides):
|
||||||
* - Read media files: GET /webdav/watch/... or /webdav/test/...
|
* - Read media files: GET /webdav/watch/... or /webdav/test/...
|
||||||
* - Write output: PUT /webdav/plugin/tmdb/...
|
* - Write output: PUT /webdav/plugin/tmdb/...
|
||||||
* - Cache: Local /cache mount (for JSON cache files)
|
* - Cache: Local /cache mount (for JSON cache files)
|
||||||
@@ -18,6 +19,7 @@
|
|||||||
* - No output mount needed on plugin containers
|
* - No output mount needed on plugin containers
|
||||||
* - Consistent read/write architecture via HTTP
|
* - Consistent read/write architecture via HTTP
|
||||||
* - Works in any orchestration environment
|
* - Works in any orchestration environment
|
||||||
|
* - Posters land in the same core the CIDs are written to, always
|
||||||
*
|
*
|
||||||
* ============================================================================
|
* ============================================================================
|
||||||
*
|
*
|
||||||
@@ -30,21 +32,12 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
import { statSync, openSync, readSync, closeSync } from 'fs';
|
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||||
import { MetaCoreClient } from './meta-core-client.js';
|
import { MetaCoreClient } from './meta-core-client.js';
|
||||||
import { readJson, writeJson } from './cache.js';
|
import { readJson, writeJson } from './cache.js';
|
||||||
import { createWebDAVClient, WebDAVClient } from './webdav-client.js';
|
import { getWebDAVClient, WebDAVClient } from './webdav-client.js';
|
||||||
|
|
||||||
// Initialize WebDAV client - required for plugin operation
|
|
||||||
const webdavClient = createWebDAVClient();
|
|
||||||
if (webdavClient) {
|
|
||||||
console.log('[tmdb] WebDAV client initialized for file access');
|
|
||||||
} else {
|
|
||||||
console.warn('[tmdb] WARNING: WEBDAV_URL not set - plugin will not be able to write output files');
|
|
||||||
}
|
|
||||||
|
|
||||||
const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
|
const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
|
||||||
|
|
||||||
@@ -52,54 +45,14 @@ const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
|
|||||||
* ============================================================================
|
* ============================================================================
|
||||||
* PLUGIN OUTPUT PATH - Written via WebDAV
|
* PLUGIN OUTPUT PATH - Written via WebDAV
|
||||||
* ============================================================================
|
* ============================================================================
|
||||||
* Output files (posters, backdrops) are written via WebDAV PUT requests.
|
* Output files (posters, backdrops) are written via WebDAV PUT requests, to the
|
||||||
* Path: /files/plugin/tmdb/<filename> (accessible at WEBDAV_URL/plugin/tmdb/)
|
* WebDAV of the meta-core that issued the /process call.
|
||||||
|
* Path: /files/plugin/tmdb/<filename>
|
||||||
* CACHE_PATH (/cache) - Local mount for plugin cache (handled by cache.ts)
|
* CACHE_PATH (/cache) - Local mount for plugin cache (handled by cache.ts)
|
||||||
* ============================================================================
|
* ============================================================================
|
||||||
*/
|
*/
|
||||||
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/tmdb';
|
const PLUGIN_OUTPUT_WEBDAV_PATH = '/files/plugin/tmdb';
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute midhash256 CID for a file (matches meta-hash algorithm)
|
|
||||||
*
|
|
||||||
* Algorithm:
|
|
||||||
* - For files <= 1MB: Hashes entire file content + 8-byte size prefix
|
|
||||||
* - For files > 1MB: Hashes middle 1MB + 8-byte size prefix
|
|
||||||
* - Returns CIDv1 with custom codec 0x1000
|
|
||||||
*/
|
|
||||||
function computeMidHash256Sync(filePath: string): string {
|
|
||||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
|
||||||
// varint encoding of 0x1000 (4096) for both codec and hash function code
|
|
||||||
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
|
||||||
|
|
||||||
// Get file size
|
|
||||||
const stats = statSync(filePath);
|
|
||||||
const fileSize = stats.size;
|
|
||||||
|
|
||||||
// Create size buffer (64-bit big-endian)
|
|
||||||
const sizeBuffer = Buffer.allocUnsafe(8);
|
|
||||||
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
|
||||||
|
|
||||||
// Extract sample data
|
|
||||||
let sampleData: Buffer;
|
|
||||||
if (fileSize <= SAMPLE_SIZE) {
|
|
||||||
// Small file: read entire content
|
|
||||||
const fd = openSync(filePath, 'r');
|
|
||||||
sampleData = Buffer.allocUnsafe(fileSize);
|
|
||||||
readSync(fd, sampleData, 0, fileSize, 0);
|
|
||||||
closeSync(fd);
|
|
||||||
} else {
|
|
||||||
// Large file: read middle 1MB
|
|
||||||
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
|
||||||
const fd = openSync(filePath, 'r');
|
|
||||||
sampleData = Buffer.allocUnsafe(SAMPLE_SIZE);
|
|
||||||
readSync(fd, sampleData, 0, SAMPLE_SIZE, middleOffset);
|
|
||||||
closeSync(fd);
|
|
||||||
}
|
|
||||||
|
|
||||||
return computeMidHash256FromData(fileSize, sampleData);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute midhash256 CID via WebDAV
|
* Compute midhash256 CID via WebDAV
|
||||||
*/
|
*/
|
||||||
@@ -170,16 +123,6 @@ function computeMidHash256FromData(fileSize: number, sampleData: Buffer): string
|
|||||||
return cid;
|
return cid;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute midhash256 CID for a file (auto-selects WebDAV or filesystem)
|
|
||||||
*/
|
|
||||||
async function computeMidHash256(filePath: string): Promise<string> {
|
|
||||||
if (webdavClient) {
|
|
||||||
return computeMidHash256WebDAV(webdavClient, filePath);
|
|
||||||
}
|
|
||||||
return computeMidHash256Sync(filePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const manifest: PluginManifest = {
|
export const manifest: PluginManifest = {
|
||||||
id: 'tmdb',
|
id: 'tmdb',
|
||||||
name: 'TMDB Metadata',
|
name: 'TMDB Metadata',
|
||||||
@@ -352,6 +295,7 @@ async function downloadImageToWebDAV(
|
|||||||
* no need to store any path in the parent file's metadata.
|
* no need to store any path in the parent file's metadata.
|
||||||
*/
|
*/
|
||||||
async function downloadAndHashImage(
|
async function downloadAndHashImage(
|
||||||
|
webdavClient: WebDAVClient | null,
|
||||||
imagePath: string,
|
imagePath: string,
|
||||||
imageType: string,
|
imageType: string,
|
||||||
title: string,
|
title: string,
|
||||||
@@ -409,6 +353,9 @@ export async function process(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
const metaCore = new MetaCoreClient(request.metaCoreUrl);
|
||||||
|
// WebDAV of the very core we're enriching, so poster blobs and the poster
|
||||||
|
// CIDs we write can never point at different cores. Cached per core URL.
|
||||||
|
const webdavClient = await getWebDAVClient(request.metaCoreUrl);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { cid, existingMeta } = request;
|
const { cid, existingMeta } = request;
|
||||||
@@ -465,6 +412,7 @@ export async function process(
|
|||||||
const isEntry = cached.data !== undefined;
|
const isEntry = cached.data !== undefined;
|
||||||
await applyTmdbData(
|
await applyTmdbData(
|
||||||
metaCore,
|
metaCore,
|
||||||
|
webdavClient,
|
||||||
cid,
|
cid,
|
||||||
isEntry ? cached.data : cached,
|
isEntry ? cached.data : cached,
|
||||||
// Legacy raw payloads carry no image CIDs → let applyTmdbData recompute.
|
// Legacy raw payloads carry no image CIDs → let applyTmdbData recompute.
|
||||||
@@ -490,7 +438,7 @@ export async function process(
|
|||||||
existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'
|
existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'
|
||||||
? 'tv'
|
? 'tv'
|
||||||
: 'movie';
|
: 'movie';
|
||||||
entry = await resolveEntryById(knownTmdbId, mediaType);
|
entry = await resolveEntryById(webdavClient, knownTmdbId, mediaType);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: no usable id → resolve by IMDB id or fuzzy title, then cache
|
// Fallback: no usable id → resolve by IMDB id or fuzzy title, then cache
|
||||||
@@ -521,7 +469,7 @@ export async function process(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (tmdbData) {
|
if (tmdbData) {
|
||||||
const images = await computeImageCids(tmdbData);
|
const images = await computeImageCids(webdavClient, tmdbData);
|
||||||
entry = { data: tmdbData, ...images };
|
entry = { data: tmdbData, ...images };
|
||||||
if (tmdbData.id) {
|
if (tmdbData.id) {
|
||||||
await writeJson(tmdbCacheKey(String(tmdbData.id), mediaTypeOf(tmdbData)), entry);
|
await writeJson(tmdbCacheKey(String(tmdbData.id), mediaTypeOf(tmdbData)), entry);
|
||||||
@@ -530,7 +478,7 @@ export async function process(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (entry) {
|
if (entry) {
|
||||||
await applyTmdbData(metaCore, cid, entry.data, {
|
await applyTmdbData(metaCore, webdavClient, cid, entry.data, {
|
||||||
posterCid: entry.posterCid,
|
posterCid: entry.posterCid,
|
||||||
backdropCid: entry.backdropCid,
|
backdropCid: entry.backdropCid,
|
||||||
});
|
});
|
||||||
@@ -594,6 +542,7 @@ function mediaTypeOf(data: any): string {
|
|||||||
// (the image filename is keyed by tmdbId, so the blob dedups across records), so
|
// (the image filename is keyed by tmdbId, so the blob dedups across records), so
|
||||||
// the CIDs can be computed once per show and reused for every release.
|
// the CIDs can be computed once per show and reused for every release.
|
||||||
async function computeImageCids(
|
async function computeImageCids(
|
||||||
|
webdavClient: WebDAVClient | null,
|
||||||
data: any
|
data: any
|
||||||
): Promise<{ posterCid: string | null; backdropCid: string | null }> {
|
): Promise<{ posterCid: string | null; backdropCid: string | null }> {
|
||||||
const tmdbId = data?.id ? String(data.id) : '';
|
const tmdbId = data?.id ? String(data.id) : '';
|
||||||
@@ -603,11 +552,11 @@ async function computeImageCids(
|
|||||||
const year = releaseDate ? releaseDate.split('-')[0] : undefined;
|
const year = releaseDate ? releaseDate.split('-')[0] : undefined;
|
||||||
const posterCid =
|
const posterCid =
|
||||||
data?.poster_path && tmdbId
|
data?.poster_path && tmdbId
|
||||||
? await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId)
|
? await downloadAndHashImage(webdavClient, data.poster_path, 'poster', displayTitle, year, tmdbId)
|
||||||
: null;
|
: null;
|
||||||
const backdropCid =
|
const backdropCid =
|
||||||
data?.backdrop_path && tmdbId
|
data?.backdrop_path && tmdbId
|
||||||
? await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId)
|
? await downloadAndHashImage(webdavClient, data.backdrop_path, 'backdrop', displayTitle, year, tmdbId)
|
||||||
: null;
|
: null;
|
||||||
return { posterCid, backdropCid };
|
return { posterCid, backdropCid };
|
||||||
}
|
}
|
||||||
@@ -617,6 +566,7 @@ async function computeImageCids(
|
|||||||
// This is the hot path for gateway records (torrent + usenet), which always
|
// This is the hot path for gateway records (torrent + usenet), which always
|
||||||
// carry a `tmdbid` anchor — so N releases of one show share one TMDB fetch.
|
// carry a `tmdbid` anchor — so N releases of one show share one TMDB fetch.
|
||||||
async function resolveEntryById(
|
async function resolveEntryById(
|
||||||
|
webdavClient: WebDAVClient | null,
|
||||||
tmdbId: string,
|
tmdbId: string,
|
||||||
mediaType: string
|
mediaType: string
|
||||||
): Promise<TmdbCacheEntry | null> {
|
): Promise<TmdbCacheEntry | null> {
|
||||||
@@ -637,7 +587,7 @@ async function resolveEntryById(
|
|||||||
if (!data) {
|
if (!data) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const images = await computeImageCids(data);
|
const images = await computeImageCids(webdavClient, data);
|
||||||
const entry: TmdbCacheEntry = { data, ...images };
|
const entry: TmdbCacheEntry = { data, ...images };
|
||||||
await writeJson(key, entry);
|
await writeJson(key, entry);
|
||||||
return entry;
|
return entry;
|
||||||
@@ -655,6 +605,7 @@ async function resolveEntryById(
|
|||||||
*/
|
*/
|
||||||
async function applyTmdbData(
|
async function applyTmdbData(
|
||||||
metaCore: MetaCoreClient,
|
metaCore: MetaCoreClient,
|
||||||
|
webdavClient: WebDAVClient | null,
|
||||||
cid: string,
|
cid: string,
|
||||||
data: any,
|
data: any,
|
||||||
images?: { posterCid: string | null; backdropCid: string | null }
|
images?: { posterCid: string | null; backdropCid: string | null }
|
||||||
@@ -750,7 +701,7 @@ async function applyTmdbData(
|
|||||||
// up as first-class file entries, so storing the CID is all that's needed.
|
// up as first-class file entries, so storing the CID is all that's needed.
|
||||||
let posterCid = images ? images.posterCid : null;
|
let posterCid = images ? images.posterCid : null;
|
||||||
if (!images && data.poster_path && tmdbId) {
|
if (!images && data.poster_path && tmdbId) {
|
||||||
posterCid = await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId);
|
posterCid = await downloadAndHashImage(webdavClient, data.poster_path, 'poster', displayTitle, year, tmdbId);
|
||||||
}
|
}
|
||||||
if (posterCid) {
|
if (posterCid) {
|
||||||
await metaCore.setProperty(cid, 'poster', posterCid);
|
await metaCore.setProperty(cid, 'poster', posterCid);
|
||||||
@@ -759,7 +710,7 @@ async function applyTmdbData(
|
|||||||
|
|
||||||
let backdropCid = images ? images.backdropCid : null;
|
let backdropCid = images ? images.backdropCid : null;
|
||||||
if (!images && data.backdrop_path && tmdbId) {
|
if (!images && data.backdrop_path && tmdbId) {
|
||||||
backdropCid = await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId);
|
backdropCid = await downloadAndHashImage(webdavClient, data.backdrop_path, 'backdrop', displayTitle, year, tmdbId);
|
||||||
}
|
}
|
||||||
if (backdropCid) {
|
if (backdropCid) {
|
||||||
await metaCore.setProperty(cid, 'backdrop', backdropCid);
|
await metaCore.setProperty(cid, 'backdrop', backdropCid);
|
||||||
|
|||||||
+54
-5
@@ -136,15 +136,64 @@ export class WebDAVClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a WebDAV client from environment variables
|
* WebDAV client for the meta-core that drove a /process call.
|
||||||
|
*
|
||||||
|
* The endpoint is read from that core's own /urls (`webdavUrlInternal`),
|
||||||
|
* so blobs always land in the /files of the core we write metadata to. A feeder
|
||||||
|
* whose plugins run beside it but enrich a remote hub core needs no per-stack
|
||||||
|
* wiring, and it is impossible to write posters into one core while pointing
|
||||||
|
* CIDs at another.
|
||||||
|
*
|
||||||
|
* WEBDAV_URL still wins when set — an explicit override for tests and for
|
||||||
|
* topologies where the core's advertised internal URL isn't reachable from here.
|
||||||
|
*
|
||||||
|
* Resolutions are cached per meta-core URL; failures are not, so a core that is
|
||||||
|
* still starting up is retried on the next task.
|
||||||
*/
|
*/
|
||||||
export function createWebDAVClient(): WebDAVClient | null {
|
const clientCache = new Map<string, Promise<WebDAVClient | null>>();
|
||||||
const webdavUrl = process.env.WEBDAV_URL;
|
|
||||||
|
|
||||||
|
async function resolveWebDAVUrl(metaCoreUrl: string): Promise<string | null> {
|
||||||
|
// /urls, not /api/urls — metaCoreUrl points at the meta-core backend, whose
|
||||||
|
// discovery routes are unprefixed (same base the /meta/* writes use).
|
||||||
|
const base = metaCoreUrl.replace(/\/$/, '');
|
||||||
|
const response = await fetch(`${base}/urls`, { signal: AbortSignal.timeout(5000) });
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`GET ${base}/urls failed: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const urls = await response.json() as { webdavUrlInternal?: string; webdavUrl?: string };
|
||||||
|
return urls.webdavUrlInternal ?? urls.webdavUrl ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getWebDAVClient(metaCoreUrl: string): Promise<WebDAVClient | null> {
|
||||||
|
const override = process.env.WEBDAV_URL;
|
||||||
|
if (override) {
|
||||||
|
return new WebDAVClient(override);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pending = clientCache.get(metaCoreUrl);
|
||||||
|
if (!pending) {
|
||||||
|
pending = resolveWebDAVUrl(metaCoreUrl).then((webdavUrl) => {
|
||||||
if (!webdavUrl) {
|
if (!webdavUrl) {
|
||||||
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable');
|
console.warn(`[webdav-client] meta-core ${metaCoreUrl} advertises no WebDAV URL`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
console.log(`[webdav-client] Using WebDAV endpoint ${webdavUrl} (from ${metaCoreUrl})`);
|
||||||
return new WebDAVClient(webdavUrl);
|
return new WebDAVClient(webdavUrl);
|
||||||
|
});
|
||||||
|
clientCache.set(metaCoreUrl, pending);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = await pending;
|
||||||
|
if (!client) {
|
||||||
|
clientCache.delete(metaCoreUrl);
|
||||||
|
}
|
||||||
|
return client;
|
||||||
|
} catch (error) {
|
||||||
|
clientCache.delete(metaCoreUrl);
|
||||||
|
console.warn(`[webdav-client] Failed to resolve WebDAV URL from ${metaCoreUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user