diff --git a/src/plugin.ts b/src/plugin.ts index cf9cac9..1602181 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -29,6 +29,15 @@ import { stringify } from 'csv-stringify/sync'; import CRC32 from 'crc-32'; import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js'; import { MetaCoreClient } from './meta-core-client.js'; +import { createWebDAVClient, WebDAVClient } from './webdav-client.js'; + +// Initialize WebDAV client if WEBDAV_URL is set +const webdavClient = createWebDAVClient(); +if (webdavClient) { + console.log('[full-hash] Using WebDAV for file access'); +} else { + console.log('[full-hash] Using direct filesystem access'); +} // Cache folder path (mounted as /cache in container) const CACHE_FOLDER_PATH = globalThis.process.env.CACHE_PATH || '/cache'; @@ -203,9 +212,9 @@ function formatAsCid(hashType: HashAlgorithm, hashHex: string): string { } /** - * Compute all hashes for a file in a single pass + * Compute all hashes for a file in a single pass (filesystem mode) */ -async function computeHashes(filePath: string, hashTypes: HashAlgorithm[]): Promise> { +async function computeHashesFromFilesystem(filePath: string, hashTypes: HashAlgorithm[]): Promise> { return new Promise((resolve, reject) => { const results = new Map(); const hashers: Map = new Map(); @@ -260,6 +269,67 @@ async function computeHashes(filePath: string, hashTypes: HashAlgorithm[]): Prom }); } +/** + * Compute all hashes for a file via WebDAV streaming + */ +async function computeHashesFromWebDAV(client: WebDAVClient, filePath: string, hashTypes: HashAlgorithm[]): Promise> { + const results = new Map(); + const hashers: Map = new Map(); + let crc32Value = 0; + const needsCrc32 = hashTypes.includes('cid_crc32'); + + // Initialize crypto hashers + for (const hashType of hashTypes) { + const algo = HASH_ALGORITHMS[hashType]; + if (algo.crypto) { + try { + hashers.set(hashType, createHash(algo.crypto)); + } catch (e) { + console.warn(`[full-hash] Hash algorithm ${algo.crypto} not available, skipping ${hashType}`); + } + } + } + + // Stream file from WebDAV + for await (const chunk of client.streamFile(filePath)) { + // Update all crypto hashers + for (const [, hasher] of hashers) { + hasher.update(chunk); + } + // Update CRC32 + if (needsCrc32) { + crc32Value = CRC32.buf(chunk, crc32Value); + } + } + + // Finalize crypto hashes + for (const [hashType, hasher] of hashers) { + const hexHash = hasher.digest('hex'); + const cid = formatAsCid(hashType, hexHash); + results.set(hashType, cid); + } + + // Finalize CRC32 + if (needsCrc32) { + const unsigned = crc32Value >>> 0; + const hexHash = unsigned.toString(16).padStart(8, '0'); + const cid = formatAsCid('cid_crc32', hexHash); + results.set('cid_crc32', cid); + } + + return results; +} + +/** + * Compute all hashes for a file in a single pass + */ +async function computeHashes(filePath: string, hashTypes: HashAlgorithm[]): Promise> { + if (webdavClient) { + return computeHashesFromWebDAV(webdavClient, filePath, hashTypes); + } + return computeHashesFromFilesystem(filePath, hashTypes); +} + export const manifest: PluginManifest = { id: 'full-hash', name: 'Full File Hash', @@ -345,9 +415,19 @@ export async function process( const hashesFromCache: Map = new Map(); // Get file stats for cache lookup - const stats = await stat(filePath); + let fileSize: number; + let mtime: string; const filename = path.basename(filePath); - const mtime = stats.mtime.toISOString(); + + if (webdavClient) { + const stats = await webdavClient.stat(filePath); + fileSize = stats.size; + mtime = stats.mtime?.toISOString() || new Date().toISOString(); + } else { + const stats = await stat(filePath); + fileSize = stats.size; + mtime = stats.mtime.toISOString(); + } // Load caches and check what's needed for (const hashType of enabledHashes) { @@ -360,7 +440,7 @@ export async function process( await loadHashCache(hashType); // Check cache - const cachedHash = getCachedHash(hashType, filename, stats.size, mtime); + const cachedHash = getCachedHash(hashType, filename, fileSize, mtime); if (cachedHash) { hashesFromCache.set(hashType, cachedHash); } else { @@ -388,7 +468,7 @@ export async function process( // Add computed hashes to cache for (const [hashType, hash] of computedHashes) { - addToCache(hashType, filename, stats.size, mtime, hash); + addToCache(hashType, filename, fileSize, mtime, hash); } // Save updated caches @@ -407,7 +487,8 @@ export async function process( const cacheHits = hashesFromCache.size; const computed = computedHashes.size; - console.log(`[full-hash] Stored ${allHashes.size} hashes (${cacheHits} cached, ${computed} computed)`); + const mode = webdavClient ? 'WebDAV' : 'filesystem'; + console.log(`[full-hash] Stored ${allHashes.size} hashes (${cacheHits} cached, ${computed} computed, ${mode})`); await sendCallback({ taskId: request.taskId, diff --git a/src/webdav-client.ts b/src/webdav-client.ts new file mode 100644 index 0000000..0b1609a --- /dev/null +++ b/src/webdav-client.ts @@ -0,0 +1,178 @@ +/** + * WebDAV Client for Plugin File Access + * + * Provides HTTP-based file access to meta-sort's WebDAV endpoint. + * Replaces direct filesystem access for containerized plugins. + */ + +export interface FileStats { + size: number; + mtime?: Date; +} + +export class WebDAVClient { + private baseUrl: string; + + constructor(baseUrl: string) { + // Remove trailing slash if present + this.baseUrl = baseUrl.replace(/\/$/, ''); + } + + /** + * Convert absolute file path to WebDAV URL + * Strips /files prefix and appends to base URL + */ + toWebDAVUrl(filePath: string): string { + // filePath is like "/files/watch/movie.mkv" + // We need to convert to "http://meta-sort-dev/webdav/watch/movie.mkv" + let relativePath = filePath; + + // Strip /files prefix if present + if (relativePath.startsWith('/files/')) { + relativePath = relativePath.substring(6); // Remove "/files" + } else if (relativePath.startsWith('/files')) { + relativePath = relativePath.substring(6); + } + + // Ensure path starts with / + if (!relativePath.startsWith('/')) { + relativePath = '/' + relativePath; + } + + return this.baseUrl + relativePath; + } + + /** + * Get file stats (size, mtime) via HTTP HEAD request + */ + async stat(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'HEAD' }); + + if (!response.ok) { + throw new Error(`WebDAV HEAD failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const contentLength = response.headers.get('content-length'); + const lastModified = response.headers.get('last-modified'); + + return { + size: contentLength ? parseInt(contentLength, 10) : 0, + mtime: lastModified ? new Date(lastModified) : undefined, + }; + } + + /** + * Stream file content for hashing (returns async iterator of chunks) + */ + async *streamFile(filePath: string): AsyncGenerator { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'GET' }); + + if (!response.ok) { + throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + if (!response.body) { + throw new Error(`WebDAV GET returned no body for ${filePath}`); + } + + const reader = response.body.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + yield Buffer.from(value); + } + } finally { + reader.releaseLock(); + } + } + + /** + * Read first N bytes of a file (for magic byte detection) + * Uses HTTP Range request + */ + async readBytes(filePath: string, start: number, end: number): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { + method: 'GET', + headers: { + 'Range': `bytes=${start}-${end}`, + }, + }); + + if (!response.ok && response.status !== 206) { + throw new Error(`WebDAV Range GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Read entire file as buffer (for small files only) + */ + async readFile(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { method: 'GET' }); + + if (!response.ok) { + throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } + + /** + * Write file via HTTP PUT + */ + async writeFile(filePath: string, data: Buffer | string): Promise { + const url = this.toWebDAVUrl(filePath); + + const response = await fetch(url, { + method: 'PUT', + body: data, + headers: { + 'Content-Type': 'application/octet-stream', + }, + }); + + if (!response.ok) { + throw new Error(`WebDAV PUT failed for ${filePath}: ${response.status} ${response.statusText}`); + } + } + + /** + * Check if file exists via HTTP HEAD + */ + async exists(filePath: string): Promise { + const url = this.toWebDAVUrl(filePath); + + try { + const response = await fetch(url, { method: 'HEAD' }); + return response.ok; + } catch { + return false; + } + } +} + +/** + * Create a WebDAV client from environment variables + */ +export function createWebDAVClient(): WebDAVClient | null { + const webdavUrl = process.env.WEBDAV_URL; + + if (!webdavUrl) { + console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable'); + return null; + } + + return new WebDAVClient(webdavUrl); +}