use webdav for files
This commit is contained in:
+60
-2
@@ -37,6 +37,15 @@ import { pipeline } from 'stream/promises';
|
|||||||
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
import type { PluginManifest, ProcessRequest, CallbackPayload } from './types.js';
|
||||||
import { MetaCoreClient } from './meta-core-client.js';
|
import { MetaCoreClient } from './meta-core-client.js';
|
||||||
import { readJson, writeJson } from './cache.js';
|
import { readJson, writeJson } from './cache.js';
|
||||||
|
import { createWebDAVClient, WebDAVClient } from './webdav-client.js';
|
||||||
|
|
||||||
|
// Initialize WebDAV client if WEBDAV_URL is set
|
||||||
|
const webdavClient = createWebDAVClient();
|
||||||
|
if (webdavClient) {
|
||||||
|
console.log('[tmdb] Using WebDAV for file access');
|
||||||
|
} else {
|
||||||
|
console.log('[tmdb] Using direct filesystem access');
|
||||||
|
}
|
||||||
|
|
||||||
const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
|
const IMAGE_BASE_URL = 'https://image.tmdb.org/t/p/original';
|
||||||
|
|
||||||
@@ -61,7 +70,7 @@ const PLUGIN_OUTPUT_PATH = '/output';
|
|||||||
* - For files > 1MB: Hashes middle 1MB + 8-byte size prefix
|
* - For files > 1MB: Hashes middle 1MB + 8-byte size prefix
|
||||||
* - Returns CIDv1 with custom codec 0x1000
|
* - Returns CIDv1 with custom codec 0x1000
|
||||||
*/
|
*/
|
||||||
function computeMidHash256(filePath: string): string {
|
function computeMidHash256Sync(filePath: string): string {
|
||||||
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||||
// varint encoding of 0x1000 (4096) for both codec and hash function code
|
// varint encoding of 0x1000 (4096) for both codec and hash function code
|
||||||
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
||||||
@@ -91,6 +100,44 @@ function computeMidHash256(filePath: string): string {
|
|||||||
closeSync(fd);
|
closeSync(fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return computeMidHash256FromData(fileSize, sampleData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute midhash256 CID via WebDAV
|
||||||
|
*/
|
||||||
|
async function computeMidHash256WebDAV(client: WebDAVClient, filePath: string): Promise<string> {
|
||||||
|
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||||
|
|
||||||
|
// Get file size via HTTP HEAD
|
||||||
|
const stats = await client.stat(filePath);
|
||||||
|
const fileSize = stats.size;
|
||||||
|
|
||||||
|
// Read sample data via HTTP Range request
|
||||||
|
let sampleData: Buffer;
|
||||||
|
if (fileSize <= SAMPLE_SIZE) {
|
||||||
|
// Small file: read entire content
|
||||||
|
sampleData = await client.readBytes(filePath, 0, fileSize - 1);
|
||||||
|
} else {
|
||||||
|
// Large file: read middle 1MB
|
||||||
|
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
||||||
|
sampleData = await client.readBytes(filePath, middleOffset, middleOffset + SAMPLE_SIZE - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return computeMidHash256FromData(fileSize, sampleData);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute midhash256 CID from file size and sample data
|
||||||
|
*/
|
||||||
|
function computeMidHash256FromData(fileSize: number, sampleData: Buffer): string {
|
||||||
|
// varint encoding of 0x1000 (4096) for both codec and hash function code
|
||||||
|
const MIDHASH_VARINT = Buffer.from([0x80, 0x20]);
|
||||||
|
|
||||||
|
// Create size buffer (64-bit big-endian)
|
||||||
|
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||||
|
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||||
|
|
||||||
// Compute SHA-256 hash of [size + sample]
|
// Compute SHA-256 hash of [size + sample]
|
||||||
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||||
const hashBuffer = createHash('sha256').update(hashInput).digest();
|
const hashBuffer = createHash('sha256').update(hashInput).digest();
|
||||||
@@ -126,6 +173,16 @@ function computeMidHash256(filePath: string): string {
|
|||||||
return cid;
|
return cid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute midhash256 CID for a file (auto-selects WebDAV or filesystem)
|
||||||
|
*/
|
||||||
|
async function computeMidHash256(filePath: string): Promise<string> {
|
||||||
|
if (webdavClient) {
|
||||||
|
return computeMidHash256WebDAV(webdavClient, filePath);
|
||||||
|
}
|
||||||
|
return computeMidHash256Sync(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
export const manifest: PluginManifest = {
|
export const manifest: PluginManifest = {
|
||||||
id: 'tmdb',
|
id: 'tmdb',
|
||||||
name: 'TMDB Metadata',
|
name: 'TMDB Metadata',
|
||||||
@@ -341,8 +398,9 @@ async function downloadAndHashImage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Compute midhash256 CID (same algorithm as meta-sort file entries)
|
// Compute midhash256 CID (same algorithm as meta-sort file entries)
|
||||||
|
// Note: For /output files, we use sync version since they're on local filesystem
|
||||||
try {
|
try {
|
||||||
const cid = computeMidHash256(localPath);
|
const cid = computeMidHash256Sync(localPath);
|
||||||
console.log(`[tmdb] Image ${imageType} CID: ${cid}`);
|
console.log(`[tmdb] Image ${imageType} CID: ${cid}`);
|
||||||
return cid;
|
return cid;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
/**
|
||||||
|
* WebDAV Client for Plugin File Access
|
||||||
|
*
|
||||||
|
* Provides HTTP-based file access to meta-sort's WebDAV endpoint.
|
||||||
|
* Replaces direct filesystem access for containerized plugins.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface FileStats {
|
||||||
|
size: number;
|
||||||
|
mtime?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class WebDAVClient {
|
||||||
|
private baseUrl: string;
|
||||||
|
|
||||||
|
constructor(baseUrl: string) {
|
||||||
|
// Remove trailing slash if present
|
||||||
|
this.baseUrl = baseUrl.replace(/\/$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert absolute file path to WebDAV URL
|
||||||
|
* Strips /files prefix and appends to base URL
|
||||||
|
*/
|
||||||
|
toWebDAVUrl(filePath: string): string {
|
||||||
|
// filePath is like "/files/watch/movie.mkv"
|
||||||
|
// We need to convert to "http://meta-sort-dev/webdav/watch/movie.mkv"
|
||||||
|
let relativePath = filePath;
|
||||||
|
|
||||||
|
// Strip /files prefix if present
|
||||||
|
if (relativePath.startsWith('/files/')) {
|
||||||
|
relativePath = relativePath.substring(6); // Remove "/files"
|
||||||
|
} else if (relativePath.startsWith('/files')) {
|
||||||
|
relativePath = relativePath.substring(6);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure path starts with /
|
||||||
|
if (!relativePath.startsWith('/')) {
|
||||||
|
relativePath = '/' + relativePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.baseUrl + relativePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file stats (size, mtime) via HTTP HEAD request
|
||||||
|
*/
|
||||||
|
async stat(filePath: string): Promise<FileStats> {
|
||||||
|
const url = this.toWebDAVUrl(filePath);
|
||||||
|
|
||||||
|
const response = await fetch(url, { method: 'HEAD' });
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`WebDAV HEAD failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = response.headers.get('content-length');
|
||||||
|
const lastModified = response.headers.get('last-modified');
|
||||||
|
|
||||||
|
return {
|
||||||
|
size: contentLength ? parseInt(contentLength, 10) : 0,
|
||||||
|
mtime: lastModified ? new Date(lastModified) : undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read first N bytes of a file (for magic byte detection)
|
||||||
|
* Uses HTTP Range request
|
||||||
|
*/
|
||||||
|
async readBytes(filePath: string, start: number, end: number): Promise<Buffer> {
|
||||||
|
const url = this.toWebDAVUrl(filePath);
|
||||||
|
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
'Range': `bytes=${start}-${end}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status !== 206) {
|
||||||
|
throw new Error(`WebDAV Range GET failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const arrayBuffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(arrayBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read entire file as buffer (for small files only)
|
||||||
|
*/
|
||||||
|
async readFile(filePath: string): Promise<Buffer> {
|
||||||
|
const url = this.toWebDAVUrl(filePath);
|
||||||
|
|
||||||
|
const response = await fetch(url, { method: 'GET' });
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`WebDAV GET failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const arrayBuffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(arrayBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write file via HTTP PUT
|
||||||
|
*/
|
||||||
|
async writeFile(filePath: string, data: Buffer | string): Promise<void> {
|
||||||
|
const url = this.toWebDAVUrl(filePath);
|
||||||
|
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: data,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/octet-stream',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`WebDAV PUT failed for ${filePath}: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if file exists via HTTP HEAD
|
||||||
|
*/
|
||||||
|
async exists(filePath: string): Promise<boolean> {
|
||||||
|
const url = this.toWebDAVUrl(filePath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, { method: 'HEAD' });
|
||||||
|
return response.ok;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a WebDAV client from environment variables
|
||||||
|
*/
|
||||||
|
export function createWebDAVClient(): WebDAVClient | null {
|
||||||
|
const webdavUrl = process.env.WEBDAV_URL;
|
||||||
|
|
||||||
|
if (!webdavUrl) {
|
||||||
|
console.warn('[webdav-client] WEBDAV_URL not set, WebDAV client unavailable');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new WebDAVClient(webdavUrl);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user