From bad6ddace2691c5fc82f59aac6961f69d9094558 Mon Sep 17 00:00:00 2001 From: Pierre Date: Fri, 6 Feb 2026 23:51:00 +0100 Subject: [PATCH] meta-sort now use webdav --- .../src/api/UnifiedAPIServer.ts | 27 +- .../src/jellyfin/JellyfinNfo.ts | 23 +- packages/meta-sort-core/src/kv/KVManager.ts | 11 + .../src/logic/MetaMeshFormat.ts | 34 +- .../src/logic/WatchedFileProcessor.ts | 26 +- .../meta-sort-core/src/webdav/WebdavClient.ts | 464 ++++++++++++++++++ packages/meta-sort-core/src/webdav/index.ts | 7 + 7 files changed, 546 insertions(+), 46 deletions(-) create mode 100644 packages/meta-sort-core/src/webdav/WebdavClient.ts create mode 100644 packages/meta-sort-core/src/webdav/index.ts diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts index a738814..384212c 100644 --- a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -16,6 +16,7 @@ import type { KVManager } from '../kv/KVManager.js'; import type { ExtendedProcessingSnapshot, IKVClientWithPubSub } from '../types/ExtendedInterfaces.js'; import { getErrorMessage } from '../types/ExtendedInterfaces.js'; import { config } from '../config/EnvConfig.js'; +import * as webdav from '../webdav/WebdavClient.js'; import type { PluginManager } from '../plugin-engine/PluginManager.js'; import type { TaskScheduler } from '../plugin-engine/TaskScheduler.js'; import type { ContainerManager, ContainerPluginScheduler, PluginCallbackPayload, GateStatus } from '../container-plugins/index.js'; @@ -237,7 +238,7 @@ export class UnifiedAPIServer { * Setup File download routes */ private setupFileRoutes(): void { - // Download file by path + // Download file by path (via WebDAV) this.app.get<{ Querystring: { path: string } }>('/api/file/download', async (request, reply) => { const { path: filePath } = request.query; @@ -246,9 +247,13 @@ export class UnifiedAPIServer { } try { - const fs = await import('fs'); const pathModule = await import('path'); + // Check if WebDAV is configured + if (!webdav.isConfigured()) { + return reply.status(503).send({ error: 'WebDAV not configured - waiting for leader discovery' }); + } + // Security: ensure the path is within FILES_PATH const filesPath = config.FILES_PATH; const absolutePath = pathModule.resolve(filePath); @@ -257,25 +262,23 @@ export class UnifiedAPIServer { return reply.status(403).send({ error: 'Access denied - path outside files directory' }); } - // Check if file exists - if (!fs.existsSync(absolutePath)) { + // Get file stats via WebDAV + const stats = await webdav.stat(absolutePath); + if (!stats || !stats.exists) { return reply.status(404).send({ error: 'File not found' }); } - // Get file stats - const stats = fs.statSync(absolutePath); - if (!stats.isFile()) { - return reply.status(400).send({ error: 'Path is not a file' }); - } - // Set headers for download const filename = pathModule.basename(absolutePath); reply.header('Content-Disposition', `attachment; filename="${encodeURIComponent(filename)}"`); reply.header('Content-Type', 'application/octet-stream'); reply.header('Content-Length', stats.size); - // Stream the file - const stream = fs.createReadStream(absolutePath); + // Stream the file from WebDAV + const stream = await webdav.createReadStream(absolutePath); + if (!stream) { + return reply.status(500).send({ error: 'Failed to create file stream' }); + } return reply.send(stream); } catch (error) { console.error('Error downloading file:', error); diff --git a/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts b/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts index e2b208a..0e9b9ba 100644 --- a/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts +++ b/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts @@ -1,10 +1,9 @@ -import {readFile, writeFile} from 'fs/promises'; import {Builder, Parser} from 'xml2js'; import {mergeMetadata} from "../logic/tool/MetadataMerger.js"; -import {existsAsync} from "@metazla/meta-hash"; import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface"; import lodash from 'lodash'; import * as xml2js from "xml2js"; +import * as webdav from '../webdav/WebdavClient.js'; export class JellyfinNfo { @@ -61,9 +60,9 @@ export class JellyfinNfo { throw new Error(`NFO file path must end with '.nfo'`); } let metadata: T = {} as T; - if (await existsAsync(nfoPath)) { + const nfoContent = await webdav.readTextFile(nfoPath); + if (nfoContent) { try { - const nfoContent = await readFile(nfoPath, {encoding: 'utf8'}); const parser = new xml2js.Parser(this.options); const result = await parser.parseStringPromise(nfoContent); metadata = result.root || result.episodedetails || result.movie || {}; @@ -85,7 +84,7 @@ export class JellyfinNfo { if (data.fileType !== "video") { return;//only process video files for writing jellyfin nfo } - if (await existsAsync(nfoPath)) { + if (await webdav.exists(nfoPath)) { await this.updateNfoFile(nfoPath, data as any); } else { throw new Error(`NFO file does not exist at path: ${nfoPath}`); @@ -100,7 +99,7 @@ export class JellyfinNfo { if (data.fileType !== "video") { return;//only process video files for writing jellyfin nfo } - if (await existsAsync(nfoPath)) { + if (await webdav.exists(nfoPath)) { await this.updateNfoFile(nfoPath, data as any); } else { // If the NFO file doesn't exist, just generate a new one @@ -146,7 +145,11 @@ export class JellyfinNfo { private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) { try { - const existingContent = await readFile(nfoFilePath, {encoding: "utf-8"}); + const existingContent = await webdav.readTextFile(nfoFilePath); + if (!existingContent) { + console.error(`Failed to read NFO content from ${nfoFilePath}`); + return; + } const parser = new Parser(this.options); let existingData = await parser.parseStringPromise(existingContent); let hasChanged = false; @@ -184,7 +187,7 @@ export class JellyfinNfo { // Convert the updated object back to XML try { const updatedContent = builder.buildObject(existingData); - await writeFile(nfoFilePath, updatedContent, {encoding: "utf-8"}); + await webdav.writeFile(nfoFilePath, updatedContent); } catch (error) { console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`); } @@ -247,7 +250,7 @@ export class JellyfinNfo { if (!nfoFilePath.endsWith('.nfo')) { throw new Error(`NFO file path must end with '.nfo'`); } - if (await existsAsync(nfoFilePath)) { + if (await webdav.exists(nfoFilePath)) { throw new Error(`NFO file already exists at path: ${nfoFilePath}`); } @@ -271,7 +274,7 @@ export class JellyfinNfo { const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}}); const nfoContent = builder.buildObject(obj); - await writeFile(nfoFilePath, nfoContent, {encoding: "utf-8"}); + await webdav.writeFile(nfoFilePath, nfoContent); } } diff --git a/packages/meta-sort-core/src/kv/KVManager.ts b/packages/meta-sort-core/src/kv/KVManager.ts index 0df09b3..0da180f 100644 --- a/packages/meta-sort-core/src/kv/KVManager.ts +++ b/packages/meta-sort-core/src/kv/KVManager.ts @@ -31,6 +31,7 @@ import { LeaderClient } from './LeaderClient.js'; import { ServiceDiscovery } from './ServiceDiscovery.js'; import { RedisKVClient } from './RedisClient.js'; import type { IKVClient, LeaderLockInfo } from './IKVClient.js'; +import { configure as configureWebdav } from '../webdav/WebdavClient.js'; interface KVManagerConfig { /** Path to META_CORE_VOLUME (e.g., /meta-core) */ @@ -122,6 +123,11 @@ export class KVManager { const leaderInfo = await this.leaderClient.waitForLeader(30000); console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`); this.kvClient = await this.createRedisClient(leaderInfo.redisUrl); + + // Configure WebDAV client with leader's WebDAV URL + if (leaderInfo.webdavUrl) { + configureWebdav(leaderInfo.webdavUrl); + } } catch (error) { console.error('[KVManager] Failed to connect to leader:', error); throw error; @@ -164,6 +170,11 @@ export class KVManager { console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`); this.kvClient = await this.createRedisClient(leaderInfo.redisUrl); + // Reconfigure WebDAV client with new leader's WebDAV URL + if (leaderInfo.webdavUrl) { + configureWebdav(leaderInfo.webdavUrl); + } + // Notify reconnect callbacks for (const callback of this.onReconnectCallbacks) { try { diff --git a/packages/meta-sort-core/src/logic/MetaMeshFormat.ts b/packages/meta-sort-core/src/logic/MetaMeshFormat.ts index 446a17e..8e39e16 100644 --- a/packages/meta-sort-core/src/logic/MetaMeshFormat.ts +++ b/packages/meta-sort-core/src/logic/MetaMeshFormat.ts @@ -1,11 +1,13 @@ -import { promises as fs } from 'fs'; import yaml from 'yaml'; +import * as webdav from '../webdav/WebdavClient.js'; const { parse, stringify } = yaml; /** * File type: * filename..meta * filename.meta (contains multiple metadata entry "mkv", "str" etc...) * folder.meta + * + * All file operations use WebDAV for access to meta-core's file system. */ export class MetaMeshFormat { @@ -14,15 +16,15 @@ export class MetaMeshFormat { throw new Error(`File path must end with '.meta'`); } try { - // Check if the file exists - const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false); + // Check if the file exists via WebDAV + const fileExists = await webdav.exists(metaDataFilePath); if (fileExists) { // If the file exists, update it await this.update(metaDataFilePath, data); } else { - // If the file does not exist, create it - await fs.writeFile(metaDataFilePath, stringify(data), 'utf8'); + // If the file does not exist, create it via WebDAV + await webdav.writeFile(metaDataFilePath, stringify(data)); } } catch (error) { console.error(`Error in createOrUpdate: ${error}`); @@ -30,18 +32,17 @@ export class MetaMeshFormat { } } - async read(metaDataFilePath: string): Promise { + async read(metaDataFilePath: string): Promise { if(!metaDataFilePath.endsWith('.meta')){ throw new Error(`File path must end with '.meta'`); } try { - const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false); + // Read the YAML file via WebDAV + const fileContent = await webdav.readTextFile(metaDataFilePath); - if (!fileExists) { + if (!fileContent) { return null; } - // Read the YAML file - const fileContent = await fs.readFile(metaDataFilePath, 'utf8'); // Parse the YAML content to a JavaScript object return parse(fileContent) as T; @@ -62,8 +63,8 @@ export class MetaMeshFormat { // Merge the existing data with the new data const updatedData = { ...existingData, ...data }; - // Write the updated data back to the file - await fs.writeFile(metaDataFilePath, stringify(updatedData), 'utf8'); + // Write the updated data back to the file via WebDAV + await webdav.writeFile(metaDataFilePath, stringify(updatedData)); } catch (error) { console.error(`Error in update: ${error}`); throw error; @@ -75,13 +76,10 @@ export class MetaMeshFormat { throw new Error(`File path must end with '.meta'`); } try { - // Write the updated data back to the file - await fs.writeFile(metaDataFilePath, stringify(data), { - encoding:'utf8', - flag:'w' // Write mode : Overwrite - }); + // Write the data to the file via WebDAV (overwrite) + await webdav.writeFile(metaDataFilePath, stringify(data)); } catch (error) { - console.error(`Error in update: ${error}`); + console.error(`Error in write: ${error}`); throw error; } } diff --git a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts index f099118..3c60940 100644 --- a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts +++ b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts @@ -1,11 +1,9 @@ -import {promises as fs} from 'fs'; import {dirname, join, parse} from 'path'; import {config} from "../config/EnvConfig.js"; import {FileProcessorInterface} from "@metazla/meta-hash"; import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js"; import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js"; import {FileType} from "@metazla/filename-tools"; -import {existsAsync} from "@metazla/meta-hash"; import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js"; import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js"; import {renamingRule} from "../config/RenamingRule.js"; @@ -16,6 +14,7 @@ import {performanceMetrics} from "../metrics/PerformanceMetrics.js"; import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js"; import type {IKVClient} from "../kv/IKVClient.js"; import { hasStreamSupport, type ExtendedFileAnalyzer, type ExtendedHashMeta } from "../types/ExtendedInterfaces.js"; +import * as webdav from '../webdav/WebdavClient.js'; // Redis Streams for reliable event delivery const EVENTS_STREAM = 'meta-sort:events'; @@ -25,7 +24,6 @@ import os from 'os'; import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js"; import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js"; import { from } from "@metazla/meta-interface"; -import { stat } from "fs/promises"; import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js"; export class WatchedFileProcessor implements FileProcessorInterface { @@ -313,8 +311,15 @@ export class WatchedFileProcessor implements FileProcessorInterface { } // STEP 1: Compute midhash256 (file identifier) - const fileStats = await stat(filePath); - const { computeMidHash256 } = await import('@metazla/meta-hash'); + // Use WebDAV for file stats and hash computation + const fileStats = await webdav.stat(filePath); + if (!fileStats || !fileStats.exists) { + console.error(`[WatchedFileProcessor] File not found via WebDAV: ${filePath}`); + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing(filePath, '', undefined, 'File not found'); + } + return; + } // Check cache first const extAnalyzer = this.fileProcessor as ExtendedFileAnalyzer; @@ -330,7 +335,16 @@ export class WatchedFileProcessor implements FileProcessorInterface { performanceMetrics.recordCacheHit('midhash256'); } else { const computeStart = performance.now(); - midHash256 = await computeMidHash256(filePath); + // Use WebDAV-based hash computation + const hash = await webdav.computeMidHash256(filePath); + if (!hash) { + console.error(`[WatchedFileProcessor] Failed to compute hash for: ${filePath}`); + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing(filePath, '', undefined, 'Hash computation failed'); + } + return; + } + midHash256 = hash; performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart)); performanceMetrics.recordCacheMiss('midhash256'); diff --git a/packages/meta-sort-core/src/webdav/WebdavClient.ts b/packages/meta-sort-core/src/webdav/WebdavClient.ts new file mode 100644 index 0000000..234a13f --- /dev/null +++ b/packages/meta-sort-core/src/webdav/WebdavClient.ts @@ -0,0 +1,464 @@ +/** + * WebDAV Client for Meta-Sort + * + * Provides HTTP-based file access to meta-core's WebDAV server. + * Files are accessed exclusively via WebDAV - no local filesystem access. + * + * The WebDAV URL is discovered automatically from the leader via LeaderClient. + */ + +import { Readable } from 'stream'; +import { createHash } from 'crypto'; +import { create } from 'multiformats/hashes/digest'; +import { CID } from 'multiformats/cid'; + +// Configuration - set dynamically after leader discovery +let WEBDAV_URL: string = ''; +const FILES_PATH = process.env.FILES_PATH || '/files'; + +/** + * Configure the WebDAV client with a URL discovered from leader. + * Called after LeaderClient connects to the leader. + */ +export function configure(webdavUrl: string): void { + if (!webdavUrl) { + return; + } + WEBDAV_URL = webdavUrl.replace(/\/$/, ''); + console.log(`[webdav-client] Configured: ${WEBDAV_URL}`); +} + +/** + * Check if WebDAV client is configured. + */ +export function isConfigured(): boolean { + return Boolean(WEBDAV_URL); +} + +/** + * Get the configured WebDAV URL. + */ +export function getWebdavUrl(): string { + return WEBDAV_URL; +} + +/** + * Convert a local file path to a WebDAV URL. + * + * @param filePath - Local file path (e.g., /files/watch/movie.mp4) + * @returns WebDAV URL or null if not configured + */ +export function toWebdavUrl(filePath: string): string | null { + if (!WEBDAV_URL) { + return null; + } + + // Strip the FILES_PATH prefix to get relative path + let relativePath = filePath; + if (filePath.startsWith(FILES_PATH + '/')) { + relativePath = filePath.slice(FILES_PATH.length); + } else if (filePath.startsWith(FILES_PATH)) { + relativePath = filePath.slice(FILES_PATH.length); + } + + // Ensure path starts with / + if (!relativePath.startsWith('/')) { + relativePath = '/' + relativePath; + } + + // URL-encode path segments (but not slashes) + const encodedPath = relativePath + .split('/') + .map(segment => encodeURIComponent(segment)) + .join('/'); + + return WEBDAV_URL + encodedPath; +} + +/** + * File stats returned from WebDAV HEAD request. + */ +export interface WebdavFileStats { + size: number; + mtime: Date; + exists: boolean; +} + +/** + * Get file stats via HTTP HEAD request to WebDAV. + * + * @param filePath - File path + * @returns File stats or null on error + */ +export async function stat(filePath: string): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot stat file'); + return null; + } + + try { + const response = await fetch(url, { + method: 'HEAD', + signal: AbortSignal.timeout(30000) + }); + + if (response.status === 404) { + return { size: 0, mtime: new Date(0), exists: false }; + } + + if (!response.ok) { + console.error(`[webdav-client] HEAD error for ${url}: ${response.status}`); + return null; + } + + const contentLength = response.headers.get('Content-Length'); + const lastModified = response.headers.get('Last-Modified'); + + return { + size: contentLength ? parseInt(contentLength, 10) : 0, + mtime: lastModified ? new Date(lastModified) : new Date(), + exists: true + }; + } catch (error) { + console.error(`[webdav-client] HEAD error for ${url}:`, error); + return null; + } +} + +/** + * Check if file exists via HTTP HEAD request to WebDAV. + * + * @param filePath - File path + * @returns True if file exists + */ +export async function exists(filePath: string): Promise { + const stats = await stat(filePath); + return stats?.exists ?? false; +} + +/** + * Read entire file from WebDAV. + * + * @param filePath - File path + * @returns File contents as Buffer, or null on error + */ +export async function readFile(filePath: string): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot read file'); + return null; + } + + try { + const response = await fetch(url, { + method: 'GET', + signal: AbortSignal.timeout(60000) + }); + + if (!response.ok) { + if (response.status !== 404) { + console.error(`[webdav-client] GET error for ${url}: ${response.status}`); + } + return null; + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } catch (error) { + console.error(`[webdav-client] GET error for ${url}:`, error); + return null; + } +} + +/** + * Read file as text from WebDAV. + * + * @param filePath - File path + * @param encoding - Text encoding (default: utf-8) + * @returns File contents as string, or null on error + */ +export async function readTextFile(filePath: string, encoding: BufferEncoding = 'utf-8'): Promise { + const buffer = await readFile(filePath); + return buffer?.toString(encoding) ?? null; +} + +/** + * Read a byte range from a file via WebDAV. + * + * @param filePath - File path + * @param start - Start byte offset (inclusive) + * @param end - End byte offset (inclusive) + * @returns Bytes in the specified range, or null on error + */ +export async function readRange(filePath: string, start: number, end: number): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot read range'); + return null; + } + + try { + const response = await fetch(url, { + method: 'GET', + headers: { + 'Range': `bytes=${start}-${end}` + }, + signal: AbortSignal.timeout(60000) + }); + + if (!response.ok && response.status !== 206) { + console.error(`[webdav-client] Range GET error for ${url}: ${response.status}`); + return null; + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } catch (error) { + console.error(`[webdav-client] Range GET error for ${url}:`, error); + return null; + } +} + +/** + * Write file to WebDAV. + * + * @param filePath - File path + * @param content - File contents (Buffer or string) + * @returns True on success + */ +export async function writeFile(filePath: string, content: Buffer | string): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot write file'); + return false; + } + + try { + const buffer = typeof content === 'string' ? Buffer.from(content, 'utf-8') : content; + + const response = await fetch(url, { + method: 'PUT', + // Node.js fetch accepts Buffer directly at runtime + body: buffer as unknown as BodyInit, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': buffer.length.toString() + }, + signal: AbortSignal.timeout(60000) + }); + + if (!response.ok && response.status !== 201 && response.status !== 204) { + console.error(`[webdav-client] PUT error for ${url}: ${response.status}`); + return false; + } + + return true; + } catch (error) { + console.error(`[webdav-client] PUT error for ${url}:`, error); + return false; + } +} + +/** + * Delete file from WebDAV. + * + * @param filePath - File path + * @returns True on success + */ +export async function deleteFile(filePath: string): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot delete file'); + return false; + } + + try { + const response = await fetch(url, { + method: 'DELETE', + signal: AbortSignal.timeout(30000) + }); + + if (!response.ok && response.status !== 204 && response.status !== 404) { + console.error(`[webdav-client] DELETE error for ${url}: ${response.status}`); + return false; + } + + return true; + } catch (error) { + console.error(`[webdav-client] DELETE error for ${url}:`, error); + return false; + } +} + +/** + * Create directory on WebDAV (MKCOL). + * + * @param dirPath - Directory path + * @returns True on success + */ +export async function mkdir(dirPath: string): Promise { + const url = toWebdavUrl(dirPath); + if (!url) { + console.error('[webdav-client] Not configured, cannot create directory'); + return false; + } + + try { + const response = await fetch(url, { + method: 'MKCOL', + signal: AbortSignal.timeout(30000) + }); + + // 201 Created, 405 Method Not Allowed (already exists), or any 2xx + if (!response.ok && response.status !== 201 && response.status !== 405) { + console.error(`[webdav-client] MKCOL error for ${url}: ${response.status}`); + return false; + } + + return true; + } catch (error) { + console.error(`[webdav-client] MKCOL error for ${url}:`, error); + return false; + } +} + +/** + * Create a readable stream from WebDAV file. + * Uses fetch with streaming response. + * + * @param filePath - File path + * @returns Readable stream or null on error + */ +export async function createReadStream(filePath: string): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot create stream'); + return null; + } + + try { + const response = await fetch(url, { + method: 'GET', + signal: AbortSignal.timeout(300000) // 5 minutes for streaming + }); + + if (!response.ok) { + console.error(`[webdav-client] Stream GET error for ${url}: ${response.status}`); + return null; + } + + if (!response.body) { + console.error(`[webdav-client] No response body for ${url}`); + return null; + } + + // Convert web ReadableStream to Node.js Readable + return Readable.fromWeb(response.body as any); + } catch (error) { + console.error(`[webdav-client] Stream error for ${url}:`, error); + return null; + } +} + +/** + * Create a readable stream for a byte range from WebDAV file. + * + * @param filePath - File path + * @param start - Start byte offset + * @param end - End byte offset + * @returns Readable stream or null on error + */ +export async function createRangeStream(filePath: string, start: number, end: number): Promise { + const url = toWebdavUrl(filePath); + if (!url) { + console.error('[webdav-client] Not configured, cannot create range stream'); + return null; + } + + try { + const response = await fetch(url, { + method: 'GET', + headers: { + 'Range': `bytes=${start}-${end}` + }, + signal: AbortSignal.timeout(300000) + }); + + if (!response.ok && response.status !== 206) { + console.error(`[webdav-client] Range stream error for ${url}: ${response.status}`); + return null; + } + + if (!response.body) { + console.error(`[webdav-client] No response body for range ${url}`); + return null; + } + + return Readable.fromWeb(response.body as any); + } catch (error) { + console.error(`[webdav-client] Range stream error for ${url}:`, error); + return null; + } +} + +/** + * Multicodec for midhash256 (custom algorithm code) + */ +const CID_MIDHASH256_CODE = 0x1000; + +/** + * Compute midhash256 for a file via WebDAV. + * + * This is a WebDAV-aware version of the midhash256 algorithm that uses + * HTTP range requests to read only the required bytes instead of the entire file. + * + * Algorithm: + * - For files <= 1MB: Hash entire file content + size prefix + * - For files > 1MB: Hash middle 1MB + size prefix + * + * @param filePath - File path + * @returns CID string or null on error + */ +export async function computeMidHash256(filePath: string): Promise { + const SAMPLE_SIZE = 1024 * 1024; // 1MB + + // Get file size via HEAD request + const stats = await stat(filePath); + if (!stats || !stats.exists) { + console.error(`[webdav-client] Cannot compute hash - file not found: ${filePath}`); + return null; + } + + const fileSize = stats.size; + + // Create size buffer (64-bit big-endian) + const sizeBuffer = Buffer.allocUnsafe(8); + sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0); + + // Read sample data via range request + let sampleData: Buffer | null; + if (fileSize <= SAMPLE_SIZE) { + // Small file: read entire content + sampleData = await readFile(filePath); + } else { + // Large file: read middle 1MB + const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2); + sampleData = await readRange(filePath, middleOffset, middleOffset + SAMPLE_SIZE - 1); + } + + if (!sampleData) { + console.error(`[webdav-client] Failed to read sample data for hash: ${filePath}`); + return null; + } + + // Compute SHA-256 hash of [size + sample] + const hashInput = Buffer.concat([sizeBuffer, sampleData]); + const hashBuffer = createHash('sha256') + .update(hashInput) + .digest(); + + // Wrap in CID format with custom multicodec 0x1000 + const digest = create(CID_MIDHASH256_CODE, hashBuffer); + return CID.createV1(CID_MIDHASH256_CODE, digest).toString(); +} diff --git a/packages/meta-sort-core/src/webdav/index.ts b/packages/meta-sort-core/src/webdav/index.ts new file mode 100644 index 0000000..60d4759 --- /dev/null +++ b/packages/meta-sort-core/src/webdav/index.ts @@ -0,0 +1,7 @@ +/** + * WebDAV module for meta-sort + * + * Provides HTTP-based file access via WebDAV, replacing direct filesystem access. + */ + +export * from './WebdavClient.js';