meta-sort now use webdav
This commit is contained in:
@@ -16,6 +16,7 @@ import type { KVManager } from '../kv/KVManager.js';
|
|||||||
import type { ExtendedProcessingSnapshot, IKVClientWithPubSub } from '../types/ExtendedInterfaces.js';
|
import type { ExtendedProcessingSnapshot, IKVClientWithPubSub } from '../types/ExtendedInterfaces.js';
|
||||||
import { getErrorMessage } from '../types/ExtendedInterfaces.js';
|
import { getErrorMessage } from '../types/ExtendedInterfaces.js';
|
||||||
import { config } from '../config/EnvConfig.js';
|
import { config } from '../config/EnvConfig.js';
|
||||||
|
import * as webdav from '../webdav/WebdavClient.js';
|
||||||
import type { PluginManager } from '../plugin-engine/PluginManager.js';
|
import type { PluginManager } from '../plugin-engine/PluginManager.js';
|
||||||
import type { TaskScheduler } from '../plugin-engine/TaskScheduler.js';
|
import type { TaskScheduler } from '../plugin-engine/TaskScheduler.js';
|
||||||
import type { ContainerManager, ContainerPluginScheduler, PluginCallbackPayload, GateStatus } from '../container-plugins/index.js';
|
import type { ContainerManager, ContainerPluginScheduler, PluginCallbackPayload, GateStatus } from '../container-plugins/index.js';
|
||||||
@@ -237,7 +238,7 @@ export class UnifiedAPIServer {
|
|||||||
* Setup File download routes
|
* Setup File download routes
|
||||||
*/
|
*/
|
||||||
private setupFileRoutes(): void {
|
private setupFileRoutes(): void {
|
||||||
// Download file by path
|
// Download file by path (via WebDAV)
|
||||||
this.app.get<{ Querystring: { path: string } }>('/api/file/download', async (request, reply) => {
|
this.app.get<{ Querystring: { path: string } }>('/api/file/download', async (request, reply) => {
|
||||||
const { path: filePath } = request.query;
|
const { path: filePath } = request.query;
|
||||||
|
|
||||||
@@ -246,9 +247,13 @@ export class UnifiedAPIServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const fs = await import('fs');
|
|
||||||
const pathModule = await import('path');
|
const pathModule = await import('path');
|
||||||
|
|
||||||
|
// Check if WebDAV is configured
|
||||||
|
if (!webdav.isConfigured()) {
|
||||||
|
return reply.status(503).send({ error: 'WebDAV not configured - waiting for leader discovery' });
|
||||||
|
}
|
||||||
|
|
||||||
// Security: ensure the path is within FILES_PATH
|
// Security: ensure the path is within FILES_PATH
|
||||||
const filesPath = config.FILES_PATH;
|
const filesPath = config.FILES_PATH;
|
||||||
const absolutePath = pathModule.resolve(filePath);
|
const absolutePath = pathModule.resolve(filePath);
|
||||||
@@ -257,25 +262,23 @@ export class UnifiedAPIServer {
|
|||||||
return reply.status(403).send({ error: 'Access denied - path outside files directory' });
|
return reply.status(403).send({ error: 'Access denied - path outside files directory' });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
// Get file stats via WebDAV
|
||||||
if (!fs.existsSync(absolutePath)) {
|
const stats = await webdav.stat(absolutePath);
|
||||||
|
if (!stats || !stats.exists) {
|
||||||
return reply.status(404).send({ error: 'File not found' });
|
return reply.status(404).send({ error: 'File not found' });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file stats
|
|
||||||
const stats = fs.statSync(absolutePath);
|
|
||||||
if (!stats.isFile()) {
|
|
||||||
return reply.status(400).send({ error: 'Path is not a file' });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set headers for download
|
// Set headers for download
|
||||||
const filename = pathModule.basename(absolutePath);
|
const filename = pathModule.basename(absolutePath);
|
||||||
reply.header('Content-Disposition', `attachment; filename="${encodeURIComponent(filename)}"`);
|
reply.header('Content-Disposition', `attachment; filename="${encodeURIComponent(filename)}"`);
|
||||||
reply.header('Content-Type', 'application/octet-stream');
|
reply.header('Content-Type', 'application/octet-stream');
|
||||||
reply.header('Content-Length', stats.size);
|
reply.header('Content-Length', stats.size);
|
||||||
|
|
||||||
// Stream the file
|
// Stream the file from WebDAV
|
||||||
const stream = fs.createReadStream(absolutePath);
|
const stream = await webdav.createReadStream(absolutePath);
|
||||||
|
if (!stream) {
|
||||||
|
return reply.status(500).send({ error: 'Failed to create file stream' });
|
||||||
|
}
|
||||||
return reply.send(stream);
|
return reply.send(stream);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error downloading file:', error);
|
console.error('Error downloading file:', error);
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
import {readFile, writeFile} from 'fs/promises';
|
|
||||||
import {Builder, Parser} from 'xml2js';
|
import {Builder, Parser} from 'xml2js';
|
||||||
import {mergeMetadata} from "../logic/tool/MetadataMerger.js";
|
import {mergeMetadata} from "../logic/tool/MetadataMerger.js";
|
||||||
import {existsAsync} from "@metazla/meta-hash";
|
|
||||||
import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface";
|
import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface";
|
||||||
import lodash from 'lodash';
|
import lodash from 'lodash';
|
||||||
import * as xml2js from "xml2js";
|
import * as xml2js from "xml2js";
|
||||||
|
import * as webdav from '../webdav/WebdavClient.js';
|
||||||
|
|
||||||
export class JellyfinNfo {
|
export class JellyfinNfo {
|
||||||
|
|
||||||
@@ -61,9 +60,9 @@ export class JellyfinNfo {
|
|||||||
throw new Error(`NFO file path must end with '.nfo'`);
|
throw new Error(`NFO file path must end with '.nfo'`);
|
||||||
}
|
}
|
||||||
let metadata: T = {} as T;
|
let metadata: T = {} as T;
|
||||||
if (await existsAsync(nfoPath)) {
|
const nfoContent = await webdav.readTextFile(nfoPath);
|
||||||
|
if (nfoContent) {
|
||||||
try {
|
try {
|
||||||
const nfoContent = await readFile(nfoPath, {encoding: 'utf8'});
|
|
||||||
const parser = new xml2js.Parser(this.options);
|
const parser = new xml2js.Parser(this.options);
|
||||||
const result = await parser.parseStringPromise(nfoContent);
|
const result = await parser.parseStringPromise(nfoContent);
|
||||||
metadata = result.root || result.episodedetails || result.movie || {};
|
metadata = result.root || result.episodedetails || result.movie || {};
|
||||||
@@ -85,7 +84,7 @@ export class JellyfinNfo {
|
|||||||
if (data.fileType !== "video") {
|
if (data.fileType !== "video") {
|
||||||
return;//only process video files for writing jellyfin nfo
|
return;//only process video files for writing jellyfin nfo
|
||||||
}
|
}
|
||||||
if (await existsAsync(nfoPath)) {
|
if (await webdav.exists(nfoPath)) {
|
||||||
await this.updateNfoFile(nfoPath, data as any);
|
await this.updateNfoFile(nfoPath, data as any);
|
||||||
} else {
|
} else {
|
||||||
throw new Error(`NFO file does not exist at path: ${nfoPath}`);
|
throw new Error(`NFO file does not exist at path: ${nfoPath}`);
|
||||||
@@ -100,7 +99,7 @@ export class JellyfinNfo {
|
|||||||
if (data.fileType !== "video") {
|
if (data.fileType !== "video") {
|
||||||
return;//only process video files for writing jellyfin nfo
|
return;//only process video files for writing jellyfin nfo
|
||||||
}
|
}
|
||||||
if (await existsAsync(nfoPath)) {
|
if (await webdav.exists(nfoPath)) {
|
||||||
await this.updateNfoFile(nfoPath, data as any);
|
await this.updateNfoFile(nfoPath, data as any);
|
||||||
} else {
|
} else {
|
||||||
// If the NFO file doesn't exist, just generate a new one
|
// If the NFO file doesn't exist, just generate a new one
|
||||||
@@ -146,7 +145,11 @@ export class JellyfinNfo {
|
|||||||
|
|
||||||
private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
|
private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
|
||||||
try {
|
try {
|
||||||
const existingContent = await readFile(nfoFilePath, {encoding: "utf-8"});
|
const existingContent = await webdav.readTextFile(nfoFilePath);
|
||||||
|
if (!existingContent) {
|
||||||
|
console.error(`Failed to read NFO content from ${nfoFilePath}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const parser = new Parser(this.options);
|
const parser = new Parser(this.options);
|
||||||
let existingData = await parser.parseStringPromise(existingContent);
|
let existingData = await parser.parseStringPromise(existingContent);
|
||||||
let hasChanged = false;
|
let hasChanged = false;
|
||||||
@@ -184,7 +187,7 @@ export class JellyfinNfo {
|
|||||||
// Convert the updated object back to XML
|
// Convert the updated object back to XML
|
||||||
try {
|
try {
|
||||||
const updatedContent = builder.buildObject(existingData);
|
const updatedContent = builder.buildObject(existingData);
|
||||||
await writeFile(nfoFilePath, updatedContent, {encoding: "utf-8"});
|
await webdav.writeFile(nfoFilePath, updatedContent);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`);
|
console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`);
|
||||||
}
|
}
|
||||||
@@ -247,7 +250,7 @@ export class JellyfinNfo {
|
|||||||
if (!nfoFilePath.endsWith('.nfo')) {
|
if (!nfoFilePath.endsWith('.nfo')) {
|
||||||
throw new Error(`NFO file path must end with '.nfo'`);
|
throw new Error(`NFO file path must end with '.nfo'`);
|
||||||
}
|
}
|
||||||
if (await existsAsync(nfoFilePath)) {
|
if (await webdav.exists(nfoFilePath)) {
|
||||||
throw new Error(`NFO file already exists at path: ${nfoFilePath}`);
|
throw new Error(`NFO file already exists at path: ${nfoFilePath}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,7 +274,7 @@ export class JellyfinNfo {
|
|||||||
|
|
||||||
const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}});
|
const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}});
|
||||||
const nfoContent = builder.buildObject(obj);
|
const nfoContent = builder.buildObject(obj);
|
||||||
await writeFile(nfoFilePath, nfoContent, {encoding: "utf-8"});
|
await webdav.writeFile(nfoFilePath, nfoContent);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import { LeaderClient } from './LeaderClient.js';
|
|||||||
import { ServiceDiscovery } from './ServiceDiscovery.js';
|
import { ServiceDiscovery } from './ServiceDiscovery.js';
|
||||||
import { RedisKVClient } from './RedisClient.js';
|
import { RedisKVClient } from './RedisClient.js';
|
||||||
import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
|
import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
|
||||||
|
import { configure as configureWebdav } from '../webdav/WebdavClient.js';
|
||||||
|
|
||||||
interface KVManagerConfig {
|
interface KVManagerConfig {
|
||||||
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
||||||
@@ -122,6 +123,11 @@ export class KVManager {
|
|||||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||||
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||||
|
|
||||||
|
// Configure WebDAV client with leader's WebDAV URL
|
||||||
|
if (leaderInfo.webdavUrl) {
|
||||||
|
configureWebdav(leaderInfo.webdavUrl);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('[KVManager] Failed to connect to leader:', error);
|
console.error('[KVManager] Failed to connect to leader:', error);
|
||||||
throw error;
|
throw error;
|
||||||
@@ -164,6 +170,11 @@ export class KVManager {
|
|||||||
console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
|
console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||||
|
|
||||||
|
// Reconfigure WebDAV client with new leader's WebDAV URL
|
||||||
|
if (leaderInfo.webdavUrl) {
|
||||||
|
configureWebdav(leaderInfo.webdavUrl);
|
||||||
|
}
|
||||||
|
|
||||||
// Notify reconnect callbacks
|
// Notify reconnect callbacks
|
||||||
for (const callback of this.onReconnectCallbacks) {
|
for (const callback of this.onReconnectCallbacks) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
import { promises as fs } from 'fs';
|
|
||||||
import yaml from 'yaml';
|
import yaml from 'yaml';
|
||||||
|
import * as webdav from '../webdav/WebdavClient.js';
|
||||||
const { parse, stringify } = yaml;
|
const { parse, stringify } = yaml;
|
||||||
/**
|
/**
|
||||||
* File type:
|
* File type:
|
||||||
* filename.<ext>.meta
|
* filename.<ext>.meta
|
||||||
* filename.meta (contains multiple metadata entry "mkv", "str" etc...)
|
* filename.meta (contains multiple metadata entry "mkv", "str" etc...)
|
||||||
* folder.meta
|
* folder.meta
|
||||||
|
*
|
||||||
|
* All file operations use WebDAV for access to meta-core's file system.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export class MetaMeshFormat {
|
export class MetaMeshFormat {
|
||||||
@@ -14,15 +16,15 @@ export class MetaMeshFormat {
|
|||||||
throw new Error(`File path must end with '.meta'`);
|
throw new Error(`File path must end with '.meta'`);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
// Check if the file exists
|
// Check if the file exists via WebDAV
|
||||||
const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
|
const fileExists = await webdav.exists(metaDataFilePath);
|
||||||
|
|
||||||
if (fileExists) {
|
if (fileExists) {
|
||||||
// If the file exists, update it
|
// If the file exists, update it
|
||||||
await this.update(metaDataFilePath, data);
|
await this.update(metaDataFilePath, data);
|
||||||
} else {
|
} else {
|
||||||
// If the file does not exist, create it
|
// If the file does not exist, create it via WebDAV
|
||||||
await fs.writeFile(metaDataFilePath, stringify(data), 'utf8');
|
await webdav.writeFile(metaDataFilePath, stringify(data));
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error in createOrUpdate: ${error}`);
|
console.error(`Error in createOrUpdate: ${error}`);
|
||||||
@@ -30,18 +32,17 @@ export class MetaMeshFormat {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async read<T>(metaDataFilePath: string): Promise<T> {
|
async read<T>(metaDataFilePath: string): Promise<T | null> {
|
||||||
if(!metaDataFilePath.endsWith('.meta')){
|
if(!metaDataFilePath.endsWith('.meta')){
|
||||||
throw new Error(`File path must end with '.meta'`);
|
throw new Error(`File path must end with '.meta'`);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
|
// Read the YAML file via WebDAV
|
||||||
|
const fileContent = await webdav.readTextFile(metaDataFilePath);
|
||||||
|
|
||||||
if (!fileExists) {
|
if (!fileContent) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
// Read the YAML file
|
|
||||||
const fileContent = await fs.readFile(metaDataFilePath, 'utf8');
|
|
||||||
|
|
||||||
// Parse the YAML content to a JavaScript object
|
// Parse the YAML content to a JavaScript object
|
||||||
return parse(fileContent) as T;
|
return parse(fileContent) as T;
|
||||||
@@ -62,8 +63,8 @@ export class MetaMeshFormat {
|
|||||||
// Merge the existing data with the new data
|
// Merge the existing data with the new data
|
||||||
const updatedData = { ...existingData, ...data };
|
const updatedData = { ...existingData, ...data };
|
||||||
|
|
||||||
// Write the updated data back to the file
|
// Write the updated data back to the file via WebDAV
|
||||||
await fs.writeFile(metaDataFilePath, stringify(updatedData), 'utf8');
|
await webdav.writeFile(metaDataFilePath, stringify(updatedData));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error in update: ${error}`);
|
console.error(`Error in update: ${error}`);
|
||||||
throw error;
|
throw error;
|
||||||
@@ -75,13 +76,10 @@ export class MetaMeshFormat {
|
|||||||
throw new Error(`File path must end with '.meta'`);
|
throw new Error(`File path must end with '.meta'`);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
// Write the updated data back to the file
|
// Write the data to the file via WebDAV (overwrite)
|
||||||
await fs.writeFile(metaDataFilePath, stringify(data), {
|
await webdav.writeFile(metaDataFilePath, stringify(data));
|
||||||
encoding:'utf8',
|
|
||||||
flag:'w' // Write mode : Overwrite
|
|
||||||
});
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error in update: ${error}`);
|
console.error(`Error in write: ${error}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
import {promises as fs} from 'fs';
|
|
||||||
import {dirname, join, parse} from 'path';
|
import {dirname, join, parse} from 'path';
|
||||||
import {config} from "../config/EnvConfig.js";
|
import {config} from "../config/EnvConfig.js";
|
||||||
import {FileProcessorInterface} from "@metazla/meta-hash";
|
import {FileProcessorInterface} from "@metazla/meta-hash";
|
||||||
import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js";
|
import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js";
|
||||||
import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js";
|
import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js";
|
||||||
import {FileType} from "@metazla/filename-tools";
|
import {FileType} from "@metazla/filename-tools";
|
||||||
import {existsAsync} from "@metazla/meta-hash";
|
|
||||||
import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js";
|
import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js";
|
||||||
import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js";
|
import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js";
|
||||||
import {renamingRule} from "../config/RenamingRule.js";
|
import {renamingRule} from "../config/RenamingRule.js";
|
||||||
@@ -16,6 +14,7 @@ import {performanceMetrics} from "../metrics/PerformanceMetrics.js";
|
|||||||
import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js";
|
import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js";
|
||||||
import type {IKVClient} from "../kv/IKVClient.js";
|
import type {IKVClient} from "../kv/IKVClient.js";
|
||||||
import { hasStreamSupport, type ExtendedFileAnalyzer, type ExtendedHashMeta } from "../types/ExtendedInterfaces.js";
|
import { hasStreamSupport, type ExtendedFileAnalyzer, type ExtendedHashMeta } from "../types/ExtendedInterfaces.js";
|
||||||
|
import * as webdav from '../webdav/WebdavClient.js';
|
||||||
|
|
||||||
// Redis Streams for reliable event delivery
|
// Redis Streams for reliable event delivery
|
||||||
const EVENTS_STREAM = 'meta-sort:events';
|
const EVENTS_STREAM = 'meta-sort:events';
|
||||||
@@ -25,7 +24,6 @@ import os from 'os';
|
|||||||
import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js";
|
import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js";
|
||||||
import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js";
|
import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js";
|
||||||
import { from } from "@metazla/meta-interface";
|
import { from } from "@metazla/meta-interface";
|
||||||
import { stat } from "fs/promises";
|
|
||||||
import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js";
|
import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js";
|
||||||
|
|
||||||
export class WatchedFileProcessor implements FileProcessorInterface {
|
export class WatchedFileProcessor implements FileProcessorInterface {
|
||||||
@@ -313,8 +311,15 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// STEP 1: Compute midhash256 (file identifier)
|
// STEP 1: Compute midhash256 (file identifier)
|
||||||
const fileStats = await stat(filePath);
|
// Use WebDAV for file stats and hash computation
|
||||||
const { computeMidHash256 } = await import('@metazla/meta-hash');
|
const fileStats = await webdav.stat(filePath);
|
||||||
|
if (!fileStats || !fileStats.exists) {
|
||||||
|
console.error(`[WatchedFileProcessor] File not found via WebDAV: ${filePath}`);
|
||||||
|
if (this.unifiedStateManager) {
|
||||||
|
this.unifiedStateManager.completeLightProcessing(filePath, '', undefined, 'File not found');
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Check cache first
|
// Check cache first
|
||||||
const extAnalyzer = this.fileProcessor as ExtendedFileAnalyzer;
|
const extAnalyzer = this.fileProcessor as ExtendedFileAnalyzer;
|
||||||
@@ -330,7 +335,16 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
|||||||
performanceMetrics.recordCacheHit('midhash256');
|
performanceMetrics.recordCacheHit('midhash256');
|
||||||
} else {
|
} else {
|
||||||
const computeStart = performance.now();
|
const computeStart = performance.now();
|
||||||
midHash256 = await computeMidHash256(filePath);
|
// Use WebDAV-based hash computation
|
||||||
|
const hash = await webdav.computeMidHash256(filePath);
|
||||||
|
if (!hash) {
|
||||||
|
console.error(`[WatchedFileProcessor] Failed to compute hash for: ${filePath}`);
|
||||||
|
if (this.unifiedStateManager) {
|
||||||
|
this.unifiedStateManager.completeLightProcessing(filePath, '', undefined, 'Hash computation failed');
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
midHash256 = hash;
|
||||||
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
||||||
performanceMetrics.recordCacheMiss('midhash256');
|
performanceMetrics.recordCacheMiss('midhash256');
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,464 @@
|
|||||||
|
/**
|
||||||
|
* WebDAV Client for Meta-Sort
|
||||||
|
*
|
||||||
|
* Provides HTTP-based file access to meta-core's WebDAV server.
|
||||||
|
* Files are accessed exclusively via WebDAV - no local filesystem access.
|
||||||
|
*
|
||||||
|
* The WebDAV URL is discovered automatically from the leader via LeaderClient.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import { create } from 'multiformats/hashes/digest';
|
||||||
|
import { CID } from 'multiformats/cid';
|
||||||
|
|
||||||
|
// Configuration - set dynamically after leader discovery
|
||||||
|
let WEBDAV_URL: string = '';
|
||||||
|
const FILES_PATH = process.env.FILES_PATH || '/files';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configure the WebDAV client with a URL discovered from leader.
|
||||||
|
* Called after LeaderClient connects to the leader.
|
||||||
|
*/
|
||||||
|
export function configure(webdavUrl: string): void {
|
||||||
|
if (!webdavUrl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
WEBDAV_URL = webdavUrl.replace(/\/$/, '');
|
||||||
|
console.log(`[webdav-client] Configured: ${WEBDAV_URL}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if WebDAV client is configured.
|
||||||
|
*/
|
||||||
|
export function isConfigured(): boolean {
|
||||||
|
return Boolean(WEBDAV_URL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the configured WebDAV URL.
|
||||||
|
*/
|
||||||
|
export function getWebdavUrl(): string {
|
||||||
|
return WEBDAV_URL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a local file path to a WebDAV URL.
|
||||||
|
*
|
||||||
|
* @param filePath - Local file path (e.g., /files/watch/movie.mp4)
|
||||||
|
* @returns WebDAV URL or null if not configured
|
||||||
|
*/
|
||||||
|
export function toWebdavUrl(filePath: string): string | null {
|
||||||
|
if (!WEBDAV_URL) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Strip the FILES_PATH prefix to get relative path
|
||||||
|
let relativePath = filePath;
|
||||||
|
if (filePath.startsWith(FILES_PATH + '/')) {
|
||||||
|
relativePath = filePath.slice(FILES_PATH.length);
|
||||||
|
} else if (filePath.startsWith(FILES_PATH)) {
|
||||||
|
relativePath = filePath.slice(FILES_PATH.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure path starts with /
|
||||||
|
if (!relativePath.startsWith('/')) {
|
||||||
|
relativePath = '/' + relativePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
// URL-encode path segments (but not slashes)
|
||||||
|
const encodedPath = relativePath
|
||||||
|
.split('/')
|
||||||
|
.map(segment => encodeURIComponent(segment))
|
||||||
|
.join('/');
|
||||||
|
|
||||||
|
return WEBDAV_URL + encodedPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* File stats returned from WebDAV HEAD request.
|
||||||
|
*/
|
||||||
|
export interface WebdavFileStats {
|
||||||
|
size: number;
|
||||||
|
mtime: Date;
|
||||||
|
exists: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get file stats via HTTP HEAD request to WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns File stats or null on error
|
||||||
|
*/
|
||||||
|
export async function stat(filePath: string): Promise<WebdavFileStats | null> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot stat file');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'HEAD',
|
||||||
|
signal: AbortSignal.timeout(30000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status === 404) {
|
||||||
|
return { size: 0, mtime: new Date(0), exists: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
console.error(`[webdav-client] HEAD error for ${url}: ${response.status}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = response.headers.get('Content-Length');
|
||||||
|
const lastModified = response.headers.get('Last-Modified');
|
||||||
|
|
||||||
|
return {
|
||||||
|
size: contentLength ? parseInt(contentLength, 10) : 0,
|
||||||
|
mtime: lastModified ? new Date(lastModified) : new Date(),
|
||||||
|
exists: true
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] HEAD error for ${url}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if file exists via HTTP HEAD request to WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns True if file exists
|
||||||
|
*/
|
||||||
|
export async function exists(filePath: string): Promise<boolean> {
|
||||||
|
const stats = await stat(filePath);
|
||||||
|
return stats?.exists ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read entire file from WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns File contents as Buffer, or null on error
|
||||||
|
*/
|
||||||
|
export async function readFile(filePath: string): Promise<Buffer | null> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot read file');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
signal: AbortSignal.timeout(60000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
if (response.status !== 404) {
|
||||||
|
console.error(`[webdav-client] GET error for ${url}: ${response.status}`);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const arrayBuffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(arrayBuffer);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] GET error for ${url}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read file as text from WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @param encoding - Text encoding (default: utf-8)
|
||||||
|
* @returns File contents as string, or null on error
|
||||||
|
*/
|
||||||
|
export async function readTextFile(filePath: string, encoding: BufferEncoding = 'utf-8'): Promise<string | null> {
|
||||||
|
const buffer = await readFile(filePath);
|
||||||
|
return buffer?.toString(encoding) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read a byte range from a file via WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @param start - Start byte offset (inclusive)
|
||||||
|
* @param end - End byte offset (inclusive)
|
||||||
|
* @returns Bytes in the specified range, or null on error
|
||||||
|
*/
|
||||||
|
export async function readRange(filePath: string, start: number, end: number): Promise<Buffer | null> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot read range');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
'Range': `bytes=${start}-${end}`
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(60000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status !== 206) {
|
||||||
|
console.error(`[webdav-client] Range GET error for ${url}: ${response.status}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const arrayBuffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(arrayBuffer);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] Range GET error for ${url}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write file to WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @param content - File contents (Buffer or string)
|
||||||
|
* @returns True on success
|
||||||
|
*/
|
||||||
|
export async function writeFile(filePath: string, content: Buffer | string): Promise<boolean> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot write file');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const buffer = typeof content === 'string' ? Buffer.from(content, 'utf-8') : content;
|
||||||
|
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'PUT',
|
||||||
|
// Node.js fetch accepts Buffer directly at runtime
|
||||||
|
body: buffer as unknown as BodyInit,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/octet-stream',
|
||||||
|
'Content-Length': buffer.length.toString()
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(60000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status !== 201 && response.status !== 204) {
|
||||||
|
console.error(`[webdav-client] PUT error for ${url}: ${response.status}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] PUT error for ${url}:`, error);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete file from WebDAV.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns True on success
|
||||||
|
*/
|
||||||
|
export async function deleteFile(filePath: string): Promise<boolean> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot delete file');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'DELETE',
|
||||||
|
signal: AbortSignal.timeout(30000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status !== 204 && response.status !== 404) {
|
||||||
|
console.error(`[webdav-client] DELETE error for ${url}: ${response.status}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] DELETE error for ${url}:`, error);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create directory on WebDAV (MKCOL).
|
||||||
|
*
|
||||||
|
* @param dirPath - Directory path
|
||||||
|
* @returns True on success
|
||||||
|
*/
|
||||||
|
export async function mkdir(dirPath: string): Promise<boolean> {
|
||||||
|
const url = toWebdavUrl(dirPath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot create directory');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'MKCOL',
|
||||||
|
signal: AbortSignal.timeout(30000)
|
||||||
|
});
|
||||||
|
|
||||||
|
// 201 Created, 405 Method Not Allowed (already exists), or any 2xx
|
||||||
|
if (!response.ok && response.status !== 201 && response.status !== 405) {
|
||||||
|
console.error(`[webdav-client] MKCOL error for ${url}: ${response.status}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] MKCOL error for ${url}:`, error);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a readable stream from WebDAV file.
|
||||||
|
* Uses fetch with streaming response.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns Readable stream or null on error
|
||||||
|
*/
|
||||||
|
export async function createReadStream(filePath: string): Promise<Readable | null> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot create stream');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
signal: AbortSignal.timeout(300000) // 5 minutes for streaming
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
console.error(`[webdav-client] Stream GET error for ${url}: ${response.status}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.body) {
|
||||||
|
console.error(`[webdav-client] No response body for ${url}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert web ReadableStream to Node.js Readable
|
||||||
|
return Readable.fromWeb(response.body as any);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] Stream error for ${url}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a readable stream for a byte range from WebDAV file.
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @param start - Start byte offset
|
||||||
|
* @param end - End byte offset
|
||||||
|
* @returns Readable stream or null on error
|
||||||
|
*/
|
||||||
|
export async function createRangeStream(filePath: string, start: number, end: number): Promise<Readable | null> {
|
||||||
|
const url = toWebdavUrl(filePath);
|
||||||
|
if (!url) {
|
||||||
|
console.error('[webdav-client] Not configured, cannot create range stream');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
'Range': `bytes=${start}-${end}`
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(300000)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status !== 206) {
|
||||||
|
console.error(`[webdav-client] Range stream error for ${url}: ${response.status}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.body) {
|
||||||
|
console.error(`[webdav-client] No response body for range ${url}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Readable.fromWeb(response.body as any);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[webdav-client] Range stream error for ${url}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Multicodec for midhash256 (custom algorithm code)
|
||||||
|
*/
|
||||||
|
const CID_MIDHASH256_CODE = 0x1000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute midhash256 for a file via WebDAV.
|
||||||
|
*
|
||||||
|
* This is a WebDAV-aware version of the midhash256 algorithm that uses
|
||||||
|
* HTTP range requests to read only the required bytes instead of the entire file.
|
||||||
|
*
|
||||||
|
* Algorithm:
|
||||||
|
* - For files <= 1MB: Hash entire file content + size prefix
|
||||||
|
* - For files > 1MB: Hash middle 1MB + size prefix
|
||||||
|
*
|
||||||
|
* @param filePath - File path
|
||||||
|
* @returns CID string or null on error
|
||||||
|
*/
|
||||||
|
export async function computeMidHash256(filePath: string): Promise<string | null> {
|
||||||
|
const SAMPLE_SIZE = 1024 * 1024; // 1MB
|
||||||
|
|
||||||
|
// Get file size via HEAD request
|
||||||
|
const stats = await stat(filePath);
|
||||||
|
if (!stats || !stats.exists) {
|
||||||
|
console.error(`[webdav-client] Cannot compute hash - file not found: ${filePath}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileSize = stats.size;
|
||||||
|
|
||||||
|
// Create size buffer (64-bit big-endian)
|
||||||
|
const sizeBuffer = Buffer.allocUnsafe(8);
|
||||||
|
sizeBuffer.writeBigUInt64BE(BigInt(fileSize), 0);
|
||||||
|
|
||||||
|
// Read sample data via range request
|
||||||
|
let sampleData: Buffer | null;
|
||||||
|
if (fileSize <= SAMPLE_SIZE) {
|
||||||
|
// Small file: read entire content
|
||||||
|
sampleData = await readFile(filePath);
|
||||||
|
} else {
|
||||||
|
// Large file: read middle 1MB
|
||||||
|
const middleOffset = Math.floor((fileSize - SAMPLE_SIZE) / 2);
|
||||||
|
sampleData = await readRange(filePath, middleOffset, middleOffset + SAMPLE_SIZE - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!sampleData) {
|
||||||
|
console.error(`[webdav-client] Failed to read sample data for hash: ${filePath}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute SHA-256 hash of [size + sample]
|
||||||
|
const hashInput = Buffer.concat([sizeBuffer, sampleData]);
|
||||||
|
const hashBuffer = createHash('sha256')
|
||||||
|
.update(hashInput)
|
||||||
|
.digest();
|
||||||
|
|
||||||
|
// Wrap in CID format with custom multicodec 0x1000
|
||||||
|
const digest = create(CID_MIDHASH256_CODE, hashBuffer);
|
||||||
|
return CID.createV1(CID_MIDHASH256_CODE, digest).toString();
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/**
|
||||||
|
* WebDAV module for meta-sort
|
||||||
|
*
|
||||||
|
* Provides HTTP-based file access via WebDAV, replacing direct filesystem access.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export * from './WebdavClient.js';
|
||||||
Reference in New Issue
Block a user