meta-fuse update
This commit is contained in:
@@ -9,8 +9,9 @@ Meta-Fuse is a read-only process in the MetaMesh ecosystem that:
|
|||||||
1. **Connects to shared KV storage** - Reads metadata from Redis/compatible KV database managed by meta-sort
|
1. **Connects to shared KV storage** - Reads metadata from Redis/compatible KV database managed by meta-sort
|
||||||
2. **Mounts virtual filesystem** - Rust-based FUSE driver creates an organized view of media files
|
2. **Mounts virtual filesystem** - Rust-based FUSE driver creates an organized view of media files
|
||||||
3. **Serves WebDAV** - Network-accessible file sharing for Windows/Mac/Linux clients
|
3. **Serves WebDAV** - Network-accessible file sharing for Windows/Mac/Linux clients
|
||||||
4. **Zero file duplication** - All content points to original files in the shared DATA volume
|
4. **Zero file duplication** - All content points to original files via WebDAV or direct volume access
|
||||||
5. **Leader-aware KV access** - Discovers and connects to the active KV database via lock file
|
5. **Leader-aware KV access** - Discovers and connects to the active KV database via lock file
|
||||||
|
6. **WebDAV file access** - Can read files from meta-sort's WebDAV server (supports SMB/rclone mounts)
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
@@ -235,11 +236,15 @@ FUSE_ALLOW_OTHER=true # Allow other users
|
|||||||
PUID=1000 # User ID for files
|
PUID=1000 # User ID for files
|
||||||
PGID=1000 # Group ID for files
|
PGID=1000 # Group ID for files
|
||||||
|
|
||||||
# WebDAV
|
# WebDAV (serving files to clients)
|
||||||
WEBDAV_PORT=8080 # WebDAV server port
|
WEBDAV_PORT=8080 # WebDAV server port
|
||||||
WEBDAV_USERNAME=metamesh # WebDAV username
|
WEBDAV_USERNAME=metamesh # WebDAV username
|
||||||
WEBDAV_PASSWORD=metamesh # WebDAV password
|
WEBDAV_PASSWORD=metamesh # WebDAV password
|
||||||
WEBDAV_READONLY=true # Read-only mode
|
WEBDAV_READONLY=true # Read-only mode
|
||||||
|
|
||||||
|
# meta-sort WebDAV Access (reading files from meta-sort)
|
||||||
|
META_SORT_WEBDAV_URL=http://meta-sort-dev/webdav # URL to access files via meta-sort's WebDAV
|
||||||
|
# Enables access to SMB/rclone mounts
|
||||||
```
|
```
|
||||||
|
|
||||||
### Docker Compose
|
### Docker Compose
|
||||||
@@ -271,6 +276,8 @@ services:
|
|||||||
- FILES_VOLUME=/files
|
- FILES_VOLUME=/files
|
||||||
- FUSE_MOUNT_POINT=/mnt/virtual
|
- FUSE_MOUNT_POINT=/mnt/virtual
|
||||||
- REDIS_PREFIX=meta-sort:
|
- REDIS_PREFIX=meta-sort:
|
||||||
|
# Optional: Read files via meta-sort WebDAV (enables SMB/rclone mount access)
|
||||||
|
- META_SORT_WEBDAV_URL=http://meta-sort/webdav
|
||||||
```
|
```
|
||||||
|
|
||||||
## API Endpoints
|
## API Endpoints
|
||||||
|
|||||||
@@ -205,12 +205,17 @@ export class APIServer {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return source path for FUSE driver to read from
|
// Return source path and/or WebDAV URL for FUSE driver to read from
|
||||||
const response: Record<string, unknown> = {
|
const response: Record<string, unknown> = {
|
||||||
sourcePath: result.sourcePath,
|
sourcePath: result.sourcePath,
|
||||||
size: result.size,
|
size: result.size,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Include WebDAV URL if available (for remote file access)
|
||||||
|
if (result.webdavUrl) {
|
||||||
|
response.webdavUrl = result.webdavUrl;
|
||||||
|
}
|
||||||
|
|
||||||
// If content is provided (virtual file), encode as base64
|
// If content is provided (virtual file), encode as base64
|
||||||
if (result.content !== null) {
|
if (result.content !== null) {
|
||||||
response.content = result.content.toString('base64');
|
response.content = result.content.toString('base64');
|
||||||
|
|||||||
@@ -40,9 +40,16 @@ async function main(): Promise<void> {
|
|||||||
vfsRefreshInterval: parseInt(process.env.VFS_REFRESH_INTERVAL ?? '30000', 10),
|
vfsRefreshInterval: parseInt(process.env.VFS_REFRESH_INTERVAL ?? '30000', 10),
|
||||||
baseUrl: process.env.BASE_URL,
|
baseUrl: process.env.BASE_URL,
|
||||||
serviceVersion: process.env.SERVICE_VERSION ?? '1.0.0',
|
serviceVersion: process.env.SERVICE_VERSION ?? '1.0.0',
|
||||||
|
// WebDAV URL for meta-sort file access (e.g., http://meta-sort-dev/webdav)
|
||||||
|
metaSortWebdavUrl: process.env.META_SORT_WEBDAV_URL,
|
||||||
};
|
};
|
||||||
|
|
||||||
logger.info(`Config: META_CORE_PATH=${config.metaCorePath}, FILES_VOLUME=${config.filesPath}`);
|
logger.info(`Config: META_CORE_PATH=${config.metaCorePath}, FILES_VOLUME=${config.filesPath}`);
|
||||||
|
if (config.metaSortWebdavUrl) {
|
||||||
|
logger.info(`WebDAV: Using meta-sort WebDAV at ${config.metaSortWebdavUrl}`);
|
||||||
|
} else {
|
||||||
|
logger.info(`WebDAV: Not configured, FUSE driver will use local filesystem`);
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize KV Manager (handles leader discovery)
|
// Initialize KV Manager (handles leader discovery)
|
||||||
const kvManager = new KVManager({
|
const kvManager = new KVManager({
|
||||||
@@ -101,6 +108,8 @@ async function main(): Promise<void> {
|
|||||||
gid: parseInt(process.env.PGID ?? '1000', 10),
|
gid: parseInt(process.env.PGID ?? '1000', 10),
|
||||||
refreshInterval: config.vfsRefreshInterval,
|
refreshInterval: config.vfsRefreshInterval,
|
||||||
configDir: process.env.CONFIG_DIR ?? '/meta-fuse/config',
|
configDir: process.env.CONFIG_DIR ?? '/meta-fuse/config',
|
||||||
|
webdavBaseUrl: config.metaSortWebdavUrl,
|
||||||
|
filesPath: config.filesPath,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Start VFS
|
// Start VFS
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ export interface FileAttributes {
|
|||||||
|
|
||||||
export interface ReadResult {
|
export interface ReadResult {
|
||||||
sourcePath: string | null;
|
sourcePath: string | null;
|
||||||
|
webdavUrl: string | null;
|
||||||
content: Buffer | null;
|
content: Buffer | null;
|
||||||
size: number;
|
size: number;
|
||||||
}
|
}
|
||||||
@@ -54,6 +55,8 @@ export interface VFSConfig {
|
|||||||
gid?: number;
|
gid?: number;
|
||||||
refreshInterval?: number;
|
refreshInterval?: number;
|
||||||
configDir?: string; // Directory for config files (renaming rules, etc.)
|
configDir?: string; // Directory for config files (renaming rules, etc.)
|
||||||
|
webdavBaseUrl?: string | null; // meta-sort WebDAV URL for file access (e.g., http://meta-sort-dev/webdav)
|
||||||
|
filesPath?: string; // Local files path prefix to strip when building WebDAV URLs
|
||||||
}
|
}
|
||||||
|
|
||||||
export class VirtualFileSystem {
|
export class VirtualFileSystem {
|
||||||
@@ -92,6 +95,8 @@ export class VirtualFileSystem {
|
|||||||
gid: config.gid ?? 1000,
|
gid: config.gid ?? 1000,
|
||||||
refreshInterval: config.refreshInterval ?? 30000, // 30 seconds
|
refreshInterval: config.refreshInterval ?? 30000, // 30 seconds
|
||||||
configDir,
|
configDir,
|
||||||
|
webdavBaseUrl: config.webdavBaseUrl ?? null,
|
||||||
|
filesPath: config.filesPath ?? '/files',
|
||||||
};
|
};
|
||||||
|
|
||||||
// Initialize root directory
|
// Initialize root directory
|
||||||
@@ -517,7 +522,7 @@ export class VirtualFileSystem {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read file (returns source path for actual reading)
|
* Read file (returns source path and/or WebDAV URL for actual reading)
|
||||||
*/
|
*/
|
||||||
read(filepath: string): ReadResult | null {
|
read(filepath: string): ReadResult | null {
|
||||||
filepath = this.normalizePath(filepath);
|
filepath = this.normalizePath(filepath);
|
||||||
@@ -527,8 +532,30 @@ export class VirtualFileSystem {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compute WebDAV URL if configured
|
||||||
|
let webdavUrl: string | null = null;
|
||||||
|
if (this.config.webdavBaseUrl && node.sourcePath) {
|
||||||
|
// Convert sourcePath (e.g., /files/watch/movie.mkv) to WebDAV URL
|
||||||
|
// Strip filesPath prefix and append to webdavBaseUrl
|
||||||
|
let relativePath = node.sourcePath;
|
||||||
|
const filesPath = this.config.filesPath;
|
||||||
|
if (relativePath.startsWith(filesPath + '/')) {
|
||||||
|
relativePath = relativePath.substring(filesPath.length);
|
||||||
|
} else if (relativePath.startsWith(filesPath)) {
|
||||||
|
relativePath = relativePath.substring(filesPath.length);
|
||||||
|
}
|
||||||
|
// Ensure path starts with /
|
||||||
|
if (!relativePath.startsWith('/')) {
|
||||||
|
relativePath = '/' + relativePath;
|
||||||
|
}
|
||||||
|
// URL-encode the path components (but not the slashes)
|
||||||
|
const encodedPath = relativePath.split('/').map(encodeURIComponent).join('/');
|
||||||
|
webdavUrl = this.config.webdavBaseUrl.replace(/\/$/, '') + encodedPath;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
sourcePath: node.sourcePath!,
|
sourcePath: node.sourcePath!,
|
||||||
|
webdavUrl,
|
||||||
content: null,
|
content: null,
|
||||||
size: node.size!,
|
size: node.size!,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ pub struct FileAttributes {
|
|||||||
pub struct ReadResult {
|
pub struct ReadResult {
|
||||||
#[serde(rename = "sourcePath")]
|
#[serde(rename = "sourcePath")]
|
||||||
pub source_path: Option<String>,
|
pub source_path: Option<String>,
|
||||||
|
#[serde(rename = "webdavUrl")]
|
||||||
|
pub webdav_url: Option<String>,
|
||||||
pub content: Option<String>,
|
pub content: Option<String>,
|
||||||
#[serde(rename = "contentEncoding")]
|
#[serde(rename = "contentEncoding")]
|
||||||
pub content_encoding: Option<String>,
|
pub content_encoding: Option<String>,
|
||||||
|
|||||||
@@ -284,6 +284,7 @@ impl ApiFS {
|
|||||||
offset: usize,
|
offset: usize,
|
||||||
size: usize,
|
size: usize,
|
||||||
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
||||||
|
// Priority 1: Inline base64 content (for virtual files)
|
||||||
if let Some(ref content_b64) = read_result.content {
|
if let Some(ref content_b64) = read_result.content {
|
||||||
let content = base64::prelude::BASE64_STANDARD.decode(content_b64)?;
|
let content = base64::prelude::BASE64_STANDARD.decode(content_b64)?;
|
||||||
let end = std::cmp::min(offset + size, content.len());
|
let end = std::cmp::min(offset + size, content.len());
|
||||||
@@ -293,6 +294,12 @@ impl ApiFS {
|
|||||||
return Ok(content[offset..end].to_vec());
|
return Ok(content[offset..end].to_vec());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Priority 2: WebDAV URL (remote file access via HTTP Range requests)
|
||||||
|
if let Some(ref webdav_url) = read_result.webdav_url {
|
||||||
|
return self.read_from_webdav(webdav_url, offset, size, read_result.size);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Local filesystem (fallback)
|
||||||
if let Some(ref source_path) = read_result.source_path {
|
if let Some(ref source_path) = read_result.source_path {
|
||||||
let mut file = File::open(source_path)?;
|
let mut file = File::open(source_path)?;
|
||||||
|
|
||||||
@@ -308,7 +315,44 @@ impl ApiFS {
|
|||||||
return Ok(buffer);
|
return Ok(buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
Err("No content or source path available".into())
|
Err("No content, WebDAV URL, or source path available".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read file content from WebDAV URL using HTTP Range requests
|
||||||
|
fn read_from_webdav(
|
||||||
|
&self,
|
||||||
|
webdav_url: &str,
|
||||||
|
offset: usize,
|
||||||
|
size: usize,
|
||||||
|
file_size: u64,
|
||||||
|
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
||||||
|
// Don't request beyond the end of the file
|
||||||
|
if offset as u64 >= file_size {
|
||||||
|
return Ok(vec![]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate the actual range to request
|
||||||
|
let end = std::cmp::min(offset as u64 + size as u64 - 1, file_size - 1);
|
||||||
|
let range_header = format!("bytes={}-{}", offset, end);
|
||||||
|
|
||||||
|
debug!("WebDAV read: {} Range: {}", webdav_url, range_header);
|
||||||
|
|
||||||
|
// Create a new HTTP client for WebDAV requests (separate from API client)
|
||||||
|
let client = reqwest::blocking::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(60))
|
||||||
|
.build()?;
|
||||||
|
|
||||||
|
let response = client
|
||||||
|
.get(webdav_url)
|
||||||
|
.header("Range", range_header)
|
||||||
|
.send()?;
|
||||||
|
|
||||||
|
if response.status().is_success() || response.status() == reqwest::StatusCode::PARTIAL_CONTENT {
|
||||||
|
let bytes = response.bytes()?;
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
} else {
|
||||||
|
Err(format!("WebDAV request failed: {} - {}", response.status(), webdav_url).into())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user