new file retriving system

README.md

@@ -6,9 +6,10 @@ A standalone Stremio addon service for streaming personal media with on-the-fly
 
 Meta-Stremio is a **process-read** service that:
 - Reads file metadata from the shared KV store (Redis)
-- Reads media files from the shared DATA volume
+- Reads media files via WebDAV (from meta-sort) or direct volume access
 - Provides HLS streaming with adaptive real-time FFmpeg transcoding
 - Serves as a Stremio addon for seamless playback
+- Supports SMB/rclone mounts via meta-sort's WebDAV endpoint
 
 ```
 ┌─────────────────────────────────────────────────────────────────────┐
@@ -65,11 +66,13 @@ meta-stremio/
 ├── CLAUDE.md                   # Development guide
 ├── Dockerfile
 ├── docker-compose.yml
-├── requirements.txt            # Just redis
+├── requirements.txt            # redis + requests
 ├── src/
 │   ├── server.py               # HTTP server + routing
 │   ├── stremio.py              # Stremio handlers + storage integration
 │   ├── transcoder.py           # HLS transcoding engine
+│   ├── fileserver.py           # File serving (direct or WebDAV)
+│   ├── webdav_client.py        # WebDAV client for meta-sort file access
 │   └── storage/
 │       ├── __init__.py
 │       ├── provider.py         # Abstract StorageProvider
@@ -142,6 +145,9 @@ python server.py
 | `SEGMENT_DURATION` | `4` | HLS segment length (seconds) |
 | `PREFETCH_SEGMENTS` | `4` | Segments to prefetch ahead |
 | `SCHEME` | `auto` | URL scheme (`http`, `https`, `auto`) |
+| `META_SORT_WEBDAV_URL` | - | WebDAV URL for file access (e.g., `http://meta-sort/webdav`) |
+
+**WebDAV File Access**: When `META_SORT_WEBDAV_URL` is set, meta-stremio reads files via HTTP from meta-sort's WebDAV server instead of direct filesystem access. This enables access to SMB/rclone mounts that are dynamically mounted inside meta-sort.
 
 ## API Endpoints
 

requirements.txt

@@ -8,3 +8,6 @@ watchdog>=4.0.0
 
 # Image processing for poster resizing
 Pillow>=10.0.0
+
+# HTTP client for WebDAV access
+requests>=2.31.0

src/fileserver.py

@@ -15,6 +15,8 @@ import os
 import io
 from typing import Optional, Tuple
 
+import webdav_client
+
 # Optional PIL for image resizing
 try:
     from PIL import Image
@@ -113,7 +115,8 @@ def serve_file(cid: str, width: Optional[int] = None) -> Tuple[Optional[bytes],
     if not file_path:
         return None, 'text/plain', 404
 
-    if not os.path.exists(file_path):
+    # Check if file exists (works with both local and WebDAV)
+    if not webdav_client.file_exists(file_path):
         print(f"[FileServer] File not found: {file_path}")
         return None, 'text/plain', 404
 
@@ -133,8 +136,11 @@ def serve_file(cid: str, width: Optional[int] = None) -> Tuple[Optional[bytes],
     content_type = content_types.get(ext, 'application/octet-stream')
 
     try:
-        with open(file_path, 'rb') as f:
+        # Read file (works with both local and WebDAV)
-            file_data = f.read()
+        file_data = webdav_client.read_file(file_path)
+        if file_data is None:
+            print(f"[FileServer] Failed to read file: {file_path}")
+            return None, 'text/plain', 500
 
         # Resize if width is specified and it's an image
         if width and PIL_AVAILABLE and content_type.startswith('image/'):

src/server.py

@@ -39,6 +39,7 @@ import transcoder
 from storage import init_service_discovery, get_service_discovery
 import fileserver
 import plugin_subscriber
+import webdav_client
 
 # Configuration
 PORT = int(os.environ.get('PORT', '7000'))
@@ -721,7 +722,8 @@ class Handler(BaseHTTPRequestHandler):
 
         full_path = None
         for candidate in candidates:
-            if os.path.exists(candidate):
+            # Check if file exists (works with both local and WebDAV)
+            if webdav_client.file_exists(candidate):
                 full_path = candidate
                 break
 
@@ -758,9 +760,9 @@ class Handler(BaseHTTPRequestHandler):
             return
 
         video_codec, audio_codec = transcoder.extract_codecs(info)
-        transcoder.segment_manager.set_codec_info(video_codec, audio_codec, filepath)
+        transcoder.segment_manager.set_codec_info(video_codec, audio_codec, full_path)
 
-        data = transcoder.get_or_transcode_segment(filepath, file_hash, audio, resolution, segment, info)
+        data = transcoder.get_or_transcode_segment(full_path, file_hash, audio, resolution, segment, info)
         if data:
             self.send_data(data, 'video/mp2t')
         else:
@@ -797,7 +799,8 @@ class Handler(BaseHTTPRequestHandler):
 
         full_path = None
         for candidate in candidates:
-            if os.path.exists(candidate):
+            # Check if file exists (works with both local and WebDAV)
+            if webdav_client.file_exists(candidate):
                 full_path = candidate
                 break
 
@@ -818,7 +821,12 @@ class Handler(BaseHTTPRequestHandler):
         }
         content_type = content_types.get(ext, 'application/octet-stream')
 
-        file_size = os.path.getsize(full_path)
+        # Get file size (works with both local and WebDAV)
+        file_size = webdav_client.get_file_size(full_path)
+        if file_size is None:
+            self.send_error(500, f"Could not get file size: {filepath}")
+            return
+
         range_header = self.headers.get('Range')
 
         if range_header:
@@ -837,16 +845,9 @@ class Handler(BaseHTTPRequestHandler):
                 self.send_header('Access-Control-Allow-Origin', '*')
                 self.end_headers()
 
-                with open(full_path, 'rb') as f:
+                # Stream the range (works with both local and WebDAV)
-                    f.seek(start)
+                for chunk in webdav_client.stream_range(full_path, start, end, file_size):
-                    remaining = length
-                    chunk_size = 64 * 1024
-                    while remaining > 0:
-                        chunk = f.read(min(chunk_size, remaining))
-                        if not chunk:
-                            break
                     self.wfile.write(chunk)
-                        remaining -= len(chunk)
                 return
 
         self.send_response(200)
@@ -856,12 +857,8 @@ class Handler(BaseHTTPRequestHandler):
         self.send_header('Access-Control-Allow-Origin', '*')
         self.end_headers()
 
-        with open(full_path, 'rb') as f:
+        # Stream the entire file (works with both local and WebDAV)
-            chunk_size = 64 * 1024
+        for chunk in webdav_client.stream_file(full_path):
-            while True:
-                chunk = f.read(chunk_size)
-                if not chunk:
-                    break
             self.wfile.write(chunk)
 
     def get_base_url(self) -> str:

src/transcoder.py

@@ -22,6 +22,8 @@ import json
 import re
 from typing import Optional
 
+import webdav_client
+
 # Configuration
 MEDIA_DIR = os.environ.get('MEDIA_DIR', '/data/media')
 CACHE_DIR = os.environ.get('CACHE_DIR', '/data/cache')
@@ -456,6 +458,10 @@ class SubtitleManager:
             print(f"[Subtitle {sub_index}] Extracting from {os.path.basename(filepath)}...")
             is_mkv = filepath.lower().endswith(('.mkv', '.mka', '.mks'))
 
+            # Use WebDAV URL if available (ffmpeg can read HTTP URLs)
+            ffmpeg_input = webdav_client.to_webdav_url(filepath) or filepath
+            use_webdav = webdav_client.WEBDAV_ENABLED and ffmpeg_input.startswith('http')
+
             # Determine intermediate format based on codec
             codec_ext_map = {
                 'subrip': 'srt', 'srt': 'srt',
@@ -467,8 +473,8 @@ class SubtitleManager:
 
             extracted = False
 
-            # Method 1: Try mkvextract for MKV files (fastest)
+            # Method 1: Try mkvextract for MKV files (fastest, but requires local file)
-            if is_mkv and track_index is not None:
+            if is_mkv and track_index is not None and not use_webdav:
                 try:
                     result = subprocess.run(
                         ['mkvextract', 'tracks', filepath, f'{track_index}:{intermediate_file}'],
@@ -483,12 +489,12 @@ class SubtitleManager:
                 except Exception as e:
                     print(f"[Subtitle {sub_index}] mkvextract failed: {e}, trying ffmpeg")
 
-            # Method 2: FFmpeg copy codec (fast demux, no re-encoding)
+            # Method 2: FFmpeg copy codec (fast demux, no re-encoding, works with WebDAV)
             if not extracted:
                 result = subprocess.run(
                     ['ffmpeg', '-y', '-hide_banner', '-loglevel', 'error',
                      '-probesize', '1M', '-analyzeduration', '1M',
-                     '-i', filepath,
+                     '-i', ffmpeg_input,
                      '-map', f'0:s:{sub_index}',
                      '-c:s', 'copy', intermediate_file],
                     capture_output=True,
@@ -496,7 +502,8 @@ class SubtitleManager:
                 )
                 if result.returncode == 0 and os.path.exists(intermediate_file):
                     extracted = True
-                    print(f"[Subtitle {sub_index}] Extracted via ffmpeg copy")
+                    mode = "WebDAV" if use_webdav else "ffmpeg copy"
+                    print(f"[Subtitle {sub_index}] Extracted via {mode}")
 
             # Convert to WebVTT if needed
             if extracted and os.path.exists(intermediate_file):
@@ -567,9 +574,13 @@ def get_file_hash(filepath: str) -> str:
 def get_video_info(filepath: str) -> dict | None:
     """Get video metadata using ffprobe."""
     full_path = os.path.join(MEDIA_DIR, filepath) if not filepath.startswith('/') else filepath
+
+    # Use WebDAV URL if available (ffprobe can read HTTP URLs)
+    ffprobe_input = webdav_client.to_webdav_url(full_path) or full_path
+
     try:
         result = subprocess.run(
-            ['ffprobe', '-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams', full_path],
+            ['ffprobe', '-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams', ffprobe_input],
             capture_output=True, text=True, timeout=30
         )
         return json.loads(result.stdout) if result.returncode == 0 else None
@@ -618,6 +629,10 @@ def transcode_segment(filepath: str, file_hash: str, audio: int, resolution: str
         return output
 
     full_path = os.path.join(MEDIA_DIR, filepath) if not filepath.startswith('/') else filepath
+
+    # Use WebDAV URL if available (ffmpeg can read HTTP URLs)
+    ffmpeg_input = webdav_client.to_webdav_url(full_path) or full_path
+
     start_offset = segment * SEGMENT_DURATION
     res_preset = RESOLUTIONS.get(resolution, RESOLUTIONS['original'])
     width, height, base_crf = res_preset
@@ -632,7 +647,7 @@ def transcode_segment(filepath: str, file_hash: str, audio: int, resolution: str
         'ffmpeg', '-y', '-hide_banner', '-loglevel', 'error',
         '-threads', str(cpu_count),
         '-ss', str(start_offset),
-        '-i', full_path,
+        '-i', ffmpeg_input,
         '-t', str(SEGMENT_DURATION),
         '-map', '0:v:0',
         '-map', f'0:a:{audio}?',  # ? = optional, don't fail if audio track doesn't exist

src/webdav_client.py

@@ -0,0 +1,248 @@
+"""
+WebDAV Client for Meta-Stremio
+
+Provides HTTP-based file access to meta-sort's WebDAV server.
+This allows meta-stremio to access files without direct volume mounts.
+
+Environment Variables:
+- META_SORT_WEBDAV_URL: Base URL for WebDAV access (e.g., http://meta-sort/webdav)
+- FILES_PATH: Local files path prefix to strip when building WebDAV URLs (default: /files)
+"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Optional, Generator
+from urllib.parse import quote
+
+import requests
+
+# Configuration
+WEBDAV_URL = os.environ.get('META_SORT_WEBDAV_URL', '').rstrip('/')
+FILES_PATH = os.environ.get('FILES_PATH', '/files')
+
+# Check if WebDAV is enabled
+WEBDAV_ENABLED = bool(WEBDAV_URL)
+
+if WEBDAV_ENABLED:
+    print(f"[WebDAV] Using meta-sort WebDAV at {WEBDAV_URL}")
+else:
+    print("[WebDAV] Not configured, using direct filesystem access")
+
+
+def to_webdav_url(file_path: str) -> Optional[str]:
+    """
+    Convert a local file path to a WebDAV URL.
+
+    Args:
+        file_path: Local file path (e.g., /files/watch/movie.mp4)
+
+    Returns:
+        WebDAV URL or None if WebDAV is not configured
+    """
+    if not WEBDAV_ENABLED:
+        return None
+
+    # Strip the FILES_PATH prefix to get relative path
+    relative_path = file_path
+    if file_path.startswith(FILES_PATH + '/'):
+        relative_path = file_path[len(FILES_PATH):]
+    elif file_path.startswith(FILES_PATH):
+        relative_path = file_path[len(FILES_PATH):]
+
+    # Ensure path starts with /
+    if not relative_path.startswith('/'):
+        relative_path = '/' + relative_path
+
+    # URL-encode path segments (but not slashes)
+    encoded_path = '/'.join(quote(segment, safe='') for segment in relative_path.split('/'))
+
+    return WEBDAV_URL + encoded_path
+
+
+def get_file_size(file_path: str) -> Optional[int]:
+    """
+    Get file size via HTTP HEAD request to WebDAV.
+
+    Args:
+        file_path: Local file path
+
+    Returns:
+        File size in bytes, or None on error
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        try:
+            return os.path.getsize(file_path)
+        except OSError:
+            return None
+
+    try:
+        response = requests.head(url, timeout=30)
+        if response.status_code == 200:
+            return int(response.headers.get('Content-Length', 0))
+        return None
+    except Exception as e:
+        print(f"[WebDAV] HEAD error for {url}: {e}")
+        return None
+
+
+def file_exists(file_path: str) -> bool:
+    """
+    Check if file exists via HTTP HEAD request to WebDAV.
+
+    Args:
+        file_path: Local file path
+
+    Returns:
+        True if file exists
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        return os.path.exists(file_path)
+
+    try:
+        response = requests.head(url, timeout=30)
+        return response.status_code == 200
+    except Exception:
+        return False
+
+
+def read_file(file_path: str) -> Optional[bytes]:
+    """
+    Read entire file from WebDAV.
+
+    Args:
+        file_path: Local file path
+
+    Returns:
+        File contents as bytes, or None on error
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        try:
+            with open(file_path, 'rb') as f:
+                return f.read()
+        except OSError:
+            return None
+
+    try:
+        response = requests.get(url, timeout=60)
+        if response.status_code == 200:
+            return response.content
+        return None
+    except Exception as e:
+        print(f"[WebDAV] GET error for {url}: {e}")
+        return None
+
+
+def read_range(file_path: str, start: int, end: int) -> Optional[bytes]:
+    """
+    Read a byte range from a file via WebDAV.
+
+    Args:
+        file_path: Local file path
+        start: Start byte offset (inclusive)
+        end: End byte offset (inclusive)
+
+    Returns:
+        Bytes in the specified range, or None on error
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        try:
+            with open(file_path, 'rb') as f:
+                f.seek(start)
+                return f.read(end - start + 1)
+        except OSError:
+            return None
+
+    try:
+        headers = {'Range': f'bytes={start}-{end}'}
+        response = requests.get(url, headers=headers, timeout=60)
+        if response.status_code in (200, 206):
+            return response.content
+        return None
+    except Exception as e:
+        print(f"[WebDAV] Range GET error for {url}: {e}")
+        return None
+
+
+def stream_file(file_path: str, chunk_size: int = 64 * 1024) -> Generator[bytes, None, None]:
+    """
+    Stream file content from WebDAV.
+
+    Args:
+        file_path: Local file path
+        chunk_size: Size of chunks to yield
+
+    Yields:
+        File content in chunks
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        try:
+            with open(file_path, 'rb') as f:
+                while True:
+                    chunk = f.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield chunk
+        except OSError:
+            return
+    else:
+        try:
+            response = requests.get(url, stream=True, timeout=300)
+            if response.status_code == 200:
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        yield chunk
+        except Exception as e:
+            print(f"[WebDAV] Stream error for {url}: {e}")
+
+
+def stream_range(file_path: str, start: int, end: int, file_size: int,
+                 chunk_size: int = 64 * 1024) -> Generator[bytes, None, None]:
+    """
+    Stream a byte range from a file via WebDAV.
+
+    Args:
+        file_path: Local file path
+        start: Start byte offset (inclusive)
+        end: End byte offset (inclusive)
+        file_size: Total file size (for local fallback)
+        chunk_size: Size of chunks to yield
+
+    Yields:
+        File content in chunks
+    """
+    url = to_webdav_url(file_path)
+    if not url:
+        # Fall back to local filesystem
+        try:
+            with open(file_path, 'rb') as f:
+                f.seek(start)
+                remaining = end - start + 1
+                while remaining > 0:
+                    chunk = f.read(min(chunk_size, remaining))
+                    if not chunk:
+                        break
+                    yield chunk
+                    remaining -= len(chunk)
+        except OSError:
+            return
+    else:
+        try:
+            headers = {'Range': f'bytes={start}-{end}'}
+            response = requests.get(url, headers=headers, stream=True, timeout=300)
+            if response.status_code in (200, 206):
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        yield chunk
+        except Exception as e:
+            print(f"[WebDAV] Stream range error for {url}: {e}")