metastar/meta-stremio
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Pierre
2026-01-11 17:27:01 +01:00
commit b9654c5b8d
20 changed files with 6635 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/storage/__init__.py
+33
View File
@@ -0,0 +1,33 @@
"""
Storage abstraction layer for meta-stremio.
Provides a unified interface for reading video metadata from different backends:
- LeaderStorage: Discovers and connects to the KV leader (recommended)
- RedisStorage: Reads from Redis with direct URL (legacy)
Also provides service discovery for inter-service communication.
"""
from .provider import StorageProvider, VideoMetadata
from .redis_storage import RedisStorage
from .leader_storage import LeaderStorage
from .leader_discovery import LeaderDiscovery, LeaderLockInfo
from .service_discovery import (
ServiceDiscovery,
ServiceInfo,
get_service_discovery,
init_service_discovery
)
__all__ = [
'StorageProvider',
'VideoMetadata',
'RedisStorage',
'LeaderStorage',
'LeaderDiscovery',
'LeaderLockInfo',
'ServiceDiscovery',
'ServiceInfo',
'get_service_discovery',
'init_service_discovery',
]
src/storage/leader_discovery.py
+300
View File
@@ -0,0 +1,300 @@
"""
Leader Discovery for meta-stremio (FOLLOWER-only)
This is a Python implementation of the leader discovery pattern used in meta-fuse.
meta-stremio never spawns Redis - it only connects to the leader's Redis instance.
Architecture:
- Reads info file at META_CORE_PATH/locks/kv-leader.info
- The leader holds flock on kv-leader.lock, writes metadata to kv-leader.info
- Parses leader info (Redis URL, HTTP URL)
- Watches for leader changes via polling and file watching
- Triggers reconnection callbacks on leader failure/change
"""
from __future__ import annotations
import os
import json
import time
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Callable
from urllib.request import urlopen
from urllib.error import URLError
# Try to import watchdog for file watching (optional)
try:
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
WATCHDOG_AVAILABLE = True
except ImportError:
WATCHDOG_AVAILABLE = False
Observer = None
FileSystemEventHandler = object
@dataclass
class LeaderLockInfo:
"""Leader lock file content format."""
host: str
api: str # Redis connection URL (e.g., redis://10.0.1.50:6379)
http: str # HTTP API URL for the leader service
timestamp: int
pid: int
class LockFileHandler(FileSystemEventHandler):
"""File system event handler for lock file changes."""
def __init__(self, callback: Callable[[], None], lock_filename: str):
self.callback = callback
self.lock_filename = lock_filename
def on_modified(self, event):
if event.src_path.endswith(self.lock_filename):
self.callback()
def on_created(self, event):
if event.src_path.endswith(self.lock_filename):
self.callback()
def on_deleted(self, event):
if event.src_path.endswith(self.lock_filename):
self.callback()
class LeaderDiscovery:
"""
Leader Discovery for meta-stremio.
Discovers and monitors the KV leader via a lock file in the shared filesystem.
"""
def __init__(
self,
meta_core_path: str,
health_check_interval: float = 5.0,
max_failures: int = 3
):
self.meta_core_path = meta_core_path
self.health_check_interval = health_check_interval
self.max_failures = max_failures
self.lock_file_path = os.path.join(meta_core_path, 'locks', 'kv-leader.info')
self.leader_info: Optional[LeaderLockInfo] = None
self._is_started = False
self._stop_event = threading.Event()
self._health_check_thread: Optional[threading.Thread] = None
self._file_observer = None
self._consecutive_failures = 0
# Event callbacks
self._on_leader_found_callback: Optional[Callable[[LeaderLockInfo], None]] = None
self._on_leader_lost_callback: Optional[Callable[[], None]] = None
def start(self) -> None:
"""Start watching for leader."""
if self._is_started:
print("[LeaderDiscovery] Already started")
return
print(f"[LeaderDiscovery] Starting, lock file: {self.lock_file_path}")
# Ensure lock directory exists
self._ensure_lock_dir()
# Try to read existing leader info
self._check_for_leader()
# Start file watcher (if watchdog is available)
self._start_file_watcher()
# Start health check thread
self._start_health_check()
self._is_started = True
def stop(self) -> None:
"""Stop watching."""
if not self._is_started:
return
print("[LeaderDiscovery] Stopping...")
self._stop_event.set()
# Stop health check thread
if self._health_check_thread and self._health_check_thread.is_alive():
self._health_check_thread.join(timeout=2.0)
# Stop file observer
if self._file_observer:
self._file_observer.stop()
self._file_observer.join(timeout=2.0)
self._file_observer = None
self._is_started = False
def _ensure_lock_dir(self) -> None:
"""Ensure lock directory exists."""
lock_dir = os.path.dirname(self.lock_file_path)
try:
os.makedirs(lock_dir, exist_ok=True)
except OSError as e:
print(f"[LeaderDiscovery] Could not create lock directory: {e}")
def _check_for_leader(self) -> None:
"""Check for leader by reading lock file."""
try:
with open(self.lock_file_path, 'r') as f:
content = f.read()
data = json.loads(content)
# Validate leader info
if data and data.get('api') and data.get('http'):
info = LeaderLockInfo(
host=data.get('host', 'unknown'),
api=data['api'],
http=data['http'],
timestamp=data.get('timestamp', 0),
pid=data.get('pid', 0)
)
is_new = (
self.leader_info is None or
self.leader_info.api != info.api
)
self.leader_info = info
self._consecutive_failures = 0
if is_new:
print(f"[LeaderDiscovery] Leader found: {info.host} at {info.api}")
if self._on_leader_found_callback:
self._on_leader_found_callback(info)
except FileNotFoundError:
# Lock file doesn't exist - no leader yet
if self.leader_info:
print("[LeaderDiscovery] Lock file removed, leader lost")
self._handle_leader_lost()
except json.JSONDecodeError as e:
print(f"[LeaderDiscovery] Error parsing lock file: {e}")
except Exception as e:
print(f"[LeaderDiscovery] Error reading lock file: {e}")
def _is_leader_healthy(self) -> bool:
"""Check if current leader is still healthy."""
if not self.leader_info:
return False
try:
health_url = f"{self.leader_info.http}/health"
with urlopen(health_url, timeout=5) as response:
return response.status == 200
except (URLError, TimeoutError, Exception):
return False
def _handle_leader_lost(self) -> None:
"""Handle leader being lost."""
self.leader_info = None
if self._on_leader_lost_callback:
self._on_leader_lost_callback()
def _start_file_watcher(self) -> None:
"""Start watching lock file for changes."""
if not WATCHDOG_AVAILABLE:
print("[LeaderDiscovery] watchdog not available, using polling only")
return
try:
lock_dir = os.path.dirname(self.lock_file_path)
handler = LockFileHandler(
callback=self._check_for_leader,
lock_filename='kv-leader.info'
)
self._file_observer = Observer()
self._file_observer.schedule(handler, lock_dir, recursive=False)
self._file_observer.start()
print(f"[LeaderDiscovery] File watcher started for {lock_dir}")
except Exception as e:
print(f"[LeaderDiscovery] Could not start file watcher: {e}")
def _start_health_check(self) -> None:
"""Start health check loop."""
def health_check_loop():
while not self._stop_event.is_set():
try:
if not self.leader_info:
# No leader known, try to find one
self._check_for_leader()
else:
# Check if current leader is healthy
healthy = self._is_leader_healthy()
if healthy:
self._consecutive_failures = 0
else:
self._consecutive_failures += 1
print(f"[LeaderDiscovery] Leader health check failed ({self._consecutive_failures}/{self.max_failures})")
if self._consecutive_failures >= self.max_failures:
print("[LeaderDiscovery] Leader appears dead, marking as lost")
self._handle_leader_lost()
# Try to find new leader
self._check_for_leader()
except Exception as e:
print(f"[LeaderDiscovery] Health check error: {e}")
self._stop_event.wait(self.health_check_interval)
self._health_check_thread = threading.Thread(
target=health_check_loop,
daemon=True,
name="leader-health-check"
)
self._health_check_thread.start()
# ========================================================================
# Event Registration
# ========================================================================
def on_leader_found(self, callback: Callable[[LeaderLockInfo], None]) -> 'LeaderDiscovery':
"""Register callback for when leader is found/changed."""
self._on_leader_found_callback = callback
# If we already have leader info, call immediately
if self.leader_info:
callback(self.leader_info)
return self
def on_leader_lost(self, callback: Callable[[], None]) -> 'LeaderDiscovery':
"""Register callback for when leader is lost."""
self._on_leader_lost_callback = callback
return self
# ========================================================================
# Getters
# ========================================================================
def get_leader_info(self) -> Optional[LeaderLockInfo]:
"""Get current leader info."""
return self.leader_info
def get_redis_url(self) -> Optional[str]:
"""Get Redis URL of current leader."""
return self.leader_info.api if self.leader_info else None
def has_leader(self) -> bool:
"""Check if a leader is known."""
return self.leader_info is not None
src/storage/leader_storage.py
+496
View File
@@ -0,0 +1,496 @@
"""
Leader-aware Storage Provider for meta-stremio.
Uses leader discovery to automatically connect to the KV leader.
Handles reconnection when the leader changes or becomes unavailable.
"""
from __future__ import annotations
import os
import threading
from typing import List, Optional, Callable
from .provider import StorageProvider, VideoMetadata
from .leader_discovery import LeaderDiscovery, LeaderLockInfo
try:
import redis
REDIS_AVAILABLE = True
except ImportError:
REDIS_AVAILABLE = False
redis = None
# Configuration from environment
META_CORE_PATH = os.environ.get('META_CORE_PATH', '/meta-core')
FILES_PATH = os.environ.get('FILES_PATH', '/files')
REDIS_PREFIX = os.environ.get('REDIS_PREFIX', 'meta-sort:')
class LeaderStorage(StorageProvider):
"""
Storage provider that discovers and connects to the KV leader.
Uses leader discovery to automatically find the Redis instance
managed by meta-sort. Handles reconnection on leader failure.
"""
def __init__(
self,
meta_core_path: str = None,
files_path: str = None,
prefix: str = None,
redis_url: str = None # Override - skip leader discovery
):
if not REDIS_AVAILABLE:
raise ImportError("redis package not installed. Run: pip install redis")
self._meta_core_path = meta_core_path or META_CORE_PATH
self._files_path = files_path or FILES_PATH
self._prefix = prefix or REDIS_PREFIX
self._override_url = redis_url # Direct Redis URL (skip leader discovery)
self._leader_discovery: Optional[LeaderDiscovery] = None
self._client: Optional[redis.Redis] = None
self._connected = False
self._lock = threading.Lock()
# Callbacks
self._on_ready_callbacks: List[Callable[[], None]] = []
self._on_disconnect_callbacks: List[Callable[[], None]] = []
def connect(self) -> None:
"""Connect to the storage backend via leader discovery."""
# If direct URL provided, skip leader discovery
if self._override_url:
print(f"[LeaderStorage] Using direct Redis URL: {self._override_url}")
self._connect_to_redis(self._override_url)
return
# Use leader discovery
self._leader_discovery = LeaderDiscovery(
meta_core_path=self._meta_core_path
)
# Set up callbacks
self._leader_discovery.on_leader_found(self._on_leader_found)
self._leader_discovery.on_leader_lost(self._on_leader_lost)
# Start discovery
self._leader_discovery.start()
def disconnect(self) -> None:
"""Disconnect from the storage backend."""
# Stop leader discovery
if self._leader_discovery:
self._leader_discovery.stop()
self._leader_discovery = None
# Disconnect Redis
self._disconnect_redis()
def is_connected(self) -> bool:
"""Check if connected to the storage backend."""
with self._lock:
if not self._connected or not self._client:
return False
try:
self._client.ping()
return True
except Exception:
self._connected = False
return False
def _on_leader_found(self, info: LeaderLockInfo) -> None:
"""Handle leader found event."""
print(f"[LeaderStorage] Connecting to leader at {info.api}...")
self._connect_to_redis(info.api)
def _on_leader_lost(self) -> None:
"""Handle leader lost event."""
print("[LeaderStorage] Leader lost, disconnecting...")
self._disconnect_redis()
self._notify_disconnect()
def _connect_to_redis(self, url: str) -> None:
"""Connect to Redis."""
with self._lock:
try:
# Disconnect existing client
self._disconnect_redis_unlocked()
# Create new client
self._client = redis.from_url(url, decode_responses=True)
self._client.ping() # Test connection
self._connected = True
print(f"[LeaderStorage] Connected to Redis at {url}")
# Notify ready
self._notify_ready()
except Exception as e:
print(f"[LeaderStorage] Failed to connect to Redis: {e}")
self._client = None
self._connected = False
def _disconnect_redis(self) -> None:
"""Disconnect from Redis."""
with self._lock:
self._disconnect_redis_unlocked()
def _disconnect_redis_unlocked(self) -> None:
"""Disconnect from Redis (without lock)."""
if self._client:
try:
self._client.close()
except Exception:
pass
self._client = None
self._connected = False
def _notify_ready(self) -> None:
"""Notify ready callbacks."""
for callback in self._on_ready_callbacks:
try:
callback()
except Exception as e:
print(f"[LeaderStorage] Error in ready callback: {e}")
def _notify_disconnect(self) -> None:
"""Notify disconnect callbacks."""
for callback in self._on_disconnect_callbacks:
try:
callback()
except Exception as e:
print(f"[LeaderStorage] Error in disconnect callback: {e}")
# ========================================================================
# Event Registration
# ========================================================================
def on_ready(self, callback: Callable[[], None]) -> 'LeaderStorage':
"""Register callback for when storage is ready."""
self._on_ready_callbacks.append(callback)
if self._connected:
callback()
return self
def on_disconnect(self, callback: Callable[[], None]) -> 'LeaderStorage':
"""Register callback for disconnection."""
self._on_disconnect_callbacks.append(callback)
return self
# ========================================================================
# Key Schema Helpers
# ========================================================================
def _get_file_key(self, hash_id: str) -> str:
"""Get the Redis key for a file hash."""
return f"{self._prefix}file:{hash_id}"
def _resolve_path(self, path: str) -> str:
"""Resolve path by prepending FILES_PATH if needed."""
if not path:
return path
if path.startswith(self._files_path):
return path
if path.startswith('/'):
return f"{self._files_path}{path}"
return f"{self._files_path}/{path}"
def _parse_nested_array(self, data: dict, prefix: str) -> list:
"""Parse nested array format (prefix/0, prefix/1, etc.) into a list."""
items = []
index = 0
while True:
key = f'{prefix}/{index}'
if key in data:
items.append(data[key])
index += 1
else:
break
return items
def _parse_video(self, hash_id: str, data: dict) -> Optional[VideoMetadata]:
"""Parse Redis hash data into VideoMetadata.
Handles both flat keys and nested keys (e.g., 'plot/eng' for description).
meta-sort stores data using a flattened nested key format.
"""
if not data:
return None
import json as jsonlib
# Parse JSON fields
audio_tracks = []
subtitles = []
genres = []
studios = []
if 'audioTracks' in data:
try:
audio_tracks = jsonlib.loads(data['audioTracks'])
except (jsonlib.JSONDecodeError, TypeError):
pass
if 'subtitles' in data:
try:
subtitles = jsonlib.loads(data['subtitles'])
except (jsonlib.JSONDecodeError, TypeError):
pass
# Parse genres - can be JSON array or nested keys (genres/0, genres/1, etc.)
if 'genres' in data:
try:
genres = jsonlib.loads(data['genres'])
except (jsonlib.JSONDecodeError, TypeError):
if isinstance(data['genres'], str):
genres = [g.strip() for g in data['genres'].split(',') if g.strip()]
else:
# Try nested key format (genres/0, genres/1, etc.)
genres = self._parse_nested_array(data, 'genres')
# Parse studios - nested keys (studio/0, studio/1, etc.)
studios = self._parse_nested_array(data, 'studio')
# Parse numeric fields
def parse_int(val):
try:
return int(val) if val else None
except (ValueError, TypeError):
return None
def parse_float(val):
try:
return float(val) if val else None
except (ValueError, TypeError):
return None
# Resolve file path with FILES_PATH
file_path = data.get('filePath', data.get('sourcePath', ''))
resolved_path = self._resolve_path(file_path)
# Normalize video type: 'tvshow' -> 'series' for Stremio compatibility
raw_type = data.get('type', data.get('videoType', 'movie'))
video_type = 'series' if raw_type == 'tvshow' else raw_type
# Get description from nested plot/eng or flat description/plot
description = data.get('description')
if not description:
description = data.get('plot')
if not description:
# Try nested format: plot/eng, plot/en, etc.
for lang in ['eng', 'en', 'english', 'und']:
description = data.get(f'plot/{lang}')
if description:
break
# Parse rating (stored as string by TMDBProcessor)
rating = parse_float(data.get('rating'))
# Parse release date
release_date = data.get('releasedate', data.get('releaseDate'))
# Parse episode title - try episodeTitle or extract from titles/eng
episode_title = data.get('episodeTitle', data.get('episodeName'))
if not episode_title:
episode_title = data.get('titles/eng', data.get('titles/en'))
# Parse tagline
tagline = data.get('tagline')
return VideoMetadata(
hash_id=hash_id,
file_path=resolved_path,
title=data.get('title', data.get('originalTitle', '')),
video_type=video_type,
year=parse_int(data.get('year', data.get('movieYear'))),
season=parse_int(data.get('season')),
episode=parse_int(data.get('episode')),
duration=parse_float(data.get('duration', data.get('fileinfo/duration'))),
width=parse_int(data.get('width', data.get('fileinfo/streamdetails/video/0/width'))),
height=parse_int(data.get('height', data.get('fileinfo/streamdetails/video/0/height'))),
video_codec=data.get('videoCodec', data.get('fileinfo/streamdetails/video/0/codec')),
audio_codec=data.get('audioCodec', data.get('fileinfo/streamdetails/audio/0/codec')),
container=data.get('container'),
file_size=parse_int(data.get('fileSize', data.get('sizeByte'))),
audio_tracks=audio_tracks,
subtitles=subtitles,
imdb_id=data.get('imdbId', data.get('imdbid')),
tmdb_id=data.get('tmdbId', data.get('tmdbid')),
poster=data.get('poster') or data.get('posterUrl'), # Fallback to posterUrl for backward compat
backdrop=data.get('backdrop') or data.get('backdropUrl'), # Fallback to backdropUrl
poster_path=data.get('posterPath'),
backdrop_path=data.get('backdropPath'),
description=description,
genres=genres,
episode_title=episode_title,
rating=rating,
release_date=release_date,
tagline=tagline,
studios=studios,
)
# ========================================================================
# StorageProvider Interface
# ========================================================================
def get_all_videos(self) -> List[VideoMetadata]:
"""Get all videos from storage."""
if not self.is_connected():
return []
videos = []
pattern = f"{self._prefix}file:*"
try:
for key in self._client.scan_iter(pattern, count=100):
# Skip index key
if '__index__' in key:
continue
# Check file type - only include video files
file_type = self._client.hget(key, 'fileType')
if file_type != 'video':
continue
hash_id = key.replace(f"{self._prefix}file:", "")
data = self._client.hgetall(key)
video = self._parse_video(hash_id, data)
if video and video.file_path:
videos.append(video)
except Exception as e:
print(f"[LeaderStorage] Error getting all videos: {e}")
# Sort by title
videos.sort(key=lambda v: v.title.lower())
return videos
def get_video_by_hash(self, hash_id: str) -> Optional[VideoMetadata]:
"""Get a video by its hash ID."""
if not self.is_connected():
return None
try:
key = self._get_file_key(hash_id)
data = self._client.hgetall(key)
return self._parse_video(hash_id, data)
except Exception as e:
print(f"[LeaderStorage] Error getting video {hash_id}: {e}")
return None
def get_videos_by_type(self, video_type: str) -> List[VideoMetadata]:
"""Get all videos of a specific type."""
videos = self.get_all_videos()
return [v for v in videos if v.video_type == video_type]
def search_videos(self, query: str) -> List[VideoMetadata]:
"""Search videos by title."""
if not query:
return self.get_all_videos()
query_lower = query.lower()
videos = self.get_all_videos()
return [v for v in videos if query_lower in v.title.lower()]
def get_video_by_imdb_id(self, imdb_id: str) -> Optional[VideoMetadata]:
"""Get a video by its IMDB ID (e.g., 'tt1727587')."""
if not self.is_connected() or not imdb_id:
return None
# Normalize IMDB ID format
imdb_id = imdb_id.lower()
if not imdb_id.startswith('tt'):
imdb_id = f"tt{imdb_id}"
try:
# Scan all videos looking for matching IMDB ID
pattern = f"{self._prefix}file:*"
for key in self._client.scan_iter(pattern, count=100):
if '__index__' in key:
continue
# Check imdbid/imdbId field
stored_imdb = self._client.hget(key, 'imdbid') or self._client.hget(key, 'imdbId')
if stored_imdb and stored_imdb.lower() == imdb_id:
hash_id = key.replace(f"{self._prefix}file:", "")
data = self._client.hgetall(key)
return self._parse_video(hash_id, data)
except Exception as e:
print(f"[LeaderStorage] Error finding video by IMDB ID {imdb_id}: {e}")
return None
def get_video_count(self) -> int:
"""Get total number of videos."""
if not self.is_connected():
return 0
try:
# Try index first
index_key = f"{self._prefix}file:__index__"
count = self._client.scard(index_key)
if count > 0:
return count
# Fallback to scanning
pattern = f"{self._prefix}file:*"
count = 0
for key in self._client.scan_iter(pattern, count=100):
if '__index__' not in key:
count += 1
return count
except Exception as e:
print(f"[LeaderStorage] Error counting videos: {e}")
return 0
def get_status(self) -> dict:
"""Get storage status for dashboard."""
leader_info = None
if self._leader_discovery:
info = self._leader_discovery.get_leader_info()
if info:
leader_info = {
'host': info.host,
'api': info.api,
'http': info.http,
}
status = {
'type': 'leader',
'connected': self.is_connected(),
'meta_core_path': self._meta_core_path,
'files_path': self._files_path,
'prefix': self._prefix,
'video_count': 0,
'leader': leader_info,
}
if status['connected']:
try:
status['video_count'] = self.get_video_count()
info = self._client.info('memory')
status['memory_used'] = info.get('used_memory_human', 'N/A')
except Exception:
pass
return status
# ========================================================================
# Pub/Sub (Future enhancement)
# ========================================================================
def on_video_added(self, callback: Callable[[VideoMetadata], None]) -> None:
"""Subscribe to video added events."""
# TODO: Implement pub/sub for real-time updates
pass
def on_video_removed(self, callback: Callable[[str], None]) -> None:
"""Subscribe to video removed events."""
# TODO: Implement pub/sub for real-time updates
pass
src/storage/provider.py
+148
View File
@@ -0,0 +1,148 @@
"""
Abstract StorageProvider interface for video metadata.
meta-stremio is a read-only consumer - it does NOT write metadata.
All writes are handled by meta-sort.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List, Optional, Callable
@dataclass
class VideoMetadata:
"""Video metadata stored in KV by meta-sort, read by meta-stremio."""
hash_id: str
file_path: str
title: str
video_type: str = 'movie' # 'movie' | 'series' | 'anime'
year: Optional[int] = None
season: Optional[int] = None
episode: Optional[int] = None
duration: Optional[float] = None
width: Optional[int] = None
height: Optional[int] = None
video_codec: Optional[str] = None
audio_codec: Optional[str] = None
container: Optional[str] = None
file_size: Optional[int] = None
audio_tracks: List[dict] = field(default_factory=list)
subtitles: List[dict] = field(default_factory=list)
imdb_id: Optional[str] = None
tmdb_id: Optional[str] = None
poster: Optional[str] = None
backdrop: Optional[str] = None
poster_path: Optional[str] = None
backdrop_path: Optional[str] = None
description: Optional[str] = None
genres: List[str] = field(default_factory=list)
# New fields for improved listings
episode_title: Optional[str] = None # Title specific to this episode
rating: Optional[float] = None # TMDB/IMDB rating (0-10)
release_date: Optional[str] = None # Full release date (YYYY-MM-DD)
tagline: Optional[str] = None # Movie/series tagline
studios: List[str] = field(default_factory=list) # Production studios
def to_dict(self) -> dict:
"""Convert to dictionary for JSON serialization."""
return {
'hash_id': self.hash_id,
'file_path': self.file_path,
'title': self.title,
'video_type': self.video_type,
'year': self.year,
'season': self.season,
'episode': self.episode,
'duration': self.duration,
'width': self.width,
'height': self.height,
'video_codec': self.video_codec,
'audio_codec': self.audio_codec,
'container': self.container,
'file_size': self.file_size,
'audio_tracks': self.audio_tracks,
'subtitles': self.subtitles,
'imdb_id': self.imdb_id,
'tmdb_id': self.tmdb_id,
'poster': self.poster,
'backdrop': self.backdrop,
'poster_path': self.poster_path,
'backdrop_path': self.backdrop_path,
'description': self.description,
'genres': self.genres,
'episode_title': self.episode_title,
'rating': self.rating,
'release_date': self.release_date,
'tagline': self.tagline,
'studios': self.studios,
}
class StorageProvider(ABC):
"""
Abstract interface for video metadata storage.
meta-stremio only reads - all writes are handled by meta-sort.
"""
@abstractmethod
def connect(self) -> None:
"""Connect to the storage backend."""
pass
@abstractmethod
def disconnect(self) -> None:
"""Disconnect from the storage backend."""
pass
@abstractmethod
def is_connected(self) -> bool:
"""Check if connected to the storage backend."""
pass
@abstractmethod
def get_all_videos(self) -> List[VideoMetadata]:
"""Get all videos from storage."""
pass
@abstractmethod
def get_video_by_hash(self, hash_id: str) -> Optional[VideoMetadata]:
"""Get a video by its hash ID."""
pass
@abstractmethod
def get_videos_by_type(self, video_type: str) -> List[VideoMetadata]:
"""Get all videos of a specific type (movie, series, anime)."""
pass
@abstractmethod
def search_videos(self, query: str) -> List[VideoMetadata]:
"""Search videos by title."""
pass
@abstractmethod
def get_video_by_imdb_id(self, imdb_id: str) -> Optional[VideoMetadata]:
"""Get a video by its IMDB ID (e.g., 'tt1727587')."""
pass
@abstractmethod
def get_video_count(self) -> int:
"""Get total number of videos in storage."""
pass
def on_video_added(self, callback: Callable[[VideoMetadata], None]) -> None:
"""Subscribe to video added events (optional)."""
pass
def on_video_removed(self, callback: Callable[[str], None]) -> None:
"""Subscribe to video removed events (optional)."""
pass
def get_status(self) -> dict:
"""Get storage status for dashboard."""
return {
'connected': self.is_connected(),
'video_count': self.get_video_count() if self.is_connected() else 0,
}
src/storage/redis_storage.py
+379
View File
@@ -0,0 +1,379 @@
"""
Redis storage provider for meta-stremio.
Reads video metadata from Redis (written by meta-sort).
Uses the same key schema as meta-sort for compatibility.
"""
from __future__ import annotations
import os
import json
from typing import List, Optional
try:
import redis
REDIS_AVAILABLE = True
except ImportError:
REDIS_AVAILABLE = False
redis = None
from .provider import StorageProvider, VideoMetadata
# Redis configuration
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379')
REDIS_PREFIX = os.environ.get('REDIS_PREFIX', 'meta-sort:')
class RedisStorage(StorageProvider):
"""
Redis storage provider.
Reads video metadata from Redis using the meta-sort key schema:
- meta-sort:file:{hashId} -> Hash containing all metadata fields
"""
def __init__(self, url: str = None, prefix: str = None):
if not REDIS_AVAILABLE:
raise ImportError("redis package not installed. Run: pip install redis")
self._url = url or REDIS_URL
self._prefix = prefix or REDIS_PREFIX
self._client: Optional[redis.Redis] = None
self._connected = False
def connect(self) -> None:
"""Connect to Redis."""
try:
self._client = redis.from_url(self._url, decode_responses=True)
# Test connection
self._client.ping()
self._connected = True
print(f"[RedisStorage] Connected to {self._url}")
except Exception as e:
self._connected = False
print(f"[RedisStorage] Failed to connect: {e}")
raise
def disconnect(self) -> None:
"""Disconnect from Redis."""
if self._client:
self._client.close()
self._client = None
self._connected = False
print("[RedisStorage] Disconnected")
def is_connected(self) -> bool:
"""Check if connected to Redis."""
if not self._connected or not self._client:
return False
try:
self._client.ping()
return True
except Exception:
self._connected = False
return False
def get_file_path_by_cid(self, cid: str) -> Optional[str]:
"""
Get the file path for a file by its CID.
Looks up file:{cid} in Redis and returns the path.
Tries 'path' field first, then falls back to 'filePath'.
This works for any file including poster images.
Returns the relative path if found, None otherwise.
"""
if not self.is_connected():
return None
try:
key = self._get_file_key(cid)
# Try 'path' field first (relative path)
path = self._client.hget(key, 'path')
if path:
return path
# Fall back to 'filePath' (full path from meta-sort)
file_path = self._client.hget(key, 'filePath')
if file_path:
# Convert absolute path to relative by removing /files/ prefix
if file_path.startswith('/files/'):
return file_path[7:] # Remove '/files/'
return file_path
return None
except Exception as e:
print(f"[RedisStorage] Error getting path for CID {cid}: {e}")
return None
def _get_file_key(self, hash_id: str) -> str:
"""Get the Redis key for a file hash."""
return f"{self._prefix}file:{hash_id}"
def _parse_video(self, hash_id: str, data: dict) -> Optional[VideoMetadata]:
"""Parse Redis hash data into VideoMetadata.
Handles both flat keys and nested keys (e.g., 'plot/eng' for description).
meta-sort stores data using a flattened nested key format.
"""
if not data:
return None
# Parse JSON fields
audio_tracks = []
subtitles = []
genres = []
studios = []
if 'audioTracks' in data:
try:
audio_tracks = json.loads(data['audioTracks'])
except (json.JSONDecodeError, TypeError):
pass
if 'subtitles' in data:
try:
subtitles = json.loads(data['subtitles'])
except (json.JSONDecodeError, TypeError):
pass
# Parse genres - can be JSON array or nested keys (genres/0, genres/1, etc.)
if 'genres' in data:
try:
genres = json.loads(data['genres'])
except (json.JSONDecodeError, TypeError):
if isinstance(data['genres'], str):
genres = [g.strip() for g in data['genres'].split(',') if g.strip()]
else:
# Try nested key format (genres/0, genres/1, etc.)
genres = self._parse_nested_array(data, 'genres')
# Parse studios - nested keys (studio/0, studio/1, etc.)
studios = self._parse_nested_array(data, 'studio')
# Parse numeric fields
def parse_int(val):
try:
return int(val) if val else None
except (ValueError, TypeError):
return None
def parse_float(val):
try:
return float(val) if val else None
except (ValueError, TypeError):
return None
# Get description from nested plot/eng or flat description/plot
description = data.get('description')
if not description:
description = data.get('plot')
if not description:
# Try nested format: plot/eng, plot/en, etc.
for lang in ['eng', 'en', 'english', 'und']:
description = data.get(f'plot/{lang}')
if description:
break
# Parse rating (stored as string by TMDBProcessor)
rating = parse_float(data.get('rating'))
# Parse release date
release_date = data.get('releasedate', data.get('releaseDate'))
# Parse episode title (for series episodes)
episode_title = data.get('episodeTitle', data.get('episodeName'))
# Parse tagline
tagline = data.get('tagline')
# Get video/audio codec and resolution from various field formats
# meta-sort stores these in fileinfo/streamdetails/video/0/... format
video_codec = data.get('videoCodec')
if not video_codec:
video_codec = data.get('fileinfo/streamdetails/video/0/codec')
audio_codec = data.get('audioCodec')
if not audio_codec:
audio_codec = data.get('fileinfo/streamdetails/audio/0/codec')
width = parse_int(data.get('width'))
if not width:
width = parse_int(data.get('fileinfo/streamdetails/video/0/width'))
height = parse_int(data.get('height'))
if not height:
height = parse_int(data.get('fileinfo/streamdetails/video/0/height'))
duration = parse_float(data.get('duration'))
if not duration:
duration = parse_float(data.get('fileinfo/duration'))
return VideoMetadata(
hash_id=hash_id,
file_path=data.get('filePath', ''),
title=data.get('title', data.get('originalTitle', '')),
video_type=data.get('type', data.get('videoType', 'movie')),
year=parse_int(data.get('year', data.get('movieYear'))),
season=parse_int(data.get('season')),
episode=parse_int(data.get('episode')),
duration=duration,
width=width,
height=height,
video_codec=video_codec,
audio_codec=audio_codec,
container=data.get('container'),
file_size=parse_int(data.get('fileSize', data.get('sizeByte'))),
audio_tracks=audio_tracks,
subtitles=subtitles,
imdb_id=data.get('imdbId', data.get('imdbid')),
tmdb_id=data.get('tmdbId', data.get('tmdbid')),
poster=data.get('poster') or data.get('posterUrl'), # Fallback to posterUrl for backward compat
backdrop=data.get('backdrop') or data.get('backdropUrl'), # Fallback to backdropUrl
poster_path=data.get('posterPath'),
backdrop_path=data.get('backdropPath'),
description=description,
genres=genres,
episode_title=episode_title,
rating=rating,
release_date=release_date,
tagline=tagline,
studios=studios,
)
def _parse_nested_array(self, data: dict, prefix: str) -> list:
"""Parse nested array format (prefix/0, prefix/1, etc.) into a list."""
items = []
index = 0
while True:
key = f'{prefix}/{index}'
if key in data:
items.append(data[key])
index += 1
else:
break
return items
def get_all_videos(self) -> List[VideoMetadata]:
"""Get all videos from Redis."""
if not self.is_connected():
return []
videos = []
pattern = f"{self._prefix}file:*"
try:
for key in self._client.scan_iter(pattern, count=100):
# Skip special keys (like file:__index__ which is a set, not a hash)
if '__index__' in key or not key.startswith(f"{self._prefix}file:"):
continue
# Check key type - only process hash keys
key_type = self._client.type(key)
if key_type != 'hash':
continue
# Check file type - only include video files
file_type = self._client.hget(key, 'fileType')
if file_type != 'video':
continue
# Extract hash_id from key
hash_id = key.replace(f"{self._prefix}file:", "")
data = self._client.hgetall(key)
video = self._parse_video(hash_id, data)
if video and video.file_path:
videos.append(video)
except Exception as e:
print(f"[RedisStorage] Error getting all videos: {e}")
# Sort by title
videos.sort(key=lambda v: v.title.lower())
return videos
def get_video_by_hash(self, hash_id: str) -> Optional[VideoMetadata]:
"""Get a video by its hash ID."""
if not self.is_connected():
return None
try:
key = self._get_file_key(hash_id)
data = self._client.hgetall(key)
return self._parse_video(hash_id, data)
except Exception as e:
print(f"[RedisStorage] Error getting video {hash_id}: {e}")
return None
def get_videos_by_type(self, video_type: str) -> List[VideoMetadata]:
"""Get all videos of a specific type."""
videos = self.get_all_videos()
return [v for v in videos if v.video_type == video_type]
def search_videos(self, query: str) -> List[VideoMetadata]:
"""Search videos by title."""
if not query:
return self.get_all_videos()
query_lower = query.lower()
videos = self.get_all_videos()
return [v for v in videos if query_lower in v.title.lower()]
def get_video_by_imdb_id(self, imdb_id: str) -> Optional[VideoMetadata]:
"""Get a video by its IMDB ID (e.g., 'tt1727587')."""
if not self.is_connected() or not imdb_id:
return None
# Normalize IMDB ID format
imdb_id = imdb_id.lower()
if not imdb_id.startswith('tt'):
imdb_id = f"tt{imdb_id}"
try:
# Scan all videos looking for matching IMDB ID
pattern = f"{self._prefix}file:*"
for key in self._client.scan_iter(pattern, count=100):
# Check imdbid/imdbId field
stored_imdb = self._client.hget(key, 'imdbid') or self._client.hget(key, 'imdbId')
if stored_imdb and stored_imdb.lower() == imdb_id:
hash_id = key.replace(f"{self._prefix}file:", "")
data = self._client.hgetall(key)
return self._parse_video(hash_id, data)
except Exception as e:
print(f"[RedisStorage] Error finding video by IMDB ID {imdb_id}: {e}")
return None
def get_video_count(self) -> int:
"""Get total number of videos."""
if not self.is_connected():
return 0
try:
pattern = f"{self._prefix}file:*"
count = 0
for _ in self._client.scan_iter(pattern, count=100):
count += 1
return count
except Exception as e:
print(f"[RedisStorage] Error counting videos: {e}")
return 0
def get_status(self) -> dict:
"""Get storage status for dashboard."""
status = {
'type': 'redis',
'connected': self.is_connected(),
'url': self._url,
'prefix': self._prefix,
'video_count': 0,
}
if status['connected']:
try:
status['video_count'] = self.get_video_count()
info = self._client.info('memory')
status['memory_used'] = info.get('used_memory_human', 'N/A')
except Exception:
pass
return status
src/storage/service_discovery.py
+367
View File
@@ -0,0 +1,367 @@
"""
Service Discovery for meta-stremio.
Python implementation of the service discovery pattern used in meta-sort.
Each service registers itself in /meta-core/services/{service-name}.json
and can discover other registered services.
Features:
- Service registration with heartbeat
- Discovery of all running services
- Stale service detection (60 second threshold)
- Automatic heartbeat loop
"""
from __future__ import annotations
import os
import json
import socket
import threading
import time
from dataclasses import dataclass, field, asdict
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Dict, Callable
from urllib.request import urlopen
from urllib.error import URLError
# Configuration
META_CORE_PATH = os.environ.get('META_CORE_PATH', '/meta-core')
SERVICE_NAME = os.environ.get('SERVICE_NAME', 'meta-stremio')
SERVICE_VERSION = os.environ.get('SERVICE_VERSION', '1.0.0')
BASE_URL = os.environ.get('BASE_URL', '')
@dataclass
class ServiceInfo:
"""Service registration info stored in JSON file."""
name: str
version: str
api: str # HTTP API URL (e.g., http://10.0.1.50:7000)
status: str = 'starting' # starting | running | stopping | stopped
pid: int = 0
hostname: str = ''
startedAt: str = ''
lastHeartbeat: str = ''
capabilities: List[str] = field(default_factory=list)
endpoints: Dict[str, str] = field(default_factory=dict)
def to_dict(self) -> dict:
"""Convert to dictionary for JSON serialization."""
return asdict(self)
class ServiceDiscovery:
"""
Service Discovery using shared filesystem.
Each service registers itself in /meta-core/services/{service-name}.json
and can discover other registered services.
"""
def __init__(
self,
meta_core_path: str = None,
service_name: str = None,
version: str = None,
api_url: str = None,
base_url: str = None,
capabilities: List[str] = None,
endpoints: Dict[str, str] = None,
heartbeat_interval: float = 30.0,
stale_threshold: float = 60.0
):
self.meta_core_path = meta_core_path or META_CORE_PATH
self.service_name = service_name or SERVICE_NAME
self.version = version or SERVICE_VERSION
# Use base_url if provided, then api_url, then env BASE_URL, then auto-detect
self.api_url = base_url or api_url or BASE_URL or self._get_default_api_url()
self.capabilities = capabilities or ['streaming', 'transcoding', 'hls']
self.endpoints = endpoints or {
'health': '/health',
'manifest': '/manifest.json',
'dashboard': '/',
'api': '/api',
}
self.heartbeat_interval = heartbeat_interval
self.stale_threshold = stale_threshold
self.services_dir = os.path.join(self.meta_core_path, 'services')
self.service_file_path = os.path.join(self.services_dir, f'{self.service_name}.json')
self._heartbeat_thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
self._is_started = False
def _get_default_api_url(self) -> str:
"""Get default API URL using local IP and PORT."""
port = os.environ.get('PORT', '7000')
ip = self._get_local_ip()
return f'http://{ip}:{port}'
def _get_local_ip(self) -> str:
"""Get the local machine's IP address."""
try:
# Create a socket to determine the outbound IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
s.close()
return ip
except Exception:
return socket.gethostname() or 'localhost'
def _ensure_services_dir(self) -> None:
"""Ensure services directory exists."""
Path(self.services_dir).mkdir(parents=True, exist_ok=True)
def _build_service_info(self, status: str = 'starting') -> ServiceInfo:
"""Build service info for this service."""
now = datetime.utcnow().isoformat() + 'Z'
return ServiceInfo(
name=self.service_name,
version=self.version,
api=self.api_url,
status=status,
pid=os.getpid(),
hostname=socket.gethostname(),
startedAt=now,
lastHeartbeat=now,
capabilities=self.capabilities,
endpoints=self.endpoints
)
def register(self) -> None:
"""Register this service."""
self._ensure_services_dir()
info = self._build_service_info('starting')
with open(self.service_file_path, 'w') as f:
json.dump(info.to_dict(), f, indent=2)
print(f'[ServiceDiscovery] Registered {self.service_name}')
def update_status(self, status: str) -> None:
"""Update service status."""
try:
with open(self.service_file_path, 'r') as f:
info = json.load(f)
info['status'] = status
info['lastHeartbeat'] = datetime.utcnow().isoformat() + 'Z'
with open(self.service_file_path, 'w') as f:
json.dump(info, f, indent=2)
except Exception as e:
print(f'[ServiceDiscovery] Failed to update status: {e}')
def heartbeat(self) -> None:
"""Send heartbeat (update lastHeartbeat timestamp)."""
try:
with open(self.service_file_path, 'r') as f:
info = json.load(f)
info['lastHeartbeat'] = datetime.utcnow().isoformat() + 'Z'
with open(self.service_file_path, 'w') as f:
json.dump(info, f, indent=2)
except FileNotFoundError:
# If file was deleted, re-register
self.register()
self.update_status('running')
except Exception as e:
print(f'[ServiceDiscovery] Heartbeat failed: {e}')
def _heartbeat_loop(self) -> None:
"""Background heartbeat loop."""
while not self._stop_event.is_set():
try:
self.heartbeat()
except Exception as e:
print(f'[ServiceDiscovery] Heartbeat error: {e}')
self._stop_event.wait(self.heartbeat_interval)
def start_heartbeat(self) -> None:
"""Start heartbeat loop in background thread."""
if self._heartbeat_thread and self._heartbeat_thread.is_alive():
return
self._stop_event.clear()
self._heartbeat_thread = threading.Thread(
target=self._heartbeat_loop,
daemon=True,
name='service-heartbeat'
)
self._heartbeat_thread.start()
def stop_heartbeat(self) -> None:
"""Stop heartbeat loop."""
self._stop_event.set()
if self._heartbeat_thread and self._heartbeat_thread.is_alive():
self._heartbeat_thread.join(timeout=2.0)
def unregister(self) -> None:
"""Unregister this service (on shutdown)."""
self.stop_heartbeat()
try:
self.update_status('stopped')
print(f'[ServiceDiscovery] Unregistered {self.service_name}')
except Exception as e:
print(f'[ServiceDiscovery] Failed to unregister: {e}')
# ========================================================================
# Discovery Methods
# ========================================================================
def _is_service_stale(self, info: dict) -> bool:
"""Check if a service is stale based on lastHeartbeat."""
try:
last_heartbeat_str = info.get('lastHeartbeat', '')
if not last_heartbeat_str:
return True
# Parse ISO timestamp
last_heartbeat_str = last_heartbeat_str.rstrip('Z')
last_heartbeat = datetime.fromisoformat(last_heartbeat_str)
age = (datetime.utcnow() - last_heartbeat).total_seconds()
return age > self.stale_threshold
except Exception:
return True
def discover_service(self, name: str) -> Optional[dict]:
"""Discover a service by name."""
service_path = os.path.join(self.services_dir, f'{name}.json')
try:
with open(service_path, 'r') as f:
info = json.load(f)
# Check if service is alive
if self._is_service_stale(info):
print(f'[ServiceDiscovery] Service {name} appears stale')
return None
return info
except FileNotFoundError:
return None
except Exception as e:
print(f'[ServiceDiscovery] Error discovering {name}: {e}')
return None
def discover_all_services(self) -> List[dict]:
"""Discover all registered services."""
services = []
try:
if not os.path.exists(self.services_dir):
return services
for filename in os.listdir(self.services_dir):
if not filename.endswith('.json'):
continue
name = filename.replace('.json', '')
service = self.discover_service(name)
if service and service.get('status') == 'running':
services.append(service)
except Exception as e:
print(f'[ServiceDiscovery] Error discovering services: {e}')
return services
def is_service_healthy(self, name: str) -> bool:
"""Check if a service is healthy via HTTP ping."""
service = self.discover_service(name)
if not service:
return False
health_endpoint = service.get('endpoints', {}).get('health', '/health')
url = f"{service['api']}{health_endpoint}"
try:
with urlopen(url, timeout=5) as response:
return response.status == 200
except Exception:
return False
# ========================================================================
# Lifecycle
# ========================================================================
def start(self) -> None:
"""Full startup sequence."""
if self._is_started:
return
# Try to register, but don't fail if filesystem is read-only
# This allows discovery-only mode when we can't write
can_register = os.access(self.services_dir, os.W_OK) if os.path.exists(self.services_dir) else False
if can_register:
try:
self.register()
self.update_status('running')
self.start_heartbeat()
print(f'[ServiceDiscovery] Started {self.service_name} at {self.api_url}')
except Exception as e:
print(f'[ServiceDiscovery] Registration failed (read-only?): {e}')
print(f'[ServiceDiscovery] Running in discovery-only mode')
else:
print(f'[ServiceDiscovery] Services directory not writable, running in discovery-only mode')
self._is_started = True
def stop(self) -> None:
"""Full shutdown sequence."""
if not self._is_started:
return
self.unregister()
self._is_started = False
# ========================================================================
# Getters
# ========================================================================
def get_service_file_path(self) -> str:
return self.service_file_path
def get_services_dir(self) -> str:
return self.services_dir
def is_started(self) -> bool:
return self._is_started
# Singleton instance
_service_discovery: Optional[ServiceDiscovery] = None
def get_service_discovery() -> ServiceDiscovery:
"""Get or create the service discovery singleton."""
global _service_discovery
if _service_discovery is None:
_service_discovery = ServiceDiscovery()
return _service_discovery
def init_service_discovery(
api_url: str = None,
base_url: str = None,
**kwargs
) -> ServiceDiscovery:
"""Initialize and start the service discovery singleton."""
global _service_discovery
_service_discovery = ServiceDiscovery(api_url=api_url, base_url=base_url, **kwargs)
_service_discovery.start()
return _service_discovery