new flat KV structure
This commit is contained in:
Pierre
@@ -29,8 +29,8 @@ class RedisStorage(StorageProvider):
|
||||
"""
|
||||
Redis storage provider.
|
||||
|
||||
Reads video metadata from Redis using the meta-sort key schema:
|
||||
- meta-sort:file:{hashId} -> Hash containing all metadata fields
|
||||
Reads video metadata from Redis using flat key schema:
|
||||
- file:{hashId}/{field} -> String value for each metadata property
|
||||
"""
|
||||
|
||||
def __init__(self, url: str = None, prefix: str = None):
|
||||
@@ -78,7 +78,7 @@ class RedisStorage(StorageProvider):
|
||||
"""
|
||||
Get the file path for a file by its CID.
|
||||
|
||||
Looks up file:{cid} in Redis and returns the path.
|
||||
Looks up file:{cid}/* in Redis and returns the path.
|
||||
Tries 'path' field first, then falls back to 'filePath'.
|
||||
This works for any file including poster images.
|
||||
|
||||
@@ -88,26 +88,41 @@ class RedisStorage(StorageProvider):
|
||||
return None
|
||||
|
||||
try:
|
||||
key = self._get_file_key(cid)
|
||||
prefix = self._get_file_key_prefix(cid)
|
||||
# Try 'path' field first (relative path)
|
||||
path = self._client.hget(key, 'path')
|
||||
path = self._client.get(f"{prefix}path")
|
||||
if path:
|
||||
return path
|
||||
return path if isinstance(path, str) else path.decode()
|
||||
# Fall back to 'filePath' (full path from meta-sort)
|
||||
file_path = self._client.hget(key, 'filePath')
|
||||
file_path = self._client.get(f"{prefix}filePath")
|
||||
if file_path:
|
||||
file_path_str = file_path if isinstance(file_path, str) else file_path.decode()
|
||||
# Convert absolute path to relative by removing /files/ prefix
|
||||
if file_path.startswith('/files/'):
|
||||
return file_path[7:] # Remove '/files/'
|
||||
return file_path
|
||||
if file_path_str.startswith('/files/'):
|
||||
return file_path_str[7:] # Remove '/files/'
|
||||
return file_path_str
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"[RedisStorage] Error getting path for CID {cid}: {e}")
|
||||
return None
|
||||
|
||||
def _get_file_key(self, hash_id: str) -> str:
|
||||
"""Get the Redis key for a file hash."""
|
||||
return f"{self._prefix}file:{hash_id}"
|
||||
def _get_file_key_prefix(self, hash_id: str) -> str:
|
||||
"""Get the Redis key prefix for a file's flat keys."""
|
||||
return f"{self._prefix}file:{hash_id}/"
|
||||
|
||||
def _get_file_metadata(self, hash_id: str) -> dict:
|
||||
"""Get all metadata for a file using flat key scan."""
|
||||
prefix = self._get_file_key_prefix(hash_id)
|
||||
data = {}
|
||||
|
||||
for key in self._client.scan_iter(f"{prefix}*", count=100):
|
||||
key_str = key if isinstance(key, str) else key.decode()
|
||||
field = key_str[len(prefix):]
|
||||
value = self._client.get(key)
|
||||
if value:
|
||||
data[field] = value if isinstance(value, str) else value.decode()
|
||||
|
||||
return data
|
||||
|
||||
def _parse_video(self, hash_id: str, data: dict) -> Optional[VideoMetadata]:
|
||||
"""Parse Redis hash data into VideoMetadata.
|
||||
@@ -280,30 +295,26 @@ class RedisStorage(StorageProvider):
|
||||
return []
|
||||
|
||||
videos = []
|
||||
pattern = f"{self._prefix}file:*"
|
||||
|
||||
try:
|
||||
for key in self._client.scan_iter(pattern, count=100):
|
||||
# Skip special keys (like file:__index__ which is a set, not a hash)
|
||||
if '__index__' in key or not key.startswith(f"{self._prefix}file:"):
|
||||
continue
|
||||
# Get all hash IDs from index
|
||||
index_key = f"{self._prefix}file:__index__"
|
||||
hash_ids = self._client.smembers(index_key)
|
||||
|
||||
# Check key type - only process hash keys
|
||||
key_type = self._client.type(key)
|
||||
if key_type != 'hash':
|
||||
continue
|
||||
for hash_id in hash_ids:
|
||||
hash_id_str = hash_id if isinstance(hash_id, str) else hash_id.decode()
|
||||
|
||||
# Get all metadata for this file
|
||||
data = self._get_file_metadata(hash_id_str)
|
||||
|
||||
# Check file type - only include video files
|
||||
file_type = self._client.hget(key, 'fileType')
|
||||
if file_type != 'video':
|
||||
if not data or data.get('fileType') != 'video':
|
||||
continue
|
||||
|
||||
# Extract hash_id from key
|
||||
hash_id = key.replace(f"{self._prefix}file:", "")
|
||||
data = self._client.hgetall(key)
|
||||
video = self._parse_video(hash_id, data)
|
||||
video = self._parse_video(hash_id_str, data)
|
||||
if video and video.file_path:
|
||||
videos.append(video)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[RedisStorage] Error getting all videos: {e}")
|
||||
|
||||
@@ -317,9 +328,8 @@ class RedisStorage(StorageProvider):
|
||||
return None
|
||||
|
||||
try:
|
||||
key = self._get_file_key(hash_id)
|
||||
data = self._client.hgetall(key)
|
||||
return self._parse_video(hash_id, data)
|
||||
data = self._get_file_metadata(hash_id)
|
||||
return self._parse_video(hash_id, data) if data else None
|
||||
except Exception as e:
|
||||
print(f"[RedisStorage] Error getting video {hash_id}: {e}")
|
||||
return None
|
||||
@@ -349,15 +359,19 @@ class RedisStorage(StorageProvider):
|
||||
imdb_id = f"tt{imdb_id}"
|
||||
|
||||
try:
|
||||
# Scan all videos looking for matching IMDB ID
|
||||
pattern = f"{self._prefix}file:*"
|
||||
for key in self._client.scan_iter(pattern, count=100):
|
||||
# Check imdbid/imdbId field
|
||||
stored_imdb = self._client.hget(key, 'imdbid') or self._client.hget(key, 'imdbId')
|
||||
if stored_imdb and stored_imdb.lower() == imdb_id:
|
||||
hash_id = key.replace(f"{self._prefix}file:", "")
|
||||
data = self._client.hgetall(key)
|
||||
return self._parse_video(hash_id, data)
|
||||
# Get from index and check each file's imdbid
|
||||
index_key = f"{self._prefix}file:__index__"
|
||||
for hash_id in self._client.smembers(index_key):
|
||||
hash_id_str = hash_id if isinstance(hash_id, str) else hash_id.decode()
|
||||
prefix = self._get_file_key_prefix(hash_id_str)
|
||||
|
||||
# Check imdbid field directly
|
||||
stored_imdb = self._client.get(f"{prefix}imdbid") or self._client.get(f"{prefix}imdbId")
|
||||
if stored_imdb:
|
||||
stored_imdb_str = stored_imdb if isinstance(stored_imdb, str) else stored_imdb.decode()
|
||||
if stored_imdb_str.lower() == imdb_id:
|
||||
data = self._get_file_metadata(hash_id_str)
|
||||
return self._parse_video(hash_id_str, data)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[RedisStorage] Error finding video by IMDB ID {imdb_id}: {e}")
|
||||
@@ -370,11 +384,9 @@ class RedisStorage(StorageProvider):
|
||||
return 0
|
||||
|
||||
try:
|
||||
pattern = f"{self._prefix}file:*"
|
||||
count = 0
|
||||
for _ in self._client.scan_iter(pattern, count=100):
|
||||
count += 1
|
||||
return count
|
||||
# Use index set for accurate count
|
||||
index_key = f"{self._prefix}file:__index__"
|
||||
return self._client.scard(index_key)
|
||||
except Exception as e:
|
||||
print(f"[RedisStorage] Error counting videos: {e}")
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user