From 888790fd11fb767f4125bda1b42da8e4de0d523f Mon Sep 17 00:00:00 2001 From: Pierre Date: Sun, 11 Jan 2026 17:26:49 +0100 Subject: [PATCH] initial commit --- .dockerignore | 6 + .github/workflows/docker-publish.yml | 68 + .gitignore | 29 + Dockerfile | 205 + README.md | 841 ++ docker/nginx.conf | 91 + docker/redis.conf | 30 + docker/supervisord.conf | 69 + package.json | 16 + packages/async-utils/LICENSE | 21 + packages/async-utils/README.md | 96 + packages/async-utils/package.json | 37 + packages/async-utils/src/Id.ts | 10 + packages/async-utils/src/Lazy.ts | 2 + packages/async-utils/src/ListenerCleaner.ts | 23 + packages/async-utils/src/MultiQueue.ts | 66 + packages/async-utils/src/PromiseQueue.ts | 80 + packages/async-utils/src/index.ts | 5 + packages/async-utils/tsconfig.json | 25 + packages/async-utils/tsup.config.js | 11 + packages/meta-sort-core/.env.example | 102 + packages/meta-sort-core/package.json | 53 + .../meta-sort-core/src/api/DuplicatesAPI.ts | 166 + packages/meta-sort-core/src/api/FuseAPI.ts | 173 + .../meta-sort-core/src/api/FuseAPIServer.ts | 345 + packages/meta-sort-core/src/api/MetricsAPI.ts | 160 + packages/meta-sort-core/src/api/README.md | 299 + .../src/api/UnifiedAPIServer.ts | 2145 +++++ .../src/api/VirtualFileSystem.ts | 627 ++ packages/meta-sort-core/src/api/index.ts | 11 + .../meta-sort-core/src/config/EnvConfig.ts | 216 + .../meta-sort-core/src/config/RenamingRule.ts | 62 + .../src/config/SupportedFileTypes.ts | 68 + .../meta-sort-core/src/config/TargetHash.ts | 21 + .../src/container-plugins/ConfigParser.ts | 372 + .../src/container-plugins/ContainerManager.ts | 771 ++ .../ContainerPluginScheduler.ts | 651 ++ .../src/container-plugins/DockerClient.ts | 417 + .../src/container-plugins/index.ts | 34 + .../src/container-plugins/types.ts | 364 + packages/meta-sort-core/src/index.ts | 308 + .../src/jellyfin/JellyfinAPI.ts | 97 + .../src/jellyfin/JellyfinNfo.ts | 278 + packages/meta-sort-core/src/jellyfin/README | 10 + .../meta-sort-core/src/jellyfin/nfoTool.ts | 22 + packages/meta-sort-core/src/kv/IKVClient.ts | 192 + packages/meta-sort-core/src/kv/KVManager.ts | 388 + .../meta-sort-core/src/kv/LeaderElection.ts | 615 ++ .../meta-sort-core/src/kv/MetadataUtils.ts | 274 + packages/meta-sort-core/src/kv/RedisClient.ts | 351 + .../meta-sort-core/src/kv/ServiceDiscovery.ts | 312 + packages/meta-sort-core/src/kv/index.ts | 18 + .../src/logic/DuplicateFinder.ts | 113 + .../src/logic/HashProcessingStateManager.ts | 143 + .../src/logic/MetaDataToFolderStruct.ts | 75 + .../src/logic/MetaMeshFormat.ts | 95 + .../src/logic/MetadataFormatRegistry.ts | 217 + .../src/logic/ProcessingStateManager.ts | 127 + .../logic/UnifiedProcessingStateManager.ts | 354 + .../src/logic/WatchedFileProcessor.ts | 620 ++ .../fileProcessor/FileAnalyzerInterface.ts | 37 + .../src/logic/fileProcessor/FileProcessor.ts | 92 + .../fileProcessor/FileProcessorInternal.ts | 84 + .../fileProcessor/FileProcessorPiscina.ts | 594 ++ .../logic/fileProcessor/SimpleFileToCid.ts | 31 + .../src/logic/fileProcessor/worker.ts | 20 + .../src/logic/pipeline/PipelineConfig.ts | 80 + .../src/logic/pipeline/StreamingPipeline.ts | 512 ++ .../src/logic/pipeline/index.ts | 2 + .../src/logic/tool/MetadataMerger.ts | 36 + .../src/metrics/PerformanceMetrics.ts | 582 ++ .../src/plugin-engine/PluginContext.ts | 392 + .../src/plugin-engine/PluginExecutor.ts | 146 + .../src/plugin-engine/PluginManager.ts | 406 + .../src/plugin-engine/TaskScheduler.ts | 634 ++ .../meta-sort-core/src/plugin-engine/index.ts | 59 + .../src/plugin-engine/plugin.test.ts | 667 ++ .../meta-sort-core/src/plugin-engine/types.ts | 429 + .../src/plugin-engine/validate.js | 210 + .../meta-sort-core/src/types/FileMetadata.ts | 21 + .../src/utils/TempIdGenerator.ts | 29 + .../src/utils/TimeoutWrapper.ts | 76 + packages/meta-sort-core/tsconfig.json | 16 + packages/meta-sort-editor/README.md | 140 + packages/meta-sort-editor/eslint.config.js | 28 + packages/meta-sort-editor/index.html | 13 + packages/meta-sort-editor/package-lock.json | 1570 ++++ packages/meta-sort-editor/package.json | 23 + packages/meta-sort-editor/src/App.css | 185 + packages/meta-sort-editor/src/App.tsx | 246 + .../meta-sort-editor/src/api/metadataApi.ts | 151 + .../src/components/BulkMetadataEditor.css | 382 + .../src/components/BulkMetadataEditor.tsx | 280 + .../src/components/DynamicFieldRenderer.css | 297 + .../src/components/DynamicFieldRenderer.tsx | 378 + .../src/components/DynamicPluginSection.css | 63 + .../src/components/DynamicPluginSection.tsx | 126 + .../src/components/FileList.css | 174 + .../src/components/FileList.tsx | 123 + .../src/components/KVTreeView.css | 247 + .../src/components/KVTreeView.tsx | 216 + .../src/components/LanguageCombobox.css | 149 + .../src/components/LanguageCombobox.tsx | 196 + .../src/components/MetadataEditor.css | 438 + .../src/components/MetadataEditor.tsx | 392 + .../src/components/SearchBar.css | 52 + .../src/components/SearchBar.tsx | 40 + packages/meta-sort-editor/src/index.css | 90 + packages/meta-sort-editor/src/main.tsx | 10 + packages/meta-sort-editor/src/types/index.ts | 155 + .../meta-sort-editor/src/utils/formatting.ts | 53 + packages/meta-sort-editor/src/vite-env.d.ts | 1 + packages/meta-sort-editor/tsconfig.json | 25 + .../meta-sort-editor/tsconfig.tsbuildinfo | 1 + packages/meta-sort-editor/vite.config.ts | 22 + packages/meta-sort-ui/index.html | 26 + packages/meta-sort-ui/package.json | 23 + packages/meta-sort-ui/public/favicon.svg | 5 + packages/meta-sort-ui/src/App.tsx | 34 + .../src/components/ServiceNav.tsx | 164 + packages/meta-sort-ui/src/index.css | 168 + packages/meta-sort-ui/src/main.tsx | 13 + .../meta-sort-ui/src/pages/Duplicates.tsx | 510 ++ packages/meta-sort-ui/src/pages/Monitor.tsx | 980 +++ packages/meta-sort-ui/src/pages/Mounts.tsx | 601 ++ packages/meta-sort-ui/src/pages/Plugins.tsx | 1261 +++ packages/meta-sort-ui/src/pages/Welcome.tsx | 140 + packages/meta-sort-ui/src/types/index.ts | 217 + packages/meta-sort-ui/src/utils/format.ts | 94 + packages/meta-sort-ui/tsconfig.json | 21 + packages/meta-sort-ui/tsconfig.node.json | 10 + packages/meta-sort-ui/vite.config.ts | 20 + packages/shared/filename-tool/.gitattributes | 6 + packages/shared/filename-tool/.gitignore | 29 + packages/shared/filename-tool/LICENSE | 21 + packages/shared/filename-tool/README.md | 147 + .../shared/filename-tool/package-lock.json | 3170 ++++++++ packages/shared/filename-tool/package.json | 45 + .../src/lib/FileNameMetaExtractor.ts | 110 + .../src/lib/FileNameVideoMetaExtractor.ts | 342 + .../shared/filename-tool/src/lib/FileType.ts | 10 + .../filename-tool/src/lib/SimpleFileType.ts | 9 + .../src/lib/config/AnimeKeywords.ts | 28 + .../src/lib/config/ExtentionsMapping.ts | 67 + .../src/lib/config/KeywordsArray.ts | 37 + .../src/lib/config/MimeTypeMapping.ts | 42 + .../lib/config/SeasonAndEpisodePatterns.ts | 47 + .../shared/filename-tool/src/lib/index.ts | 19 + .../src/lib/tools/FileNameCleaner.ts | 177 + .../src/lib/tools/FileNameStringOperation.ts | 25 + .../src/lib/tools/FileTypeConfigurable.ts | 77 + .../src/lib/tools/StringArrayOperation.ts | 52 + .../src/lib/tools/TwoStringsOperation.ts | 80 + .../src/lib/tools/folder/FolderTool.ts | 26 + .../src/lib/tools/lang/LanguageTool.ts | 43 + .../tools/video/SerieFilesNamaAnalysisTool.ts | 209 + .../src/tests/FileNameMetaExtractor.test.ts | 15 + .../tests/FileNameVideoMetaExtractor.test.ts | 241 + .../filename-tool/src/tests/main.test.ts | 7 + packages/shared/filename-tool/tsconfig.json | 22 + packages/shared/filename-tool/tsup.config.js | 15 + .../shared/filename-tool/vitest.config.ts | 11 + packages/shared/meta-hash/.gitattributes | 6 + packages/shared/meta-hash/.gitignore | 31 + packages/shared/meta-hash/LICENSE | 21 + packages/shared/meta-hash/README.md | 175 + packages/shared/meta-hash/package.json | 57 + packages/shared/meta-hash/pnpm-lock.yaml | 2341 ++++++ .../src/lib/file-id/BrowserHasher.ts | 49 + .../meta-hash/src/lib/file-id/BtihV2Hasher.ts | 200 + .../meta-hash/src/lib/file-id/CodeTable.ts | 12 + .../meta-hash/src/lib/file-id/ComputeHash.ts | 50 + .../meta-hash/src/lib/file-id/Crc32Hash.ts | 18 + .../meta-hash/src/lib/file-id/CreateHasher.ts | 31 + .../src/lib/file-id/CreateHasherBrowser.ts | 31 + .../meta-hash/src/lib/file-id/FastHash.ts | 104 + .../lib/file-id/FileIDComputerInterface.ts | 5 + .../src/lib/file-id/FileIDComputerWorker.ts | 31 + .../src/lib/file-id/MagnetLinkGenerator.ts | 183 + .../src/lib/file-id/ShaComputeWorker.ts | 19 + .../meta-hash/src/lib/file-id/SimpleHash.ts | 5 + .../folder-watcher/FileProcessorInterface.ts | 50 + .../src/lib/folder-watcher/FolderWatcher.ts | 170 + .../src/lib/folder-watcher/PollingWatcher.ts | 168 + .../src/lib/hash-compute/HashComputer.ts | 30 + .../src/lib/hash-compute/HashComputerFile.ts | 32 + .../hash-compute/HashComputerIndexCache.ts | 37 + .../lib/hash-compute/HashComputerWorker.ts | 29 + .../src/lib/hash-compute/HashIndexManager.ts | 305 + .../src/lib/hash-compute/MultiHashData.ts | 51 + .../shared/meta-hash/src/lib/index-browser.ts | 4 + .../meta-hash/src/lib/index-interface.ts | 3 + packages/shared/meta-hash/src/lib/index.ts | 20 + .../meta-hash/src/lib/utils/ExistsAsync.ts | 12 + .../shared/meta-hash/src/tests/btihv2.test.ts | 236 + .../meta-hash/src/tests/magnet-link.test.ts | 175 + .../shared/meta-hash/src/tests/main.test.ts | 97 + packages/shared/meta-hash/src/tests/test.txt | 10 + packages/shared/meta-hash/tsconfig.json | 30 + packages/shared/meta-hash/tsup.config.js | 17 + packages/shared/meta-hash/vitest.config.ts | 17 + packages/shared/meta-interface/.gitattributes | 6 + packages/shared/meta-interface/.gitignore | 29 + packages/shared/meta-interface/.mocharc.json | 4 + packages/shared/meta-interface/LICENSE | 21 + packages/shared/meta-interface/README.md | 176 + packages/shared/meta-interface/package.json | 40 + packages/shared/meta-interface/pnpm-lock.yaml | 1637 ++++ .../meta-interface/src/lib/MetaData.test.ts | 50 + .../meta-interface/src/lib/MetaDataAPI.ts | 115 + .../meta-interface/src/lib/MetaDataName.ts | 74 + .../shared/meta-interface/src/lib/Readme.md | 1 + .../shared/meta-interface/src/lib/index.ts | 26 + .../src/lib/metadata-type/AnimeMeta.ts | 9 + .../src/lib/metadata-type/DebugMeta.ts | 7 + .../src/lib/metadata-type/FileNameMeta.ts | 9 + .../src/lib/metadata-type/FileStatMeta.ts | 12 + .../src/lib/metadata-type/GenreMeta.ts | 9 + .../src/lib/metadata-type/HashMeta.ts | 32 + .../src/lib/metadata-type/JellyfinMeta.ts | 64 + .../src/lib/metadata-type/LanguagesMeta.ts | 10 + .../src/lib/metadata-type/MediaStreamMeta.ts | 59 + .../src/lib/metadata-type/SubtitleMeta.ts | 23 + .../src/lib/metadata-type/TagsMeta.ts | 9 + .../src/lib/metadata-type/TorrentMeta.ts | 25 + .../src/lib/metadata-type/VersionMeta.ts | 7 + .../src/lib/metadata-type/VideoMeta.ts | 24 + .../src/lib/metadata-type/basic/ArrayKey.ts | 6 + .../src/lib/metadata-type/basic/FileRef.ts | 4 + .../lib/metadata-type/basic/LanguageData.ts | 12 + .../src/lib/metadata-type/basic/RecordSet.ts | 2 + .../lib/metadata-type/basic/SimpleFileType.ts | 2 + .../lib/metadata-type/basic/StringBoolean.ts | 1 + .../src/lib/metadata-type/basic/VideoType.ts | 1 + .../meta-interface/src/lib/metamesh.xsd | 14 + packages/shared/meta-interface/tsconfig.json | 18 + .../tsup.config.bundled_o1epw1yp7n.mjs | 0 packages/shared/meta-interface/tsup.config.js | 15 + pnpm-lock.yaml | 7036 +++++++++++++++++ pnpm-workspace.yaml | 6 + 240 files changed, 48691 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker/nginx.conf create mode 100644 docker/redis.conf create mode 100644 docker/supervisord.conf create mode 100644 package.json create mode 100644 packages/async-utils/LICENSE create mode 100644 packages/async-utils/README.md create mode 100644 packages/async-utils/package.json create mode 100644 packages/async-utils/src/Id.ts create mode 100644 packages/async-utils/src/Lazy.ts create mode 100644 packages/async-utils/src/ListenerCleaner.ts create mode 100644 packages/async-utils/src/MultiQueue.ts create mode 100644 packages/async-utils/src/PromiseQueue.ts create mode 100644 packages/async-utils/src/index.ts create mode 100644 packages/async-utils/tsconfig.json create mode 100644 packages/async-utils/tsup.config.js create mode 100644 packages/meta-sort-core/.env.example create mode 100644 packages/meta-sort-core/package.json create mode 100644 packages/meta-sort-core/src/api/DuplicatesAPI.ts create mode 100644 packages/meta-sort-core/src/api/FuseAPI.ts create mode 100644 packages/meta-sort-core/src/api/FuseAPIServer.ts create mode 100644 packages/meta-sort-core/src/api/MetricsAPI.ts create mode 100644 packages/meta-sort-core/src/api/README.md create mode 100644 packages/meta-sort-core/src/api/UnifiedAPIServer.ts create mode 100644 packages/meta-sort-core/src/api/VirtualFileSystem.ts create mode 100644 packages/meta-sort-core/src/api/index.ts create mode 100644 packages/meta-sort-core/src/config/EnvConfig.ts create mode 100644 packages/meta-sort-core/src/config/RenamingRule.ts create mode 100644 packages/meta-sort-core/src/config/SupportedFileTypes.ts create mode 100644 packages/meta-sort-core/src/config/TargetHash.ts create mode 100644 packages/meta-sort-core/src/container-plugins/ConfigParser.ts create mode 100644 packages/meta-sort-core/src/container-plugins/ContainerManager.ts create mode 100644 packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts create mode 100644 packages/meta-sort-core/src/container-plugins/DockerClient.ts create mode 100644 packages/meta-sort-core/src/container-plugins/index.ts create mode 100644 packages/meta-sort-core/src/container-plugins/types.ts create mode 100644 packages/meta-sort-core/src/index.ts create mode 100644 packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts create mode 100644 packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts create mode 100644 packages/meta-sort-core/src/jellyfin/README create mode 100644 packages/meta-sort-core/src/jellyfin/nfoTool.ts create mode 100644 packages/meta-sort-core/src/kv/IKVClient.ts create mode 100644 packages/meta-sort-core/src/kv/KVManager.ts create mode 100644 packages/meta-sort-core/src/kv/LeaderElection.ts create mode 100644 packages/meta-sort-core/src/kv/MetadataUtils.ts create mode 100644 packages/meta-sort-core/src/kv/RedisClient.ts create mode 100644 packages/meta-sort-core/src/kv/ServiceDiscovery.ts create mode 100644 packages/meta-sort-core/src/kv/index.ts create mode 100644 packages/meta-sort-core/src/logic/DuplicateFinder.ts create mode 100644 packages/meta-sort-core/src/logic/HashProcessingStateManager.ts create mode 100644 packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts create mode 100644 packages/meta-sort-core/src/logic/MetaMeshFormat.ts create mode 100644 packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts create mode 100644 packages/meta-sort-core/src/logic/ProcessingStateManager.ts create mode 100644 packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts create mode 100644 packages/meta-sort-core/src/logic/WatchedFileProcessor.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts create mode 100644 packages/meta-sort-core/src/logic/fileProcessor/worker.ts create mode 100644 packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts create mode 100644 packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts create mode 100644 packages/meta-sort-core/src/logic/pipeline/index.ts create mode 100644 packages/meta-sort-core/src/logic/tool/MetadataMerger.ts create mode 100644 packages/meta-sort-core/src/metrics/PerformanceMetrics.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/PluginContext.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/PluginManager.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/index.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/plugin.test.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/types.ts create mode 100644 packages/meta-sort-core/src/plugin-engine/validate.js create mode 100644 packages/meta-sort-core/src/types/FileMetadata.ts create mode 100644 packages/meta-sort-core/src/utils/TempIdGenerator.ts create mode 100644 packages/meta-sort-core/src/utils/TimeoutWrapper.ts create mode 100644 packages/meta-sort-core/tsconfig.json create mode 100644 packages/meta-sort-editor/README.md create mode 100644 packages/meta-sort-editor/eslint.config.js create mode 100644 packages/meta-sort-editor/index.html create mode 100644 packages/meta-sort-editor/package-lock.json create mode 100644 packages/meta-sort-editor/package.json create mode 100644 packages/meta-sort-editor/src/App.css create mode 100644 packages/meta-sort-editor/src/App.tsx create mode 100644 packages/meta-sort-editor/src/api/metadataApi.ts create mode 100644 packages/meta-sort-editor/src/components/BulkMetadataEditor.css create mode 100644 packages/meta-sort-editor/src/components/BulkMetadataEditor.tsx create mode 100644 packages/meta-sort-editor/src/components/DynamicFieldRenderer.css create mode 100644 packages/meta-sort-editor/src/components/DynamicFieldRenderer.tsx create mode 100644 packages/meta-sort-editor/src/components/DynamicPluginSection.css create mode 100644 packages/meta-sort-editor/src/components/DynamicPluginSection.tsx create mode 100644 packages/meta-sort-editor/src/components/FileList.css create mode 100644 packages/meta-sort-editor/src/components/FileList.tsx create mode 100644 packages/meta-sort-editor/src/components/KVTreeView.css create mode 100644 packages/meta-sort-editor/src/components/KVTreeView.tsx create mode 100644 packages/meta-sort-editor/src/components/LanguageCombobox.css create mode 100644 packages/meta-sort-editor/src/components/LanguageCombobox.tsx create mode 100644 packages/meta-sort-editor/src/components/MetadataEditor.css create mode 100644 packages/meta-sort-editor/src/components/MetadataEditor.tsx create mode 100644 packages/meta-sort-editor/src/components/SearchBar.css create mode 100644 packages/meta-sort-editor/src/components/SearchBar.tsx create mode 100644 packages/meta-sort-editor/src/index.css create mode 100644 packages/meta-sort-editor/src/main.tsx create mode 100644 packages/meta-sort-editor/src/types/index.ts create mode 100644 packages/meta-sort-editor/src/utils/formatting.ts create mode 100644 packages/meta-sort-editor/src/vite-env.d.ts create mode 100644 packages/meta-sort-editor/tsconfig.json create mode 100644 packages/meta-sort-editor/tsconfig.tsbuildinfo create mode 100644 packages/meta-sort-editor/vite.config.ts create mode 100644 packages/meta-sort-ui/index.html create mode 100644 packages/meta-sort-ui/package.json create mode 100644 packages/meta-sort-ui/public/favicon.svg create mode 100644 packages/meta-sort-ui/src/App.tsx create mode 100644 packages/meta-sort-ui/src/components/ServiceNav.tsx create mode 100644 packages/meta-sort-ui/src/index.css create mode 100644 packages/meta-sort-ui/src/main.tsx create mode 100644 packages/meta-sort-ui/src/pages/Duplicates.tsx create mode 100644 packages/meta-sort-ui/src/pages/Monitor.tsx create mode 100644 packages/meta-sort-ui/src/pages/Mounts.tsx create mode 100644 packages/meta-sort-ui/src/pages/Plugins.tsx create mode 100644 packages/meta-sort-ui/src/pages/Welcome.tsx create mode 100644 packages/meta-sort-ui/src/types/index.ts create mode 100644 packages/meta-sort-ui/src/utils/format.ts create mode 100644 packages/meta-sort-ui/tsconfig.json create mode 100644 packages/meta-sort-ui/tsconfig.node.json create mode 100644 packages/meta-sort-ui/vite.config.ts create mode 100644 packages/shared/filename-tool/.gitattributes create mode 100644 packages/shared/filename-tool/.gitignore create mode 100644 packages/shared/filename-tool/LICENSE create mode 100644 packages/shared/filename-tool/README.md create mode 100644 packages/shared/filename-tool/package-lock.json create mode 100644 packages/shared/filename-tool/package.json create mode 100644 packages/shared/filename-tool/src/lib/FileNameMetaExtractor.ts create mode 100644 packages/shared/filename-tool/src/lib/FileNameVideoMetaExtractor.ts create mode 100644 packages/shared/filename-tool/src/lib/FileType.ts create mode 100644 packages/shared/filename-tool/src/lib/SimpleFileType.ts create mode 100644 packages/shared/filename-tool/src/lib/config/AnimeKeywords.ts create mode 100644 packages/shared/filename-tool/src/lib/config/ExtentionsMapping.ts create mode 100644 packages/shared/filename-tool/src/lib/config/KeywordsArray.ts create mode 100644 packages/shared/filename-tool/src/lib/config/MimeTypeMapping.ts create mode 100644 packages/shared/filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts create mode 100644 packages/shared/filename-tool/src/lib/index.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/FileNameCleaner.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/FileNameStringOperation.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/FileTypeConfigurable.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/StringArrayOperation.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/TwoStringsOperation.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/folder/FolderTool.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/lang/LanguageTool.ts create mode 100644 packages/shared/filename-tool/src/lib/tools/video/SerieFilesNamaAnalysisTool.ts create mode 100644 packages/shared/filename-tool/src/tests/FileNameMetaExtractor.test.ts create mode 100644 packages/shared/filename-tool/src/tests/FileNameVideoMetaExtractor.test.ts create mode 100644 packages/shared/filename-tool/src/tests/main.test.ts create mode 100644 packages/shared/filename-tool/tsconfig.json create mode 100644 packages/shared/filename-tool/tsup.config.js create mode 100644 packages/shared/filename-tool/vitest.config.ts create mode 100644 packages/shared/meta-hash/.gitattributes create mode 100644 packages/shared/meta-hash/.gitignore create mode 100644 packages/shared/meta-hash/LICENSE create mode 100644 packages/shared/meta-hash/README.md create mode 100644 packages/shared/meta-hash/package.json create mode 100644 packages/shared/meta-hash/pnpm-lock.yaml create mode 100644 packages/shared/meta-hash/src/lib/file-id/BrowserHasher.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/BtihV2Hasher.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/CodeTable.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/ComputeHash.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/Crc32Hash.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/CreateHasher.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/CreateHasherBrowser.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/FastHash.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/FileIDComputerInterface.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/FileIDComputerWorker.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/MagnetLinkGenerator.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/ShaComputeWorker.ts create mode 100644 packages/shared/meta-hash/src/lib/file-id/SimpleHash.ts create mode 100644 packages/shared/meta-hash/src/lib/folder-watcher/FileProcessorInterface.ts create mode 100644 packages/shared/meta-hash/src/lib/folder-watcher/FolderWatcher.ts create mode 100644 packages/shared/meta-hash/src/lib/folder-watcher/PollingWatcher.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/HashComputer.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/HashComputerFile.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/HashComputerIndexCache.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/HashComputerWorker.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts create mode 100644 packages/shared/meta-hash/src/lib/hash-compute/MultiHashData.ts create mode 100644 packages/shared/meta-hash/src/lib/index-browser.ts create mode 100644 packages/shared/meta-hash/src/lib/index-interface.ts create mode 100644 packages/shared/meta-hash/src/lib/index.ts create mode 100644 packages/shared/meta-hash/src/lib/utils/ExistsAsync.ts create mode 100644 packages/shared/meta-hash/src/tests/btihv2.test.ts create mode 100644 packages/shared/meta-hash/src/tests/magnet-link.test.ts create mode 100644 packages/shared/meta-hash/src/tests/main.test.ts create mode 100644 packages/shared/meta-hash/src/tests/test.txt create mode 100644 packages/shared/meta-hash/tsconfig.json create mode 100644 packages/shared/meta-hash/tsup.config.js create mode 100644 packages/shared/meta-hash/vitest.config.ts create mode 100644 packages/shared/meta-interface/.gitattributes create mode 100644 packages/shared/meta-interface/.gitignore create mode 100644 packages/shared/meta-interface/.mocharc.json create mode 100644 packages/shared/meta-interface/LICENSE create mode 100644 packages/shared/meta-interface/README.md create mode 100644 packages/shared/meta-interface/package.json create mode 100644 packages/shared/meta-interface/pnpm-lock.yaml create mode 100644 packages/shared/meta-interface/src/lib/MetaData.test.ts create mode 100644 packages/shared/meta-interface/src/lib/MetaDataAPI.ts create mode 100644 packages/shared/meta-interface/src/lib/MetaDataName.ts create mode 100644 packages/shared/meta-interface/src/lib/Readme.md create mode 100644 packages/shared/meta-interface/src/lib/index.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/AnimeMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/DebugMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/FileNameMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/FileStatMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/GenreMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/HashMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/JellyfinMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/LanguagesMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/MediaStreamMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/SubtitleMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/TagsMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/TorrentMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/VersionMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/VideoMeta.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/ArrayKey.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/FileRef.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/LanguageData.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/RecordSet.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/SimpleFileType.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/StringBoolean.ts create mode 100644 packages/shared/meta-interface/src/lib/metadata-type/basic/VideoType.ts create mode 100644 packages/shared/meta-interface/src/lib/metamesh.xsd create mode 100644 packages/shared/meta-interface/tsconfig.json create mode 100644 packages/shared/meta-interface/tsup.config.bundled_o1epw1yp7n.mjs create mode 100644 packages/shared/meta-interface/tsup.config.js create mode 100644 pnpm-lock.yaml create mode 100644 pnpm-workspace.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5518407 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +node_modules +**/node_modules +**/.pnpm-store +packages/*/dist +*.log +.git diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..b5948ac --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,68 @@ +name: Build and Push Docker Image + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Tag for the image (optional, defaults to sha)' + required: false + type: string + meta_core_version: + description: 'meta-core version to use (default: latest)' + required: false + type: string + default: 'latest' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive # For plugins + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + META_CORE_IMAGE=ghcr.io/worph/meta-core:${{ inputs.meta_core_version || 'latest' }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d472a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Build output +dist/ + +# Dependencies +node_modules/ + +# Environment +.env +.env.local + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Test data +test-data/ + +# Logs +*.log +npm-debug.log* + +# Cache +.cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..efa34bd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,205 @@ +# meta-sort standalone Docker image +# Includes: Redis, nginx, rclone, FFmpeg, Node.js backend, React UI, and Editor +# +# Build standalone: +# docker build -t meta-sort . +# +# Build with custom meta-core (for development): +# docker build --build-arg META_CORE_IMAGE=meta-core:local -t meta-sort . + +# Stage 0: Get meta-core binary from published image +ARG META_CORE_IMAGE=ghcr.io/worph/meta-core:latest +FROM ${META_CORE_IMAGE} AS meta-core + +# Stage 1: Build UI +FROM node:21-alpine AS ui-builder + +WORKDIR /build + +# Copy UI package and install +COPY packages/meta-sort-ui/package.json ./ +RUN npm install + +# Copy UI source and build +COPY packages/meta-sort-ui/ ./ +RUN npm run build + +# Stage 2: Build Editor +FROM node:21-alpine AS editor-builder + +WORKDIR /build + +# Copy editor package and install +COPY packages/meta-sort-editor/package.json ./ +RUN npm install + +# Copy editor source and build +COPY packages/meta-sort-editor/ ./ +RUN npm run build + +# Stage 3: Build Backend using pnpm workspace +FROM node:21-alpine AS backend-builder + +# Install pnpm +RUN npm install -g pnpm + +WORKDIR /build + +# Copy workspace configuration +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ + +# Copy all packages for the workspace +COPY packages/shared/ ./packages/shared/ +COPY packages/async-utils/ ./packages/async-utils/ +COPY packages/meta-sort-core/ ./packages/meta-sort-core/ + +# Install all dependencies +RUN pnpm install --frozen-lockfile || pnpm install + +# Build shared dependencies first +RUN pnpm --filter="@metazla/meta-interface" build || true +RUN pnpm --filter="@metazla/meta-hash" build || true +RUN pnpm --filter="@metazla/filename-tools" build || true + +# Build async-utils +RUN pnpm --filter="@worph/async-utils" build || true + +# Build the core package +RUN pnpm --filter="@meta-sort/core" build + +# Stage 4: Runtime +FROM ubuntu:22.04 + +# Container registry metadata +LABEL org.opencontainers.image.source=https://github.com/worph/meta-sort +LABEL org.opencontainers.image.description="MetaMesh file processing and metadata extraction service" +LABEL org.opencontainers.image.licenses=MIT + +# Avoid prompts during install +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + gnupg \ + ca-certificates \ + nginx \ + redis-server \ + supervisor \ + ffmpeg \ + apache2-utils \ + fuse3 \ + unzip \ + cifs-utils \ + nfs-common \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 21 +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - \ + && apt-get install -y nodejs + +# Install rclone +RUN curl https://rclone.org/install.sh | bash + +# Create directories +RUN mkdir -p \ + /app/backend \ + /app/ui \ + /app/editor \ + /data/watch \ + /data/cache \ + /data/redis \ + /data/mounts \ + /meta-core/db/redis \ + /meta-core/locks \ + /meta-core/services \ + /files \ + /var/log/supervisor \ + /var/log/nginx \ + /root/.config/rclone + +# Copy built UI +COPY --from=ui-builder /build/dist /app/ui +RUN chmod -R 755 /app/ui + +# Copy built Editor +COPY --from=editor-builder /build/dist /app/editor +RUN chmod -R 755 /app/editor + +# Copy built backend with all dependencies +COPY --from=backend-builder /build/packages/meta-sort-core/dist /app/backend/dist +COPY --from=backend-builder /build/packages/meta-sort-core/package.json /app/backend/ + +# Copy plugins directory +COPY --from=backend-builder /build/packages/meta-sort-core/plugins /app/backend/plugins + +# Copy node_modules (has all deps including workspace packages) +COPY --from=backend-builder /build/node_modules /app/node_modules + +# Recreate ALL pnpm symlinks (pnpm stores in .pnpm, symlinks at top level) +RUN for pkg_path in /app/node_modules/.pnpm/*/node_modules/*; do \ + pkg_name=$(basename "$pkg_path"); \ + if [ ! -L "/app/node_modules/$pkg_name" ] && [ ! -d "/app/node_modules/$pkg_name" ]; then \ + ln -sf "$pkg_path" /app/node_modules/$pkg_name 2>/dev/null || true; \ + fi; \ + done + +# Also handle scoped packages (@scope/name) +RUN for scope_path in /app/node_modules/.pnpm/*/node_modules/@*; do \ + if [ -d "$scope_path" ]; then \ + scope_name=$(basename "$scope_path"); \ + mkdir -p "/app/node_modules/$scope_name" 2>/dev/null; \ + for pkg_path in "$scope_path"/*; do \ + pkg_name=$(basename "$pkg_path"); \ + if [ ! -L "/app/node_modules/$scope_name/$pkg_name" ] && [ ! -d "/app/node_modules/$scope_name/$pkg_name" ]; then \ + ln -sf "$pkg_path" /app/node_modules/$scope_name/$pkg_name 2>/dev/null || true; \ + fi; \ + done; \ + fi; \ + done + +# Copy built workspace packages (overwrite symlinks with actual dist) +COPY --from=backend-builder /build/packages/shared/meta-hash/dist /app/node_modules/@metazla/meta-hash/dist +COPY --from=backend-builder /build/packages/shared/meta-hash/package.json /app/node_modules/@metazla/meta-hash/ +COPY --from=backend-builder /build/packages/shared/meta-interface/dist /app/node_modules/@metazla/meta-interface/dist +COPY --from=backend-builder /build/packages/shared/meta-interface/package.json /app/node_modules/@metazla/meta-interface/ +COPY --from=backend-builder /build/packages/shared/filename-tool/dist /app/node_modules/@metazla/filename-tools/dist +COPY --from=backend-builder /build/packages/shared/filename-tool/package.json /app/node_modules/@metazla/filename-tools/ +COPY --from=backend-builder /build/packages/async-utils/dist /app/node_modules/@worph/async-utils/dist +COPY --from=backend-builder /build/packages/async-utils/package.json /app/node_modules/@worph/async-utils/ + +# Copy meta-core sidecar binary +COPY --from=meta-core /usr/local/bin/meta-core /usr/local/bin/meta-core +RUN chmod +x /usr/local/bin/meta-core + +# Copy configuration files +COPY docker/nginx.conf /etc/nginx/nginx.conf +COPY docker/redis.conf /etc/redis/redis.conf +COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# Create nginx htpasswd file (default: admin/admin) +RUN htpasswd -bc /etc/nginx/.htpasswd admin admin + +# Environment variables +ENV NODE_ENV=production \ + NODE_PATH=/app/node_modules \ + WATCH_PATHS=/data/watch \ + META_CORE_PATH=/meta-core \ + FILES_PATH=/files \ + REDIS_URL=redis://127.0.0.1:6379 \ + REDIS_PREFIX=meta-sort: \ + API_HOST=0.0.0.0 \ + API_PORT=3000 + +# Set working directory for backend +WORKDIR /app/backend + +# Expose port 80 (nginx) +EXPOSE 80 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD curl -f http://localhost/health || exit 1 + +# Start supervisord (manages redis, backend, nginx) +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..c643acd --- /dev/null +++ b/README.md @@ -0,0 +1,841 @@ +# meta-sort + +A standalone file sorting and metadata extraction service for the MetaMesh ecosystem. meta-sort monitors watch folders, extracts metadata from media files, stores it in a distributed key-value store, and manages remote drive mounting for shared access. + +## Overview + +meta-sort is a **process-write** service that serves as the primary data ingestion point for the MetaMesh platform. It: + +1. **Watches folders** for new media files (video, audio, subtitles) +2. **Extracts metadata** using FFmpeg and filename parsing +3. **Computes content identifiers** using midhash256 (fast) and SHA-256 (background) +4. **Stores metadata** in a distributed KV store (Redis or etcd) +5. **Manages remote mounts** (NFS, SMB, cloud storage) for shared access + +--- + +## Meta-Core Architecture + +MetaMesh uses **two shared volumes** for operation: + +### 1. META_CORE_VOLUME - Infrastructure + +The infrastructure volume stores all service coordination data: + +| Context | Path | Description | +|---------|------|-------------| +| **Inside Container** | `/meta-core` | Mounted shared volume | +| **Production (Host)** | `/DATA/Apps/meta-core` | External shared drive mount point | +| **Development** | `./data/meta-core` | Local development directory | + +``` +META_CORE_VOLUME (/meta-core or /DATA/Apps/meta-core) +├── db/ # KV Database storage +│ ├── redis/ # Redis persistence (RDB + AOF) +│ │ ├── dump.rdb # Redis snapshot +│ │ └── appendonly.aof # Redis append-only log +│ └── etcd/ # etcd data (alternative) +│ └── member/ +│ +├── locks/ # Distributed locking +│ └── kv-leader.lock # Leader election lock file +│ # Contains: {"host":"service-name","api":"http://host:port"} +│ +├── services/ # Service discovery registry +│ ├── meta-sort.json # {"name":"meta-sort","api":"http://...","status":"running","pid":123} +│ ├── meta-fuse.json # {"name":"meta-fuse","api":"http://...","status":"running","pid":456} +│ ├── meta-stremio.json # {"name":"meta-stremio","api":"http://...","status":"running","pid":789} +│ └── meta-orbit.json # {"name":"meta-orbit","api":"http://...","status":"running","pid":321} +│ +└── config/ # Shared configuration + ├── remote-storage.conf # Remote mount definitions + └── smb-credentials.txt # SMB credentials (chmod 600) +``` + +### 2. FILES_VOLUME - Shared Media Drive + +The **FILES_VOLUME** (canonical name: `/files`) is the shared drive for all media files. This is where all actual media content resides, and file paths in the KV database reference this volume. + +| Context | Path | Description | +|---------|------|-------------| +| **Inside Container** | `/files` | Mounted shared media volume | +| **Production (Host)** | Varies (NFS/SMB/local) | External shared drive | +| **Development** | `./data/files` | Local development directory | + +``` +FILES_VOLUME (/files) +├── media1/ # Local folder mounted from host +│ ├── Movies/ +│ │ └── Inception (2010)/ +│ │ └── Inception.mkv +│ └── TV Shows/ +│ └── Breaking Bad/ +│ └── Season 01/ +│ +├── media2/ # Another local media folder +│ └── Anime/ +│ └── Steins Gate/ +│ +├── smb/ # SMB/CIFS remote mount +│ └── NAS-Media/ +│ └── Movies/ +│ +├── nfs/ # NFS remote mount +│ └── server-media/ +│ +└── gdrive/ # rclone remote mount (Google Drive) + └── Cloud-Media/ +``` + +### File Path References + +All file paths stored in the KV database are **relative to FILES_VOLUME**: + +``` +KV Key: /file/{midhash256}/filePath +KV Value: "media1/Movies/Inception (2010)/Inception.mkv" + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Relative to FILES_VOLUME (/files) + +Full path construction: + Container: /files + /media1/Movies/... = /files/media1/Movies/Inception (2010)/Inception.mkv + Any host: $FILES_VOLUME + /media1/Movies/... = $FILES_VOLUME/media1/Movies/Inception (2010)/Inception.mkv +``` + +This allows services to be deployed on different hosts while sharing the same volume. The FILES_VOLUME can be: +- A local folder bind mount +- An NFS share mounted on the host +- An SMB/CIFS share +- A FUSE mount (rclone, sshfs, etc.) + +### Mount Management + +meta-sort manages remote mounts **inside** the FILES_VOLUME for shared access: + +```bash +# meta-sort mounts remote storage INTO /files +/files/smb/nas-share ← SMB mount managed by meta-sort +/files/nfs/server ← NFS mount managed by meta-sort +/files/gdrive/ ← rclone mount managed by meta-sort + +# Other folders are bind mounts from host +/files/media1 ← Host folder mounted into container +/files/media2 ← Another host folder +``` + +All services access files through `/files/*`, regardless of whether the underlying storage is local, NFS, SMB, or cloud. + +--- + +## Leader Election and Lock Mechanism + +### How It Works + +MetaMesh uses a **filesystem-based distributed consensus** mechanism using `flock(2)`. The shared filesystem acts as the consensus layer. + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Leader Election via flock │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Step 1: Service Startup │ +│ ┌─────────────┐ │ +│ │ Service │ ──▶ Try to acquire flock on /meta-core/locks/kv-leader.lock│ +│ └─────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ flock() Result │ │ +│ │ ┌─────────────────────┐ ┌──────────────────────────────────┐ │ │ +│ │ │ Lock Acquired │ │ Lock Blocked │ │ │ +│ │ │ ────────────── │ │ ───────────── │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ BECOME LEADER │ │ BECOME FOLLOWER │ │ │ +│ │ │ 1. Spawn Redis │ │ 1. Read lock file │ │ │ +│ │ │ 2. Write API │ │ 2. Extract leader API │ │ │ +│ │ │ endpoint to │ │ 3. Connect to leader's │ │ │ +│ │ │ lock file │ │ Redis │ │ │ +│ │ │ 3. Accept clients │ │ 4. Start reconnect loop │ │ │ +│ │ └─────────────────────┘ └──────────────────────────────────┘ │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Lock File Format + +The lock file at `/meta-core/locks/kv-leader.lock` contains JSON: + +```json +{ + "host": "meta-sort-instance-1", + "api": "redis://10.0.1.50:6379", + "http": "http://10.0.1.50:3000", + "timestamp": 1703808000000, + "pid": 12345 +} +``` + +### flock Implementation + +```typescript +import { open, flock } from 'fs'; +import { promisify } from 'util'; + +const LOCK_FILE = '/meta-core/locks/kv-leader.lock'; + +async function tryBecomeLeader(): Promise { + const fd = await promisify(open)(LOCK_FILE, 'w'); + + try { + // LOCK_EX = exclusive lock, LOCK_NB = non-blocking + await promisify(flock)(fd, 'exnb'); // Throws if cannot acquire + + // We are the leader - spawn KV database and write our endpoint + await spawnRedis(); + await fs.writeFile(LOCK_FILE, JSON.stringify({ + host: hostname(), + api: `redis://${myIP}:6379`, + http: `http://${myIP}:3000`, + timestamp: Date.now(), + pid: process.pid + })); + + return true; // We are leader + } catch (err) { + if (err.code === 'EAGAIN' || err.code === 'EWOULDBLOCK') { + // Lock held by another process - become follower + return false; + } + throw err; + } +} +``` + +### Failure Detection and Recovery + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ Failure Detection Loop │ +├──────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Follower Service: │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ while (true) { │ │ +│ │ try { │ │ +│ │ await redis.ping(); // Health check │ │ +│ │ await sleep(5000); // Wait 5 seconds │ │ +│ │ } catch (connectionError) { │ │ +│ │ // Leader might be down │ │ +│ │ await tryBecomeLeader(); // Attempt to take over │ │ +│ │ if (isLeader) break; │ │ +│ │ await reconnectToNewLeader(); // Or connect to new leader │ │ +│ │ } │ │ +│ │ } │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ When Leader Dies: │ +│ 1. flock is automatically released by OS │ +│ 2. First follower to call flock() wins │ +│ 3. New leader spawns Redis, writes new endpoint │ +│ 4. Other followers detect connection failure, read new endpoint │ +│ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Service Discovery + +### Registration Mechanism + +Each service registers itself in `/meta-core/services/{service-name}.json`: + +```json +{ + "name": "meta-sort", + "version": "1.0.0", + "api": "http://10.0.1.50:3000", + "status": "running", + "pid": 12345, + "hostname": "meta-sort-instance-1", + "startedAt": "2024-12-28T10:00:00Z", + "lastHeartbeat": "2024-12-28T12:30:00Z", + "capabilities": ["write", "mount", "monitor"], + "endpoints": { + "health": "/health", + "api": "/api", + "metrics": "/api/metrics" + } +} +``` + +### Service Lifecycle + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ Service Lifecycle │ +├──────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Startup: │ +│ 1. Create/update /meta-core/services/{name}.json │ +│ 2. Set status = "starting" │ +│ 3. Initialize (connect to KV, etc.) │ +│ 4. Set status = "running" │ +│ 5. Start heartbeat loop (update lastHeartbeat every 30s) │ +│ │ +│ Running: │ +│ - Heartbeat updates lastHeartbeat timestamp │ +│ - Other services read this file to discover endpoints │ +│ - stale detection: if (now - lastHeartbeat > 60s) → service dead │ +│ │ +│ Shutdown: │ +│ 1. Set status = "stopping" │ +│ 2. Graceful shutdown (finish current work) │ +│ 3. Delete /meta-core/services/{name}.json OR set status = "stopped" │ +│ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +### Discovering Other Services + +```typescript +async function discoverService(name: string): Promise { + const servicePath = `/meta-core/services/${name}.json`; + + try { + const content = await fs.readFile(servicePath, 'utf-8'); + const service = JSON.parse(content); + + // Check if service is alive (heartbeat within last 60 seconds) + const lastHeartbeat = new Date(service.lastHeartbeat).getTime(); + const isAlive = Date.now() - lastHeartbeat < 60_000; + + if (!isAlive) { + console.warn(`Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`); + return null; + } + + return service; + } catch (err) { + if (err.code === 'ENOENT') { + return null; // Service not registered + } + throw err; + } +} +``` + +--- + +## MetaMesh Ecosystem Architecture + +### Service Hierarchy + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ MetaMesh Ecosystem │ +│ │ +│ Shared Volumes: │ +│ ┌──────────────────────────────────┐ ┌──────────────────────────────────┐ │ +│ │ META_CORE_VOLUME │ │ FILES_VOLUME │ │ +│ │ (/meta-core) │ │ (/files) │ │ +│ │ ┌────────┐ ┌────────┐ ┌──────┐ │ │ ┌────────┐ ┌────────┐ ┌──────┐ │ │ +│ │ │ db/ │ │ locks/ │ │svc/ │ │ │ │media1/ │ │ smb/ │ │ nfs/ │ │ │ +│ │ │(Redis) │ │(flock) │ │(JSON)│ │ │ │(local) │ │(mount) │ │(mnt) │ │ │ +│ │ └────┬───┘ └───┬────┘ └──┬───┘ │ │ └───┬────┘ └───┬────┘ └──┬───┘ │ │ +│ └───────┼─────────┼─────────┼─────┘ └──────┼──────────┼─────────┼─────┘ │ +│ │ │ │ │ │ │ │ +│ ▼ ▼ ▼ ▼ ▼ ▼ │ +│ ┌───────────────────────────────────────────────────────────────────────┐│ +│ │ ││ +│ │ [1] meta-sort PROCESS-WRITE ││ +│ │ ├── Watches /files/* for new media files ││ +│ │ ├── Extracts metadata → writes to KV (Redis in /meta-core/db) ││ +│ │ ├── Manages remote mounts INTO /files (NFS, SMB, rclone) ││ +│ │ ├── Leader election (spawns Redis if leader) ││ +│ │ └── Serves monitoring UI (meta-sort-editor) ││ +│ │ ││ +│ │ [2] meta-fuse PROCESS-READ ││ +│ │ ├── Reads metadata from KV (Redis) ││ +│ │ ├── Builds virtual filesystem from metadata ││ +│ │ ├── Serves FUSE mount + WebDAV ││ +│ │ └── File paths resolve to /files/* ││ +│ │ ││ +│ │ [3] meta-stremio PROCESS-READ ││ +│ │ ├── Reads metadata from KV (Redis) ││ +│ │ ├── Stremio addon server (manifest, catalog, streams) ││ +│ │ ├── HLS transcoding via FFmpeg ││ +│ │ └── Video files from /files/* ││ +│ │ ││ +│ │ [4] meta-orbit SHARING-READ-WRITE ││ +│ │ ├── P2P metadata synchronization (libp2p + OrbitDB) ││ +│ │ ├── Decentralized search across network ││ +│ │ ├── Reads from local KV, shares to network ││ +│ │ └── Receives from network, writes to local KV ││ +│ │ ││ +│ └───────────────────────────────────────────────────────────────────────┘│ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Data Flow + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Data Flow │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ File Discovery: │ +│ │ +│ /files/media1/Movies/Inception (2010)/Inception.mkv │ +│ │ │ +│ ▼ │ +│ [meta-sort] ──▶ Extract metadata ──▶ Write to Redis (in /meta-core/db) │ +│ │ │ │ +│ │ Key: file:{midhash256} │ │ +│ │ filePath: "media1/Movies/Inception (2010)/Inception.mkv" │ +│ │ title: "Inception" │ +│ │ year: "2010" │ +│ │ duration: 8880 │ +│ │ │ │ +│ ▼ ▼ │ +│ │ +│ File Access: │ +│ │ +│ [meta-fuse] ◀── Read from Redis │ +│ │ │ +│ ▼ │ +│ Virtual FS: /mnt/virtual/Movies/Inception (2010)/Inception.mkv │ +│ │ ↓ │ +│ │ symlink to /files/media1/Movies/Inception (2010)/... │ +│ │ │ +│ [meta-stremio] ◀── Read from Redis │ +│ │ │ +│ ▼ │ +│ Stream: GET /stremio/stream/{midhash256}/playlist.m3u8 │ +│ │ ↓ │ +│ │ transcode /files/media1/Movies/Inception (2010)/... │ +│ │ │ +│ [meta-orbit] ◀──▶ Sync with P2P network │ +│ │ │ +│ ▼ │ +│ Share metadata to peers / Receive metadata from peers │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Deployment + +### Docker Compose (Production) + +```yaml +version: '3.8' + +services: + meta-sort: + image: metazla/meta-sort:latest + volumes: + # Infrastructure volume (KV DB, locks, services) + - /DATA/Apps/meta-core:/meta-core:rw + # Shared media volume + - /DATA/Apps/meta-core/files:/files:rw,shared + # OR mount multiple sources into /files + # - /mnt/nas/media:/files/nas:ro + # - /mnt/local/media:/files/local:ro + environment: + - META_CORE_PATH=/meta-core + - FILES_PATH=/files + - WATCH_PATHS=/files + - REDIS_DATA_DIR=/meta-core/db/redis + privileged: true # Required for rclone FUSE mounts + + meta-fuse: + image: metazla/meta-fuse:latest + volumes: + # Infrastructure (read-only for followers) + - /DATA/Apps/meta-core:/meta-core:ro + # Shared media volume (read-only) + - /DATA/Apps/meta-core/files:/files:ro + # FUSE mount output + - /mnt/metamesh:/mnt/metamesh:rw,shared + environment: + - META_CORE_PATH=/meta-core + - FILES_PATH=/files + privileged: true # Required for FUSE + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + + meta-stremio: + image: metazla/meta-stremio:latest + volumes: + # Infrastructure (read-only) + - /DATA/Apps/meta-core:/meta-core:ro + # Shared media volume (read-only) + - /DATA/Apps/meta-core/files:/files:ro + environment: + - META_CORE_PATH=/meta-core + - FILES_PATH=/files + ports: + - "7000:7000" + + meta-orbit: + image: metazla/meta-orbit:latest + volumes: + # Infrastructure (read-write for P2P sync) + - /DATA/Apps/meta-core:/meta-core:rw + environment: + - META_CORE_PATH=/meta-core + ports: + - "4001:4001" # libp2p +``` + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `META_CORE_PATH` | `/meta-core` | Path to META_CORE_VOLUME (infrastructure) | +| `FILES_PATH` | `/files` | Path to FILES_VOLUME (shared media) | +| `REDIS_URL` | Auto-discover | Redis connection (from leader lock file) | +| `WATCH_PATHS` | `$FILES_PATH` | Folders to watch for media | +| `SERVICE_NAME` | Container hostname | Unique service identifier | + +--- + +## File Processing Pipeline + +meta-sort uses a **plugin-based architecture** with a two-queue task scheduler: + +``` +┌──────────────┐ ┌────────────────────────────────────────────────────┐ +│ Discovery │ │ Task Scheduler │ +│ (Stream) │───▶│ ┌─────────────────┐ ┌──────────────────────────┐ │ +└──────────────┘ │ │ Fast Queue │ │ Background Queue │ │ + Watch folder │ │ (16 workers) │ │ (4 workers) │ │ + + manual scan │ │ <1s tasks │ │ >=1s tasks │ │ + │ └─────────────────┘ └──────────────────────────┘ │ + └────────────────────────────────────────────────────┘ +``` + +Tasks are automatically classified and scheduled based on plugin dependencies: + +``` +Discovery → file-info → ffmpeg → filename-parser → jellyfin-nfo → tmdb + ↓ + anime-detector ← language ← subtitle ← torrent ← full-hash +``` + +--- + +## Plugin System + +Plugins are the core processing units that extract and enrich metadata. Each plugin: +- Has a `manifest.yml` defining metadata schema, config, and dependencies +- Runs in either **fast** (high concurrency) or **background** (low concurrency) queue +- Can depend on other plugins to ensure correct execution order + +### Built-in Plugins + +| Plugin | Queue | Priority | Description | +|--------|-------|----------|-------------| +| **file-info** | fast | 10 | Basic file info (type, MIME, size, midhash256) | +| **ffmpeg** | fast | 15 | Video/audio/subtitle stream metadata via FFprobe | +| **filename-parser** | fast | 20 | Parse title, season, episode, year, quality from filename | +| **jellyfin-nfo** | fast | 25 | Parse Jellyfin/Kodi NFO metadata files | +| **tmdb** | background | 30 | Fetch metadata from The Movie Database API | +| **anime-detector** | fast | 35 | Detect anime content from keywords and audio tracks | +| **language** | fast | 40 | Aggregate languages from all streams (ISO 639-3) | +| **subtitle** | fast | 45 | Find sibling subtitle files and detect language | +| **torrent** | fast | 50 | Parse .torrent file metadata | +| **full-hash** | background | 100 | Compute SHA-256 hash of entire file | + +### Plugin Manifest Example + +```yaml +id: ffmpeg +name: FFmpeg Metadata +version: 1.0.0 +description: Extracts video/audio/subtitle stream metadata using FFprobe +priority: 15 +color: "#4CAF50" +defaultQueue: fast + +dependencies: + - file-info + +config: {} + +schema: + fileinfo/duration: + label: Duration + type: string + readonly: true + fileinfo/streamdetails/video: + label: Video Streams + type: json + readonly: true + pattern: "fileinfo/streamdetails/video/{n}/*" +``` + +### Task Queue Architecture + +- **Fast Queue** (16 workers): High-throughput for quick operations (<1s) + - File info extraction, FFprobe, filename parsing, NFO reading +- **Background Queue** (4 workers): Low-concurrency for slow operations (>=1s) + - TMDB API calls, full SHA-256 hashing + - Automatically pauses when fast queue has work, resumes when idle + +### Plugin Management + +Plugins can be enabled/disabled and configured via: +- **Dashboard**: http://localhost:8180/ → Plugins tab +- **Config file**: `plugins.json` in cache directory + +--- + +## Processing States + +1. **Pending** - File discovered in watch folder +2. **Processing** - Plugins running (fast queue first, then background) +3. **Done** - All plugins completed successfully + +--- + +## Package Structure + +``` +meta-sort/ +├── packages/ +│ ├── meta-sort-core/ # Core processing service (@meta-sort/core) +│ │ ├── plugins/ # Built-in plugins +│ │ │ ├── file-info/ # Basic file information +│ │ │ ├── ffmpeg/ # FFprobe metadata extraction +│ │ │ ├── filename-parser/# Filename parsing +│ │ │ ├── jellyfin-nfo/ # NFO file parsing +│ │ │ ├── tmdb/ # TMDB API integration +│ │ │ ├── anime-detector/ # Anime content detection +│ │ │ ├── language/ # Language aggregation +│ │ │ ├── subtitle/ # Subtitle file finder +│ │ │ ├── torrent/ # Torrent file parsing +│ │ │ └── full-hash/ # SHA-256 full file hashing +│ │ ├── src/ +│ │ │ ├── api/ # Fastify REST API (UnifiedAPIServer) +│ │ │ ├── config/ # Environment configuration +│ │ │ ├── jellyfin/ # Jellyfin integration +│ │ │ ├── kv/ # KV manager with leader election (Redis) +│ │ │ ├── logic/ # File processing logic +│ │ │ ├── metrics/ # Performance tracking +│ │ │ ├── plugin-engine/ # Plugin loader and task scheduler +│ │ │ ├── types/ # TypeScript types +│ │ │ ├── utils/ # Utility functions +│ │ │ └── index.ts # Entry point +│ │ └── package.json +│ │ +│ ├── meta-sort-editor/ # Metadata editor UI (@meta-sort/editor) +│ │ └── ... # React/Vite app +│ │ +│ ├── meta-sort-ui/ # Monitoring dashboard (@meta-sort/ui) +│ │ └── ... # React/Vite app +│ │ +│ └── async-utils/ # Async utility library +│ └── ... # MultiQueue, PromiseQueue, etc. +│ +├── docker/ +│ ├── nginx.conf # Reverse proxy config +│ ├── redis.conf # Redis configuration +│ └── supervisord.conf # Process management +│ +├── Dockerfile +├── docker-compose.yml +├── package.json # Workspace root +├── pnpm-workspace.yaml +└── README.md +``` + +### Key Components (meta-sort-core) + +| Component | File | Purpose | +|-----------|------|---------| +| Entry Point | `src/index.ts` | Initializes KV manager, plugin engine, API server, file watching | +| File Processor | `src/logic/WatchedFileProcessor.ts` | File discovery and plugin task scheduling | +| Plugin Loader | `src/plugin-engine/PluginLoader.ts` | Loads plugins from manifest.yml, manages plugin lifecycle | +| Task Scheduler | `src/plugin-engine/TaskScheduler.ts` | Two-queue scheduler (fast/background) with dependency resolution | +| KV Manager | `src/kv/KVManager.ts` | Leader election via flock, Redis spawning, service discovery | +| Unified API | `src/api/UnifiedAPIServer.ts` | Fastify REST API for monitoring and control | +| Performance Metrics | `src/metrics/PerformanceMetrics.ts` | Tracks processing throughput and timing | +| Config | `src/config/EnvConfig.ts` | Environment variable parsing | + +--- + +## Core Features + +### 1. Folder Watching + +- **Real-time monitoring** using chokidar for file system events +- **Streaming discovery** via async generators for efficient memory usage +- **Configurable watch paths** supporting local and remote mounted directories +- **File type filtering** (video, audio, subtitle, torrent files) + +### 2. Plugin-Based Metadata Extraction + +Metadata extraction is handled by plugins that can be enabled/disabled and configured: + +- **file-info** - Basic file info, MIME type, midhash256 content identifier +- **ffmpeg** - Duration, codec, resolution, bitrate, audio tracks, subtitles via FFprobe +- **filename-parser** - Extract title, year, season, episode, quality from filenames +- **jellyfin-nfo** - Parse NFO sidecar files (Jellyfin/Kodi format) +- **tmdb** - Fetch rich metadata from The Movie Database API +- **anime-detector** - Detect anime content from keywords and audio tracks +- **language** - Aggregate languages from all streams (ISO 639-3) +- **subtitle** - Find sibling subtitle files and detect language +- **torrent** - Parse .torrent file metadata +- **full-hash** - Compute SHA-256 hash of entire file (background) + +### 3. KV Storage + +- **Nested key architecture** for property-level granularity +- **Leader election** via flock for high availability +- **Service discovery** via shared filesystem +- **CRDT-compatible** for distributed synchronization + +``` +Key Structure: +/file/{midhash256}/title → "Inception" +/file/{midhash256}/year → "2010" +/file/{midhash256}/video/codec → "h265" +/file/{midhash256}/audio/0/language → "en" +/file/{midhash256}/filePath → "media1/Movies/Inception (2010)/Inception.mkv" + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Relative to FILES_VOLUME (/files) +``` + +### 4. Remote Mount Management + +- **NFS shares** - Network file systems +- **SMB/CIFS** - Windows network shares with credentials +- **rclone** - Cloud storage (Google Drive, Dropbox, S3, etc.) +- **Automatic mounting** on startup +- **Mounts created inside FILES_VOLUME** for shared access by all services + +Mounts are created inside `/files/` (FILES_VOLUME) so all services can access them: + +```bash +# Example mount structure +/files/ # FILES_VOLUME +├── media1/ # Local folder bind mount from host +├── media2/ # Another local folder +├── smb/ # SMB/CIFS remote mounts +│ └── nas-share/ # SMB: //server/share +├── nfs/ # NFS remote mounts +│ └── server-media/ # NFS: nas.local:/volume1/media +└── gdrive/ # rclone: gdrive:Media +``` + +--- + +## Configuration + +### Environment Variables + +```bash +# Volume paths +META_CORE_PATH=/meta-core # Infrastructure volume +FILES_PATH=/files # Shared media volume + +# Watch paths (comma-separated, relative to FILES_PATH or absolute) +WATCH_PATHS=/files + +# Processing configuration +MAX_WORKER_THREADS=16 # Defaults to CPU count +LIGHT_PROCESSING_TIMEOUT_MS=120000 +HASH_PROCESSING_TIMEOUT_MS=7200000 + +# Remote mount configuration +REMOTE_STORAGE_CONFIG=/meta-core/config/remote-storage.conf +``` + +### Remote Storage Configuration + +Create `/meta-core/config/remote-storage.conf`: + +```conf +# NFS mount (into FILES_VOLUME) +nfs://nas.local:/volume1/media /files/nfs/nas-media nfs defaults,rw 0 0 + +# SMB mount (credentials in separate file) +//server/share /files/smb/nas-share cifs credentials=/meta-core/config/smb-creds.txt,uid=1000 0 0 + +# rclone mount +gdrive: /files/gdrive rclone vfs-cache-mode=full 0 0 +``` + +--- + +## API + +### REST Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/health` | GET | Service health check | +| `/api/scan/trigger` | POST | Trigger manual folder scan | +| `/api/processing/status` | GET | Current pipeline state | +| `/api/metrics` | GET | Performance metrics | +| `/api/mounts` | GET | List mounted remote storage | +| `/api/mounts` | POST | Add new remote mount | +| `/api/mounts/{id}` | DELETE | Remove remote mount | + +### Processing Status Response + +```json +{ + "totalPending": 150, + "totalLightProcessing": 16, + "totalHashProcessing": 8, + "totalDone": 45000, + "totalFilesInVFS": 45150, + "throughput": { + "filesPerSecond": 34, + "avgLightProcessingMs": 470, + "avgHashProcessingMs": 15000 + } +} +``` + +--- + +## Usage + +### Development + +```bash +# Install dependencies +pnpm install + +# Build all packages +pnpm run build + +# Run in development mode +pnpm run dev +``` + +### Docker + +```bash +# Build and run with Docker Compose +docker-compose up -d + +# Check health +curl http://localhost/health +curl http://localhost/api/processing/status +``` + +--- + +## License + +MIT License - see LICENSE file for details. diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..b1bffcd --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,91 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml; + + # Upstream servers + upstream api { + server 127.0.0.1:3000; + } + + upstream rclone { + server 127.0.0.1:5572; + } + + server { + listen 80; + server_name _; + + # Static files (UI) + root /app/ui; + index index.html; + + # SPA routing - serve index.html for any non-file request + location / { + try_files $uri $uri/ /index.html; + } + + # Editor UI - SPA with base path /editor/ + location /editor/ { + alias /app/editor/; + index index.html; + try_files $uri $uri/ /editor/index.html; + } + + # Redirect /editor to /editor/ + location = /editor { + return 301 /editor/; + } + + # API routes + location /api/ { + proxy_pass http://api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Health check (direct to API) + location /health { + proxy_pass http://api/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # rclone web UI + location /rclone/ { + proxy_pass http://rclone/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Basic auth for rclone + auth_basic "rclone"; + auth_basic_user_file /etc/nginx/.htpasswd; + } + } +} diff --git a/docker/redis.conf b/docker/redis.conf new file mode 100644 index 0000000..9464ec6 --- /dev/null +++ b/docker/redis.conf @@ -0,0 +1,30 @@ +# Redis configuration for meta-sort +# Optimized for metadata storage + +# Basic settings +bind 0.0.0.0 +port 6379 +protected-mode no +daemonize no + +# Persistence +appendonly yes +appendfsync everysec +dir /data/redis + +# Memory management +maxmemory 256mb +maxmemory-policy allkeys-lru + +# Logging +loglevel notice +logfile "" + +# Performance +tcp-keepalive 300 +timeout 0 + +# Disable dangerous commands in production +rename-command FLUSHDB "" +rename-command FLUSHALL "" +rename-command DEBUG "" diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 0000000..8f6a0e9 --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,69 @@ +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +loglevel=info + +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpc_interface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[program:meta-core] +command=/usr/local/bin/meta-core +priority=5 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +startsecs=3 +environment= + META_CORE_PATH="/meta-core", + FILES_PATH="/files", + SERVICE_NAME="meta-sort", + SERVICE_VERSION="1.0.0", + META_CORE_HTTP_PORT="9000", + REDIS_PORT="6379" + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +priority=20 +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/nginx.log +stderr_logfile=/var/log/supervisor/nginx_error.log +startsecs=2 + +[program:rclone] +command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin +priority=30 +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/rclone.log +stderr_logfile=/var/log/supervisor/rclone_error.log +startsecs=3 + +[program:meta-sort] +command=/usr/bin/node /app/backend/dist/index.js +directory=/app/backend +priority=40 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +startsecs=5 +environment= + REDIS_URL="redis://127.0.0.1:6379", + REDIS_PREFIX="", + API_HOST="0.0.0.0", + API_PORT="3000" diff --git a/package.json b/package.json new file mode 100644 index 0000000..e338373 --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "name": "meta-sort", + "version": "1.0.0", + "private": true, + "description": "Standalone file sorting and metadata extraction service", + "scripts": { + "build": "pnpm -r build", + "dev": "pnpm -r --parallel dev", + "start": "pnpm --filter @meta-sort/core start", + "start:core": "pnpm --filter @meta-sort/core start", + "start:ui": "pnpm --filter @meta-sort/ui dev", + "test": "pnpm --filter @meta-sort/core test", + "lint": "pnpm -r lint" + }, + "license": "MIT" +} diff --git a/packages/async-utils/LICENSE b/packages/async-utils/LICENSE new file mode 100644 index 0000000..f3621b9 --- /dev/null +++ b/packages/async-utils/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Pierre-Henri + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/async-utils/README.md b/packages/async-utils/README.md new file mode 100644 index 0000000..50a7d51 --- /dev/null +++ b/packages/async-utils/README.md @@ -0,0 +1,96 @@ +# Async Utils + +A TypeScript library providing utility classes and functions to simplify working with asynchronous operations and promises. + +## Installation + +```bash +yarn add github:worph/async-utils#main +``` + +## Features + +- **Lazy**: A utility type for lazy initialization of asynchronous operations. +- **ListenerCleaner**: Manages cleanup functions, allowing for easy resource management. +- **PromiseQueue**: A queue system for promises, allowing for sequential execution and cancellation. +- **MultiQueue**: Manages multiple `PromiseQueue` instances, distributing tasks to optimize concurrency. +- **Id Generation**: Utility function for generating random IDs. + +## Usage + +### Lazy Initialization + +```typescript +import { Lazy } from "@worph/async-utils"; + +const lazyValue: Lazy = async () => { + // some asynchronous operation + return 42; +}; + +// Usage +lazyValue().then(value => console.log(value)); +``` + +### Listener Cleaner + +```typescript +import { ListenerCleaner } from "worph/async-utils"; + +const cleaner = new ListenerCleaner(); +cleaner.add(() => console.log("Cleanup action")); + +// Trigger all cleanup actions +cleaner.cleanUp(); +``` + +### Promise Queue + +```typescript +import { PromiseQueue } from "@worph/async-utils"; + +const queue = new PromiseQueue(); + +queue.add(async () => { + // some asynchronous task + return 1; +}); + +// Wait for the queue to be empty +queue.awaitQueueEmpty().then(() => console.log("Queue empty")); +``` + +### Multi Queue + +```typescript +import { MultiQueue } from "@worph/async-utils"; + +const multiQueue = new MultiQueue(2); // 2 concurrent tasks + +multiQueue.add(async () => { + // some asynchronous task + return 1; +}); + +// Add more tasks... + +// Wait for all queues to be empty +multiQueue.awaitQueueEmpty().then(() => console.log("All queues empty")); +``` + +### Generating an ID + +```typescript +import { makeid } from "@worph/async-utils"; + +const id = makeid(10); // Generates a random 10 character string +console.log(id); +``` + +## Contributing + +Contributions are welcome! Please submit a pull request or open an issue for discussion. + +## License + +This project is licensed under the MIT License - see the LICENSE file for details. diff --git a/packages/async-utils/package.json b/packages/async-utils/package.json new file mode 100644 index 0000000..17b7588 --- /dev/null +++ b/packages/async-utils/package.json @@ -0,0 +1,37 @@ +{ + "name": "@worph/async-utils", + "version": "1.0.0", + "description": "", + "repository": { + "type": "git", + "url": "git+https://github.com/ApteroSAS/async-utils.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/ApteroSAS/async-utils/issues" + }, + "homepage": "https://github.com/ApteroSAS/async-utils#readme", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "scripts": { + "build": "tsup src/index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + }, + "devDependencies": { + "@types/jest": "^29.5.1", + "jest": "^29.5.0", + "tsup": "^6.7.0", + "typescript": "^5.0.4" + } +} diff --git a/packages/async-utils/src/Id.ts b/packages/async-utils/src/Id.ts new file mode 100644 index 0000000..1ee9081 --- /dev/null +++ b/packages/async-utils/src/Id.ts @@ -0,0 +1,10 @@ +export function makeid(length) { + let result = ''; + let characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let charactersLength = characters.length; + for ( let i = 0; i < length; i++ ) { + result += characters.charAt(Math.floor(Math.random() * + charactersLength)); + } + return result; +} \ No newline at end of file diff --git a/packages/async-utils/src/Lazy.ts b/packages/async-utils/src/Lazy.ts new file mode 100644 index 0000000..8ccdbe7 --- /dev/null +++ b/packages/async-utils/src/Lazy.ts @@ -0,0 +1,2 @@ + +export type Lazy = () => Promise; diff --git a/packages/async-utils/src/ListenerCleaner.ts b/packages/async-utils/src/ListenerCleaner.ts new file mode 100644 index 0000000..2327373 --- /dev/null +++ b/packages/async-utils/src/ListenerCleaner.ts @@ -0,0 +1,23 @@ +export class ListenerCleaner { + cleaners: (() => void)[] = []; + + add(cleanerCallback: () => void) { + this.cleaners.push(cleanerCallback); + } + + cleaner(): () => void { + return () => { + this.cleanUp(); + } + } + + /** + * Call all the cleaner callbacks and reset this cleaner to be reused. + */ + cleanUp() { + for (const cleaner of this.cleaners) { + cleaner(); + } + this.cleaners = []; + } +} diff --git a/packages/async-utils/src/MultiQueue.ts b/packages/async-utils/src/MultiQueue.ts new file mode 100644 index 0000000..1c62ad6 --- /dev/null +++ b/packages/async-utils/src/MultiQueue.ts @@ -0,0 +1,66 @@ +import {PromiseQueue} from "./PromiseQueue"; + +export class MultiQueue { + private readonly queues:PromiseQueue[]; + + /** + * + * @param concurrentTask + */ + constructor( + concurrentTask:number=1, + ) { + if (concurrentTask<1){ + throw new Error("concurrentTask must be >=1"); + } + this.queues = []; + for(let i=0;i()); + } + } + + public getQueueSize(): number { + let ret = 0; + for(let queue of this.queues){ + ret+=queue.getQueueSize(); + } + return ret; + } + + public async awaitQueueEmpty(){ + for(let queue of this.queues){ + await queue.awaitQueueEmpty(); + } + } + + /** + * Add a task to the queue. The queue with the smallest task size is used + * @param task + * @param id + **/ + public add(task: () => Promise, id: string = null): { id: string, promise: Promise } { + let queue = this.queues[0]; + for(let i=1;i { + private nextPromise: Promise = Promise.resolve(null); + private canceled = {}; + private queueSize = 0; + private queueID = 0; + + /** + * + * @param onlyLast if true all task are cancel and only the last one is keep alive (a task started still finishes) + */ + constructor( + private onlyLast: boolean = false + ) { + this.cancelAll(); + } + + getQueueSize(): number { + return this.queueSize; + } + + async awaitQueueEmpty(){ + await this.nextPromise;//await the current promise + while (this.queueSize!=0){//await all the fowllofing promise + await this.nextPromise; + } + } + + add(task: () => Promise, id: string = null): { id: string, promise: Promise } { + if (!id) { + id = "promise-" + makeid(8); + } + let lastPromise = this.nextPromise; + this.queueSize++; + let currentID = this.queueID; + let currentPromise=null; + currentPromise = (async () => { + this.queueSize--; + try { + await lastPromise; + }catch (e) { + /* ignore error of previous promise*/ + } + if(currentID!==this.queueID){ + throw new Error("canceled caused by new queue"); + } + if (this.canceled[id]) { + delete this.canceled[id]; + throw new Error("canceled by user"); + } + if(this.onlyLast && currentPromise!=this.nextPromise){ + throw new Error("canceled caused by not last"); + } + //execute task and return result + return task(); + })(); + currentPromise.catch(e=>{ + /* we catch the exception this way to avoid an uncaught exception in the console*/ + }); + this.nextPromise = currentPromise + return {id, promise: this.nextPromise}; + } + + cancel(id: string) { + this.canceled[id] = true; + } + + /** + * The last task in progress is not canceled + */ + async cancelAll(){ + this.queueID++;//create a new queue will cancel all previous task except the one in progress + await this.awaitQueueEmpty(); + this.nextPromise = Promise.resolve(null); + this.canceled = {}; + this.queueSize = 0; + } + +} diff --git a/packages/async-utils/src/index.ts b/packages/async-utils/src/index.ts new file mode 100644 index 0000000..c3aa8e5 --- /dev/null +++ b/packages/async-utils/src/index.ts @@ -0,0 +1,5 @@ +export * from "./ListenerCleaner"; +export * from "./PromiseQueue"; +export * from "./MultiQueue"; +export * from "./Id"; +export * from "./Lazy"; diff --git a/packages/async-utils/tsconfig.json b/packages/async-utils/tsconfig.json new file mode 100644 index 0000000..a81c579 --- /dev/null +++ b/packages/async-utils/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "experimentalDecorators": true,//needed for inversify + "module": "ESNext", + "target": "ESNext", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "noImplicitAny": false, + "noLib": false, + "skipLibCheck": true, + "removeComments": true, + "sourceMap": true, + "outDir": "./dist" + }, + "include": [ + "./src" + ], + "exclude": [ + "node_modules", + "test/*", + "examples/*" + ] +} diff --git a/packages/async-utils/tsup.config.js b/packages/async-utils/tsup.config.js new file mode 100644 index 0000000..85bb752 --- /dev/null +++ b/packages/async-utils/tsup.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsup' + +export default defineConfig({ + entry: ['src/index.ts'], + splitting: false, + sourcemap: true, + clean: true, + dts:true, + minify:true, + format: ['cjs', 'esm'], +}) diff --git a/packages/meta-sort-core/.env.example b/packages/meta-sort-core/.env.example new file mode 100644 index 0000000..557a417 --- /dev/null +++ b/packages/meta-sort-core/.env.example @@ -0,0 +1,102 @@ +# meta-sort-core Environment Configuration +# Copy this file to .env and adjust values as needed + +# ============================================================================= +# Required Settings +# ============================================================================= + +# Comma-separated list of folders to watch for media files (REQUIRED) +WATCH_FOLDER_LIST=/files + +# ============================================================================= +# Volume Paths (New Architecture) +# ============================================================================= + +# Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services +# This is where Redis data, leader election locks, and service discovery files are stored +META_CORE_PATH=/meta-core + +# Path to FILES_VOLUME - shared media volume +# All file paths in the KV database are relative to this path +FILES_PATH=/files + +# ============================================================================= +# KV Storage Configuration +# ============================================================================= + +# Use legacy etcd mode instead of Redis (default: false) +# Set to 'true' to use etcd instead of Redis +USE_ETCD=false + +# etcd API URL (only used if USE_ETCD=true) +# ETCD_URL=http://localhost:2379 + +# Direct Redis URL (skip leader election if provided) +# If set, the service will connect directly to this Redis instance +# instead of participating in leader election +# REDIS_URL=redis://localhost:6379 + +# Redis port for leader to listen on (default: 6379) +REDIS_PORT=6379 + +# ============================================================================= +# Service Identity +# ============================================================================= + +# Service name for leader election and service discovery +SERVICE_NAME=meta-sort + +# Service version +SERVICE_VERSION=1.0.0 + +# ============================================================================= +# API Server +# ============================================================================= + +# FUSE API server port (default: 3000) +FUSE_API_PORT=3000 + +# FUSE API server host (default: localhost) +FUSE_API_HOST=0.0.0.0 + +# ============================================================================= +# Processing Configuration +# ============================================================================= + +# Maximum number of worker threads for Piscina worker pool +# Default: CPU count +# MAX_WORKER_THREADS=16 + +# Interval in milliseconds for polling watch folders (0 to disable) +# WATCH_FOLDER_POOLING_INTERVAL_MS=0 + +# Metadata formats to generate: 'meta' (.meta YAML), 'jellyfin' (.nfo XML) +METADATA_FORMATS=meta + +# ============================================================================= +# Cache Configuration +# ============================================================================= + +# Path to hash index cache +INDEX_FOLDER_PATH=/data/cache/hash + +# Path to plugin cache folder +CACHE_FOLDER_PATH=/data/cache/plugins + +# ============================================================================= +# Jellyfin Integration (Optional) +# ============================================================================= + +# Jellyfin server endpoint +# JELLYFIN_ENDPOINT=http://localhost:8096 + +# Jellyfin API key +# JELLYFIN_API_KEY=your-api-key-here + +# ============================================================================= +# Polling Paths (Optional) +# ============================================================================= + +# JSON array of polling path configurations for paths that don't support inotify +# Example: '[{"path": "/files/smb/downloads", "intervalMs": 60000}]' +# POLLING_PATHS=[] diff --git a/packages/meta-sort-core/package.json b/packages/meta-sort-core/package.json new file mode 100644 index 0000000..b66fcbf --- /dev/null +++ b/packages/meta-sort-core/package.json @@ -0,0 +1,53 @@ +{ + "name": "@meta-sort/core", + "version": "1.0.0", + "description": "Core file sorting and metadata extraction service", + "main": "dist/index.js", + "type": "module", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "start": "node dist/index.js", + "exec": "node dist/index.js", + "test": "mocha", + "lint": "eslint src --ext ts" + }, + "license": "MIT", + "dependencies": { + "@fastify/cors": "^11.1.0", + "@fastify/static": "^8.3.0", + "@metazla/filename-tools": "github:worph/filename-tool", + "@metazla/meta-hash": "workspace:*", + "@metazla/meta-interface": "workspace:*", + "@worph/async-utils": "workspace:*", + "axios": "^1.6.7", + "bencode": "^4.0.0", + "chokidar": "^3.5.3", + "dockerode": "^4.0.2", + "dotenv": "^16.4.1", + "fastify": "^5.6.1", + "fluent-ffmpeg": "^2.1.2", + "franc-min": "^6.2.0", + "ioredis": "^5.4.1", + "iso-639-1": "^3.1.2", + "lodash": "^4.17.21", + "multiformats": "^13.1.0", + "p-queue": "^8.0.1", + "piscina": "^4.4.0", + "tslog": "^4.9.2", + "wanakana": "^5.3.1", + "xml2js": "^0.6.2", + "yaml": "^2.4.3" + }, + "devDependencies": { + "@types/chai": "^4.3.11", + "@types/dockerode": "^3.3.31", + "@types/fluent-ffmpeg": "^2.1.24", + "@types/lodash": "^4.17.0", + "@types/mocha": "^10.0.6", + "@types/node": "^20.11.16", + "chai": "^5.0.3", + "mocha": "^10.2.0", + "typescript": "^5.3.3" + } +} diff --git a/packages/meta-sort-core/src/api/DuplicatesAPI.ts b/packages/meta-sort-core/src/api/DuplicatesAPI.ts new file mode 100644 index 0000000..f928070 --- /dev/null +++ b/packages/meta-sort-core/src/api/DuplicatesAPI.ts @@ -0,0 +1,166 @@ +/** + * HTTP REST API Server for Duplicate Files + * + * Provides endpoints for accessing duplicate file detection results. + * Separate from FUSE API to maintain proper separation of concerns. + */ + +import * as http from 'http'; + +export interface DuplicatesAPIConfig { + /** Port to listen on (default: 3003) */ + port?: number; + /** Host to bind to (default: 'localhost') */ + host?: string; + /** Enable CORS (default: true) */ + enableCors?: boolean; + /** Mount path for duplicates endpoint (default: '/duplicates') */ + mountPath?: string; +} + +export class DuplicatesAPI { + private server: http.Server | null = null; + private config: Required; + + constructor(config: DuplicatesAPIConfig = {}) { + this.config = { + port: config.port ?? 3003, + host: config.host ?? 'localhost', + enableCors: config.enableCors ?? true, + mountPath: config.mountPath ?? '/duplicates' + }; + } + + /** + * Start the HTTP server + */ + public async start(): Promise { + return new Promise((resolve, reject) => { + this.server = http.createServer(this.handleRequest.bind(this)); + + this.server.on('error', reject); + + this.server.listen(this.config.port, this.config.host, () => { + console.log(`Duplicates API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`); + resolve(); + }); + }); + } + + /** + * Stop the HTTP server + */ + public async stop(): Promise { + if (!this.server) { + return; + } + + return new Promise((resolve, reject) => { + this.server!.close((err) => { + if (err) { + reject(err); + } else { + console.log('Duplicates API stopped'); + this.server = null; + resolve(); + } + }); + }); + } + + /** + * Handle incoming HTTP requests + */ + private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { + // CORS headers + if (this.config.enableCors) { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type'); + + if (req.method === 'OPTIONS') { + res.writeHead(204); + res.end(); + return; + } + } + + try { + const url = new URL(req.url!, `http://${req.headers.host}`); + const pathname = url.pathname; + const method = req.method!; + + // Route the request + if (method === 'GET' && pathname === this.config.mountPath) { + await this.handleDuplicates(res); + } else if (method === 'GET' && pathname === '/health') { + this.handleHealth(res); + } else { + this.sendError(res, 404, 'Not Found'); + } + } catch (error) { + console.error('Error handling request:', error); + this.sendError(res, 500, 'Internal Server Error', error); + } + } + + /** + * Send JSON response + */ + private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void { + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(data, null, 2)); + } + + /** + * Send error response + */ + private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void { + this.sendJson(res, { + error: message, + details: error?.message ?? error + }, statusCode); + } + + /** + * GET /duplicates + * Get duplicate files + * NOTE: This API is deprecated. Use UnifiedAPIServer's /api/duplicates endpoint instead. + * Disk-based duplicate logging has been removed for performance. + */ + private async handleDuplicates(res: http.ServerResponse): Promise { + try { + // Return empty results - duplicate detection now happens in-memory via UnifiedAPIServer + const hashDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = []; + const titleDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = []; + + this.sendJson(res, { + hashDuplicates, + titleDuplicates, + stats: { + hashGroupCount: 0, + hashFileCount: 0, + titleGroupCount: 0, + titleFileCount: 0 + }, + deprecated: true, + message: 'This API is deprecated. Use UnifiedAPIServer /api/duplicates endpoint instead.' + }); + } catch (error) { + console.error('Error in duplicates endpoint:', error); + this.sendError(res, 500, 'Failed to get duplicate data', error); + } + } + + /** + * GET /health + * Health check endpoint + */ + private handleHealth(res: http.ServerResponse): void { + this.sendJson(res, { + status: 'ok', + service: 'duplicates-api', + timestamp: new Date().toISOString() + }); + } +} diff --git a/packages/meta-sort-core/src/api/FuseAPI.ts b/packages/meta-sort-core/src/api/FuseAPI.ts new file mode 100644 index 0000000..a999cb5 --- /dev/null +++ b/packages/meta-sort-core/src/api/FuseAPI.ts @@ -0,0 +1,173 @@ +/** + * FUSE API for meta-mesh + * + * This API provides a virtual filesystem interface that can be consumed by a FUSE driver. + * It exposes the computed folder structure without creating actual hardlinks. + */ + +import { HashMeta } from '@metazla/meta-interface'; +import { Stats } from 'fs'; + +/** + * Represents a file or directory in the virtual filesystem + */ +export interface VirtualNode { + /** Virtual path in the output filesystem */ + path: string; + /** Node type */ + type: 'file' | 'directory'; + /** File name */ + name: string; + /** Parent directory path */ + parent: string; +} + +/** + * Represents a regular file in the virtual filesystem + */ +export interface VirtualFile extends VirtualNode { + type: 'file'; + /** Source file path on the real filesystem (null for virtual files like .meta) */ + sourcePath: string | null; + /** File size in bytes */ + size: number; + /** Last modified time */ + mtime: Date; + /** Creation time */ + ctime: Date; + /** File permissions (octal) */ + mode: number; + /** Associated metadata */ + metadata?: HashMeta; +} + +/** + * Represents a directory in the virtual filesystem + */ +export interface VirtualDirectory extends VirtualNode { + type: 'directory'; + /** Child nodes (file/directory names) */ + children: string[]; + /** Directory permissions (octal) */ + mode: number; + /** Last modified time */ + mtime: Date; +} + +/** + * File attributes returned by getattr() - compatible with FUSE stat structure + */ +export interface FileAttributes { + /** File size in bytes */ + size: number; + /** File mode (permissions + type) */ + mode: number; + /** Last modified time (Unix timestamp in seconds) */ + mtime: number; + /** Last access time (Unix timestamp in seconds) */ + atime: number; + /** Creation/change time (Unix timestamp in seconds) */ + ctime: number; + /** Number of hard links */ + nlink: number; + /** User ID */ + uid: number; + /** Group ID */ + gid: number; +} + +/** + * Read result for file content + */ +export interface ReadResult { + /** For regular files: source path to read from. For virtual files: null */ + sourcePath: string | null; + /** For virtual files: generated content. For regular files: null */ + content: Buffer | null; + /** Total file size */ + size: number; +} + +/** + * FUSE API operations + */ +export interface IFuseAPI { + /** + * List directory contents + * @param path Virtual directory path (e.g., "/Anime/Naruto/Season 1") + * @returns Array of file/directory names in the directory, or null if not found + */ + readdir(path: string): string[] | null; + + /** + * Get file or directory attributes + * @param path Virtual file or directory path + * @returns File attributes, or null if not found + */ + getattr(path: string): FileAttributes | null; + + /** + * Check if a path exists + * @param path Virtual path + * @returns true if path exists, false otherwise + */ + exists(path: string): boolean; + + /** + * Get read information for a file + * @param path Virtual file path + * @returns Read result with source path or content, or null if not found + */ + read(path: string): ReadResult | null; + + /** + * Get metadata for a file + * @param path Virtual file path + * @returns HashMeta if available, null otherwise + */ + getMetadata(path: string): HashMeta | null; + + /** + * Get the complete virtual filesystem tree + * @returns Root directory node + */ + getTree(): VirtualDirectory; + + /** + * Refresh the virtual filesystem from current state + * Called after processing completes to update the VFS + */ + refresh(): Promise; +} + +/** + * Events emitted by the FUSE API + */ +export interface FuseAPIEvents { + /** Emitted when the virtual filesystem is updated */ + 'vfs-updated': () => void; + /** Emitted when a file is added */ + 'file-added': (path: string) => void; + /** Emitted when a file is updated (e.g., metadata changed) */ + 'file-updated': (path: string) => void; + /** Emitted when a file is removed */ + 'file-removed': (path: string) => void; + /** Emitted when a directory is added */ + 'directory-added': (path: string) => void; +} + +/** + * Configuration for FUSE API + */ +export interface FuseAPIConfig { + /** Base path for the virtual filesystem (default: "/") */ + basePath?: string; + /** Default file permissions (default: 0o644) */ + fileMode?: number; + /** Default directory permissions (default: 0o755) */ + directoryMode?: number; + /** User ID for files (default: process.getuid()) */ + uid?: number; + /** Group ID for files (default: process.getgid()) */ + gid?: number; +} diff --git a/packages/meta-sort-core/src/api/FuseAPIServer.ts b/packages/meta-sort-core/src/api/FuseAPIServer.ts new file mode 100644 index 0000000..f4130bf --- /dev/null +++ b/packages/meta-sort-core/src/api/FuseAPIServer.ts @@ -0,0 +1,345 @@ +/** + * HTTP REST API Server for FUSE operations + * + * Exposes the VirtualFileSystem over HTTP for consumption by a FUSE driver. + */ + +import * as http from 'http'; +import { VirtualFileSystem } from './VirtualFileSystem.js'; +import { FileAttributes, ReadResult } from './FuseAPI.js'; + +export interface FuseAPIServerConfig { + /** Port to listen on (default: 3000) */ + port?: number; + /** Host to bind to (default: 'localhost') */ + host?: string; + /** Enable CORS (default: false) */ + enableCors?: boolean; +} + +export class FuseAPIServer { + private server: http.Server | null = null; + private vfs: VirtualFileSystem; + private config: Required; + + constructor(vfs: VirtualFileSystem, config: FuseAPIServerConfig = {}) { + this.vfs = vfs; + this.config = { + port: config.port ?? 3000, + host: config.host ?? 'localhost', + enableCors: config.enableCors ?? false + }; + } + + /** + * Start the HTTP server + */ + public async start(): Promise { + return new Promise((resolve, reject) => { + this.server = http.createServer(this.handleRequest.bind(this)); + + this.server.on('error', reject); + + this.server.listen(this.config.port, this.config.host, () => { + console.log(`FUSE API Server listening on http://${this.config.host}:${this.config.port}`); + resolve(); + }); + }); + } + + /** + * Stop the HTTP server + */ + public async stop(): Promise { + if (!this.server) { + return; + } + + return new Promise((resolve, reject) => { + this.server!.close((err) => { + if (err) { + reject(err); + } else { + console.log('FUSE API Server stopped'); + this.server = null; + resolve(); + } + }); + }); + } + + /** + * Handle incoming HTTP requests + */ + private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { + // CORS headers + if (this.config.enableCors) { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type'); + + if (req.method === 'OPTIONS') { + res.writeHead(204); + res.end(); + return; + } + } + + try { + const url = new URL(req.url!, `http://${req.headers.host}`); + const pathname = url.pathname; + const method = req.method!; + + // Route the request + if (method === 'GET' && pathname === '/api/fuse/health') { + this.handleHealth(res); + } else if (method === 'GET' && pathname === '/api/fuse/stats') { + this.handleStats(res); + } else if (method === 'POST' && pathname === '/api/fuse/readdir') { + await this.handleReaddir(req, res); + } else if (method === 'POST' && pathname === '/api/fuse/getattr') { + await this.handleGetattr(req, res); + } else if (method === 'POST' && pathname === '/api/fuse/exists') { + await this.handleExists(req, res); + } else if (method === 'POST' && pathname === '/api/fuse/read') { + await this.handleRead(req, res); + } else if (method === 'POST' && pathname === '/api/fuse/metadata') { + await this.handleMetadata(req, res); + } else if (method === 'GET' && pathname === '/api/fuse/tree') { + this.handleTree(res); + } else if (method === 'GET' && pathname === '/api/fuse/files') { + this.handleFiles(res); + } else if (method === 'GET' && pathname === '/api/fuse/directories') { + this.handleDirectories(res); + } else if (method === 'POST' && pathname === '/api/fuse/refresh') { + await this.handleRefresh(req, res); + } else { + this.sendError(res, 404, 'Not Found'); + } + } catch (error) { + console.error('Error handling request:', error); + this.sendError(res, 500, 'Internal Server Error', error); + } + } + + /** + * Parse JSON body from request + */ + private async parseBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk.toString(); + }); + req.on('end', () => { + try { + resolve(JSON.parse(body)); + } catch (error) { + reject(new Error('Invalid JSON')); + } + }); + req.on('error', reject); + }); + } + + /** + * Send JSON response + */ + private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void { + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(data)); + } + + /** + * Send error response + */ + private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void { + this.sendJson(res, { + error: message, + details: error?.message ?? error + }, statusCode); + } + + // API Handlers + + /** + * GET /api/health + * Health check endpoint + */ + private handleHealth(res: http.ServerResponse): void { + this.sendJson(res, { + status: 'ok', + timestamp: new Date().toISOString() + }); + } + + /** + * GET /api/stats + * Get VFS statistics + */ + private handleStats(res: http.ServerResponse): void { + const stats = this.vfs.getStats(); + this.sendJson(res, stats); + } + + /** + * POST /api/readdir + * List directory contents + * Body: { path: string } + * Response: { entries: string[] } | { error: string } + */ + private async handleReaddir(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const { path } = await this.parseBody(req); + + if (typeof path !== 'string') { + this.sendError(res, 400, 'Missing or invalid "path" parameter'); + return; + } + + const entries = this.vfs.readdir(path); + + if (entries === null) { + this.sendError(res, 404, 'Directory not found'); + return; + } + + this.sendJson(res, { entries }); + } + + /** + * POST /api/getattr + * Get file attributes + * Body: { path: string } + * Response: FileAttributes | { error: string } + */ + private async handleGetattr(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const { path } = await this.parseBody(req); + + if (typeof path !== 'string') { + this.sendError(res, 400, 'Missing or invalid "path" parameter'); + return; + } + + const attrs = this.vfs.getattr(path); + + if (attrs === null) { + this.sendError(res, 404, 'Path not found'); + return; + } + + this.sendJson(res, attrs); + } + + /** + * POST /api/exists + * Check if path exists + * Body: { path: string } + * Response: { exists: boolean } + */ + private async handleExists(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const { path } = await this.parseBody(req); + + if (typeof path !== 'string') { + this.sendError(res, 400, 'Missing or invalid "path" parameter'); + return; + } + + const exists = this.vfs.exists(path); + this.sendJson(res, { exists }); + } + + /** + * POST /api/read + * Read file + * Body: { path: string } + * Response: ReadResult | { error: string } + */ + private async handleRead(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const { path } = await this.parseBody(req); + + if (typeof path !== 'string') { + this.sendError(res, 400, 'Missing or invalid "path" parameter'); + return; + } + + const result = this.vfs.read(path); + + if (result === null) { + this.sendError(res, 404, 'File not found'); + return; + } + + // Convert Buffer to base64 for JSON serialization + const response: any = { + sourcePath: result.sourcePath, + size: result.size + }; + + if (result.content !== null) { + response.content = result.content.toString('base64'); + response.contentEncoding = 'base64'; + } + + this.sendJson(res, response); + } + + /** + * POST /api/metadata + * Get file metadata + * Body: { path: string } + * Response: HashMeta | { error: string } + */ + private async handleMetadata(req: http.IncomingMessage, res: http.ServerResponse): Promise { + const { path } = await this.parseBody(req); + + if (typeof path !== 'string') { + this.sendError(res, 400, 'Missing or invalid "path" parameter'); + return; + } + + const metadata = this.vfs.getMetadata(path); + + if (metadata === null) { + this.sendError(res, 404, 'Metadata not found'); + return; + } + + this.sendJson(res, metadata); + } + + /** + * GET /api/tree + * Get complete VFS tree + */ + private handleTree(res: http.ServerResponse): void { + const tree = this.vfs.getTree(); + this.sendJson(res, tree); + } + + /** + * GET /api/files + * Get all file paths + */ + private handleFiles(res: http.ServerResponse): void { + const files = this.vfs.getAllFiles(); + this.sendJson(res, { files }); + } + + /** + * GET /api/directories + * Get all directory paths + */ + private handleDirectories(res: http.ServerResponse): void { + const directories = this.vfs.getAllDirectories(); + this.sendJson(res, { directories }); + } + + /** + * POST /api/refresh + * Refresh the VFS (no-op, VFS is updated by processing pipeline) + */ + private async handleRefresh(req: http.IncomingMessage, res: http.ServerResponse): Promise { + await this.vfs.refresh(); + this.sendJson(res, { status: 'ok' }); + } +} diff --git a/packages/meta-sort-core/src/api/MetricsAPI.ts b/packages/meta-sort-core/src/api/MetricsAPI.ts new file mode 100644 index 0000000..e836ee1 --- /dev/null +++ b/packages/meta-sort-core/src/api/MetricsAPI.ts @@ -0,0 +1,160 @@ +/** + * HTTP REST API Server for Performance Metrics + * + * Provides a dedicated endpoint for exposing application performance metrics. + * Separate from FUSE API to maintain proper separation of concerns. + */ + +import * as http from 'http'; +import { performanceMetrics } from '../metrics/PerformanceMetrics.js'; + +export interface MetricsAPIConfig { + /** Port to listen on (default: 3001) */ + port?: number; + /** Host to bind to (default: 'localhost') */ + host?: string; + /** Enable CORS (default: true for metrics) */ + enableCors?: boolean; + /** Mount path for metrics endpoint (default: '/metrics') */ + mountPath?: string; +} + +export class MetricsAPI { + private server: http.Server | null = null; + private config: Required; + + constructor(config: MetricsAPIConfig = {}) { + this.config = { + port: config.port ?? 3001, + host: config.host ?? 'localhost', + enableCors: config.enableCors ?? true, + mountPath: config.mountPath ?? '/metrics' + }; + } + + /** + * Start the HTTP server + */ + public async start(): Promise { + return new Promise((resolve, reject) => { + this.server = http.createServer(this.handleRequest.bind(this)); + + this.server.on('error', reject); + + this.server.listen(this.config.port, this.config.host, () => { + console.log(`Metrics API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`); + resolve(); + }); + }); + } + + /** + * Stop the HTTP server + */ + public async stop(): Promise { + if (!this.server) { + return; + } + + return new Promise((resolve, reject) => { + this.server!.close((err) => { + if (err) { + reject(err); + } else { + console.log('Metrics API stopped'); + this.server = null; + resolve(); + } + }); + }); + } + + /** + * Handle incoming HTTP requests + */ + private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { + // CORS headers + if (this.config.enableCors) { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type'); + + if (req.method === 'OPTIONS') { + res.writeHead(204); + res.end(); + return; + } + } + + try { + const url = new URL(req.url!, `http://${req.headers.host}`); + const pathname = url.pathname; + const method = req.method!; + + // Route the request + if (method === 'GET' && pathname === this.config.mountPath) { + this.handleMetrics(res); + } else if (method === 'GET' && pathname === '/health') { + this.handleHealth(res); + } else if (method === 'POST' && pathname === '/reset') { + this.handleReset(res); + } else { + this.sendError(res, 404, 'Not Found'); + } + } catch (error) { + console.error('Error handling request:', error); + this.sendError(res, 500, 'Internal Server Error', error); + } + } + + /** + * Send JSON response + */ + private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void { + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(data, null, 2)); + } + + /** + * Send error response + */ + private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void { + this.sendJson(res, { + error: message, + details: error?.message ?? error + }, statusCode); + } + + /** + * GET /metrics + * Get performance metrics + */ + private handleMetrics(res: http.ServerResponse): void { + const metrics = performanceMetrics.getMetrics(); + this.sendJson(res, metrics); + } + + /** + * GET /health + * Health check endpoint + */ + private handleHealth(res: http.ServerResponse): void { + this.sendJson(res, { + status: 'ok', + service: 'metrics-api', + timestamp: new Date().toISOString() + }); + } + + /** + * POST /reset + * Reset all metrics + */ + private handleReset(res: http.ServerResponse): void { + performanceMetrics.reset(); + this.sendJson(res, { + status: 'ok', + message: 'Metrics reset successfully' + }); + } +} diff --git a/packages/meta-sort-core/src/api/README.md b/packages/meta-sort-core/src/api/README.md new file mode 100644 index 0000000..0b1bb66 --- /dev/null +++ b/packages/meta-sort-core/src/api/README.md @@ -0,0 +1,299 @@ +# FUSE API for Meta-Mesh + +This module provides a FUSE-compatible HTTP API that runs alongside meta-mesh's hardlink system. The API exposes a virtual filesystem that can be consumed by a future FUSE driver. + +## Overview + +The FUSE API runs **always-on alongside hardlinks**, providing: + +- **Dual output**: Both hardlinks (for local access) and virtual filesystem (for FUSE driver) +- **Dynamic updates**: VirtualFileSystem rebuilds on every processing cycle +- **Virtual .meta files**: Metadata files are generated on-the-fly without disk I/O +- **HTTP REST API**: Can be consumed by local or remote FUSE drivers +- **Zero configuration**: Always enabled, no opt-in required + +## Architecture + +### Components + +1. **VirtualFileSystem** - In-memory representation of the output directory structure +2. **FuseAPIServer** - HTTP REST API server exposing FUSE operations +3. **FuseAPI** - TypeScript interfaces defining the API contract + +### Data Flow + +``` +Input Files → File Processing → Metadata Extraction → Duplicate Removal + ↓ +Virtual Folder Structure Computation (diff generation) + ↓ + ├─→ VirtualFileSystem Build (in-memory) → FUSE API Server → Future FUSE Driver + │ + └─→ Hardlink Creation (on disk) → Direct File Access +``` + +Both outputs are generated in parallel from the same computed folder structure. + +## Configuration + +The FUSE API is **always enabled** and runs alongside hardlinks. You can optionally configure the server port and host: + +```bash +# FUSE API Server configuration (optional) +FUSE_API_PORT=3000 # Default: 3000 +FUSE_API_HOST=localhost # Default: localhost +``` + +The API server starts automatically when meta-mesh starts. + +## API Endpoints + +The FUSE API Server exposes the following HTTP REST endpoints: + +### Health & Info + +- **GET /api/health** - Health check + ```json + { "status": "ok", "timestamp": "2025-01-15T10:30:00.000Z" } + ``` + +- **GET /api/stats** - VFS statistics + ```json + { + "fileCount": 1250, + "directoryCount": 45, + "totalSize": 52428800, + "metaFileCount": 1250 + } + ``` + +### Directory Operations + +- **POST /api/readdir** - List directory contents + ```json + // Request + { "path": "/Anime/Naruto/Season 1" } + + // Response + { "entries": ["Naruto S01E01.mkv", "Naruto S01E01.mkv.meta", ...] } + ``` + +### File Operations + +- **POST /api/getattr** - Get file attributes + ```json + // Request + { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" } + + // Response + { + "size": 524288000, + "mode": 33188, // 0o100644 (regular file, rw-r--r--) + "mtime": 1705315200, + "atime": 1705315200, + "ctime": 1705315200, + "nlink": 1, + "uid": 1000, + "gid": 1000 + } + ``` + +- **POST /api/exists** - Check if path exists + ```json + // Request + { "path": "/Anime/Naruto" } + + // Response + { "exists": true } + ``` + +- **POST /api/read** - Read file + ```json + // Request + { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" } + + // Response (regular file) + { + "sourcePath": "/data/input/naruto_01.mkv", + "size": 524288000 + } + + // Response (virtual .meta file) + { + "sourcePath": null, + "content": "..base64-encoded content..", + "contentEncoding": "base64", + "size": 1024 + } + ``` + +- **POST /api/metadata** - Get file metadata + ```json + // Request + { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" } + + // Response + { + "title": "Naruto", + "season": "01", + "episode": "01", + "anime": true, + "sha256": "abc123...", + // ... full HashMeta object + } + ``` + +### Tree Operations + +- **GET /api/tree** - Get complete VFS tree structure +- **GET /api/files** - Get all file paths +- **GET /api/directories** - Get all directory paths +- **POST /api/refresh** - Refresh VFS (no-op, VFS updates automatically) + +## Usage Example + +### Starting meta-mesh with FUSE API + +The FUSE API starts automatically with meta-mesh: + +```bash +# Run meta-mesh (FUSE API starts automatically on port 3000) +pnpm run start:mesh + +# Or customize the port +export FUSE_API_PORT=8080 +pnpm run start:mesh +``` + +You'll see output like: +``` +FUSE API Server started on http://localhost:3000 +VirtualFileSystem build took 15ms +VirtualFileSystem: 1250 files (1250 .meta), 45 directories, 50.00 MB +``` + +### Consuming the API + +```typescript +import { VirtualFileSystem, FuseAPIServer } from '@metazla/meta-mesh/api'; + +// Create VFS instance +const vfs = new VirtualFileSystem({ + fileMode: 0o644, + directoryMode: 0o755, + uid: process.getuid(), + gid: process.getgid() +}); + +// Build from computed structure +vfs.buildFromComputed(sourceToDestMap, metadataMap); + +// Start API server +const server = new FuseAPIServer(vfs, { + port: 3000, + host: 'localhost' +}); + +await server.start(); +``` + +### FUSE Driver Integration + +A FUSE driver would make HTTP requests to these endpoints: + +```typescript +// Example pseudo-code for FUSE driver +async function readdir(path: string): Promise { + const response = await fetch('http://localhost:3000/api/readdir', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path }) + }); + + const { entries } = await response.json(); + return entries; +} + +async function read(path: string): Promise { + const response = await fetch('http://localhost:3000/api/read', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path }) + }); + + const result = await response.json(); + + if (result.sourcePath) { + // Regular file - read from source + return fs.readFileSync(result.sourcePath); + } else { + // Virtual file - use provided content + return Buffer.from(result.content, 'base64'); + } +} +``` + +## Virtual .meta Files + +`.meta` files are entirely virtual and generated on-the-fly: + +- **No disk I/O**: Content is serialized from in-memory metadata +- **Always up-to-date**: Reflects current metadata state +- **YAML format**: Uses `MetaMeshFormat.serialize()` for consistency +- **No source path**: `sourcePath` is `null`, `content` contains the YAML + +## Events + +The VirtualFileSystem emits events for monitoring: + +```typescript +vfs.on('vfs-updated', () => { + console.log('Virtual filesystem updated'); +}); + +vfs.on('file-added', (path: string) => { + console.log(`File added: ${path}`); +}); + +vfs.on('file-removed', (path: string) => { + console.log(`File removed: ${path}`); +}); + +vfs.on('directory-added', (path: string) => { + console.log(`Directory added: ${path}`); +}); +``` + +## Performance + +The VirtualFileSystem is highly performant: + +- **In-memory**: All operations are O(1) or O(n) in-memory +- **No disk I/O**: Except for reading source files when FUSE driver requests content +- **Lazy .meta generation**: Content is only serialized when requested +- **Fast updates**: Rebuilding the VFS takes milliseconds + +## Dual Output: Hardlinks + FUSE API + +Meta-mesh now provides **both** output methods simultaneously: + +| Feature | Hardlinks | FUSE API | Best For | +|---------|-----------|----------|----------| +| Disk usage | 1 inode per file | 0 (virtual) | Hardlinks: local use | +| Update speed | Slow (file I/O) | Fast (in-memory) | FUSE: frequent updates | +| .meta files | Written to disk | Virtual (on-demand) | FUSE: dynamic metadata | +| Remote access | No | Yes (HTTP) | FUSE: network drives | +| Compatibility | Any filesystem | Requires FUSE driver | Hardlinks: broad compat | +| Access method | Direct file access | HTTP + FUSE driver | Hardlinks: native apps | + +**Use hardlinks** for local applications like Jellyfin, Plex, Kodi that read files directly. + +**Use FUSE API** for future FUSE drivers that want dynamic, virtual filesystem access. + +## Future Enhancements + +- **WebSocket support**: Real-time updates for FUSE drivers +- **Authentication**: API key or JWT-based auth +- **Compression**: Gzip/Brotli for API responses +- **Caching**: ETag/Last-Modified headers +- **Batch operations**: Multi-file operations in single request diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts new file mode 100644 index 0000000..cbba3ca --- /dev/null +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -0,0 +1,2145 @@ +/** + * Unified HTTP API Server using Fastify + * + * Consolidates FUSE API, Metrics API, and Duplicates API into a single server. + * This simplifies deployment and reduces the number of ports needed. + */ + +import Fastify, { FastifyInstance } from 'fastify'; +import fastifyCors from '@fastify/cors'; +import { execSync } from 'child_process'; +import { VirtualFileSystem } from './VirtualFileSystem.js'; +import { performanceMetrics } from '../metrics/PerformanceMetrics.js'; +import { UnifiedProcessingStateManager } from '../logic/UnifiedProcessingStateManager.js'; +import { DuplicateResult } from '../logic/DuplicateFinder.js'; +import type { IKVClient } from '../kv/IKVClient.js'; +import type { KVManager } from '../kv/KVManager.js'; +import { config } from '../config/EnvConfig.js'; +import type { PluginManager } from '../plugin-engine/PluginManager.js'; +import type { TaskScheduler } from '../plugin-engine/TaskScheduler.js'; +import type { ContainerManager, ContainerPluginScheduler, PluginCallbackPayload } from '../container-plugins/index.js'; + +export interface UnifiedAPIServerConfig { + /** Port to listen on (default: 3000) */ + port?: number; + /** Host to bind to (default: 'localhost') */ + host?: string; + /** Enable CORS (default: true) */ + enableCors?: boolean; +} + +export class UnifiedAPIServer { + private app: FastifyInstance; + private vfs: VirtualFileSystem; + private unifiedStateManager: UnifiedProcessingStateManager | null = null; + private getDuplicateResult: (() => DuplicateResult | null) | null = null; + private kvClient: IKVClient | null = null; + private kvManager: KVManager | null = null; + private triggerScanCallback: (() => Promise) | null = null; + private getQueueStatus: (() => any) | null = null; + private fastQueueConcurrency: number | undefined = undefined; + private backgroundQueueConcurrency: number | undefined = undefined; + private config: Required; + private getPluginManager: (() => PluginManager | null) | null = null; + private getTaskScheduler: (() => TaskScheduler | null) | null = null; + private containerManager: ContainerManager | null = null; + private containerPluginScheduler: ContainerPluginScheduler | null = null; + + // Cache for total file size (avoid running du on every request) + private totalSizeCache: { value: number; timestamp: number } | null = null; + private static TOTAL_SIZE_CACHE_TTL_MS = 30000; // 30 seconds + + constructor( + vfs: VirtualFileSystem, + config: UnifiedAPIServerConfig = {}, + unifiedStateManager?: UnifiedProcessingStateManager, + getDuplicateResult?: () => DuplicateResult | null, + kvClient?: IKVClient, + triggerScanCallback?: () => Promise, + backgroundQueueConcurrency?: number, + fastQueueConcurrency?: number, + getQueueStatus?: () => any, + kvManager?: KVManager, + getPluginManager?: () => PluginManager | null, + getTaskScheduler?: () => TaskScheduler | null + ) { + this.vfs = vfs; + this.unifiedStateManager = unifiedStateManager || null; + this.getDuplicateResult = getDuplicateResult || null; + this.kvClient = kvClient || null; + this.kvManager = kvManager || null; + this.triggerScanCallback = triggerScanCallback || null; + this.getQueueStatus = getQueueStatus || null; + this.backgroundQueueConcurrency = backgroundQueueConcurrency; + this.fastQueueConcurrency = fastQueueConcurrency; + this.getPluginManager = getPluginManager || null; + this.getTaskScheduler = getTaskScheduler || null; + this.config = { + port: config.port ?? 3000, + host: config.host ?? 'localhost', + enableCors: config.enableCors ?? true + }; + + // Initialize Fastify + this.app = Fastify({ + logger: false, + bodyLimit: 10 * 1024 * 1024 // 10MB + }); + + this.setupMiddleware(); + this.setupRoutes(); + } + + /** + * Setup middleware (CORS, static files, etc.) + */ + private setupMiddleware(): void { + // CORS + if (this.config.enableCors) { + this.app.register(fastifyCors, { + origin: '*', + methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'] + }); + } + } + + /** + * Get total size of watch folders using du command (cached) + */ + private getTotalWatchFolderSize(): number { + // Check cache + const now = Date.now(); + if (this.totalSizeCache && (now - this.totalSizeCache.timestamp) < UnifiedAPIServer.TOTAL_SIZE_CACHE_TTL_MS) { + return this.totalSizeCache.value; + } + + // Calculate total size using du -sb on each watch folder + let totalSize = 0; + const watchFolders = config.WATCH_FOLDER_LIST?.split(',').map(f => f.trim()).filter(f => f) || []; + + for (const folder of watchFolders) { + try { + // du -sb gives total bytes for directory + const output = execSync(`du -sb "${folder}" 2>/dev/null || echo "0"`, { + encoding: 'utf8', + timeout: 10000 // 10s timeout per folder + }); + const bytes = parseInt(output.split('\t')[0], 10); + if (!isNaN(bytes)) { + totalSize += bytes; + } + } catch (error) { + // Skip folders that fail (permission issues, doesn't exist, etc.) + console.warn(`Failed to get size for folder ${folder}:`, error); + } + } + + // Update cache + this.totalSizeCache = { value: totalSize, timestamp: now }; + return totalSize; + } + + /** + * Setup all API routes + */ + private setupRoutes(): void { + // Root health check + this.app.get('/health', async (request, reply) => { + return { + status: 'ok', + timestamp: new Date().toISOString(), + services: { + fuse: 'ok', + metrics: 'ok', + duplicates: 'ok', + unifiedProcessing: this.unifiedStateManager ? 'ok' : 'unavailable' + } + }; + }); + + // API health check (for dashboard UI) + this.app.get('/api/health', async (request, reply) => { + const redisOk = this.kvClient ? await this.kvClient.health() : false; + const metrics = performanceMetrics.getMetrics(); + + return { + status: redisOk ? 'ok' : 'degraded', + redis: redisOk, + uptime: Math.floor(metrics.uptime / 1000), // in seconds + version: '1.0.0', + timestamp: new Date().toISOString() + }; + }); + + // FUSE API routes (under /api/fuse) + this.setupFuseRoutes(); + + // Metrics API routes (under /api/metrics) + this.setupMetricsRoutes(); + + // Duplicates API routes (under /api/duplicates) + this.setupDuplicatesRoutes(); + + // Metadata API routes (under /api/metadata) - Nested key architecture + if (this.kvClient) { + this.setupMetadataRoutes(); + } + + // KV Browser API routes (under /api/kv) - For editor UI + if (this.kvClient) { + this.setupKVBrowserRoutes(); + } + + // Unified processing state routes (under /api/processing) + if (this.unifiedStateManager) { + this.setupUnifiedProcessingRoutes(); + } + + // Scan trigger routes (under /api/scan) + this.setupScanRoutes(); + + // Stats routes (under /api/stats) - Redis stats for monitor UI + if (this.kvClient) { + this.setupStatsRoutes(); + } + + // Service discovery routes (under /api/services) + if (this.kvManager) { + this.setupServicesRoutes(); + } + + // Plugin management routes (under /api/plugins) + // Always register - routes check for plugin manager at request time + this.setupPluginRoutes(); + + // Container plugin routes (under /api/plugins/containers and /api/plugins/callback) + this.setupContainerPluginRoutes(); + + // Meta-core compatible routes (under /meta/) - for container plugins to write metadata + if (this.kvClient) { + this.setupMetaCoreRoutes(); + } + + // File download routes (under /api/file) + this.setupFileRoutes(); + } + + /** + * Setup File download routes + */ + private setupFileRoutes(): void { + // Download file by path + this.app.get<{ Querystring: { path: string } }>('/api/file/download', async (request, reply) => { + const { path: filePath } = request.query; + + if (!filePath) { + return reply.status(400).send({ error: 'Missing path parameter' }); + } + + try { + const fs = await import('fs'); + const pathModule = await import('path'); + + // Security: ensure the path is within FILES_PATH + const filesPath = config.FILES_PATH; + const absolutePath = pathModule.resolve(filePath); + + if (!absolutePath.startsWith(filesPath)) { + return reply.status(403).send({ error: 'Access denied - path outside files directory' }); + } + + // Check if file exists + if (!fs.existsSync(absolutePath)) { + return reply.status(404).send({ error: 'File not found' }); + } + + // Get file stats + const stats = fs.statSync(absolutePath); + if (!stats.isFile()) { + return reply.status(400).send({ error: 'Path is not a file' }); + } + + // Set headers for download + const filename = pathModule.basename(absolutePath); + reply.header('Content-Disposition', `attachment; filename="${encodeURIComponent(filename)}"`); + reply.header('Content-Type', 'application/octet-stream'); + reply.header('Content-Length', stats.size); + + // Stream the file + const stream = fs.createReadStream(absolutePath); + return reply.send(stream); + } catch (error) { + console.error('Error downloading file:', error); + return reply.status(500).send({ + error: 'Failed to download file', + details: (error as Error).message + }); + } + }); + } + + /** + * Setup FUSE API routes + */ + private setupFuseRoutes(): void { + // Health check + this.app.get('/api/fuse/health', async (request, reply) => { + return { + status: 'ok', + timestamp: new Date().toISOString() + }; + }); + + // VFS statistics + this.app.get('/api/fuse/stats', async (request, reply) => { + return this.vfs.getStats(); + }); + + // Database statistics (KV) + this.app.get('/api/fuse/db-stats', async (request, reply) => { + if (!this.kvClient) { + return reply.status(503).send({ + error: 'KV client not available' + }); + } + + try { + // Count files in database with /file/ prefix + const fileCount = await this.kvClient.countKeysWithPrefix('/file/'); + return { + fileCount, + source: 'KV' + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to retrieve database stats', + details: error.message + }); + } + }); + + // List directory contents + this.app.post<{ Body: { path: string } }>('/api/fuse/readdir', async (request, reply) => { + const { path } = request.body; + + if (typeof path !== 'string') { + return reply.status(400).send({ + error: 'Missing or invalid "path" parameter' + }); + } + + const entries = this.vfs.readdir(path); + + if (entries === null) { + return reply.status(404).send({ error: 'Directory not found' }); + } + + return { entries }; + }); + + // Get file attributes + this.app.post<{ Body: { path: string } }>('/api/fuse/getattr', async (request, reply) => { + const { path } = request.body; + + if (typeof path !== 'string') { + return reply.status(400).send({ + error: 'Missing or invalid "path" parameter' + }); + } + + const attrs = this.vfs.getattr(path); + + if (attrs === null) { + return reply.status(404).send({ error: 'Path not found' }); + } + + return attrs; + }); + + // Check if path exists + this.app.post<{ Body: { path: string } }>('/api/fuse/exists', async (request, reply) => { + const { path } = request.body; + + if (typeof path !== 'string') { + return reply.status(400).send({ + error: 'Missing or invalid "path" parameter' + }); + } + + const exists = this.vfs.exists(path); + return { exists }; + }); + + // Read file + this.app.post<{ Body: { path: string } }>('/api/fuse/read', async (request, reply) => { + const { path } = request.body; + + if (typeof path !== 'string') { + return reply.status(400).send({ + error: 'Missing or invalid "path" parameter' + }); + } + + const result = this.vfs.read(path); + + if (result === null) { + return reply.status(404).send({ error: 'File not found' }); + } + + // Convert Buffer to base64 for JSON serialization + const response: any = { + sourcePath: result.sourcePath, + size: result.size + }; + + if (result.content !== null) { + response.content = result.content.toString('base64'); + response.contentEncoding = 'base64'; + } + + return response; + }); + + // Get file metadata + this.app.post<{ Body: { path: string } }>('/api/fuse/metadata', async (request, reply) => { + const { path } = request.body; + + if (typeof path !== 'string') { + return reply.status(400).send({ + error: 'Missing or invalid "path" parameter' + }); + } + + const metadata = this.vfs.getMetadata(path); + + if (metadata === null) { + return reply.status(404).send({ error: 'Metadata not found' }); + } + + return metadata; + }); + + // Get complete VFS tree + this.app.get('/api/fuse/tree', async (request, reply) => { + return this.vfs.getTree(); + }); + + // Get all file paths + this.app.get('/api/fuse/files', async (request, reply) => { + const files = this.vfs.getAllFiles(); + return { files }; + }); + + // Get all source file paths (for Stremio addon validation) + this.app.get('/api/fuse/source-paths', async (request, reply) => { + const files = this.vfs.getAllFiles(); + const sourcePaths: string[] = []; + + for (const file of files) { + // Skip virtual metadata files (.meta, .nfo, .xml) + if (file.endsWith('.meta') || file.endsWith('.nfo') || file.endsWith('.xml')) { + continue; + } + + const result = this.vfs.read(file); + if (result && result.sourcePath) { + sourcePaths.push(result.sourcePath); + } + } + + return { sourcePaths, count: sourcePaths.length }; + }); + + // Get all directory paths + this.app.get('/api/fuse/directories', async (request, reply) => { + const directories = this.vfs.getAllDirectories(); + return { directories }; + }); + + // Refresh VFS + this.app.post('/api/fuse/refresh', async (request, reply) => { + await this.vfs.refresh(); + return { status: 'ok' }; + }); + } + + /** + * Setup Metrics API routes + */ + private setupMetricsRoutes(): void { + // Get performance metrics + this.app.get('/api/metrics', async (request, reply) => { + return performanceMetrics.getMetrics(); + }); + } + + /** + * Setup Duplicates API routes + */ + private setupDuplicatesRoutes(): void { + // Get duplicate files (from stored duplicate data) + this.app.get('/api/duplicates', async (request, reply) => { + try { + // Get stored duplicate result from WatchedFileProcessor + const duplicateResult = this.getDuplicateResult ? this.getDuplicateResult() : null; + + if (!duplicateResult) { + // No duplicates found yet (processing hasn't completed) + return { + hashDuplicates: [], + titleDuplicates: [], + stats: { + hashGroupCount: 0, + hashFileCount: 0, + titleGroupCount: 0, + titleFileCount: 0 + } + }; + } + + return { + hashDuplicates: duplicateResult.hashDuplicates, + titleDuplicates: duplicateResult.titleDuplicates, + stats: { + hashGroupCount: duplicateResult.hashDuplicates.length, + hashFileCount: duplicateResult.hashDuplicates.reduce((sum, g) => sum + g.files.length, 0), + titleGroupCount: duplicateResult.titleDuplicates.length, + titleFileCount: duplicateResult.titleDuplicates.reduce((sum, g) => sum + g.files.length, 0) + } + }; + } catch (error) { + console.error('Error retrieving duplicates:', error); + return reply.status(500).send({ + error: 'Failed to retrieve duplicates', + details: (error as Error).message + }); + } + }); + + // Refresh duplicates (trigger re-computation) + this.app.post('/api/duplicates/refresh', async (request, reply) => { + // Currently just acknowledges the request + // The duplicate detection runs automatically during processing + return { + status: 'ok', + message: 'Duplicate refresh triggered' + }; + }); + } + + /** + * Setup Metadata API routes (nested key architecture) + * Provides access to file metadata stored in KV using the nested key structure + */ + private setupMetadataRoutes(): void { + if (!this.kvClient) { + return; + } + + // Get all hash IDs (all files in KV) + this.app.get('/api/metadata/hash-ids', async (request, reply) => { + try { + const hashIds = await this.kvClient!.getAllHashIds(); + return { + hashIds, + count: hashIds.length + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to retrieve hash IDs', + details: error.message + }); + } + }); + + // List all files with metadata (for Files view) + this.app.get<{ + Querystring: { limit?: string; offset?: string } + }>('/api/metadata/list', async (request, reply) => { + const limit = parseInt(request.query.limit || '100', 10); + const offset = parseInt(request.query.offset || '0', 10); + + try { + const allHashIds = await this.kvClient!.getAllHashIds(); + const paginatedIds = allHashIds.slice(offset, offset + limit); + + const files = await Promise.all( + paginatedIds.map(async (hashId) => { + const metadata = await this.kvClient!.getMetadataFlat(hashId); + if (!metadata) return null; + + // Extract fields for Files view + return { + hashId, + filename: metadata.fileName || metadata.filename || 'Unknown', + path: metadata.filePath || metadata.path || '', + size: metadata.fileSize || metadata.size || 0, + mimeType: metadata.mimeType, + mediaType: metadata.videoType === 'tvshow' || metadata.videoType === 'movie' ? 'video' : metadata.mediaType, + duration: metadata.fileinfo?.streamdetails?.video?.[0]?.durationinseconds || metadata.duration, + resolution: metadata.fileinfo?.streamdetails?.video?.[0]?.width && metadata.fileinfo?.streamdetails?.video?.[0]?.height + ? `${metadata.fileinfo.streamdetails.video[0].width}x${metadata.fileinfo.streamdetails.video[0].height}` + : metadata.resolution, + codec: metadata.fileinfo?.streamdetails?.video?.[0]?.codec || metadata.codec, + createdAt: metadata.createdAt, + processedAt: metadata.processedAt || metadata.mtime + }; + }) + ); + + return { + files: files.filter(f => f !== null), + total: allHashIds.length, + limit, + offset + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to list files', + details: error.message + }); + } + }); + + // Get complete metadata for a file by hash ID + this.app.get<{ Params: { hashId: string } }>('/api/metadata/:hashId', async (request, reply) => { + const { hashId } = request.params; + + if (!hashId) { + return reply.status(400).send({ + error: 'Missing hash ID parameter' + }); + } + + try { + const metadata = await this.kvClient!.getMetadataFlat(hashId); + + if (!metadata) { + return reply.status(404).send({ + error: 'Metadata not found for hash ID', + hashId + }); + } + + return metadata; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to retrieve metadata', + details: error.message + }); + } + }); + + // Get a specific property for a file by hash ID and property path + this.app.get<{ + Params: { hashId: string }; + Querystring: { property: string } + }>('/api/metadata/:hashId/property', async (request, reply) => { + const { hashId } = request.params; + const { property } = request.query; + + if (!hashId || !property) { + return reply.status(400).send({ + error: 'Missing hash ID or property parameter' + }); + } + + try { + const key = `/file/${hashId}/${property}`; + const value = await this.kvClient!.getProperty(key); + + if (value === null) { + return reply.status(404).send({ + error: 'Property not found', + hashId, + property + }); + } + + return { + hashId, + property, + value + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to retrieve property', + details: error.message + }); + } + }); + + // Get all properties matching a specific property path across all files + // Example: /api/metadata/query?property=video/codec to get all video codecs + this.app.get<{ + Querystring: { property: string } + }>('/api/metadata/query', async (request, reply) => { + const { property } = request.query; + + if (!property) { + return reply.status(400).send({ + error: 'Missing property parameter' + }); + } + + try { + // Get all hash IDs + const hashIds = await this.kvClient!.getAllHashIds(); + + // Query each file for the specified property + const results: Array<{ hashId: string; value: string }> = []; + + for (const hashId of hashIds) { + const key = `/file/${hashId}/${property}`; + const value = await this.kvClient!.getProperty(key); + + if (value !== null) { + results.push({ hashId, value }); + } + } + + return { + property, + count: results.length, + results + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to query property', + details: error.message + }); + } + }); + + // Update complete metadata for a file by hash ID + this.app.put<{ + Params: { hashId: string }; + Body: any + }>('/api/metadata/:hashId', async (request, reply) => { + const { hashId } = request.params; + const metadata = request.body; + + if (!hashId) { + return reply.status(400).send({ + error: 'Missing hash ID parameter' + }); + } + + if (!metadata || typeof metadata !== 'object') { + return reply.status(400).send({ + error: 'Invalid metadata: must be an object' + }); + } + + try { + // Store the updated metadata (excludes internal processing status) + await this.kvClient!.setMetadataFlat(hashId, metadata, ['processingStatus']); + + return { + status: 'ok', + hashId, + message: 'Metadata updated successfully' + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to update metadata', + details: error.message + }); + } + }); + + // Update a specific property for a file by hash ID + this.app.put<{ + Params: { hashId: string }; + Body: { property: string; value: string } + }>('/api/metadata/:hashId/property', async (request, reply) => { + const { hashId } = request.params; + const { property, value } = request.body; + + if (!hashId || !property) { + return reply.status(400).send({ + error: 'Missing hash ID or property parameter' + }); + } + + if (value === undefined || value === null) { + return reply.status(400).send({ + error: 'Missing value parameter' + }); + } + + try { + const key = `/file/${hashId}/${property}`; + await this.kvClient!.setProperty(key, String(value)); + + return { + status: 'ok', + hashId, + property, + value, + message: 'Property updated successfully' + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to update property', + details: error.message + }); + } + }); + + // Search for files by various criteria + this.app.post<{ + Body: { + query?: string; // Search in title, filename, originaltitle + hashId?: string; // Exact hash ID match + property?: string; // Search by specific property + propertyValue?: string; // Value to match for property search + limit?: number; + } + }>('/api/metadata/search', async (request, reply) => { + const { query, hashId, property, propertyValue, limit = 100 } = request.body; + + try { + // If hashId is provided, do exact match + if (hashId) { + const metadata = await this.kvClient!.getMetadataFlat(hashId); + if (metadata) { + return { + results: [{ hashId, metadata }], + count: 1 + }; + } + return { results: [], count: 0 }; + } + + // Get all hash IDs + const hashIds = await this.kvClient!.getAllHashIds(); + const results: Array<{ hashId: string; metadata: any }> = []; + + // Search through files + for (const id of hashIds) { + if (results.length >= limit) break; + + const metadata = await this.kvClient!.getMetadataFlat(id); + if (!metadata) continue; + + let matches = false; + + // Property search + if (property && propertyValue !== undefined) { + const value = this.getNestedProperty(metadata, property); + if (value && String(value).toLowerCase().includes(String(propertyValue).toLowerCase())) { + matches = true; + } + } + + // General query search (searches title, filename, originaltitle, filepath, and hashId) + if (query && !property) { + const searchFields = ['title', 'fileName', 'originaltitle', 'showtitle', 'filePath']; + const queryLower = query.toLowerCase(); + + // Check if query matches hashId + if (id.toLowerCase().includes(queryLower)) { + matches = true; + } + + // Check other fields + if (!matches) { + for (const field of searchFields) { + const value = this.getNestedProperty(metadata, field); + if (value && String(value).toLowerCase().includes(queryLower)) { + matches = true; + break; + } + } + } + } + + if (matches || (!query && !property)) { + results.push({ hashId: id, metadata }); + } + } + + return { + results, + count: results.length, + total: hashIds.length + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to search metadata', + details: error.message + }); + } + }); + + // Batch update multiple files + this.app.post<{ + Body: { + updates: Array<{ + hashId: string; + metadata?: any; // Complete metadata update + properties?: Record; // Property updates + }> + } + }>('/api/metadata/batch', async (request, reply) => { + const { updates } = request.body; + + if (!updates || !Array.isArray(updates)) { + return reply.status(400).send({ + error: 'Invalid updates: must be an array' + }); + } + + try { + const results: Array<{ hashId: string; status: string; error?: string }> = []; + + for (const update of updates) { + const { hashId, metadata, properties } = update; + + if (!hashId) { + results.push({ + hashId: 'unknown', + status: 'error', + error: 'Missing hash ID' + }); + continue; + } + + try { + // Update complete metadata + if (metadata) { + await this.kvClient!.setMetadataFlat(hashId, metadata, ['processingStatus']); + } + + // Update individual properties + if (properties) { + for (const [property, value] of Object.entries(properties)) { + const key = `/file/${hashId}/${property}`; + await this.kvClient!.setProperty(key, String(value)); + } + } + + results.push({ + hashId, + status: 'ok' + }); + } catch (error: any) { + results.push({ + hashId, + status: 'error', + error: error.message + }); + } + } + + const successCount = results.filter(r => r.status === 'ok').length; + const errorCount = results.filter(r => r.status === 'error').length; + + return { + status: 'completed', + total: results.length, + success: successCount, + errors: errorCount, + results + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to batch update metadata', + details: error.message + }); + } + }); + + // Delete metadata for a file by hash ID + this.app.delete<{ Params: { hashId: string } }>('/api/metadata/:hashId', async (request, reply) => { + const { hashId } = request.params; + + if (!hashId) { + return reply.status(400).send({ + error: 'Missing hash ID parameter' + }); + } + + try { + const deletedCount = await this.kvClient!.deleteMetadataFlat(hashId); + + return { + status: 'ok', + hashId, + deletedKeys: deletedCount + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to delete metadata', + details: error.message + }); + } + }); + } + + /** + * Setup KV Browser API routes (for editor UI) + * Provides raw access to browse Redis keys and values + */ + private setupKVBrowserRoutes(): void { + if (!this.kvClient) { + return; + } + + // Get KV database info + this.app.get('/api/kv/info', async (request, reply) => { + try { + const hashIds = await this.kvClient!.getAllHashIds(); + const redis = (this.kvClient as any).getRedisClient?.(); + + let memoryUsage = 'N/A'; + if (redis) { + try { + const info = await redis.info('memory'); + const match = info.match(/used_memory_human:(\S+)/); + if (match) { + memoryUsage = match[1]; + } + } catch { + // Ignore memory info errors + } + } + + // Get total size of watch folders (cached, uses du command) + const totalSize = this.getTotalWatchFolderSize(); + + return { + prefix: 'file:', + fileCount: hashIds.length, + keyCount: hashIds.length, + totalSize, + memoryUsage + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get KV info', + details: error.message + }); + } + }); + + // List keys with cursor pagination + this.app.get<{ + Querystring: { cursor?: string; count?: string } + }>('/api/kv/keys', async (request, reply) => { + const cursor = request.query.cursor || '0'; + const count = parseInt(request.query.count || '50', 10); + + try { + const allHashIds = await this.kvClient!.getAllHashIds(); + + // Simple offset-based pagination using cursor as offset + const offset = parseInt(cursor, 10) || 0; + const keys = allHashIds.slice(offset, offset + count).map(id => `file:${id}`); + const nextOffset = offset + keys.length; + const hasMore = nextOffset < allHashIds.length; + + return { + keys, + cursor: hasMore ? String(nextOffset) : '0', + hasMore + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to list keys', + details: error.message + }); + } + }); + + // Get value for a specific key + this.app.get<{ + Params: { key: string } + }>('/api/kv/key/:key', async (request, reply) => { + const { key } = request.params; + + if (!key) { + return reply.status(400).send({ + error: 'Missing key parameter' + }); + } + + try { + // Extract hash ID from key (format: file:{hashId}) + const hashIdMatch = key.match(/^file:(.+)$/); + if (!hashIdMatch) { + return reply.status(400).send({ + error: 'Invalid key format. Expected: file:{hashId}' + }); + } + + const hashId = hashIdMatch[1]; + const metadata = await this.kvClient!.getMetadataFlat(hashId); + + if (!metadata) { + return reply.status(404).send({ + error: 'Key not found', + key + }); + } + + return { + key, + type: 'hash', + value: metadata + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get key value', + details: error.message + }); + } + }); + } + + /** + * Setup Unified Processing State API routes (4-state pipeline) + */ + private setupUnifiedProcessingRoutes(): void { + if (!this.unifiedStateManager) { + return; + } + + // Get active processing queue (files currently being processed) + this.app.get('/api/processing/queue', async (request, reply) => { + const items: Array<{ path: string; phase: 'light' | 'hash'; startTime?: number }> = []; + + // Get files in light processing + const lightProcessing = this.unifiedStateManager!.getLightProcessingFiles(); + for (const [filePath, info] of lightProcessing) { + items.push({ + path: filePath, + phase: 'light', + startTime: info.lightProcessingStartedAt + }); + } + + // Get files in hash processing + const hashProcessing = this.unifiedStateManager!.getHashProcessingFiles(); + for (const [filePath, info] of hashProcessing) { + items.push({ + path: filePath, + phase: 'hash', + startTime: info.hashProcessingStartedAt + }); + } + + return { items }; + }); + + // Get unified processing status snapshot (4 tabs: pending, lightProcessing, hashProcessing, done) + this.app.get('/api/processing/status', async (request, reply) => { + // Get ALL file paths currently in VFS (most accurate count) + const allVfsFiles = this.vfs.getAllFiles(); + + // Count only actual media files (not virtual metadata files like .meta, .nfo) + const mediaFiles = allVfsFiles.filter(f => { + return !f.endsWith('.meta') && !f.endsWith('.nfo') && !f.endsWith('.xml'); + }); + const actualMediaFileCount = mediaFiles.length; + + // Count files with completed hash computation + const filesWithHash = this.vfs.countFilesWithHash(); + + // Get snapshot with actual VFS counts + const snapshot = this.unifiedStateManager!.getSnapshot(actualMediaFileCount, this.fastQueueConcurrency, this.backgroundQueueConcurrency); + + // Override totalDone with actual count of files with hashes in VFS + // (not just current session, but ALL files in VFS with completed hash) + snapshot.totalDone = filesWithHash; + + // Add queue status if available + if (this.getQueueStatus) { + const queueStatus = this.getQueueStatus(); + (snapshot as any).queueStatus = queueStatus; + + // Add calculated fields that the UI needs (make API self-describing) + // Pipeline: validation (preProcess) → lightProcessing (fast) → hashProcessing (background) + const preProcessRunning = queueStatus.preProcessQueue?.pending || 0; + const preProcessPending = queueStatus.preProcessQueue?.size || 0; + const fastQueueRunning = queueStatus.fastQueue?.running || 0; + const fastQueuePending = queueStatus.fastQueue?.pending || 0; + const backgroundQueueRunning = queueStatus.backgroundQueue?.running || 0; + const backgroundQueuePending = queueStatus.backgroundQueue?.pending || 0; + + (snapshot as any).computed = { + // Pre-process queue (validation - quick extension checks) + preProcessRunning, + preProcessPending, + preProcessPaused: queueStatus.preProcessQueue?.isPaused || false, + + // Fast plugin queue (plugins with <1s avg execution) + fastQueueRunning, + fastQueuePending, + fastQueueTotal: fastQueueRunning + fastQueuePending + (queueStatus.fastQueue?.size || 0), + fastQueuePaused: queueStatus.fastQueue?.isPaused || false, + + // Background plugin queue (plugins with >=1s avg execution, e.g., full-hash, tmdb) + backgroundQueueRunning, + backgroundQueuePending, + backgroundQueueTotal: backgroundQueueRunning + backgroundQueuePending + (queueStatus.backgroundQueue?.size || 0), + backgroundQueuePaused: queueStatus.backgroundQueue?.isPaused || false, + + // Legacy compatibility fields (UI may still reference these) + actualRunningHashWorkers: backgroundQueueRunning, + actualRunningLightWorkers: preProcessRunning + fastQueueRunning, + trueHashQueueSize: backgroundQueuePending, + hashQueuePaused: queueStatus.backgroundQueue?.isPaused || false, + lightQueuePaused: queueStatus.preProcessQueue?.isPaused || false + }; + } + + return snapshot; + }); + } + + + /** + * Setup Stats API routes (Redis stats for monitor UI) + */ + private setupStatsRoutes(): void { + if (!this.kvClient) { + return; + } + + // Get Redis/KV stats (for monitor UI) + this.app.get('/api/stats', async (request, reply) => { + try { + const hashIds = await this.kvClient!.getAllHashIds(); + const redis = (this.kvClient as any).getRedisClient?.(); + + let memoryUsage = 'N/A'; + let memoryUsageBytes = 0; + if (redis) { + try { + const info = await redis.info('memory'); + const match = info.match(/used_memory_human:(\S+)/); + const bytesMatch = info.match(/used_memory:(\d+)/); + if (match) { + memoryUsage = match[1]; + } + if (bytesMatch) { + memoryUsageBytes = parseInt(bytesMatch[1], 10); + } + } catch { + // Ignore memory info errors + } + } + + // Get total size of watch folders (cached, uses du command) + const totalSize = this.getTotalWatchFolderSize(); + + return { + fileCount: hashIds.length, + keyCount: hashIds.length, + totalSize, + memoryUsage, + memoryUsageBytes + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get stats', + details: error.message + }); + } + }); + } + + /** + * Setup Scan Trigger API routes + */ + private setupScanRoutes(): void { + // Trigger manual scan + this.app.post('/api/scan/trigger', async (request, reply) => { + if (!this.triggerScanCallback) { + return reply.status(503).send({ + status: 'error', + message: 'Scan trigger not available' + }); + } + + try { + console.log('Manual scan triggered via API'); + // Don't await - trigger and return immediately + this.triggerScanCallback().catch(error => { + console.error('Error during triggered scan:', error); + }); + + return { + status: 'ok', + message: 'Scan triggered successfully' + }; + } catch (error: any) { + console.error('Error triggering scan:', error); + return reply.status(500).send({ + status: 'error', + message: 'Failed to trigger scan', + details: error.message + }); + } + }); + } + + /** + * Setup Services Discovery API routes + * Returns list of discovered services for dashboard navigation + */ + private setupServicesRoutes(): void { + if (!this.kvManager) { + return; + } + + // Get discovered services (for inter-service navigation) + this.app.get('/api/services', async (request, reply) => { + const services: Array<{ + name: string; + url: string; + api: string; + status: string; + capabilities: string[]; + version: string; + }> = []; + + try { + const serviceDiscovery = this.kvManager!.getServiceDiscovery(); + if (serviceDiscovery) { + const allServices = await serviceDiscovery.discoverAllServices(); + + for (const svc of allServices) { + // Build dashboard URL from API URL + const apiUrl = svc.api || ''; + const dashboardPath = svc.endpoints?.dashboard || '/'; + + services.push({ + name: svc.name || 'Unknown', + url: apiUrl + dashboardPath, + api: apiUrl, + status: svc.status || 'unknown', + capabilities: svc.capabilities || [], + version: svc.version || '', + }); + } + } + } catch (error: any) { + console.error('[Services API] Error discovering services:', error); + } + + // Get leader info + const leaderInfo = this.kvManager!.getLeaderInfo(); + const isLeader = this.kvManager!.isLeader(); + + return { + services, + current: 'meta-sort', + leader: leaderInfo ? { + host: leaderInfo.host, + api: leaderInfo.api, + http: leaderInfo.http, + } : null, + isLeader, + }; + }); + } + + /** + * Setup Plugin Management API routes + */ + private setupPluginRoutes(): void { + // Get all plugins with their status, config, and schema + this.app.get('/api/plugins', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ + error: 'Plugin manager not yet initialized', + message: 'Plugins are loaded on first file processing' + }); + } + + try { + const plugins = pluginManager.getPlugins(); + const executionOrder = pluginManager.getExecutionOrder(); + + return { + plugins: plugins.map(p => ({ + id: p.id, + name: p.name, + version: p.version, + description: p.description, + priority: p.priority, + color: p.color, + dependencies: p.dependencies, + active: p.active, + status: p.status, + error: p.error, + config: p.config, + configSchema: p.configSchema, + metadataSchema: p.metadataSchema + })), + executionOrder, + activeCount: plugins.filter(p => p.active).length, + totalCount: plugins.length + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get plugins', + details: error.message + }); + } + }); + + // Get plugin timings from performance metrics + this.app.get('/api/plugins/timings', async (request, reply) => { + try { + const metrics = performanceMetrics.getMetrics(); + const pluginTimings = metrics.pluginProcessingTimes || {}; + + // Convert to array format with stats + const timings = Object.entries(pluginTimings).map(([pluginId, data]: [string, any]) => ({ + pluginId, + totalCalls: data.count || 0, + totalTime: data.total || 0, + avgTime: data.count > 0 ? Math.round(data.total / data.count) : 0, + minTime: data.min || 0, + maxTime: data.max || 0 + })); + + return { + timings, + lastReset: metrics.startTime + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get plugin timings', + details: error.message + }); + } + }); + + // Activate a plugin + this.app.post<{ Params: { pluginId: string } }>('/api/plugins/:pluginId/activate', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + const { pluginId } = request.params; + + try { + await pluginManager.activate(pluginId); + return { + status: 'ok', + message: `Plugin ${pluginId} activated successfully`, + executionOrder: pluginManager.getExecutionOrder() + }; + } catch (error: any) { + return reply.status(400).send({ + error: 'Failed to activate plugin', + details: error.message + }); + } + }); + + // Deactivate a plugin + this.app.post<{ Params: { pluginId: string } }>('/api/plugins/:pluginId/deactivate', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + const { pluginId } = request.params; + + try { + await pluginManager.deactivate(pluginId); + return { + status: 'ok', + message: `Plugin ${pluginId} deactivated successfully`, + executionOrder: pluginManager.getExecutionOrder() + }; + } catch (error: any) { + return reply.status(400).send({ + error: 'Failed to deactivate plugin', + details: error.message + }); + } + }); + + // Update plugin configuration + this.app.put<{ + Params: { pluginId: string }; + Body: Record + }>('/api/plugins/:pluginId/config', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + const { pluginId } = request.params; + const newConfig = request.body; + + try { + await pluginManager.updateConfig(pluginId, newConfig); + const plugins = pluginManager.getPlugins(); + const plugin = plugins.find(p => p.id === pluginId); + + return { + status: 'ok', + message: `Plugin ${pluginId} configuration updated`, + config: plugin?.config + }; + } catch (error: any) { + return reply.status(400).send({ + error: 'Failed to update plugin config', + details: error.message + }); + } + }); + + // Clear plugin cache + this.app.post<{ Params: { pluginId: string } }>('/api/plugins/:pluginId/clear-cache', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + const { pluginId } = request.params; + + // Container plugins don't have local caches + return { + status: 'ok', + message: `Cache clear not applicable for container plugin ${pluginId}` + }; + }); + + // Rescan plugins (reload from container manager) + this.app.post('/api/plugins/rescan', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + try { + // Reload container plugins from container manager + await pluginManager.loadContainerPlugins(); + const plugins = pluginManager.getPlugins(); + return { + status: 'ok', + message: 'Plugin rescan complete', + pluginCount: plugins.length + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to rescan plugins', + details: error.message + }); + } + }); + + // Recompute plugin on all files (runs plugin with forceRecompute flag) + this.app.post<{ Params: { pluginId: string } }>('/api/plugins/:pluginId/recompute', async (request, reply) => { + const pluginManager = this.getPluginManager?.(); + if (!pluginManager) { + return reply.status(503).send({ error: 'Plugin manager not yet initialized' }); + } + + const taskScheduler = this.getTaskScheduler?.(); + if (!taskScheduler) { + return reply.status(503).send({ error: 'Task scheduler not yet initialized' }); + } + + if (!this.kvClient) { + return reply.status(503).send({ error: 'KV client not available' }); + } + + const { pluginId } = request.params; + + // Verify plugin exists and is active + const plugins = pluginManager.getPlugins(); + const plugin = plugins.find(p => p.id === pluginId); + + if (!plugin) { + return reply.status(404).send({ error: `Plugin '${pluginId}' not found` }); + } + + if (!plugin.active) { + return reply.status(400).send({ error: `Plugin '${pluginId}' is not active` }); + } + + try { + // Get all files from KV + const hashIds = await this.kvClient.getAllHashIds(); + console.log(`[Recompute] Starting recompute for plugin '${pluginId}' on ${hashIds.length} files`); + + // Prepare file data with metadata for TaskScheduler + const files: Array<{ filePath: string; fileHash: string; kvData?: Record }> = []; + + for (const hashId of hashIds) { + const metadata = await this.kvClient.getMetadataFlat(hashId); + if (metadata && metadata.filePath) { + // Convert nested metadata to flat key-value pairs for KVStore + const kvData: Record = {}; + const flattenObj = (obj: any, prefix = '') => { + for (const [key, value] of Object.entries(obj)) { + const fullKey = prefix ? `${prefix}/${key}` : key; + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + flattenObj(value, fullKey); + } else if (value !== undefined && value !== null) { + kvData[fullKey] = String(value); + } + } + }; + flattenObj(metadata); + + files.push({ + fileHash: hashId, + filePath: metadata.filePath, + kvData + }); + } + } + + // Create and enqueue tasks with forceRecompute option + const tasks = taskScheduler.createTasksForPluginOnFiles( + pluginId, + files, + { forceRecompute: true } + ); + + // Enqueue all tasks + taskScheduler.enqueueTasks(tasks); + + console.log(`[Recompute] Queued ${tasks.length} tasks for plugin '${pluginId}'`); + + return { + status: 'ok', + message: `Recompute triggered for plugin '${pluginId}'`, + filesQueued: tasks.length, + totalFiles: hashIds.length + }; + } catch (error: any) { + console.error(`[Recompute] Error triggering recompute for plugin '${pluginId}':`, error); + return reply.status(500).send({ + error: 'Failed to trigger recompute', + details: error.message + }); + } + }); + } + + /** + * Setup Container Plugin API routes + * Handles container plugin management and task callbacks + */ + private setupContainerPluginRoutes(): void { + // Callback endpoint for container plugins to report task completion + this.app.post<{ + Body: PluginCallbackPayload + }>('/api/plugins/callback', async (request, reply) => { + if (!this.containerPluginScheduler) { + return reply.status(503).send({ + error: 'Container plugin scheduler not initialized' + }); + } + + const payload = request.body; + + if (!payload || !payload.taskId) { + return reply.status(400).send({ + error: 'Invalid callback payload: missing taskId' + }); + } + + try { + const handled = this.containerPluginScheduler.handleCallback(payload); + + if (!handled) { + return reply.status(404).send({ + error: 'Unknown task ID', + taskId: payload.taskId + }); + } + + return { + status: 'ok', + taskId: payload.taskId + }; + } catch (error: any) { + console.error('[ContainerPluginCallback] Error handling callback:', error); + return reply.status(500).send({ + error: 'Failed to handle callback', + details: error.message + }); + } + }); + + // List container plugin status + this.app.get('/api/plugins/containers', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ + error: 'Container manager not initialized', + message: 'Container plugins are disabled or not yet initialized' + }); + } + + try { + const status = this.containerManager.getStatus(); + const queueStatus = this.containerPluginScheduler?.getQueueStatus(); + + return { + initialized: status.initialized, + pluginCount: status.plugins.length, + plugins: status.plugins.map(p => ({ + id: p.pluginId, + instanceCount: p.instances, + healthyCount: p.healthyInstances, + enabled: p.enabled, + image: p.image + })), + queue: queueStatus || null + }; + } catch (error: any) { + console.error('[ContainerPlugins] Error getting status:', error); + return reply.status(500).send({ + error: 'Failed to get container plugin status', + details: error.message + }); + } + }); + + // Get container plugin manifest + this.app.get<{ + Params: { pluginId: string } + }>('/api/plugins/containers/:pluginId/manifest', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + const { pluginId } = request.params; + + try { + const manifest = this.containerManager.getPluginManifest(pluginId); + + if (!manifest) { + return reply.status(404).send({ + error: `Container plugin '${pluginId}' not found` + }); + } + + return manifest; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get plugin manifest', + details: error.message + }); + } + }); + + // Restart a container plugin instance + this.app.post<{ + Params: { pluginId: string }; + Querystring: { instance?: string } + }>('/api/plugins/containers/:pluginId/restart', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + const { pluginId } = request.params; + + try { + // Restart all instances of the plugin + await this.containerManager.restartPlugin(pluginId); + return { + status: 'ok', + message: `Plugin '${pluginId}' restarted` + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to restart container plugin', + details: error.message + }); + } + }); + + // Stop a container plugin (not supported - use restart instead) + this.app.post<{ + Params: { pluginId: string } + }>('/api/plugins/containers/:pluginId/stop', async (request, reply) => { + return reply.status(501).send({ + error: 'Stop operation not supported', + message: 'Use restart to reload plugin instances' + }); + }); + + // Start a container plugin (not supported - plugins auto-start) + this.app.post<{ + Params: { pluginId: string } + }>('/api/plugins/containers/:pluginId/start', async (request, reply) => { + return reply.status(501).send({ + error: 'Start operation not supported', + message: 'Plugins are started automatically during initialization. Use restart to reload.' + }); + }); + + // Get container logs (not implemented) + this.app.get<{ + Params: { pluginId: string }; + Querystring: { instance?: string; tail?: string } + }>('/api/plugins/containers/:pluginId/logs', async (request, reply) => { + return reply.status(501).send({ + error: 'Log retrieval not implemented', + message: 'Use docker logs command directly for container logs' + }); + }); + } + + /** + * Setup meta-core compatible API routes + * These routes allow container plugins to write metadata directly to Redis + * Matches the meta-core Go sidecar API for compatibility + */ + private setupMetaCoreRoutes(): void { + if (!this.kvClient) { + return; + } + + // GET /meta/{hash} - Get all metadata for a file + this.app.get<{ + Params: { hash: string } + }>('/meta/:hash', async (request, reply) => { + const { hash } = request.params; + + try { + const metadata = await this.kvClient!.getMetadataFlat(hash); + return { metadata: metadata || {} }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get metadata', + details: error.message + }); + } + }); + + // GET /meta/{hash}/{key} - Get single property (supports nested keys with slashes) + this.app.get<{ + Params: { hash: string; '*': string } + }>('/meta/:hash/*', async (request, reply) => { + const { hash } = request.params; + const key = request.params['*']; + + if (!key) { + return reply.status(400).send({ error: 'Missing key' }); + } + + try { + const metadata = await this.kvClient!.getMetadataFlat(hash); + if (!metadata) { + return reply.status(404).send({ error: 'Hash not found' }); + } + + const value = metadata[key]; + if (value === undefined) { + return reply.status(404).send({ error: 'Property not found' }); + } + + return { value }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to get property', + details: error.message + }); + } + }); + + // PUT /meta/{hash}/{key} - Set single property + this.app.put<{ + Params: { hash: string; '*': string }; + Body: { value: string } + }>('/meta/:hash/*', async (request, reply) => { + const { hash } = request.params; + const key = request.params['*']; + const { value } = request.body || {}; + + if (!key) { + return reply.status(400).send({ error: 'Missing key' }); + } + + if (value === undefined) { + return reply.status(400).send({ error: 'Missing value in body' }); + } + + try { + // Get existing metadata and merge + const existing = await this.kvClient!.getMetadataFlat(hash) || {}; + existing[key] = String(value); + await this.kvClient!.setMetadataFlat(hash, existing); + + return { status: 'ok' }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to set property', + details: error.message + }); + } + }); + + // DELETE /meta/{hash}/{key} - Delete single property + this.app.delete<{ + Params: { hash: string; '*': string } + }>('/meta/:hash/*', async (request, reply) => { + const { hash } = request.params; + const key = request.params['*']; + + if (!key) { + return reply.status(400).send({ error: 'Missing key' }); + } + + try { + const existing = await this.kvClient!.getMetadataFlat(hash); + if (!existing) { + return { status: 'ok' }; // Nothing to delete + } + + delete existing[key]; + await this.kvClient!.setMetadataFlat(hash, existing); + + return { status: 'ok' }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to delete property', + details: error.message + }); + } + }); + + // PATCH /meta/{hash} - Merge metadata (partial update) + this.app.patch<{ + Params: { hash: string }; + Body: Record + }>('/meta/:hash', async (request, reply) => { + const { hash } = request.params; + const newMetadata = request.body; + + if (!newMetadata || typeof newMetadata !== 'object') { + return reply.status(400).send({ error: 'Body must be an object with key-value pairs' }); + } + + try { + // Get existing metadata and merge + const existing = await this.kvClient!.getMetadataFlat(hash) || {}; + const merged = { ...existing, ...newMetadata }; + await this.kvClient!.setMetadataFlat(hash, merged); + + return { status: 'ok' }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to merge metadata', + details: error.message + }); + } + }); + + // POST /meta/{hash}/_add/{key} - Add value to a set field (comma-separated) + this.app.post<{ + Params: { hash: string; key: string }; + Body: { value: string } + }>('/meta/:hash/_add/:key', async (request, reply) => { + const { hash, key } = request.params; + const { value } = request.body || {}; + + if (!value) { + return reply.status(400).send({ error: 'Missing value in body' }); + } + + try { + const existing = await this.kvClient!.getMetadataFlat(hash) || {}; + const currentValue = existing[key] || ''; + + // Parse existing as comma-separated set + const values = new Set(currentValue ? currentValue.split(',').map(v => v.trim()) : []); + values.add(String(value)); + + existing[key] = Array.from(values).join(','); + await this.kvClient!.setMetadataFlat(hash, existing); + + return { status: 'ok' }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to add to set', + details: error.message + }); + } + }); + + // Health check for meta-core compatibility + this.app.get('/meta-health', async () => { + return { status: 'ok', service: 'meta-sort-kv' }; + }); + + // POST /file/cid - Compute CID for a file (for poster/backdrop hashing) + this.app.post<{ Body: { path: string } }>('/file/cid', async (request, reply) => { + const { path: relativePath } = request.body; + + if (!relativePath) { + reply.status(400); + return { error: 'Bad Request', message: 'path is required' }; + } + + // Construct full path + const fullPath = `${config.FILES_PATH}/${relativePath}`; + + // Security check: ensure path is within FILES_PATH + const path = await import('path'); + const absFilesPath = path.resolve(config.FILES_PATH); + const absFullPath = path.resolve(fullPath); + if (!absFullPath.startsWith(absFilesPath)) { + reply.status(400); + return { error: 'Bad Request', message: 'path must be within files directory' }; + } + + try { + const fs = await import('fs/promises'); + const crypto = await import('crypto'); + + // Check if file exists + const stats = await fs.stat(fullPath); + if (stats.isDirectory()) { + reply.status(400); + return { error: 'Bad Request', message: 'path is a directory, not a file' }; + } + + // Read file and compute SHA-256 + const fileBuffer = await fs.readFile(fullPath); + const hashBytes = crypto.createHash('sha256').update(fileBuffer).digest(); + + // Build CIDv1 with raw codec (0x55) and sha256 multihash + // CIDv1 format: version (0x01) + codec (0x55) + multihash + // Multihash format: hash-code (0x12) + length (0x20) + hash + const cidBytes = Buffer.concat([ + Buffer.from([0x01]), // CIDv1 + Buffer.from([0x55]), // raw codec + Buffer.from([0x12]), // sha256 code + Buffer.from([0x20]), // 32 bytes + hashBytes + ]); + + // Encode as base32lower with 'b' prefix (multibase) + // Note: Node's base32 encoding uses uppercase, so we lowercase it + const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; + let cid = 'b'; + let bits = 0; + let value = 0; + for (const byte of cidBytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + cid += base32Chars[(value >> bits) & 0x1f]; + } + } + if (bits > 0) { + cid += base32Chars[(value << (5 - bits)) & 0x1f]; + } + + return { + cid, + path: relativePath, + size: stats.size + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + reply.status(404); + return { error: 'Not Found', message: 'file not found' }; + } + reply.status(500); + return { error: 'Internal Server Error', message: (error as Error).message }; + } + }); + + console.log('[UnifiedAPIServer] Meta-core compatible routes registered at /meta/*'); + console.log('[UnifiedAPIServer] File CID endpoint registered at POST /file/cid'); + } + + /** + * Set container plugin manager and scheduler (called after initialization) + */ + public setContainerPluginManager( + containerManager: ContainerManager, + scheduler: ContainerPluginScheduler + ): void { + this.containerManager = containerManager; + this.containerPluginScheduler = scheduler; + console.log('[UnifiedAPIServer] Container plugin manager and scheduler set'); + } + + /** + * Start the HTTP server + */ + public async start(): Promise { + try { + await this.app.listen({ + port: this.config.port, + host: this.config.host + }); + console.log(`Unified API Server listening on http://${this.config.host}:${this.config.port}`); + console.log(` - FUSE API: http://${this.config.host}:${this.config.port}/api/fuse/*`); + console.log(` - Metrics API: http://${this.config.host}:${this.config.port}/api/metrics`); + console.log(` - Duplicates API: http://${this.config.host}:${this.config.port}/api/duplicates`); + if (this.kvClient) { + console.log(` - Metadata API: http://${this.config.host}:${this.config.port}/api/metadata/* (GET/PUT/DELETE/POST)`); + console.log(` - KV Browser API: http://${this.config.host}:${this.config.port}/api/kv/* (GET)`); + } + if (this.unifiedStateManager) { + console.log(` - Unified Processing API (4 states): http://${this.config.host}:${this.config.port}/api/processing/status`); + } + console.log(` - Scan API: http://${this.config.host}:${this.config.port}/api/scan/trigger (POST)`); + } catch (error) { + console.error('Error starting Unified API Server:', error); + throw error; + } + } + + /** + * Stop the HTTP server + */ + public async stop(): Promise { + try { + await this.app.close(); + console.log('Unified API Server stopped'); + } catch (error) { + console.error('Error stopping Unified API Server:', error); + throw error; + } + } + + /** + * Get the Fastify instance (for testing or advanced usage) + */ + public getApp(): FastifyInstance { + return this.app; + } + + /** + * Helper method to get a nested property value from an object + * Supports dot notation: "video.codec", "audio.0.language" + */ + private getNestedProperty(obj: any, path: string): any { + if (!obj || !path) return undefined; + + const keys = path.split('.'); + let current = obj; + + for (const key of keys) { + if (current === null || current === undefined) { + return undefined; + } + current = current[key]; + } + + return current; + } +} diff --git a/packages/meta-sort-core/src/api/VirtualFileSystem.ts b/packages/meta-sort-core/src/api/VirtualFileSystem.ts new file mode 100644 index 0000000..be8dbd1 --- /dev/null +++ b/packages/meta-sort-core/src/api/VirtualFileSystem.ts @@ -0,0 +1,627 @@ +/** + * VirtualFileSystem - In-memory representation of the virtual output filesystem + * + * This class maintains the virtual directory structure and provides FUSE-compatible operations. + */ + +import { EventEmitter } from 'events'; +import * as path from 'path'; +import * as fs from 'fs'; +import { HashMeta } from '@metazla/meta-interface'; +import { + IFuseAPI, + VirtualDirectory, + VirtualFile, + FileAttributes, + ReadResult, + FuseAPIConfig, + FuseAPIEvents +} from './FuseAPI.js'; +import { MetaMeshFormat } from '../logic/MetaMeshFormat.js'; +import { MetadataFormatRegistry, IMetadataFormatHandler } from '../logic/MetadataFormatRegistry.js'; +import { config } from '../config/EnvConfig.js'; + +interface VFSNode { + type: 'file' | 'directory'; + name: string; + parent: string | null; + // For files + sourcePath?: string | null; + size?: number; + mtime?: Date; + ctime?: Date; + metadata?: HashMeta; + // For virtual metadata files + formatHandler?: IMetadataFormatHandler; + // For directories + children?: Set; +} + +export class VirtualFileSystem extends EventEmitter implements IFuseAPI { + private nodes: Map = new Map(); + private config: Required; + private metaFormat: MetaMeshFormat = new MetaMeshFormat(); + private formatRegistry: MetadataFormatRegistry = new MetadataFormatRegistry(); + private activeFormatHandlers: IMetadataFormatHandler[] = []; + + // Cached stats - updated incrementally on each modification + private cachedStats = { + fileCount: 0, + directoryCount: 0, + totalSize: 0, + metaFileCount: 0, + metadataFileCount: 0 + }; + + constructor(fuseConfig: FuseAPIConfig = {}) { + super(); + this.config = { + basePath: fuseConfig.basePath ?? '/', + fileMode: fuseConfig.fileMode ?? 0o644, + directoryMode: fuseConfig.directoryMode ?? 0o755, + uid: fuseConfig.uid ?? process.getuid(), + gid: fuseConfig.gid ?? process.getgid() + }; + + // Initialize active format handlers from configuration + this.activeFormatHandlers = this.formatRegistry.getHandlersForFormats(config.METADATA_FORMATS); + + if (this.activeFormatHandlers.length > 0) { + console.log(`VirtualFileSystem: Active metadata formats: ${this.activeFormatHandlers.map(h => h.formatType).join(', ')}`); + } else { + console.log('VirtualFileSystem: No metadata formats configured'); + } + + // Initialize root directory + this.nodes.set('/', { + type: 'directory', + name: '', + parent: null, + children: new Set() + }); + + // Initialize cached stats with root directory + this.cachedStats.directoryCount = 1; + } + + /** + * Build the virtual filesystem from a computed folder structure + * @param computed Map of source paths to destination paths + * @param metadata Map of source paths to HashMeta + * @param basePath Base path to strip from destination paths (e.g., OUTPUT_FOLDER) + */ + public buildFromComputed( + computed: Map, + metadata: Map, + basePath?: string + ): void { + // Clear existing structure (except root) + this.nodes.clear(); + this.nodes.set('/', { + type: 'directory', + name: '', + parent: null, + children: new Set() + }); + + // Reset cached stats (root directory will be counted) + this.cachedStats = { + fileCount: 0, + directoryCount: 1, // Root directory + totalSize: 0, + metaFileCount: 0, + metadataFileCount: 0 + }; + + // Normalize basePath if provided + const normalizedBasePath = basePath ? path.normalize(basePath) : null; + + // Add all files and their parent directories + for (const [sourcePath, destPath] of computed.entries()) { + // Strip basePath from destPath to create relative virtual path + let virtualPath = destPath; + if (normalizedBasePath && destPath.startsWith(normalizedBasePath)) { + virtualPath = destPath.slice(normalizedBasePath.length); + // Ensure path starts with / + if (!virtualPath.startsWith('/')) { + virtualPath = '/' + virtualPath; + } + } + this.addFile(virtualPath, sourcePath, metadata.get(sourcePath)); + } + + this.emit('vfs-updated'); + } + + /** + * Add a file to the virtual filesystem, creating parent directories as needed + */ + public addFile(virtualPath: string, sourcePath: string, metadata?: HashMeta): void { + // Normalize path + virtualPath = this.normalizePath(virtualPath); + + // Create parent directories + const dirname = path.dirname(virtualPath); + this.ensureDirectory(dirname); + + // Try to use metadata for file stats (avoid disk I/O during fast load) + let size: number; + let mtime: Date; + let ctime: Date; + + if (metadata && (metadata as any).sizeByte) { + // Use cached metadata (fast path - no disk I/O) + size = parseInt((metadata as any).sizeByte, 10); + // Use current time as placeholder for mtime/ctime (will be corrected on next scan) + mtime = new Date(); + ctime = new Date(); + } else { + // Fallback: stat the file (slow path - disk I/O) + try { + const stats = fs.statSync(sourcePath); + size = stats.size; + mtime = stats.mtime; + ctime = stats.ctime; + } catch (error) { + console.warn(`Cannot stat source file: ${sourcePath}`, error); + return; + } + } + + // Add the file node + const filename = path.basename(virtualPath); + this.nodes.set(virtualPath, { + type: 'file', + name: filename, + parent: dirname, + sourcePath, + size, + mtime, + ctime, + metadata + }); + + // Add to parent's children + const parentNode = this.nodes.get(dirname); + if (parentNode && parentNode.type === 'directory') { + parentNode.children!.add(filename); + } + + // Update cached stats + this.cachedStats.fileCount++; + this.cachedStats.totalSize += size; + + // Also add corresponding metadata files based on configured formats + this.addMetadataFiles(virtualPath, metadata, mtime); + + this.emit('file-added', virtualPath); + } + + /** + * Add virtual metadata files for all configured formats + */ + private addMetadataFiles(mediaPath: string, metadata: HashMeta | undefined, mtime: Date): void { + if (!metadata) return; + + // Create a metadata file for each configured format + for (const handler of this.activeFormatHandlers) { + const metaPath = `${mediaPath}${handler.extension}`; + const dirname = path.dirname(metaPath); + const filename = path.basename(metaPath); + + // Pre-generate metadata content to calculate size + const metaContent = handler.serialize(metadata); + const metaBuffer = Buffer.from(metaContent, 'utf-8'); + + // Add the virtual metadata file node + this.nodes.set(metaPath, { + type: 'file', + name: filename, + parent: dirname, + sourcePath: null, // Virtual file, no source + size: metaBuffer.length, + mtime: mtime, + ctime: mtime, + metadata, + formatHandler: handler + }); + + // Add to parent's children + const parentNode = this.nodes.get(dirname); + if (parentNode && parentNode.type === 'directory') { + parentNode.children!.add(filename); + } + + // Update cached stats + this.cachedStats.fileCount++; + this.cachedStats.totalSize += metaBuffer.length; + this.cachedStats.metadataFileCount++; + // Legacy: Count .meta files separately for backward compatibility + if (handler.extension === '.meta') { + this.cachedStats.metaFileCount++; + } + } + } + + /** + * Ensure a directory exists in the VFS, creating parent directories as needed + */ + private ensureDirectory(dirPath: string): void { + dirPath = this.normalizePath(dirPath); + + if (dirPath === '/' || this.nodes.has(dirPath)) { + return; + } + + // Recursively create parent + const parentPath = path.dirname(dirPath); + this.ensureDirectory(parentPath); + + // Create this directory + const dirName = path.basename(dirPath); + this.nodes.set(dirPath, { + type: 'directory', + name: dirName, + parent: parentPath, + children: new Set() + }); + + // Add to parent's children + const parentNode = this.nodes.get(parentPath); + if (parentNode && parentNode.type === 'directory') { + parentNode.children!.add(dirName); + } + + // Update cached stats + this.cachedStats.directoryCount++; + + this.emit('directory-added', dirPath); + } + + /** + * Normalize path to use forward slashes and ensure leading slash + */ + private normalizePath(p: string): string { + // Convert to forward slashes + p = p.split(path.sep).join('/'); + + // Ensure leading slash + if (!p.startsWith('/')) { + p = '/' + p; + } + + // Remove trailing slash (except for root) + if (p !== '/' && p.endsWith('/')) { + p = p.slice(0, -1); + } + + return p; + } + + // IFuseAPI implementation + + public readdir(dirPath: string): string[] | null { + dirPath = this.normalizePath(dirPath); + const node = this.nodes.get(dirPath); + + if (!node || node.type !== 'directory') { + return null; + } + + return Array.from(node.children!); + } + + public getattr(filepath: string): FileAttributes | null { + filepath = this.normalizePath(filepath); + const node = this.nodes.get(filepath); + + if (!node) { + return null; + } + + if (node.type === 'directory') { + return { + size: 4096, // Standard directory size + mode: 0o040000 | this.config.directoryMode, // S_IFDIR | permissions + mtime: Date.now() / 1000, + atime: Date.now() / 1000, + ctime: Date.now() / 1000, + nlink: 2, // . and .. + uid: this.config.uid, + gid: this.config.gid + }; + } else { + return { + size: node.size!, + mode: 0o100000 | this.config.fileMode, // S_IFREG | permissions + mtime: node.mtime!.getTime() / 1000, + atime: node.mtime!.getTime() / 1000, + ctime: node.ctime!.getTime() / 1000, + nlink: 1, + uid: this.config.uid, + gid: this.config.gid + }; + } + } + + public exists(filepath: string): boolean { + filepath = this.normalizePath(filepath); + return this.nodes.has(filepath); + } + + public read(filepath: string): ReadResult | null { + filepath = this.normalizePath(filepath); + const node = this.nodes.get(filepath); + + if (!node || node.type !== 'file') { + return null; + } + + // Virtual metadata file (any format) + if (node.sourcePath === null && node.metadata && node.formatHandler) { + const metaContent = node.formatHandler.serialize(node.metadata); + const metaBuffer = Buffer.from(metaContent, 'utf-8'); + + return { + sourcePath: null, + content: metaBuffer, + size: metaBuffer.length + }; + } + + // Regular file - return source path for reading + return { + sourcePath: node.sourcePath!, + content: null, + size: node.size! + }; + } + + public getMetadata(filepath: string): HashMeta | null { + filepath = this.normalizePath(filepath); + const node = this.nodes.get(filepath); + + if (!node || node.type !== 'file') { + return null; + } + + return node.metadata ?? null; + } + + public getTree(): VirtualDirectory { + const rootNode = this.nodes.get('/')!; + + return { + path: '/', + type: 'directory', + name: '', + parent: '', + children: Array.from(rootNode.children!), + mode: this.config.directoryMode, + mtime: new Date() + }; + } + + public async refresh(): Promise { + // Refresh is triggered by external calls after processing + // The buildFromComputed method handles the actual refresh + this.emit('vfs-updated'); + } + + /** + * Get all file paths in the VFS + */ + public getAllFiles(): string[] { + const files: string[] = []; + for (const [filepath, node] of this.nodes.entries()) { + if (node.type === 'file') { + files.push(filepath); + } + } + return files; + } + + /** + * Get all directory paths in the VFS + */ + public getAllDirectories(): string[] { + const dirs: string[] = []; + for (const [dirpath, node] of this.nodes.entries()) { + if (node.type === 'directory') { + dirs.push(dirpath); + } + } + return dirs; + } + + /** + * Get statistics about the VFS (returns cached values, updated incrementally) + */ + public getStats(): { + fileCount: number; + directoryCount: number; + totalSize: number; + metaFileCount: number; + metadataFileCount: number; + } { + // Return cached stats - no iteration needed! + return { ...this.cachedStats }; + } + + /** + * Count files in VFS that have completed hash computation + * A file is considered "done" if hashStatus === 'complete' (all hashes computed without errors) + */ + public countFilesWithHash(): number { + let count = 0; + for (const [filepath, node] of this.nodes.entries()) { + // Only count actual media files (not directories, not metadata files) + if (node.type === 'file' && node.sourcePath && node.metadata) { + // Check if file has completed processing (both light and hash phases) + if ((node.metadata as any).processingStatus === 'complete') { + count++; + } + } + } + return count; + } + + /** + * Update metadata for an existing file in the VFS (e.g., when hash computation completes) + * This also regenerates the metadata files with updated content + */ + public updateFileMetadata(virtualPath: string, metadata: HashMeta): void { + virtualPath = this.normalizePath(virtualPath); + const node = this.nodes.get(virtualPath); + + if (!node || node.type !== 'directory') { + // Update file metadata + if (node) { + node.metadata = metadata; + + // Regenerate metadata files with updated content + if (node.mtime) { + // Get old metadata file sizes before regenerating + const oldSizes = new Map(); + for (const handler of this.activeFormatHandlers) { + const metaPath = `${virtualPath}${handler.extension}`; + const oldMetaNode = this.nodes.get(metaPath); + if (oldMetaNode) { + oldSizes.set(metaPath, oldMetaNode.size ?? 0); + } + } + + // Regenerate metadata files (this will overwrite the nodes) + this.addMetadataFilesUpdate(virtualPath, metadata, node.mtime, oldSizes); + } + + this.emit('file-updated', virtualPath); + } + } + } + + /** + * Add/update virtual metadata files for all configured formats (used during updates) + * This version adjusts cached stats based on size differences + */ + private addMetadataFilesUpdate( + mediaPath: string, + metadata: HashMeta, + mtime: Date, + oldSizes: Map + ): void { + // Create a metadata file for each configured format + for (const handler of this.activeFormatHandlers) { + const metaPath = `${mediaPath}${handler.extension}`; + const dirname = path.dirname(metaPath); + const filename = path.basename(metaPath); + + // Pre-generate metadata content to calculate size + const metaContent = handler.serialize(metadata); + const metaBuffer = Buffer.from(metaContent, 'utf-8'); + const newSize = metaBuffer.length; + + const oldSize = oldSizes.get(metaPath); + const isNew = oldSize === undefined; + + // Update the virtual metadata file node + this.nodes.set(metaPath, { + type: 'file', + name: filename, + parent: dirname, + sourcePath: null, // Virtual file, no source + size: newSize, + mtime: mtime, + ctime: mtime, + metadata, + formatHandler: handler + }); + + // Add to parent's children (no-op if already exists) + const parentNode = this.nodes.get(dirname); + if (parentNode && parentNode.type === 'directory') { + parentNode.children!.add(filename); + } + + // Update cached stats + if (isNew) { + // New metadata file + this.cachedStats.fileCount++; + this.cachedStats.totalSize += newSize; + this.cachedStats.metadataFileCount++; + if (handler.extension === '.meta') { + this.cachedStats.metaFileCount++; + } + } else { + // Update existing - adjust size difference + this.cachedStats.totalSize += (newSize - oldSize); + } + } + } + + /** + * Remove a file and its associated metadata files from the VFS + */ + public removeFile(virtualPath: string): void { + virtualPath = this.normalizePath(virtualPath); + const node = this.nodes.get(virtualPath); + + if (!node || node.type !== 'file') { + return; + } + + // Update cached stats for main file + this.cachedStats.fileCount--; + this.cachedStats.totalSize -= node.size ?? 0; + + // Remove the main file + this.nodes.delete(virtualPath); + + // Remove from parent's children + const parentNode = this.nodes.get(node.parent!); + if (parentNode && parentNode.type === 'directory') { + parentNode.children!.delete(node.name); + } + + // Remove associated metadata files (.nfo, .meta, etc.) + for (const handler of this.activeFormatHandlers) { + const metaPath = `${virtualPath}${handler.extension}`; + const metaNode = this.nodes.get(metaPath); + + if (metaNode) { + // Update cached stats for metadata file + this.cachedStats.fileCount--; + this.cachedStats.totalSize -= metaNode.size ?? 0; + this.cachedStats.metadataFileCount--; + if (handler.extension === '.meta') { + this.cachedStats.metaFileCount--; + } + + this.nodes.delete(metaPath); + + // Remove from parent's children + const metaParent = this.nodes.get(metaNode.parent!); + if (metaParent && metaParent.type === 'directory') { + metaParent.children!.delete(metaNode.name); + } + } + } + + this.emit('file-removed', virtualPath); + } + + /** + * Check if a file exists in the VFS and return its metadata + */ + public getFileMetadata(virtualPath: string): HashMeta | undefined { + virtualPath = this.normalizePath(virtualPath); + const node = this.nodes.get(virtualPath); + return node?.metadata; + } +} + +// Type-safe event emitter +export interface VirtualFileSystem { + on(event: K, listener: FuseAPIEvents[K]): this; + emit(event: K, ...args: Parameters): boolean; +} diff --git a/packages/meta-sort-core/src/api/index.ts b/packages/meta-sort-core/src/api/index.ts new file mode 100644 index 0000000..a27cd20 --- /dev/null +++ b/packages/meta-sort-core/src/api/index.ts @@ -0,0 +1,11 @@ +/** + * API exports + * + * This module exports all API components including FUSE API, Metrics API, and Unified API Server. + */ + +export { IFuseAPI, VirtualNode, VirtualFile, VirtualDirectory, FileAttributes, ReadResult, FuseAPIConfig, FuseAPIEvents } from './FuseAPI.js'; +export { VirtualFileSystem } from './VirtualFileSystem.js'; +export { FuseAPIServer, FuseAPIServerConfig } from './FuseAPIServer.js'; +export { MetricsAPI, MetricsAPIConfig } from './MetricsAPI.js'; +export { UnifiedAPIServer, UnifiedAPIServerConfig } from './UnifiedAPIServer.js'; diff --git a/packages/meta-sort-core/src/config/EnvConfig.ts b/packages/meta-sort-core/src/config/EnvConfig.ts new file mode 100644 index 0000000..753494a --- /dev/null +++ b/packages/meta-sort-core/src/config/EnvConfig.ts @@ -0,0 +1,216 @@ +import dotenv from 'dotenv'; +dotenv.config(); + +export interface PollingPathConfig { + path: string; + intervalMs: number; +} + +interface EnvConfig { + /** Comma-separated list of folders to watch for changes.(mandatory) */ + WATCH_FOLDER_LIST: string; + + /** Path to the index file within the NFO CID library. */ + INDEX_FOLDER_PATH: string; + + /** Interval in milliseconds to update/check for changes. (default 30000) */ + UPDATE_INTERVAL_MS: number; + + /** Interval in milliseconds for polling watch folders. Set to 0 to disable polling. (default 0) */ + WATCH_FOLDER_POOLING_INTERVAL_MS: number; + + /** CHMOD for new folder format (default 0o777) */ + CHMOD_FOR_NEW_FOLDER: number; + + /** Path to the cache folder. */ + CACHE_FOLDER_PATH: string; + + /** jellyfin */ + JELLYFIN_ENDPOINT: string; + JELLYFIN_API_KEY: string; + + /** FUSE API server port (default 3000) */ + FUSE_API_PORT: number; + + /** FUSE API server host (default 'localhost') */ + FUSE_API_HOST: string; + + /** Comma-separated list of metadata formats to generate in virtual filesystem (default 'meta'). Options: 'meta' (.meta YAML files), 'jellyfin' (.nfo XML files). Example: 'meta,jellyfin' or 'jellyfin' or '' (no metadata files) */ + METADATA_FORMATS: string[]; + + /** Maximum number of worker threads for Piscina worker pool AND max concurrent file processing (default: CPU count) */ + MAX_WORKER_THREADS?: number; + + /** JSON array of polling path configurations. Example: '[{"path": "/data/watch/smb/downloads", "intervalMs": 60000}]' (optional) */ + POLLING_PATHS?: PollingPathConfig[]; + + // ======================================================================== + // KV Storage Configuration (NEW) + // ======================================================================== + + /** Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services (default '/meta-core') */ + META_CORE_PATH: string; + + /** Path to FILES_VOLUME - shared media volume (default '/files') */ + FILES_PATH: string; + + /** Service name for leader election and service discovery (default 'meta-sort') */ + SERVICE_NAME: string; + + /** Service version (default '1.0.0') */ + SERVICE_VERSION: string; + + /** Redis port (default 6379) */ + REDIS_PORT: number; + + /** Direct Redis URL (skip leader election if provided) */ + REDIS_URL?: string; + + /** Base URL for service discovery (e.g., 'http://localhost:3000' or 'http://meta-sort:80') */ + BASE_URL?: string; + + /** + * Hostname to advertise in lock file for other services to connect. + * Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution. + * If not set, auto-detects local IP (may be unstable across container restarts). + */ + ADVERTISE_HOST?: string; + + // ======================================================================== + // TaskScheduler Configuration (Plugin Queue System) + // ======================================================================== + + /** Concurrency for fast plugin queue (plugins with <1s avg execution time). Default: 32 */ + FAST_QUEUE_CONCURRENCY: number; + + /** Concurrency for background plugin queue (plugins with >=1s avg execution time). Default: 8 */ + BACKGROUND_QUEUE_CONCURRENCY: number; + + /** Threshold in ms to classify plugin as fast vs background. Default: 1000 */ + FAST_THRESHOLD_MS: number; + + /** Minimum samples before using measured timing for classification. Default: 10 */ + MIN_SAMPLES_FOR_MEASUREMENT: number; + + // ======================================================================== + // Container Plugins Configuration + // All plugins run as Docker containers for isolation and flexibility + // ======================================================================== + + /** Path to container plugins config file (default: '/app/plugins.yml') */ + CONTAINER_PLUGINS_CONFIG: string; + + /** Docker socket path (default: '/var/run/docker.sock') */ + DOCKER_SOCKET_PATH: string; + + /** Callback URL for container plugins (default: 'http://meta-sort:8180') */ + CONTAINER_CALLBACK_URL: string; + + /** meta-core URL for container plugins (default: 'http://meta-sort:9000') */ + CONTAINER_META_CORE_URL: string; + + /** Docker network for plugin containers (default: 'meta-network') */ + CONTAINER_NETWORK: string; + + /** + * Plugin file mounts - maps subpaths to host paths for /files in plugin containers. + * Format: "subpath:host_path,subpath2:host_path2" + * Example: "watch:/data/media,test:/app/test/media" + * Results in /files/watch and /files/test mounted in plugin containers (read-only). + */ + PLUGIN_FILE_MOUNTS?: string; + + /** Host path for plugin cache folder - each plugin gets its own subdirectory at /cache (optional, dev only) */ + PLUGIN_CACHE_FOLDER?: string; + + /** + * Host path for plugin output folders - each plugin gets /files/plugin/ mount (READ-WRITE) + * Plugins decide whether to use this folder - we always mount it if this env is set + */ + PLUGIN_OUTPUT_FOLDER?: string; + + /** Docker Compose project name for grouping containers in Docker Desktop (optional). When set, plugin containers appear grouped as a stack. */ + PLUGIN_STACK_NAME?: string; +} + +// Parse POLLING_PATHS from JSON string +function parsePollingPaths(): PollingPathConfig[] | undefined { + if (!process.env.POLLING_PATHS) { + return undefined; + } + + try { + const parsed = JSON.parse(process.env.POLLING_PATHS); + if (!Array.isArray(parsed)) { + console.error('POLLING_PATHS must be a JSON array'); + return undefined; + } + + // Validate structure + const valid = parsed.every(item => + item && typeof item === 'object' && + typeof item.path === 'string' && + typeof item.intervalMs === 'number' + ); + + if (!valid) { + console.error('POLLING_PATHS entries must have {path: string, intervalMs: number}'); + return undefined; + } + + return parsed as PollingPathConfig[]; + } catch (error) { + console.error('Failed to parse POLLING_PATHS:', error); + return undefined; + } +} + +export const config: EnvConfig = { + WATCH_FOLDER_LIST: process.env.WATCH_FOLDER_LIST, + INDEX_FOLDER_PATH: process.env.INDEX_FOLDER_PATH || '/data/cache/hash',//default /data/cache/hash + UPDATE_INTERVAL_MS: parseInt(process.env.UPDATE_INTERVAL_MS || "30000", 10),//default 30000 + WATCH_FOLDER_POOLING_INTERVAL_MS: parseInt(process.env.WATCH_FOLDER_POOLING_INTERVAL_MS || "0", 10),//default 0 + CHMOD_FOR_NEW_FOLDER: process.env.CHMOD_FOR_NEW_FOLDER && parseInt(process.env.CHMOD_FOR_NEW_FOLDER, 8),//default 0o777 (this is the default for mkdir) + CACHE_FOLDER_PATH: process.env.CACHE_FOLDER_PATH || '/data/cache/plugins',//default /data/cache/plugins + JELLYFIN_ENDPOINT: process.env.JELLYFIN_ENDPOINT!, + JELLYFIN_API_KEY: process.env.JELLYFIN_API_KEY!, + FUSE_API_PORT: parseInt(process.env.FUSE_API_PORT || "3000", 10),//default 3000 + FUSE_API_HOST: process.env.FUSE_API_HOST || 'localhost',//default localhost + METADATA_FORMATS: (process.env.METADATA_FORMATS || 'meta').split(',').map(f => f.trim()).filter(f => f.length > 0),//default 'meta' + MAX_WORKER_THREADS: process.env.MAX_WORKER_THREADS ? parseInt(process.env.MAX_WORKER_THREADS, 10) : undefined, + POLLING_PATHS: parsePollingPaths(), + + // KV Storage Configuration + META_CORE_PATH: process.env.META_CORE_PATH || '/meta-core', + FILES_PATH: process.env.FILES_PATH || '/files', + SERVICE_NAME: process.env.SERVICE_NAME || 'meta-sort', + SERVICE_VERSION: process.env.SERVICE_VERSION || '1.0.0', + REDIS_PORT: parseInt(process.env.REDIS_PORT || "6379", 10), + REDIS_URL: process.env.REDIS_URL, + BASE_URL: process.env.BASE_URL, + ADVERTISE_HOST: process.env.ADVERTISE_HOST, + + // TaskScheduler Configuration + FAST_QUEUE_CONCURRENCY: parseInt(process.env.FAST_QUEUE_CONCURRENCY || "32", 10), + BACKGROUND_QUEUE_CONCURRENCY: parseInt(process.env.BACKGROUND_QUEUE_CONCURRENCY || "8", 10), + FAST_THRESHOLD_MS: parseInt(process.env.FAST_THRESHOLD_MS || "1000", 10), + MIN_SAMPLES_FOR_MEASUREMENT: parseInt(process.env.MIN_SAMPLES_FOR_MEASUREMENT || "10", 10), + + // Container Plugins Configuration + CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml', + DOCKER_SOCKET_PATH: process.env.DOCKER_SOCKET_PATH || '/var/run/docker.sock', + CONTAINER_CALLBACK_URL: process.env.CONTAINER_CALLBACK_URL || 'http://meta-sort:8180', + CONTAINER_META_CORE_URL: process.env.CONTAINER_META_CORE_URL || 'http://meta-sort:9000', + CONTAINER_NETWORK: process.env.CONTAINER_NETWORK || 'meta-network', + PLUGIN_FILE_MOUNTS: process.env.PLUGIN_FILE_MOUNTS, + PLUGIN_CACHE_FOLDER: process.env.PLUGIN_CACHE_FOLDER, + PLUGIN_OUTPUT_FOLDER: process.env.PLUGIN_OUTPUT_FOLDER, + PLUGIN_STACK_NAME: process.env.PLUGIN_STACK_NAME, +}; + +if(process.env.TEST!=='true') { + if (!config.WATCH_FOLDER_LIST) { + console.error('Invalid configuration: WATCH_FOLDER_LIST is required'); + } +} + diff --git a/packages/meta-sort-core/src/config/RenamingRule.ts b/packages/meta-sort-core/src/config/RenamingRule.ts new file mode 100644 index 0000000..902e5a7 --- /dev/null +++ b/packages/meta-sort-core/src/config/RenamingRule.ts @@ -0,0 +1,62 @@ +import { + AnimeMeta, FileNameMeta, + FileStatMeta, + HashMeta, + LanguagesMeta, SubtitleFileMeta, + VersionMeta, + VideoMeta +} from "@metazla/meta-interface"; + +type Local = VideoMeta & HashMeta & FileStatMeta & AnimeMeta & VersionMeta & LanguagesMeta & SubtitleFileMeta & FileNameMeta; + +/** + * return the path relative to the root folder + */ +export function renamingRule(metamesh: Local, filepath?: string): string | null { + let supported = ['video', 'subtitle', 'torrent']; + + //check if filetype is one of the suported ones + if (!supported.includes(metamesh.fileType)) { + return null; + } + + // Extract the file extension from the original file path + const extension = metamesh.extension; + if (!extension) { + throw new Error(`No extension found for file: ${filepath}`); + } + + const title = metamesh?.titles?.eng || metamesh.originalTitle; + if (!title) { + if (metamesh.fileType !== "torrent") { + //don't warn for torrent files because they don't always have a title (eg representing a folder of files) + throw new Error(`No title found for file: ${filepath}`); + } + return null; + } + + + let newPath = ""; + + const season = metamesh.season ? ("S" + String(metamesh.season).padStart(2, '0')) : ""; + const seasonSpace = metamesh.season ? ` ${season}` : ""; + const episode = metamesh.episode ? ("E" + String(metamesh.episode).padStart(2, '0')) : ""; + const version = metamesh.version ? ` ${metamesh.version}` : ""; + const year = metamesh.movieYear ? ` (${metamesh.movieYear})` : ""; + const subtitle = metamesh.fileType === "subtitle" && metamesh.subtitleLanguage ? `.${metamesh.subtitleLanguage}` : ""; + const fileName = `${title}${seasonSpace}${episode}${year}${subtitle}${version}.${extension}`; + + // Simplified structure: TV Shows for series, Movies for standalone content + // Note: season can be 0 (special episodes), so check for null/undefined explicitly + const hasSeason = metamesh.season !== null && metamesh.season !== undefined; + const hasEpisode = metamesh.episode !== null && metamesh.episode !== undefined; + + if (metamesh.extra) { + newPath = `TV Shows/${title}/extra/${fileName}`; + } else if (hasSeason && hasEpisode) { + newPath = `TV Shows/${title}/${season}/${fileName}`; + } else { + newPath = `Movies/${title}${year}/${fileName}`; + } + return newPath; +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/config/SupportedFileTypes.ts b/packages/meta-sort-core/src/config/SupportedFileTypes.ts new file mode 100644 index 0000000..bd4a12e --- /dev/null +++ b/packages/meta-sort-core/src/config/SupportedFileTypes.ts @@ -0,0 +1,68 @@ +/** + * Supported file extensions for Meta Mesh + * Application-specific configuration + */ + +export const SUPPORTED_VIDEO_EXTENSIONS = new Set([ + '.mkv', + '.mp4', + '.avi', + '.mov', + '.wmv', + '.flv', + '.webm', + '.m4v', + '.mpg', + '.mpeg', + '.m2ts', + '.ts', + '.vob', + '.3gp', + '.ogv', + '.divx', + '.xvid', + '.rm', + '.rmvb' +]); + +export const SUPPORTED_SUBTITLE_EXTENSIONS = new Set([ + '.srt', + '.ass', + '.ssa', + '.sub', + '.vtt', + '.idx' +]); + +export const SUPPORTED_METADATA_EXTENSIONS = new Set([ + '.torrent' +]); + +export const SUPPORTED_IMAGE_EXTENSIONS = new Set([ + '.jpg', + '.jpeg', + '.png', + '.webp', + '.gif', + '.bmp', + '.tiff', + '.tif' +]); + +/** + * All supported file extensions + */ +export const SUPPORTED_EXTENSIONS = new Set([ + ...SUPPORTED_VIDEO_EXTENSIONS, + ...SUPPORTED_SUBTITLE_EXTENSIONS, + ...SUPPORTED_METADATA_EXTENSIONS, + ...SUPPORTED_IMAGE_EXTENSIONS +]); + +/** + * Check if file extension is supported + */ +export function isSupportedExtension(filePath: string): boolean { + const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase(); + return SUPPORTED_EXTENSIONS.has(ext); +} diff --git a/packages/meta-sort-core/src/config/TargetHash.ts b/packages/meta-sort-core/src/config/TargetHash.ts new file mode 100644 index 0000000..bc3785c --- /dev/null +++ b/packages/meta-sort-core/src/config/TargetHash.ts @@ -0,0 +1,21 @@ +import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash"; + +// Target hashes for Piscina worker (full-file hash computation) +// Note: midhash256 is NOT included here because it's computed in the main thread +// during light processing, not by the Piscina worker +export const targetHash = [ + CID_ALGORITHM_NAMES.sha1, // For torrent verification + CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility + CID_ALGORITHM_NAMES.md5, // For compatibility + CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash + CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash + CID_ALGORITHM_NAMES.crc32, // Fast checksum + CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52) +] + +// Target hashes for index-cid cache (includes all hashes we want to cache) +// This includes midhash256 which is computed separately from the worker hashes +export const targetHashForIndex = [ + ...targetHash, + CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing) +] \ No newline at end of file diff --git a/packages/meta-sort-core/src/container-plugins/ConfigParser.ts b/packages/meta-sort-core/src/container-plugins/ConfigParser.ts new file mode 100644 index 0000000..ea1f9f7 --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/ConfigParser.ts @@ -0,0 +1,372 @@ +/** + * Container Plugin Configuration Parser + * + * Parses plugins.yml configuration files for container plugins. + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; +import YAML from 'yaml'; +import type { + ContainerPluginsConfig, + ContainerPluginConfig, + ContainerResourceLimits, + ContainerMount, + ContainerHealthCheck, +} from './types.js'; +import type { TaskQueueType } from '../plugin-engine/types.js'; + +/** + * Parse error with context + */ +export class ConfigParseError extends Error { + constructor( + message: string, + public readonly filePath: string, + public readonly cause?: Error + ) { + super(`${message} (file: ${filePath})`); + this.name = 'ConfigParseError'; + } +} + +/** + * Default configuration values + */ +const DEFAULTS = { + version: '1.0', + instances: 1, + enabled: true, + network: false, + healthCheck: { + interval: '30s', + timeout: '5s', + retries: 3, + }, +} as const; + +/** + * Substitute environment variables in a string + * Supports: ${VAR}, ${VAR:-default}, ${VAR:=default} + */ +function substituteEnvVars(value: string): string { + // Match ${VAR}, ${VAR:-default}, or ${VAR:=default} + const envPattern = /\$\{([A-Z_][A-Z0-9_]*)(?:(:?[-=])([^}]*))?\}/gi; + + return value.replace(envPattern, (match, varName, operator, defaultValue) => { + const envValue = process.env[varName]; + + if (envValue !== undefined && envValue !== '') { + return envValue; + } + + // Handle default values + if (operator === ':-' || operator === ':=') { + return defaultValue || ''; + } + if (operator === '-' || operator === '=') { + // Only use default if variable is unset (not if empty) + if (envValue === undefined) { + return defaultValue || ''; + } + return ''; + } + + // No default, return empty string + return ''; + }); +} + +/** + * Recursively substitute env vars in an object + */ +function substituteEnvVarsInObject(obj: unknown): unknown { + if (typeof obj === 'string') { + return substituteEnvVars(obj); + } + + if (Array.isArray(obj)) { + return obj.map(item => substituteEnvVarsInObject(item)); + } + + if (obj !== null && typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = substituteEnvVarsInObject(value); + } + return result; + } + + return obj; +} + +/** + * Validate resource limits + */ +function validateResourceLimits( + resources: unknown, + pluginId: string +): ContainerResourceLimits | undefined { + if (!resources || typeof resources !== 'object') { + return undefined; + } + + const r = resources as Record; + const result: ContainerResourceLimits = {}; + + if (r.memory !== undefined) { + if (typeof r.memory !== 'string') { + throw new Error(`Plugin '${pluginId}': resources.memory must be a string (e.g., '512m', '2g')`); + } + // Validate format + if (!/^\d+[kmgKMG]?$/.test(r.memory)) { + throw new Error(`Plugin '${pluginId}': invalid memory format '${r.memory}' (use e.g., '512m', '2g')`); + } + result.memory = r.memory; + } + + if (r.cpus !== undefined) { + const cpus = Number(r.cpus); + if (isNaN(cpus) || cpus <= 0) { + throw new Error(`Plugin '${pluginId}': resources.cpus must be a positive number`); + } + result.cpus = cpus; + } + + return Object.keys(result).length > 0 ? result : undefined; +} + +/** + * Validate volume mounts + */ +function validateMounts(mounts: unknown, pluginId: string): ContainerMount[] | undefined { + if (!mounts) { + return undefined; + } + + if (!Array.isArray(mounts)) { + throw new Error(`Plugin '${pluginId}': mounts must be an array`); + } + + return mounts.map((mount, index) => { + if (!mount || typeof mount !== 'object') { + throw new Error(`Plugin '${pluginId}': mount[${index}] must be an object`); + } + + const m = mount as Record; + + if (typeof m.source !== 'string' || !m.source) { + throw new Error(`Plugin '${pluginId}': mount[${index}].source is required`); + } + + if (typeof m.target !== 'string' || !m.target) { + throw new Error(`Plugin '${pluginId}': mount[${index}].target is required`); + } + + return { + source: m.source, + target: m.target, + readonly: m.readonly !== false, // Default to true + }; + }); +} + +/** + * Validate health check config + */ +function validateHealthCheck( + healthCheck: unknown, + pluginId: string +): ContainerHealthCheck | undefined { + if (!healthCheck || typeof healthCheck !== 'object') { + return undefined; + } + + const h = healthCheck as Record; + const result: ContainerHealthCheck = {}; + + if (h.interval !== undefined) { + if (typeof h.interval !== 'string') { + throw new Error(`Plugin '${pluginId}': healthCheck.interval must be a string (e.g., '30s')`); + } + result.interval = h.interval; + } + + if (h.timeout !== undefined) { + if (typeof h.timeout !== 'string') { + throw new Error(`Plugin '${pluginId}': healthCheck.timeout must be a string (e.g., '5s')`); + } + result.timeout = h.timeout; + } + + if (h.retries !== undefined) { + const retries = Number(h.retries); + if (isNaN(retries) || retries < 0 || !Number.isInteger(retries)) { + throw new Error(`Plugin '${pluginId}': healthCheck.retries must be a non-negative integer`); + } + result.retries = retries; + } + + return Object.keys(result).length > 0 ? result : undefined; +} + +/** + * Validate a single plugin configuration + */ +function validatePluginConfig( + pluginId: string, + config: unknown +): ContainerPluginConfig { + if (!config || typeof config !== 'object') { + throw new Error(`Plugin '${pluginId}': configuration must be an object`); + } + + const c = config as Record; + + // Required: image + if (typeof c.image !== 'string' || !c.image) { + throw new Error(`Plugin '${pluginId}': 'image' is required and must be a non-empty string`); + } + + // Optional: enabled (default: true) + const enabled = c.enabled !== false; + + // Optional: instances (default: 1) + let instances: number = DEFAULTS.instances; + if (c.instances !== undefined) { + instances = Number(c.instances); + if (isNaN(instances) || instances < 1 || !Number.isInteger(instances)) { + throw new Error(`Plugin '${pluginId}': 'instances' must be a positive integer`); + } + } + + // Optional: defaultQueue + let defaultQueue: TaskQueueType | undefined; + if (c.defaultQueue !== undefined) { + if (c.defaultQueue !== 'fast' && c.defaultQueue !== 'background') { + throw new Error(`Plugin '${pluginId}': 'defaultQueue' must be 'fast' or 'background'`); + } + defaultQueue = c.defaultQueue; + } + + // Optional: network (default: false) + const network = c.network === true; + + return { + enabled, + image: c.image, + instances, + resources: validateResourceLimits(c.resources, pluginId), + config: c.config && typeof c.config === 'object' + ? c.config as Record + : undefined, + network, + mounts: validateMounts(c.mounts, pluginId), + healthCheck: validateHealthCheck(c.healthCheck, pluginId), + defaultQueue, + }; +} + +/** + * Parse and validate container plugins configuration + */ +export function parseConfig(rawConfig: unknown, filePath: string): ContainerPluginsConfig { + if (!rawConfig || typeof rawConfig !== 'object') { + throw new ConfigParseError('Configuration must be an object', filePath); + } + + const config = rawConfig as Record; + + // Optional: version + const version = typeof config.version === 'string' + ? config.version + : DEFAULTS.version; + + // Required: plugins + if (!config.plugins || typeof config.plugins !== 'object') { + throw new ConfigParseError("'plugins' section is required and must be an object", filePath); + } + + const plugins: Record = {}; + + for (const [pluginId, pluginConfig] of Object.entries(config.plugins)) { + // Validate plugin ID format + if (!/^[a-z][a-z0-9-]*$/.test(pluginId)) { + throw new ConfigParseError( + `Invalid plugin ID '${pluginId}': must be lowercase alphanumeric with hyphens, starting with a letter`, + filePath + ); + } + + try { + plugins[pluginId] = validatePluginConfig(pluginId, pluginConfig); + } catch (error) { + throw new ConfigParseError( + error instanceof Error ? error.message : String(error), + filePath + ); + } + } + + return { version, plugins }; +} + +/** + * Load and parse plugins.yml configuration file + */ +export async function loadConfig(configPath: string): Promise { + // Check if file exists + try { + await fs.access(configPath, fs.constants.R_OK); + } catch { + throw new ConfigParseError(`Configuration file not found or not readable`, configPath); + } + + // Read file + let content: string; + try { + content = await fs.readFile(configPath, 'utf-8'); + } catch (error) { + throw new ConfigParseError( + 'Failed to read configuration file', + configPath, + error instanceof Error ? error : undefined + ); + } + + // Parse YAML + let rawConfig: unknown; + try { + rawConfig = YAML.parse(content); + } catch (error) { + throw new ConfigParseError( + 'Invalid YAML syntax', + configPath, + error instanceof Error ? error : undefined + ); + } + + // Substitute environment variables + const substituted = substituteEnvVarsInObject(rawConfig); + + // Validate and return + return parseConfig(substituted, configPath); +} + +/** + * Get enabled plugins from config + */ +export function getEnabledPlugins( + config: ContainerPluginsConfig +): Array<{ id: string; config: ContainerPluginConfig }> { + return Object.entries(config.plugins) + .filter(([, pluginConfig]) => (pluginConfig as ContainerPluginConfig).enabled !== false) + .map(([id, pluginConfig]) => ({ id, config: pluginConfig as ContainerPluginConfig })); +} + +/** + * Get default config path + */ +export function getDefaultConfigPath(): string { + return process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml'; +} diff --git a/packages/meta-sort-core/src/container-plugins/ContainerManager.ts b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts new file mode 100644 index 0000000..6886d75 --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts @@ -0,0 +1,771 @@ +/** + * Container Manager + * + * Manages the lifecycle of plugin containers including spawning, + * health checking, configuration, and cleanup. + */ + +import { EventEmitter } from 'events'; +import { DockerClient } from './DockerClient.js'; +import { + loadConfig, + getEnabledPlugins, +} from './ConfigParser.js'; +import type { + ContainerPluginsConfig, + ContainerPluginConfig, + ContainerPluginManifest, + ContainerPluginInstance, + ContainerInstanceStatus, + ContainerManagerStatus, + PluginHealthResponse, + PluginConfigureResponse, +} from './types.js'; +import { config } from '../config/EnvConfig.js'; + +/** + * Container manager events + */ +export interface ContainerManagerEvents { + 'plugin:starting': { pluginId: string; instanceIndex: number }; + 'plugin:started': { pluginId: string; instance: ContainerPluginInstance }; + 'plugin:healthy': { pluginId: string; instance: ContainerPluginInstance }; + 'plugin:unhealthy': { pluginId: string; instance: ContainerPluginInstance; error: string }; + 'plugin:stopped': { pluginId: string; instanceIndex: number }; + 'plugin:error': { pluginId: string; error: Error }; + 'manager:initialized': { pluginCount: number }; + 'manager:shutdown': {}; +} + +/** + * Container naming helper + */ +function getContainerName(pluginId: string, instanceIndex: number): string { + return `meta-plugin-${pluginId}-${instanceIndex}`; +} + +/** + * Container Manager + */ +/** + * ============================================================================ + * PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS + * ============================================================================ + * + * Each plugin container has exactly 3 mount points (hardcoded): + * + * /files (READ-ONLY) - Files to process, same structure as meta-sort + * /output (READ-WRITE) - Plugin-generated files + * /cache (READ-WRITE) - Plugin-specific cache + * + * Host paths are configured via environment variables: + * - PLUGIN_FILE_MOUNTS: Maps subpaths under /files to host paths + * Format: "subpath:host_path,subpath2:host_path2" + * Example: "watch:/data/media,test:/app/test/media" + * Results in: /files/watch, /files/test mounted in container + * + * - PLUGIN_CACHE_FOLDER: Base path for /cache mounts + * - PLUGIN_OUTPUT_FOLDER: Base path for /output mounts + * + * SECURITY: Plugins have read-only access to /files. + * Only /cache and /output are writable. + * + * ============================================================================ + */ + +/** + * File mount configuration - maps container path to host path + */ +interface FileMountConfig { + containerPath: string; + hostPath: string; +} + +/** + * Parse PLUGIN_FILE_MOUNTS env var into mount configs + * Format: "container_path:host_path,container_path2:host_path2" + */ +function parseFileMounts(mountsStr?: string): FileMountConfig[] { + if (!mountsStr) { + return []; + } + + const mounts: FileMountConfig[] = []; + const pairs = mountsStr.split(',').map(s => s.trim()).filter(s => s); + + for (const pair of pairs) { + const [containerPath, hostPath] = pair.split(':').map(s => s.trim()); + if (containerPath && hostPath) { + mounts.push({ containerPath, hostPath }); + } else { + console.warn(`[ContainerManager] Invalid mount format: "${pair}" - expected "container_path:host_path"`); + } + } + + return mounts; +} + +export class ContainerManager extends EventEmitter { + private dockerClient: DockerClient; + private pluginsConfig: ContainerPluginsConfig | null = null; + private instances: Map = new Map(); + private healthCheckIntervals: Map = new Map(); + private roundRobinCounters: Map = new Map(); + private initialized: boolean = false; + private shutdownInProgress: boolean = false; + private fileMounts: FileMountConfig[] = []; + private pluginCacheFolder?: string; + private pluginOutputFolder?: string; + private stackName?: string; + + constructor( + private configPath: string = config.CONTAINER_PLUGINS_CONFIG, + private network: string = config.CONTAINER_NETWORK, + private filesPath: string = config.FILES_PATH, + private callbackUrl: string = config.CONTAINER_CALLBACK_URL, + private metaCoreUrl: string = config.CONTAINER_META_CORE_URL, + dockerClient?: DockerClient + ) { + super(); + this.dockerClient = dockerClient || new DockerClient(config.DOCKER_SOCKET_PATH); + + // Parse file mounts from PLUGIN_FILE_MOUNTS env var + this.fileMounts = parseFileMounts(config.PLUGIN_FILE_MOUNTS); + + // Configure plugin cache folder (READ-WRITE) + this.pluginCacheFolder = config.PLUGIN_CACHE_FOLDER; + + // Configure plugin output folder (READ-WRITE for /plugin-files) + this.pluginOutputFolder = config.PLUGIN_OUTPUT_FOLDER; + + // Configure stack name for Docker Desktop grouping + this.stackName = config.PLUGIN_STACK_NAME; + } + + /** + * Initialize the container manager + */ + async initialize(): Promise { + if (this.initialized) { + return; + } + + console.log('[ContainerManager] Initializing...'); + + // Log configured host mounts + console.log('[ContainerManager] Plugin mount configuration (hardcoded paths):'); + console.log(' /files (RO) - Files to process:'); + if (this.fileMounts.length > 0) { + for (const mount of this.fileMounts) { + const targetPath = mount.containerPath.startsWith('/files') + ? mount.containerPath + : `/files/${mount.containerPath}`; + console.log(` ${targetPath} -> ${mount.hostPath}`); + } + } else { + console.warn(' (not configured) - Set PLUGIN_FILE_MOUNTS'); + } + console.log(` /cache (RW) - Plugin cache: ${this.pluginCacheFolder || '(not configured)'}`); + console.log(` /output (RW) - Plugin output: ${this.pluginOutputFolder || '(not configured)'}`) + + // Initialize Docker client + await this.dockerClient.initialize(); + + // Ensure network exists + await this.dockerClient.ensureNetwork(this.network); + + // Load configuration + this.pluginsConfig = await loadConfig(this.configPath); + + // Get enabled plugins + const enabledPlugins = getEnabledPlugins(this.pluginsConfig); + + console.log(`[ContainerManager] Found ${enabledPlugins.length} enabled plugins`); + + // Clean up any stale containers from previous runs + await this.cleanupStaleContainers(); + + // Spawn containers for each enabled plugin + for (const { id, config: pluginConfig } of enabledPlugins) { + try { + await this.spawnPlugin(id, pluginConfig); + } catch (error) { + console.error(`[ContainerManager] Failed to spawn plugin '${id}':`, error); + this.emit('plugin:error', { + pluginId: id, + error: error instanceof Error ? error : new Error(String(error)), + }); + } + } + + this.initialized = true; + this.emit('manager:initialized', { pluginCount: enabledPlugins.length }); + + console.log('[ContainerManager] Initialization complete'); + } + + /** + * Shutdown the container manager + */ + async shutdown(): Promise { + if (this.shutdownInProgress) { + return; + } + + this.shutdownInProgress = true; + console.log('[ContainerManager] Shutting down...'); + + // Stop health checks + for (const interval of this.healthCheckIntervals.values()) { + clearInterval(interval); + } + this.healthCheckIntervals.clear(); + + // Stop all plugin containers + for (const [pluginId, instances] of this.instances) { + for (const instance of instances) { + try { + await this.stopInstance(instance); + } catch (error) { + console.error( + `[ContainerManager] Error stopping ${instance.containerName}:`, + error + ); + } + } + } + + this.instances.clear(); + this.initialized = false; + this.shutdownInProgress = false; + this.emit('manager:shutdown', {}); + + console.log('[ContainerManager] Shutdown complete'); + } + + /** + * Spawn all instances for a plugin + */ + async spawnPlugin(pluginId: string, pluginConfig: ContainerPluginConfig): Promise { + const instanceCount = pluginConfig.instances || 1; + const instances: ContainerPluginInstance[] = []; + + console.log( + `[ContainerManager] Spawning ${instanceCount} instance(s) for plugin '${pluginId}'` + ); + + // Pull image if needed + const imageExists = await this.dockerClient.imageExists(pluginConfig.image); + if (!imageExists) { + console.log(`[ContainerManager] Pulling image ${pluginConfig.image}...`); + await this.dockerClient.pullImage(pluginConfig.image, (event) => { + if (event.status) { + console.log(`[ContainerManager] Pull: ${event.status}`); + } + }); + } + + // Spawn each instance + for (let i = 0; i < instanceCount; i++) { + this.emit('plugin:starting', { pluginId, instanceIndex: i }); + + const instance = await this.spawnInstance(pluginId, pluginConfig, i); + instances.push(instance); + + this.emit('plugin:started', { pluginId, instance }); + } + + this.instances.set(pluginId, instances); + + // Start health checking + this.startHealthCheck(pluginId); + + // Wait for at least one instance to be healthy + await this.waitForHealthy(pluginId, 30000); + + // Fetch manifest from first healthy instance + const healthyInstance = this.getHealthyInstance(pluginId); + if (healthyInstance) { + try { + const manifest = await this.fetchManifest(healthyInstance); + // Update all instances with manifest + for (const inst of instances) { + inst.manifest = manifest; + } + } catch (error) { + console.warn( + `[ContainerManager] Failed to fetch manifest for '${pluginId}':`, + error + ); + } + + // Send configuration + if (pluginConfig.config) { + try { + await this.sendConfig(healthyInstance, pluginConfig.config); + } catch (error) { + console.warn( + `[ContainerManager] Failed to configure '${pluginId}':`, + error + ); + } + } + } + } + + /** + * Spawn a single container instance + */ + private async spawnInstance( + pluginId: string, + pluginConfig: ContainerPluginConfig, + instanceIndex: number + ): Promise { + const containerName = getContainerName(pluginId, instanceIndex); + + // Check if container already exists + const existingContainers = await this.dockerClient.listContainers({ + name: [containerName], + }); + + if (existingContainers.length > 0) { + const existing = existingContainers[0]; + // Remove existing container + console.log(`[ContainerManager] Removing existing container ${containerName}`); + await this.dockerClient.removeContainer(existing.Id, true); + } + + /** + * PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS: + * /files (READ-ONLY) - Files to process + * /output (READ-WRITE) - Plugin-generated files + * /cache (READ-WRITE) - Plugin-specific cache + */ + const mounts: Array<{ source: string; target: string; readonly: boolean }> = []; + + // /files/* (READ-ONLY) - Mount each configured subpath under /files + for (const fileMount of this.fileMounts) { + // fileMount.containerPath is the subpath (e.g., "watch", "test") + // Mount at /files/{subpath} + const targetPath = fileMount.containerPath.startsWith('/files') + ? fileMount.containerPath + : `/files/${fileMount.containerPath}`; + mounts.push({ + source: fileMount.hostPath, + target: targetPath, + readonly: true, + }); + } + + if (this.fileMounts.length === 0) { + console.warn(`[ContainerManager] WARNING: No file mounts configured. Plugin '${pluginId}' won't have access to files.`); + } + + // /cache (READ-WRITE) - Plugin-specific cache + if (this.pluginCacheFolder) { + mounts.push({ + source: `${this.pluginCacheFolder}/${pluginId}`, + target: '/cache', + readonly: false, + }); + } + + // /output (READ-WRITE) - Plugin-generated files + if (this.pluginOutputFolder) { + mounts.push({ + source: `${this.pluginOutputFolder}/${pluginId}`, + target: '/output', + readonly: false, + }); + } + + // Create container + const containerId = await this.dockerClient.createContainer({ + image: pluginConfig.image, + name: containerName, + network: this.network, + resources: pluginConfig.resources, + mounts, + labels: { + 'meta-mesh.plugin': 'true', + 'meta-mesh.plugin.id': pluginId, + 'meta-mesh.plugin.instance': String(instanceIndex), + }, + env: { + PLUGIN_ID: pluginId, + META_CORE_URL: this.metaCoreUrl, + CALLBACK_URL: this.callbackUrl, + }, + // Docker Desktop grouping + stackName: this.stackName, + serviceName: pluginId, + }); + + // Start container + await this.dockerClient.startContainer(containerId); + + // Create instance record + const instance: ContainerPluginInstance = { + pluginId, + containerId, + containerName, + baseUrl: `http://${containerName}:8080`, + instanceIndex, + status: 'starting', + tasksProcessed: 0, + tasksFailed: 0, + }; + + console.log(`[ContainerManager] Started container ${containerName}`); + + return instance; + } + + /** + * Stop a container instance + */ + private async stopInstance(instance: ContainerPluginInstance): Promise { + console.log(`[ContainerManager] Stopping ${instance.containerName}...`); + + try { + await this.dockerClient.stopContainer(instance.containerId, 10); + } catch (error) { + // Container might already be stopped + } + + try { + await this.dockerClient.removeContainer(instance.containerId, true); + } catch (error) { + // Container might already be removed + } + + instance.status = 'stopped'; + this.emit('plugin:stopped', { + pluginId: instance.pluginId, + instanceIndex: instance.instanceIndex, + }); + } + + /** + * Start health check loop for a plugin + */ + private startHealthCheck(pluginId: string, intervalMs: number = 30000): void { + // Clear existing interval if any + const existing = this.healthCheckIntervals.get(pluginId); + if (existing) { + clearInterval(existing); + } + + const checkHealth = async () => { + const instances = this.instances.get(pluginId); + if (!instances) return; + + for (const instance of instances) { + if (instance.status === 'stopped') continue; + + try { + const healthy = await this.checkHealth(instance); + const previousStatus = instance.status; + + if (healthy) { + instance.status = 'healthy'; + instance.lastHealthCheck = Date.now(); + + if (previousStatus !== 'healthy') { + this.emit('plugin:healthy', { pluginId, instance }); + } + } else { + instance.status = 'unhealthy'; + instance.lastError = 'Health check failed'; + + if (previousStatus === 'healthy') { + this.emit('plugin:unhealthy', { + pluginId, + instance, + error: 'Health check failed', + }); + } + } + } catch (error) { + instance.status = 'unhealthy'; + instance.lastError = error instanceof Error ? error.message : String(error); + + this.emit('plugin:unhealthy', { + pluginId, + instance, + error: instance.lastError, + }); + } + } + }; + + // Initial check + checkHealth(); + + // Schedule periodic checks + const interval = setInterval(checkHealth, intervalMs); + this.healthCheckIntervals.set(pluginId, interval); + } + + /** + * Check health of a container instance + */ + async checkHealth(instance: ContainerPluginInstance): Promise { + try { + const response = await fetch(`${instance.baseUrl}/health`, { + method: 'GET', + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + return false; + } + + const data = (await response.json()) as PluginHealthResponse; + return data.status === 'healthy' && data.ready === true; + } catch { + return false; + } + } + + /** + * Wait for at least one healthy instance + */ + private async waitForHealthy(pluginId: string, timeoutMs: number): Promise { + const startTime = Date.now(); + const checkInterval = 1000; + + while (Date.now() - startTime < timeoutMs) { + const instances = this.instances.get(pluginId); + if (!instances) return; + + for (const instance of instances) { + const healthy = await this.checkHealth(instance); + if (healthy) { + instance.status = 'healthy'; + instance.lastHealthCheck = Date.now(); + this.emit('plugin:healthy', { pluginId, instance }); + return; + } + } + + await new Promise((resolve) => setTimeout(resolve, checkInterval)); + } + + console.warn(`[ContainerManager] Timeout waiting for healthy instance of '${pluginId}'`); + } + + /** + * Fetch manifest from a container instance + */ + async fetchManifest(instance: ContainerPluginInstance): Promise { + const response = await fetch(`${instance.baseUrl}/manifest`, { + method: 'GET', + signal: AbortSignal.timeout(10000), + }); + + if (!response.ok) { + throw new Error(`Failed to fetch manifest: ${response.status}`); + } + + return response.json() as Promise; + } + + /** + * Send configuration to a container instance + */ + async sendConfig( + instance: ContainerPluginInstance, + pluginConfig: Record + ): Promise { + const response = await fetch(`${instance.baseUrl}/configure`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ config: pluginConfig }), + signal: AbortSignal.timeout(10000), + }); + + if (!response.ok) { + throw new Error(`Failed to configure plugin: ${response.status}`); + } + + const data = (await response.json()) as PluginConfigureResponse; + if (!data.success) { + throw new Error(`Plugin configuration failed: ${data.error}`); + } + } + + /** + * Get a healthy instance for a plugin (round-robin) + */ + getHealthyInstance(pluginId: string): ContainerPluginInstance | null { + const instances = this.instances.get(pluginId); + if (!instances || instances.length === 0) { + return null; + } + + const healthyInstances = instances.filter((i) => i.status === 'healthy'); + if (healthyInstances.length === 0) { + return null; + } + + // Round-robin selection + const counter = this.roundRobinCounters.get(pluginId) || 0; + const selected = healthyInstances[counter % healthyInstances.length]; + this.roundRobinCounters.set(pluginId, counter + 1); + + return selected; + } + + /** + * Get all instances for a plugin + */ + getInstances(pluginId: string): ContainerPluginInstance[] { + return this.instances.get(pluginId) || []; + } + + /** + * Get all plugins + */ + getPluginIds(): string[] { + return Array.from(this.instances.keys()); + } + + /** + * Check if a plugin is a container plugin + */ + isContainerPlugin(pluginId: string): boolean { + return this.instances.has(pluginId); + } + + /** + * Get plugin manifest + */ + getPluginManifest(pluginId: string): ContainerPluginManifest | undefined { + const instances = this.instances.get(pluginId); + if (!instances || instances.length === 0) { + return undefined; + } + return instances[0].manifest; + } + + /** + * Get manager status + */ + getStatus(): ContainerManagerStatus { + const plugins: ContainerManagerStatus['plugins'] = []; + + for (const [pluginId, instances] of this.instances) { + const pluginConfig = this.pluginsConfig?.plugins[pluginId]; + const healthyCount = instances.filter((i) => i.status === 'healthy').length; + + plugins.push({ + pluginId, + enabled: pluginConfig?.enabled !== false, + image: pluginConfig?.image || 'unknown', + instances: instances.length, + healthyInstances: healthyCount, + manifest: instances[0]?.manifest, + }); + } + + const totalInstances = Array.from(this.instances.values()).reduce( + (sum, instances) => sum + instances.length, + 0 + ); + + const healthyInstances = Array.from(this.instances.values()).reduce( + (sum, instances) => sum + instances.filter((i) => i.status === 'healthy').length, + 0 + ); + + return { + initialized: this.initialized, + pluginCount: this.instances.size, + runningContainers: totalInstances, + healthyContainers: healthyInstances, + plugins, + }; + } + + /** + * Restart a plugin (stop and respawn all instances) + */ + async restartPlugin(pluginId: string): Promise { + console.log(`[ContainerManager] Restarting plugin '${pluginId}'...`); + + const instances = this.instances.get(pluginId); + const pluginConfig = this.pluginsConfig?.plugins[pluginId]; + + if (!instances || !pluginConfig) { + throw new Error(`Plugin '${pluginId}' not found`); + } + + // Stop health check + const interval = this.healthCheckIntervals.get(pluginId); + if (interval) { + clearInterval(interval); + this.healthCheckIntervals.delete(pluginId); + } + + // Stop all instances + for (const instance of instances) { + await this.stopInstance(instance); + } + + this.instances.delete(pluginId); + + // Respawn + await this.spawnPlugin(pluginId, pluginConfig); + } + + /** + * Clean up stale containers from previous runs + */ + private async cleanupStaleContainers(): Promise { + console.log('[ContainerManager] Cleaning up stale containers...'); + + const staleContainers = await this.dockerClient.listPluginContainers(); + + for (const container of staleContainers) { + console.log( + `[ContainerManager] Removing stale container ${container.Names?.[0] || container.Id}` + ); + try { + await this.dockerClient.removeContainer(container.Id, true); + } catch (error) { + console.warn( + `[ContainerManager] Failed to remove stale container:`, + error + ); + } + } + } + + /** + * Record task completion for statistics + */ + recordTaskCompletion(pluginId: string, instanceName: string, success: boolean): void { + const instances = this.instances.get(pluginId); + if (!instances) return; + + const instance = instances.find((i) => i.containerName === instanceName); + if (!instance) return; + + if (success) { + instance.tasksProcessed++; + } else { + instance.tasksFailed++; + } + } +} + +/** + * Create a container manager instance + */ +export function createContainerManager( + configPath?: string, + network?: string +): ContainerManager { + return new ContainerManager(configPath, network); +} diff --git a/packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts b/packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts new file mode 100644 index 0000000..225bee8 --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts @@ -0,0 +1,651 @@ +/** + * Container Plugin Scheduler + * + * Handles dispatching tasks to containerized plugins and + * processing completion callbacks. + */ + +import { EventEmitter } from 'events'; +import PQueue from 'p-queue'; +import { randomUUID } from 'crypto'; +import type { ContainerManager } from './ContainerManager.js'; +import type { + ContainerTask, + ContainerTaskStatus, + PluginProcessRequest, + PluginProcessResponse, + PluginCallbackPayload, + CallbackStatus, +} from './types.js'; +import type { TaskQueueType } from '../plugin-engine/types.js'; +import type { IKVClient } from '../kv/IKVClient.js'; +import { config } from '../config/EnvConfig.js'; + +/** + * Scheduler events + */ +export interface ContainerSchedulerEvents { + 'task:created': { task: ContainerTask }; + 'task:dispatched': { task: ContainerTask }; + 'task:completed': { task: ContainerTask }; + 'task:failed': { task: ContainerTask; error: string }; + 'task:timeout': { task: ContainerTask }; + 'file:complete': { fileHash: string; filePath: string }; +} + +/** + * Pending callback info + */ +interface PendingCallback { + task: ContainerTask; + resolve: (result: PluginCallbackPayload) => void; + reject: (error: Error) => void; + timeoutHandle: NodeJS.Timeout; +} + +/** + * Container Plugin Scheduler + */ +export class ContainerPluginScheduler extends EventEmitter { + private containerManager: ContainerManager; + private fastQueue: PQueue; + private backgroundQueue: PQueue; + private pendingTasks: Map = new Map(); + private pendingCallbacks: Map = new Map(); + private fileTasks: Map> = new Map(); // fileHash -> taskIds + private fileCompletedPlugins: Map> = new Map(); // fileHash -> completed pluginIds + private dependencyWaiters: Map void>> = new Map(); // "fileHash:pluginId" -> resolve callbacks + private callbackUrl: string; + private metaCoreUrl: string; + private defaultTimeout: number = 60000; // 60 seconds + private kvClient: IKVClient | null = null; + + constructor( + containerManager: ContainerManager, + options?: { + fastConcurrency?: number; + backgroundConcurrency?: number; + callbackUrl?: string; + metaCoreUrl?: string; + defaultTimeout?: number; + kvClient?: IKVClient; + } + ) { + super(); + this.containerManager = containerManager; + + this.fastQueue = new PQueue({ + concurrency: options?.fastConcurrency ?? config.FAST_QUEUE_CONCURRENCY, + }); + + this.backgroundQueue = new PQueue({ + concurrency: options?.backgroundConcurrency ?? config.BACKGROUND_QUEUE_CONCURRENCY, + }); + + this.callbackUrl = options?.callbackUrl ?? `${config.CONTAINER_CALLBACK_URL}/api/plugins/callback`; + this.metaCoreUrl = options?.metaCoreUrl ?? config.CONTAINER_META_CORE_URL; + this.defaultTimeout = options?.defaultTimeout ?? 60000; + this.kvClient = options?.kvClient ?? null; + } + + /** + * Wait for dependency plugins to complete for a file + */ + private async waitForDependencies( + fileHash: string, + dependencies: string[], + timeoutMs: number = 30000 + ): Promise { + if (!dependencies || dependencies.length === 0) { + return; + } + + const completedPlugins = this.fileCompletedPlugins.get(fileHash) || new Set(); + const pendingDeps = dependencies.filter(dep => !completedPlugins.has(dep)); + + if (pendingDeps.length === 0) { + return; // All dependencies already completed + } + + // Wait for each pending dependency + const waitPromises = pendingDeps.map(depPluginId => { + return new Promise((resolve, reject) => { + const key = `${fileHash}:${depPluginId}`; + + // Check again in case it completed while we were setting up + const currentCompleted = this.fileCompletedPlugins.get(fileHash); + if (currentCompleted?.has(depPluginId)) { + resolve(); + return; + } + + // Set up timeout + const timeoutHandle = setTimeout(() => { + // Remove waiter on timeout + const waiters = this.dependencyWaiters.get(key); + if (waiters) { + const idx = waiters.indexOf(resolve); + if (idx >= 0) waiters.splice(idx, 1); + } + // Resolve anyway to not block - plugin will get stale metadata but won't hang + console.warn(`[ContainerScheduler] Timeout waiting for dependency '${depPluginId}' for file ${fileHash}`); + resolve(); + }, timeoutMs); + + // Register waiter + if (!this.dependencyWaiters.has(key)) { + this.dependencyWaiters.set(key, []); + } + this.dependencyWaiters.get(key)!.push(() => { + clearTimeout(timeoutHandle); + resolve(); + }); + }); + }); + + await Promise.all(waitPromises); + } + + /** + * Mark a plugin as completed for a file and notify waiters + */ + private markPluginCompleted(fileHash: string, pluginId: string, filePath?: string): void { + // Track completion + if (!this.fileCompletedPlugins.has(fileHash)) { + this.fileCompletedPlugins.set(fileHash, new Set()); + } + this.fileCompletedPlugins.get(fileHash)!.add(pluginId); + + // Notify waiters + const key = `${fileHash}:${pluginId}`; + const waiters = this.dependencyWaiters.get(key); + if (waiters) { + for (const resolve of waiters) { + resolve(); + } + this.dependencyWaiters.delete(key); + } + + // Publish to Redis for external subscribers (e.g., meta-stremio) + this.publishPluginComplete(fileHash, pluginId, filePath || ''); + } + + /** + * Publish plugin completion event to Redis + * Channel: meta-sort:plugin:complete + * + * External services (like meta-stremio) can subscribe to this channel + * to be notified when specific plugins complete for a file. + */ + private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise { + if (!this.kvClient) { + return; + } + + try { + const message = JSON.stringify({ + fileHash, + pluginId, + filePath, + timestamp: Date.now() + }); + await (this.kvClient as any).publish('meta-sort:plugin:complete', message); + } catch (error) { + // Silent fail - pub/sub is optional + } + } + + /** + * Create a task for a container plugin + */ + createTask( + pluginId: string, + fileHash: string, + filePath: string, + dependencies: string[] = [], + queue: TaskQueueType = 'fast' + ): ContainerTask { + const task: ContainerTask = { + id: randomUUID(), + fileHash, + filePath, + pluginId, + dependencies, + queue, + status: 'pending', + createdAt: Date.now(), + }; + + this.pendingTasks.set(task.id, task); + + // Track tasks by file + if (!this.fileTasks.has(fileHash)) { + this.fileTasks.set(fileHash, new Set()); + } + this.fileTasks.get(fileHash)!.add(task.id); + + this.emit('task:created', { task }); + + return task; + } + + /** + * Enqueue a task for execution + */ + enqueueTask(task: ContainerTask): void { + const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue; + + queue.add(async () => { + await this.executeTask(task); + }); + } + + /** + * Execute a task (dispatch to container) + */ + private async executeTask(task: ContainerTask): Promise { + // Wait for dependencies to complete first + if (task.dependencies && task.dependencies.length > 0) { + task.status = 'waiting'; + await this.waitForDependencies(task.fileHash, task.dependencies); + } + + const instance = this.containerManager.getHealthyInstance(task.pluginId); + + if (!instance) { + task.status = 'failed'; + task.error = `No healthy instance available for plugin '${task.pluginId}'`; + this.emit('task:failed', { task, error: task.error }); + this.checkFileCompletion(task.fileHash); + return; + } + + task.status = 'dispatched'; + task.dispatchedAt = Date.now(); + task.instanceName = instance.containerName; + + this.emit('task:dispatched', { task }); + + try { + // Fetch fresh metadata after dependencies complete + // This ensures we get metadata written by dependency plugins + const existingMeta = await this.fetchExistingMetadata(task.fileHash); + + // Build process request + const request: PluginProcessRequest = { + taskId: task.id, + cid: task.fileHash, + filePath: task.filePath, + callbackUrl: this.callbackUrl, + metaCoreUrl: this.metaCoreUrl, + existingMeta, + }; + + // Dispatch to container + const response = await this.dispatchToContainer(instance.baseUrl, request); + + if (response.status !== 'accepted') { + throw new Error(response.error || 'Task rejected by plugin'); + } + + // Wait for callback with timeout + const manifest = instance.manifest; + const timeout = manifest?.timeout ?? this.defaultTimeout; + + await this.waitForCallback(task, timeout); + + } catch (error) { + task.status = 'failed'; + task.error = error instanceof Error ? error.message : String(error); + task.completedAt = Date.now(); + + if (task.dispatchedAt) { + task.duration = task.completedAt - task.dispatchedAt; + } + + this.containerManager.recordTaskCompletion( + task.pluginId, + task.instanceName!, + false + ); + + this.emit('task:failed', { task, error: task.error }); + } + + this.checkFileCompletion(task.fileHash); + } + + /** + * Dispatch task to container plugin + */ + private async dispatchToContainer( + baseUrl: string, + request: PluginProcessRequest + ): Promise { + const response = await fetch(`${baseUrl}/process`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal: AbortSignal.timeout(10000), // 10s timeout for dispatch + }); + + if (!response.ok) { + throw new Error(`Plugin returned ${response.status}: ${await response.text()}`); + } + + return response.json() as Promise; + } + + /** + * Wait for callback from container + */ + private waitForCallback(task: ContainerTask, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timeoutHandle = setTimeout(() => { + this.pendingCallbacks.delete(task.id); + task.status = 'timeout'; + task.completedAt = Date.now(); + if (task.dispatchedAt) { + task.duration = task.completedAt - task.dispatchedAt; + } + this.emit('task:timeout', { task }); + reject(new Error(`Task timeout after ${timeoutMs}ms`)); + }, timeoutMs); + + this.pendingCallbacks.set(task.id, { + task, + resolve, + reject, + timeoutHandle, + }); + }); + } + + /** + * Handle callback from container plugin + */ + handleCallback(payload: PluginCallbackPayload): boolean { + const pending = this.pendingCallbacks.get(payload.taskId); + + if (!pending) { + console.warn( + `[ContainerScheduler] Received callback for unknown task: ${payload.taskId}` + ); + return false; + } + + // Clear timeout + clearTimeout(pending.timeoutHandle); + this.pendingCallbacks.delete(payload.taskId); + + const task = pending.task; + task.completedAt = Date.now(); + task.duration = payload.duration; + + if (payload.status === 'completed') { + task.status = 'completed'; + + this.containerManager.recordTaskCompletion( + task.pluginId, + task.instanceName!, + true + ); + + // Mark plugin as completed for dependency tracking + this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath); + + this.emit('task:completed', { task }); + pending.resolve(payload); + + } else if (payload.status === 'failed') { + task.status = 'failed'; + task.error = payload.error; + + this.containerManager.recordTaskCompletion( + task.pluginId, + task.instanceName!, + false + ); + + // Still mark as completed for dependency purposes (don't block other plugins) + this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath); + + this.emit('task:failed', { task, error: payload.error || 'Unknown error' }); + pending.reject(new Error(payload.error || 'Task failed')); + + } else if (payload.status === 'skipped') { + task.status = 'completed'; // Treat skipped as completed + task.error = payload.reason; + + this.containerManager.recordTaskCompletion( + task.pluginId, + task.instanceName!, + true + ); + + // Mark plugin as completed for dependency tracking + this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath); + + this.emit('task:completed', { task }); + pending.resolve(payload); + } + + return true; + } + + /** + * Check if all tasks for a file are complete + */ + private checkFileCompletion(fileHash: string): void { + const taskIds = this.fileTasks.get(fileHash); + if (!taskIds) return; + + let allComplete = true; + let filePath: string | undefined; + + for (const taskId of taskIds) { + const task = this.pendingTasks.get(taskId); + if (!task) continue; + + filePath = task.filePath; + + if ( + task.status !== 'completed' && + task.status !== 'failed' && + task.status !== 'timeout' + ) { + allComplete = false; + break; + } + } + + if (allComplete && filePath) { + // Clean up task tracking + for (const taskId of taskIds) { + this.pendingTasks.delete(taskId); + } + this.fileTasks.delete(fileHash); + + // Clean up dependency tracking + this.fileCompletedPlugins.delete(fileHash); + // Clean up any stale dependency waiters for this file + for (const key of this.dependencyWaiters.keys()) { + if (key.startsWith(`${fileHash}:`)) { + this.dependencyWaiters.delete(key); + } + } + + this.emit('file:complete', { fileHash, filePath }); + } + } + + /** + * Fetch existing metadata for a file + */ + private async fetchExistingMetadata(fileHash: string): Promise> { + try { + const response = await fetch(`${this.metaCoreUrl}/meta/${fileHash}`, { + method: 'GET', + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + return {}; + } + + const data = await response.json() as { metadata?: Record }; + return data.metadata || {}; + } catch { + return {}; + } + } + + /** + * Get queue status + */ + getQueueStatus(): { + fast: { pending: number; running: number }; + background: { pending: number; running: number }; + pendingTasks: number; + pendingCallbacks: number; + } { + return { + fast: { + pending: this.fastQueue.pending, + running: this.fastQueue.pending - this.fastQueue.size, + }, + background: { + pending: this.backgroundQueue.pending, + running: this.backgroundQueue.pending - this.backgroundQueue.size, + }, + pendingTasks: this.pendingTasks.size, + pendingCallbacks: this.pendingCallbacks.size, + }; + } + + /** + * Get pending tasks for a file + */ + getPendingTasksForFile(fileHash: string): ContainerTask[] { + const taskIds = this.fileTasks.get(fileHash); + if (!taskIds) return []; + + return Array.from(taskIds) + .map(id => this.pendingTasks.get(id)) + .filter((t): t is ContainerTask => t !== undefined); + } + + /** + * Cancel all tasks for a file + */ + cancelFile(fileHash: string): void { + const taskIds = this.fileTasks.get(fileHash); + if (!taskIds) return; + + for (const taskId of taskIds) { + const pending = this.pendingCallbacks.get(taskId); + if (pending) { + clearTimeout(pending.timeoutHandle); + pending.reject(new Error('Task cancelled')); + this.pendingCallbacks.delete(taskId); + } + this.pendingTasks.delete(taskId); + } + + this.fileTasks.delete(fileHash); + } + + /** + * Clear all queues + */ + clear(): void { + this.fastQueue.clear(); + this.backgroundQueue.clear(); + + for (const pending of this.pendingCallbacks.values()) { + clearTimeout(pending.timeoutHandle); + pending.reject(new Error('Scheduler cleared')); + } + + this.pendingCallbacks.clear(); + this.pendingTasks.clear(); + this.fileTasks.clear(); + this.fileCompletedPlugins.clear(); + this.dependencyWaiters.clear(); + } + + /** + * Pause queues + */ + pause(): void { + this.fastQueue.pause(); + this.backgroundQueue.pause(); + } + + /** + * Resume queues + */ + resume(): void { + this.fastQueue.start(); + this.backgroundQueue.start(); + } + + /** + * Dispatch tasks to all enabled plugins for a file + * This creates and enqueues tasks for all plugins that should process this file + */ + async dispatchAllPlugins( + fileHash: string, + filePath: string, + existingMeta: Record + ): Promise { + const status = this.containerManager.getStatus(); + + if (!status.initialized || status.pluginCount === 0) { + return; // No plugins available + } + + console.log(`[ContainerScheduler] Dispatching tasks for ${fileHash} to ${status.pluginCount} plugins`); + + // Get plugin info and dispatch tasks + for (const pluginStatus of status.plugins) { + if (!pluginStatus.enabled || pluginStatus.healthyInstances === 0) { + continue; // Skip disabled or unhealthy plugins + } + + const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId); + const queue: TaskQueueType = manifest?.defaultQueue || 'fast'; + const dependencies = manifest?.dependencies || []; + + // Create and enqueue task + const task = this.createTask( + pluginStatus.pluginId, + fileHash, + filePath, + dependencies, + queue + ); + + // Store existing metadata in task for dispatch + (task as any).existingMeta = existingMeta; + + this.enqueueTask(task); + } + } +} + +/** + * Create a container plugin scheduler + */ +export function createContainerPluginScheduler( + containerManager: ContainerManager, + options?: { + fastConcurrency?: number; + backgroundConcurrency?: number; + callbackUrl?: string; + metaCoreUrl?: string; + kvClient?: IKVClient; + } +): ContainerPluginScheduler { + return new ContainerPluginScheduler(containerManager, options); +} diff --git a/packages/meta-sort-core/src/container-plugins/DockerClient.ts b/packages/meta-sort-core/src/container-plugins/DockerClient.ts new file mode 100644 index 0000000..23146b5 --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/DockerClient.ts @@ -0,0 +1,417 @@ +/** + * Docker Client + * + * Wrapper around the Docker Engine API for container management. + * Requires: npm install dockerode @types/dockerode + */ + +import Docker from 'dockerode'; +import type { Container, ContainerInfo, ContainerCreateOptions } from 'dockerode'; +import type { Readable } from 'stream'; +import type { ContainerSpawnOptions, ContainerResourceLimits } from './types.js'; + +/** + * Docker client for container operations + */ +export class DockerClient { + private docker: Docker; + private initialized: boolean = false; + + constructor(socketPath: string = '/var/run/docker.sock') { + this.docker = new Docker({ socketPath }); + } + + /** + * Initialize and verify Docker connection + */ + async initialize(): Promise { + try { + await this.docker.ping(); + this.initialized = true; + } catch (error) { + throw new Error(`Failed to connect to Docker: ${error instanceof Error ? error.message : error}`); + } + } + + /** + * Check if Docker is connected + */ + isInitialized(): boolean { + return this.initialized; + } + + /** + * Get Docker info + */ + async getInfo(): Promise { + return this.docker.info(); + } + + /** + * Pull a Docker image + */ + async pullImage(imageName: string, onProgress?: (event: any) => void): Promise { + return new Promise((resolve, reject) => { + this.docker.pull(imageName, (err: Error | null, stream: NodeJS.ReadableStream) => { + if (err) { + reject(err); + return; + } + + this.docker.modem.followProgress( + stream, + (err: Error | null) => { + if (err) { + reject(err); + } else { + resolve(); + } + }, + onProgress + ); + }); + }); + } + + /** + * Check if an image exists locally + */ + async imageExists(imageName: string): Promise { + try { + await this.docker.getImage(imageName).inspect(); + return true; + } catch { + return false; + } + } + + /** + * Create a container + */ + async createContainer(options: ContainerSpawnOptions): Promise { + // Build labels, adding Docker Compose labels if stackName is provided + const labels: Record = { + 'meta-mesh.plugin': 'true', + ...options.labels, + }; + + // Add compose labels for Docker Desktop grouping + if (options.stackName) { + labels['com.docker.compose.project'] = options.stackName; + if (options.serviceName) { + labels['com.docker.compose.service'] = options.serviceName; + } + } + + const createOptions: ContainerCreateOptions = { + Image: options.image, + name: options.name, + Env: options.env + ? Object.entries(options.env).map(([k, v]) => `${k}=${v}`) + : undefined, + Labels: labels, + HostConfig: { + Binds: options.mounts?.map(m => + `${m.source}:${m.target}:${m.readonly ? 'ro' : 'rw'}` + ), + NetworkMode: options.network, + Memory: options.resources?.memory + ? parseMemoryLimit(options.resources.memory) + : undefined, + NanoCpus: options.resources?.cpus + ? Math.floor(options.resources.cpus * 1e9) + : undefined, + RestartPolicy: { + Name: 'unless-stopped', + }, + }, + ExposedPorts: { + '8080/tcp': {}, + }, + }; + + const container = await this.docker.createContainer(createOptions); + return container.id; + } + + /** + * Start a container + */ + async startContainer(containerId: string): Promise { + const container = this.docker.getContainer(containerId); + await container.start(); + } + + /** + * Stop a container + */ + async stopContainer(containerId: string, timeout: number = 10): Promise { + const container = this.docker.getContainer(containerId); + await container.stop({ t: timeout }); + } + + /** + * Remove a container + */ + async removeContainer(containerId: string, force: boolean = false): Promise { + const container = this.docker.getContainer(containerId); + await container.remove({ force }); + } + + /** + * Get container info + */ + async inspectContainer(containerId: string): Promise { + const container = this.docker.getContainer(containerId); + return container.inspect(); + } + + /** + * Check if a container exists + */ + async containerExists(containerId: string): Promise { + try { + await this.inspectContainer(containerId); + return true; + } catch { + return false; + } + } + + /** + * Check if a container is running + */ + async isContainerRunning(containerId: string): Promise { + try { + const info = await this.inspectContainer(containerId); + return info.State?.Running === true; + } catch { + return false; + } + } + + /** + * Get container IP address on a specific network + */ + async getContainerIP(containerId: string, network: string): Promise { + try { + const info = await this.inspectContainer(containerId); + const networkSettings = info.NetworkSettings?.Networks?.[network]; + return networkSettings?.IPAddress || null; + } catch { + return null; + } + } + + /** + * List containers with optional filters + */ + async listContainers(filters?: { + label?: string[]; + name?: string[]; + status?: string[]; + }): Promise { + const dockerFilters: Record = {}; + + if (filters?.label) { + dockerFilters.label = filters.label; + } + if (filters?.name) { + dockerFilters.name = filters.name; + } + if (filters?.status) { + dockerFilters.status = filters.status; + } + + return this.docker.listContainers({ + all: true, + filters: Object.keys(dockerFilters).length > 0 ? dockerFilters : undefined, + }); + } + + /** + * List plugin containers (created by meta-sort) + */ + async listPluginContainers(): Promise { + return this.listContainers({ + label: ['meta-mesh.plugin=true'], + }); + } + + /** + * Get container logs + */ + async getContainerLogs( + containerId: string, + options?: { + tail?: number; + since?: number; + stdout?: boolean; + stderr?: boolean; + } + ): Promise { + const container = this.docker.getContainer(containerId); + const logs = await container.logs({ + stdout: options?.stdout ?? true, + stderr: options?.stderr ?? true, + tail: options?.tail ?? 100, + since: options?.since, + }); + + // Convert buffer to string + if (Buffer.isBuffer(logs)) { + return logs.toString('utf-8'); + } + return logs as unknown as string; + } + + /** + * Stream container logs + */ + async streamContainerLogs( + containerId: string, + onLog: (log: string) => void, + options?: { + stdout?: boolean; + stderr?: boolean; + } + ): Promise<() => void> { + const container = this.docker.getContainer(containerId); + const stream = await container.logs({ + stdout: options?.stdout ?? true, + stderr: options?.stderr ?? true, + follow: true, + tail: 0, + }); + + const readable = stream as Readable; + readable.on('data', (chunk: Buffer) => { + onLog(chunk.toString('utf-8')); + }); + + return () => { + readable.destroy(); + }; + } + + /** + * Check if a network exists + */ + async networkExists(networkName: string): Promise { + try { + const networks = await this.docker.listNetworks({ + filters: { name: [networkName] }, + }); + return networks.some(n => n.Name === networkName); + } catch { + return false; + } + } + + /** + * Create a network if it doesn't exist + */ + async ensureNetwork(networkName: string): Promise { + if (await this.networkExists(networkName)) { + return; + } + + await this.docker.createNetwork({ + Name: networkName, + Driver: 'bridge', + Labels: { + 'meta-mesh': 'true', + }, + }); + } + + /** + * Connect container to network + */ + async connectToNetwork(containerId: string, networkName: string): Promise { + const network = this.docker.getNetwork(networkName); + await network.connect({ Container: containerId }); + } + + /** + * Disconnect container from network + */ + async disconnectFromNetwork(containerId: string, networkName: string): Promise { + const network = this.docker.getNetwork(networkName); + await network.disconnect({ Container: containerId }); + } + + /** + * Execute a command in a container + */ + async exec( + containerId: string, + cmd: string[], + options?: { workingDir?: string } + ): Promise<{ exitCode: number; output: string }> { + const container = this.docker.getContainer(containerId); + + const exec = await container.exec({ + Cmd: cmd, + AttachStdout: true, + AttachStderr: true, + WorkingDir: options?.workingDir, + }); + + const stream = await exec.start({ Detach: false }); + + return new Promise((resolve, reject) => { + let output = ''; + + stream.on('data', (chunk: Buffer) => { + output += chunk.toString('utf-8'); + }); + + stream.on('end', async () => { + try { + const info = await exec.inspect(); + resolve({ + exitCode: info.ExitCode ?? -1, + output, + }); + } catch (err) { + reject(err); + } + }); + + stream.on('error', reject); + }); + } +} + +/** + * Parse memory limit string to bytes + * Supports: '512m', '1g', '1024k' + */ +function parseMemoryLimit(limit: string): number { + const match = limit.toLowerCase().match(/^(\d+)([kmg]?)$/); + if (!match) { + throw new Error(`Invalid memory limit format: ${limit}`); + } + + const value = parseInt(match[1], 10); + const unit = match[2] || ''; + + switch (unit) { + case 'k': + return value * 1024; + case 'm': + return value * 1024 * 1024; + case 'g': + return value * 1024 * 1024 * 1024; + default: + return value; + } +} + +/** + * Create a new Docker client instance + */ +export function createDockerClient(socketPath?: string): DockerClient { + return new DockerClient(socketPath); +} diff --git a/packages/meta-sort-core/src/container-plugins/index.ts b/packages/meta-sort-core/src/container-plugins/index.ts new file mode 100644 index 0000000..f78167d --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/index.ts @@ -0,0 +1,34 @@ +/** + * Container Plugins Module + * + * Provides support for running plugins as Docker containers. + */ + +// Types +export * from './types.js'; + +// Configuration +export { + loadConfig, + parseConfig, + getEnabledPlugins, + getDefaultConfigPath, + ConfigParseError, +} from './ConfigParser.js'; + +// Docker Client +export { DockerClient, createDockerClient } from './DockerClient.js'; + +// Container Manager +export { + ContainerManager, + createContainerManager, + type ContainerManagerEvents, +} from './ContainerManager.js'; + +// Container Plugin Scheduler +export { + ContainerPluginScheduler, + createContainerPluginScheduler, + type ContainerSchedulerEvents, +} from './ContainerPluginScheduler.js'; diff --git a/packages/meta-sort-core/src/container-plugins/types.ts b/packages/meta-sort-core/src/container-plugins/types.ts new file mode 100644 index 0000000..649ee1a --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/types.ts @@ -0,0 +1,364 @@ +/** + * Container Plugin System Type Definitions + * + * Types for containerized plugins that run in Docker containers + * and communicate via HTTP APIs. + */ + +import type { ConfigField, SchemaField, TaskQueueType } from '../plugin-engine/types.js'; + +// ============================================================================= +// Plugin Configuration (stored in plugins.yml) +// ============================================================================= + +/** + * Resource limits for a container plugin + */ +export interface ContainerResourceLimits { + /** Memory limit (e.g., '512m', '2g') */ + memory?: string; + /** CPU limit (e.g., 1.0, 2.5) */ + cpus?: number; +} + +/** + * Volume mount configuration + */ +export interface ContainerMount { + /** Source path (relative to FILES_PATH) */ + source: string; + /** Target path inside container */ + target: string; + /** Mount as read-only (default: true) */ + readonly?: boolean; +} + +/** + * Health check configuration + */ +export interface ContainerHealthCheck { + /** Health check interval (e.g., '30s', '1m') */ + interval?: string; + /** Health check timeout (e.g., '5s') */ + timeout?: string; + /** Number of retries before marking unhealthy */ + retries?: number; +} + +/** + * Single container plugin configuration (from plugins.yml) + */ +export interface ContainerPluginConfig { + /** Whether this plugin is enabled */ + enabled?: boolean; + /** Docker image name and tag */ + image: string; + /** Number of container instances to spawn (default: 1) */ + instances?: number; + /** Resource limits */ + resources?: ContainerResourceLimits; + /** Plugin configuration values (sent to /configure endpoint) */ + config?: Record; + /** Whether plugin needs external network access (default: false) */ + network?: boolean; + /** Additional volume mounts */ + mounts?: ContainerMount[]; + /** Health check settings */ + healthCheck?: ContainerHealthCheck; + /** Default queue override (normally from manifest) */ + defaultQueue?: TaskQueueType; +} + +/** + * Root configuration file structure (plugins.yml) + */ +export interface ContainerPluginsConfig { + /** Config version */ + version?: string; + /** Plugin configurations keyed by plugin ID */ + plugins: Record; +} + +// ============================================================================= +// Plugin Manifest (returned by plugin container via GET /manifest) +// ============================================================================= + +/** + * File filter criteria for plugin + */ +export interface PluginFileFilter { + /** MIME type patterns (e.g., 'video/*', 'image/jpeg') */ + mimeTypes?: string[]; + /** File extension patterns (e.g., 'mkv', 'mp4') */ + extensions?: string[]; + /** Minimum file size in bytes */ + minSize?: number; + /** Maximum file size in bytes */ + maxSize?: number; +} + +/** + * UI display hints + */ +export interface PluginUIConfig { + /** Color for UI display (e.g., '#4CAF50') */ + color?: string; + /** Icon name (optional) */ + icon?: string; +} + +/** + * Container plugin manifest (from GET /manifest) + */ +export interface ContainerPluginManifest { + /** Unique plugin identifier */ + id: string; + /** Human-readable name */ + name: string; + /** Semantic version */ + version: string; + /** Description */ + description?: string; + /** Author or organization */ + author?: string; + /** Source repository URL */ + repository?: string; + + /** Execution priority (lower = runs earlier, 10-100 range) */ + priority?: number; + /** Default queue assignment */ + defaultQueue?: TaskQueueType; + /** Max processing time in milliseconds */ + timeout?: number; + + /** Plugin IDs that must complete before this one */ + dependencies?: string[]; + + /** File filter criteria */ + filter?: PluginFileFilter; + + /** Configuration schema (aliased as config for compatibility) */ + config?: Record; + /** Metadata output schema (aliased as schema for compatibility) */ + schema?: Record; + + /** UI display color */ + color?: string; + + /** UI display hints */ + ui?: PluginUIConfig; +} + +// ============================================================================= +// Container Plugin Instance (runtime state) +// ============================================================================= + +/** + * Container instance status + */ +export type ContainerInstanceStatus = 'starting' | 'healthy' | 'unhealthy' | 'stopped' | 'error'; + +/** + * Runtime state of a container plugin instance + */ +export interface ContainerPluginInstance { + /** Plugin ID */ + pluginId: string; + /** Docker container ID */ + containerId: string; + /** Container name (e.g., 'meta-plugin-ffmpeg-0') */ + containerName: string; + /** HTTP base URL (e.g., 'http://meta-plugin-ffmpeg-0:8080') */ + baseUrl: string; + /** Instance index (0, 1, 2, ...) */ + instanceIndex: number; + /** Current status */ + status: ContainerInstanceStatus; + /** Cached manifest from container */ + manifest?: ContainerPluginManifest; + /** Last successful health check timestamp */ + lastHealthCheck?: number; + /** Last error message */ + lastError?: string; + /** Number of tasks processed */ + tasksProcessed: number; + /** Number of failed tasks */ + tasksFailed: number; +} + +// ============================================================================= +// Plugin HTTP API Types +// ============================================================================= + +/** + * Health check response from GET /health + */ +export interface PluginHealthResponse { + status: 'healthy' | 'unhealthy'; + ready: boolean; + version: string; + message?: string; +} + +/** + * Configure request body for POST /configure + */ +export interface PluginConfigureRequest { + config: Record; +} + +/** + * Configure response from POST /configure + */ +export interface PluginConfigureResponse { + success: boolean; + error?: string; +} + +/** + * Process request body for POST /process + */ +export interface PluginProcessRequest { + /** Unique task identifier */ + taskId: string; + /** Content identifier (midhash256) */ + cid: string; + /** Absolute path to file in container */ + filePath: string; + /** Callback URL for completion notification */ + callbackUrl: string; + /** meta-core API base URL */ + metaCoreUrl: string; + /** Current metadata (flat key-value map) */ + existingMeta: Record; +} + +/** + * Process response from POST /process (immediate) + */ +export interface PluginProcessResponse { + status: 'accepted' | 'rejected'; + taskId: string; + error?: string; +} + +/** + * Callback status + */ +export type CallbackStatus = 'completed' | 'failed' | 'skipped'; + +/** + * Callback payload sent by plugin to meta-sort + */ +export interface PluginCallbackPayload { + /** Task identifier */ + taskId: string; + /** Plugin ID */ + pluginId: string; + /** Content identifier */ + cid: string; + /** Task status */ + status: CallbackStatus; + /** Processing duration in milliseconds */ + duration: number; + /** Error message (if status is 'failed') */ + error?: string; + /** Skip reason (if status is 'skipped') */ + reason?: string; +} + +// ============================================================================= +// Container Manager Types +// ============================================================================= + +/** + * Options for spawning a container + */ +export interface ContainerSpawnOptions { + /** Docker image */ + image: string; + /** Container name */ + name: string; + /** Environment variables */ + env?: Record; + /** Resource limits */ + resources?: ContainerResourceLimits; + /** Volume mounts */ + mounts?: Array<{ + source: string; + target: string; + readonly?: boolean; + }>; + /** Docker network to join */ + network?: string; + /** Labels for the container */ + labels?: Record; + /** Stack name for grouping containers in Docker Desktop (adds compose labels) */ + stackName?: string; + /** Service name for compose grouping (usually the plugin ID) */ + serviceName?: string; +} + +/** + * Container manager status + */ +export interface ContainerManagerStatus { + /** Whether manager is initialized */ + initialized: boolean; + /** Total number of plugin types */ + pluginCount: number; + /** Total running containers */ + runningContainers: number; + /** Healthy containers */ + healthyContainers: number; + /** Per-plugin status */ + plugins: Array<{ + pluginId: string; + enabled: boolean; + image: string; + instances: number; + healthyInstances: number; + manifest?: ContainerPluginManifest; + }>; +} + +// ============================================================================= +// Container Task Types +// ============================================================================= + +/** + * Container task status + */ +export type ContainerTaskStatus = 'pending' | 'waiting' | 'dispatched' | 'processing' | 'completed' | 'failed' | 'timeout'; + +/** + * Container plugin task + */ +export interface ContainerTask { + /** Unique task ID (UUID) */ + id: string; + /** File hash (midhash256) */ + fileHash: string; + /** File path */ + filePath: string; + /** Target plugin ID */ + pluginId: string; + /** Plugin dependencies */ + dependencies: string[]; + /** Task queue */ + queue: TaskQueueType; + /** Task status */ + status: ContainerTaskStatus; + /** Timestamp when created */ + createdAt: number; + /** Timestamp when dispatched to container */ + dispatchedAt?: number; + /** Timestamp when completed */ + completedAt?: number; + /** Container instance that processed this task */ + instanceName?: string; + /** Processing duration in ms */ + duration?: number; + /** Error message if failed */ + error?: string; +} diff --git a/packages/meta-sort-core/src/index.ts b/packages/meta-sort-core/src/index.ts new file mode 100644 index 0000000..991fd11 --- /dev/null +++ b/packages/meta-sort-core/src/index.ts @@ -0,0 +1,308 @@ +import {FolderWatcher, PollingWatcher} from "@metazla/meta-hash"; +import {config} from "./config/EnvConfig.js"; + +import os from 'os'; +import {existsSync} from 'fs'; +import {WatchedFileProcessor} from "./logic/WatchedFileProcessor.js"; +import {UnifiedAPIServer} from "./api/UnifiedAPIServer.js"; +import {StreamingPipeline} from "./logic/pipeline/StreamingPipeline.js"; +import {SUPPORTED_EXTENSIONS} from "./config/SupportedFileTypes.js"; +import {performanceMetrics} from "./metrics/PerformanceMetrics.js"; +import {KVManager} from "./kv/KVManager.js"; +import {getPluginManagerInstance, initializePluginManager} from "./logic/fileProcessor/FileProcessorPiscina.js"; +import {ContainerManager} from "./container-plugins/ContainerManager.js"; +import {ContainerPluginScheduler} from "./container-plugins/ContainerPluginScheduler.js"; + +try { + // Get current user information + const userInfo = os.userInfo(); + // Display the current user and UID + console.log(`[node] Current User: ${userInfo.username}`); + console.log(`[node] User ID (UID): ${userInfo.uid}`); +}catch (error) { + /* ignore */ +} + +console.log(`BUILD_VERSION: ${process.env.BUILD_VERSION || 'dev'}`); + +//display the config +console.log(config); + +process.umask(0);//This unable the process to create files with 777 permissions + +let fileProcessor = new WatchedFileProcessor(); +let folders = config.WATCH_FOLDER_LIST; + +// KV Manager for Redis-based storage with leader election (optional) +let kvManager: KVManager | null = null; + +// Container Manager for containerized plugins +let containerManager: ContainerManager | null = null; +let containerPluginScheduler: ContainerPluginScheduler | null = null; + +// Calculate concurrency for pipeline stages +const cpuCount = os.cpus().length; +// Smart defaults optimized for network storage + HDD scenarios: +// - Fast queue: CPU (fast operations: filename parsing, stats) +// - Background queue: CPU / 2 (slow operations: SHA-256, FFmpeg) +const defaultBackgroundWorkers = Math.max(1, Math.floor(cpuCount / 2)); +const validationConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS * 2 : cpuCount * 2; +const fastQueueConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS : cpuCount; +const backgroundQueueConcurrency = config.MAX_WORKER_THREADS || defaultBackgroundWorkers; + +console.log(`Pipeline concurrency: validation=${validationConcurrency}, fastQueue=${fastQueueConcurrency}, backgroundQueue=${backgroundQueueConcurrency} (cpuCount=${cpuCount}, defaultBackgroundWorkers=${defaultBackgroundWorkers})`); + +// Create streaming pipeline +const pipeline = new StreamingPipeline({ + supportedExtensions: SUPPORTED_EXTENSIONS, + validationConcurrency, + fastQueueConcurrency, + backgroundQueueConcurrency, + strictMimeValidation: false, // Skip MIME for speed + fileProcessor: fileProcessor.fileProcessor, + stateManager: fileProcessor.unifiedStateManager, + kvClient: fileProcessor.getKVClient(), + virtualFileSystem: fileProcessor.getVirtualFileSystem(), + metaDataToFolderStruct: fileProcessor.metaDataToFolderStruct +}); + +// Connect pipeline to fileProcessor for queue status monitoring +fileProcessor.setPipeline(pipeline); + +// Create generic file discovery +const folderWatcher = new FolderWatcher(); + +// Parse folder list +const folderList = folders.split(',').map(f => f.trim()).filter(f => f.length > 0); +console.log(`Watching folders: ${folderList.join(', ')}`); + +// VFS and state manager are available immediately +const vfs = fileProcessor.getVirtualFileSystem(); +const unifiedStateManager = fileProcessor.getUnifiedStateManager(); +const getDuplicateResult = () => fileProcessor.getDuplicateResult(); + +// API Server will be initialized after KV is ready (deferred to async init) +let apiServer: UnifiedAPIServer | null = null; + +// Shared progressive discovery and cleanup logic (used by both startup and manual scan) +const runProgressiveDiscovery = async () => { + console.log(`[Discovery] Progressive scan mode: ${folderList.length} folders`); + + // Process folders one at a time for progressive cleanup + for (const folder of folderList) { + console.log(`[Discovery] Scanning folder: ${folder}`); + + // Track discovered files for this folder + const discoveredFilesInFolder = new Set(); + + // Wrap discovery stream to track files + async function* trackDiscoveredFiles(stream: AsyncGenerator) { + for await (const filePath of stream) { + discoveredFilesInFolder.add(filePath); + yield filePath; + } + } + + const discoveryStream = folderWatcher.discoverFiles([folder]); + const trackedStream = trackDiscoveredFiles(discoveryStream); + + await pipeline.start(trackedStream); + + // Progressive cleanup: clean up after each folder completes + console.log(`[Discovery] Folder scan complete: ${folder}. Found ${discoveredFilesInFolder.size} files. Running cleanup...`); + await fileProcessor.cleanupStaleEntries([folder], discoveredFilesInFolder); + console.log(`[Discovery] Cleanup complete for folder: ${folder}`); + } + + console.log(`[Discovery] All folders scanned and cleaned up`); +}; + +// Manual scan trigger (called from API) +// Note: Stremio updates happen automatically via incremental add/remove notifications +// during file processing (like FUSE), so no explicit refresh needed here +const triggerScan = async () => { + console.log('Triggering manual scan from API...'); + await runProgressiveDiscovery(); +}; + +// Global error handlers - prevent crashes from unforeseen errors +process.on('uncaughtException', (error: Error) => { + console.error('❌ [CRITICAL] Uncaught Exception:', error); + console.error('Stack:', error.stack); + // Log but don't exit - allow process to continue + // Monitor for worker health issues +}); + +process.on('unhandledRejection', (reason: any, promise: Promise) => { + console.error('❌ [CRITICAL] Unhandled Rejection at:', promise); + console.error('Reason:', reason); + // Log but don't exit - allow process to continue +}); + +// Graceful shutdown +process.on('SIGINT', async () => { + console.log('\nShutting down...'); + if (apiServer) { + await apiServer.stop(); + } + + // Stop Container Manager if running + if (containerManager) { + console.log('[Shutdown] Stopping Container Manager...'); + await containerManager.shutdown(); + } + + // Stop KV Manager if running + if (kvManager) { + console.log('[Shutdown] Stopping KV Manager...'); + await kvManager.stop(); + } + + process.exit(0); +}); + +// Start initial discovery and processing (streaming pipeline) +(async () => { + try { + // Initialize KV Manager if configured (Redis + leader election) + if (config.META_CORE_PATH) { + console.log('[Startup] Initializing KV Manager (Redis + leader election)...'); + kvManager = new KVManager({ + metaCorePath: config.META_CORE_PATH, + filesPath: config.FILES_PATH, + serviceName: config.SERVICE_NAME, + version: config.SERVICE_VERSION, + apiPort: config.FUSE_API_PORT, + redisPort: config.REDIS_PORT, + redisUrl: config.REDIS_URL, + baseUrl: config.BASE_URL, + capabilities: ['write', 'mount', 'monitor'] + }); + + // Wait for KV to be ready + await kvManager.start(); + await kvManager.waitForReady(); + + // Pass KV client to file processor + const client = kvManager.getClient(); + if (client) { + fileProcessor.setKVClient(client); + console.log(`[Startup] KV Manager ready (role: ${kvManager.isLeader() ? 'LEADER' : 'FOLLOWER'})`); + } + } + + // Start Unified API Server AFTER KV is ready (so kvClient is available) + const kvClient = fileProcessor.getKVClient(); + apiServer = new UnifiedAPIServer(vfs, { + port: config.FUSE_API_PORT, + host: config.FUSE_API_HOST, + enableCors: true, + }, unifiedStateManager, getDuplicateResult, kvClient || undefined, triggerScan, backgroundQueueConcurrency, fastQueueConcurrency, () => fileProcessor.getQueueStatus(), kvManager || undefined, getPluginManagerInstance, () => fileProcessor.getTaskScheduler()); + + await apiServer.start(); + console.log('Unified API Server started successfully'); + + // Initialize plugin manager early so it's available for API + console.log('[Startup] Initializing plugin manager...'); + const pluginManager = await initializePluginManager(); + console.log('[Startup] Plugin manager initialized'); + + // Initialize container plugins if config file exists + if (existsSync(config.CONTAINER_PLUGINS_CONFIG)) { + console.log('[Startup] Initializing container plugins...'); + try { + containerManager = new ContainerManager(); + await containerManager.initialize(); + + // Connect container manager to plugin manager + pluginManager.setContainerManager(containerManager); + await pluginManager.loadContainerPlugins(); + + // Create container plugin scheduler + containerPluginScheduler = new ContainerPluginScheduler(containerManager, { + fastConcurrency: config.FAST_QUEUE_CONCURRENCY, + backgroundConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY, + kvClient: kvClient || undefined, + }); + + // Connect to API server (for callback handling) + apiServer.setContainerPluginManager(containerManager, containerPluginScheduler); + console.log('[Startup] ContainerPluginScheduler connected to API server'); + + const status = containerManager.getStatus(); + console.log(`[Startup] Container plugins initialized: ${status.pluginCount} plugins, ${status.healthyContainers}/${status.runningContainers} healthy`); + } catch (error) { + console.error('[Startup] Failed to initialize container plugins:', error); + console.log('[Startup] Continuing without container plugins...'); + } + } else { + console.log(`[Startup] Container plugins config not found at ${config.CONTAINER_PLUGINS_CONFIG}, skipping`); + } + + // Initialize hash index cache (loads existing hashes from disk) + // This also initializes the TaskScheduler + console.log('[Startup] Initializing hash index cache...'); + await fileProcessor.initialize(); + console.log('[Startup] Hash index cache initialized'); + + // Connect ContainerPluginScheduler to TaskScheduler and Pipeline + if (containerPluginScheduler) { + const taskScheduler = fileProcessor.getTaskScheduler(); + if (taskScheduler) { + taskScheduler.setContainerPluginScheduler(containerPluginScheduler); + console.log('[Startup] ContainerPluginScheduler connected to TaskScheduler'); + } + + // Connect to StreamingPipeline for file processing dispatch + pipeline.setContainerPluginScheduler(containerPluginScheduler); + console.log('[Startup] ContainerPluginScheduler connected to StreamingPipeline'); + } + + // First, rebuild VFS from KV to restore state from previous runs + console.log('[Startup] Rebuilding VirtualFileSystem from KV...'); + await fileProcessor.rebuildVFSFromKV(); + console.log('[Startup] VFS rebuild completed'); + + console.log('[Startup] Starting initial file discovery with progressive cleanup...'); + await runProgressiveDiscovery(); + console.log('[Startup] Initial discovery completed, processing continues in background'); + + // Log cache performance summary + const metrics = performanceMetrics.getMetrics(); + if (metrics.cacheHits.midhash256 || metrics.cacheMisses.midhash256) { + const hits = metrics.cacheHits.midhash256 || 0; + const misses = metrics.cacheMisses.midhash256 || 0; + const total = hits + misses; + const hitRate = total > 0 ? ((hits / total) * 100).toFixed(1) : '0.0'; + console.log(`[Cache] midhash256 cache: ${hits} hits, ${misses} misses (${hitRate}% hit rate)`); + } + + // Setup file watching for changes + console.log('[Startup] Setting up file watching...'); + folderWatcher.watch(folderList, { + onAdd: (filePath) => pipeline.handleFileAdded(filePath), + onChange: (filePath) => pipeline.handleFileChanged(filePath), + onUnlink: (filePath) => pipeline.handleFileDeleted(filePath) + }, { + interval: config.WATCH_FOLDER_POOLING_INTERVAL_MS || 30000, + stabilityThreshold: 30000, + pollInterval: 5000 + }); + console.log('[Startup] File watching active'); + } catch (error) { + console.error('[Startup] Error during initial discovery:', error); + } +})(); + +// Start PollingWatcher for path-specific polling (if configured) +if (config.POLLING_PATHS && config.POLLING_PATHS.length > 0) { + console.log(`PollingWatcher: ${config.POLLING_PATHS.length} paths configured`); + const pollingWatcher = new PollingWatcher(fileProcessor, config.POLLING_PATHS); + pollingWatcher.start().then(() => { + console.log('PollingWatcher started successfully'); + }).catch((error) => { + console.error('Error starting PollingWatcher:', error); + }); +} else { + console.log('PollingWatcher: No polling paths configured (skipped)'); +} diff --git a/packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts b/packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts new file mode 100644 index 0000000..a6eedb1 --- /dev/null +++ b/packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts @@ -0,0 +1,97 @@ +import {config} from "../config/EnvConfig.js"; +import axios from "axios"; + +export class JellyfinAPI { + + constructor() { + + } + + async getMetaData(filepath: string): Promise { + console.log("geteMetaData"); + } + + private async sendAuthedRequest(url: string, method: string, data?: any, params?: any) { + return (await axios({ + method: method, + url: url, + headers: { + 'X-MediaBrowser-Token': config.JELLYFIN_API_KEY + }, + data: data, + params: params + })).data; + } + + private async getRootUserId(): Promise { + const users = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Users`, 'GET'); + for (const user of users) { + if (user.Policy.IsAdministrator) { + return user.Id; + } + } + return users[0].Id; + } + + private async getItemIdByFilePath(filePath: string): Promise { + try { + let user = await this.getRootUserId(); + const items = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Items`, 'GET', null, { + Path: encodeURIComponent(filePath), + userId: user + }); + + if (items.Items && items.Items.length > 0) { + return items.Items[0].Id; + } else { + console.log(`Item with file path "${filePath}" not found.`); + return null; + } + } catch (error) { + console.error('Error fetching item ID by file path:', error.message); + return null; + } + } + + jellyApiAvailable(): boolean { + return !!config.JELLYFIN_API_KEY && !!config.JELLYFIN_ENDPOINT; + } + + /** takes around 30s to complete (if light update) */ + async refreshLibrary(): Promise{ + try { + const response = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Library/Refresh`, "POST",null, { + headers: { + 'X-MediaBrowser-Token': config.JELLYFIN_API_KEY + } + }); + } catch (error) { + console.error('Error refreshing library:', error.message); + } + } + + async notifyJellyfin(filePath): Promise { + const itemId = await this.getItemIdByFilePath(filePath); + if(!itemId) { + return; + } + // Fetch the public system info + const response = await axios.post(`${config.JELLYFIN_ENDPOINT}/Items/${itemId}/Refresh`, { + + },{ + headers: { + 'X-MediaBrowser-Token': config.JELLYFIN_API_KEY + } + }); + console.log(response.data); + } + + async systemEndpoint() { + const response = await axios.get(`${config.JELLYFIN_ENDPOINT}/System/Endpoint`, { + headers: { + 'X-MediaBrowser-Token': config.JELLYFIN_API_KEY + } + }); + console.log(response.data); + } +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts b/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts new file mode 100644 index 0000000..e6c96d7 --- /dev/null +++ b/packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts @@ -0,0 +1,278 @@ +import {readFile, writeFile} from 'fs/promises'; +import {Builder, Parser} from 'xml2js'; +import {mergeMetadata} from "../logic/tool/MetadataMerger.js"; +import {existsAsync} from "@metazla/meta-hash"; +import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface"; +import lodash from 'lodash'; +import * as xml2js from "xml2js"; + +export class JellyfinNfo { + + options = {explicitArray: true, ignoreAttrs: true}; + + /** + * Convert a JSON XML object back to a regular JSON object where arrays containing `set` are converted back to Set objects + * @param input - The input JSON XML object + */ + private convertFromJsonXml(input: any): any { + const output: any = {}; + + for (const [key, value] of Object.entries(input)) { + if (!Array.isArray(value) || value.length === 0) { + throw new Error("Unexpected format: each property should be an array."); + } + + const firstElement = value[0]; + + if (firstElement && typeof firstElement === 'object') { + if ('set' in firstElement) { + // Convert array from 'set' back to Set object + output[key] = new Set(firstElement.set); + } else { + // Recursively convert from JsonXml for nested objects + output[key] = this.convertFromJsonXml(firstElement); + } + } else { + // Primitive value, simply assign it as it is the only element in the array + output[key] = firstElement; + } + } + + return output; + } + + /** + * Filter only known jellyfin supported data + * @param data + */ + filterData(data: any): any { + const supportedFields = ["originalTitle", "season", "episode", "videoType", "fileType"] + const result = {}; + for (const key of supportedFields) { + if (data[key]) { + result[key] = data[key]; + } + } + return result; + } + + async read(nfoPath: string): Promise { + if (!nfoPath.endsWith('.nfo')) { + throw new Error(`NFO file path must end with '.nfo'`); + } + let metadata: T = {} as T; + if (await existsAsync(nfoPath)) { + try { + const nfoContent = await readFile(nfoPath, {encoding: 'utf8'}); + const parser = new xml2js.Parser(this.options); + const result = await parser.parseStringPromise(nfoContent); + metadata = result.root || result.episodedetails || result.movie || {}; + metadata = this.convertFromJsonXml(metadata); + } catch (error) { + console.error(`Error parsing NFO file ${nfoPath}`); + } + } + + // we always read the file for meta data but ignore its content except for cid which are long to compute + return metadata; + } + + async update(nfoPath: string, data: T): Promise { + data = this.filterData(data); + if (!nfoPath.endsWith('.nfo')) { + throw new Error(`NFO file path must end with '.nfo'`); + } + if (data.fileType !== "video") { + return;//only process video files for writing jellyfin nfo + } + if (await existsAsync(nfoPath)) { + await this.updateNfoFile(nfoPath, data as any); + } else { + throw new Error(`NFO file does not exist at path: ${nfoPath}`); + } + } + + public async createOrUpdate(nfoPath: string, data: T): Promise { + data = this.filterData(data); + if (!nfoPath.endsWith('.nfo')) { + throw new Error(`NFO file path must end with '.nfo'`); + } + if (data.fileType !== "video") { + return;//only process video files for writing jellyfin nfo + } + if (await existsAsync(nfoPath)) { + await this.updateNfoFile(nfoPath, data as any); + } else { + // If the NFO file doesn't exist, just generate a new one + await this.generateNfoFile(nfoPath, data as any); + } + } + + /** + * Convert a regular JSON object to an XML JSON object (add arrays) + * @param input + */ + private convertToJsonXml(input: any): any { + const output: any = {}; + + // Use Object.entries to loop through each key-value pair in the input object + for (const [key, value] of Object.entries(input)) { + if (!this.assertLegalName(key)) { + throw new Error(`Illegal character in key: ${key}`); + } + if (value instanceof Set) { + // If the value is a Set, convert it to an array and then to XmlJson format + const array = Array.from(value as any); + if (array.length > 0) { + output[key] = [{set: array}]; + } + } else if (value instanceof Array) { + throw new Error("unsupported"); + } else if (typeof value === 'object' && value !== null) { + // For nested objects, recursively convert + let jsonXml = this.convertToJsonXml(value); + if (jsonXml instanceof Array) { + output[key] = jsonXml; + } else { + output[key] = [jsonXml]; + } + } else { + output[key] = [value]; + } + } + + return output; + } + + private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) { + try { + const existingContent = await readFile(nfoFilePath, {encoding: "utf-8"}); + const parser = new Parser(this.options); + let existingData = await parser.parseStringPromise(existingContent); + let hasChanged = false; + + // 1st try to update the type of the metadata (root can be "root" or "movie" or "episodedetails") and + // according to the 'jelly' parameter and the isMovieOrTvShow function + const targetRoot = (this.isMovieOrTvShow(data) === "tvshow" ? 'episodedetails' : "movie"); + if (existingData.root) { + existingData[targetRoot] = existingData.root; + delete existingData.root; + hasChanged = true; + } else if (existingData.movie && "movie" !== targetRoot) { + existingData[targetRoot] = existingData.movie; + delete existingData.movie; + hasChanged = true; + } else if (existingData.episodedetails && "episodedetails" !== targetRoot) { + existingData[targetRoot] = existingData.episodedetails; + delete existingData.episodedetails; + hasChanged = true; + } + + //2nd update the data + let root = existingData[targetRoot]; + if (typeof root !== 'object') { + root = {};//don't allow primitive type in root + hasChanged = true; + } + const convertedData = this.convertToJsonXml(data); + hasChanged = mergeMetadata(root, convertedData, nfoFilePath) || hasChanged; + root = this.sanitizeData(root); + existingData[targetRoot] = root; + + if (hasChanged) { + //const builder = new Builder({ headless: true, renderOpts: { 'pretty': true } }); + const builder = new Builder(); + // Convert the updated object back to XML + try { + const updatedContent = builder.buildObject(existingData); + await writeFile(nfoFilePath, updatedContent, {encoding: "utf-8"}); + } catch (error) { + console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`); + } + } + } catch (error) { + console.error(`Failed to parse existing NFO content ${nfoFilePath} : ${error}`); + return; + } + } + + private isMovieOrTvShow(data: VideoMeta): "tvshow" | "movie" { + if (data.videoType) { + return data.videoType; + } else if (data.episode) { + return "tvshow"; + } else { + return "movie"; + } + } + + private assertLegalChar(str): boolean { + return str.match(/[\0\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/); + }; + + private assertLegalName(str: string) { + if (this.assertLegalChar(str)) { + return false; + } + const regex = /^([:A-Z_a-z\xC0-\xD6\xD8-\xF6\xF8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])([\x2D\.0-:A-Z_a-z\xB7\xC0-\xD6\xD8-\xF6\xF8-\u037D\u037F-\u1FFF\u200C\u200D\u203F\u2040\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])*$/; + if (!str.match(regex)) { + return false; + } + return true; + }; + + private sanitizeData(data: any): any { + const sanitize = (value: any): any => { + if (typeof value === 'string') { + // Escaping the string values + return lodash.escape(value); + } else if (Array.isArray(value)) { + throw new Error(`Arrays are not supported in NFO files`); + } else if (typeof value === 'object' && value !== null) { + // Recursively sanitize each property in the object + let result = {}; + for (const key of Object.keys(value)) { + if (this.assertLegalName(key)) { + result[key] = value[key]; + } + } + return result; + } + return value; + }; + + return sanitize(data); + } + + private async generateNfoFile(nfoFilePath: string, data: JellyfinMeta) { + if (!nfoFilePath.endsWith('.nfo')) { + throw new Error(`NFO file path must end with '.nfo'`); + } + if (await existsAsync(nfoFilePath)) { + throw new Error(`NFO file already exists at path: ${nfoFilePath}`); + } + + const rootElementName = data.videoType === "tvshow" ? 'episodedetails' : "movie"; + /*const schemaAttributes = jelly ? {} : { + 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + 'xsi:schemaLocation': 'http://www.example.com/schema http://www.example.com/schema/mediaFileSchema.xsd' + };*/ + const schemaAttributes = {}; + + const convertedData = this.convertToJsonXml(data); + const sanitizedData = this.sanitizeData(convertedData); + + // Prepare the data object with the correct root element and attributes + const obj = { + [rootElementName]: { + ...({$: schemaAttributes}), + ...sanitizedData + } + }; + + const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}}); + const nfoContent = builder.buildObject(obj); + await writeFile(nfoFilePath, nfoContent, {encoding: "utf-8"}); + } + +} diff --git a/packages/meta-sort-core/src/jellyfin/README b/packages/meta-sort-core/src/jellyfin/README new file mode 100644 index 0000000..7f660e8 --- /dev/null +++ b/packages/meta-sort-core/src/jellyfin/README @@ -0,0 +1,10 @@ +TODO: +Jellyfin nfo format (only for video) + - nfo to metadata (this is used as lib) + - metadata to nfo + - create or update nfo + +meta yml to nfo + - nfo to metadata (this is used as lib) + - metadata to nfo + - create or update nfo diff --git a/packages/meta-sort-core/src/jellyfin/nfoTool.ts b/packages/meta-sort-core/src/jellyfin/nfoTool.ts new file mode 100644 index 0000000..ef2d55e --- /dev/null +++ b/packages/meta-sort-core/src/jellyfin/nfoTool.ts @@ -0,0 +1,22 @@ +import { parse, sep } from "path"; + +/** + * Get the nfo file path for a given file path (replace the extension with .nfo) + * @param filePath + * @param ext - The extension to replace with, default is "nfo" + * @returns The nfo file path + */ +export function filePathReplace(filePath: string, ext: string = "nfo"): string { + const { dir, name } = parse(filePath); + return `${dir}${sep}${name}.${ext}`; +} + +/** + * Get the nfo file path for a given folder path (add .nfo to the folder path) + * @param folderPath + * @param ext + * @returns The nfo file path + */ +export function filePathAdd(folderPath: string, ext: string = "nfo"): string { + return `${folderPath}.${ext}`; +} diff --git a/packages/meta-sort-core/src/kv/IKVClient.ts b/packages/meta-sort-core/src/kv/IKVClient.ts new file mode 100644 index 0000000..6952498 --- /dev/null +++ b/packages/meta-sort-core/src/kv/IKVClient.ts @@ -0,0 +1,192 @@ +/** + * KV Client Interface - Abstraction layer for key-value storage + * + * Redis-based key-value storage with nested key architecture. + * Uses property-level granularity for efficient updates. + * + * Key Structure: + * /file/{hashId}/{property/path} → value + * + * Example: + * /file/midhash256:abc123/title → "Inception" + * /file/midhash256:abc123/video/codec → "h265" + */ + +export interface KeyValuePair { + key: string; + value: string; +} + +export interface IKVClient { + // ======================================================================== + // Basic Operations + // ======================================================================== + + /** + * Set a key-value pair (value is JSON serialized) + */ + set(key: string, value: any): Promise; + + /** + * Get a value by key (value is JSON deserialized) + */ + get(key: string): Promise; + + /** + * Delete a key + */ + delete(key: string): Promise; + + /** + * Count keys with a given prefix + */ + countKeysWithPrefix(prefix: string): Promise; + + /** + * Health check - verifies KV store is accessible + */ + health(): Promise; + + // ======================================================================== + // Nested Key Architecture Methods + // ======================================================================== + + /** + * Set a single property value (raw string, not JSON) + */ + setProperty(key: string, value: string): Promise; + + /** + * Get a single property value (raw string) + */ + getProperty(key: string): Promise; + + /** + * Get all key-value pairs with a given prefix + */ + getRange(prefix: string): Promise; + + /** + * Set multiple key-value pairs atomically (or as close as possible) + */ + setMultiple(pairs: KeyValuePair[]): Promise; + + /** + * Delete all keys with a given prefix + */ + deleteRange(prefix: string): Promise; + + // ======================================================================== + // High-Level Metadata Operations + // ======================================================================== + + /** + * Store file metadata using nested key architecture + * Flattens the metadata object and stores each property as a separate key + */ + setMetadataFlat(hashId: string, metadata: any, excludeFields?: string[]): Promise; + + /** + * Retrieve file metadata - reconstructs from individual property keys + */ + getMetadataFlat(hashId: string): Promise; + + /** + * Get a specific property from a file's metadata + */ + getMetadata(hashId: string, propertyPath: string): Promise; + + /** + * Delete all metadata for a file + */ + deleteMetadataFlat(hashId: string): Promise; + + /** + * Get all unique hash IDs (files) stored in KV + */ + getAllHashIds(): Promise; + + // ======================================================================== + // Lifecycle + // ======================================================================== + + /** + * Close connections and cleanup resources + */ + close(): Promise; +} + +/** + * KV Client configuration + */ +export interface KVClientConfig { + /** Backend type */ + type: 'redis'; + + /** Connection URL (e.g., redis://localhost:6379) */ + url: string; + + /** Operation timeout in ms (default: 30000) */ + timeout?: number; + + /** Key prefix for all operations (default: '' for none) */ + prefix?: string; +} + +/** + * Leader lock file content format + */ +export interface LeaderLockInfo { + /** Hostname of the leader */ + host: string; + + /** Redis connection URL (e.g., redis://10.0.1.50:6379) */ + api: string; + + /** HTTP API URL for the leader service */ + http: string; + + /** Configured BASE_URL for stable service discovery (optional, falls back to http) */ + baseUrl?: string; + + /** Timestamp when leadership was acquired */ + timestamp: number; + + /** Process ID of the leader */ + pid: number; +} + +/** + * Service registration info + */ +export interface ServiceInfo { + /** Service name (e.g., 'meta-sort', 'meta-fuse') */ + name: string; + + /** Service version */ + version: string; + + /** HTTP API URL */ + api: string; + + /** Current status */ + status: 'starting' | 'running' | 'stopping' | 'stopped'; + + /** Process ID */ + pid: number; + + /** Hostname */ + hostname: string; + + /** Start timestamp (ISO string) */ + startedAt: string; + + /** Last heartbeat timestamp (ISO string) */ + lastHeartbeat: string; + + /** Service capabilities */ + capabilities: string[]; + + /** Named endpoints */ + endpoints: Record; +} diff --git a/packages/meta-sort-core/src/kv/KVManager.ts b/packages/meta-sort-core/src/kv/KVManager.ts new file mode 100644 index 0000000..cf8aa55 --- /dev/null +++ b/packages/meta-sort-core/src/kv/KVManager.ts @@ -0,0 +1,388 @@ +/** + * KV Manager - Unified manager for KV storage, leader election, and service discovery + * + * This is the main entry point for the KV subsystem. It: + * 1. Participates in leader election (spawns Redis if leader) + * 2. Creates and manages the KV client (Redis) + * 3. Handles service discovery registration + * 4. Manages reconnection on leader failure + * + * Usage: + * ```typescript + * const kvManager = new KVManager({ + * metaCorePath: '/meta-core', + * filesPath: '/files', + * serviceName: 'meta-sort', + * apiPort: 3000 + * }); + * + * await kvManager.start(); + * const client = kvManager.getClient(); + * // Use client... + * await kvManager.stop(); + * ``` + */ + +import { networkInterfaces } from 'os'; +import { LeaderElection } from './LeaderElection.js'; +import { ServiceDiscovery } from './ServiceDiscovery.js'; +import { RedisKVClient } from './RedisClient.js'; +import type { IKVClient, LeaderLockInfo } from './IKVClient.js'; + +interface KVManagerConfig { + /** Path to META_CORE_VOLUME (e.g., /meta-core) */ + metaCorePath: string; + + /** Path to FILES_VOLUME (e.g., /files) */ + filesPath: string; + + /** Service name (e.g., 'meta-sort', 'meta-fuse') */ + serviceName: string; + + /** Service version */ + version?: string; + + /** HTTP API port */ + apiPort: number; + + /** Redis port (default: 6379) */ + redisPort?: number; + + /** Service capabilities */ + capabilities?: string[]; + + /** Whether to skip leader election and use provided Redis URL */ + redisUrl?: string; + + /** Base URL for service discovery (overrides auto-detected URL) */ + baseUrl?: string; + + /** + * Hostname to advertise in lock file for other services to connect. + * Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution. + */ + advertiseHost?: string; +} + +export class KVManager { + private config: KVManagerConfig; + private leaderElection: LeaderElection | null = null; + private serviceDiscovery: ServiceDiscovery | null = null; + private kvClient: IKVClient | null = null; + private isStarted = false; + private isShuttingDown = false; + + // Event callbacks + private onReadyCallbacks: (() => void)[] = []; + private onReconnectCallbacks: (() => void)[] = []; + + constructor(config: KVManagerConfig) { + this.config = { + version: '1.0.0', + redisPort: 6379, + capabilities: [], + ...config + }; + } + + /** + * Get the local IP for API URL construction + */ + private getLocalIP(): string { + const interfaces = networkInterfaces(); + + for (const name of Object.keys(interfaces)) { + const addrs = interfaces[name]; + if (!addrs) continue; + + for (const addr of addrs) { + if (addr.family === 'IPv4' && !addr.internal) { + return addr.address; + } + } + } + + return 'localhost'; + } + + /** + * Create Redis client from URL + */ + private async createRedisClient(redisUrl: string): Promise { + const client = new RedisKVClient({ + url: redisUrl, + timeout: 30000, + prefix: '' // No prefix for shared access + }); + + await client.connect(); + return client; + } + + /** + * Start the KV manager + */ + async start(): Promise { + if (this.isStarted) { + console.warn('[KVManager] Already started'); + return; + } + + console.log(`[KVManager] Starting for ${this.config.serviceName}...`); + + // Check for direct Redis URL (skip leader election) + if (this.config.redisUrl) { + console.log(`[KVManager] Using direct Redis URL: ${this.config.redisUrl}`); + this.kvClient = await this.createRedisClient(this.config.redisUrl); + + // Initialize service discovery even with direct Redis URL + const apiUrl = this.config.baseUrl || `http://localhost:${this.config.apiPort}`; + this.serviceDiscovery = new ServiceDiscovery({ + metaCorePath: this.config.metaCorePath, + serviceName: this.config.serviceName, + version: this.config.version!, + apiUrl, + capabilities: this.config.capabilities, + endpoints: { + health: '/health', + dashboard: '/', + api: '/api', + metrics: '/api/metrics', + processing: '/api/processing/status' + } + }); + await this.serviceDiscovery.start(); + + this.isStarted = true; + this.notifyReady(); + return; + } + + // Initialize leader election + const localIP = this.getLocalIP(); + + this.leaderElection = new LeaderElection({ + metaCorePath: this.config.metaCorePath, + serviceName: this.config.serviceName, + apiPort: this.config.apiPort, + redisPort: this.config.redisPort, + baseUrl: this.config.baseUrl + }); + + // Setup leader election callbacks + this.leaderElection.onLeader(async () => { + console.log('[KVManager] We are the leader, connecting to local Redis...'); + + // Connect to local Redis (spawned by leader election) + const redisUrl = `redis://127.0.0.1:${this.config.redisPort}`; + this.kvClient = await this.createRedisClient(redisUrl); + + this.notifyReady(); + }); + + this.leaderElection.onFollower(async (leaderInfo: LeaderLockInfo) => { + console.log(`[KVManager] We are a follower, connecting to leader at ${leaderInfo.api}...`); + + // Connect to leader's Redis + this.kvClient = await this.createRedisClient(leaderInfo.api); + + this.notifyReady(); + }); + + this.leaderElection.onLostLeader(async () => { + console.log('[KVManager] Leader lost, disconnecting...'); + + if (this.kvClient) { + await this.kvClient.close(); + this.kvClient = null; + } + + // Leader election will handle re-election and reconnection + }); + + // Initialize service discovery + // Use BASE_URL if provided, otherwise fall back to auto-detected local IP + const apiUrl = this.config.baseUrl || `http://${localIP}:${this.config.apiPort}`; + + this.serviceDiscovery = new ServiceDiscovery({ + metaCorePath: this.config.metaCorePath, + serviceName: this.config.serviceName, + version: this.config.version!, + apiUrl, + capabilities: this.config.capabilities, + endpoints: { + health: '/health', + dashboard: '/', + api: '/api', + metrics: '/api/metrics', + processing: '/api/processing/status' + } + }); + + // Start leader election first + await this.leaderElection.start(); + + // Start service discovery + await this.serviceDiscovery.start(); + + this.isStarted = true; + } + + /** + * Stop the KV manager + */ + async stop(): Promise { + if (!this.isStarted) return; + + console.log('[KVManager] Stopping...'); + this.isShuttingDown = true; + + // Stop service discovery + if (this.serviceDiscovery) { + await this.serviceDiscovery.stop(); + this.serviceDiscovery = null; + } + + // Close KV client + if (this.kvClient) { + await this.kvClient.close(); + this.kvClient = null; + } + + // Stop leader election (releases lock, stops Redis if leader) + if (this.leaderElection) { + await this.leaderElection.stop(); + this.leaderElection = null; + } + + this.isStarted = false; + console.log('[KVManager] Stopped'); + } + + /** + * Notify ready callbacks + */ + private notifyReady(): void { + for (const callback of this.onReadyCallbacks) { + try { + callback(); + } catch (error) { + console.error('[KVManager] Error in ready callback:', error); + } + } + } + + /** + * Register a callback for when the KV client is ready + */ + onReady(callback: () => void): this { + this.onReadyCallbacks.push(callback); + + // If already ready, call immediately + if (this.kvClient) { + callback(); + } + + return this; + } + + /** + * Register a callback for reconnection events + */ + onReconnect(callback: () => void): this { + this.onReconnectCallbacks.push(callback); + return this; + } + + /** + * Wait for the KV client to be ready + */ + async waitForReady(timeoutMs: number = 30000): Promise { + if (this.kvClient) return; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error(`KV client not ready after ${timeoutMs}ms`)); + }, timeoutMs); + + this.onReady(() => { + clearTimeout(timeout); + resolve(); + }); + }); + } + + // ======================================================================== + // Getters + // ======================================================================== + + /** + * Get the KV client (may be null if not yet connected) + */ + getClient(): IKVClient | null { + return this.kvClient; + } + + /** + * Get the KV client (throws if not connected) + */ + requireClient(): IKVClient { + if (!this.kvClient) { + throw new Error('KV client not available. Call start() and wait for ready.'); + } + return this.kvClient; + } + + /** + * Check if this service is the leader + */ + isLeader(): boolean { + return this.leaderElection?.isLeader() ?? false; + } + + /** + * Check if this service is a follower + */ + isFollower(): boolean { + return this.leaderElection?.isFollower() ?? false; + } + + /** + * Get the current leader info + */ + getLeaderInfo(): LeaderLockInfo | null { + return this.leaderElection?.getLeaderInfo() ?? null; + } + + /** + * Get service discovery instance + */ + getServiceDiscovery(): ServiceDiscovery | null { + return this.serviceDiscovery; + } + + /** + * Discover another service + */ + async discoverService(name: string) { + return this.serviceDiscovery?.discoverService(name) ?? null; + } + + /** + * Check if the KV store is healthy + */ + async isHealthy(): Promise { + if (!this.kvClient) return false; + return this.kvClient.health(); + } + + /** + * Get paths configuration + */ + getPaths() { + return { + metaCorePath: this.config.metaCorePath, + filesPath: this.config.filesPath + }; + } +} diff --git a/packages/meta-sort-core/src/kv/LeaderElection.ts b/packages/meta-sort-core/src/kv/LeaderElection.ts new file mode 100644 index 0000000..5d00a86 --- /dev/null +++ b/packages/meta-sort-core/src/kv/LeaderElection.ts @@ -0,0 +1,615 @@ +/** + * Leader Election using flock on shared filesystem + * + * This implements a distributed leader election mechanism using: + * 1. flock (advisory file locking) for mutual exclusion + * 2. Shared filesystem as the consensus layer + * 3. Separate info file for service discovery + * + * Architecture: + * - Service tries to acquire exclusive flock on /meta-core/locks/kv-leader.lock + * - If acquired → becomes LEADER, spawns Redis, writes endpoint to info file + * - If blocked → becomes FOLLOWER, reads leader endpoint from info file + * - flock automatically releases when process dies (no stale lock issues!) + * + * Key advantage over O_EXCL approach: + * - Lock is automatically released when process exits/crashes + * - No need for timestamp-based stale detection + * - No race conditions during takeover + */ + +import { promises as fs } from 'fs'; +import { dirname } from 'path'; +import { hostname, networkInterfaces } from 'os'; +import { spawn, ChildProcess, execSync } from 'child_process'; +import * as IORedis from 'ioredis'; +import type { LeaderLockInfo } from './IKVClient.js'; + +// ESM/CJS interop for ioredis +const Redis = (IORedis as any).default ?? IORedis; + +interface LeaderElectionConfig { + /** Path to META_CORE_VOLUME (e.g., /meta-core) */ + metaCorePath: string; + + /** Service name for identification */ + serviceName: string; + + /** HTTP API port */ + apiPort: number; + + /** Redis port (default: 6379) */ + redisPort?: number; + + /** Redis data directory (relative to META_CORE_VOLUME/db/redis) */ + redisDataDir?: string; + + /** Health check interval in ms (default: 5000) */ + healthCheckInterval?: number; + + /** Base URL for stable service discovery (e.g., http://localhost:8180) */ + baseUrl?: string; +} + +type LeaderRole = 'leader' | 'follower' | 'unknown'; + +export class LeaderElection { + private config: LeaderElectionConfig; + private lockFilePath: string; + private infoFilePath: string; + private flockProcess: ChildProcess | null = null; + private role: LeaderRole = 'unknown'; + private leaderInfo: LeaderLockInfo | null = null; + private redisProcess: ChildProcess | null = null; + private healthCheckTimer: NodeJS.Timeout | null = null; + private isShuttingDown = false; + + // Event callbacks + private onBecomeLeader?: () => void; + private onBecomeFollower?: (leaderInfo: LeaderLockInfo) => void; + private onLeaderLost?: () => void; + + constructor(config: LeaderElectionConfig) { + this.config = { + redisPort: 6379, + healthCheckInterval: 5000, + ...config + }; + + this.lockFilePath = `${this.config.metaCorePath}/locks/kv-leader.lock`; + this.infoFilePath = `${this.config.metaCorePath}/locks/kv-leader.info`; + } + + /** + * Get the local machine's IP address + */ + private getLocalIP(): string { + const interfaces = networkInterfaces(); + + // Prefer non-internal IPv4 addresses + for (const name of Object.keys(interfaces)) { + const addrs = interfaces[name]; + if (!addrs) continue; + + for (const addr of addrs) { + if (addr.family === 'IPv4' && !addr.internal) { + return addr.address; + } + } + } + + // Fallback to hostname or localhost + return hostname() || 'localhost'; + } + + /** + * Build leader info for this service + */ + private buildLeaderInfo(): LeaderLockInfo { + const ip = this.getLocalIP(); + return { + host: hostname(), + api: `redis://${ip}:${this.config.redisPort}`, + http: `http://${ip}:${this.config.apiPort}`, + baseUrl: this.config.baseUrl, + timestamp: Date.now(), + pid: process.pid + }; + } + + /** + * Ensure lock directory exists + */ + private async ensureLockDir(): Promise { + const lockDir = dirname(this.lockFilePath); + await fs.mkdir(lockDir, { recursive: true }); + } + + /** + * Try to acquire the leader lock using flock + * Returns true if lock acquired (we are leader), false otherwise + * + * Uses `flock -n -x` for non-blocking exclusive lock attempt. + * The lock is held by keeping a child process alive with the lock fd open. + */ + private async tryAcquireLock(): Promise { + await this.ensureLockDir(); + + // Ensure lock file exists (flock needs an existing file) + try { + await fs.writeFile(this.lockFilePath, '', { flag: 'a' }); + } catch { + // Ignore if file already exists + } + + return new Promise((resolve) => { + // Spawn a process that tries to acquire the lock and holds it + // Using flock with file descriptor: open fd 200 on lock file, then sleep forever + // The lock is held as long as fd 200 is open (process alive) + this.flockProcess = spawn('bash', [ + '-c', + `exec 200>"${this.lockFilePath}" && flock -n -x 200 && echo LOCKED && while true; do sleep 86400; done` + ], { + stdio: ['ignore', 'pipe', 'pipe'] + }); + + let resolved = false; + + // Check stdout for "LOCKED" message indicating we got the lock + this.flockProcess.stdout?.on('data', (data) => { + const output = data.toString().trim(); + if (output === 'LOCKED' && !resolved) { + resolved = true; + console.log('[LeaderElection] Acquired flock on', this.lockFilePath); + resolve(true); + } + }); + + this.flockProcess.stderr?.on('data', (data) => { + const msg = data.toString().trim(); + if (msg) { + console.error('[LeaderElection] flock stderr:', msg); + } + }); + + // If process exits quickly, we didn't get the lock + this.flockProcess.on('exit', (code) => { + if (!resolved) { + resolved = true; + console.log('[LeaderElection] Could not acquire flock (lock held by another process)'); + this.flockProcess = null; + resolve(false); + } else if (!this.isShuttingDown && this.role === 'leader') { + // Lock process died unexpectedly while we were leader + console.error('[LeaderElection] flock process died unexpectedly!'); + this.flockProcess = null; + this.handleLockLost(); + } + }); + + // Timeout: if we don't get the lock within 2 seconds, assume it's held + setTimeout(() => { + if (!resolved) { + resolved = true; + if (this.flockProcess) { + this.flockProcess.kill(); + this.flockProcess = null; + } + console.log('[LeaderElection] Timeout acquiring flock (lock held by another process)'); + resolve(false); + } + }, 2000); + }); + } + + /** + * Handle losing the lock unexpectedly + */ + private async handleLockLost(): Promise { + if (this.isShuttingDown) return; + + console.log('[LeaderElection] Lost leadership, cleaning up...'); + this.role = 'unknown'; + + // Stop Redis + await this.stopRedis(); + + // Notify callback + this.onLeaderLost?.(); + + // Try to re-elect + const acquired = await this.tryAcquireLock(); + if (acquired) { + await this.transitionToLeader(); + } else { + await this.transitionToFollower(); + } + } + + /** + * Check if the lock is currently held by someone + * Uses flock -n which fails immediately if lock is held + */ + private isLockHeld(): boolean { + try { + // Try to acquire lock non-blocking - if it succeeds, no one holds it + execSync(`flock -n -x "${this.lockFilePath}" -c "exit 0"`, { + timeout: 1000, + stdio: 'ignore' + }); + // Lock was free (we briefly held it and released) + return false; + } catch { + // Lock is held by someone else + return true; + } + } + + /** + * Read leader info from info file + */ + private async readLeaderInfo(): Promise { + try { + const content = await fs.readFile(this.infoFilePath, 'utf-8'); + return JSON.parse(content) as LeaderLockInfo; + } catch { + return null; + } + } + + /** + * Write leader info to info file (atomic write) + */ + private async writeLeaderInfo(): Promise { + this.leaderInfo = this.buildLeaderInfo(); + + // Atomic write: write to temp file, then rename + const tempPath = `${this.infoFilePath}.tmp`; + await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2)); + await fs.rename(tempPath, this.infoFilePath); + + console.log('[LeaderElection] Wrote leader info to', this.infoFilePath); + } + + /** + * Update leader timestamp (heartbeat) using atomic write + */ + private async updateLeaderTimestamp(): Promise { + if (this.role !== 'leader' || !this.leaderInfo) return; + + this.leaderInfo.timestamp = Date.now(); + try { + // Atomic write: write to temp file, then rename + const tempPath = `${this.infoFilePath}.tmp`; + await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2)); + await fs.rename(tempPath, this.infoFilePath); + } catch (error) { + console.error('[LeaderElection] Failed to update timestamp:', error); + } + } + + /** + * Spawn Redis server as child process + */ + private async spawnRedis(): Promise { + const redisDataDir = `${this.config.metaCorePath}/db/redis`; + + // Ensure Redis data directory exists + await fs.mkdir(redisDataDir, { recursive: true }); + + console.log(`[LeaderElection] Spawning Redis on port ${this.config.redisPort}...`); + + // Spawn Redis server + this.redisProcess = spawn('redis-server', [ + '--port', String(this.config.redisPort), + '--bind', '0.0.0.0', + '--dir', redisDataDir, + '--appendonly', 'yes', + '--appendfilename', 'appendonly.aof', + '--dbfilename', 'dump.rdb', + '--save', '60', '1', // Save after 60 seconds if at least 1 key changed + '--loglevel', 'warning' + ], { + stdio: ['ignore', 'pipe', 'pipe'], + detached: false + }); + + this.redisProcess.stdout?.on('data', (data) => { + console.log(`[Redis] ${data.toString().trim()}`); + }); + + this.redisProcess.stderr?.on('data', (data) => { + console.error(`[Redis] ${data.toString().trim()}`); + }); + + this.redisProcess.on('exit', (code, signal) => { + console.log(`[Redis] Process exited with code ${code}, signal ${signal}`); + this.redisProcess = null; + + if (!this.isShuttingDown && this.role === 'leader') { + // Redis crashed - try to restart + console.log('[LeaderElection] Redis crashed, attempting restart...'); + this.spawnRedis().catch(console.error); + } + }); + + // Wait for Redis to be ready + await this.waitForRedisReady(); + } + + /** + * Wait for Redis to be ready to accept connections + */ + private async waitForRedisReady(): Promise { + const maxAttempts = 30; + const ip = this.getLocalIP(); + + for (let i = 0; i < maxAttempts; i++) { + try { + const client = new Redis({ + host: ip, + port: this.config.redisPort, + connectTimeout: 1000, + maxRetriesPerRequest: 1 + }); + + const pong = await client.ping(); + await client.quit(); + + if (pong === 'PONG') { + console.log('[LeaderElection] Redis is ready'); + return; + } + } catch { + // Redis not ready yet + } + + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + throw new Error('Redis failed to start within timeout'); + } + + /** + * Stop Redis server + */ + private async stopRedis(): Promise { + if (this.redisProcess) { + console.log('[LeaderElection] Stopping Redis...'); + this.redisProcess.kill('SIGTERM'); + + // Wait for graceful shutdown + await new Promise((resolve) => { + const timeout = setTimeout(() => { + if (this.redisProcess) { + this.redisProcess.kill('SIGKILL'); + } + resolve(); + }, 5000); + + this.redisProcess!.on('exit', () => { + clearTimeout(timeout); + resolve(); + }); + }); + + this.redisProcess = null; + } + } + + /** + * Release the leader lock by killing the flock process + */ + private async releaseLock(): Promise { + if (this.flockProcess) { + console.log('[LeaderElection] Releasing flock...'); + + // Kill the flock holder process - this releases the lock + this.flockProcess.kill('SIGTERM'); + + // Wait briefly for graceful exit + await new Promise((resolve) => { + const timeout = setTimeout(() => { + if (this.flockProcess) { + this.flockProcess.kill('SIGKILL'); + } + resolve(); + }, 1000); + + if (this.flockProcess) { + this.flockProcess.on('exit', () => { + clearTimeout(timeout); + resolve(); + }); + } else { + clearTimeout(timeout); + resolve(); + } + }); + + this.flockProcess = null; + } + + // Clean up info file (optional, but good hygiene) + try { + await fs.unlink(this.infoFilePath); + } catch { + // Ignore if file doesn't exist + } + } + + /** + * Start health check loop + */ + private startHealthCheck(): void { + this.healthCheckTimer = setInterval(async () => { + if (this.isShuttingDown) return; + + try { + if (this.role === 'leader') { + // Update timestamp as heartbeat + await this.updateLeaderTimestamp(); + + // Verify we still hold the lock + if (!this.flockProcess || this.flockProcess.exitCode !== null) { + console.error('[LeaderElection] Lost flock process!'); + await this.handleLockLost(); + } + } else if (this.role === 'follower') { + // Check if lock is still held (leader still alive) + if (!this.isLockHeld()) { + console.log('[LeaderElection] Lock is free, attempting to become leader...'); + this.onLeaderLost?.(); + + // Try to become leader + const acquired = await this.tryAcquireLock(); + if (acquired) { + await this.transitionToLeader(); + } else { + // Another service became leader - reconnect + await this.transitionToFollower(); + } + } + } + } catch (error) { + console.error('[LeaderElection] Health check error:', error); + } + }, this.config.healthCheckInterval); + } + + /** + * Transition to leader role + */ + private async transitionToLeader(): Promise { + console.log('[LeaderElection] Becoming LEADER'); + this.role = 'leader'; + + // Write leader info BEFORE spawning Redis so followers can find us + await this.writeLeaderInfo(); + + // Spawn Redis + await this.spawnRedis(); + + // Notify callback + this.onBecomeLeader?.(); + } + + /** + * Transition to follower role + */ + private async transitionToFollower(): Promise { + console.log('[LeaderElection] Becoming FOLLOWER'); + this.role = 'follower'; + + // Wait a bit for leader to write info file + await new Promise(resolve => setTimeout(resolve, 500)); + + // Read leader info + this.leaderInfo = await this.readLeaderInfo(); + + if (this.leaderInfo) { + console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`); + this.onBecomeFollower?.(this.leaderInfo); + } else { + console.warn('[LeaderElection] Could not read leader info, will retry...'); + // Retry reading after a delay + setTimeout(async () => { + this.leaderInfo = await this.readLeaderInfo(); + if (this.leaderInfo) { + console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`); + this.onBecomeFollower?.(this.leaderInfo); + } + }, 2000); + } + } + + /** + * Start the leader election process + */ + async start(): Promise { + console.log(`[LeaderElection] Starting election for ${this.config.serviceName}...`); + console.log(`[LeaderElection] Lock file: ${this.lockFilePath}`); + console.log(`[LeaderElection] Info file: ${this.infoFilePath}`); + + // Try to become leader + const acquired = await this.tryAcquireLock(); + + if (acquired) { + await this.transitionToLeader(); + } else { + await this.transitionToFollower(); + } + + // Start health check + this.startHealthCheck(); + } + + /** + * Stop and cleanup + */ + async stop(): Promise { + console.log('[LeaderElection] Stopping...'); + this.isShuttingDown = true; + + // Stop health check + if (this.healthCheckTimer) { + clearInterval(this.healthCheckTimer); + this.healthCheckTimer = null; + } + + // If we were leader, cleanup + if (this.role === 'leader') { + await this.stopRedis(); + await this.releaseLock(); + } + + this.role = 'unknown'; + console.log('[LeaderElection] Stopped'); + } + + // ======================================================================== + // Event Registration + // ======================================================================== + + onLeader(callback: () => void): this { + this.onBecomeLeader = callback; + return this; + } + + onFollower(callback: (leaderInfo: LeaderLockInfo) => void): this { + this.onBecomeFollower = callback; + return this; + } + + onLostLeader(callback: () => void): this { + this.onLeaderLost = callback; + return this; + } + + // ======================================================================== + // Getters + // ======================================================================== + + getRole(): LeaderRole { + return this.role; + } + + isLeader(): boolean { + return this.role === 'leader'; + } + + isFollower(): boolean { + return this.role === 'follower'; + } + + getLeaderInfo(): LeaderLockInfo | null { + return this.leaderInfo; + } + + getRedisUrl(): string | null { + if (this.leaderInfo) { + return this.leaderInfo.api; + } + return null; + } +} diff --git a/packages/meta-sort-core/src/kv/MetadataUtils.ts b/packages/meta-sort-core/src/kv/MetadataUtils.ts new file mode 100644 index 0000000..24cf136 --- /dev/null +++ b/packages/meta-sort-core/src/kv/MetadataUtils.ts @@ -0,0 +1,274 @@ +/** + * Utility functions for converting between nested JSON metadata and flat key-value pairs. + * Implements the hierarchical nested key architecture for KV storage (Redis, etc.) + */ + +import type { KeyValuePair } from './IKVClient.js'; + +/** + * Flattens a nested metadata object into an array of key-value pairs. + * Each property becomes a separate key with a hierarchical path. + * + * @param metadata - The nested metadata object to flatten + * @param prefix - The key prefix (e.g., "/file/{hashId}") + * @param excludeFields - Fields to exclude from flattening (e.g., internal tracking fields) + * @returns Array of key-value pairs suitable for KV storage + * + * @example + * flattenMetadata({ title: "Inception", video: { codec: "h265" } }, "/file/abc123") + * // Returns: + * // [ + * // { key: "/file/abc123/title", value: "Inception" }, + * // { key: "/file/abc123/video/codec", value: "h265" } + * // ] + */ +export function flattenMetadata( + metadata: any, + prefix: string, + excludeFields: string[] = [] +): KeyValuePair[] { + const pairs: KeyValuePair[] = []; + + function flatten(obj: any, path: string) { + if (obj === null || obj === undefined) { + // Store null/undefined as empty string + pairs.push({ key: path, value: '' }); + return; + } + + if (typeof obj !== 'object') { + // Primitive value - store as string + pairs.push({ key: path, value: String(obj) }); + return; + } + + if (Array.isArray(obj)) { + // Handle arrays with numeric indices + obj.forEach((item, index) => { + flatten(item, `${path}/${index}`); + }); + return; + } + + // Handle objects recursively + Object.keys(obj).forEach((key) => { + // Skip excluded fields + if (excludeFields.includes(key)) { + return; + } + + const value = obj[key]; + const newPath = path ? `${path}/${key}` : key; + + flatten(value, newPath); + }); + } + + flatten(metadata, prefix); + return pairs; +} + +/** + * Reconstructs a nested metadata object from flat key-value pairs. + * Reverses the flattening process to restore the original structure. + * + * @param pairs - Array of key-value pairs from KV storage + * @param prefix - The key prefix to strip (e.g., "/file/{hashId}") + * @returns Reconstructed nested metadata object + * + * @example + * reconstructMetadata([ + * { key: "/file/abc123/title", value: "Inception" }, + * { key: "/file/abc123/video/codec", value: "h265" } + * ], "/file/abc123") + * // Returns: { title: "Inception", video: { codec: "h265" } } + */ +export function reconstructMetadata( + pairs: KeyValuePair[], + prefix: string +): any { + const result: any = {}; + + pairs.forEach(({ key, value }) => { + // Strip the prefix to get the property path + if (!key.startsWith(prefix)) { + return; // Skip keys that don't match the prefix + } + + const propertyPath = key.slice(prefix.length + 1); // +1 to skip the leading slash + if (!propertyPath) { + return; // Skip if no property path after prefix + } + + const parts = propertyPath.split('/'); + let current = result; + + // Navigate/create the nested structure + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const isLast = i === parts.length - 1; + + if (isLast) { + // Last part - set the value + current[part] = parseValue(value); + } else { + // Intermediate part - create object or array as needed + const nextPart = parts[i + 1]; + const isNextPartNumeric = /^\d+$/.test(nextPart); + + if (!(part in current)) { + // Create array if next part is numeric, otherwise create object + current[part] = isNextPartNumeric ? [] : {}; + } + + current = current[part]; + } + } + }); + + // Post-process: Convert objects with only numeric keys to arrays + return convertNumericObjectsToArrays(result); +} + +/** + * Recursively converts objects with only numeric sequential keys to arrays. + * This handles cases where arrays were stored with numeric indices. + */ +function convertNumericObjectsToArrays(obj: any): any { + if (obj === null || typeof obj !== 'object') { + return obj; + } + + if (Array.isArray(obj)) { + // Already an array, recursively process elements + return obj.map(convertNumericObjectsToArrays); + } + + const keys = Object.keys(obj); + + // Check if all keys are numeric and sequential (0, 1, 2, ...) + const allNumeric = keys.every(key => /^\d+$/.test(key)); + + if (allNumeric && keys.length > 0) { + // Sort keys numerically + const sortedKeys = keys.map(Number).sort((a, b) => a - b); + + // Check if keys are sequential starting from 0 + const isSequential = sortedKeys.every((key, index) => key === index); + + if (isSequential) { + // Convert to array + const arr = sortedKeys.map(index => + convertNumericObjectsToArrays(obj[index.toString()]) + ); + return arr; + } + } + + // Regular object - recursively process values + const result: any = {}; + for (const key of keys) { + result[key] = convertNumericObjectsToArrays(obj[key]); + } + return result; +} + +/** + * Parses a string value from KV storage into the appropriate JavaScript type. + * Handles numbers, booleans, null, and strings. + * + * @param value - The string value from KV storage + * @returns Parsed value with appropriate type + */ +function parseValue(value: string): any { + // Handle empty string as null + if (value === '') { + return null; + } + + // Handle booleans + if (value === 'true') return true; + if (value === 'false') return false; + + // Handle null + if (value === 'null') return null; + + // Try to parse as number, but be conservative + // Only parse as number if it's a valid integer or floating point without leading zeros + // This preserves strings like "5.1" (channel layout), "23.976" (frame rate) + // Note: In actual storage, these will be stored as strings, so this only matters + // when the original value was numeric. For metadata consistency, we convert + // what looks like numbers to numbers. + const num = Number(value); + if (!isNaN(num) && value.trim() !== '' && /^-?\d+(\.\d+)?$/.test(value)) { + return num; + } + + // Return as string + return value; +} + +/** + * Extracts the hash ID from a file key. + * + * @param key - Full key like "/file/{hashId}/property/path" + * @returns The hash ID or null if not a valid file key + * + * @example + * extractHashId("/file/sha256:abc123/title") // Returns "sha256:abc123" + */ +export function extractHashId(key: string): string | null { + const match = key.match(/^\/file\/([^/]+)/); + return match ? match[1] : null; +} + +/** + * Builds a property key path. + * + * @param hashId - The file's hash ID + * @param propertyPath - The property path (e.g., "video/codec") + * @returns Full KV key path + * + * @example + * buildPropertyKey("sha256:abc123", "video/codec") + * // Returns "/file/sha256:abc123/video/codec" + */ +export function buildPropertyKey(hashId: string, propertyPath: string): string { + return `/file/${hashId}/${propertyPath}`; +} + +/** + * Builds a file prefix for range queries. + * + * @param hashId - The file's hash ID + * @returns Prefix for querying all properties of a file + * + * @example + * buildFilePrefix("sha256:abc123") // Returns "/file/sha256:abc123" + */ +export function buildFilePrefix(hashId: string): string { + return `/file/${hashId}`; +} + +/** + * Groups key-value pairs by hash ID. + * Useful for batch processing multiple files' metadata. + * + * @param pairs - Array of key-value pairs + * @returns Map of hash ID to array of key-value pairs + */ +export function groupByHashId(pairs: KeyValuePair[]): Map { + const grouped = new Map(); + + pairs.forEach((pair) => { + const hashId = extractHashId(pair.key); + if (hashId) { + if (!grouped.has(hashId)) { + grouped.set(hashId, []); + } + grouped.get(hashId)!.push(pair); + } + }); + + return grouped; +} diff --git a/packages/meta-sort-core/src/kv/RedisClient.ts b/packages/meta-sort-core/src/kv/RedisClient.ts new file mode 100644 index 0000000..ba9108e --- /dev/null +++ b/packages/meta-sort-core/src/kv/RedisClient.ts @@ -0,0 +1,351 @@ +/** + * Redis KV Client - Implements IKVClient interface using Redis + * + * Uses Redis hashes for efficient nested key storage: + * - Each file's metadata is stored as a Redis hash + * - Hash key: file:{hashId} + * - Hash field: property/path (e.g., "video/codec") + * + * Benefits of Redis hashes: + * - Single command to get all file metadata (HGETALL) + * - Single command to set multiple properties (HMSET) + * - Atomic operations on file metadata + */ + +import * as IORedis from 'ioredis'; +import type { IKVClient, KeyValuePair } from './IKVClient.js'; +import { + flattenMetadata, + reconstructMetadata, + buildFilePrefix, + buildPropertyKey, +} from './MetadataUtils.js'; + +// ESM/CJS interop for ioredis +const Redis = (IORedis as any).default ?? IORedis; +type RedisInstance = InstanceType; + +interface RedisClientConfig { + url: string; + timeout?: number; + prefix?: string; +} + +export class RedisKVClient implements IKVClient { + private redis: RedisInstance; + private timeout: number; + private prefix: string; + private isConnected: boolean = false; + + constructor(config: RedisClientConfig) { + this.timeout = config.timeout || 30000; + this.prefix = config.prefix || ''; + + // Parse Redis URL and create client + this.redis = new Redis(config.url, { + commandTimeout: this.timeout, + connectTimeout: this.timeout, + lazyConnect: true, + maxRetriesPerRequest: 3, + enableOfflineQueue: true, + retryStrategy: (times) => { + const delay = Math.min(times * 200, 2000); + console.log(`[Redis] Retry attempt ${times}, waiting ${delay}ms`); + return delay; + } + }); + + // Event handlers + this.redis.on('connect', () => { + this.isConnected = true; + console.log('[Redis] Connected'); + }); + + this.redis.on('error', (err) => { + console.error('[Redis] Error:', err.message); + }); + + this.redis.on('close', () => { + this.isConnected = false; + console.log('[Redis] Connection closed'); + }); + } + + /** + * Connect to Redis (call before using other methods) + */ + async connect(): Promise { + if (this.isConnected) return; + await this.redis.connect(); + } + + /** + * Build the full Redis key with optional prefix + */ + private buildKey(key: string): string { + return this.prefix ? `${this.prefix}${key}` : key; + } + + // ======================================================================== + // Basic Operations + // ======================================================================== + + async set(key: string, value: any): Promise { + const fullKey = this.buildKey(key); + await this.redis.set(fullKey, JSON.stringify(value)); + } + + async get(key: string): Promise { + const fullKey = this.buildKey(key); + const value = await this.redis.get(fullKey); + if (value === null) return null; + try { + return JSON.parse(value); + } catch { + return value; + } + } + + async delete(key: string): Promise { + const fullKey = this.buildKey(key); + await this.redis.del(fullKey); + } + + async countKeysWithPrefix(prefix: string): Promise { + const fullPrefix = this.buildKey(prefix); + // Use SCAN to count keys (don't use KEYS in production) + let count = 0; + let cursor = '0'; + + do { + const [nextCursor, keys] = await this.redis.scan( + cursor, + 'MATCH', + `${fullPrefix}*`, + 'COUNT', + 1000 + ); + cursor = nextCursor; + count += keys.length; + } while (cursor !== '0'); + + return count; + } + + async health(): Promise { + try { + const result = await this.redis.ping(); + return result === 'PONG'; + } catch { + return false; + } + } + + // ======================================================================== + // Nested Key Architecture Methods + // ======================================================================== + + async setProperty(key: string, value: string): Promise { + const fullKey = this.buildKey(key); + await this.redis.set(fullKey, value); + } + + async getProperty(key: string): Promise { + const fullKey = this.buildKey(key); + return await this.redis.get(fullKey); + } + + async getRange(prefix: string): Promise { + const fullPrefix = this.buildKey(prefix); + const pairs: KeyValuePair[] = []; + let cursor = '0'; + + do { + const [nextCursor, keys] = await this.redis.scan( + cursor, + 'MATCH', + `${fullPrefix}*`, + 'COUNT', + 1000 + ); + cursor = nextCursor; + + if (keys.length > 0) { + // Get values for all found keys + const values = await this.redis.mget(...keys); + keys.forEach((key, index) => { + const value = values[index]; + if (value !== null) { + // Strip prefix to get original key + const originalKey = this.prefix + ? key.slice(this.prefix.length) + : key; + pairs.push({ key: originalKey, value }); + } + }); + } + } while (cursor !== '0'); + + return pairs; + } + + async setMultiple(pairs: KeyValuePair[]): Promise { + if (pairs.length === 0) return; + + // Use pipeline for efficient batch operations + const pipeline = this.redis.pipeline(); + + for (const pair of pairs) { + const fullKey = this.buildKey(pair.key); + pipeline.set(fullKey, pair.value); + } + + await pipeline.exec(); + } + + async deleteRange(prefix: string): Promise { + const fullPrefix = this.buildKey(prefix); + let deletedCount = 0; + let cursor = '0'; + + do { + const [nextCursor, keys] = await this.redis.scan( + cursor, + 'MATCH', + `${fullPrefix}*`, + 'COUNT', + 1000 + ); + cursor = nextCursor; + + if (keys.length > 0) { + const deleted = await this.redis.del(...keys); + deletedCount += deleted; + } + } while (cursor !== '0'); + + return deletedCount; + } + + // ======================================================================== + // High-Level Metadata Operations (Using Redis Hashes) + // ======================================================================== + + /** + * Store file metadata using Redis hash + * More efficient than individual keys - single HMSET command + */ + async setMetadataFlat( + hashId: string, + metadata: any, + excludeFields: string[] = [] + ): Promise { + const prefix = buildFilePrefix(hashId); + const pairs = flattenMetadata(metadata, prefix, excludeFields); + + if (pairs.length === 0) return; + + // Convert to Redis hash format + // Hash key: file:{hashId} + // Hash fields: property paths (without /file/{hashId} prefix) + const hashKey = this.buildKey(`file:${hashId}`); + const hashData: Record = {}; + + for (const pair of pairs) { + // Extract field name by removing the prefix + const field = pair.key.slice(prefix.length + 1); // +1 for trailing / + if (field) { + hashData[field] = pair.value; + } + } + + if (Object.keys(hashData).length > 0) { + await this.redis.hmset(hashKey, hashData); + } + + // Also store in key index for getAllHashIds + await this.redis.sadd(this.buildKey('file:__index__'), hashId); + } + + async getMetadataFlat(hashId: string): Promise { + const hashKey = this.buildKey(`file:${hashId}`); + const hashData = await this.redis.hgetall(hashKey); + + if (!hashData || Object.keys(hashData).length === 0) { + return null; + } + + // Convert Redis hash back to KeyValuePair format for reconstruction + const prefix = buildFilePrefix(hashId); + const pairs: KeyValuePair[] = Object.entries(hashData).map(([field, value]) => ({ + key: `${prefix}/${field}`, + value: value as string + })); + + return reconstructMetadata(pairs, prefix); + } + + async getMetadata(hashId: string, propertyPath: string): Promise { + const hashKey = this.buildKey(`file:${hashId}`); + const value = await this.redis.hget(hashKey, propertyPath); + + if (value === null) return null; + + try { + return JSON.parse(value); + } catch { + return value; + } + } + + async deleteMetadataFlat(hashId: string): Promise { + const hashKey = this.buildKey(`file:${hashId}`); + const fieldCount = await this.redis.hlen(hashKey); + + await this.redis.del(hashKey); + await this.redis.srem(this.buildKey('file:__index__'), hashId); + + return fieldCount; + } + + async getAllHashIds(): Promise { + const indexKey = this.buildKey('file:__index__'); + return await this.redis.smembers(indexKey); + } + + // ======================================================================== + // Pub/Sub Operations + // ======================================================================== + + /** + * Publish a message to a Redis channel + * Used for notifying other services (meta-fuse) about file changes + * Note: Channel names are NOT prefixed - they use their own namespace + */ + async publish(channel: string, message: string): Promise { + return await this.redis.publish(channel, message); + } + + // ======================================================================== + // Lifecycle + // ======================================================================== + + async close(): Promise { + await this.redis.quit(); + this.isConnected = false; + } + + /** + * Check if client is connected + */ + isHealthy(): boolean { + return this.isConnected; + } + + /** + * Get the underlying Redis client (for advanced operations) + */ + getRedisClient(): RedisInstance { + return this.redis; + } +} diff --git a/packages/meta-sort-core/src/kv/ServiceDiscovery.ts b/packages/meta-sort-core/src/kv/ServiceDiscovery.ts new file mode 100644 index 0000000..cef4ded --- /dev/null +++ b/packages/meta-sort-core/src/kv/ServiceDiscovery.ts @@ -0,0 +1,312 @@ +/** + * Service Discovery using shared filesystem + * + * Each service registers itself in /meta-core/services/{service-name}.json + * with its API endpoint, status, and heartbeat timestamp. + * + * Services can discover each other by reading these JSON files. + * Stale detection: if lastHeartbeat > 60 seconds ago, service is considered dead. + */ + +import { promises as fs } from 'fs'; +import { hostname } from 'os'; +import { join } from 'path'; +import type { ServiceInfo } from './IKVClient.js'; + +interface ServiceDiscoveryConfig { + /** Path to META_CORE_VOLUME (e.g., /meta-core) */ + metaCorePath: string; + + /** Service name */ + serviceName: string; + + /** Service version */ + version: string; + + /** HTTP API URL */ + apiUrl: string; + + /** Service capabilities */ + capabilities?: string[]; + + /** Named endpoints */ + endpoints?: Record; + + /** Heartbeat interval in ms (default: 30000) */ + heartbeatInterval?: number; + + /** Stale threshold in ms (default: 60000) */ + staleThreshold?: number; +} + +export class ServiceDiscovery { + private config: ServiceDiscoveryConfig; + private servicesDir: string; + private serviceFilePath: string; + private heartbeatTimer: NodeJS.Timeout | null = null; + private isShuttingDown = false; + + constructor(config: ServiceDiscoveryConfig) { + this.config = { + heartbeatInterval: 30000, + staleThreshold: 60000, + capabilities: [], + endpoints: {}, + ...config + }; + + this.servicesDir = `${this.config.metaCorePath}/services`; + this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}.json`; + } + + /** + * Ensure services directory exists + */ + private async ensureServicesDir(): Promise { + await fs.mkdir(this.servicesDir, { recursive: true }); + } + + /** + * Build service info for this service + */ + private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo { + return { + name: this.config.serviceName, + version: this.config.version, + api: this.config.apiUrl, + status, + pid: process.pid, + hostname: hostname(), + startedAt: new Date().toISOString(), + lastHeartbeat: new Date().toISOString(), + capabilities: this.config.capabilities || [], + endpoints: this.config.endpoints || {} + }; + } + + /** + * Register this service + */ + async register(): Promise { + await this.ensureServicesDir(); + + const info = this.buildServiceInfo('starting'); + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + + console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}`); + } + + /** + * Update service status + */ + async updateStatus(status: ServiceInfo['status']): Promise { + try { + const content = await fs.readFile(this.serviceFilePath, 'utf-8'); + const info = JSON.parse(content) as ServiceInfo; + + info.status = status; + info.lastHeartbeat = new Date().toISOString(); + + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + } catch (error) { + console.error('[ServiceDiscovery] Failed to update status:', error); + } + } + + /** + * Send heartbeat (update lastHeartbeat timestamp) + */ + async heartbeat(): Promise { + try { + const content = await fs.readFile(this.serviceFilePath, 'utf-8'); + const info = JSON.parse(content) as ServiceInfo; + + info.lastHeartbeat = new Date().toISOString(); + + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + } catch (error) { + // If file was deleted, re-register + if ((error as any).code === 'ENOENT') { + await this.register(); + await this.updateStatus('running'); + } else { + console.error('[ServiceDiscovery] Heartbeat failed:', error); + } + } + } + + /** + * Start heartbeat loop + */ + startHeartbeat(): void { + this.heartbeatTimer = setInterval(async () => { + if (this.isShuttingDown) return; + await this.heartbeat(); + }, this.config.heartbeatInterval); + } + + /** + * Stop heartbeat loop + */ + stopHeartbeat(): void { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + } + + /** + * Unregister this service (on shutdown) + */ + async unregister(): Promise { + this.isShuttingDown = true; + this.stopHeartbeat(); + + try { + // Update status to stopped before removing + await this.updateStatus('stopped'); + + // Optionally remove the file (or keep for debugging) + // await fs.unlink(this.serviceFilePath); + + console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}`); + } catch (error) { + console.error('[ServiceDiscovery] Failed to unregister:', error); + } + } + + // ======================================================================== + // Discovery Methods + // ======================================================================== + + /** + * Discover a service by name + */ + async discoverService(name: string): Promise { + const servicePath = join(this.servicesDir, `${name}.json`); + + try { + const content = await fs.readFile(servicePath, 'utf-8'); + const service = JSON.parse(content) as ServiceInfo; + + // Check if service is alive (heartbeat within threshold) + const lastHeartbeat = new Date(service.lastHeartbeat).getTime(); + const isAlive = Date.now() - lastHeartbeat < this.config.staleThreshold!; + + if (!isAlive) { + console.warn(`[ServiceDiscovery] Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`); + return null; + } + + return service; + } catch (error) { + if ((error as any).code === 'ENOENT') { + return null; // Service not registered + } + throw error; + } + } + + /** + * Discover all registered services + */ + async discoverAllServices(): Promise { + try { + const files = await fs.readdir(this.servicesDir); + const services: ServiceInfo[] = []; + + for (const file of files) { + if (!file.endsWith('.json')) continue; + + const name = file.replace('.json', ''); + const service = await this.discoverService(name); + + if (service && service.status === 'running') { + services.push(service); + } + } + + return services; + } catch (error) { + if ((error as any).code === 'ENOENT') { + return []; // Services directory doesn't exist yet + } + throw error; + } + } + + /** + * Wait for a service to become available + */ + async waitForService( + name: string, + timeoutMs: number = 30000, + pollInterval: number = 1000 + ): Promise { + const startTime = Date.now(); + + while (Date.now() - startTime < timeoutMs) { + const service = await this.discoverService(name); + if (service && service.status === 'running') { + return service; + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + + throw new Error(`Service ${name} not available after ${timeoutMs}ms`); + } + + /** + * Check if a service is healthy + */ + async isServiceHealthy(name: string): Promise { + const service = await this.discoverService(name); + if (!service) return false; + + // Try to ping the service's health endpoint + const healthEndpoint = service.endpoints.health || '/health'; + const url = `${service.api}${healthEndpoint}`; + + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(5000) + }); + return response.ok; + } catch { + return false; + } + } + + // ======================================================================== + // Lifecycle + // ======================================================================== + + /** + * Full startup sequence + */ + async start(): Promise { + await this.register(); + await this.updateStatus('running'); + this.startHeartbeat(); + } + + /** + * Full shutdown sequence + */ + async stop(): Promise { + await this.unregister(); + } + + // ======================================================================== + // Getters + // ======================================================================== + + getServiceFilePath(): string { + return this.serviceFilePath; + } + + getServicesDir(): string { + return this.servicesDir; + } +} diff --git a/packages/meta-sort-core/src/kv/index.ts b/packages/meta-sort-core/src/kv/index.ts new file mode 100644 index 0000000..71ba5ab --- /dev/null +++ b/packages/meta-sort-core/src/kv/index.ts @@ -0,0 +1,18 @@ +/** + * KV Module - Unified key-value storage with leader election + * + * This module provides: + * - IKVClient interface for KV storage abstraction + * - RedisKVClient for Redis-based storage + * - LeaderElection for distributed leader election via flock + * - ServiceDiscovery for service registration and discovery + * - KVManager for unified management of all components + * - MetadataUtils for flattening/reconstructing nested metadata + */ + +export * from './IKVClient.js'; +export * from './RedisClient.js'; +export * from './LeaderElection.js'; +export * from './ServiceDiscovery.js'; +export * from './KVManager.js'; +export * from './MetadataUtils.js'; diff --git a/packages/meta-sort-core/src/logic/DuplicateFinder.ts b/packages/meta-sort-core/src/logic/DuplicateFinder.ts new file mode 100644 index 0000000..d92b450 --- /dev/null +++ b/packages/meta-sort-core/src/logic/DuplicateFinder.ts @@ -0,0 +1,113 @@ +import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash"; +import {VideoMeta, HashMeta, VersionMeta, FileNameMeta, FileStatMeta} from "@metazla/meta-interface"; +import {renamingRule} from "../config/RenamingRule.js"; + +type Local = VideoMeta & HashMeta & VersionMeta & FileNameMeta & FileStatMeta; + +export interface DuplicateGroup { + key: string; + files: string[]; + virtualPath?: string; +} + +export interface DuplicateResult { + hashDuplicates: DuplicateGroup[]; + titleDuplicates: DuplicateGroup[]; +} + +export class DuplicateFinder { + async findDuplicates(database: Map): Promise { + let [hashDuplicates, titleDuplicates] = await Promise.all([ + this.findDuplicatesGeneric(database, this.hashKeyExtractor, 'remove'), + this.findDuplicatesGeneric(database, this.pathExtractor, 'version') + ]); + + return { + hashDuplicates, + titleDuplicates + }; + } + + private hashKeyExtractor(metadata: Local): string | null { + return metadata[CID_ALGORITHM_NAMES.sha256] ?? null; + } + + private pathExtractor(metadata: Local): string | null { + //case insensitive + try { + return renamingRule(metadata, '')?.toLowerCase() ?? null; + } catch (e) { + return null; + } + } + + private async findDuplicatesGeneric( + database: Map, + keyExtractor: (metadata: Local) => string | null, + action?: 'remove' | 'version' + ): Promise { + const duplicates = new Map(); + const duplicateGroups: DuplicateGroup[] = []; + + for (const [filePath, metadata] of database) { + const key = keyExtractor(metadata); + if (key) { + const files = duplicates.get(key) ?? []; + files.push(filePath); + duplicates.set(key, files); + } + } + + for (const [key, files] of duplicates) { + if (files.length > 1) { + // Store duplicate group before modifying database + const sortedFiles = [...files].sort(); + + // Generate virtualPath for both hash and title duplicates + let virtualPath: string | undefined; + if (action === 'version') { + // For title duplicates, use the key (normalized title path) + virtualPath = key; + } else if (action === 'remove') { + // For hash duplicates, use renamingRule on the first file's metadata + const firstFile = sortedFiles[0]; + const metadata = database.get(firstFile); + if (metadata) { + try { + virtualPath = renamingRule(metadata, ''); + } catch (e) { + // If renamingRule fails, virtualPath remains undefined + } + } + } + + duplicateGroups.push({ + key, + files: sortedFiles, + virtualPath + }); + + //alphabetical order + const remainingFiles = files.slice(1).sort(); + //rename all problematic entries V2, V3, V4, V5 + let i = 2; + for (const file of remainingFiles) { + if (action === 'remove') { + database.delete(file); + } else if (action === 'version') { + let definition = database.get(file); + definition.version = "V" + i; + if (definition.fileType !== "subtitle") {//allow multiple version of subtitle in the same folder + definition.extra = "true";//mark as extra so it don't pollute the main list + } + i++; + } else { + throw new Error(); + } + } + } + } + + return duplicateGroups; + } +} diff --git a/packages/meta-sort-core/src/logic/HashProcessingStateManager.ts b/packages/meta-sort-core/src/logic/HashProcessingStateManager.ts new file mode 100644 index 0000000..8e10247 --- /dev/null +++ b/packages/meta-sort-core/src/logic/HashProcessingStateManager.ts @@ -0,0 +1,143 @@ +/** + * HashProcessingStateManager + * + * Tracks the state of background hash computation operations: + * - Pending: Files queued for hash computation + * - Computing: Files currently having hashes computed + * - Complete: Recently completed hash computations + */ + +export type HashStatus = 'pending' | 'computing' | 'complete' | 'error'; + +export interface HashProcessingState { + filePath: string; + tempId?: string; + hashStatus: HashStatus; + queuedAt?: number; + startedAt?: number; + completedAt?: number; + processingTime?: number; + hash?: string; + error?: string; + virtualPath?: string; +} + +export interface HashProcessingStatusSnapshot { + pending: HashProcessingState[]; + computing: HashProcessingState[]; + complete: HashProcessingState[]; + totalPending: number; + totalComputing: number; + totalComplete: number; +} + +export class HashProcessingStateManager { + private pending: Map = new Map(); + private computing: Map = new Map(); + private complete: HashProcessingState[] = []; + private maxCompleteHistory = 100; // Keep last 100 completed hashes + + /** + * Add a file to the pending hash queue + */ + addPending(filePath: string, tempId?: string, virtualPath?: string): void { + this.pending.set(filePath, { + filePath, + tempId, + virtualPath, + hashStatus: 'pending', + queuedAt: Date.now() + }); + } + + /** + * Move a file from pending to computing + */ + startComputing(filePath: string): void { + const pendingState = this.pending.get(filePath); + this.pending.delete(filePath); + + this.computing.set(filePath, { + filePath, + tempId: pendingState?.tempId, + virtualPath: pendingState?.virtualPath, + hashStatus: 'computing', + queuedAt: pendingState?.queuedAt, + startedAt: Date.now() + }); + } + + /** + * Move a file from computing to complete + */ + completeComputing( + filePath: string, + processingTime: number, + hash?: string, + error?: string, + virtualPath?: string + ): void { + const computingState = this.computing.get(filePath); + this.computing.delete(filePath); + + const completedState: HashProcessingState = { + filePath, + tempId: computingState?.tempId, + virtualPath: virtualPath || computingState?.virtualPath, + hashStatus: error ? 'error' : 'complete', + queuedAt: computingState?.queuedAt, + startedAt: computingState?.startedAt, + completedAt: Date.now(), + processingTime, + hash, + error + }; + + // Add to complete list (most recent first) + this.complete.unshift(completedState); + + // Trim history if needed + if (this.complete.length > this.maxCompleteHistory) { + this.complete = this.complete.slice(0, this.maxCompleteHistory); + } + } + + /** + * Remove a file from all states (e.g., when deleted) + */ + removeFile(filePath: string): void { + this.pending.delete(filePath); + this.computing.delete(filePath); + this.complete = this.complete.filter(f => f.filePath !== filePath); + } + + /** + * Get a snapshot of the current hash processing state + */ + getSnapshot(): HashProcessingStatusSnapshot { + return { + pending: Array.from(this.pending.values()), + computing: Array.from(this.computing.values()), + complete: this.complete.slice(0, 50), // Return only last 50 for API + totalPending: this.pending.size, + totalComputing: this.computing.size, + totalComplete: this.complete.length + }; + } + + /** + * Clear all states (useful for testing or reset) + */ + clear(): void { + this.pending.clear(); + this.computing.clear(); + this.complete = []; + } + + /** + * Get the total number of files in all states + */ + getTotalFileCount(): number { + return this.pending.size + this.computing.size + this.complete.length; + } +} diff --git a/packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts b/packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts new file mode 100644 index 0000000..dd1d7b3 --- /dev/null +++ b/packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts @@ -0,0 +1,75 @@ +import * as path from 'path'; +import { + HashMeta, +} from "@metazla/meta-interface"; + +type Local = HashMeta; + +export class MetaDataToFolderStruct { + + public renamingRule: (metamesh: Local, filepath: string) => string | null; + + constructor(renamingRule: (metamesh: Local, filepath: string) => string | null) { + this.renamingRule = renamingRule; + } + + /** + * Compute virtual filesystem structure for FUSE/WebDAV + * Generates organized paths based on metadata without creating physical files + * @param data Map of file paths to metadata + * @private + */ + private computeVirtualStructure(data: Map): Map { + const virtualMap = new Map(); + + for (const [filepath, metamesh] of data) { + // Compute the virtual path for the file + let virtualPath: string; + try { + virtualPath = this.renamingRule(metamesh, filepath); + }catch (e){ + console.warn(e.message || e); + } + if (!virtualPath) { + continue; + } + virtualPath = this.sanitizePath(virtualPath); + virtualMap.set(filepath, virtualPath); + } + + console.log(`Computed virtual structure for ${virtualMap.size} files`); + return virtualMap; + } + + sanitizePath(newPath: string): string { + // First, isolate any potential Windows drive letter at the start of the path + const driveMatch = newPath.match(/^[a-zA-Z]:/); + const drive = driveMatch ? driveMatch[0] : ''; + + // Define illegal characters for Windows and potentially problematic ones for Unix + const illegalChars = /[<>:"|?*]/g; // Null character added for Unix-like systems + + // Replace illegal characters with an underscore in the path, excluding the drive part + const pathWithoutDrive = newPath.slice(drive.length); + const sanitizedPath = pathWithoutDrive.replace(illegalChars, ''); + + // Combine the drive (if any) and the sanitized path + const cleanPath = drive + sanitizedPath; + + // Normalize the clean path to handle slashes appropriately + return path.normalize(cleanPath); + } + + /** + * Generate virtual filesystem structure for FUSE/WebDAV + * Returns only the computed virtual paths without checking physical filesystem + * @param data Map of file paths to metadata + */ + public generateVirtualStructure(data: Map): Map { + const start = performance.now(); + const computed = this.computeVirtualStructure(data); + console.log(`Compute virtual structure took ${performance.now() - start}ms`); + return computed; + } + +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/MetaMeshFormat.ts b/packages/meta-sort-core/src/logic/MetaMeshFormat.ts new file mode 100644 index 0000000..446a17e --- /dev/null +++ b/packages/meta-sort-core/src/logic/MetaMeshFormat.ts @@ -0,0 +1,95 @@ +import { promises as fs } from 'fs'; +import yaml from 'yaml'; +const { parse, stringify } = yaml; +/** + * File type: + * filename..meta + * filename.meta (contains multiple metadata entry "mkv", "str" etc...) + * folder.meta + */ + +export class MetaMeshFormat { + async createOrUpdate(metaDataFilePath: string, data: T): Promise { + if(!metaDataFilePath.endsWith('.meta')){ + throw new Error(`File path must end with '.meta'`); + } + try { + // Check if the file exists + const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false); + + if (fileExists) { + // If the file exists, update it + await this.update(metaDataFilePath, data); + } else { + // If the file does not exist, create it + await fs.writeFile(metaDataFilePath, stringify(data), 'utf8'); + } + } catch (error) { + console.error(`Error in createOrUpdate: ${error}`); + throw error; + } + } + + async read(metaDataFilePath: string): Promise { + if(!metaDataFilePath.endsWith('.meta')){ + throw new Error(`File path must end with '.meta'`); + } + try { + const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false); + + if (!fileExists) { + return null; + } + // Read the YAML file + const fileContent = await fs.readFile(metaDataFilePath, 'utf8'); + + // Parse the YAML content to a JavaScript object + return parse(fileContent) as T; + } catch (error) { + console.error(`Error in read: ${error}`); + throw error; + } + } + + async update(metaDataFilePath: string, data: T): Promise { + if(!metaDataFilePath.endsWith('.meta')){ + throw new Error(`File path must end with '.meta'`); + } + try { + // Read the existing file content + const existingData = await this.read(metaDataFilePath) || {}; + + // Merge the existing data with the new data + const updatedData = { ...existingData, ...data }; + + // Write the updated data back to the file + await fs.writeFile(metaDataFilePath, stringify(updatedData), 'utf8'); + } catch (error) { + console.error(`Error in update: ${error}`); + throw error; + } + } + + async write(metaDataFilePath: string, data: T): Promise { + if(!metaDataFilePath.endsWith('.meta')){ + throw new Error(`File path must end with '.meta'`); + } + try { + // Write the updated data back to the file + await fs.writeFile(metaDataFilePath, stringify(data), { + encoding:'utf8', + flag:'w' // Write mode : Overwrite + }); + } catch (error) { + console.error(`Error in update: ${error}`); + throw error; + } + } + + /** + * Serialize data to YAML string (for virtual .meta files) + */ + serialize(data: T): string { + return stringify(data); + } +} diff --git a/packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts b/packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts new file mode 100644 index 0000000..fe3af31 --- /dev/null +++ b/packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts @@ -0,0 +1,217 @@ +import yaml from 'yaml'; +const { stringify: yamlStringify } = yaml; +import { Builder } from 'xml2js'; + +/** + * Supported metadata format types + */ +export enum MetadataFormatType { + META = 'meta', + JELLYFIN = 'jellyfin' +} + +/** + * Interface for metadata format handlers + * Note: metadata parameter uses `any` type because it combines multiple metadata interfaces + * (HashMeta, JellyfinMeta, VideoMeta, etc.) and TypeScript doesn't have a single type for this union + */ +export interface IMetadataFormatHandler { + /** + * The file extension for this format (e.g., '.meta', '.nfo') + */ + readonly extension: string; + + /** + * The format type identifier + */ + readonly formatType: MetadataFormatType; + + /** + * Serialize metadata to string format + * @param metadata The metadata to serialize (combination of various metadata interfaces) + * @returns Serialized string content + */ + serialize(metadata: any): string; +} + +/** + * Handler for .meta YAML format + */ +export class MetaFormatHandler implements IMetadataFormatHandler { + readonly extension = '.meta'; + readonly formatType = MetadataFormatType.META; + + serialize(metadata: any): string { + return yamlStringify(metadata); + } +} + +/** + * Handler for Jellyfin .nfo XML format + */ +export class JellyfinFormatHandler implements IMetadataFormatHandler { + readonly extension = '.nfo'; + readonly formatType = MetadataFormatType.JELLYFIN; + + serialize(metadata: any): string { + // Only include Jellyfin-compatible fields + const jellyfinData = this.filterJellyfinFields(metadata); + + // Determine root element based on video type + const rootElementName = this.isMovieOrTvShow(jellyfinData) === "tvshow" ? 'episodedetails' : "movie"; + + // Convert to XML-friendly format + const xmlData = this.convertToJsonXml(jellyfinData); + + // Build XML + const obj = { + [rootElementName]: xmlData + }; + + const builder = new Builder({ + xmldec: { version: '1.0', encoding: 'UTF-8', standalone: true } + }); + + return builder.buildObject(obj); + } + + private filterJellyfinFields(metadata: any): any { + const jellyfinFields: any = {}; + + // Video metadata + if (metadata.videoType) jellyfinFields.videoType = metadata.videoType; + if (metadata.season) jellyfinFields.season = metadata.season; + if (metadata.episode) jellyfinFields.episode = metadata.episode; + if (metadata.originalTitle) jellyfinFields.originalTitle = metadata.originalTitle; + if (metadata.movieYear) jellyfinFields.year = metadata.movieYear; + + // Titles (first title or original title) + if (metadata.titles) { + const titleEntries = Object.entries(metadata.titles); + if (titleEntries.length > 0) { + jellyfinFields.title = titleEntries[0][1]; + } + } + + // IDs + if (metadata.imdbid) jellyfinFields.imdbid = metadata.imdbid; + if (metadata.tmdbid) jellyfinFields.tmdbid = metadata.tmdbid; + if (metadata.anidbid) jellyfinFields.anidbid = metadata.anidbid; + + // Additional fields + if (metadata.rating) jellyfinFields.rating = metadata.rating; + if (metadata.plot) jellyfinFields.plot = metadata.plot; + if (metadata.studio) jellyfinFields.studio = metadata.studio; + if (metadata.releasedate) jellyfinFields.releasedate = metadata.releasedate; + if (metadata.mpaa) jellyfinFields.mpaa = metadata.mpaa; + + // Sets (genres, tags, languages) + if (metadata.genres) jellyfinFields.genre = metadata.genres; + if (metadata.tags) jellyfinFields.tag = metadata.tags; + if (metadata.languages) jellyfinFields.language = metadata.languages; + + // Art + if (metadata.art) { + if (metadata.art.poster) jellyfinFields.poster = metadata.art.poster; + if (metadata.art.fanart) jellyfinFields.fanart = metadata.art.fanart; + } + + return jellyfinFields; + } + + private isMovieOrTvShow(metadata: any): "tvshow" | "movie" { + if (metadata.videoType) { + return metadata.videoType; + } else if (metadata.episode) { + return "tvshow"; + } else { + return "movie"; + } + } + + private convertToJsonXml(input: any): any { + const output: any = {}; + + for (const [key, value] of Object.entries(input)) { + if (value instanceof Set) { + // Convert Set to array of items for XML + const array = Array.from(value as any); + if (array.length > 0) { + output[key] = array; + } + } else if (value instanceof Array) { + output[key] = value; + } else if (typeof value === 'object' && value !== null) { + // For nested objects, recursively convert + output[key] = [this.convertToJsonXml(value)]; + } else { + output[key] = [value]; + } + } + + return output; + } +} + +/** + * Registry for metadata format handlers + */ +export class MetadataFormatRegistry { + private handlers: Map = new Map(); + + constructor() { + // Register default handlers + this.registerHandler(new MetaFormatHandler()); + this.registerHandler(new JellyfinFormatHandler()); + } + + /** + * Register a new format handler + */ + registerHandler(handler: IMetadataFormatHandler): void { + this.handlers.set(handler.formatType, handler); + } + + /** + * Get handler by format type + */ + getHandler(formatType: MetadataFormatType): IMetadataFormatHandler | undefined { + return this.handlers.get(formatType); + } + + /** + * Get handler by format name (case-insensitive) + */ + getHandlerByName(formatName: string): IMetadataFormatHandler | undefined { + const normalizedName = formatName.toLowerCase(); + for (const [type, handler] of this.handlers.entries()) { + if (type.toLowerCase() === normalizedName) { + return handler; + } + } + return undefined; + } + + /** + * Get all registered handlers + */ + getAllHandlers(): IMetadataFormatHandler[] { + return Array.from(this.handlers.values()); + } + + /** + * Get handlers for specified format names + */ + getHandlersForFormats(formatNames: string[]): IMetadataFormatHandler[] { + const handlers: IMetadataFormatHandler[] = []; + for (const name of formatNames) { + const handler = this.getHandlerByName(name); + if (handler) { + handlers.push(handler); + } else { + console.warn(`Unknown metadata format: ${name}. Supported formats: ${Array.from(this.handlers.keys()).join(', ')}`); + } + } + return handlers; + } +} diff --git a/packages/meta-sort-core/src/logic/ProcessingStateManager.ts b/packages/meta-sort-core/src/logic/ProcessingStateManager.ts new file mode 100644 index 0000000..4a693a6 --- /dev/null +++ b/packages/meta-sort-core/src/logic/ProcessingStateManager.ts @@ -0,0 +1,127 @@ +/** + * ProcessingStateManager + * + * Tracks the state of file processing operations: + * - Pending: Files queued for processing + * - Processing: Files currently being processed + * - Processed: Recently completed files + */ + +export interface FileProcessingState { + filePath: string; + state: 'pending' | 'processing' | 'processed'; + queuedAt?: number; + startedAt?: number; + completedAt?: number; + processingTime?: number; + error?: string; + virtualPath?: string; +} + +export interface ProcessingStatusSnapshot { + pending: FileProcessingState[]; + processing: FileProcessingState[]; + processed: FileProcessingState[]; + totalPending: number; + totalProcessing: number; + totalProcessed: number; +} + +export class ProcessingStateManager { + private pending: Map = new Map(); + private processing: Map = new Map(); + private processed: FileProcessingState[] = []; + private maxProcessedHistory = 100; // Keep last 100 processed files + + /** + * Add a file to the pending queue + */ + addPending(filePath: string): void { + this.pending.set(filePath, { + filePath, + state: 'pending', + queuedAt: Date.now() + }); + } + + /** + * Move a file from pending to processing + */ + startProcessing(filePath: string): void { + const pendingState = this.pending.get(filePath); + this.pending.delete(filePath); + + this.processing.set(filePath, { + filePath, + state: 'processing', + queuedAt: pendingState?.queuedAt, + startedAt: Date.now() + }); + } + + /** + * Move a file from processing to processed + */ + completeProcessing(filePath: string, processingTime: number, error?: string, virtualPath?: string): void { + const processingState = this.processing.get(filePath); + this.processing.delete(filePath); + + const completedState: FileProcessingState = { + filePath, + state: 'processed', + queuedAt: processingState?.queuedAt, + startedAt: processingState?.startedAt, + completedAt: Date.now(), + processingTime, + error, + virtualPath + }; + + // Add to processed list (most recent first) + this.processed.unshift(completedState); + + // Trim history if needed + if (this.processed.length > this.maxProcessedHistory) { + this.processed = this.processed.slice(0, this.maxProcessedHistory); + } + } + + /** + * Remove a file from all states (e.g., when deleted) + */ + removeFile(filePath: string): void { + this.pending.delete(filePath); + this.processing.delete(filePath); + this.processed = this.processed.filter(f => f.filePath !== filePath); + } + + /** + * Get a snapshot of the current processing state + */ + getSnapshot(): ProcessingStatusSnapshot { + return { + pending: Array.from(this.pending.values()), + processing: Array.from(this.processing.values()), + processed: this.processed.slice(0, 50), // Return only last 50 for API + totalPending: this.pending.size, + totalProcessing: this.processing.size, + totalProcessed: this.processed.length + }; + } + + /** + * Clear all states (useful for testing or reset) + */ + clear(): void { + this.pending.clear(); + this.processing.clear(); + this.processed = []; + } + + /** + * Get the total number of files in all states + */ + getTotalFileCount(): number { + return this.pending.size + this.processing.size + this.processed.length; + } +} diff --git a/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts b/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts new file mode 100644 index 0000000..d4d7a4a --- /dev/null +++ b/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts @@ -0,0 +1,354 @@ +/** + * UnifiedProcessingStateManager + * + * Tracks files through the complete processing pipeline with 4 states: + * 1. Pending: Files discovered in filesystem (not yet processed) + * 2. Light Processing: Quick metadata extraction + midhash256 computation (~20ms) + * 3. Hash Processing: SHA-256 computation for compatibility (slow, background) + * 4. Done: Fully processed files + * + * Key change from old system: + * - midhash256 (fast) is computed in lightProcessing and becomes the permanent file ID + * - SHA-256 (slow) is computed in hashProcessing as additional metadata + * - Files become accessible in VFS immediately after lightProcessing completes + */ + +import { performanceMetrics } from '../metrics/PerformanceMetrics.js'; + +export type FileState = 'pending' | 'lightProcessing' | 'hashProcessing' | 'done'; + +export interface UnifiedFileState { + filePath: string; + state: FileState; + hash?: string; // midhash256 (set after lightProcessing) + virtualPath?: string; + error?: string; + + // Retry tracking + retryCount?: number; + lastRetryAt?: number; + + // Timestamps + discoveredAt?: number; + lightProcessingStartedAt?: number; + lightProcessingCompletedAt?: number; + hashProcessingStartedAt?: number; + hashProcessingCompletedAt?: number; + + // Processing times + lightProcessingTime?: number; + hashProcessingTime?: number; + totalProcessingTime?: number; +} + +export interface UnifiedProcessingSnapshot { + pending: UnifiedFileState[]; + lightProcessing: UnifiedFileState[]; + hashProcessing: UnifiedFileState[]; + done: UnifiedFileState[]; + + totalPending: number; + totalLightProcessing: number; + totalHashProcessing: number; + totalDone: number; + totalFilesInVFS: number; // Total number of files accessible in the Virtual File System + + // Configuration + fastQueueConcurrency?: number; // Number of fast queue workers + backgroundQueueConcurrency?: number; // Number of background queue workers +} + +export class UnifiedProcessingStateManager { + private pending: Map = new Map(); + private lightProcessing: Map = new Map(); + private hashProcessing: Map = new Map(); + private done: UnifiedFileState[] = []; + private maxDoneHistory = 100; // Keep last 100 completed files + private totalProcessedCount = 0; // Track total number of completed files (not limited) + + /** + * Add a file to pending state (file discovered) + */ + addPending(filePath: string): void { + this.pending.set(filePath, { + filePath, + state: 'pending', + discoveredAt: Date.now() + }); + } + + /** + * Move file from pending to light processing + */ + startLightProcessing(filePath: string): void { + const pendingState = this.pending.get(filePath); + const existingState = this.lightProcessing.get(filePath); + this.pending.delete(filePath); + + this.lightProcessing.set(filePath, { + filePath, + state: 'lightProcessing', + discoveredAt: pendingState?.discoveredAt || existingState?.discoveredAt, + lightProcessingStartedAt: Date.now(), + retryCount: existingState?.retryCount || 0 + }); + } + + /** + * Complete light processing and move to hash processing queue + */ + completeLightProcessing( + filePath: string, + hash: string, // midhash256 + virtualPath?: string, + error?: string + ): void { + const lightState = this.lightProcessing.get(filePath); + this.lightProcessing.delete(filePath); + + const now = Date.now(); + const lightProcessingTime = lightState?.lightProcessingStartedAt + ? now - lightState.lightProcessingStartedAt + : undefined; + + // Record light processing time metric + if (lightProcessingTime !== undefined && !error) { + performanceMetrics.recordMetadataProcessing(lightProcessingTime); + } + + if (error) { + // If error, mark as done with error + this.completeHashProcessing(filePath, hash, virtualPath, error); + return; + } + + // Move to hash processing queue + this.hashProcessing.set(filePath, { + filePath, + state: 'hashProcessing', + hash, + virtualPath, + discoveredAt: lightState?.discoveredAt, + lightProcessingStartedAt: lightState?.lightProcessingStartedAt, + lightProcessingCompletedAt: now, + lightProcessingTime, + retryCount: lightState?.retryCount || 0 + }); + } + + /** + * Start hash processing (SHA-256 computation) + */ + startHashProcessing(filePath: string): void { + const hashState = this.hashProcessing.get(filePath); + if (hashState) { + hashState.hashProcessingStartedAt = Date.now(); + } + } + + /** + * Complete hash processing and move to done + */ + completeHashProcessing( + filePath: string, + hash?: string, + virtualPath?: string, + error?: string + ): void { + const hashState = this.hashProcessing.get(filePath); + this.hashProcessing.delete(filePath); + + const now = Date.now(); + const hashProcessingTime = hashState?.hashProcessingStartedAt + ? now - hashState.hashProcessingStartedAt + : undefined; + + const totalProcessingTime = hashState?.discoveredAt + ? now - hashState.discoveredAt + : undefined; + + // Record hash processing time metric + if (hashProcessingTime !== undefined && !error) { + performanceMetrics.recordHashProcessing(hashProcessingTime); + } + + const doneState: UnifiedFileState = { + filePath, + state: 'done', + hash: hash || hashState?.hash, + virtualPath: virtualPath || hashState?.virtualPath, + error, + discoveredAt: hashState?.discoveredAt, + lightProcessingStartedAt: hashState?.lightProcessingStartedAt, + lightProcessingCompletedAt: hashState?.lightProcessingCompletedAt, + hashProcessingStartedAt: hashState?.hashProcessingStartedAt, + hashProcessingCompletedAt: now, + lightProcessingTime: hashState?.lightProcessingTime, + hashProcessingTime, + totalProcessingTime + }; + + // Add to done list (most recent first) + this.done.unshift(doneState); + + // Increment total processed counter (tracks all files, not limited) + this.totalProcessedCount++; + + // Trim history if needed + if (this.done.length > this.maxDoneHistory) { + this.done = this.done.slice(0, this.maxDoneHistory); + } + } + + /** + * Remove a file from all states (e.g., when deleted) + */ + removeFile(filePath: string): void { + this.pending.delete(filePath); + this.lightProcessing.delete(filePath); + this.hashProcessing.delete(filePath); + this.done = this.done.filter(f => f.filePath !== filePath); + } + + /** + * Get a snapshot of the current processing state + * @param vfsFileCount - Optional total number of files in the Virtual File System + * @param fastQueueConcurrency - Number of fast queue workers + * @param backgroundQueueConcurrency - Number of background queue workers + */ + getSnapshot(vfsFileCount: number = 0, fastQueueConcurrency?: number, backgroundQueueConcurrency?: number): UnifiedProcessingSnapshot { + // Limit returned arrays to prevent slow API responses with large queues + const MAX_ITEMS_PER_STATE = 100; + + // Efficiently extract only the needed items without creating full arrays + const getPendingSample = (): UnifiedFileState[] => { + const result: UnifiedFileState[] = []; + let count = 0; + for (const item of this.pending.values()) { + if (count >= MAX_ITEMS_PER_STATE) break; + result.push(item); + count++; + } + return result; + }; + + const getLightProcessingSample = (): UnifiedFileState[] => { + const result: UnifiedFileState[] = []; + let count = 0; + for (const item of this.lightProcessing.values()) { + if (count >= MAX_ITEMS_PER_STATE) break; + result.push(item); + count++; + } + return result; + }; + + const getHashProcessingSample = (): UnifiedFileState[] => { + const result: UnifiedFileState[] = []; + let count = 0; + for (const item of this.hashProcessing.values()) { + if (count >= MAX_ITEMS_PER_STATE) break; + result.push(item); + count++; + } + return result; + }; + + return { + pending: getPendingSample(), + lightProcessing: getLightProcessingSample(), + hashProcessing: getHashProcessingSample(), + done: this.done.slice(0, 50), // Return only last 50 for API + + totalPending: this.pending.size, + totalLightProcessing: this.lightProcessing.size, + totalHashProcessing: this.hashProcessing.size, + totalDone: this.totalProcessedCount, // Total count of all processed files (not limited to history) + totalFilesInVFS: vfsFileCount, // Total number of files accessible in the Virtual File System + + fastQueueConcurrency, + backgroundQueueConcurrency + }; + } + + /** + * Get the state of a file in any processing queue + */ + getFileState(filePath: string): UnifiedFileState | undefined { + return this.pending.get(filePath) || + this.lightProcessing.get(filePath) || + this.hashProcessing.get(filePath); + } + + /** + * Clear all states (useful for testing or reset) + */ + clear(): void { + this.pending.clear(); + this.lightProcessing.clear(); + this.hashProcessing.clear(); + this.done = []; + this.totalProcessedCount = 0; + } + + /** + * Get the total number of files in all states + */ + getTotalFileCount(): number { + return this.pending.size + this.lightProcessing.size + this.hashProcessing.size + this.done.length; + } + + /** + * Retry processing for a file (move back to pending with incremented retry count) + */ + retryProcessing(filePath: string): boolean { + const lightState = this.lightProcessing.get(filePath); + const hashState = this.hashProcessing.get(filePath); + const state = lightState || hashState; + + if (!state) { + return false; + } + + const retryCount = (state.retryCount || 0) + 1; + + this.lightProcessing.delete(filePath); + this.hashProcessing.delete(filePath); + + this.pending.set(filePath, { + filePath, + state: 'pending', + discoveredAt: state.discoveredAt, + retryCount, + lastRetryAt: Date.now() + }); + + return true; + } + + /** + * Get the retry count for a file + */ + getRetryCount(filePath: string): number { + const lightState = this.lightProcessing.get(filePath); + const hashState = this.hashProcessing.get(filePath); + const pendingState = this.pending.get(filePath); + return lightState?.retryCount || hashState?.retryCount || pendingState?.retryCount || 0; + } + + /** + * Get all files currently in light processing + * Returns the internal Map for direct iteration + */ + getLightProcessingFiles(): Map { + return this.lightProcessing; + } + + /** + * Get all files currently in hash processing + * Returns the internal Map for direct iteration + */ + getHashProcessingFiles(): Map { + return this.hashProcessing; + } +} diff --git a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts new file mode 100644 index 0000000..d6144c9 --- /dev/null +++ b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts @@ -0,0 +1,620 @@ +import {promises as fs} from 'fs'; +import {dirname, join, parse} from 'path'; +import {config} from "../config/EnvConfig.js"; +import {FileProcessorInterface} from "@metazla/meta-hash"; +import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js"; +import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js"; +import {FileType} from "@metazla/filename-tools"; +import {existsAsync} from "@metazla/meta-hash"; +import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js"; +import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js"; +import {renamingRule} from "../config/RenamingRule.js"; +import {MetaMeshFormat} from "./MetaMeshFormat.js"; +import {JellyfinAPI} from "../jellyfin/JellyfinAPI.js"; +import {VirtualFileSystem} from "../api/VirtualFileSystem.js"; +import {performanceMetrics} from "../metrics/PerformanceMetrics.js"; +import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js"; +import type {IKVClient} from "../kv/IKVClient.js"; +import PQueue from "p-queue"; +import os from 'os'; +import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js"; +import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js"; +import { from } from "@metazla/meta-interface"; +import { stat } from "fs/promises"; +import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js"; + +export class WatchedFileProcessor implements FileProcessorInterface { + jellyfinAPI = new JellyfinAPI(); + metaMeshFormat = new MetaMeshFormat(); + metaDataToFolderStruct = new MetaDataToFolderStruct(renamingRule); + virtualFileSystem: VirtualFileSystem; + unifiedStateManager = new UnifiedProcessingStateManager(); + //fileProcessor:FileAnalyzerInterface = new FileProcessor();//TODO only use for debug + fileProcessor:FileAnalyzerInterface = new FileProcessorPiscina(); + duplicateFinder = new DuplicateFinder(); + duplicateResult: DuplicateResult | null = null; + fileType = new FileType(); + supported = ['video','subtitle','torrent']; + + // Task-based plugin scheduler (replaces two-queue system) + private taskScheduler: TaskScheduler | null = null; + + // Pre-processing queue for midhash256 computation (before plugins) + private preProcessQueue: PQueue; + + // Reference to StreamingPipeline for queue status + private pipeline: StreamingPipeline | null = null; + + constructor() { + // Always initialize VirtualFileSystem for FUSE API + this.virtualFileSystem = new VirtualFileSystem({ + fileMode: 0o644, + directoryMode: config.CHMOD_FOR_NEW_FOLDER || 0o755 + }); + + // Pass the unified state manager to the fileProcessor + if (this.fileProcessor instanceof FileProcessorPiscina) { + this.fileProcessor.setUnifiedStateManager(this.unifiedStateManager); + } + + // Pre-processing queue for midhash256 computation + const cpuCount = os.cpus().length; + const preProcessConcurrency = config.MAX_WORKER_THREADS || cpuCount * 2; + this.preProcessQueue = new PQueue({concurrency: preProcessConcurrency, autoStart: true}); + + console.log(`WatchedFileProcessor: TaskScheduler-based system initialized`); + console.log(` - Pre-process queue: ${preProcessConcurrency} workers (midhash256)`); + console.log(` - TaskScheduler handles plugin execution (fast/background queues)`); + } + + getVirtualFileSystem(): VirtualFileSystem { + return this.virtualFileSystem; + } + + getUnifiedStateManager(): UnifiedProcessingStateManager { + return this.unifiedStateManager; + } + + getDuplicateResult(): DuplicateResult | null { + return this.duplicateResult; + } + + /** + * Get the KV client + */ + getKVClient(): IKVClient | null { + if (this.fileProcessor instanceof FileProcessorPiscina) { + return this.fileProcessor.getKVClient(); + } + return null; + } + + /** + * Set the KV client + * This is called by KVManager after successful connection + */ + setKVClient(client: IKVClient): void { + if (this.fileProcessor instanceof FileProcessorPiscina) { + this.fileProcessor.setKVClient(client); + } + } + + /** + * Set the StreamingPipeline reference for queue status + * This is called by index.ts after creating the pipeline + */ + setPipeline(pipeline: StreamingPipeline): void { + this.pipeline = pipeline; + console.log('[WatchedFileProcessor] Pipeline connected for queue status'); + } + + /** + * Get current queue status for monitoring UI + * Uses StreamingPipeline queues which do the actual processing: + * Pipeline flow: validation → lightProcessing → hashProcessing + * + * Mapping to UI: + * - Pre-Process = validation (quick extension checks, high concurrency) + * - Fast Queue = lightProcessing (midhash256 + metadata extraction) + * - Background Queue = hashProcessing (SHA-256 full file hash) + */ + getQueueStatus() { + // Use StreamingPipeline stats if available (this is the actual processing engine) + if (this.pipeline) { + const pipelineStats = this.pipeline.getStats(); + const queues = pipelineStats.queues; + + // Map StreamingPipeline queues to the 3-queue architecture UI expects + // PQueue: .pending = currently running, .size = waiting in queue + // API expects: .pending = running, .size = waiting + const result = { + preProcessQueue: { + // validation = pre-process (quick extension checks) + size: queues.validation.size, // waiting in queue + pending: queues.validation.pending, // running jobs + isPaused: false + }, + fastQueue: { + // fastQueue = fast queue (midhash256 + metadata) + running: queues.fastQueue.pending, // running jobs + pending: queues.fastQueue.size, // waiting in queue + size: queues.fastQueue.size, + isPaused: false + }, + backgroundQueue: { + // backgroundQueue = background (SHA-256 computation) + running: queues.backgroundQueue.pending, // running jobs + pending: queues.backgroundQueue.size, // waiting in queue + size: queues.backgroundQueue.size, + isPaused: false + } + }; + return result; + } + + // Fallback: use TaskScheduler if available (for future plugin-based architecture) + if (this.taskScheduler) { + const status = this.taskScheduler.getQueueStatus(); + return { + preProcessQueue: { + size: this.preProcessQueue.pending, + pending: this.preProcessQueue.size, + isPaused: this.preProcessQueue.isPaused + }, + fastQueue: { + size: status.fast.ready, + pending: status.fast.pending, + running: status.fast.running, + isPaused: status.fast.isPaused + }, + backgroundQueue: { + size: status.background.ready, + pending: status.background.pending, + running: status.background.running, + isPaused: status.background.isPaused + } + }; + } + + // Fallback: empty status + return { + preProcessQueue: { size: 0, pending: 0, isPaused: false }, + fastQueue: { size: 0, pending: 0, running: 0, isPaused: false }, + backgroundQueue: { size: 0, pending: 0, running: 0, isPaused: false } + }; + } + + /** + * Get the TaskScheduler instance + */ + getTaskScheduler(): TaskScheduler | null { + return this.taskScheduler; + } + + async initialize() { + await this.fileProcessor.init(); + + // Initialize TaskScheduler with PluginManager using config values + const pluginManager = await initializePluginManager(); + this.taskScheduler = createTaskScheduler(pluginManager, performanceMetrics, { + fastQueueConcurrency: config.FAST_QUEUE_CONCURRENCY, + backgroundQueueConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY, + fastThresholdMs: config.FAST_THRESHOLD_MS, + minSamplesForMeasurement: config.MIN_SAMPLES_FOR_MEASUREMENT, + }); + + // Set up event listeners for VFS updates + this.setupTaskSchedulerEvents(); + + console.log(`[WatchedFileProcessor] TaskScheduler initialized with:`); + console.log(` - Fast queue concurrency: ${config.FAST_QUEUE_CONCURRENCY}`); + console.log(` - Background queue concurrency: ${config.BACKGROUND_QUEUE_CONCURRENCY}`); + console.log(` - Fast threshold: ${config.FAST_THRESHOLD_MS}ms`); + console.log(` - Min samples for measurement: ${config.MIN_SAMPLES_FOR_MEASUREMENT}`); + } + + /** + * Set up TaskScheduler event listeners for VFS integration and Redis pub/sub + */ + private setupTaskSchedulerEvents(): void { + if (!this.taskScheduler) return; + + // When a task completes, check if we should update VFS + this.taskScheduler.on('task:complete', (event: any) => { + const { task } = event; + + // Update VFS when filename-parser completes (has basic metadata) + if (task.pluginId === 'filename-parser') { + this.updateVFSForFile(task.fileHash, task.filePath); + } + + // Publish plugin completion event to Redis + this.publishPluginComplete(task.fileHash, task.pluginId, task.filePath); + }); + + // When all tasks for a file complete + this.taskScheduler.on('file:complete', (event: any) => { + const { fileHash, filePath } = event; + console.log(`[TaskScheduler] All plugins complete for ${filePath}`); + + // Final VFS update with complete metadata + this.updateVFSForFile(fileHash, filePath); + + // Update unified state manager + if (this.unifiedStateManager) { + this.unifiedStateManager.completeHashProcessing(filePath, fileHash); + } + }); + } + + /** + * Publish plugin completion event to Redis + * Channel: meta-sort:plugin:complete + */ + private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise { + const kvClient = this.getKVClient(); + if (!kvClient || typeof (kvClient as any).publish !== 'function') { + return; + } + + try { + const message = JSON.stringify({ + fileHash, + pluginId, + filePath, + timestamp: Date.now() + }); + await (kvClient as any).publish('meta-sort:plugin:complete', message); + } catch (error) { + // Silent fail - pub/sub is optional + } + } + + /** + * Update VFS for a file with current metadata + */ + private updateVFSForFile(fileHash: string, filePath: string): void { + const metadata = this.fileProcessor.getDatabase().get(filePath); + if (metadata && metadata.cid_midhash256) { + try { + const virtualPath = this.metaDataToFolderStruct.renamingRule(metadata as any, ''); + this.virtualFileSystem.addFile(virtualPath, filePath, metadata); + } catch (e) { + // Renaming rule may fail for incomplete metadata + } + } + } + + markPending(filePath: string): void { + // STATE 1: Mark as pending when file is discovered (before queue) + this.unifiedStateManager.addPending(filePath); + } + + async queueFile(filePath: string): Promise { + // TaskScheduler-based processing + // Phase 1: Pre-process (midhash256 computation) + // Phase 2: Plugin execution via TaskScheduler (fast/background queues) + + // Debug: log queue state before adding + console.log(`[DEBUG queueFile] Before add: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`); + + await this.preProcessQueue.add(async () => { + console.log(`[DEBUG queueFile] Task started: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`); + try { + // STATE: Start LIGHT processing + if (this.unifiedStateManager) { + this.unifiedStateManager.startLightProcessing(filePath); + } + + // STEP 1: Compute midhash256 (file identifier) + const fileStats = await stat(filePath); + const { computeMidHash256 } = await import('@metazla/meta-hash'); + + // Check cache first + const indexLine = (this.fileProcessor as any).indexManager?.getCidForFile( + filePath, + fileStats.size, + fileStats.mtime.toISOString() + ); + + let midHash256: string; + if (indexLine?.cid_midhash256) { + midHash256 = indexLine.cid_midhash256; + performanceMetrics.recordCacheHit('midhash256'); + } else { + const computeStart = performance.now(); + midHash256 = await computeMidHash256(filePath); + performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart)); + performanceMetrics.recordCacheMiss('midhash256'); + } + + // STEP 2: Initialize metadata with midhash256 + const metadata: any = { + cid_midhash256: midHash256, + filePath, + processingStatus: 'processing' + }; + + // Store in local database + this.fileProcessor.getDatabase().set(filePath, metadata); + + // STEP 3: Create KV store for plugins + const metaDataNode = from(metadata); + const kv = new MetadataNodeKVStore(metaDataNode); + + // STEP 4: Create and enqueue plugin tasks + if (this.taskScheduler) { + const tasks = this.taskScheduler.createTasksForFile(filePath, midHash256, kv); + this.taskScheduler.enqueueTasks(tasks); + } + + // STATE: Light processing setup complete + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing( + filePath, + midHash256, + undefined // Virtual path will be computed by plugins + ); + } + } catch (error) { + console.error(`Failed pre-processing for ${filePath}:`, error); + + // Mark processing as failed + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing( + filePath, + '', + undefined, + error instanceof Error ? error.message : String(error) + ); + } + } + }); + } + + async processFile(current: number, queueSize: number, filePath: string): Promise { + // This method is now a wrapper for queueFile for backwards compatibility + // In the new single-phase model, queueFile handles everything + await this.queueFile(filePath); + } + + async finalize(): Promise { + try { + console.log(`Virtual filesystem update start`); + const start = performance.now(); + if (this.fileProcessor.getDatabase().size === 0) { + console.log(`No files to process`); + return; + } + let startTime = performance.now(); + const newDuplicateResult = await this.duplicateFinder.findDuplicates(this.fileProcessor.getDatabase());//find duplicates and removes them to avoid conflict + const duplicateTime = Math.ceil(performance.now() - startTime); + const totalDuplicates = newDuplicateResult.hashDuplicates.length + newDuplicateResult.titleDuplicates.length; + console.log(`Duplicate find took ${duplicateTime}ms - found ${newDuplicateResult.hashDuplicates.length} hash groups, ${newDuplicateResult.titleDuplicates.length} title groups`); + + // Only update duplicateResult if we found new duplicates, or if it's the first run + if (totalDuplicates > 0 || this.duplicateResult === null) { + this.duplicateResult = newDuplicateResult; + } + + performanceMetrics.recordDuplicateDetection(duplicateTime, totalDuplicates); + + startTime = performance.now(); + console.log(`Generating virtual structure for ${this.fileProcessor.getDatabase().size} files`); + const virtualStructure = this.metaDataToFolderStruct.generateVirtualStructure(this.fileProcessor.getDatabase()); + const virtualStructureTime = Math.ceil(performance.now() - startTime); + console.log(`Virtual structure generation took ${virtualStructureTime}ms`); + performanceMetrics.recordVirtualStructureGeneration(virtualStructureTime); + + // Build VirtualFileSystem for FUSE/WebDAV API + startTime = performance.now(); + this.virtualFileSystem.buildFromComputed(virtualStructure, this.fileProcessor.getDatabase()); + const vfsBuildTime = Math.ceil(performance.now() - startTime); + console.log(`VirtualFileSystem build took ${vfsBuildTime}ms`); + performanceMetrics.recordVFSUpdate(vfsBuildTime); + const stats = this.virtualFileSystem.getStats(); + console.log(`VirtualFileSystem: ${stats.fileCount} files (${stats.metaFileCount} .meta), ${stats.directoryCount} directories, ${(stats.totalSize / 1024 / 1024).toFixed(2)} MB`); + + if(this.jellyfinAPI.jellyApiAvailable()) { + await this.jellyfinAPI.refreshLibrary(); + console.log(`Jellyfin refreshed`); + } + + console.log(`Virtual filesystem update took ${Math.ceil(performance.now() - start)}ms`); + } catch (error) { + console.error(error); + } + } + + async canProcessFile(filePath: string) { + const fileTypeResult = await this.fileType.getFileType(filePath); + //check if filetype is one of the suported ones + return this.supported.includes(fileTypeResult); + } + + async deleteFile(filePath: string) { + this.fileProcessor.deleteFile(filePath); + this.unifiedStateManager.removeFile(filePath); + } + + /** + * Clean up stale entries from in-memory database for scanned directories + * + * This is called after discovery scans a folder. It removes files from + * in-memory DB that belong to the scanned folder but were not seen during + * the scan (meaning they no longer exist on disk). + * + * Design Philosophy: + * - Discovery is the source of truth for file existence + * - Empty/missing folders → remove all child paths from memory + * - KV keeps everything (distributed metadata archive) + * + * @param scannedFolders - Array of folder paths that were completely scanned + * @param discoveredFiles - Set of file paths that were found during scan + */ + async cleanupStaleEntries(scannedFolders: string[], discoveredFiles: Set): Promise { + console.log(`[Cleanup] Starting cleanup for ${scannedFolders.length} scanned folders...`); + const startTime = performance.now(); + + let removedCount = 0; + const db = this.fileProcessor.getDatabase(); + const removedHashIds: string[] = []; // Collect hashIds for Stremio notification + + for (const folder of scannedFolders) { + const normalizedFolder = folder.endsWith('/') ? folder : folder + '/'; + + // Find all in-memory files that belong to this folder + const filesToCheck: string[] = []; + for (const [filePath] of db.entries()) { + if (filePath.startsWith(normalizedFolder) || filePath === folder) { + filesToCheck.push(filePath); + } + } + + // Remove files that weren't discovered (no longer exist) + for (const filePath of filesToCheck) { + if (!discoveredFiles.has(filePath)) { + // File is in memory but wasn't seen during scan → removed/offline + const metadata = db.get(filePath); + + // Collect hashId before deletion (for Stremio notification) + if (metadata && metadata.cid_midhash256) { + removedHashIds.push(metadata.cid_midhash256); + } + + db.delete(filePath); + this.unifiedStateManager.removeFile(filePath); + removedCount++; + } + } + } + + const elapsed = Math.ceil(performance.now() - startTime); + console.log(`[Cleanup] Removed ${removedCount} stale entries in ${elapsed}ms`); + + // If we removed files, rebuild VFS to update structure + if (removedCount > 0) { + console.log('[Cleanup] Rebuilding VFS after cleanup...'); + await this.finalize(); + + // Notify Stremio addon to remove videos (incremental updates like FUSE) + if (removedHashIds.length > 0) { + console.log(`[Cleanup] Notifying Stremio to remove ${removedHashIds.length} videos...`); + for (const hashId of removedHashIds) { + this.notifyStremioRemove(hashId).catch(() => { + // Silent fail - Stremio addon may not be running + }); + } + } + } + } + + /** + * Rebuild VirtualFileSystem from KV data + * This is called on startup to restore the VFS from persistent storage + * + * Fast-load strategy: Load all metadata from KV WITHOUT disk validation. + * Discovery will later validate file existence and clean up stale entries. + * + * Design Philosophy: + * - KV = distributed metadata archive (permanent record) + * - In-memory DB = current VFS state (only present files) + * - Discovery = source of truth (validates and cleans) + */ + async rebuildVFSFromKV(): Promise { + const kvClient = this.getKVClient(); + if (!kvClient) { + console.warn('[VFS Rebuild] KV client not available, skipping VFS rebuild'); + return; + } + + try { + console.log('[VFS Rebuild] Starting fast-load from KV...'); + const startTime = performance.now(); + + // Get all hash IDs from KV + const hashIds = await kvClient.getAllHashIds(); + console.log(`[VFS Rebuild] Found ${hashIds.length} files in KV`); + + if (hashIds.length === 0) { + console.log('[VFS Rebuild] No files in KV, VFS remains empty'); + return; + } + + // Batch fetch all metadata in parallel (no disk validation) + console.log('[VFS Rebuild] Fetching metadata in parallel...'); + const metadataList = await Promise.all( + hashIds.map(async (hashId) => { + try { + return await kvClient.getMetadataFlat(hashId); + } catch (error) { + console.error(`[VFS Rebuild] Failed to load metadata for ${hashId}:`, error); + return null; + } + }) + ); + + // Populate in-memory database without disk validation + let loadedCount = 0; + for (const metadata of metadataList) { + if (metadata && metadata.filePath) { + // Mark as unverified - discovery will validate later + (metadata as any)._lastVerified = 0; + this.fileProcessor.getDatabase().set(metadata.filePath, metadata); + loadedCount++; + } + } + + console.log(`[VFS Rebuild] Loaded ${loadedCount}/${hashIds.length} files into memory (unverified)`); + + // Build VFS immediately - may contain stale entries until discovery validates + if (this.fileProcessor.getDatabase().size > 0) { + await this.finalize(); + console.log(`[VFS Rebuild] VFS rebuilt in ${Math.ceil(performance.now() - startTime)}ms`); + console.log('[VFS Rebuild] Discovery will validate file existence in background'); + } else { + console.log('[VFS Rebuild] No files loaded, VFS remains empty'); + } + } catch (error) { + console.error('[VFS Rebuild] Error rebuilding VFS from KV:', error); + } + } + + /** + * Notify Stremio addon to remove video (incremental update like FUSE) + */ + private async notifyStremioRemove(hashId: string): Promise { + try { + const response = await fetch('http://localhost:7000/remove', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ hashId }), + signal: AbortSignal.timeout(5000) // 5 second timeout + }); + + if (!response.ok) { + // Silent fail - Stremio may not be running + return; + } + } catch (error: any) { + // Silent fail - Stremio addon may not be running or network issue + } + } + + /** + * Shutdown: clean up resources + */ + async shutdown(): Promise { + console.log('[WatchedFileProcessor] Shutting down...'); + + // Clear pre-process queue + this.preProcessQueue.clear(); + + // Shutdown TaskScheduler + if (this.taskScheduler) { + await this.taskScheduler.shutdown(); + } + + console.log('[WatchedFileProcessor] Shutdown complete'); + } +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts new file mode 100644 index 0000000..a285076 --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts @@ -0,0 +1,37 @@ +import {HashMeta} from "@metazla/meta-interface"; +import {FileMetadata} from "../../types/FileMetadata.js"; + +export interface FileAnalyzerInterface { + /** + * Full file processing - computes midhash256 and extracts all metadata + * Since midhash256 is instant (< 1s), files appear in VFS immediately with permanent ID + */ + processFile(filePath: string, current: number, queueSize: number):Promise; + + /** + * Light processing phase - fast metadata extraction + midhash256 + * File becomes accessible in VFS with permanent ID after this phase + */ + processLightPhase(filePath: string, current: number, queueSize: number):Promise; + + /** + * Hash processing phase - background hash computation + * Computes full hashes (SHA-256, SHA-1, MD5, CRC32) and updates KV + */ + processHashPhase(filePath: string, current: number, queueSize: number):Promise; + + /** + * Get the metadata database + */ + getDatabase(): Map; + + /** + * Delete a file from the database + */ + deleteFile(filePath: string): Promise | void; + + /** + * Initialize the file analyzer + */ + init():Promise; +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts new file mode 100644 index 0000000..8428964 --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts @@ -0,0 +1,92 @@ +import {HashIndexManager} from "@metazla/meta-hash"; +import {config} from "../../config/EnvConfig.js"; +import {stat} from "fs/promises"; +import {targetHash} from "../../config/TargetHash.js"; +import {HashMeta} from "@metazla/meta-interface"; +import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js"; +import {FileProcessorInternal} from "./FileProcessorInternal.js"; +import {globalHashIndex} from "./SimpleFileToCid.js"; +import {FileMetadata} from "../../types/FileMetadata.js"; + + + +export class FileProcessor implements FileAnalyzerInterface{ + database = new Map();//Map + indexManager = globalHashIndex; + fileProcessor = new FileProcessorInternal(); + + constructor() { + } + + async init() { + await this.indexManager.init(true); + console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry'); + } + + /** + * Quick metadata processing - extracts metadata without computing content hash + * Note: Plugin processing is now handled by container plugins via TaskScheduler + */ + async processFileQuick(filePath: string): Promise { + // Initialize metadata with processing status + const metadata: FileMetadata = { + processingStatus: 'processing' + }; + + // Add to database immediately + // Plugin processing will be done asynchronously by container plugins + this.database.set(filePath, metadata); + + return metadata; + } + + getDatabase(): Map { + return this.database as Map; + } + + async processFile(filePath: string, current: number, queueSize: number) { + try { + if (this.database.has(filePath)) { + return; + } + + const stats = await stat(filePath); + let indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + let hash: HashMeta = {}; + if(indexLine) { + for (const hashKey of targetHash) { + if (indexLine[hashKey]) { + hash[hashKey] = indexLine[hashKey]; + } + } + } + + // -------------------------------------------- + // Hash logic compute and get the hash - use cache etc... + // -------------------------------------------- + const result = await this.fileProcessor.processFile(filePath,hash); + //const result = await this.fileProcessor.processFile(filePath,params.hash); + + //save the metadata to the cache or disk + this.database.set(filePath, result.metadata); + this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), result.metadata); + //console.log(`File [${current}/${queueSize}] ${filePath} took ${result.time}ms`); + } catch (e) { + console.error(`Error processing file [No Thread] ${filePath}:`, e); + } + } + + async processLightPhase(filePath: string, current: number, queueSize: number) { + // Legacy FileProcessor doesn't support split phases - just use processFile + await this.processFile(filePath, current, queueSize); + } + + async processHashPhase(filePath: string, current: number, queueSize: number) { + // Legacy FileProcessor doesn't support split phases - no-op + // (processFile already did everything) + } + + deleteFile(filePath: string) { + this.database.delete(filePath); + } +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts new file mode 100644 index 0000000..e8e430c --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts @@ -0,0 +1,84 @@ +import { HashComputerFile } from "@metazla/meta-hash"; +import { targetHash } from "../../config/TargetHash.js"; +import { DebugMeta, HashMeta } from "@metazla/meta-interface"; +import { MetaMeshFormat } from "../MetaMeshFormat.js"; + +type Local = HashMeta & DebugMeta; + +export class FileProcessorInternal { + hashComputer = new HashComputerFile(targetHash); + metaMeshFormat = new MetaMeshFormat(); + + async processFile(filePath: string, hash: HashMeta): Promise<{ + time: number, + metadata: HashMeta, + hashTimings?: Record, + pluginTimings?: Record + }> { + const start = performance.now(); + const hashTimings: Record = {}; + const pluginTimings: Record = {}; + + // -------------------------------------------- + // Hash logic compute and get the hash - use cache etc... + // -------------------------------------------- + const metaDataPath = `${filePath}.meta`; + let metadata: HashMeta = await this.metaMeshFormat.read(metaDataPath) || {}; // Read the reference file + // try to update the sha values from the index + if (hash) { + for (let key in hash) { + if (!metadata[key] && hash[key]) { + metadata[key] = hash[key]; + } + } + } + + // Time consuming - compute hashes and track timing + try { + // Determine which hashes need to be computed + const neededHashes = this.hashComputer['targetHash'].filter(hashName => !metadata[hashName]); + + if (neededHashes.length > 0) { + // Measure total time for hash computation + const hashStart = performance.now(); + + // Compute all needed hashes at once (more efficient than individual) + await this.hashComputer.computeMissingHash(filePath, metadata); + + const totalHashTime = Math.ceil(performance.now() - hashStart); + + // Distribute the time evenly among computed hashes + // This is an approximation - actual individual hash times may vary + const avgTimePerHash = totalHashTime / neededHashes.length; + for (const hashAlgorithm of neededHashes) { + hashTimings[hashAlgorithm] = Math.ceil(avgTimePerHash); + } + } + } catch (hashError) { + console.error(`[Worker] Hash computation failed for ${filePath}:`, hashError); + throw new Error(`Hash computation failed: ${hashError instanceof Error ? hashError.message : String(hashError)}`); + } + + // Plugin processing is now handled by container plugins via TaskScheduler + // This internal processor only handles hash computation + + // Validate all required hashes are present before returning + const missingHashesAfterCompute: string[] = []; + for (const hashKey of this.hashComputer['targetHash']) { + if (!metadata[hashKey]) { + missingHashesAfterCompute.push(hashKey); + } + } + + if (missingHashesAfterCompute.length > 0) { + throw new Error(`[Worker] Hash validation failed - missing hashes: ${missingHashesAfterCompute.join(', ')}`); + } + + return { + time: Math.ceil(performance.now() - start), + metadata, + hashTimings, + pluginTimings + }; + } +} diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts new file mode 100644 index 0000000..69e8fb1 --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts @@ -0,0 +1,594 @@ +import {Piscina} from "piscina"; +import {Input, Output} from "./worker.js"; +import {HashIndexManager} from "@metazla/meta-hash"; +import {config} from "../../config/EnvConfig.js"; +import {stat} from "fs/promises"; +// Note: Full hashes are now computed by the full-hash container plugin +import {HashMeta} from "@metazla/meta-interface"; +import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js"; +import {globalHashIndex} from "./SimpleFileToCid.js"; +import {performanceMetrics} from "../../metrics/PerformanceMetrics.js"; +import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js"; +import {renamingRule} from "../../config/RenamingRule.js"; +import {FileMetadata, ProcessingStatus} from "../../types/FileMetadata.js"; +import { PluginManager } from "../../plugin-engine/index.js"; +import type {IKVClient} from "../../kv/IKVClient.js"; +import * as path from 'path'; +import { fileURLToPath } from 'url'; +import os from 'os'; + +// Get the directory of this file to resolve plugin paths +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Singleton PluginManager instance +let pluginManagerInstance: PluginManager | null = null; +let pluginManagerInitPromise: Promise | null = null; + +/** + * Get the plugin manager instance (for API access) + * Returns null if not yet initialized + */ +export function getPluginManagerInstance(): PluginManager | null { + return pluginManagerInstance; +} + +/** + * Set the plugin manager instance (called by container initialization) + */ +export function setPluginManagerInstance(manager: PluginManager): void { + pluginManagerInstance = manager; +} + +/** + * Initialize the plugin manager on startup + * Note: With container-based plugins, the PluginManager is initialized + * by the ContainerManager. This function creates a basic instance + * for state management if container plugins are not yet ready. + */ +export async function initializePluginManager(): Promise { + if (pluginManagerInstance) { + return pluginManagerInstance; + } + + // Create basic plugin manager for state management + const statePath = path.join(config.CACHE_FOLDER_PATH, 'plugins.json'); + const cacheDir = path.join(config.CACHE_FOLDER_PATH, 'plugin-cache'); + + console.log(`[PluginManager] Initializing with:`); + console.log(` - State path: ${statePath}`); + console.log(` - Cache dir: ${cacheDir}`); + + pluginManagerInstance = new PluginManager({ + statePath, + cacheDir, + }); + + console.log(`[PluginManager] Basic instance created (waiting for container plugins)`); + return pluginManagerInstance; +} + +export class FileProcessorPiscina implements FileAnalyzerInterface{ + private piscina: Piscina; + database = new Map();//Map + indexManager = globalHashIndex; + private unifiedStateManager: UnifiedProcessingStateManager | null = null; + + // Unified KV client (Redis) + private kvClient: IKVClient | null = null; + + // Track running hash workers for abort capability + private runningHashTasks = new Map(); + + constructor() { + // Construct the URL for the current module + let distFolder = import.meta.dirname; + distFolder = distFolder.replace('src', 'dist'); + distFolder = distFolder + "/worker.js"; + const workerUrl = new URL(distFolder,"file://").href; + + // Set maxThreads: Smart default uses half CPU cores to avoid starving the system + // (especially important for network storage + HDD scenarios) + const cpuCount = os.cpus().length; + const defaultWorkers = Math.max(1, Math.floor(cpuCount / 2)); + const maxThreads = config.MAX_WORKER_THREADS || defaultWorkers; + + this.piscina = new Piscina({ + maxThreads, + //maxThreads: 1,//TODO note for DEBUG must be set to 1 to avoid memory issues + filename:workerUrl + }); + + console.log(`Piscina worker pool initialized with ${maxThreads} threads (cpuCount=${cpuCount}, defaultWorkers=${defaultWorkers})`); + } + + async init() { + await this.indexManager.init(true); + console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry'); + } + + /** + * Single-phase metadata processing - computes midhash256 immediately and extracts metadata + * Files appear in VFS instantly with their permanent midhash256 ID + * + * @param filePath - Path to the file to process + * @param current - Current file number (for progress reporting) + * @param queueSize - Total queue size (for progress reporting) + * @returns FileMetadata with midhash256 and all metadata + */ + + setUnifiedStateManager(manager: UnifiedProcessingStateManager): void { + this.unifiedStateManager = manager; + } + + getDatabase(): Map { + return this.database as Map; + } + + /** + * Abort all running hash workers immediately (for priority enforcement) + * @returns Number of workers aborted + */ + abortAllHashWorkers(): number { + const abortedCount = this.runningHashTasks.size; + + for (const [filePath, controller] of this.runningHashTasks.entries()) { + controller.abort(); + console.log(`[Abort] Hash worker aborted: ${filePath}`); + } + + this.runningHashTasks.clear(); + return abortedCount; + } + + /** + * Gracefully shutdown - close KV client + */ + async shutdown(): Promise { + if (this.kvClient) { + console.log('[FileProcessor] Closing KV client...'); + await this.kvClient.close(); + console.log('[FileProcessor] KV client closed'); + } + } + + async processFile(filePath: string, current: number, queueSize: number) { + const startTime = Date.now(); + + try { + // STATE: Start LIGHT processing (fast: midhash256 + metadata extraction) + if (this.unifiedStateManager) { + this.unifiedStateManager.startLightProcessing(filePath); + } + + const stats = await stat(filePath); + + // STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files) + let midHash256: string | undefined; + const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + + if (indexLine && indexLine.cid_midhash256) { + // Cache hit! Use cached midhash256 (no network I/O needed) + midHash256 = indexLine.cid_midhash256; + performanceMetrics.recordCacheHit('midhash256'); + } else { + // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives) + const computeStart = performance.now(); + const { computeMidHash256 } = await import('@metazla/meta-hash'); + midHash256 = await computeMidHash256(filePath); + const computeTime = Math.ceil(performance.now() - computeStart); + + // Record midhash256 computation time (excludes cache hits) + performanceMetrics.recordHashComputation('cid_midhash256', computeTime); + performanceMetrics.recordCacheMiss('midhash256'); + } + + // Initialize metadata with midhash256 + const metadata: FileMetadata = { + cid_midhash256: midHash256, + processingStatus: 'processing' + }; + + // STEP 2: Plugin metadata extraction is now handled by container plugins + // The TaskScheduler will dispatch tasks to container plugins asynchronously + // Basic metadata processing continues below + + // Generate virtualPath from metadata (needed for VFS access) + let virtualPath: string | undefined; + try { + virtualPath = renamingRule(metadata as any, ''); + } catch (e) { + // If renamingRule fails, virtualPath remains undefined + } + + // STEP 3: Collision detection and store preliminary metadata in KV + let collisionDetected = false; + if (this.kvClient) { + try { + const hashId = midHash256; // Use midhash256 as primary ID + + if (hashId) { + // COLLISION DETECTION: Check if this midhash256 already exists + const existingFilePath = await this.kvClient.getMetadata(hashId, 'filePath'); + + if (existingFilePath && existingFilePath !== filePath) { + // COLLISION DETECTED: Different file with same midhash256! + collisionDetected = true; + console.warn(`⚠️ MIDHASH256 COLLISION DETECTED!`); + console.warn(` Hash: ${hashId}`); + console.warn(` Existing file: ${existingFilePath}`); + console.warn(` New file: ${filePath}`); + console.warn(` Action: Skipping VFS addition, forcing full hash computation`); + + // Do NOT store in KV with midhash256 + // File will be processed with full SHA-256 instead (see below) + } else { + // No collision or same file - safe to store + const finalMetadata = { + ...metadata, + filePath + }; + + // Store using nested key architecture + await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']); + } + } + } catch (metaError) { + console.error(`Failed to store metadata in KV:`, metaError); + } + } + + // STATE: Light processing complete, move to HASH processing queue + // If collision detected, skip VFS addition (virtualPath = undefined) + if (this.unifiedStateManager) { + if (collisionDetected) { + // Collision: Do NOT complete light processing with virtualPath + // This prevents file from being added to VFS + this.unifiedStateManager.completeLightProcessing( + filePath, + midHash256, + undefined // No virtualPath = not added to VFS + ); + } else { + // No collision: File becomes accessible in VFS + this.unifiedStateManager.completeLightProcessing( + filePath, + midHash256, + virtualPath + ); + } + // Start hash processing phase immediately + this.unifiedStateManager.startHashProcessing(filePath); + } + + // STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the + // container plugin "full-hash" in background queue - not here. + // Only midhash256 is computed in-process for fast file identification. + + // Mark processing as complete + metadata.processingStatus = 'complete'; + this.database.set(filePath, metadata); + + // STEP 5: Update metadata in KV with full hashes + if (this.kvClient) { + try { + // Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256 + let hashId: string; + if (collisionDetected && metadata['cid_sha2-256']) { + // Collision: Use full SHA-256 as primary ID + hashId = metadata['cid_sha2-256']; + console.log(` Using SHA-256 as primary ID for collision: ${hashId}`); + } else { + // Normal: Use midhash256 as primary ID + hashId = midHash256; + } + + if (hashId) { + // Prepare metadata with filePath added + const finalMetadata = { + ...metadata, + filePath + }; + + // Store using nested key architecture + await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']); + } + } catch (metaError) { + console.error(`Failed to store metadata in KV:`, metaError); + } + } + + // STATE: Hash processing complete, move to done + if (this.unifiedStateManager) { + this.unifiedStateManager.completeHashProcessing( + filePath, + midHash256, + virtualPath + ); + } + } catch (e) { + console.error(`Error processing file ${filePath}:`, e); + + // Mark processing as failed (complete hash processing with error) + if (this.unifiedStateManager) { + this.unifiedStateManager.completeHashProcessing( + filePath, + undefined, + undefined, + e instanceof Error ? e.message : String(e) + ); + } + } + } + + /** + * Light Processing Phase - Fast processing to make file accessible in VFS + * - Compute midhash256 (< 1s) + * - Extract metadata (filename, FFmpeg analysis) + * - Store in KV + * - File becomes accessible in VFS with midhash256 ID + * + * @deprecated Use TaskScheduler-based processing via WatchedFileProcessor.queueFile() instead. + * This method is kept for backward compatibility with StreamingPipeline. + */ + async processLightPhase(filePath: string, current: number, queueSize: number) { + try { + // STATE: Start LIGHT processing + if (this.unifiedStateManager) { + this.unifiedStateManager.startLightProcessing(filePath); + } + + const stats = await stat(filePath); + + // STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files) + let midHash256: string | undefined; + const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + + if (indexLine && indexLine.cid_midhash256) { + // Cache hit! Use cached midhash256 (no network I/O needed) + midHash256 = indexLine.cid_midhash256; + performanceMetrics.recordCacheHit('midhash256'); + } else { + // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives) + const computeStart = performance.now(); + const { computeMidHash256 } = await import('@metazla/meta-hash'); + midHash256 = await computeMidHash256(filePath); + const computeTime = Math.ceil(performance.now() - computeStart); + + // Record midhash256 computation time (excludes cache hits) + performanceMetrics.recordHashComputation('cid_midhash256', computeTime); + performanceMetrics.recordCacheMiss('midhash256'); + } + + // Initialize metadata with midhash256 + const metadata: FileMetadata = { + cid_midhash256: midHash256, + processingStatus: 'processing' + }; + + // STEP 2: Plugin metadata extraction is now handled by container plugins + // The TaskScheduler will dispatch tasks to container plugins asynchronously + + // Generate virtualPath from metadata (needed for VFS access) + let virtualPath: string | undefined; + try { + virtualPath = renamingRule(metadata as any, ''); + } catch (e) { + // If renamingRule fails, virtualPath remains undefined + } + + // STEP 3: Store metadata in KV (file becomes accessible in VFS) + if (this.kvClient) { + try { + const hashId = midHash256; // Use midhash256 as primary ID + + if (hashId) { + // Prepare metadata with filePath added + const finalMetadata = { + ...metadata, + filePath + }; + + // Store using nested key architecture + await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']); + } + } catch (metaError) { + console.error(`Failed to store metadata in KV:`, metaError); + } + } + + // Store in local database + this.database.set(filePath, metadata); + + // STATE: Light processing complete, move to HASH processing queue + // File is now accessible in VFS with midhash256 ID + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing( + filePath, + midHash256, + virtualPath + ); + } + } catch (e) { + console.error(`Error in light processing for ${filePath}:`, e); + + // Mark processing as failed + if (this.unifiedStateManager) { + this.unifiedStateManager.completeLightProcessing( + filePath, + '', + undefined, + e instanceof Error ? e.message : String(e) + ); + } + throw e; + } + } + + /** + * Hash Processing Phase - Background hash computation + * - Compute full hashes (SHA-256, SHA-1, MD5, CRC32) + * - Update KV with additional hash metadata + * - Mark processing as complete + * - Supports abortion for priority enforcement + * + * @deprecated Use the full-hash plugin via TaskScheduler instead. + * This method is kept for backward compatibility with StreamingPipeline. + */ + async processHashPhase(filePath: string, current: number, queueSize: number) { + // Create abort controller for this task + const abortController = new AbortController(); + this.runningHashTasks.set(filePath, abortController); + + try { + // STATE: Start HASH processing + if (this.unifiedStateManager) { + this.unifiedStateManager.startHashProcessing(filePath); + } + + const stats = await stat(filePath); + const metadata = this.database.get(filePath); + if (!metadata) { + throw new Error('Metadata not found - light processing may have failed'); + } + + const midHash256 = metadata.cid_midhash256; + if (!midHash256) { + throw new Error('midhash256 not found in metadata'); + } + + // Generate virtualPath (should already be done but ensure it's available) + let virtualPath: string | undefined; + try { + virtualPath = renamingRule(metadata as any, ''); + } catch (e) { + // If renamingRule fails, virtualPath remains undefined + } + + // Check if collision was detected during light processing + // If the midhash256 key exists in KV with a different file path, collision occurred + let collisionDetected = false; + if (this.kvClient) { + try { + const existingFilePath = await this.kvClient.getMetadata(midHash256, 'filePath'); + if (existingFilePath && existingFilePath !== filePath) { + collisionDetected = true; + console.log(` Collision detected in hash phase for ${filePath}`); + } + } catch (e) { + // Ignore errors - if key doesn't exist, no collision + } + } + + // Check if aborted before processing + if (abortController.signal.aborted) { + throw new Error('AbortError'); + } + + // STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the + // container plugin "full-hash" in background queue - not here. + // Only midhash256 is computed in-process for fast file identification. + + // Mark processing as complete + metadata.processingStatus = 'complete'; + this.database.set(filePath, metadata); + + // STEP 5: Update metadata in KV (full hashes will be added later by container plugin) + if (this.kvClient) { + try { + // Use midhash256 as primary ID + // Note: SHA-256 is computed later by the full-hash container plugin + const hashId = midHash256; + + if (hashId) { + // Prepare metadata with filePath added + const finalMetadata = { + ...metadata, + filePath + }; + + // Store using nested key architecture + await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']); + } + } catch (metaError) { + console.error(`Failed to store metadata in KV:`, metaError); + } + } + + // STATE: Hash processing complete, move to done + if (this.unifiedStateManager) { + this.unifiedStateManager.completeHashProcessing( + filePath, + midHash256, + virtualPath + ); + } + } catch (e) { + // Handle abort errors gracefully + if (e instanceof Error && (e.message === 'AbortError' || e.name === 'AbortError')) { + // Task was aborted for priority - re-queue it + console.log(`[Abort] Hash processing aborted for ${filePath} - will retry when idle`); + + // Don't mark as failed, just remove from running tasks + // The task will remain in hash queue and retry later + if (this.unifiedStateManager) { + // Move back to hash processing start (will retry) + this.unifiedStateManager.startHashProcessing(filePath); + } + } else { + console.error(`Error in hash processing for ${filePath}:`, e); + + // Mark hash processing as failed for real errors + if (this.unifiedStateManager) { + this.unifiedStateManager.completeHashProcessing( + filePath, + undefined, + undefined, + e instanceof Error ? e.message : String(e) + ); + } + } + throw e; + } finally { + // Always remove from running tasks + this.runningHashTasks.delete(filePath); + } + } + + async deleteFile(filePath: string) { + const metadata = this.database.get(filePath); + this.database.delete(filePath); + + // Delete from KV if configured (using nested key architecture) + if (this.kvClient && metadata) { + try { + const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5; + + // Delete all properties for this file using nested key architecture + if (hashId) { + const deletedCount = await this.kvClient.deleteMetadataFlat(hashId); + console.log(`Metadata deleted from KV: /file/${hashId} (${deletedCount} keys)`); + } + } catch (metaError) { + // Log error but don't fail the entire process + console.error(`Failed to delete metadata from KV:`, metaError); + } + } + } + + /** + * Set the KV client + * This is called by KVManager after successful connection + */ + setKVClient(client: IKVClient): void { + this.kvClient = client; + console.log('[FileProcessor] KV client set'); + } + + /** + * Get the KV client + */ + getKVClient(): IKVClient | null { + return this.kvClient; + } +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts b/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts new file mode 100644 index 0000000..e2fb83a --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts @@ -0,0 +1,31 @@ +import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash"; +import {HashIndexManager} from "@metazla/meta-hash"; +import {config} from "../../config/EnvConfig.js"; +import {targetHash, targetHashForIndex} from "../../config/TargetHash.js"; +import {stat} from "fs/promises"; + + +export const globalHashIndex = new HashIndexManager(config.INDEX_FOLDER_PATH, targetHashForIndex); +export const globalHashComputer = new HashComputerFile(targetHash); + +class SimpleFileToCid{ + + async getcid(filePath: string): Promise { + try { + const stats = await stat(filePath); + let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString()); + if(indexLine) { + return indexLine[CID_ALGORITHM_NAMES.sha256]; + }else { + let hashs = {}; + await globalHashComputer.computeMissingHash(filePath, hashs); + globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs); + return hashs[CID_ALGORITHM_NAMES.sha256]; + } + } catch (e) { + return null; + } + } +} + +export const simpleFileToCid = new SimpleFileToCid(); \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/fileProcessor/worker.ts b/packages/meta-sort-core/src/logic/fileProcessor/worker.ts new file mode 100644 index 0000000..d6b23d5 --- /dev/null +++ b/packages/meta-sort-core/src/logic/fileProcessor/worker.ts @@ -0,0 +1,20 @@ +import {FileProcessorInternal} from "./FileProcessorInternal.js"; +import {HashMeta} from "@metazla/meta-interface"; + +const fileProcessor = new FileProcessorInternal(); + +export interface Input { + filePath: string; + hash:HashMeta; +} + +export interface Output { + time: number; + metadata: HashMeta; + hashTimings?: Record; // Per-hash computation times in ms + pluginTimings?: Record; // Per-plugin processing times in ms +} + +export default async function processFile({ filePath,hash}: Input): Promise { + return fileProcessor.processFile(filePath,hash); +} \ No newline at end of file diff --git a/packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts b/packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts new file mode 100644 index 0000000..7d3b94d --- /dev/null +++ b/packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts @@ -0,0 +1,80 @@ +import {FileAnalyzerInterface} from "../fileProcessor/FileAnalyzerInterface.js"; +import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js"; +import type {IKVClient} from "../../kv/IKVClient.js"; +import {VirtualFileSystem} from "../../api/VirtualFileSystem.js"; +import {MetaDataToFolderStruct} from "../MetaDataToFolderStruct.js"; +import type {ContainerPluginScheduler} from "../../container-plugins/ContainerPluginScheduler.js"; + +/** + * Configuration for StreamingPipeline + */ +export interface PipelineConfig { + /** + * Supported file extensions (application-specific) + */ + supportedExtensions: Set; + + /** + * Concurrency for validation stage (extension + optional MIME check) + * Default: 32 (I/O bound - fast checks) + */ + validationConcurrency: number; + + /** + * Concurrency for fast queue stage (filename parsing, stats) + * Default: 16 (CPU/I/O balanced) + */ + fastQueueConcurrency: number; + + /** + * Concurrency for background queue stage (hash computation, FFmpeg) + * Default: 16 (CPU bound) + */ + backgroundQueueConcurrency: number; + + /** + * Enable strict MIME validation (reads file header) + * Default: false (skip for performance) + */ + strictMimeValidation: boolean; + + /** + * File processor (handles light and heavy processing) + */ + fileProcessor: FileAnalyzerInterface; + + /** + * State manager (tracks processing states) + */ + stateManager: UnifiedProcessingStateManager; + + /** + * Optional KV client for metadata storage + */ + kvClient?: IKVClient; + + /** + * Virtual filesystem (for instant file visibility) + */ + virtualFileSystem: VirtualFileSystem; + + /** + * Metadata to folder structure converter (for virtual path generation) + */ + metaDataToFolderStruct: MetaDataToFolderStruct; + + /** + * Optional container plugin scheduler for dispatching tasks to containerized plugins + */ + containerPluginScheduler?: ContainerPluginScheduler; +} + +/** + * Default pipeline configuration + */ +export const DEFAULT_PIPELINE_CONFIG: Partial = { + validationConcurrency: 32, + fastQueueConcurrency: 16, + backgroundQueueConcurrency: 16, + strictMimeValidation: false +}; diff --git a/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts new file mode 100644 index 0000000..7bcf5f3 --- /dev/null +++ b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts @@ -0,0 +1,512 @@ +import * as path from 'path'; +import PQueue from 'p-queue'; +import {PipelineConfig} from './PipelineConfig.js'; +import {withTimeoutAndRetry, TimeoutConfig} from '../../utils/TimeoutWrapper.js'; +import {performanceMetrics} from '../../metrics/PerformanceMetrics.js'; + +// Pub/sub channel for batched updates to meta-fuse +const UPDATE_CHANNEL = 'meta-sort:file:batch'; + +// Batch update interval (5 seconds) +const BATCH_INTERVAL_MS = 5000; + +interface BatchChange { + action: 'add' | 'update' | 'remove'; + hashId: string; +} + +/** + * Streaming Pipeline for file processing + * Files flow through two stages as they're discovered: + * Discovery → Validation → Processing (midhash256 + metadata) + * + * With midhash256 being instant (< 1s), files appear in VFS immediately + * + * Publishes batched updates to meta-fuse via Redis pub/sub every 5 seconds. + */ +export class StreamingPipeline { + private validationQueue: PQueue; + private fastQueue: PQueue; + private backgroundQueue: PQueue; + private config: PipelineConfig; + + private discoveredCount = 0; + private validatedCount = 0; + private fastProcessedCount = 0; + private backgroundProcessedCount = 0; + + // Batch buffer for pub/sub notifications + private batchBuffer: BatchChange[] = []; + private batchTimer: NodeJS.Timeout | null = null; + + // Retry configuration + private readonly MAX_PROCESSING_RETRIES = 10; + + // Fast queue timeout: Fast metadata extraction + midhash256 + private readonly fastQueueTimeoutConfig: TimeoutConfig = { + baseTimeout: 120000, // 2 minutes base timeout + timeoutMultiplier: 1.5, // Increase by 1.5x each retry + maxTimeout: 600000 // Max 10 minutes + }; + + // Background queue timeout: Full file hashing over NAS (can be very slow) + private readonly backgroundQueueTimeoutConfig: TimeoutConfig = { + baseTimeout: 7200000, // 2 hours base timeout (for NAS) + timeoutMultiplier: 1.5, // Increase by 1.5x each retry + maxTimeout: 14400000 // Max 4 hours + }; + + constructor(config: PipelineConfig) { + this.config = config; + + // Stage 1: Validation (high concurrency - fast I/O checks) + this.validationQueue = new PQueue({ + concurrency: config.validationConcurrency, + autoStart: true + }); + + // Stage 2: Fast Queue - metadata + midhash256 (high concurrency, fast) + this.fastQueue = new PQueue({ + concurrency: config.fastQueueConcurrency, + autoStart: true + }); + + // Stage 3: Background Queue - SHA-256 computation (lower concurrency, CPU-intensive) + this.backgroundQueue = new PQueue({ + concurrency: config.backgroundQueueConcurrency, + autoStart: true + }); + + console.log(`[Pipeline] Initialized with concurrency: validation=${config.validationConcurrency}, fastQueue=${config.fastQueueConcurrency}, backgroundQueue=${config.backgroundQueueConcurrency}`); + + // Start batch publish timer + this.startBatchTimer(); + } + + /** + * Start the batch publish timer + * Publishes accumulated changes to meta-fuse every 5 seconds + */ + private startBatchTimer(): void { + this.batchTimer = setInterval(() => { + this.flushBatch().catch(err => { + console.error('[Pipeline] Failed to flush batch:', err.message); + }); + }, BATCH_INTERVAL_MS); + } + + /** + * Stop the batch publish timer + */ + stopBatchTimer(): void { + if (this.batchTimer) { + clearInterval(this.batchTimer); + this.batchTimer = null; + } + } + + /** + * Queue a change for batched notification + */ + private queueChange(action: 'add' | 'update' | 'remove', hashId: string): void { + // Deduplicate: remove existing entries for same hashId + this.batchBuffer = this.batchBuffer.filter(c => c.hashId !== hashId); + this.batchBuffer.push({ action, hashId }); + } + + /** + * Flush the batch buffer and publish to Redis + */ + private async flushBatch(): Promise { + if (this.batchBuffer.length === 0) { + return; + } + + const changes = [...this.batchBuffer]; + this.batchBuffer = []; + + try { + const kvClient = this.config.kvClient; + if (kvClient && typeof (kvClient as any).publish === 'function') { + const message = JSON.stringify({ + timestamp: Date.now(), + changes + }); + await (kvClient as any).publish(UPDATE_CHANNEL, message); + console.log(`[Pipeline] Published batch update: ${changes.length} changes`); + } + } catch (error: any) { + // Put changes back in buffer on failure + this.batchBuffer = [...changes, ...this.batchBuffer]; + console.warn(`[Pipeline] Failed to publish batch: ${error.message}`); + } + } + + /** + * Start processing files from discovery stream + * Processing begins immediately as files are discovered + * + * @param discoveryStream - Async generator yielding file paths + */ + async start(discoveryStream: AsyncGenerator): Promise { + console.log('[Pipeline] Starting file processing stream...'); + const startTime = Date.now(); + + try { + // Consume discovery stream + for await (const filePath of discoveryStream) { + this.discoveredCount++; + + // Stage 1: Validation (fire and forget) + this.validationQueue.add(() => this.validateFile(filePath)) + .catch(err => console.error(`[Pipeline] Validation error for ${filePath}:`, err.message)); + + // Log progress every 1000 files + if (this.discoveredCount % 1000 === 0) { + this.logProgress(); + } + } + + const elapsed = Date.now() - startTime; + console.log(`[Pipeline] Discovery completed: ${this.discoveredCount} files found in ${elapsed}ms`); + console.log(`[Pipeline] Processing continues in background...`); + this.logProgress(); + } catch (error: any) { + console.error(`[Pipeline] Error in discovery stream:`, error.message); + throw error; + } + } + + /** + * Stage 1: Validation + * - Quick extension check (no file I/O) + * - Optional MIME validation (reads file header) + * - Mark file as pending + */ + private async validateFile(filePath: string): Promise { + try { + // Extension check (instant, no I/O) + const ext = path.extname(filePath).toLowerCase(); + if (!this.config.supportedExtensions.has(ext)) { + // Skip unsupported file types + return; + } + + // Optional strict MIME validation + // Note: Skipped for performance - extension check is sufficient + // If needed, could add MIME validation via external library + + // Mark as pending + this.config.stateManager.addPending(filePath); + this.validatedCount++; + + // Stage 2: Fast Queue - metadata + midhash256 (fire and forget) + this.fastQueue.add(() => this.processLightPhase(filePath)) + .catch(err => console.error(`[Pipeline] Fast queue error for ${filePath}:`, err.message)); + } catch (error: any) { + console.error(`[Pipeline] Validation failed for ${filePath}:`, error.message); + } + } + + /** + * Stage 2: Fast Queue Phase + * - Extract metadata (filename parsing, FFmpeg analysis) + * - Compute midhash256 (< 1s) + * - Store to KV (file becomes accessible in VFS) + * - Queue for background processing + * - Includes timeout and retry logic + */ + private async processLightPhase(filePath: string): Promise { + try { + // Get current retry count + const retryCount = this.config.stateManager.getRetryCount(filePath); + + // Call light processing with timeout + const current = this.fastProcessedCount + 1; + const queueSize = this.discoveredCount; + + await withTimeoutAndRetry( + () => this.config.fileProcessor.processLightPhase(filePath, current, queueSize), + retryCount, + this.fastQueueTimeoutConfig, + `Fast queue processing file ${filePath}` + ); + + // After light processing, add to VFS + const metadata = this.config.fileProcessor.getDatabase().get(filePath); + if (metadata && metadata.cid_midhash256) { + // Generate virtual path from metadata + const virtualPath = this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath); + + if (virtualPath) { + // Add to VFS immediately (file appears with permanent midhash256 ID) + this.config.virtualFileSystem.addFile(virtualPath, filePath, metadata); + //console.log(`✓ File added to VFS: ${filePath} → ${virtualPath} (midhash256: ${metadata.cid_midhash256})`); + + // Queue for pub/sub notification to meta-fuse + if (metadata.cid_midhash256) { + this.queueChange('add', metadata.cid_midhash256); + } + + // Notify Stremio addon to add video (incremental update like FUSE) + if (metadata.cid_midhash256) { + this.notifyStremioAdd(metadata.cid_midhash256).catch(err => { + // Silent fail - Stremio addon may not be running + }); + } + } else { + // File cannot be added to VFS (e.g., unsupported type, missing title) + console.log(`⚠ File skipped from VFS (no virtual path): ${filePath}`); + } + + // Dispatch tasks to container plugins (fire and forget) + this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata); + + // Stage 3: Queue for background processing (SHA-256 computation) + this.backgroundQueue.add(() => this.processHashPhase(filePath)) + .catch(err => console.error(`[Pipeline] Background queue error for ${filePath}:`, err.message)); + } + + this.fastProcessedCount++; + } catch (error: any) { + // Handle timeout or processing error + const retryCount = this.config.stateManager.getRetryCount(filePath); + + if (retryCount < this.MAX_PROCESSING_RETRIES) { + // Retry: move back to pending queue with incremented retry count + const success = this.config.stateManager.retryProcessing(filePath); + if (success) { + console.warn(`[Pipeline] Fast queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying with longer timeout. Error: ${error.message}`); + + // Re-queue for fast processing + this.fastQueue.add(() => this.processLightPhase(filePath)) + .catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message)); + return; + } + } + + // Max retries exceeded or retry failed - mark as permanently failed + console.error(`[Pipeline] Fast queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`); + + // Record failed file in metrics + performanceMetrics.recordFailedFile( + filePath, + error.message || 'Unknown error', + retryCount + 1, + 'processing' + ); + + // Remove from processing + this.config.stateManager.removeFile(filePath); + } + } + + /** + * Stage 3: Background Queue Phase + * - Compute full hashes (SHA-256, SHA-1, MD5, CRC32) + * - Update KV with additional hash metadata + * - Mark processing as complete + * - Includes timeout and retry logic + */ + private async processHashPhase(filePath: string): Promise { + try { + // Get current retry count + const retryCount = this.config.stateManager.getRetryCount(filePath); + + // Call hash processing with timeout + const current = this.backgroundProcessedCount + 1; + const queueSize = this.discoveredCount; + + await withTimeoutAndRetry( + () => this.config.fileProcessor.processHashPhase(filePath, current, queueSize), + retryCount, + this.backgroundQueueTimeoutConfig, + `Background queue processing file ${filePath}` + ); + + this.backgroundProcessedCount++; + } catch (error: any) { + // Handle timeout or processing error + const retryCount = this.config.stateManager.getRetryCount(filePath); + + if (retryCount < this.MAX_PROCESSING_RETRIES) { + // Retry: move back to background queue with incremented retry count + console.warn(`[Pipeline] Background queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying. Error: ${error.message}`); + + // Re-queue for background processing + this.backgroundQueue.add(() => this.processHashPhase(filePath)) + .catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message)); + return; + } + + // Max retries exceeded - mark as permanently failed + console.error(`[Pipeline] Background queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`); + + // Record failed file in metrics + performanceMetrics.recordFailedFile( + filePath, + error.message || 'Unknown error', + retryCount + 1, + 'hash' + ); + + // Complete with error + this.config.stateManager.completeHashProcessing(filePath, undefined, undefined, error.message); + } + } + + /** + * Handle file added event (from chokidar) + */ + async handleFileAdded(filePath: string): Promise { + + // Process through validation stage + this.validationQueue.add(() => this.validateFile(filePath)) + .catch(err => console.error(`[Pipeline] Error processing added file ${filePath}:`, err.message)); + } + + /** + * Handle file changed event (from chokidar) + */ + async handleFileChanged(filePath: string): Promise { + + // Reprocess through validation stage + this.config.stateManager.removeFile(filePath); + this.validationQueue.add(() => this.validateFile(filePath)) + .catch(err => console.error(`[Pipeline] Error processing changed file ${filePath}:`, err.message)); + } + + /** + * Handle file deleted event (from chokidar) + */ + async handleFileDeleted(filePath: string): Promise { + // Get hashId before removal for pub/sub notification + const metadata = this.config.fileProcessor.getDatabase().get(filePath); + const hashId = (metadata as any)?.cid_midhash256; + + // Remove from state and call cleanup + if (this.config.fileProcessor.deleteFile) { + this.config.fileProcessor.deleteFile(filePath); + } + this.config.stateManager.removeFile(filePath); + + // Queue for pub/sub notification to meta-fuse + if (hashId) { + this.queueChange('remove', hashId); + } + } + + /** + * Get pipeline statistics + */ + getStats() { + return { + discovered: this.discoveredCount, + validated: this.validatedCount, + fastProcessed: this.fastProcessedCount, + backgroundProcessed: this.backgroundProcessedCount, + queues: { + validation: { + pending: this.validationQueue.pending, + size: this.validationQueue.size + }, + fastQueue: { + pending: this.fastQueue.pending, + size: this.fastQueue.size + }, + backgroundQueue: { + pending: this.backgroundQueue.pending, + size: this.backgroundQueue.size + } + }, + state: this.config.stateManager.getSnapshot() + }; + } + + /** + * Log current pipeline progress + */ + private logProgress(): void { + const stats = this.getStats(); + } + + /** + * Notify Stremio addon to add video (incremental update like FUSE) + */ + private async notifyStremioAdd(hashId: string): Promise { + try { + const response = await fetch('http://localhost:7000/add', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ hashId }), + signal: AbortSignal.timeout(5000) // 5 second timeout + }); + + if (!response.ok) { + // Silent fail - Stremio may not be running + return; + } + } catch (error: any) { + // Silent fail - Stremio addon may not be running or network issue + // This is expected during development when Stremio isn't started + } + } + + /** + * Notify Stremio addon to remove video (incremental update like FUSE) + */ + private async notifyStremioRemove(hashId: string): Promise { + try { + const response = await fetch('http://localhost:7000/remove', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ hashId }), + signal: AbortSignal.timeout(5000) // 5 second timeout + }); + + if (!response.ok) { + // Silent fail - Stremio may not be running + return; + } + } catch (error: any) { + // Silent fail - Stremio addon may not be running or network issue + } + } + + /** + * Set the container plugin scheduler (called after initialization) + */ + setContainerPluginScheduler(scheduler: import('../../container-plugins/ContainerPluginScheduler.js').ContainerPluginScheduler): void { + (this.config as any).containerPluginScheduler = scheduler; + console.log('[Pipeline] Container plugin scheduler connected'); + } + + /** + * Dispatch tasks to container plugins for metadata extraction + * This is fire-and-forget - container plugins write results directly to meta-core + */ + private dispatchContainerPluginTasks(filePath: string, hashId: string, existingMeta: Record): void { + const scheduler = this.config.containerPluginScheduler; + if (!scheduler) { + return; // Container plugins not configured + } + + try { + // Convert metadata to flat string map for plugin API + const metaFlat: Record = {}; + for (const [key, value] of Object.entries(existingMeta)) { + if (value != null) { + metaFlat[key] = String(value); + } + } + + // Dispatch tasks to all enabled plugins + scheduler.dispatchAllPlugins(hashId, filePath, metaFlat) + .catch(err => { + console.error(`[Pipeline] Failed to dispatch container plugin tasks for ${filePath}:`, err.message); + }); + } catch (error: any) { + console.error(`[Pipeline] Error dispatching container plugin tasks for ${filePath}:`, error.message); + } + } +} diff --git a/packages/meta-sort-core/src/logic/pipeline/index.ts b/packages/meta-sort-core/src/logic/pipeline/index.ts new file mode 100644 index 0000000..fe4f986 --- /dev/null +++ b/packages/meta-sort-core/src/logic/pipeline/index.ts @@ -0,0 +1,2 @@ +export { StreamingPipeline } from './StreamingPipeline.js'; +export { PipelineConfig, DEFAULT_PIPELINE_CONFIG } from './PipelineConfig.js'; diff --git a/packages/meta-sort-core/src/logic/tool/MetadataMerger.ts b/packages/meta-sort-core/src/logic/tool/MetadataMerger.ts new file mode 100644 index 0000000..bd70d24 --- /dev/null +++ b/packages/meta-sort-core/src/logic/tool/MetadataMerger.ts @@ -0,0 +1,36 @@ +import lodash from 'lodash'; +import {HashMeta} from "@metazla/meta-interface"; + +/** + * Merge the new metadata into the old metadata + * @param existingMetadata + * @param newMetadata + * @param id + * @returns {boolean} true if the metadata has changed + */ +export function mergeMetadata(existingMetadata: any, newMetadata: any, id?:string): boolean{ + let hasChanged = false; + const mergeLogic = (existingVal:any, newVal:any, key:string) => { + if ((lodash.isObject(existingVal) || lodash.isObject(newVal)) || + (lodash.isArray(existingVal) || lodash.isArray(newVal))) { + // Recursively merge objects and arrays + return lodash.mergeWith(existingVal, newVal, mergeLogic); + } + + if (existingVal) { + if (!lodash.isEqual(""+existingVal, ""+newVal)){ + //conflict on primitive types + console.warn(`Conflict found for '${id}'. Existing value: '${existingVal}', New value: '${newVal}'.`); + } + // Keep the original value (especially in case of conflict) + return existingVal; + }else { + if(newVal !== undefined) { + hasChanged = true; + } + return newVal; + } + }; + lodash.mergeWith(existingMetadata, newMetadata, mergeLogic); + return hasChanged; +} diff --git a/packages/meta-sort-core/src/metrics/PerformanceMetrics.ts b/packages/meta-sort-core/src/metrics/PerformanceMetrics.ts new file mode 100644 index 0000000..334f20f --- /dev/null +++ b/packages/meta-sort-core/src/metrics/PerformanceMetrics.ts @@ -0,0 +1,582 @@ +/** + * Performance metrics tracker for meta-mesh + * Tracks detailed performance statistics and timing information + */ + +export interface MetricEntry { + timestamp: number; + value: number; + label?: string; +} + +export interface FailedFileEntry { + filePath: string; + reason: string; + timestamp: number; + retryCount: number; + stage: 'metadata' | 'hash' | 'processing'; +} + +export interface HashTimingStats { + total: number; + count: number; + average: number; + min: number; + max: number; +} + +export interface PluginTimingStats { + total: number; + count: number; + average: number; + min: number; + max: number; +} + +export interface PerformanceMetricsData { + // File processing metrics + totalFilesProcessed: number; + averageFileProcessingTime: number; + minFileProcessingTime: number; + maxFileProcessingTime: number; + recentFileProcessingTimes: MetricEntry[]; + + // Metadata (light processing) metrics + totalMetadataProcessed: number; + averageMetadataProcessingTime: number; + minMetadataProcessingTime: number; + maxMetadataProcessingTime: number; + + // Hash processing metrics + totalHashProcessed: number; + averageHashProcessingTime: number; + minHashProcessingTime: number; + maxHashProcessingTime: number; + + // Per-hash-algorithm timing metrics (excludes cache hits) + hashComputationTimes: Record; + + // Per-plugin timing metrics + pluginProcessingTimes: Record; + + // VFS update metrics + totalVFSUpdates: number; + averageVFSUpdateTime: number; + lastVFSUpdateTime: number; + recentVFSUpdateTimes: MetricEntry[]; + + // Duplicate detection metrics + averageDuplicateDetectionTime: number; + lastDuplicateDetectionTime: number; + totalDuplicatesFound: number; + + // Virtual structure generation metrics + averageVirtualStructureTime: number; + lastVirtualStructureTime: number; + + // Failed files tracking + failedFiles: FailedFileEntry[]; + totalFailedFiles: number; + totalMetadataFailures: number; + totalHashFailures: number; + + // System metrics + uptime: number; + startTime: number; + lastActivityTime: number; + + // Memory and performance + memoryUsage?: NodeJS.MemoryUsage; + + // Processing rate + filesPerSecond: number; + totalProcessingTime: number; + + // Cache metrics + cacheHits: Record; + cacheMisses: Record; + cacheHitRate: Record; +} + +export class PerformanceMetrics { + private startTime: number; + private lastActivityTime: number; + + // File processing tracking - running statistics + private recentFileProcessing: MetricEntry[] = []; + private totalFilesProcessed: number = 0; + private fileProcessingSum: number = 0; + private fileProcessingMin: number = Infinity; + private fileProcessingMax: number = 0; + + // Metadata (light processing) tracking - running statistics + private totalMetadataProcessed: number = 0; + private metadataProcessingSum: number = 0; + private metadataProcessingMin: number = Infinity; + private metadataProcessingMax: number = 0; + + // Hash processing tracking - running statistics + private totalHashProcessed: number = 0; + private hashProcessingSum: number = 0; + private hashProcessingMin: number = Infinity; + private hashProcessingMax: number = 0; + + // VFS update tracking - running statistics + private recentVFSUpdates: MetricEntry[] = []; + private totalVFSUpdates: number = 0; + private vfsUpdateSum: number = 0; + private lastVFSUpdateTime: number = 0; + + // Duplicate detection tracking - running statistics + private totalDuplicatesFound: number = 0; + private duplicateDetectionSum: number = 0; + private duplicateDetectionCount: number = 0; + private lastDuplicateDetectionTime: number = 0; + + // Virtual structure generation tracking - running statistics + private virtualStructureSum: number = 0; + private virtualStructureCount: number = 0; + private lastVirtualStructureTime: number = 0; + + // Failed files tracking + private failedFiles: FailedFileEntry[] = []; + private totalMetadataFailures: number = 0; + private totalHashFailures: number = 0; + + // Cache metrics tracking + private cacheHits: Record = {}; + private cacheMisses: Record = {}; + + // Per-hash-algorithm timing tracking (excludes cache hits) + private hashTimings: Record = {}; + + // Per-plugin timing tracking + private pluginTimings: Record = {}; + + // Configuration + private maxRecentEntries: number = 100; + private maxFailedFilesHistory: number = 50; + + constructor() { + this.startTime = Date.now(); + this.lastActivityTime = this.startTime; + } + + /** + * Record a file processing event + */ + recordFileProcessing(filePath: string, timeMs: number): void { + // Update running statistics + this.fileProcessingSum += timeMs; + this.fileProcessingMin = Math.min(this.fileProcessingMin, timeMs); + this.fileProcessingMax = Math.max(this.fileProcessingMax, timeMs); + this.totalFilesProcessed++; + + // Keep recent list for display only + this.recentFileProcessing.push({ + timestamp: Date.now(), + value: timeMs, + label: filePath + }); + + if (this.recentFileProcessing.length > this.maxRecentEntries) { + this.recentFileProcessing.shift(); + } + + this.lastActivityTime = Date.now(); + } + + /** + * Record a metadata (light processing) event + */ + recordMetadataProcessing(timeMs: number): void { + // Update running statistics + this.metadataProcessingSum += timeMs; + this.metadataProcessingMin = Math.min(this.metadataProcessingMin, timeMs); + this.metadataProcessingMax = Math.max(this.metadataProcessingMax, timeMs); + this.totalMetadataProcessed++; + + this.lastActivityTime = Date.now(); + } + + /** + * Record a hash processing event + */ + recordHashProcessing(timeMs: number): void { + // Update running statistics + this.hashProcessingSum += timeMs; + this.hashProcessingMin = Math.min(this.hashProcessingMin, timeMs); + this.hashProcessingMax = Math.max(this.hashProcessingMax, timeMs); + this.totalHashProcessed++; + + this.lastActivityTime = Date.now(); + } + + /** + * Record a VFS update event + */ + recordVFSUpdate(timeMs: number): void { + // Update running statistics + this.vfsUpdateSum += timeMs; + this.lastVFSUpdateTime = timeMs; + this.totalVFSUpdates++; + + // Keep recent list for display only + this.recentVFSUpdates.push({ + timestamp: Date.now(), + value: timeMs + }); + + if (this.recentVFSUpdates.length > this.maxRecentEntries) { + this.recentVFSUpdates.shift(); + } + + this.lastActivityTime = Date.now(); + } + + /** + * Record a duplicate detection event + */ + recordDuplicateDetection(timeMs: number, duplicatesFound: number): void { + // Update running statistics + this.duplicateDetectionSum += timeMs; + this.duplicateDetectionCount++; + this.lastDuplicateDetectionTime = timeMs; + this.totalDuplicatesFound += duplicatesFound; + + this.lastActivityTime = Date.now(); + } + + /** + * Record a virtual structure generation event + */ + recordVirtualStructureGeneration(timeMs: number): void { + // Update running statistics + this.virtualStructureSum += timeMs; + this.virtualStructureCount++; + this.lastVirtualStructureTime = timeMs; + + this.lastActivityTime = Date.now(); + } + + /** + * Record a cache hit event + */ + recordCacheHit(cacheType: string): void { + if (!this.cacheHits[cacheType]) { + this.cacheHits[cacheType] = 0; + } + this.cacheHits[cacheType]++; + this.lastActivityTime = Date.now(); + } + + /** + * Record a cache miss event + */ + recordCacheMiss(cacheType: string): void { + if (!this.cacheMisses[cacheType]) { + this.cacheMisses[cacheType] = 0; + } + this.cacheMisses[cacheType]++; + this.lastActivityTime = Date.now(); + } + + /** + * Record hash computation time for a specific algorithm (excludes cache hits) + * @param hashAlgorithm - Hash algorithm name (e.g., 'cid_sha2-256', 'cid_midhash256') + * @param timeMs - Time taken to compute the hash in milliseconds + */ + recordHashComputation(hashAlgorithm: string, timeMs: number): void { + if (!this.hashTimings[hashAlgorithm]) { + this.hashTimings[hashAlgorithm] = { + sum: 0, + count: 0, + min: Infinity, + max: 0 + }; + } + + const stats = this.hashTimings[hashAlgorithm]; + stats.sum += timeMs; + stats.count++; + stats.min = Math.min(stats.min, timeMs); + stats.max = Math.max(stats.max, timeMs); + + this.lastActivityTime = Date.now(); + } + + /** + * Record plugin processing time + * @param pluginName - Name of the plugin + * @param timeMs - Time taken by the plugin in milliseconds + */ + recordPluginProcessing(pluginName: string, timeMs: number): void { + if (!this.pluginTimings[pluginName]) { + this.pluginTimings[pluginName] = { + sum: 0, + count: 0, + min: Infinity, + max: 0 + }; + } + + const stats = this.pluginTimings[pluginName]; + stats.sum += timeMs; + stats.count++; + stats.min = Math.min(stats.min, timeMs); + stats.max = Math.max(stats.max, timeMs); + + this.lastActivityTime = Date.now(); + } + + /** + * Record a failed file processing event + */ + recordFailedFile(filePath: string, reason: string, retryCount: number, stage: 'metadata' | 'hash' | 'processing'): void { + const failedEntry: FailedFileEntry = { + filePath, + reason, + timestamp: Date.now(), + retryCount, + stage + }; + + // Add to failed files list (most recent first) + this.failedFiles.unshift(failedEntry); + + // Trim history if needed + if (this.failedFiles.length > this.maxFailedFilesHistory) { + this.failedFiles = this.failedFiles.slice(0, this.maxFailedFilesHistory); + } + + // Update failure counters + if (stage === 'metadata') { + this.totalMetadataFailures++; + } else { + this.totalHashFailures++; + } + + this.lastActivityTime = Date.now(); + } + + /** + * Get all metrics data - O(1) complexity using cached running statistics + */ + getMetrics(): PerformanceMetricsData & { + // UI compatibility aliases + totalFilesDiscovered: number; + totalFilesFailed: number; + averageLightProcessingMs: number; + averageHashProcessingMs: number; + } { + const now = Date.now(); + const uptime = now - this.startTime; + const totalFailedFiles = this.totalMetadataFailures + this.totalHashFailures; + const avgMetadata = this.totalMetadataProcessed > 0 + ? this.metadataProcessingSum / this.totalMetadataProcessed + : 0; + const avgHash = this.totalHashProcessed > 0 + ? this.hashProcessingSum / this.totalHashProcessed + : 0; + + return { + // File processing metrics - O(1) from running statistics + totalFilesProcessed: this.totalFilesProcessed, + + // UI compatibility aliases + totalFilesDiscovered: this.totalFilesProcessed + totalFailedFiles, + totalFilesFailed: totalFailedFiles, + averageLightProcessingMs: avgMetadata, + averageHashProcessingMs: avgHash, + averageFileProcessingTime: this.totalFilesProcessed > 0 + ? this.fileProcessingSum / this.totalFilesProcessed + : 0, + minFileProcessingTime: this.fileProcessingMin === Infinity ? 0 : this.fileProcessingMin, + maxFileProcessingTime: this.fileProcessingMax, + recentFileProcessingTimes: this.recentFileProcessing.slice(-20), // Last 20 files + + // Metadata (light processing) metrics - O(1) from running statistics + totalMetadataProcessed: this.totalMetadataProcessed, + averageMetadataProcessingTime: this.totalMetadataProcessed > 0 + ? this.metadataProcessingSum / this.totalMetadataProcessed + : 0, + minMetadataProcessingTime: this.metadataProcessingMin === Infinity ? 0 : this.metadataProcessingMin, + maxMetadataProcessingTime: this.metadataProcessingMax, + + // Hash processing metrics - O(1) from running statistics + totalHashProcessed: this.totalHashProcessed, + averageHashProcessingTime: this.totalHashProcessed > 0 + ? this.hashProcessingSum / this.totalHashProcessed + : 0, + minHashProcessingTime: this.hashProcessingMin === Infinity ? 0 : this.hashProcessingMin, + maxHashProcessingTime: this.hashProcessingMax, + + // Per-hash-algorithm timing metrics (excludes cache hits) + hashComputationTimes: Object.keys(this.hashTimings).reduce((acc, hashAlgorithm) => { + const stats = this.hashTimings[hashAlgorithm]; + acc[hashAlgorithm] = { + total: stats.sum, + count: stats.count, + average: stats.count > 0 ? stats.sum / stats.count : 0, + min: stats.min === Infinity ? 0 : stats.min, + max: stats.max + }; + return acc; + }, {} as Record), + + // Per-plugin timing metrics + pluginProcessingTimes: Object.keys(this.pluginTimings).reduce((acc, pluginName) => { + const stats = this.pluginTimings[pluginName]; + acc[pluginName] = { + total: stats.sum, + count: stats.count, + average: stats.count > 0 ? stats.sum / stats.count : 0, + min: stats.min === Infinity ? 0 : stats.min, + max: stats.max + }; + return acc; + }, {} as Record), + + // VFS update metrics - O(1) from running statistics + totalVFSUpdates: this.totalVFSUpdates, + averageVFSUpdateTime: this.totalVFSUpdates > 0 + ? this.vfsUpdateSum / this.totalVFSUpdates + : 0, + lastVFSUpdateTime: this.lastVFSUpdateTime, + recentVFSUpdateTimes: this.recentVFSUpdates.slice(-10), // Last 10 updates + + // Duplicate detection metrics - O(1) from running statistics + averageDuplicateDetectionTime: this.duplicateDetectionCount > 0 + ? this.duplicateDetectionSum / this.duplicateDetectionCount + : 0, + lastDuplicateDetectionTime: this.lastDuplicateDetectionTime, + totalDuplicatesFound: this.totalDuplicatesFound, + + // Virtual structure generation metrics - O(1) from running statistics + averageVirtualStructureTime: this.virtualStructureCount > 0 + ? this.virtualStructureSum / this.virtualStructureCount + : 0, + lastVirtualStructureTime: this.lastVirtualStructureTime, + + // Failed files tracking + failedFiles: this.failedFiles, + totalFailedFiles: this.totalMetadataFailures + this.totalHashFailures, + totalMetadataFailures: this.totalMetadataFailures, + totalHashFailures: this.totalHashFailures, + + // System metrics + uptime: uptime, + startTime: this.startTime, + lastActivityTime: this.lastActivityTime, + + // Memory usage + memoryUsage: process.memoryUsage(), + + // Processing rate + filesPerSecond: uptime > 0 ? (this.totalFilesProcessed / (uptime / 1000)) : 0, + totalProcessingTime: this.fileProcessingSum, + + // Cache metrics + cacheHits: { ...this.cacheHits }, + cacheMisses: { ...this.cacheMisses }, + cacheHitRate: Object.keys(this.cacheHits).reduce((acc, key) => { + const hits = this.cacheHits[key] || 0; + const misses = this.cacheMisses[key] || 0; + const total = hits + misses; + acc[key] = total > 0 ? (hits / total) * 100 : 0; + return acc; + }, {} as Record) + }; + } + + /** + * Get timing stats for a specific plugin + * @param pluginId - Plugin identifier + * @returns Plugin timing stats or null if no data + */ + getPluginTiming(pluginId: string): PluginTimingStats | null { + const stats = this.pluginTimings[pluginId]; + if (!stats || stats.count === 0) { + return null; + } + return { + total: stats.sum, + count: stats.count, + average: stats.sum / stats.count, + min: stats.min === Infinity ? 0 : stats.min, + max: stats.max + }; + } + + /** + * Reset all metrics + */ + reset(): void { + // Reset file processing metrics + this.recentFileProcessing = []; + this.totalFilesProcessed = 0; + this.fileProcessingSum = 0; + this.fileProcessingMin = Infinity; + this.fileProcessingMax = 0; + + // Reset metadata processing metrics + this.totalMetadataProcessed = 0; + this.metadataProcessingSum = 0; + this.metadataProcessingMin = Infinity; + this.metadataProcessingMax = 0; + + // Reset hash processing metrics + this.totalHashProcessed = 0; + this.hashProcessingSum = 0; + this.hashProcessingMin = Infinity; + this.hashProcessingMax = 0; + + // Reset VFS update metrics + this.recentVFSUpdates = []; + this.totalVFSUpdates = 0; + this.vfsUpdateSum = 0; + this.lastVFSUpdateTime = 0; + + // Reset duplicate detection metrics + this.totalDuplicatesFound = 0; + this.duplicateDetectionSum = 0; + this.duplicateDetectionCount = 0; + this.lastDuplicateDetectionTime = 0; + + // Reset virtual structure generation metrics + this.virtualStructureSum = 0; + this.virtualStructureCount = 0; + this.lastVirtualStructureTime = 0; + + // Reset failed files tracking + this.failedFiles = []; + this.totalMetadataFailures = 0; + this.totalHashFailures = 0; + + // Reset cache metrics + this.cacheHits = {}; + this.cacheMisses = {}; + + // Reset hash computation timing + this.hashTimings = {}; + + // Reset plugin processing timing + this.pluginTimings = {}; + + this.startTime = Date.now(); + this.lastActivityTime = this.startTime; + } +} + +// Global singleton instance +export const performanceMetrics = new PerformanceMetrics(); diff --git a/packages/meta-sort-core/src/plugin-engine/PluginContext.ts b/packages/meta-sort-core/src/plugin-engine/PluginContext.ts new file mode 100644 index 0000000..587a26e --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/PluginContext.ts @@ -0,0 +1,392 @@ +/** + * Plugin Context Implementation + * + * Provides KV store, cache, and logger implementations for plugins. + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { existsSync, mkdirSync } from 'fs'; +import type { MetadataNode } from '@metazla/meta-interface'; +import type { + PluginKVStore, + PluginCache, + PluginLogger, + PluginContext, + PluginLifecycleContext, +} from './types.js'; + +// ============================================================================= +// KV Store Implementation +// ============================================================================= + +/** + * In-memory KV store implementation with path-style keys + */ +export class KVStore implements PluginKVStore { + private data: Map = new Map(); + + constructor(initialData?: Map | Record) { + if (initialData) { + if (initialData instanceof Map) { + this.data = new Map(initialData); + } else { + for (const [key, value] of Object.entries(initialData)) { + if (typeof value === 'string') { + this.data.set(key, value); + } + } + } + } + } + + get(key: string): string | undefined { + return this.data.get(key); + } + + set(key: string, value: string): void { + if (key === undefined || key === null || key === '') { + return; + } + if (value === undefined || value === null) { + return; + } + this.data.set(key, String(value)); + } + + delete(key: string): void { + this.data.delete(key); + } + + keys(prefix?: string): string[] { + const allKeys = Array.from(this.data.keys()); + if (!prefix) { + return allKeys; + } + return allKeys.filter(k => k.startsWith(prefix)); + } + + entries(prefix?: string): Array<[string, string]> { + const allEntries = Array.from(this.data.entries()); + if (!prefix) { + return allEntries; + } + return allEntries.filter(([k]) => k.startsWith(prefix)); + } + + /** + * Get all data as a plain object (for serialization) + */ + toObject(): Record { + const obj: Record = {}; + this.data.forEach((value, key) => { + obj[key] = value; + }); + return obj; + } + + /** + * Get the underlying Map + */ + toMap(): Map { + return new Map(this.data); + } + + /** + * Clear all data + */ + clear(): void { + this.data.clear(); + } +} + +// ============================================================================= +// MetadataNode KV Store Adapter +// ============================================================================= + +/** + * KV store that wraps MetadataNode from @metazla/meta-interface. + * This ensures plugins output the same nested object format as the old processors. + * + * Path conversion: + * - 'fileType' -> metadata.at('fileType') + * - 'fileinfo/duration' -> metadata.at('fileinfo').at('duration') + * - 'fileinfo/streamdetails/video/0/codec' -> metadata.at('fileinfo').at('streamdetails').at('video').at('0').at('codec') + */ +export class MetadataNodeKVStore implements PluginKVStore { + private metadata: MetadataNode; + + constructor(metadata: MetadataNode) { + this.metadata = metadata; + } + + /** + * Navigate to a nested node using path segments + */ + private navigate(key: string): MetadataNode { + const parts = key.split('/'); + let node = this.metadata; + for (const part of parts) { + node = node.at(part); + } + return node; + } + + get(key: string): string | undefined { + try { + const value = this.navigate(key).get(); + if (value === undefined || value === null) { + return undefined; + } + if (typeof value === 'string') { + return value; + } + if (typeof value === 'object') { + // Return undefined for objects (indicates path to nested data) + return undefined; + } + return String(value); + } catch { + return undefined; + } + } + + set(key: string, value: string): void { + if (key === undefined || key === null || key === '') { + return; + } + if (value === undefined || value === null) { + return; + } + try { + this.navigate(key).set(String(value)); + } catch (e) { + console.error(`[MetadataNodeKVStore] Error setting ${key}:`, e); + } + } + + delete(key: string): void { + // MetadataNode doesn't have a delete method, so we set to undefined + try { + this.navigate(key).set(undefined as any); + } catch { + // Ignore errors + } + } + + /** + * Get keys is not easily supported by MetadataNode. + * This returns an empty array - plugins should track their own keys if needed. + */ + keys(prefix?: string): string[] { + // MetadataNode doesn't expose iteration over keys + // Plugins that need to iterate should use the metadata object directly + console.warn('[MetadataNodeKVStore] keys() not supported - use metadata.get() for object inspection'); + return []; + } + + entries(prefix?: string): Array<[string, string]> { + console.warn('[MetadataNodeKVStore] entries() not supported - use metadata.get() for object inspection'); + return []; + } + + /** + * Get the underlying MetadataNode + */ + getMetadataNode(): MetadataNode { + return this.metadata; + } + + /** + * Add a value to a set (for RecordSet fields like genres, tags) + */ + add(key: string, value: string): void { + if (key === undefined || key === null || key === '') { + return; + } + if (value === undefined || value === null) { + return; + } + try { + this.navigate(key).add(value); + } catch (e) { + console.error(`[MetadataNodeKVStore] Error adding to ${key}:`, e); + } + } + + /** + * Get full object at a path (for inspecting nested structures) + */ + getObject(key: string): any { + try { + return this.navigate(key).get(); + } catch { + return undefined; + } + } +} + +// ============================================================================= +// Cache Implementation +// ============================================================================= + +/** + * File-based cache implementation for plugins + */ +export class PluginCacheImpl implements PluginCache { + private cacheDir: string; + private initialized: boolean = false; + + constructor(cacheDir: string) { + this.cacheDir = cacheDir; + } + + private ensureDir(): void { + if (!this.initialized) { + if (!existsSync(this.cacheDir)) { + mkdirSync(this.cacheDir, { recursive: true }); + } + this.initialized = true; + } + } + + getPath(filename: string): string { + this.ensureDir(); + return path.join(this.cacheDir, filename); + } + + async readJson(filename: string): Promise { + try { + const filePath = this.getPath(filename); + const data = await fs.readFile(filePath, 'utf8'); + return JSON.parse(data) as T; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return null; + } + throw error; + } + } + + async writeJson(filename: string, data: unknown): Promise { + this.ensureDir(); + const filePath = this.getPath(filename); + const dir = path.dirname(filePath); + if (!existsSync(dir)) { + await fs.mkdir(dir, { recursive: true }); + } + await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf8'); + } + + async exists(filename: string): Promise { + try { + await fs.access(this.getPath(filename)); + return true; + } catch { + return false; + } + } + + async delete(filename: string): Promise { + try { + await fs.unlink(this.getPath(filename)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } + + async clear(): Promise { + try { + const files = await fs.readdir(this.cacheDir); + await Promise.all( + files.map(file => fs.unlink(path.join(this.cacheDir, file)).catch(() => {})) + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } +} + +// ============================================================================= +// Logger Implementation +// ============================================================================= + +/** + * Scoped logger implementation for plugins + */ +export class PluginLoggerImpl implements PluginLogger { + private prefix: string; + + constructor(pluginId: string) { + this.prefix = `[plugin:${pluginId}]`; + } + + debug(message: string, ...args: unknown[]): void { + console.debug(this.prefix, message, ...args); + } + + info(message: string, ...args: unknown[]): void { + console.info(this.prefix, message, ...args); + } + + warn(message: string, ...args: unknown[]): void { + console.warn(this.prefix, message, ...args); + } + + error(message: string, ...args: unknown[]): void { + console.error(this.prefix, message, ...args); + } +} + +// ============================================================================= +// Context Factory +// ============================================================================= + +/** + * Create a lifecycle context for a plugin + */ +export function createLifecycleContext( + pluginId: string, + config: Record, + cacheDir: string +): PluginLifecycleContext { + return { + config, + cacheDir, + log: new PluginLoggerImpl(pluginId), + }; +} + +/** + * Create a full plugin context for file processing + */ +export function createPluginContext( + pluginId: string, + filePath: string, + kv: PluginKVStore, + config: Record, + cacheDir: string, + metadata?: any +): PluginContext { + const ctx: PluginContext = { + kv, + filePath, + config, + cacheDir, + cache: new PluginCacheImpl(cacheDir), + log: new PluginLoggerImpl(pluginId), + }; + + // If kv is a MetadataNodeKVStore, extract the MetadataNode + if (metadata) { + ctx.metadata = metadata; + } else if (kv instanceof MetadataNodeKVStore) { + ctx.metadata = kv.getMetadataNode(); + } + + return ctx; +} diff --git a/packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts b/packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts new file mode 100644 index 0000000..c960cf0 --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts @@ -0,0 +1,146 @@ +/** + * Plugin Executor + * + * Handles dependency resolution for the container-based plugin system. + */ + +import type { PluginManifest } from './types.js'; + +// ============================================================================= +// Dependency Resolution +// ============================================================================= + +/** + * Result of dependency resolution + */ +export interface DependencyResolution { + order: string[]; + errors: string[]; +} + +/** + * Perform topological sort on plugins based on dependencies. + * Returns execution order (plugins that must run first come first). + * + * @param plugins Map of plugin id to manifest + * @param activePluginIds Set of active plugin ids to include in the order + */ +export function resolveDependencyOrder( + plugins: Map, + activePluginIds: Set +): DependencyResolution { + const errors: string[] = []; + const order: string[] = []; + const visited = new Set(); + const visiting = new Set(); // For cycle detection + + // Filter to only active plugins + const activePlugins = new Map(); + activePluginIds.forEach(pluginId => { + const manifest = plugins.get(pluginId); + if (manifest) { + activePlugins.set(pluginId, manifest); + } + }); + + // DFS visit function + function visit(pluginId: string): boolean { + if (visited.has(pluginId)) { + return true; + } + if (visiting.has(pluginId)) { + errors.push(`Circular dependency detected involving plugin '${pluginId}'`); + return false; + } + + const manifest = activePlugins.get(pluginId); + if (!manifest) { + // Plugin not found or not active + return true; + } + + visiting.add(pluginId); + + // Visit dependencies first + for (const depId of manifest.dependencies || []) { + if (!activePlugins.has(depId)) { + // Dependency is not active + if (plugins.has(depId)) { + errors.push(`Plugin '${pluginId}' depends on '${depId}' which is not active`); + } else { + errors.push(`Plugin '${pluginId}' depends on '${depId}' which does not exist`); + } + continue; + } + if (!visit(depId)) { + return false; + } + } + + visiting.delete(pluginId); + visited.add(pluginId); + order.push(pluginId); + + return true; + } + + // Visit all active plugins + Array.from(activePluginIds).forEach(pluginId => { + if (!visited.has(pluginId)) { + visit(pluginId); + } + }); + + return { order, errors }; +} + +/** + * Check if a plugin can be activated (all dependencies are active) + */ +export function canActivate( + pluginId: string, + plugins: Map, + activePluginIds: Set +): { canActivate: boolean; missingDeps: string[] } { + const manifest = plugins.get(pluginId); + if (!manifest) { + return { canActivate: false, missingDeps: [] }; + } + + const missingDeps: string[] = []; + for (const depId of manifest.dependencies || []) { + if (!activePluginIds.has(depId)) { + missingDeps.push(depId); + } + } + + return { + canActivate: missingDeps.length === 0, + missingDeps, + }; +} + +/** + * Check if a plugin can be deactivated (no active plugins depend on it) + */ +export function canDeactivate( + pluginId: string, + plugins: Map, + activePluginIds: Set +): { canDeactivate: boolean; dependents: string[] } { + const dependents: string[] = []; + + activePluginIds.forEach(activeId => { + if (activeId === pluginId) return; + + const manifest = plugins.get(activeId); + if (manifest?.dependencies?.includes(pluginId)) { + dependents.push(activeId); + } + }); + + return { + canDeactivate: dependents.length === 0, + dependents, + }; +} diff --git a/packages/meta-sort-core/src/plugin-engine/PluginManager.ts b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts new file mode 100644 index 0000000..61f8019 --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts @@ -0,0 +1,406 @@ +/** + * Plugin Manager + * + * Main orchestrator for the container-based plugin system. + * Handles plugin manifests, lifecycle management, and state persistence. + * All plugins run as Docker containers - there is no local plugin execution. + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { existsSync, mkdirSync } from 'fs'; +import { EventEmitter } from 'events'; +import type { + PluginManifest, + PluginState, + PluginStateEntry, + PluginInfo, + LoadedPlugin, + PluginManagerEvent, + PluginKVStore, +} from './types.js'; +import { KVStore } from './PluginContext.js'; +import { + resolveDependencyOrder, + canActivate, + canDeactivate, +} from './PluginExecutor.js'; +import type { ContainerManager, ContainerPluginManifest } from '../container-plugins/index.js'; + +// ============================================================================= +// Plugin Manager +// ============================================================================= + +export interface PluginManagerOptions { + /** Path to plugins.json state file */ + statePath: string; + /** Base directory for plugin caches */ + cacheDir: string; + /** Container manager for containerized plugins (required) */ + containerManager?: ContainerManager; +} + +export class PluginManager extends EventEmitter { + private statePath: string; + private cacheDir: string; + + // Runtime state + private manifests: Map = new Map(); + private loadedPlugins: Map = new Map(); + private activePluginIds: Set = new Set(); + private configs: Map> = new Map(); + private executionOrder: string[] = []; + + // Container plugins (all plugins are container-based) + private containerManager: ContainerManager | null = null; + + constructor(options: PluginManagerOptions) { + super(); + this.statePath = options.statePath; + this.cacheDir = options.cacheDir; + this.containerManager = options.containerManager || null; + + // Ensure cache directory exists + if (!existsSync(this.cacheDir)) { + mkdirSync(this.cacheDir, { recursive: true }); + } + } + + // ========================================================================= + // Initialization + // ========================================================================= + + /** + * Initialize the plugin manager with container plugins. + * This loads plugin manifests from the ContainerManager and sets up the execution order. + */ + async initialize(): Promise { + if (!this.containerManager) { + console.warn('[PluginManager] No container manager set, no plugins will be available'); + return; + } + + await this.loadContainerPlugins(); + console.log(`[PluginManager] Initialized with ${this.manifests.size} container plugins`); + } + + // ========================================================================= + // Lifecycle Management + // ========================================================================= + + /** + * Activate a plugin + */ + async activate(pluginId: string): Promise { + const manifest = this.manifests.get(pluginId); + if (!manifest) { + throw new Error(`Plugin '${pluginId}' not found`); + } + if (this.activePluginIds.has(pluginId)) { + return; // Already active + } + + // Check dependencies + const { canActivate: can, missingDeps } = canActivate( + pluginId, + this.manifests, + this.activePluginIds + ); + if (!can) { + throw new Error( + `Cannot activate '${pluginId}': missing dependencies: ${missingDeps.join(', ')}` + ); + } + + // Mark as active + this.activePluginIds.add(pluginId); + this.rebuildExecutionOrder(); + + // Save state + await this.saveState(); + + // Emit event + this.emit('plugin:activated', { pluginId }); + } + + /** + * Deactivate a plugin + */ + async deactivate(pluginId: string): Promise { + if (!this.activePluginIds.has(pluginId)) { + return; // Already inactive + } + + // Check if other plugins depend on this one + const { canDeactivate: can, dependents } = canDeactivate( + pluginId, + this.manifests, + this.activePluginIds + ); + if (!can) { + throw new Error( + `Cannot deactivate '${pluginId}': other plugins depend on it: ${dependents.join(', ')}` + ); + } + + // Mark as inactive + this.activePluginIds.delete(pluginId); + this.rebuildExecutionOrder(); + + // Save state + await this.saveState(); + + // Emit event + this.emit('plugin:deactivated', { pluginId }); + } + + /** + * Update plugin configuration + */ + async updateConfig(pluginId: string, config: Record): Promise { + const manifest = this.manifests.get(pluginId); + if (!manifest) { + throw new Error(`Plugin '${pluginId}' not found`); + } + + // Store the config + this.configs.set(pluginId, config); + + // Save state + await this.saveState(); + + // Emit event + this.emit('plugin:config-changed', { pluginId, config }); + } + + // ========================================================================= + // Container Plugin Support + // ========================================================================= + + /** + * Set the container manager (can be set after construction) + */ + setContainerManager(containerManager: ContainerManager): void { + this.containerManager = containerManager; + console.log('[PluginManager] Container manager set'); + } + + /** + * Check if a plugin is a containerized plugin (all plugins are container-based) + */ + isContainerPlugin(pluginId: string): boolean { + return this.manifests.has(pluginId); + } + + /** + * Load plugin manifests from ContainerManager + * Should be called after container plugins are initialized + */ + async loadContainerPlugins(): Promise { + if (!this.containerManager) { + return; + } + + const status = this.containerManager.getStatus(); + if (!status.initialized) { + console.warn('[PluginManager] Container manager not initialized, skipping plugins'); + return; + } + + // Load state for plugin active status + const state = await this.loadState(); + + for (const pluginStatus of status.plugins) { + const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId); + if (!manifest) continue; + + const pluginId = pluginStatus.pluginId; + + // Convert container manifest to local manifest format + const localManifest: PluginManifest = { + id: manifest.id, + name: manifest.name, + version: manifest.version, + description: manifest.description || '', + author: manifest.author, + dependencies: manifest.dependencies || [], + schema: manifest.schema || {}, + config: manifest.config, + priority: manifest.priority || 100, + color: manifest.color, + defaultQueue: manifest.defaultQueue || 'background', + }; + + this.manifests.set(pluginId, localManifest); + + // Container plugins are always "loaded" but execution is remote + this.loadedPlugins.set(pluginId, { + manifest: localManifest, + instance: null as any, // No local instance for container plugins + status: 'loaded', + pluginDir: '', // No local dir + }); + + // Check if should be active + const stateEntry = state.plugins[pluginId]; + if (stateEntry?.active !== false) { + // Default to active for plugins + this.activePluginIds.add(pluginId); + } + + // Store config + const config = stateEntry?.config || {}; + this.configs.set(pluginId, config); + + console.log(`[PluginManager] Loaded plugin: ${pluginId}`); + } + + // Rebuild execution order + this.rebuildExecutionOrder(); + await this.saveState(); + } + + /** + * Get the container manager instance + */ + getContainerManager(): ContainerManager | null { + return this.containerManager; + } + + // ========================================================================= + // State Management + // ========================================================================= + + /** + * Get all discovered plugins + */ + getPlugins(): PluginInfo[] { + const plugins: PluginInfo[] = []; + + this.manifests.forEach((manifest, pluginId) => { + const loaded = this.loadedPlugins.get(pluginId); + const config = this.configs.get(pluginId) || {}; + + plugins.push({ + id: pluginId, + name: manifest.name, + version: manifest.version, + description: manifest.description, + priority: manifest.priority, + color: manifest.color, + dependencies: manifest.dependencies || [], + active: this.activePluginIds.has(pluginId), + config, + configSchema: manifest.config, + metadataSchema: manifest.schema, + status: loaded?.status || 'unloaded', + error: loaded?.error, + isContainer: true, // All plugins are container-based + }); + }); + + return plugins; + } + + /** + * Get only active plugins + */ + getActivePlugins(): PluginInfo[] { + return this.getPlugins().filter(p => p.active); + } + + /** + * Get execution order (dependency-sorted active plugin IDs) + */ + getExecutionOrder(): string[] { + return [...this.executionOrder]; + } + + /** + * Create a new KV store for file processing + */ + createKVStore(initialData?: Record): KVStore { + return new KVStore(initialData); + } + + // ========================================================================= + // Shutdown + // ========================================================================= + + /** + * Shutdown: clear plugin state + */ + async shutdown(): Promise { + this.manifests.clear(); + this.loadedPlugins.clear(); + this.activePluginIds.clear(); + this.executionOrder = []; + console.log('[PluginManager] Shutdown complete'); + } + + // ========================================================================= + // Private Helpers + // ========================================================================= + + private rebuildExecutionOrder(): void { + const { order, errors } = resolveDependencyOrder( + this.manifests, + this.activePluginIds + ); + + if (errors.length > 0) { + for (const error of errors) { + console.warn(`Dependency resolution warning: ${error}`); + } + } + + this.executionOrder = order; + } + + private async loadState(): Promise { + try { + if (existsSync(this.statePath)) { + const content = await fs.readFile(this.statePath, 'utf8'); + return JSON.parse(content) as PluginState; + } + } catch (error) { + console.error('Failed to load plugin state:', error); + } + return { plugins: {} }; + } + + private async saveState(): Promise { + const state: PluginState = { plugins: {} }; + + this.manifests.forEach((_, pluginId) => { + state.plugins[pluginId] = { + active: this.activePluginIds.has(pluginId), + config: this.configs.get(pluginId) || {}, + }; + }); + + const dir = path.dirname(this.statePath); + if (!existsSync(dir)) { + await fs.mkdir(dir, { recursive: true }); + } + + await fs.writeFile(this.statePath, JSON.stringify(state, null, 2), 'utf8'); + } +} + +// ============================================================================= +// Factory Function +// ============================================================================= + +/** + * Create a plugin manager with default paths based on config + */ +export function createPluginManager( + statePath: string, + cacheDir: string +): PluginManager { + return new PluginManager({ + statePath, + cacheDir, + }); +} diff --git a/packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts b/packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts new file mode 100644 index 0000000..a211b8a --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts @@ -0,0 +1,634 @@ +/** + * Task Scheduler + * + * Manages plugin task execution across fast and background queues. + * All plugins run as Docker containers - task execution is delegated to ContainerPluginScheduler. + */ + +import { EventEmitter } from 'events'; +import PQueue from 'p-queue'; +import type { + PluginTask, + PluginTaskStatus, + PluginTaskResult, + TaskQueueType, + QueueStatus, + TaskSchedulerEvent, + PluginManifest, + PluginKVStore, +} from './types.js'; +import type { PluginManager } from './PluginManager.js'; +import type { PerformanceMetrics, PluginTimingStats } from '../metrics/PerformanceMetrics.js'; +import type { ContainerPluginScheduler } from '../container-plugins/index.js'; +import { KVStore } from './PluginContext.js'; + +// ============================================================================= +// Configuration +// ============================================================================= + +export interface TaskSchedulerConfig { + /** Concurrency for fast queue (default: 32) */ + fastQueueConcurrency?: number; + /** Concurrency for background queue (default: 8) */ + backgroundQueueConcurrency?: number; + /** Threshold in ms for fast classification (default: 1000) */ + fastThresholdMs?: number; + /** Minimum samples before using measured timing (default: 10) */ + minSamplesForMeasurement?: number; +} + +const DEFAULT_CONFIG: Required = { + fastQueueConcurrency: 32, + backgroundQueueConcurrency: 8, + fastThresholdMs: 1000, + minSamplesForMeasurement: 10, +}; + +// ============================================================================= +// Task Scheduler +// ============================================================================= + +export class TaskScheduler extends EventEmitter { + private pluginManager: PluginManager; + private performanceMetrics: PerformanceMetrics; + private config: Required; + + // Queues + private fastQueue: PQueue; + private backgroundQueue: PQueue; + + // Task tracking + private pendingTasks: Map = new Map(); + private completedTasks: Set = new Set(); + private fileTasks: Map> = new Map(); // fileHash -> taskIds + private fileKVStores: Map = new Map(); // fileHash -> KV store + + // Container plugin support + private containerPluginScheduler: ContainerPluginScheduler | null = null; + + // Statistics + private totalTasksCreated: number = 0; + private totalTasksCompleted: number = 0; + private totalTasksFailed: number = 0; + + constructor( + pluginManager: PluginManager, + performanceMetrics: PerformanceMetrics, + config: TaskSchedulerConfig = {} + ) { + super(); + this.pluginManager = pluginManager; + this.performanceMetrics = performanceMetrics; + this.config = { ...DEFAULT_CONFIG, ...config }; + + // Initialize fast queue + this.fastQueue = new PQueue({ + concurrency: this.config.fastQueueConcurrency, + }); + + // Initialize background queue (starts paused) + this.backgroundQueue = new PQueue({ + concurrency: this.config.backgroundQueueConcurrency, + autoStart: false, + }); + + // When fast queue becomes idle, start background queue + this.fastQueue.on('idle', () => { + this.onFastQueueIdle(); + }); + } + + /** + * Set the container plugin scheduler (called after initialization) + */ + setContainerPluginScheduler(scheduler: ContainerPluginScheduler): void { + this.containerPluginScheduler = scheduler; + + // Forward container scheduler events to this scheduler + scheduler.on('task:completed', ({ task }) => { + this.onContainerTaskComplete(task.id, task.fileHash, true); + }); + scheduler.on('task:failed', ({ task }) => { + this.onContainerTaskComplete(task.id, task.fileHash, false); + }); + scheduler.on('file:complete', ({ fileHash, filePath }) => { + // Container file completion - check if local tasks are also done + this.checkFileComplete(fileHash); + }); + + console.log('[TaskScheduler] Container plugin scheduler set'); + } + + /** + * Handle completion of a container plugin task + */ + private onContainerTaskComplete(taskId: string, fileHash: string, success: boolean): void { + // Move from pending to completed + this.pendingTasks.delete(taskId); + this.completedTasks.add(taskId); + + if (success) { + this.totalTasksCompleted++; + } else { + this.totalTasksFailed++; + } + + // Schedule dependent tasks + const task = { id: taskId, fileHash, pluginId: taskId.split(':')[1] } as PluginTask; + this.scheduleDependentTasks(task); + + // Check if file is complete + this.checkFileComplete(fileHash); + } + + // ========================================================================= + // Task Creation + // ========================================================================= + + /** + * Create tasks for all active plugins for a file + * @param filePath - Absolute path to the file + * @param fileHash - File identity hash (midhash256) + * @param kv - KV store for the file (will be shared across all plugins) + */ + createTasksForFile(filePath: string, fileHash: string, kv?: PluginKVStore): PluginTask[] { + const tasks: PluginTask[] = []; + const executionOrder = this.pluginManager.getExecutionOrder(); + const plugins = this.pluginManager.getPlugins(); + const pluginMap = new Map(plugins.map(p => [p.id, p])); + + // Initialize file task tracking + if (!this.fileTasks.has(fileHash)) { + this.fileTasks.set(fileHash, new Set()); + } + const fileTaskIds = this.fileTasks.get(fileHash)!; + + // Store KV for this file (create new if not provided) + if (!this.fileKVStores.has(fileHash)) { + this.fileKVStores.set(fileHash, kv || new KVStore()); + } + + for (const pluginId of executionOrder) { + const plugin = pluginMap.get(pluginId); + if (!plugin || !plugin.active) continue; + + const taskId = `${fileHash}:${pluginId}`; + + // Skip if task already exists + if (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId)) { + continue; + } + + const timing = this.performanceMetrics.getPluginTiming(pluginId); + const queue = this.classifyPlugin(pluginId); + + const task: PluginTask = { + id: taskId, + fileHash, + filePath, + pluginId, + dependencies: plugin.dependencies || [], + queue, + priority: timing?.average || 0, + estimatedTimeMs: timing?.average || 0, + status: 'pending', + createdAt: Date.now(), + }; + + tasks.push(task); + fileTaskIds.add(taskId); + this.totalTasksCreated++; + + this.emitEvent({ type: 'task:created', task }); + } + + return tasks; + } + + /** + * Create a task for a specific plugin on a single file + * Used for recompute scenarios where we want to run just one plugin + * @param pluginId - Plugin to execute + * @param filePath - Absolute path to the file + * @param fileHash - File identity hash (midhash256) + * @param options - Runtime options merged into plugin config (e.g., { forceRecompute: true }) + * @param kvData - Optional initial KV data (flattened metadata from Redis) + * @returns The created task, or null if plugin not found/not active + */ + createTaskForPlugin( + pluginId: string, + filePath: string, + fileHash: string, + options?: Record, + kvData?: Record + ): PluginTask | null { + const plugins = this.pluginManager.getPlugins(); + const plugin = plugins.find(p => p.id === pluginId); + + if (!plugin || !plugin.active) { + console.warn(`[TaskScheduler] Plugin '${pluginId}' not found or not active`); + return null; + } + + // Use a unique task ID for recompute tasks to avoid conflicts + const taskId = options?.forceRecompute + ? `${fileHash}:${pluginId}:recompute:${Date.now()}` + : `${fileHash}:${pluginId}`; + + // Skip if non-recompute task already exists + if (!options?.forceRecompute && (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId))) { + return null; + } + + // Initialize file task tracking if needed + if (!this.fileTasks.has(fileHash)) { + this.fileTasks.set(fileHash, new Set()); + } + const fileTaskIds = this.fileTasks.get(fileHash)!; + + // Initialize or update KV store for this file + // For recompute, we want to load existing metadata so plugin can see current state + if (!this.fileKVStores.has(fileHash)) { + this.fileKVStores.set(fileHash, new KVStore(kvData)); + } else if (kvData) { + // Update existing KV store with provided data + const existingKv = this.fileKVStores.get(fileHash)!; + for (const [key, value] of Object.entries(kvData)) { + existingKv.set(key, value); + } + } + + const timing = this.performanceMetrics.getPluginTiming(pluginId); + const queue = this.classifyPlugin(pluginId); + + const task: PluginTask = { + id: taskId, + fileHash, + filePath, + pluginId, + dependencies: plugin.dependencies || [], + queue, + priority: timing?.average || 0, + estimatedTimeMs: timing?.average || 0, + status: 'pending', + createdAt: Date.now(), + options, + }; + + fileTaskIds.add(taskId); + this.totalTasksCreated++; + this.emitEvent({ type: 'task:created', task }); + + return task; + } + + /** + * Create tasks for a specific plugin on multiple files + * Used for "recompute plugin on all files" feature + * @param pluginId - Plugin to execute + * @param files - Array of { filePath, fileHash, kvData? } objects + * @param options - Runtime options merged into plugin config + * @returns Array of created tasks + */ + createTasksForPluginOnFiles( + pluginId: string, + files: Array<{ filePath: string; fileHash: string; kvData?: Record }>, + options?: Record + ): PluginTask[] { + const tasks: PluginTask[] = []; + + for (const { filePath, fileHash, kvData } of files) { + const task = this.createTaskForPlugin(pluginId, filePath, fileHash, options, kvData); + if (task) { + tasks.push(task); + } + } + + console.log(`[TaskScheduler] Created ${tasks.length} tasks for plugin '${pluginId}'`); + return tasks; + } + + /** + * Enqueue a task for execution + * All plugins are container-based, so all tasks go to the container scheduler + */ + enqueueTask(task: PluginTask): void { + this.pendingTasks.set(task.id, task); + this.enqueueContainerTask(task); + } + + /** + * Enqueue multiple tasks + */ + enqueueTasks(tasks: PluginTask[]): void { + for (const task of tasks) { + this.enqueueTask(task); + } + } + + /** + * Enqueue a task to the container plugin scheduler + */ + private enqueueContainerTask(task: PluginTask): void { + if (!this.containerPluginScheduler) { + console.warn( + `[TaskScheduler] Container plugin task ${task.id} cannot be scheduled: no container scheduler` + ); + task.status = 'failed'; + task.result = { + success: false, + timeMs: 0, + error: 'Container plugin scheduler not available', + }; + this.pendingTasks.delete(task.id); + this.completedTasks.add(task.id); + this.totalTasksFailed++; + return; + } + + // Create container task and enqueue + const containerTask = this.containerPluginScheduler.createTask( + task.pluginId, + task.fileHash, + task.filePath, + task.dependencies, + task.queue + ); + + // Map the local task ID to the container task ID + // Update our pending tasks map with the container task ID + this.pendingTasks.delete(task.id); + this.pendingTasks.set(containerTask.id, { + ...task, + id: containerTask.id, // Use container task ID + }); + + // Update file task tracking + const fileTaskIds = this.fileTasks.get(task.fileHash); + if (fileTaskIds) { + fileTaskIds.delete(task.id); + fileTaskIds.add(containerTask.id); + } + + // Enqueue to container scheduler + this.containerPluginScheduler.enqueueTask(containerTask); + } + + // ========================================================================= + // Classification + // ========================================================================= + + /** + * Classify a plugin as fast or background based on timing data and manifest + */ + classifyPlugin(pluginId: string): TaskQueueType { + const plugins = this.pluginManager.getPlugins(); + const plugin = plugins.find(p => p.id === pluginId); + const timing = this.performanceMetrics.getPluginTiming(pluginId); + + // Get manifest default queue (need to access via plugin manager internals) + // For now, we'll use a workaround - check if plugin has defaultQueue in manifest + const manifest = this.getPluginManifest(pluginId); + const defaultQueue = manifest?.defaultQueue || 'fast'; + + // If we have enough samples, use measured timing + if (timing && timing.count >= this.config.minSamplesForMeasurement) { + return timing.average < this.config.fastThresholdMs ? 'fast' : 'background'; + } + + // Otherwise use manifest default + return defaultQueue; + } + + // ========================================================================= + // Task Completion + // ========================================================================= + + private scheduleDependentTasks(completedTask: PluginTask): void { + // Container scheduler handles dependency resolution + // This method is kept for compatibility but doesn't need to do anything + } + + private checkFileComplete(fileHash: string): void { + const fileTaskIds = this.fileTasks.get(fileHash); + if (!fileTaskIds) return; + + // Check if all tasks for this file are completed + let filePath = ''; + for (const taskId of fileTaskIds) { + if (!this.completedTasks.has(taskId)) { + return; // Still have pending/running tasks + } + // Try to get filePath from any task + if (!filePath) { + const task = this.pendingTasks.get(taskId); + if (task) filePath = task.filePath; + } + } + + // All tasks complete + this.emitEvent({ + type: 'file:complete', + fileHash, + filePath, + }); + + // Clean up file tracking + this.fileTasks.delete(fileHash); + this.fileKVStores.delete(fileHash); + + // Clean up completed task IDs for this file to prevent memory leak + for (const taskId of fileTaskIds) { + this.completedTasks.delete(taskId); + } + } + + // ========================================================================= + // Queue Control + // ========================================================================= + + private onFastQueueIdle(): void { + this.emitEvent({ type: 'queue:fast:idle' }); + + // Start background queue if it has pending work + if (this.backgroundQueue.size > 0 || this.backgroundQueue.pending > 0) { + console.log('[TaskScheduler] Fast queue idle, starting background queue'); + this.backgroundQueue.start(); + this.emitEvent({ type: 'queue:background:started' }); + } + } + + // ========================================================================= + // Status & Metrics + // ========================================================================= + + /** + * Get status of both queues + */ + getQueueStatus(): { fast: QueueStatus; background: QueueStatus } { + return { + fast: this.buildQueueStatus('fast'), + background: this.buildQueueStatus('background'), + }; + } + + private buildQueueStatus(queueType: TaskQueueType): QueueStatus { + const queue = queueType === 'fast' ? this.fastQueue : this.backgroundQueue; + + let pending = 0; + let ready = 0; + let running = 0; + let completed = 0; + let failed = 0; + + // Count pending tasks for this queue type + for (const [, task] of this.pendingTasks) { + if (task.queue === queueType) { + if (task.status === 'pending') pending++; + else if (task.status === 'ready') ready++; + else if (task.status === 'running') running++; + } + } + + // Count completed tasks + for (const taskId of this.completedTasks) { + const [, pluginId] = taskId.split(':'); + if (this.classifyPlugin(pluginId) === queueType) { + completed++; + } + } + + return { + pending, + ready: queue.size, // Tasks in queue waiting to run + running: queue.pending, // Tasks currently executing + completed, + failed, // Would need separate tracking + isPaused: queue.isPaused, + }; + } + + /** + * Get pending tasks for a specific file + */ + getPendingTasksForFile(fileHash: string): PluginTask[] { + const tasks: PluginTask[] = []; + const fileTaskIds = this.fileTasks.get(fileHash); + + if (fileTaskIds) { + for (const taskId of fileTaskIds) { + const task = this.pendingTasks.get(taskId); + if (task) { + tasks.push(task); + } + } + } + + return tasks; + } + + /** + * Get the KV store for a file + */ + getFileKVStore(fileHash: string): PluginKVStore | undefined { + return this.fileKVStores.get(fileHash); + } + + /** + * Cancel processing for a file (removes all pending tasks) + */ + cancelFile(fileHash: string): void { + const fileTaskIds = this.fileTasks.get(fileHash); + if (!fileTaskIds) return; + + // Remove pending tasks + for (const taskId of fileTaskIds) { + this.pendingTasks.delete(taskId); + } + + // Clean up file tracking + this.fileTasks.delete(fileHash); + this.fileKVStores.delete(fileHash); + + console.log(`[TaskScheduler] Cancelled processing for file ${fileHash}`); + } + + /** + * Get scheduler statistics + */ + getStats(): { + totalTasksCreated: number; + totalTasksCompleted: number; + totalTasksFailed: number; + pendingTaskCount: number; + completedTaskCount: number; + filesInProgress: number; + } { + return { + totalTasksCreated: this.totalTasksCreated, + totalTasksCompleted: this.totalTasksCompleted, + totalTasksFailed: this.totalTasksFailed, + pendingTaskCount: this.pendingTasks.size, + completedTaskCount: this.completedTasks.size, + filesInProgress: this.fileTasks.size, + }; + } + + // ========================================================================= + // Shutdown + // ========================================================================= + + /** + * Gracefully shutdown the scheduler + */ + async shutdown(): Promise { + console.log('[TaskScheduler] Shutting down...'); + + // Clear queues + this.fastQueue.clear(); + this.backgroundQueue.clear(); + + // Wait for in-progress tasks to complete + await Promise.all([ + this.fastQueue.onIdle(), + this.backgroundQueue.onIdle(), + ]); + + // Clear tracking + this.pendingTasks.clear(); + this.completedTasks.clear(); + this.fileTasks.clear(); + this.fileKVStores.clear(); + + console.log('[TaskScheduler] Shutdown complete'); + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private getPluginManifest(pluginId: string): PluginManifest | undefined { + // Access manifests from PluginManager + const manifests = (this.pluginManager as any).manifests as Map; + return manifests.get(pluginId); + } + + private emitEvent(event: TaskSchedulerEvent): void { + this.emit(event.type, event); + } +} + +// ============================================================================= +// Factory Function +// ============================================================================= + +/** + * Create a task scheduler instance + */ +export function createTaskScheduler( + pluginManager: PluginManager, + performanceMetrics: PerformanceMetrics, + config?: TaskSchedulerConfig +): TaskScheduler { + return new TaskScheduler(pluginManager, performanceMetrics, config); +} diff --git a/packages/meta-sort-core/src/plugin-engine/index.ts b/packages/meta-sort-core/src/plugin-engine/index.ts new file mode 100644 index 0000000..328d121 --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/index.ts @@ -0,0 +1,59 @@ +/** + * Plugin System + * + * Main entry point for the meta-sort container-based plugin system. + * All plugins run as Docker containers - there is no local plugin execution. + */ + +// Types +export type { + // Manifest types + ConfigFieldType, + SchemaFieldType, + SelectOption, + ConfigField, + SchemaField, + PluginManifest, + TaskQueueType, + // State types + PluginStateEntry, + PluginState, + PluginStatus, + PluginInfo, + // Context types + PluginKVStore, + PluginCache, + PluginLogger, + PluginLifecycleContext, + PluginContext, + // Plugin interface + Plugin, + // Internal types + LoadedPlugin, + PluginManagerEvent, + // Task types + PluginTaskStatus, + PluginTaskResult, + PluginTask, + QueueStatus, + TaskSchedulerEvent, +} from './types.js'; + +// Context implementations (KVStore is still used for local state) +export { KVStore, MetadataNodeKVStore } from './PluginContext.js'; + +// Dependency resolution utilities +export { + resolveDependencyOrder, + canActivate, + canDeactivate, +} from './PluginExecutor.js'; +export type { DependencyResolution } from './PluginExecutor.js'; + +// Plugin Manager +export { PluginManager, createPluginManager } from './PluginManager.js'; +export type { PluginManagerOptions } from './PluginManager.js'; + +// Task Scheduler +export { TaskScheduler, createTaskScheduler } from './TaskScheduler.js'; +export type { TaskSchedulerConfig } from './TaskScheduler.js'; diff --git a/packages/meta-sort-core/src/plugin-engine/plugin.test.ts b/packages/meta-sort-core/src/plugin-engine/plugin.test.ts new file mode 100644 index 0000000..725bbb9 --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/plugin.test.ts @@ -0,0 +1,667 @@ +/** + * Plugin System Unit Tests + */ + +import { expect } from 'chai'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { existsSync, mkdirSync, rmSync } from 'fs'; +import { fileURLToPath } from 'url'; + +import { + KVStore, + PluginCacheImpl, + PluginLoggerImpl, + parseManifest, + validateManifest, + resolveDependencyOrder, + canActivate, + canDeactivate, + PluginManager, +} from './index.js'; +import type { PluginManifest, Plugin, PluginContext } from './types.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const TEST_DIR = path.join(__dirname, '../../test-plugins'); +const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache'); +const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins.json'); + +// ============================================================================= +// Test Helpers +// ============================================================================= + +async function setupTestDir() { + // Clean up + if (existsSync(TEST_DIR)) { + rmSync(TEST_DIR, { recursive: true }); + } + if (existsSync(TEST_CACHE_DIR)) { + rmSync(TEST_CACHE_DIR, { recursive: true }); + } + if (existsSync(TEST_STATE_PATH)) { + rmSync(TEST_STATE_PATH); + } + + mkdirSync(TEST_DIR, { recursive: true }); + mkdirSync(TEST_CACHE_DIR, { recursive: true }); +} + +async function cleanupTestDir() { + if (existsSync(TEST_DIR)) { + rmSync(TEST_DIR, { recursive: true }); + } + if (existsSync(TEST_CACHE_DIR)) { + rmSync(TEST_CACHE_DIR, { recursive: true }); + } + if (existsSync(TEST_STATE_PATH)) { + rmSync(TEST_STATE_PATH); + } +} + +async function createTestPlugin( + id: string, + manifest: Partial, + processCode: string = 'ctx.kv.set("test/processed", "true");' +): Promise { + const pluginDir = path.join(TEST_DIR, id); + mkdirSync(pluginDir, { recursive: true }); + + // Write manifest + const fullManifest = { + id, + name: manifest.name || id, + version: manifest.version || '1.0.0', + ...manifest, + }; + + const manifestYaml = `id: ${fullManifest.id} +name: ${fullManifest.name} +version: ${fullManifest.version} +${fullManifest.description ? `description: ${fullManifest.description}` : ''} +${fullManifest.dependencies?.length ? `dependencies:\n${fullManifest.dependencies.map(d => ` - ${d}`).join('\n')}` : ''} +`; + + await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml); + + // Write plugin code + const pluginCode = ` +export default { + async process(ctx) { + ${processCode} + } +}; +`; + await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode); + + return pluginDir; +} + +// ============================================================================= +// KVStore Tests +// ============================================================================= + +describe('KVStore', () => { + it('should set and get values', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + expect(kv.get('file/name')).to.equal('test.mkv'); + }); + + it('should return undefined for missing keys', () => { + const kv = new KVStore(); + expect(kv.get('missing/key')).to.be.undefined; + }); + + it('should delete keys', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.delete('file/name'); + expect(kv.get('file/name')).to.be.undefined; + }); + + it('should list all keys', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.set('video/codec', 'h264'); + kv.set('audio/0/codec', 'aac'); + + const keys = kv.keys(); + expect(keys).to.have.lengthOf(3); + expect(keys).to.include('file/name'); + expect(keys).to.include('video/codec'); + expect(keys).to.include('audio/0/codec'); + }); + + it('should filter keys by prefix', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.set('audio/0/codec', 'aac'); + kv.set('audio/0/language', 'eng'); + kv.set('audio/1/codec', 'dts'); + + const audioKeys = kv.keys('audio/'); + expect(audioKeys).to.have.lengthOf(3); + expect(audioKeys).to.include('audio/0/codec'); + expect(audioKeys).to.include('audio/0/language'); + expect(audioKeys).to.include('audio/1/codec'); + }); + + it('should return entries', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.set('video/codec', 'h264'); + + const entries = kv.entries(); + expect(entries).to.have.lengthOf(2); + expect(entries).to.deep.include(['file/name', 'test.mkv']); + expect(entries).to.deep.include(['video/codec', 'h264']); + }); + + it('should filter entries by prefix', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.set('audio/0/codec', 'aac'); + kv.set('audio/1/codec', 'dts'); + + const audioEntries = kv.entries('audio/'); + expect(audioEntries).to.have.lengthOf(2); + }); + + it('should initialize with data', () => { + const kv = new KVStore({ 'file/name': 'test.mkv', 'video/codec': 'h264' }); + expect(kv.get('file/name')).to.equal('test.mkv'); + expect(kv.get('video/codec')).to.equal('h264'); + }); + + it('should convert to object', () => { + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + kv.set('video/codec', 'h264'); + + const obj = kv.toObject(); + expect(obj).to.deep.equal({ + 'file/name': 'test.mkv', + 'video/codec': 'h264', + }); + }); + + it('should ignore null/undefined values', () => { + const kv = new KVStore(); + kv.set('test', null as any); + kv.set('test2', undefined as any); + expect(kv.keys()).to.have.lengthOf(0); + }); + + it('should ignore empty keys', () => { + const kv = new KVStore(); + kv.set('', 'value'); + expect(kv.keys()).to.have.lengthOf(0); + }); +}); + +// ============================================================================= +// PluginCache Tests +// ============================================================================= + +describe('PluginCache', () => { + const cacheDir = path.join(TEST_CACHE_DIR, 'test-plugin'); + + beforeEach(async () => { + if (existsSync(cacheDir)) { + rmSync(cacheDir, { recursive: true }); + } + }); + + afterEach(async () => { + if (existsSync(cacheDir)) { + rmSync(cacheDir, { recursive: true }); + } + }); + + it('should write and read JSON', async () => { + const cache = new PluginCacheImpl(cacheDir); + const data = { foo: 'bar', num: 42 }; + + await cache.writeJson('test.json', data); + const result = await cache.readJson('test.json'); + + expect(result).to.deep.equal(data); + }); + + it('should return null for missing files', async () => { + const cache = new PluginCacheImpl(cacheDir); + const result = await cache.readJson('missing.json'); + expect(result).to.be.null; + }); + + it('should check file existence', async () => { + const cache = new PluginCacheImpl(cacheDir); + + expect(await cache.exists('test.json')).to.be.false; + + await cache.writeJson('test.json', { foo: 'bar' }); + expect(await cache.exists('test.json')).to.be.true; + }); + + it('should delete files', async () => { + const cache = new PluginCacheImpl(cacheDir); + await cache.writeJson('test.json', { foo: 'bar' }); + + await cache.delete('test.json'); + expect(await cache.exists('test.json')).to.be.false; + }); + + it('should clear all files', async () => { + const cache = new PluginCacheImpl(cacheDir); + await cache.writeJson('test1.json', { a: 1 }); + await cache.writeJson('test2.json', { b: 2 }); + + await cache.clear(); + + expect(await cache.exists('test1.json')).to.be.false; + expect(await cache.exists('test2.json')).to.be.false; + }); + + it('should get full path', () => { + const cache = new PluginCacheImpl(cacheDir); + const fullPath = cache.getPath('test.json'); + expect(fullPath).to.equal(path.join(cacheDir, 'test.json')); + }); +}); + +// ============================================================================= +// Manifest Parsing Tests +// ============================================================================= + +describe('Manifest Parsing', () => { + beforeEach(setupTestDir); + afterEach(cleanupTestDir); + + it('should parse valid manifest', async () => { + const manifestPath = path.join(TEST_DIR, 'manifest.yml'); + await fs.writeFile( + manifestPath, + `id: test-plugin +name: Test Plugin +version: 1.0.0 +description: A test plugin +dependencies: + - other-plugin +config: + apiKey: + type: string + required: true + secret: true + language: + type: string + default: en-US +` + ); + + const manifest = await parseManifest(manifestPath); + + expect(manifest.id).to.equal('test-plugin'); + expect(manifest.name).to.equal('Test Plugin'); + expect(manifest.version).to.equal('1.0.0'); + expect(manifest.description).to.equal('A test plugin'); + expect(manifest.dependencies).to.deep.equal(['other-plugin']); + expect(manifest.config?.apiKey?.type).to.equal('string'); + expect(manifest.config?.apiKey?.secret).to.be.true; + expect(manifest.config?.language?.default).to.equal('en-US'); + }); + + it('should validate manifest - valid', () => { + const manifest: PluginManifest = { + id: 'test-plugin', + name: 'Test Plugin', + version: '1.0.0', + }; + + const errors = validateManifest(manifest); + expect(errors).to.have.lengthOf(0); + }); + + it('should validate manifest - invalid id', () => { + const manifest: PluginManifest = { + id: 'Test_Plugin', + name: 'Test Plugin', + version: '1.0.0', + }; + + const errors = validateManifest(manifest); + expect(errors).to.have.lengthOf(1); + expect(errors[0]).to.include('Invalid plugin id'); + }); + + it('should validate manifest - invalid version', () => { + const manifest: PluginManifest = { + id: 'test-plugin', + name: 'Test Plugin', + version: 'invalid', + }; + + const errors = validateManifest(manifest); + expect(errors).to.have.lengthOf(1); + expect(errors[0]).to.include('Invalid version'); + }); +}); + +// ============================================================================= +// Dependency Resolution Tests +// ============================================================================= + +describe('Dependency Resolution', () => { + it('should resolve simple dependency chain', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }], + ]); + + const active = new Set(['a', 'b', 'c']); + const { order, errors } = resolveDependencyOrder(plugins, active); + + expect(errors).to.have.lengthOf(0); + expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b')); + expect(order.indexOf('b')).to.be.lessThan(order.indexOf('c')); + }); + + it('should handle diamond dependency', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['a'] }], + ['d', { id: 'd', name: 'D', version: '1.0.0', dependencies: ['b', 'c'] }], + ]); + + const active = new Set(['a', 'b', 'c', 'd']); + const { order, errors } = resolveDependencyOrder(plugins, active); + + expect(errors).to.have.lengthOf(0); + expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b')); + expect(order.indexOf('a')).to.be.lessThan(order.indexOf('c')); + expect(order.indexOf('b')).to.be.lessThan(order.indexOf('d')); + expect(order.indexOf('c')).to.be.lessThan(order.indexOf('d')); + }); + + it('should detect circular dependencies', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['c'] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }], + ]); + + const active = new Set(['a', 'b', 'c']); + const { errors } = resolveDependencyOrder(plugins, active); + + expect(errors.length).to.be.greaterThan(0); + expect(errors[0]).to.include('Circular dependency'); + }); + + it('should report missing dependencies', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['missing'] }], + ]); + + const active = new Set(['a']); + const { errors } = resolveDependencyOrder(plugins, active); + + expect(errors.length).to.be.greaterThan(0); + expect(errors[0]).to.include('does not exist'); + }); + + it('should report inactive dependencies', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ]); + + const active = new Set(['b']); // 'a' is not active + const { errors } = resolveDependencyOrder(plugins, active); + + expect(errors.length).to.be.greaterThan(0); + expect(errors[0]).to.include('not active'); + }); +}); + +// ============================================================================= +// Activation/Deactivation Tests +// ============================================================================= + +describe('Activation Checks', () => { + it('should allow activation when dependencies are met', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ]); + + const active = new Set(['a']); + const { canActivate: can, missingDeps } = canActivate('b', plugins, active); + + expect(can).to.be.true; + expect(missingDeps).to.have.lengthOf(0); + }); + + it('should prevent activation when dependencies are missing', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ]); + + const active = new Set(); // 'a' is not active + const { canActivate: can, missingDeps } = canActivate('b', plugins, active); + + expect(can).to.be.false; + expect(missingDeps).to.deep.equal(['a']); + }); + + it('should allow deactivation when no dependents', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ]); + + const active = new Set(['a', 'b']); + const { canDeactivate: can, dependents } = canDeactivate('b', plugins, active); + + expect(can).to.be.true; + expect(dependents).to.have.lengthOf(0); + }); + + it('should prevent deactivation when has dependents', () => { + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ]); + + const active = new Set(['a', 'b']); + const { canDeactivate: can, dependents } = canDeactivate('a', plugins, active); + + expect(can).to.be.false; + expect(dependents).to.deep.equal(['b']); + }); +}); + +// ============================================================================= +// Plugin Manager Integration Tests +// ============================================================================= + +describe('PluginManager', () => { + beforeEach(setupTestDir); + afterEach(async () => { + await cleanupTestDir(); + }); + + it('should scan and discover plugins', async () => { + await createTestPlugin('test-plugin', { + name: 'Test Plugin', + version: '1.0.0', + }); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + const plugins = await manager.scanPlugins(); + + expect(plugins).to.have.lengthOf(1); + expect(plugins[0].id).to.equal('test-plugin'); + expect(plugins[0].name).to.equal('Test Plugin'); + }); + + it('should load and activate plugins', async () => { + await createTestPlugin('plugin-a', { name: 'Plugin A' }); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager.scanPlugins(); + await manager.loadAll(); + + // Activate + await manager.activate('plugin-a'); + + const plugins = manager.getPlugins(); + expect(plugins[0].active).to.be.true; + + const order = manager.getExecutionOrder(); + expect(order).to.include('plugin-a'); + }); + + it('should respect dependency order in execution', async () => { + await createTestPlugin('plugin-a', { name: 'Plugin A' }, 'ctx.kv.set("order/a", Date.now().toString());'); + await createTestPlugin( + 'plugin-b', + { name: 'Plugin B', dependencies: ['plugin-a'] }, + 'ctx.kv.set("order/b", Date.now().toString());' + ); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager.scanPlugins(); + await manager.loadAll(); + await manager.activate('plugin-a'); + await manager.activate('plugin-b'); + + const order = manager.getExecutionOrder(); + expect(order.indexOf('plugin-a')).to.be.lessThan(order.indexOf('plugin-b')); + }); + + it('should process files through plugins', async () => { + await createTestPlugin( + 'test-plugin', + { name: 'Test Plugin' }, + ` + ctx.kv.set('file/processed', 'true'); + ctx.kv.set('file/path', ctx.filePath); + ` + ); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager.scanPlugins(); + await manager.loadAll(); + await manager.activate('test-plugin'); + + const kv = manager.createKVStore(); + const result = await manager.processFile('/test/file.mkv', kv); + + expect(result.success).to.be.true; + expect(kv.get('file/processed')).to.equal('true'); + expect(kv.get('file/path')).to.equal('/test/file.mkv'); + }); + + it('should persist state across restarts', async () => { + await createTestPlugin('test-plugin', { name: 'Test Plugin' }); + + // First run + const manager1 = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager1.scanPlugins(); + await manager1.loadAll(); + await manager1.activate('test-plugin'); + await manager1.updateConfig('test-plugin', { setting: 'value' }); + await manager1.shutdown(); + + // Second run + const manager2 = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager2.scanPlugins(); + await manager2.loadAll(); + + const plugins = manager2.getPlugins(); + expect(plugins[0].active).to.be.true; + expect(plugins[0].config.setting).to.equal('value'); + + await manager2.shutdown(); + }); + + it('should prevent deactivation of plugin with dependents', async () => { + await createTestPlugin('plugin-a', { name: 'Plugin A' }); + await createTestPlugin('plugin-b', { name: 'Plugin B', dependencies: ['plugin-a'] }); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + await manager.scanPlugins(); + await manager.loadAll(); + await manager.activate('plugin-a'); + await manager.activate('plugin-b'); + + try { + await manager.deactivate('plugin-a'); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as Error).message).to.include('other plugins depend on it'); + } + + await manager.shutdown(); + }); + + it('should emit events on activation/deactivation', async () => { + await createTestPlugin('test-plugin', { name: 'Test Plugin' }); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + const events: string[] = []; + manager.on('plugin:activated', (e) => events.push(`activated:${e.pluginId}`)); + manager.on('plugin:deactivated', (e) => events.push(`deactivated:${e.pluginId}`)); + + await manager.scanPlugins(); + await manager.loadAll(); + await manager.activate('test-plugin'); + await manager.deactivate('test-plugin'); + + expect(events).to.deep.equal(['activated:test-plugin', 'deactivated:test-plugin']); + + await manager.shutdown(); + }); +}); diff --git a/packages/meta-sort-core/src/plugin-engine/types.ts b/packages/meta-sort-core/src/plugin-engine/types.ts new file mode 100644 index 0000000..a644375 --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/types.ts @@ -0,0 +1,429 @@ +/** + * Plugin System Type Definitions + * + * This module defines all types for the meta-sort plugin system. + */ + +// ============================================================================= +// Plugin Manifest Types +// ============================================================================= + +/** + * Configuration field type in manifest + */ +export type ConfigFieldType = 'string' | 'number' | 'boolean' | 'select'; + +/** + * Schema field type for editor UI + */ +export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString'; + +/** + * Select option for config/schema fields + */ +export interface SelectOption { + value: string; + label: string; +} + +/** + * Configuration field definition in manifest + */ +export interface ConfigField { + type: ConfigFieldType; + label?: string; + required?: boolean; + default?: unknown; + secret?: boolean; + options?: SelectOption[]; +} + +/** + * Metadata schema field definition (for editor UI) + */ +export interface SchemaField { + /** Display label */ + label: string; + /** Field type for form control */ + type?: SchemaFieldType; + /** Read-only field (not editable) */ + readonly?: boolean; + /** Whether field is indexed (has {n} pattern) */ + indexed?: boolean; + /** Options for select type */ + options?: SelectOption[]; + /** Hint text shown below field */ + hint?: string; + /** URL template for clickable values (use {value} as placeholder) */ + link?: string; + /** Pattern for JSON type to aggregate matching keys */ + pattern?: string; + /** Minimum value for number type */ + min?: number; + /** Maximum value for number type */ + max?: number; + /** Placeholder text */ + placeholder?: string; +} + +/** + * Task queue type for plugin classification + */ +export type TaskQueueType = 'fast' | 'background'; + +/** + * Plugin manifest (parsed from manifest.yml) + */ +export interface PluginManifest { + id: string; + name: string; + version: string; + description?: string; + /** Plugin author */ + author?: string; + /** Display priority (lower = higher priority) */ + priority?: number; + /** Color for UI display */ + color?: string; + dependencies?: string[]; + config?: Record; + schema?: Record; + /** + * Default queue for this plugin. + * - 'fast': High concurrency queue for quick operations (<1s) + * - 'background': Low concurrency queue for slow operations (>=1s) + * Default: 'fast' + */ + defaultQueue?: TaskQueueType; +} + +// ============================================================================= +// Plugin State Types +// ============================================================================= + +/** + * Plugin state entry (stored in plugins.json) + */ +export interface PluginStateEntry { + active: boolean; + config: Record; +} + +/** + * Complete plugin state file structure + */ +export interface PluginState { + plugins: Record; +} + +/** + * Plugin status + */ +export type PluginStatus = 'loaded' | 'error' | 'unloaded'; + +/** + * Plugin info (runtime representation) + */ +export interface PluginInfo { + id: string; + name: string; + version: string; + description?: string; + /** Display priority (lower = higher priority) */ + priority?: number; + /** Color for UI display */ + color?: string; + dependencies: string[]; + active: boolean; + config: Record; + configSchema?: Record; + metadataSchema?: Record; + status: PluginStatus; + error?: string; + /** True if this is a containerized plugin */ + isContainer?: boolean; +} + +// ============================================================================= +// Plugin Context Types (injected into plugins) +// ============================================================================= + +/** + * Key-value store for file metadata + */ +export interface PluginKVStore { + /** + * Get a value by key + * @param key Path-style key, e.g., "video/codec" or "audio/0/language" + */ + get(key: string): string | undefined; + + /** + * Set a value. Only string values are supported. + * @param key Path-style key + * @param value String value + */ + set(key: string, value: string): void; + + /** + * Delete a key + */ + delete(key: string): void; + + /** + * Get all keys, optionally filtered by prefix + * @param prefix Optional prefix filter, e.g., "audio/" + */ + keys(prefix?: string): string[]; + + /** + * Get all key-value pairs, optionally filtered by prefix + */ + entries(prefix?: string): Array<[string, string]>; + + /** + * Add a value to a set (for RecordSet fields like genres, tags). + * Optional - only available with MetadataNodeKVStore. + */ + add?(key: string, value: string): void; + + /** + * Get the full object at a path (for inspecting nested structures). + * Optional - only available with MetadataNodeKVStore. + */ + getObject?(key: string): any; +} + +/** + * Cache utilities for persistent plugin storage + */ +export interface PluginCache { + /** + * Get full path for a cache file + * @param filename Relative filename within cache directory + */ + getPath(filename: string): string; + + /** + * Read and parse a JSON cache file + * @returns Parsed data or null if not found + */ + readJson(filename: string): Promise; + + /** + * Write data as JSON to cache file + */ + writeJson(filename: string, data: unknown): Promise; + + /** + * Check if a cache file exists + */ + exists(filename: string): Promise; + + /** + * Delete a cache file + */ + delete(filename: string): Promise; + + /** + * Clear all files in the plugin's cache directory + */ + clear(): Promise; +} + +/** + * Logger interface for plugins + */ +export interface PluginLogger { + debug(message: string, ...args: unknown[]): void; + info(message: string, ...args: unknown[]): void; + warn(message: string, ...args: unknown[]): void; + error(message: string, ...args: unknown[]): void; +} + +/** + * Context passed to plugin lifecycle hooks + */ +export interface PluginLifecycleContext { + /** Plugin configuration from plugins.json */ + config: Record; + /** Absolute path to plugin's cache directory */ + cacheDir: string; + /** Scoped logger */ + log: PluginLogger; +} + +/** + * Context passed to plugin process function + */ +export interface PluginContext extends PluginLifecycleContext { + /** Key-value storage for the current file's metadata */ + kv: PluginKVStore; + /** Absolute path to the file being processed */ + filePath: string; + /** Cache utilities for persistent storage */ + cache: PluginCache; + /** + * Direct access to MetadataNode for plugins that need it. + * Use this for compatibility with existing libraries that expect MetadataNode. + * Only available when using MetadataNodeKVStore. + */ + metadata?: any; +} + +// ============================================================================= +// Plugin Interface +// ============================================================================= + +/** + * Plugin interface that all plugins must implement + */ +export interface Plugin { + /** + * Called once when the plugin is loaded at startup. + * Use for one-time initialization. + */ + onLoad?(ctx: PluginLifecycleContext): Promise; + + /** + * Called when the plugin is activated. + * Triggered at startup (if previously active) or from dashboard. + */ + onActivate?(ctx: PluginLifecycleContext): Promise; + + /** + * Called when the plugin is deactivated from the dashboard. + * Use for cleanup, releasing resources. + */ + onDeactivate?(ctx: PluginLifecycleContext): Promise; + + /** + * Called when the plugin is unloaded at shutdown. + */ + onUnload?(ctx: PluginLifecycleContext): Promise; + + /** + * Process a single file. Called for each file in the library. + * Must be implemented by all plugins. + */ + process(ctx: PluginContext): Promise; +} + +// ============================================================================= +// Plugin Manager Types +// ============================================================================= + +/** + * Loaded plugin internal representation + */ +export interface LoadedPlugin { + manifest: PluginManifest; + instance: Plugin; + status: PluginStatus; + error?: string; + pluginDir: string; +} + +/** + * Processing result for a single file + */ +export interface ProcessingResult { + success: boolean; + timings: Record; + errors: Array<{ pluginId: string; error: string }>; +} + +/** + * Plugin manager events + */ +export type PluginManagerEvent = + | { type: 'plugin:activated'; pluginId: string } + | { type: 'plugin:deactivated'; pluginId: string } + | { type: 'plugin:config-changed'; pluginId: string; config: Record } + | { type: 'plugin:error'; pluginId: string; error: Error }; + +// ============================================================================= +// Plugin Task Types (for TaskScheduler) +// ============================================================================= + +/** + * Status of a plugin task + */ +export type PluginTaskStatus = 'pending' | 'ready' | 'running' | 'completed' | 'failed'; + +/** + * Result of a plugin task execution + */ +export interface PluginTaskResult { + success: boolean; + timeMs: number; + /** KV updates to apply (key-value pairs) */ + kvUpdates?: Record; + error?: string; +} + +/** + * Plugin task representing a single plugin execution for a file + */ +export interface PluginTask { + /** Unique task ID: `${fileHash}:${pluginId}` */ + id: string; + /** File identity hash (midhash256) */ + fileHash: string; + /** Absolute path to the file */ + filePath: string; + /** Plugin to execute */ + pluginId: string; + /** Plugin IDs this task depends on */ + dependencies: string[]; + /** Target queue (from classification) */ + queue: TaskQueueType; + /** Priority based on avg execution time (lower = higher priority) */ + priority: number; + /** Estimated execution time in ms (from performanceMetrics) */ + estimatedTimeMs: number; + /** Current task status */ + status: PluginTaskStatus; + /** Timestamp when task was created */ + createdAt: number; + /** Timestamp when task started executing */ + startedAt?: number; + /** Timestamp when task completed */ + completedAt?: number; + /** Execution result */ + result?: PluginTaskResult; + /** Runtime options merged into plugin config (e.g., forceRecompute) */ + options?: Record; +} + +/** + * Queue status information + */ +export interface QueueStatus { + /** Number of tasks pending (waiting for dependencies) */ + pending: number; + /** Number of tasks ready (in queue, waiting for worker) */ + ready: number; + /** Number of tasks currently running */ + running: number; + /** Number of tasks completed */ + completed: number; + /** Number of tasks failed */ + failed: number; + /** Whether the queue is currently paused */ + isPaused: boolean; +} + +/** + * Task scheduler events + */ +export type TaskSchedulerEvent = + | { type: 'task:created'; task: PluginTask } + | { type: 'task:ready'; task: PluginTask } + | { type: 'task:started'; task: PluginTask } + | { type: 'task:complete'; task: PluginTask } + | { type: 'task:failed'; task: PluginTask; error: string } + | { type: 'file:complete'; fileHash: string; filePath: string } + | { type: 'queue:fast:idle' } + | { type: 'queue:background:started' } + | { type: 'queue:background:paused' }; diff --git a/packages/meta-sort-core/src/plugin-engine/validate.js b/packages/meta-sort-core/src/plugin-engine/validate.js new file mode 100644 index 0000000..b6b39bd --- /dev/null +++ b/packages/meta-sort-core/src/plugin-engine/validate.js @@ -0,0 +1,210 @@ +/** + * Plugin System Validation Script + * + * Simple validation to ensure the plugin system works correctly. + * Run with: node dist/plugin2/validate.js + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { existsSync, mkdirSync, rmSync } from 'fs'; +import { fileURLToPath } from 'url'; + +import { + KVStore, + PluginCacheImpl, + resolveDependencyOrder, + canActivate, + canDeactivate, + PluginManager, +} from './index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const TEST_DIR = path.join(__dirname, '../../test-plugins-validation'); +const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache-validation'); +const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins-validation.json'); + +let passed = 0; +let failed = 0; + +function assert(condition, message) { + if (condition) { + passed++; + console.log(` ✓ ${message}`); + } else { + failed++; + console.log(` ✗ ${message}`); + } +} + +async function cleanup() { + if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + if (existsSync(TEST_CACHE_DIR)) rmSync(TEST_CACHE_DIR, { recursive: true }); + if (existsSync(TEST_STATE_PATH)) rmSync(TEST_STATE_PATH); +} + +async function setup() { + await cleanup(); + mkdirSync(TEST_DIR, { recursive: true }); + mkdirSync(TEST_CACHE_DIR, { recursive: true }); +} + +async function createTestPlugin(id, deps = []) { + const pluginDir = path.join(TEST_DIR, id); + mkdirSync(pluginDir, { recursive: true }); + + const manifestYaml = `id: ${id} +name: ${id} +version: 1.0.0 +${deps.length ? `dependencies:\n${deps.map(d => ` - ${d}`).join('\n')}` : ''} +`; + await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml); + + const pluginCode = ` +export default { + async process(ctx) { + ctx.kv.set("${id}/processed", "true"); + } +}; +`; + await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode); +} + +async function testKVStore() { + console.log('\nTesting KVStore...'); + + const kv = new KVStore(); + kv.set('file/name', 'test.mkv'); + assert(kv.get('file/name') === 'test.mkv', 'set and get value'); + + assert(kv.get('missing/key') === undefined, 'missing key returns undefined'); + + kv.set('audio/0/codec', 'aac'); + kv.set('audio/0/language', 'eng'); + kv.set('audio/1/codec', 'dts'); + const audioKeys = kv.keys('audio/'); + assert(audioKeys.length === 3, 'keys with prefix filters correctly'); + + kv.delete('file/name'); + assert(kv.get('file/name') === undefined, 'delete removes key'); + + const obj = kv.toObject(); + assert(typeof obj === 'object', 'toObject returns object'); +} + +async function testPluginCache() { + console.log('\nTesting PluginCache...'); + + const cacheDir = path.join(TEST_CACHE_DIR, 'test-cache'); + const cache = new PluginCacheImpl(cacheDir); + + await cache.writeJson('test.json', { foo: 'bar' }); + const data = await cache.readJson('test.json'); + assert(data?.foo === 'bar', 'writeJson and readJson work'); + + const exists = await cache.exists('test.json'); + assert(exists === true, 'exists returns true for existing file'); + + const notExists = await cache.exists('missing.json'); + assert(notExists === false, 'exists returns false for missing file'); + + await cache.delete('test.json'); + const afterDelete = await cache.exists('test.json'); + assert(afterDelete === false, 'delete removes file'); +} + +async function testDependencyResolution() { + console.log('\nTesting Dependency Resolution...'); + + const plugins = new Map([ + ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }], + ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }], + ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }], + ]); + + const active = new Set(['a', 'b', 'c']); + const { order, errors } = resolveDependencyOrder(plugins, active); + + assert(errors.length === 0, 'no errors for valid dependency chain'); + assert(order.indexOf('a') < order.indexOf('b'), 'a comes before b'); + assert(order.indexOf('b') < order.indexOf('c'), 'b comes before c'); + + // Test can activate + const { canActivate: can, missingDeps } = canActivate('b', plugins, new Set(['a'])); + assert(can === true, 'can activate when dependencies met'); + + const { canActivate: cannot, missingDeps: missing } = canActivate('b', plugins, new Set()); + assert(cannot === false, 'cannot activate when dependencies missing'); + assert(missing.includes('a'), 'reports missing dependency'); + + // Test can deactivate + const { canDeactivate: canDeact } = canDeactivate('c', plugins, active); + assert(canDeact === true, 'can deactivate when no dependents'); + + const { canDeactivate: cannotDeact, dependents } = canDeactivate('a', plugins, active); + assert(cannotDeact === false, 'cannot deactivate when has dependents'); + assert(dependents.includes('b'), 'reports dependent plugin'); +} + +async function testPluginManager() { + console.log('\nTesting PluginManager...'); + + await createTestPlugin('plugin-a'); + await createTestPlugin('plugin-b', ['plugin-a']); + + const manager = new PluginManager({ + pluginsDir: TEST_DIR, + statePath: TEST_STATE_PATH, + cacheDir: TEST_CACHE_DIR, + }); + + const plugins = await manager.scanPlugins(); + assert(plugins.length === 2, 'scans and discovers plugins'); + + await manager.loadAll(); + assert(manager.getPlugins().length === 2, 'loads all plugins'); + + await manager.activate('plugin-a'); + await manager.activate('plugin-b'); + assert(manager.getActivePlugins().length === 2, 'activates plugins'); + + const order = manager.getExecutionOrder(); + assert(order.indexOf('plugin-a') < order.indexOf('plugin-b'), 'respects dependency order'); + + const kv = manager.createKVStore(); + const result = await manager.processFile('/test/file.mkv', kv); + assert(result.success === true, 'processes file successfully'); + assert(kv.get('plugin-a/processed') === 'true', 'plugin-a executed'); + assert(kv.get('plugin-b/processed') === 'true', 'plugin-b executed'); + + await manager.deactivate('plugin-b'); + assert(manager.getActivePlugins().length === 1, 'deactivates plugin'); + + await manager.shutdown(); +} + +async function run() { + console.log('Plugin System Validation\n========================'); + + try { + await setup(); + await testKVStore(); + await testPluginCache(); + await testDependencyResolution(); + await testPluginManager(); + await cleanup(); + + console.log(`\n========================`); + console.log(`Results: ${passed} passed, ${failed} failed`); + + if (failed > 0) { + process.exit(1); + } + } catch (error) { + console.error('\nValidation error:', error); + await cleanup(); + process.exit(1); + } +} + +run(); diff --git a/packages/meta-sort-core/src/types/FileMetadata.ts b/packages/meta-sort-core/src/types/FileMetadata.ts new file mode 100644 index 0000000..d75d800 --- /dev/null +++ b/packages/meta-sort-core/src/types/FileMetadata.ts @@ -0,0 +1,21 @@ +/** + * FileMetadata - Extended metadata for file processing + * + * Wraps HashMeta with application-specific fields for tracking + * file processing state. + */ + +import { HashMeta } from '@metazla/meta-interface'; + +export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error'; + +/** + * Extended file metadata that includes processing state + * Extends the generic HashMeta with meta-mesh specific fields + * + * Note: No longer uses tempId - midhash256 is computed immediately and serves as permanent ID + */ +export interface FileMetadata extends HashMeta { + /** Current status of file processing */ + processingStatus?: ProcessingStatus; +} diff --git a/packages/meta-sort-core/src/utils/TempIdGenerator.ts b/packages/meta-sort-core/src/utils/TempIdGenerator.ts new file mode 100644 index 0000000..c64a8fc --- /dev/null +++ b/packages/meta-sort-core/src/utils/TempIdGenerator.ts @@ -0,0 +1,29 @@ +/** + * TempIdGenerator + * + * Generates temporary IDs for files before content hash is computed. + * Uses a deterministic approach based on file path, size, and mtime. + */ + +import { createHash } from 'crypto'; +import { Stats } from 'fs'; + +/** + * Generate a temporary ID for a file based on its path and stats + * This is deterministic - same file will always get same temp ID + * + * @param filePath - Absolute path to the file + * @param stats - File stats from fs.stat() + * @returns Temporary ID as MD5 hash (without prefix - prefix added by storage layer) + */ +export function generateTempId(filePath: string, stats: Stats): string { + // Create deterministic input from file metadata (NOT content) + const input = `${filePath}:${stats.size}:${stats.mtimeMs}`; + + // Fast MD5 hash of the input + const hash = createHash('md5').update(input).digest('hex'); + + // Return hash without prefix - storage layer adds "tmp:" prefix + return hash; +} + diff --git a/packages/meta-sort-core/src/utils/TimeoutWrapper.ts b/packages/meta-sort-core/src/utils/TimeoutWrapper.ts new file mode 100644 index 0000000..8836c94 --- /dev/null +++ b/packages/meta-sort-core/src/utils/TimeoutWrapper.ts @@ -0,0 +1,76 @@ +/** + * Timeout Wrapper Utility + * + * Provides timeout functionality for async operations with configurable retry logic + */ + +export interface TimeoutConfig { + /** Base timeout in milliseconds (default: 30000ms = 30s) */ + baseTimeout: number; + /** Timeout multiplier for each retry (default: 1.5x) */ + timeoutMultiplier: number; + /** Maximum timeout in milliseconds (default: 600000ms = 10min) */ + maxTimeout: number; +} + +const DEFAULT_CONFIG: TimeoutConfig = { + baseTimeout: 30000, // 30 seconds + timeoutMultiplier: 1.5, + maxTimeout: 600000 // 10 minutes +}; + +/** + * Calculate timeout for a specific retry attempt + */ +export function calculateTimeout(retryCount: number, config: TimeoutConfig = DEFAULT_CONFIG): number { + const timeout = config.baseTimeout * Math.pow(config.timeoutMultiplier, retryCount); + return Math.min(timeout, config.maxTimeout); +} + +/** + * Execute an async function with timeout + * @param fn - Function to execute + * @param timeoutMs - Timeout in milliseconds + * @param label - Label for error messages + * @returns Promise that rejects if timeout is exceeded + */ +export async function withTimeout( + fn: () => Promise, + timeoutMs: number, + label: string = 'Operation' +): Promise { + let timeoutHandle: NodeJS.Timeout; + + const timeoutPromise = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => { + reject(new Error(`${label} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }); + + try { + const result = await Promise.race([fn(), timeoutPromise]); + clearTimeout(timeoutHandle!); + return result; + } catch (error) { + clearTimeout(timeoutHandle!); + throw error; + } +} + +/** + * Execute an async function with timeout and retry logic + * @param fn - Function to execute + * @param retryCount - Current retry count (0-based) + * @param config - Timeout configuration + * @param label - Label for error messages + * @returns Promise that resolves with result or rejects with timeout error + */ +export async function withTimeoutAndRetry( + fn: () => Promise, + retryCount: number, + config: TimeoutConfig = DEFAULT_CONFIG, + label: string = 'Operation' +): Promise { + const timeout = calculateTimeout(retryCount, config); + return withTimeout(fn, timeout, label); +} diff --git a/packages/meta-sort-core/tsconfig.json b/packages/meta-sort-core/tsconfig.json new file mode 100644 index 0000000..499f861 --- /dev/null +++ b/packages/meta-sort-core/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "resolveJsonModule": true, + "sourceMap": true, + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": false, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "**/*.spec.ts", "**/*.test.ts"] +} diff --git a/packages/meta-sort-editor/README.md b/packages/meta-sort-editor/README.md new file mode 100644 index 0000000..d66fc08 --- /dev/null +++ b/packages/meta-sort-editor/README.md @@ -0,0 +1,140 @@ +# @metazla/meta-editor + +User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface. + +## Features + +- **Search**: Find files by title, filename, or hash ID +- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more +- **Batch Operations**: Update multiple files simultaneously +- **Real-time**: Direct integration with etcd via REST API +- **User-friendly**: Clean, intuitive interface with form validation + +## Development + +### Prerequisites + +- Node.js 21.6.2+ +- pnpm 10.19.0+ +- Meta-mesh API server running (provides the `/api/metadata/*` endpoints) + +### Running Locally + +```bash +# From the root directory +pnpm run start:editor + +# Or from the package directory +cd packages/meta-editor +pnpm run dev +``` + +The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`. + +### Building + +```bash +# From the root directory +pnpm run build:editor + +# Or from the package directory +cd packages/meta-editor +pnpm run build +``` + +The built files will be in `dist/` directory. + +## Docker Deployment + +In the Docker development environment, the editor is automatically available at: + +``` +http://localhost/editor +``` + +The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`. + +## API Integration + +The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API: + +- `POST /api/metadata/search` - Search for files +- `GET /api/metadata/:hashId` - Get file metadata +- `PUT /api/metadata/:hashId` - Update complete metadata +- `PUT /api/metadata/:hashId/property` - Update specific property +- `POST /api/metadata/batch` - Batch update multiple files + +See `src/api/metadataApi.ts` for the API client implementation. + +## Usage + +### Searching for Files + +1. Select search type: "Search by Title/Filename" or "Search by Hash ID" +2. Enter your search query +3. Click "Search" +4. Results will appear in the left sidebar + +### Editing Metadata + +1. Select a file from the search results +2. Modify any fields in the editor form +3. Click "Save Changes" to persist updates +4. Use "Reset" to discard changes + +### Supported Metadata Fields + +**Basic Information:** +- Title, Original Title, Show Title +- Video Type (Movie, TV, Anime, Other) +- Year, Season, Episode + +**Jellyfin Metadata:** +- Plot, IMDb ID, TMDb ID, AniDB ID +- Director, Studio, Rating, Runtime +- Genres, Premiered Date, Release Date + +**File Information:** +- Filename, File Size, MIME Type +- Hash values (SHA-256, MD5, etc.) - read-only + +## Project Structure + +``` +packages/meta-editor/ +├── src/ +│ ├── api/ # API client for metadata operations +│ ├── components/ # React components +│ │ ├── SearchBar.tsx # Search interface +│ │ ├── FileList.tsx # Search results list +│ │ └── MetadataEditor.tsx # Main editing form +│ ├── types/ # TypeScript type definitions +│ ├── utils/ # Utility functions +│ ├── App.tsx # Main application component +│ ├── main.tsx # Entry point +│ └── index.css # Global styles +├── index.html +├── vite.config.ts +├── tsconfig.json +└── package.json +``` + +## Technology Stack + +- **React 18** - UI framework +- **TypeScript 5.7** - Type safety +- **Vite 6** - Build tool and dev server +- **Native CSS** - Styling (no CSS framework) + +## Contributing + +When adding new features: + +1. Add type definitions to `src/types/index.ts` +2. Update API client in `src/api/metadataApi.ts` +3. Create/update components as needed +4. Update this README with new features + +## License + +Part of the MetaMesh project. diff --git a/packages/meta-sort-editor/eslint.config.js b/packages/meta-sort-editor/eslint.config.js new file mode 100644 index 0000000..092408a --- /dev/null +++ b/packages/meta-sort-editor/eslint.config.js @@ -0,0 +1,28 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' + +export default tseslint.config( + { ignores: ['dist'] }, + { + extends: [js.configs.recommended, ...tseslint.configs.recommended], + files: ['**/*.{ts,tsx}'], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + plugins: { + 'react-hooks': reactHooks, + 'react-refresh': reactRefresh, + }, + rules: { + ...reactHooks.configs.recommended.rules, + 'react-refresh/only-export-components': [ + 'warn', + { allowConstantExport: true }, + ], + }, + }, +) diff --git a/packages/meta-sort-editor/index.html b/packages/meta-sort-editor/index.html new file mode 100644 index 0000000..7fd12a8 --- /dev/null +++ b/packages/meta-sort-editor/index.html @@ -0,0 +1,13 @@ + + + + + + + MetaMesh Metadata Editor + + +
+ + + diff --git a/packages/meta-sort-editor/package-lock.json b/packages/meta-sort-editor/package-lock.json new file mode 100644 index 0000000..7bc4dea --- /dev/null +++ b/packages/meta-sort-editor/package-lock.json @@ -0,0 +1,1570 @@ +{ + "name": "@meta-sort/editor", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@meta-sort/editor", + "version": "0.1.0", + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.48", + "@types/react-dom": "^18.2.18", + "@vitejs/plugin-react": "^4.2.1", + "typescript": "^5.3.3", + "vite": "^5.0.12" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", + "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", + "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helpers": "^7.28.4", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", + "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", + "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", + "dev": true, + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.28.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", + "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.5", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "dev": true, + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.54.0.tgz", + "integrity": "sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.54.0.tgz", + "integrity": "sha512-Skx39Uv+u7H224Af+bDgNinitlmHyQX1K/atIA32JP3JQw6hVODX5tkbi2zof/E69M1qH2UoN3Xdxgs90mmNYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.54.0.tgz", + "integrity": "sha512-k43D4qta/+6Fq+nCDhhv9yP2HdeKeP56QrUUTW7E6PhZP1US6NDqpJj4MY0jBHlJivVJD5P8NxrjuobZBJTCRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.54.0.tgz", + "integrity": "sha512-cOo7biqwkpawslEfox5Vs8/qj83M/aZCSSNIWpVzfU2CYHa2G3P1UN5WF01RdTHSgCkri7XOlTdtk17BezlV3A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.54.0.tgz", + "integrity": "sha512-miSvuFkmvFbgJ1BevMa4CPCFt5MPGw094knM64W9I0giUIMMmRYcGW/JWZDriaw/k1kOBtsWh1z6nIFV1vPNtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.54.0.tgz", + "integrity": "sha512-KGXIs55+b/ZfZsq9aR026tmr/+7tq6VG6MsnrvF4H8VhwflTIuYh+LFUlIsRdQSgrgmtM3fVATzEAj4hBQlaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.54.0.tgz", + "integrity": "sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.54.0.tgz", + "integrity": "sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.54.0.tgz", + "integrity": "sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.54.0.tgz", + "integrity": "sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.54.0.tgz", + "integrity": "sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.54.0.tgz", + "integrity": "sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.54.0.tgz", + "integrity": "sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.54.0.tgz", + "integrity": "sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.54.0.tgz", + "integrity": "sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.54.0.tgz", + "integrity": "sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.54.0.tgz", + "integrity": "sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.54.0.tgz", + "integrity": "sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.54.0.tgz", + "integrity": "sha512-c2V0W1bsKIKfbLMBu/WGBz6Yci8nJ/ZJdheE0EwB73N3MvHYKiKGs3mVilX4Gs70eGeDaMqEob25Tw2Gb9Nqyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.54.0.tgz", + "integrity": "sha512-woEHgqQqDCkAzrDhvDipnSirm5vxUXtSKDYTVpZG3nUdW/VVB5VdCYA2iReSj/u3yCZzXID4kuKG7OynPnB3WQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.54.0.tgz", + "integrity": "sha512-dzAc53LOuFvHwbCEOS0rPbXp6SIhAf2txMP5p6mGyOXXw5mWY8NGGbPMPrs4P1WItkfApDathBj/NzMLUZ9rtQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.54.0.tgz", + "integrity": "sha512-hYT5d3YNdSh3mbCU1gwQyPgQd3T2ne0A3KG8KSBdav5TiBg6eInVmV+TeR5uHufiIgSFg0XsOWGW5/RhNcSvPg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true + }, + "node_modules/@types/react": { + "version": "18.3.27", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz", + "integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==", + "dev": true, + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.11", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.11.tgz", + "integrity": "sha512-Sg0xJUNDU1sJNGdfGWhVHX0kkZ+HWcvmVymJbj6NSgZZmW/8S9Y2HQ5euytnIgakgxN6papOAWiwDo1ctFDcoQ==", + "dev": true, + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001762", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001762.tgz", + "integrity": "sha512-PxZwGNvH7Ak8WX5iXzoK1KPZttBXNPuaOvI2ZYU7NrlM+d9Ov+TUvlLOBNGzVXAntMSMMlJPd+jY6ovrVjSmUw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.267", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.267.tgz", + "integrity": "sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.54.0.tgz", + "integrity": "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.54.0", + "@rollup/rollup-android-arm64": "4.54.0", + "@rollup/rollup-darwin-arm64": "4.54.0", + "@rollup/rollup-darwin-x64": "4.54.0", + "@rollup/rollup-freebsd-arm64": "4.54.0", + "@rollup/rollup-freebsd-x64": "4.54.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.54.0", + "@rollup/rollup-linux-arm-musleabihf": "4.54.0", + "@rollup/rollup-linux-arm64-gnu": "4.54.0", + "@rollup/rollup-linux-arm64-musl": "4.54.0", + "@rollup/rollup-linux-loong64-gnu": "4.54.0", + "@rollup/rollup-linux-ppc64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-musl": "4.54.0", + "@rollup/rollup-linux-s390x-gnu": "4.54.0", + "@rollup/rollup-linux-x64-gnu": "4.54.0", + "@rollup/rollup-linux-x64-musl": "4.54.0", + "@rollup/rollup-openharmony-arm64": "4.54.0", + "@rollup/rollup-win32-arm64-msvc": "4.54.0", + "@rollup/rollup-win32-ia32-msvc": "4.54.0", + "@rollup/rollup-win32-x64-gnu": "4.54.0", + "@rollup/rollup-win32-x64-msvc": "4.54.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + } + } +} diff --git a/packages/meta-sort-editor/package.json b/packages/meta-sort-editor/package.json new file mode 100644 index 0000000..c891c2a --- /dev/null +++ b/packages/meta-sort-editor/package.json @@ -0,0 +1,23 @@ +{ + "name": "@meta-sort/editor", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "iso-639-3": "^3.0.1", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.48", + "@types/react-dom": "^18.2.18", + "@vitejs/plugin-react": "^4.2.1", + "typescript": "^5.3.3", + "vite": "^5.0.12" + } +} diff --git a/packages/meta-sort-editor/src/App.css b/packages/meta-sort-editor/src/App.css new file mode 100644 index 0000000..9cf5ff3 --- /dev/null +++ b/packages/meta-sort-editor/src/App.css @@ -0,0 +1,185 @@ +.app { + min-height: 100vh; + display: flex; + flex-direction: column; +} + +.app-header { + padding: 2rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.1); + background-color: rgba(0, 0, 0, 0.2); +} + +.app-header h1 { + margin: 0 0 0.5rem 0; + color: #646cff; +} + +.app-description { + margin: 0; + color: rgba(255, 255, 255, 0.6); +} + +.app-main { + flex: 1; + padding: 2rem; + max-width: 1400px; + width: 100%; + margin: 0 auto; +} + +.app-error { + padding: 1rem; + background-color: rgba(255, 0, 0, 0.1); + border: 1px solid rgba(255, 0, 0, 0.3); + border-radius: 8px; + color: #ff6b6b; + margin-bottom: 1rem; +} + +.app-loading { + text-align: center; + padding: 3rem 2rem; + color: rgba(255, 255, 255, 0.7); + font-size: 1.1rem; +} + +.app-welcome { + text-align: center; + padding: 4rem 2rem; + color: rgba(255, 255, 255, 0.6); +} + +.app-welcome h2 { + margin: 0 0 1rem 0; + color: rgba(255, 255, 255, 0.8); +} + +.app-welcome-features { + margin-top: 1rem; + margin-bottom: 1.5rem; + font-size: 0.85rem; + color: rgba(255, 255, 255, 0.5); +} + +.app-kv-browser { + margin-top: 1.5rem; + text-align: left; + max-width: 900px; + margin-left: auto; + margin-right: auto; +} + +.app-content { + display: grid; + grid-template-columns: 350px 1fr; + gap: 1.5rem; + margin-top: 1rem; +} + +.app-sidebar { + min-height: 400px; +} + +.app-editor { + min-height: 400px; +} + +.app-no-selection { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + min-height: 400px; + background-color: rgba(255, 255, 255, 0.05); + border-radius: 8px; + color: rgba(255, 255, 255, 0.5); + text-align: center; + padding: 2rem; +} + +.app-footer { + padding: 1.5rem 2rem; + border-top: 1px solid rgba(255, 255, 255, 0.1); + text-align: center; + color: rgba(255, 255, 255, 0.5); + font-size: 0.9rem; +} + +.app-footer a { + color: #646cff; + text-decoration: none; +} + +.app-footer a:hover { + text-decoration: underline; +} + +@media (max-width: 1024px) { + .app-content { + grid-template-columns: 1fr; + } + + .app-sidebar { + order: 2; + } + + .app-editor { + order: 1; + } +} + +@media (max-width: 768px) { + .app-header, + .app-main, + .app-footer { + padding: 1rem; + } + + .app-content { + gap: 1rem; + } +} + +@media (prefers-color-scheme: light) { + .app-header { + border-bottom-color: rgba(0, 0, 0, 0.1); + background-color: rgba(0, 0, 0, 0.02); + } + + .app-description { + color: rgba(0, 0, 0, 0.6); + } + + .app-error { + background-color: rgba(255, 0, 0, 0.1); + border-color: rgba(255, 0, 0, 0.3); + color: #cc0000; + } + + .app-loading { + color: rgba(0, 0, 0, 0.7); + } + + .app-welcome { + color: rgba(0, 0, 0, 0.6); + } + + .app-welcome h2 { + color: rgba(0, 0, 0, 0.8); + } + + .app-welcome-features { + color: rgba(0, 0, 0, 0.5); + } + + .app-no-selection { + background-color: rgba(0, 0, 0, 0.02); + color: rgba(0, 0, 0, 0.5); + } + + .app-footer { + border-top-color: rgba(0, 0, 0, 0.1); + color: rgba(0, 0, 0, 0.5); + } +} diff --git a/packages/meta-sort-editor/src/App.tsx b/packages/meta-sort-editor/src/App.tsx new file mode 100644 index 0000000..d0f5e64 --- /dev/null +++ b/packages/meta-sort-editor/src/App.tsx @@ -0,0 +1,246 @@ +import { useState, useEffect } from 'react'; +import { SearchBar } from './components/SearchBar'; +import { FileList } from './components/FileList'; +import { MetadataEditor } from './components/MetadataEditor'; +import { BulkMetadataEditor } from './components/BulkMetadataEditor'; +import { KVTreeView } from './components/KVTreeView'; +import { MetadataAPI } from './api/metadataApi'; +import { SearchResult } from './types'; +import './App.css'; + +function App() { + const [searchResults, setSearchResults] = useState([]); + const [selectedHashId, setSelectedHashId] = useState(); + const [selectedHashIds, setSelectedHashIds] = useState>(new Set()); + const [bulkEditMode, setBulkEditMode] = useState(false); + const [isSearching, setIsSearching] = useState(false); + const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false); + const [error, setError] = useState(null); + + // Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...) + const getHashFromUrl = (): string | null => { + const path = window.location.pathname; + const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i); + if (fileMatch) { + return fileMatch[1]; + } + // Fallback to query param for backwards compatibility + const params = new URLSearchParams(window.location.search); + return params.get('hash'); + }; + + // Update URL when file is selected + const updateUrlForFile = (hashId: string | undefined) => { + const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, ''); + if (hashId) { + const newPath = `${basePath}/file/${hashId}`; + window.history.pushState({ hashId }, '', newPath); + } else { + window.history.pushState({}, '', basePath || '/editor/'); + } + }; + + // Check for hash parameter in URL on component mount + useEffect(() => { + const loadHashFromUrl = async () => { + const hashParam = getHashFromUrl(); + + if (hashParam) { + try { + setIsLoadingDirectHash(true); + setError(null); + + // Try to get metadata for this hash + const metadata = await MetadataAPI.getMetadata(hashParam); + + // Create a search result from the metadata + const searchResult: SearchResult = { + hashId: hashParam, + metadata: metadata, + score: 1.0, + }; + + // Set the search results and select this file + setSearchResults([searchResult]); + setSelectedHashId(hashParam); + } catch (err: any) { + setError(`Failed to load file with hash ${hashParam}: ${err.message}`); + setSearchResults([]); + setSelectedHashId(undefined); + } finally { + setIsLoadingDirectHash(false); + } + } + }; + + loadHashFromUrl(); + + // Handle browser back/forward + const handlePopState = (event: PopStateEvent) => { + const hashId = event.state?.hashId || getHashFromUrl(); + if (hashId) { + setSelectedHashId(hashId); + } else { + setSelectedHashId(undefined); + } + }; + window.addEventListener('popstate', handlePopState); + return () => window.removeEventListener('popstate', handlePopState); + }, []); // Empty dependency array means this runs once on mount + + const handleSearch = async (query: string) => { + try { + setIsSearching(true); + setError(null); + + // Unified search - searches across all fields + const response = await MetadataAPI.search({ query }); + + setSearchResults(response.results); + + // Auto-select first result if available + if (response.results.length > 0) { + setSelectedHashId(response.results[0].hashId); + } else { + setSelectedHashId(undefined); + } + } catch (err: any) { + setError(err.message); + setSearchResults([]); + setSelectedHashId(undefined); + } finally { + setIsSearching(false); + } + }; + + const handleSelectFile = (hashId: string) => { + if (!bulkEditMode) { + setSelectedHashId(hashId); + setSelectedHashIds(new Set()); + updateUrlForFile(hashId); + } + }; + + const handleToggleSelection = (hashId: string) => { + const newSelection = new Set(selectedHashIds); + if (newSelection.has(hashId)) { + newSelection.delete(hashId); + } else { + newSelection.add(hashId); + } + setSelectedHashIds(newSelection); + }; + + const handleSelectAll = () => { + const allHashIds = new Set(searchResults.map(r => r.hashId)); + setSelectedHashIds(allHashIds); + }; + + const handleClearSelection = () => { + setSelectedHashIds(new Set()); + setBulkEditMode(false); + }; + + const handleBulkEditMode = (enabled: boolean) => { + setBulkEditMode(enabled); + if (enabled) { + setSelectedHashId(undefined); + } else { + setSelectedHashIds(new Set()); + } + }; + + const handleMetadataSaved = () => { + // Optionally refresh the search results to reflect changes + console.log('Metadata saved successfully'); + }; + + const handleBulkMetadataSaved = () => { + console.log('Bulk metadata saved successfully'); + handleClearSelection(); + }; + + return ( +
+
+

MetaMesh Metadata Editor

+

+ Search and edit metadata for your media files +

+
+ +
+ + + {error &&
{error}
} + + {isLoadingDirectHash && ( +
Loading file metadata...
+ )} + + {!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && ( +
+

Welcome to MetaMesh Metadata Editor

+

Search for files by title, filename, file path, or hash ID to get started.

+

+ Unified search across all fields | Single file editing | Bulk editing | Direct access via URL +

+
+ +
+
+ )} + + {searchResults.length > 0 && ( +
+
+ +
+
+ {bulkEditMode && selectedHashIds.size > 0 ? ( + + ) : selectedHashId ? ( + + ) : ( +
+ {bulkEditMode + ? 'Select files from the list to bulk edit metadata' + : 'Select a file from the list to edit its metadata'} +
+ )} +
+
+ )} +
+ + +
+ ); +} + +export default App; diff --git a/packages/meta-sort-editor/src/api/metadataApi.ts b/packages/meta-sort-editor/src/api/metadataApi.ts new file mode 100644 index 0000000..deb209f --- /dev/null +++ b/packages/meta-sort-editor/src/api/metadataApi.ts @@ -0,0 +1,151 @@ +import { + FileMetadata, + SearchResponse, + UpdateResult, + BatchUpdateResponse, +} from '../types'; + +const API_BASE = '/api/metadata'; + +export class MetadataAPI { + /** + * Search for files by query, hashId, or property + */ + static async search(params: { + query?: string; + hashId?: string; + property?: string; + propertyValue?: string; + limit?: number; + }): Promise { + const response = await fetch(`${API_BASE}/search`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(params), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Search failed'); + } + + return response.json(); + } + + /** + * Get metadata for a specific file by hash ID + */ + static async getMetadata(hashId: string): Promise { + const response = await fetch(`${API_BASE}/${hashId}`); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to get metadata'); + } + + return response.json(); + } + + /** + * Update complete metadata for a file + */ + static async updateMetadata( + hashId: string, + metadata: FileMetadata + ): Promise { + const response = await fetch(`${API_BASE}/${hashId}`, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(metadata), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to update metadata'); + } + + return response.json(); + } + + /** + * Update a specific property for a file + */ + static async updateProperty( + hashId: string, + property: string, + value: string + ): Promise { + const response = await fetch(`${API_BASE}/${hashId}/property`, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ property, value }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to update property'); + } + + return response.json(); + } + + /** + * Batch update multiple files + */ + static async batchUpdate( + updates: Array<{ + hashId: string; + metadata?: FileMetadata; + properties?: Record; + }> + ): Promise { + const response = await fetch(`${API_BASE}/batch`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ updates }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Batch update failed'); + } + + return response.json(); + } + + /** + * Delete metadata for a file + */ + static async deleteMetadata(hashId: string): Promise { + const response = await fetch(`${API_BASE}/${hashId}`, { + method: 'DELETE', + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to delete metadata'); + } + } + + /** + * Get all hash IDs + */ + static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> { + const response = await fetch(`${API_BASE}/hash-ids`); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to get hash IDs'); + } + + return response.json(); + } +} diff --git a/packages/meta-sort-editor/src/components/BulkMetadataEditor.css b/packages/meta-sort-editor/src/components/BulkMetadataEditor.css new file mode 100644 index 0000000..5a461ff --- /dev/null +++ b/packages/meta-sort-editor/src/components/BulkMetadataEditor.css @@ -0,0 +1,382 @@ +.bulk-metadata-editor { + background-color: rgba(255, 255, 255, 0.05); + border-radius: 8px; + padding: 1.5rem; +} + +.bulk-metadata-editor-header { + margin-bottom: 1rem; + padding-bottom: 1rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.1); +} + +.bulk-metadata-editor-header h2 { + margin: 0 0 0.5rem 0; + font-size: 1.5rem; +} + +.bulk-info { + margin: 0 0 1rem 0; + color: rgba(255, 255, 255, 0.6); + font-size: 0.95rem; +} + +.bulk-metadata-editor-actions { + display: flex; + gap: 0.5rem; +} + +.bulk-metadata-editor-actions button.primary { + background-color: #646cff; + color: white; +} + +.bulk-metadata-editor-actions button.primary:hover:not(:disabled) { + background-color: #535bf2; +} + +.bulk-metadata-editor-message { + padding: 1rem; + border-radius: 4px; + margin-bottom: 1rem; +} + +.bulk-metadata-editor-message.success { + background-color: rgba(0, 200, 0, 0.1); + border: 1px solid rgba(0, 200, 0, 0.3); + color: #00ff00; +} + +.bulk-metadata-editor-message.error { + background-color: rgba(255, 0, 0, 0.1); + border: 1px solid rgba(255, 0, 0, 0.3); + color: #ff6b6b; +} + +.bulk-metadata-editor-form { + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +.bulk-metadata-section { + border: 1px solid rgba(255, 255, 255, 0.1); + border-left: 4px solid #646cff; + border-radius: 8px; + padding: 1.5rem; + background-color: rgba(0, 0, 0, 0.2); +} + +.bulk-metadata-section-header { + margin-bottom: 1.5rem; + padding-bottom: 0.75rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.1); +} + +.bulk-metadata-section-header h3 { + margin: 0 0 0.25rem 0; + font-size: 1.1rem; + color: #646cff; +} + +.section-hint { + margin: 0.25rem 0 0 0; + font-size: 0.85rem; + color: rgba(255, 255, 255, 0.5); + font-style: italic; +} + +.section-hint-bulk { + margin: 0.5rem 0 0 0; + font-size: 0.8rem; + color: rgba(255, 255, 255, 0.4); + font-weight: 500; +} + +/* Plugin-based bulk fields */ +.bulk-fields-container { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.bulk-field-wrapper { + border: 1px solid rgba(255, 255, 255, 0.05); + border-radius: 4px; + padding: 0.75rem; + background-color: rgba(0, 0, 0, 0.1); + transition: all 0.2s ease; +} + +.bulk-field-wrapper.has-value { + border-color: rgba(100, 108, 255, 0.3); + background-color: rgba(100, 108, 255, 0.05); +} + +.bulk-field-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.5rem; +} + +.field-key { + font-size: 0.85rem; + font-weight: 600; + color: rgba(255, 255, 255, 0.6); + text-transform: lowercase; + font-family: 'Courier New', monospace; +} + +.bulk-field-wrapper.has-value .field-key { + color: #646cff; +} + +.bulk-field-input { + /* Plugin component will render inside this */ +} + +/* Legacy bulk fields (for compatibility) */ +.bulk-metadata-field { + display: flex; + flex-direction: column; + gap: 0.5rem; + margin-bottom: 1rem; +} + +.bulk-metadata-field:last-child { + margin-bottom: 0; +} + +.field-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.field-header label { + font-size: 0.9rem; + font-weight: 500; + color: rgba(255, 255, 255, 0.7); +} + +.clear-field-btn { + width: 20px; + height: 20px; + padding: 0; + border: 1px solid rgba(255, 0, 0, 0.3); + background-color: rgba(255, 0, 0, 0.1); + color: #ff6b6b; + border-radius: 50%; + cursor: pointer; + font-size: 16px; + line-height: 1; + transition: all 0.2s; + display: flex; + align-items: center; + justify-content: center; +} + +.clear-field-btn:hover { + background-color: rgba(255, 0, 0, 0.2); + border-color: rgba(255, 0, 0, 0.5); +} + +.bulk-metadata-field input, +.bulk-metadata-field textarea, +.bulk-metadata-field select { + width: 100%; +} + +.bulk-metadata-field-row { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 1rem; + margin-bottom: 1rem; +} + +.bulk-metadata-field-row:last-child { + margin-bottom: 0; +} + +/* Modified fields summary */ +.modified-fields { + background-color: rgba(100, 108, 255, 0.05); + border-color: rgba(100, 108, 255, 0.3); +} + +.modified-fields h3 { + color: #646cff; +} + +.modified-fields-list { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.modified-field-item { + display: flex; + gap: 0.75rem; + align-items: center; + padding: 0.75rem; + background-color: rgba(100, 108, 255, 0.05); + border: 1px solid rgba(100, 108, 255, 0.2); + border-radius: 4px; +} + +.field-name { + font-weight: 600; + color: #646cff; + min-width: 150px; + font-family: 'Courier New', monospace; + font-size: 0.9rem; +} + +.field-value { + flex: 1; + word-break: break-word; + color: rgba(255, 255, 255, 0.8); + font-size: 0.9rem; +} + +.remove-field-btn { + padding: 0.25rem 0.75rem; + font-size: 0.85rem; + background-color: rgba(255, 0, 0, 0.1); + border: 1px solid rgba(255, 0, 0, 0.3); + color: #ff6b6b; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s ease; + white-space: nowrap; +} + +.remove-field-btn:hover { + background-color: rgba(255, 0, 0, 0.2); + border-color: rgba(255, 0, 0, 0.5); +} + +@media (max-width: 768px) { + .bulk-metadata-editor-header { + flex-direction: column; + align-items: flex-start; + gap: 1rem; + } + + .bulk-metadata-editor-actions { + width: 100%; + flex-direction: column; + } + + .bulk-metadata-editor-actions button { + width: 100%; + } + + .bulk-metadata-field-row { + grid-template-columns: 1fr; + } + + .modified-field-item { + flex-direction: column; + align-items: flex-start; + } + + .field-name { + min-width: auto; + } + + .remove-field-btn { + align-self: flex-end; + } +} + +@media (prefers-color-scheme: light) { + .bulk-metadata-editor { + background-color: rgba(0, 0, 0, 0.02); + } + + .bulk-metadata-editor-header { + border-bottom-color: rgba(0, 0, 0, 0.1); + } + + .bulk-info { + color: rgba(0, 0, 0, 0.6); + } + + .bulk-metadata-section { + border-color: rgba(0, 0, 0, 0.1); + background-color: rgba(0, 0, 0, 0.02); + } + + .bulk-metadata-section-header { + border-bottom-color: rgba(0, 0, 0, 0.1); + } + + .section-hint { + color: rgba(0, 0, 0, 0.5); + } + + .section-hint-bulk { + color: rgba(0, 0, 0, 0.4); + } + + .bulk-field-wrapper { + border-color: rgba(0, 0, 0, 0.1); + background-color: rgba(0, 0, 0, 0.02); + } + + .bulk-field-wrapper.has-value { + border-color: rgba(100, 108, 255, 0.3); + background-color: rgba(100, 108, 255, 0.05); + } + + .field-key { + color: rgba(0, 0, 0, 0.6); + } + + .bulk-field-wrapper.has-value .field-key { + color: #646cff; + } + + .field-header label { + color: rgba(0, 0, 0, 0.7); + } + + .field-value { + color: rgba(0, 0, 0, 0.8); + } + + .bulk-metadata-editor-message.success { + background-color: rgba(0, 200, 0, 0.1); + border-color: rgba(0, 200, 0, 0.3); + color: #008800; + } + + .bulk-metadata-editor-message.error { + background-color: rgba(255, 0, 0, 0.1); + border-color: rgba(255, 0, 0, 0.3); + color: #cc0000; + } + + .modified-fields { + background-color: rgba(100, 108, 255, 0.05); + border-color: rgba(100, 108, 255, 0.3); + } + + .modified-field-item { + background-color: rgba(100, 108, 255, 0.05); + border-color: rgba(100, 108, 255, 0.2); + } + + .clear-field-btn { + border-color: rgba(255, 0, 0, 0.3); + background-color: rgba(255, 0, 0, 0.1); + color: #cc0000; + } + + .clear-field-btn:hover { + background-color: rgba(255, 0, 0, 0.2); + border-color: rgba(255, 0, 0, 0.5); + } +} diff --git a/packages/meta-sort-editor/src/components/BulkMetadataEditor.tsx b/packages/meta-sort-editor/src/components/BulkMetadataEditor.tsx new file mode 100644 index 0000000..f054584 --- /dev/null +++ b/packages/meta-sort-editor/src/components/BulkMetadataEditor.tsx @@ -0,0 +1,280 @@ +import { useState, useEffect } from 'react'; +import { FileMetadata, PluginSchema } from '../types'; +import { MetadataAPI } from '../api/metadataApi'; +import { DynamicFieldRenderer } from './DynamicFieldRenderer'; +import './BulkMetadataEditor.css'; + +interface PluginsResponse { + plugins: PluginSchema[]; +} + +interface BulkMetadataEditorProps { + hashIds: string[]; + onSaved?: () => void; + onCancel?: () => void; +} + +export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) { + const [plugins, setPlugins] = useState([]); + const [commonValues, setCommonValues] = useState>({}); + const [editedFields, setEditedFields] = useState>({}); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [error, setError] = useState(null); + const [successMessage, setSuccessMessage] = useState(null); + + // Load plugins and metadata on mount + useEffect(() => { + const loadData = async () => { + try { + setIsLoading(true); + setError(null); + + // Fetch plugins + const pluginsResponse = await fetch('/api/plugins'); + if (pluginsResponse.ok) { + const data: PluginsResponse = await pluginsResponse.json(); + // Sort by priority + const sorted = [...data.plugins].sort((a, b) => + (a.priority ?? 100) - (b.priority ?? 100) + ); + setPlugins(sorted); + } + + // Load metadata for all selected files + const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId)); + const metadata = await Promise.all(metadataPromises); + + // Calculate common values + const commons: Record = {}; + + // Get all editable fields from all plugins + const allFields: string[] = []; + plugins.forEach((plugin) => { + if (plugin.metadataSchema) { + Object.entries(plugin.metadataSchema).forEach(([key, field]) => { + if (!field.readonly) { + allFields.push(key); + } + }); + } + }); + + allFields.forEach((field) => { + const values = metadata.map((m) => m[field as keyof FileMetadata]); + + // Filter out undefined and null values + const definedValues = values.filter((v) => v !== undefined && v !== null); + + if (definedValues.length === 0) { + return; + } + + // Safely stringify values + const uniqueValues = Array.from( + new Set( + definedValues.map((v) => { + try { + return JSON.stringify(v); + } catch { + return String(v); + } + }) + ) + ); + + if (uniqueValues.length === 1) { + try { + commons[field] = JSON.parse(uniqueValues[0]); + } catch { + commons[field] = uniqueValues[0]; + } + } else { + commons[field] = '*'; + } + }); + + setCommonValues(commons); + } catch (err: any) { + setError(`Failed to load data: ${err.message}`); + } finally { + setIsLoading(false); + } + }; + + loadData(); + }, [hashIds]); + + const handleFieldChange = (field: string, value: any) => { + setEditedFields({ + ...editedFields, + [field]: value, + }); + }; + + const handleClearField = (field: string) => { + const newFields = { ...editedFields }; + delete newFields[field]; + setEditedFields(newFields); + }; + + const handleSave = async () => { + if (Object.keys(editedFields).length === 0) { + setError('No fields have been edited'); + return; + } + + try { + setIsSaving(true); + setError(null); + setSuccessMessage(null); + + // Build batch update payload + const updates = hashIds.map((hashId) => ({ + hashId, + metadata: editedFields, + })); + + const result = await MetadataAPI.batchUpdate(updates); + + if (result.errors > 0) { + setError( + `Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.` + ); + console.error('Batch update errors:', result.results.filter((r) => r.error)); + } else { + setSuccessMessage(`Successfully updated ${result.success} files!`); + } + + if (onSaved) { + onSaved(); + } + + setTimeout(() => setSuccessMessage(null), 3000); + } catch (err: any) { + setError(err.message); + } finally { + setIsSaving(false); + } + }; + + const hasChanges = Object.keys(editedFields).length > 0; + + // Filter to only editable plugins (those with non-readonly fields) + const editablePlugins = plugins.filter((plugin) => { + if (!plugin.metadataSchema) return false; + return Object.values(plugin.metadataSchema).some((field) => !field.readonly); + }); + + if (isLoading) { + return ( +
+
+ Loading metadata for {hashIds.length} files... +
+
+ ); + } + + // Build display metadata (edited values or common values) + const displayMetadata: Record = { ...commonValues, ...editedFields }; + + return ( +
+
+

Bulk Edit Metadata

+

+ Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will + be updated. Fields showing "*" have different values across files. +

+
+ {onCancel && ( + + )} + +
+
+ + {successMessage && ( +
{successMessage}
+ )} + {error &&
{error}
} + +
+ {editablePlugins.map((plugin) => { + if (!plugin.metadataSchema) return null; + + // Get editable fields for this plugin + const editableFields = Object.entries(plugin.metadataSchema).filter( + ([, field]) => !field.readonly + ); + + if (editableFields.length === 0) return null; + + return ( +
+
+

{plugin.name}

+ {plugin.description && ( +

{plugin.description}

+ )} +

+ Common values shown. "*" indicates different values. Edit to update all files. +

+
+ +
+ {editableFields.map(([key, field]) => ( + + ))} +
+
+ ); + })} + + {/* Modified Fields Summary */} + {hasChanges && ( +
+

Fields to be Updated ({Object.keys(editedFields).length})

+
+ {Object.entries(editedFields).map(([key, value]) => ( +
+ {key}: + + {Array.isArray(value) + ? value.join(', ') + : typeof value === 'object' + ? JSON.stringify(value) + : String(value || '(empty)')} + + +
+ ))} +
+
+ )} +
+
+ ); +} diff --git a/packages/meta-sort-editor/src/components/DynamicFieldRenderer.css b/packages/meta-sort-editor/src/components/DynamicFieldRenderer.css new file mode 100644 index 0000000..fac5e4f --- /dev/null +++ b/packages/meta-sort-editor/src/components/DynamicFieldRenderer.css @@ -0,0 +1,297 @@ +.dynamic-field { + display: flex; + flex-direction: column; + gap: 0.5rem; + margin-bottom: 0.5rem; +} + +.dynamic-field:last-child { + margin-bottom: 0; +} + +.dynamic-field label { + font-size: 0.9rem; + font-weight: 500; + color: rgba(255, 255, 255, 0.7); +} + +.dynamic-field input[type="text"], +.dynamic-field input[type="number"], +.dynamic-field select, +.dynamic-field textarea { + width: 100%; + padding: 0.5rem 0.75rem; + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 4px; + font-size: 0.9rem; + background: rgba(0, 0, 0, 0.3); + color: #fff; + transition: border-color 0.2s; +} + +.dynamic-field input:focus, +.dynamic-field select:focus, +.dynamic-field textarea:focus { + outline: none; + border-color: #646cff; +} + +.dynamic-field input:disabled, +.dynamic-field select:disabled, +.dynamic-field textarea:disabled { + background: rgba(0, 0, 0, 0.2); + color: rgba(255, 255, 255, 0.5); + cursor: not-allowed; +} + +.dynamic-field textarea { + resize: vertical; + min-height: 80px; + font-family: inherit; +} + +.field-hint { + display: block; + font-size: 0.8rem; + color: rgba(255, 255, 255, 0.4); +} + +.checkbox-label { + display: flex; + align-items: center; + gap: 0.5rem; + cursor: pointer; + font-weight: normal; + color: rgba(255, 255, 255, 0.7); +} + +.checkbox-label input[type="checkbox"] { + width: 1.1rem; + height: 1.1rem; + cursor: pointer; + accent-color: #646cff; +} + +.checkbox-label input:disabled { + cursor: not-allowed; + opacity: 0.5; +} + +.readonly-value { + padding: 0.5rem 0.75rem; + background: rgba(0, 0, 0, 0.2); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 4px; + color: rgba(255, 255, 255, 0.7); + font-family: 'Courier New', monospace; + font-size: 0.9rem; + word-break: break-all; +} + +.field-link { + color: #646cff; + text-decoration: none; +} + +.field-link:hover { + text-decoration: underline; +} + +/* CID field styling */ +.cid-field .cid-value-container { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.cid-field .cid-value { + flex: 1; + padding: 0.5rem 0.75rem; + background: rgba(0, 0, 0, 0.3); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 4px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace; + font-size: 0.85rem; + color: #a0d8ef; + cursor: pointer; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.cid-field .cid-value:hover { + background: rgba(0, 0, 0, 0.4); + border-color: #646cff; +} + +.cid-field .cid-open-button { + display: inline-flex; + align-items: center; + padding: 0.5rem 0.75rem; + background: #646cff; + color: #fff; + border-radius: 4px; + text-decoration: none; + font-size: 0.85rem; + font-weight: 500; + white-space: nowrap; + transition: background 0.2s; +} + +.cid-field .cid-open-button:hover { + background: #535bf2; +} + +/* JSON field styling */ +.json-field .json-viewer { + background: rgba(0, 0, 0, 0.4); + color: #d4d4d4; + padding: 1rem; + border-radius: 4px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace; + font-size: 0.85rem; + overflow-x: auto; + max-height: 300px; + overflow-y: auto; + margin: 0; + white-space: pre-wrap; + word-break: break-all; + border: 1px solid rgba(255, 255, 255, 0.1); +} + +/* Language string field styling */ +.language-string-field .language-entries { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.language-string-field .language-entry { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.language-string-field .lang-input { + flex: 1; + min-width: 0; +} + +.language-string-field .lang-code { + display: inline-block; + width: 50px; + padding: 0.4rem 0.5rem; + background: rgba(100, 108, 255, 0.2); + border: 1px solid rgba(100, 108, 255, 0.3); + border-radius: 4px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace; + font-size: 0.8rem; + color: #a0d8ef; + text-align: center; + flex-shrink: 0; +} + +.language-string-field .lang-text { + flex: 1; + padding: 0.4rem 0.75rem; + background: rgba(0, 0, 0, 0.2); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 4px; + color: rgba(255, 255, 255, 0.8); + font-size: 0.9rem; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.language-string-field .lang-remove-btn { + width: 32px; + height: 32px; + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: center; + background: rgba(255, 100, 100, 0.2); + border: 1px solid rgba(255, 100, 100, 0.3); + border-radius: 4px; + color: #ff6b6b; + font-size: 1.2rem; + cursor: pointer; + transition: background 0.2s, border-color 0.2s; +} + +.language-string-field .lang-remove-btn:hover { + background: rgba(255, 100, 100, 0.3); + border-color: rgba(255, 100, 100, 0.5); +} + +.language-string-field .lang-add-btn { + align-self: flex-start; + padding: 0.4rem 0.75rem; + background: rgba(100, 108, 255, 0.2); + border: 1px solid rgba(100, 108, 255, 0.3); + border-radius: 4px; + color: #646cff; + font-size: 0.85rem; + cursor: pointer; + transition: background 0.2s, border-color 0.2s; +} + +.language-string-field .lang-add-btn:hover { + background: rgba(100, 108, 255, 0.3); + border-color: rgba(100, 108, 255, 0.5); +} + +/* Light theme */ +@media (prefers-color-scheme: light) { + .dynamic-field label { + color: rgba(0, 0, 0, 0.7); + } + + .dynamic-field input[type="text"], + .dynamic-field input[type="number"], + .dynamic-field select, + .dynamic-field textarea { + border-color: rgba(0, 0, 0, 0.1); + background: #fff; + color: #333; + } + + .dynamic-field input:disabled, + .dynamic-field select:disabled, + .dynamic-field textarea:disabled { + background: #f5f5f5; + color: rgba(0, 0, 0, 0.5); + } + + .field-hint { + color: rgba(0, 0, 0, 0.5); + } + + .checkbox-label { + color: rgba(0, 0, 0, 0.7); + } + + .readonly-value { + background: #f8f9fa; + border-color: #e9ecef; + color: #495057; + } + + .json-field .json-viewer { + background: #1e1e1e; + border-color: rgba(0, 0, 0, 0.2); + } + + .language-string-field .lang-code { + background: rgba(100, 108, 255, 0.1); + color: #4a50c8; + } + + .language-string-field .lang-text { + background: #f8f9fa; + border-color: #e9ecef; + color: #495057; + } +} diff --git a/packages/meta-sort-editor/src/components/DynamicFieldRenderer.tsx b/packages/meta-sort-editor/src/components/DynamicFieldRenderer.tsx new file mode 100644 index 0000000..9a0d885 --- /dev/null +++ b/packages/meta-sort-editor/src/components/DynamicFieldRenderer.tsx @@ -0,0 +1,378 @@ +import { SchemaField, FileMetadata } from '../types'; +import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox'; +import './DynamicFieldRenderer.css'; + +interface DynamicFieldRendererProps { + fieldKey: string; + schema: SchemaField; + metadata: FileMetadata; + onChange: (key: string, value: any) => void; + readonly?: boolean; +} + +/** + * Renders a single metadata field based on its schema definition + */ +export function DynamicFieldRenderer({ + fieldKey, + schema, + metadata, + onChange, + readonly = false, +}: DynamicFieldRendererProps) { + const isReadonly = readonly || schema.readonly; + const fieldType = schema.type || 'string'; + + // Get value from metadata using path-style key + const getValue = (): any => { + // Handle path-style keys like "stream/duration" or "nfo/title" + const parts = fieldKey.split('/'); + let value: any = metadata; + for (const part of parts) { + if (value === undefined || value === null) return undefined; + value = value[part]; + } + return value; + }; + + // Set value in metadata using path-style key + const setValue = (newValue: any) => { + onChange(fieldKey, newValue); + }; + + const value = getValue(); + + // Render link if configured + const renderValueWithLink = (displayValue: string) => { + if (schema.link && displayValue) { + const url = schema.link.replace('{value}', encodeURIComponent(displayValue)); + return ( + + {displayValue} + + ); + } + return displayValue; + }; + + // Render based on field type + switch (fieldType) { + case 'boolean': + return ( +
+ + {schema.hint && {schema.hint}} +
+ ); + + case 'number': + return ( +
+ + setValue(e.target.value ? Number(e.target.value) : undefined)} + disabled={isReadonly} + min={schema.min} + max={schema.max} + placeholder={schema.placeholder} + /> + {schema.hint && {schema.hint}} +
+ ); + + case 'select': + return ( +
+ + + {schema.hint && {schema.hint}} +
+ ); + + case 'text': + return ( +
+ +