move feature from meta-sort to meta-core

This commit is contained in:
Pierre
2026-01-25 22:01:25 +01:00
parent 768942d28b
commit 7d9d502912
69 changed files with 12831 additions and 13 deletions
+186
View File
@@ -0,0 +1,186 @@
package api
import (
"net/http"
"net/url"
"strconv"
"strings"
"github.com/gorilla/mux"
)
// KVInfoResponse is the response for GET /api/kv/info
type KVInfoResponse struct {
Prefix string `json:"prefix"`
FileCount int `json:"fileCount"`
KeyCount int `json:"keyCount"`
TotalSize int64 `json:"totalSize"`
MemoryUsage string `json:"memoryUsage"`
}
// KVKeysResponse is the response for GET /api/kv/keys
type KVKeysResponse struct {
Keys []string `json:"keys"`
Cursor string `json:"cursor"`
HasMore bool `json:"hasMore"`
}
// KVKeyValueResponse is the response for GET /api/kv/key/{key}
type KVKeyValueResponse struct {
Key string `json:"key"`
Type string `json:"type"`
Value map[string]string `json:"value"`
}
// handleKVInfo handles GET /api/kv/info
func (s *Server) handleKVInfo(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Get all hash IDs to count files
hashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Get memory usage
memoryUsage, err := s.storage.GetMemoryInfo()
if err != nil {
memoryUsage = "N/A"
}
// Calculate total size from sizeByte metadata (if available)
var totalSize int64
for _, hashID := range hashIDs {
sizeStr, err := s.storage.GetProperty(hashID, "sizeByte")
if err == nil && sizeStr != "" {
if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil {
totalSize += size
}
}
}
response := KVInfoResponse{
Prefix: "file:",
FileCount: len(hashIDs),
KeyCount: len(hashIDs),
TotalSize: totalSize,
MemoryUsage: memoryUsage,
}
writeJSON(w, http.StatusOK, response)
}
// handleKVKeys handles GET /api/kv/keys
func (s *Server) handleKVKeys(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Parse query parameters
cursorStr := r.URL.Query().Get("cursor")
countStr := r.URL.Query().Get("count")
cursor := 0
if cursorStr != "" {
if c, err := strconv.Atoi(cursorStr); err == nil {
cursor = c
}
}
count := 50
if countStr != "" {
if c, err := strconv.Atoi(countStr); err == nil && c > 0 {
count = c
}
}
// Get all hash IDs
allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Simple offset-based pagination
var keys []string
end := cursor + count
if end > len(allHashIDs) {
end = len(allHashIDs)
}
if cursor < len(allHashIDs) {
for _, id := range allHashIDs[cursor:end] {
keys = append(keys, "file:"+id)
}
}
nextCursor := cursor + len(keys)
hasMore := nextCursor < len(allHashIDs)
response := KVKeysResponse{
Keys: keys,
Cursor: strconv.Itoa(nextCursor),
HasMore: hasMore,
}
if !hasMore {
response.Cursor = "0"
}
writeJSON(w, http.StatusOK, response)
}
// handleKVGetKey handles GET /api/kv/key/{key}
func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
key := vars["key"]
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
// URL decode the key (in case it contains encoded characters)
decodedKey, err := url.PathUnescape(key)
if err != nil {
decodedKey = key
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Extract hash ID from key (format: file:{hashId})
if !strings.HasPrefix(decodedKey, "file:") {
writeError(w, http.StatusBadRequest, "invalid key format, expected: file:{hashId}")
return
}
hashID := strings.TrimPrefix(decodedKey, "file:")
// Get metadata for this hash ID
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if metadata == nil {
writeError(w, http.StatusNotFound, "key not found")
return
}
response := KVKeyValueResponse{
Key: decodedKey,
Type: "hash",
Value: metadata,
}
writeJSON(w, http.StatusOK, response)
}
+568
View File
@@ -0,0 +1,568 @@
package api
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
)
// HashIDsResponse is the response for GET /api/metadata/hash-ids
type HashIDsResponse struct {
HashIds []string `json:"hashIds"`
Count int `json:"count"`
}
// MetadataListItem represents a file in the metadata list
type MetadataListItem struct {
HashID string `json:"hashId"`
Title string `json:"title,omitempty"`
FileName string `json:"fileName,omitempty"`
FilePath string `json:"filePath,omitempty"`
Type string `json:"type,omitempty"`
Year string `json:"year,omitempty"`
}
// MetadataListResponse is the response for GET /api/metadata/list
type MetadataListResponse struct {
Items []MetadataListItem `json:"items"`
Count int `json:"count"`
Total int `json:"total"`
}
// MetadataSearchRequest is the request body for POST /api/metadata/search
type MetadataSearchRequest struct {
Query string `json:"query,omitempty"`
HashID string `json:"hashId,omitempty"`
Property string `json:"property,omitempty"`
PropertyValue string `json:"propertyValue,omitempty"`
Limit int `json:"limit,omitempty"`
}
// MetadataSearchResult is a single search result
type MetadataSearchResult struct {
HashID string `json:"hashId"`
Metadata map[string]string `json:"metadata"`
}
// MetadataSearchResponse is the response for POST /api/metadata/search
type MetadataSearchResponse struct {
Results []MetadataSearchResult `json:"results"`
Count int `json:"count"`
Total int `json:"total"`
}
// MetadataBatchUpdateRequest is the request body for POST /api/metadata/batch
type MetadataBatchUpdateRequest struct {
Updates []MetadataBatchItem `json:"updates"`
}
// MetadataBatchItem is a single item in a batch update
type MetadataBatchItem struct {
HashID string `json:"hashId"`
Metadata map[string]string `json:"metadata,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
}
// MetadataBatchResult is a single result from a batch update
type MetadataBatchResult struct {
HashID string `json:"hashId"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
}
// MetadataBatchResponse is the response for POST /api/metadata/batch
type MetadataBatchResponse struct {
Status string `json:"status"`
Total int `json:"total"`
Success int `json:"success"`
Errors int `json:"errors"`
Results []MetadataBatchResult `json:"results"`
}
// PropertyUpdateRequest is the request body for PUT /api/metadata/{hashId}/property
type PropertyUpdateRequest struct {
Property string `json:"property"`
Value string `json:"value"`
}
// handleGetHashIds handles GET /api/metadata/hash-ids
func (s *Server) handleGetHashIds(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
hashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
response := HashIDsResponse{
HashIds: hashIDs,
Count: len(hashIDs),
}
writeJSON(w, http.StatusOK, response)
}
// handleListMetadata handles GET /api/metadata/list
func (s *Server) handleListMetadata(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Parse pagination parameters
offsetStr := r.URL.Query().Get("offset")
limitStr := r.URL.Query().Get("limit")
offset := 0
if offsetStr != "" {
if o, err := strconv.Atoi(offsetStr); err == nil {
offset = o
}
}
limit := 100
if limitStr != "" {
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 {
limit = l
}
}
// Get all hash IDs
allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
total := len(allHashIDs)
// Apply pagination
end := offset + limit
if end > total {
end = total
}
var items []MetadataListItem
if offset < total {
for _, hashID := range allHashIDs[offset:end] {
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil || metadata == nil {
continue
}
item := MetadataListItem{
HashID: hashID,
Title: metadata["title"],
FileName: metadata["fileName"],
FilePath: metadata["filePath"],
Type: metadata["type"],
Year: metadata["year"],
}
items = append(items, item)
}
}
response := MetadataListResponse{
Items: items,
Count: len(items),
Total: total,
}
writeJSON(w, http.StatusOK, response)
}
// handleGetMetadataByHashId handles GET /api/metadata/{hashId}
func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if metadata == nil {
writeError(w, http.StatusNotFound, "metadata not found")
return
}
response := MetadataResponse{
HashID: hashID,
Metadata: metadata,
}
writeJSON(w, http.StatusOK, response)
}
// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId}
func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
// Exclude internal processing status field
delete(metadata, "processingStatus")
if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"message": "Metadata updated successfully",
})
}
// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId}
func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
deletedCount, err := s.storage.DeleteMetadata(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"deletedKeys": deletedCount,
})
}
// handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X
func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
property := r.URL.Query().Get("property")
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if property == "" {
writeError(w, http.StatusBadRequest, "property parameter is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
value, err := s.storage.GetProperty(hashID, property)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if value == "" {
writeError(w, http.StatusNotFound, "property not found")
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"hashId": hashID,
"property": property,
"value": value,
})
}
// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property
func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req PropertyUpdateRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Property == "" {
writeError(w, http.StatusBadRequest, "property is required")
return
}
if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"property": req.Property,
"value": req.Value,
"message": "Property updated successfully",
})
}
// handleSearchMetadata handles POST /api/metadata/search
func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req MetadataSearchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Limit == 0 {
req.Limit = 100
}
// If hashId is provided, do exact match
if req.HashID != "" {
metadata, err := s.storage.GetMetadataFlat(req.HashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if metadata != nil {
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: []MetadataSearchResult{{HashID: req.HashID, Metadata: metadata}},
Count: 1,
Total: 1,
})
return
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: []MetadataSearchResult{},
Count: 0,
Total: 0,
})
return
}
// Get all hash IDs
allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
var results []MetadataSearchResult
total := len(allHashIDs)
// Search fields for query
searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
for _, hashID := range allHashIDs {
if len(results) >= req.Limit {
break
}
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil || metadata == nil {
continue
}
matches := false
// Property search
if req.Property != "" && req.PropertyValue != "" {
value, ok := metadata[req.Property]
if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) {
matches = true
}
}
// General query search
if req.Query != "" && req.Property == "" {
queryLower := strings.ToLower(req.Query)
// Check if query matches hashId
if strings.Contains(strings.ToLower(hashID), queryLower) {
matches = true
}
// Check search fields
if !matches {
for _, field := range searchFields {
if value, ok := metadata[field]; ok {
if strings.Contains(strings.ToLower(value), queryLower) {
matches = true
break
}
}
}
}
}
// If no query or property filter, return all
if req.Query == "" && req.Property == "" {
matches = true
}
if matches {
results = append(results, MetadataSearchResult{
HashID: hashID,
Metadata: metadata,
})
}
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: total,
})
}
// handleBatchUpdate handles POST /api/metadata/batch
func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req MetadataBatchUpdateRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if len(req.Updates) == 0 {
writeError(w, http.StatusBadRequest, "updates array is required")
return
}
var results []MetadataBatchResult
successCount := 0
errorCount := 0
for _, update := range req.Updates {
if update.HashID == "" {
results = append(results, MetadataBatchResult{
HashID: "unknown",
Status: "error",
Error: "Missing hash ID",
})
errorCount++
continue
}
var updateErr error
// Update complete metadata
if update.Metadata != nil {
// Exclude internal processing status
delete(update.Metadata, "processingStatus")
updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata)
}
// Update individual properties
if updateErr == nil && update.Properties != nil {
for property, value := range update.Properties {
if err := s.storage.SetProperty(update.HashID, property, value); err != nil {
updateErr = err
break
}
}
}
if updateErr != nil {
results = append(results, MetadataBatchResult{
HashID: update.HashID,
Status: "error",
Error: updateErr.Error(),
})
errorCount++
} else {
results = append(results, MetadataBatchResult{
HashID: update.HashID,
Status: "ok",
})
successCount++
}
}
writeJSON(w, http.StatusOK, MetadataBatchResponse{
Status: "completed",
Total: len(results),
Success: successCount,
Errors: errorCount,
Results: results,
})
}
// handleClearMetadata handles POST /api/metadata/clear
func (s *Server) handleClearMetadata(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
deletedCount, err := s.storage.ClearAllMetadata()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Metadata cleared",
"deletedCount": deletedCount,
})
}
+80 -6
View File
@@ -11,17 +11,24 @@ import (
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/mounts"
"github.com/metazla/meta-core/internal/storage"
"github.com/metazla/meta-core/internal/watcher"
)
// Server is the HTTP API server for meta-core
type Server struct {
config *config.Config
election *leader.Election
discovery *discovery.Service
storage *storage.Client
router *mux.Router
server *http.Server
config *config.Config
election *leader.Election
discovery *discovery.Service
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
router *mux.Router
server *http.Server
}
// NewServer creates a new API server
@@ -39,6 +46,27 @@ func NewServer(
router: mux.NewRouter(),
}
// Initialize mounts manager
mountsManager, err := mounts.NewManager(cfg)
if err != nil {
log.Printf("[API] Warning: failed to initialize mounts manager: %v", err)
} else {
s.mountsManager = mountsManager
s.mountsHandlers = mounts.NewHandlers(mountsManager)
}
// Initialize file watcher (if enabled)
if cfg.EnableFileWatcher && len(cfg.WatchFolderList) > 0 {
s.watcherDispatcher = watcher.NewDispatcher()
fileWatcher, err := watcher.NewWatcher(cfg, s.watcherDispatcher)
if err != nil {
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
} else {
s.fileWatcher = fileWatcher
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
}
}
s.setupRoutes()
return s
}
@@ -51,6 +79,23 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
s.router.HandleFunc("/role", s.handleRole).Methods("GET")
// Metadata Editor API routes (must be before /meta/{hash} routes)
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
// KV Browser API routes
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
// Metadata operations - base endpoints
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
@@ -78,6 +123,21 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
// Mount management routes (if manager initialized)
if s.mountsHandlers != nil {
s.mountsHandlers.RegisterRoutes(s.router)
log.Println("[API] Mount management routes registered at /api/mounts/*")
}
log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
log.Println("[API] KV Browser routes registered at /api/kv/*")
// File watcher routes (if watcher initialized)
if s.watcherHandlers != nil {
s.watcherHandlers.RegisterRoutes(s.router)
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
}
// Add middleware
s.router.Use(loggingMiddleware)
s.router.Use(corsMiddleware)
@@ -103,11 +163,25 @@ func (s *Server) Start() error {
}
}()
// Start file watcher (if initialized)
if s.fileWatcher != nil {
if err := s.fileWatcher.Start(); err != nil {
log.Printf("[API] Warning: failed to start file watcher: %v", err)
}
}
return nil
}
// Stop gracefully stops the HTTP server
func (s *Server) Stop() error {
// Stop file watcher
if s.fileWatcher != nil {
if err := s.fileWatcher.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop file watcher: %v", err)
}
}
if s.server == nil {
return nil
}
+56 -1
View File
@@ -3,6 +3,7 @@ package config
import (
"os"
"strconv"
"strings"
)
// Config holds all configuration for meta-core
@@ -28,11 +29,20 @@ type Config struct {
HealthCheckIntervalMS int // Health check interval in ms (default: 5000)
HeartbeatIntervalMS int // Service heartbeat interval in ms (default: 30000)
StaleThresholdMS int // Stale service threshold in ms (default: 60000)
// File watcher configuration
WatchFolderList []string // List of folders to watch for file changes
WatchIntervalMS int // Polling interval for network mounts (default: 1000)
DebounceMS int // File change debounce time (default: 30000)
EnableFileWatcher bool // Enable file watcher (default: true)
// Mount configuration
MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
}
// Load creates a Config from environment variables
func Load() *Config {
return &Config{
cfg := &Config{
MetaCorePath: getEnv("META_CORE_PATH", "/meta-core"),
FilesPath: getEnv("FILES_PATH", "/files"),
ServiceName: getEnv("SERVICE_NAME", "meta-core"),
@@ -45,7 +55,19 @@ func Load() *Config {
HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
HeartbeatIntervalMS: getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
StaleThresholdMS: getEnvInt("STALE_THRESHOLD_MS", 60000),
WatchIntervalMS: getEnvInt("WATCH_INTERVAL_MS", 1000),
DebounceMS: getEnvInt("DEBOUNCE_MS", 30000),
EnableFileWatcher: getEnvBool("ENABLE_FILE_WATCHER", true),
}
// Parse watch folder list (comma-separated)
watchFolders := getEnv("WATCH_FOLDER_LIST", "/files/")
cfg.WatchFolderList = parseCommaSeparated(watchFolders)
// Set mounts directory
cfg.MountsDir = cfg.MetaCorePath + "/mounts"
return cfg
}
// LockFilePath returns the path to the leader lock file
@@ -68,6 +90,16 @@ func (c *Config) ServicesDir() string {
return c.MetaCorePath + "/services"
}
// MountsFilePath returns the path to the mounts configuration file
func (c *Config) MountsFilePath() string {
return c.MountsDir + "/mounts.json"
}
// MountsErrorDir returns the path to mount error files
func (c *Config) MountsErrorDir() string {
return c.MountsDir + "/errors"
}
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
@@ -83,3 +115,26 @@ func getEnvInt(key string, defaultValue int) int {
}
return defaultValue
}
func getEnvBool(key string, defaultValue bool) bool {
if value := os.Getenv(key); value != "" {
lower := strings.ToLower(value)
return lower == "true" || lower == "1" || lower == "yes"
}
return defaultValue
}
func parseCommaSeparated(s string) []string {
if s == "" {
return []string{}
}
parts := strings.Split(s, ",")
result := make([]string, 0, len(parts))
for _, part := range parts {
trimmed := strings.TrimSpace(part)
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
+16 -6
View File
@@ -90,7 +90,16 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
hostname, _ := os.Hostname()
ip := getLocalIP()
apiBase := fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
// Use BASE_URL for external access (dashboard navigation), fall back to internal IP
var apiBase string
if s.config.BaseURL != "" {
apiBase = s.config.BaseURL
} else {
apiBase = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
}
// Internal URLs always use container IP for container-to-container communication
internalBase := fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
metaCoreBase := fmt.Sprintf("http://%s:%d", ip, s.config.HTTPPort)
return &ServiceInfo{
@@ -104,15 +113,16 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
Capabilities: []string{"meta-core"},
Endpoints: map[string]string{
// meta-core sidecar endpoints (port 9000)
// meta-core sidecar endpoints (port 9000) - always internal
"health": metaCoreBase + "/health",
"meta": metaCoreBase + "/meta",
"leader": metaCoreBase + "/leader",
"services": metaCoreBase + "/services",
// main service endpoints (port 80)
"api": apiBase + "/api",
"webdav": apiBase + "/webdav",
"callback": apiBase + "/api/plugins/callback",
// main service endpoints - external for dashboard
"api": apiBase + "/api",
"webdav": apiBase + "/webdav",
// callback uses internal IP for container-to-container plugin communication
"callback": internalBase + "/api/plugins/callback",
},
}
}
+222
View File
@@ -0,0 +1,222 @@
package mounts
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
)
// Handlers provides HTTP handlers for mount operations
type Handlers struct {
manager *Manager
}
// NewHandlers creates new mount handlers
func NewHandlers(manager *Manager) *Handlers {
return &Handlers{manager: manager}
}
// RegisterRoutes registers all mount-related routes
func (h *Handlers) RegisterRoutes(r *mux.Router) {
// Mount management
r.HandleFunc("/api/mounts", h.handleListMounts).Methods("GET")
r.HandleFunc("/api/mounts", h.handleCreateMount).Methods("POST")
r.HandleFunc("/api/mounts/rclone/remotes", h.handleListRcloneRemotes).Methods("GET")
r.HandleFunc("/api/mounts/{id}", h.handleGetMount).Methods("GET")
r.HandleFunc("/api/mounts/{id}", h.handleDeleteMount).Methods("DELETE")
r.HandleFunc("/api/mounts/{id}/mount", h.handleRequestMount).Methods("POST")
r.HandleFunc("/api/mounts/{id}/unmount", h.handleRequestUnmount).Methods("POST")
r.HandleFunc("/api/mounts/{id}/safe-unmount", h.handleSafeUnmount).Methods("POST")
}
// handleListMounts handles GET /api/mounts
func (h *Handlers) handleListMounts(w http.ResponseWriter, r *http.Request) {
mounts, err := h.manager.ListMounts()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, MountsListResponse{Mounts: mounts})
}
// handleGetMount handles GET /api/mounts/{id}
func (h *Handlers) handleGetMount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
mount, err := h.manager.GetMount(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if mount == nil {
writeError(w, http.StatusNotFound, "mount not found")
return
}
writeJSON(w, http.StatusOK, MountResponse{Mount: mount})
}
// handleCreateMount handles POST /api/mounts
func (h *Handlers) handleCreateMount(w http.ResponseWriter, r *http.Request) {
var req CreateMountRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
mount, err := h.manager.CreateMount(&req)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusCreated, MountResponse{Mount: mount})
}
// handleDeleteMount handles DELETE /api/mounts/{id}
func (h *Handlers) handleDeleteMount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
if err := h.manager.DeleteMount(id); err != nil {
if err.Error() == "mount not found" {
writeError(w, http.StatusNotFound, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, StatusResponse{Status: "ok"})
}
// handleRequestMount handles POST /api/mounts/{id}/mount
func (h *Handlers) handleRequestMount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
if err := h.manager.RequestMount(id); err != nil {
if err.Error() == "mount not found" {
writeError(w, http.StatusNotFound, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, StatusResponse{
Status: "ok",
Message: "Mount requested",
})
}
// handleRequestUnmount handles POST /api/mounts/{id}/unmount
func (h *Handlers) handleRequestUnmount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
if err := h.manager.RequestUnmount(id); err != nil {
if err.Error() == "mount not found" {
writeError(w, http.StatusNotFound, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, StatusResponse{
Status: "ok",
Message: "Unmount requested",
})
}
// handleSafeUnmount handles POST /api/mounts/{id}/safe-unmount
func (h *Handlers) handleSafeUnmount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
// Get timeout from query param (default 60000ms)
timeoutStr := r.URL.Query().Get("timeout")
timeoutMS := 60000
if timeoutStr != "" {
if parsed, err := parseIntOrDefault(timeoutStr, 60000); err == nil {
timeoutMS = parsed
}
}
// Get mount info
mount, err := h.manager.GetMount(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if mount == nil {
writeError(w, http.StatusNotFound, "mount not found")
return
}
// Request unmount
if err := h.manager.RequestUnmount(id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Wait for unmount
success := h.manager.WaitForUnmount(mount.MountPath, timeoutMS)
if !success {
writeJSON(w, http.StatusOK, StatusResponse{
Status: "warning",
Message: "Unmount requested but mount may still be attached",
})
return
}
writeJSON(w, http.StatusOK, StatusResponse{
Status: "ok",
Message: "Safe unmount completed",
})
}
// handleListRcloneRemotes handles GET /api/mounts/rclone/remotes
func (h *Handlers) handleListRcloneRemotes(w http.ResponseWriter, r *http.Request) {
remotes, err := h.manager.ListRcloneRemotes()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, RcloneRemotesResponse{Remotes: remotes})
}
// Helper functions
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
func writeError(w http.ResponseWriter, status int, message string) {
writeJSON(w, status, map[string]string{
"error": http.StatusText(status),
"message": message,
})
}
func parseIntOrDefault(s string, defaultVal int) (int, error) {
var result int
if s == "" {
return defaultVal, nil
}
err := json.Unmarshal([]byte(s), &result)
if err != nil {
return defaultVal, nil
}
return result, nil
}
+478
View File
@@ -0,0 +1,478 @@
package mounts
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/metazla/meta-core/internal/config"
)
// Manager handles mount configuration and status
type Manager struct {
config *config.Config
mu sync.RWMutex
filesPath string
mountsDir string
}
// NewManager creates a new mount manager
func NewManager(cfg *config.Config) (*Manager, error) {
m := &Manager{
config: cfg,
filesPath: cfg.FilesPath,
mountsDir: cfg.MountsDir,
}
// Ensure directories exist
if err := m.ensureDirs(); err != nil {
return nil, fmt.Errorf("failed to create mount directories: %w", err)
}
return m, nil
}
// ensureDirs creates required directories
func (m *Manager) ensureDirs() error {
dirs := []string{
m.mountsDir,
m.config.MountsErrorDir(),
}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
}
return nil
}
// readConfig reads the mounts configuration file
func (m *Manager) readConfig() (*MountsFile, error) {
m.mu.RLock()
defer m.mu.RUnlock()
data, err := os.ReadFile(m.config.MountsFilePath())
if err != nil {
if os.IsNotExist(err) {
return &MountsFile{Version: 1, Mounts: []MountConfig{}}, nil
}
return nil, err
}
var mountsFile MountsFile
if err := json.Unmarshal(data, &mountsFile); err != nil {
return nil, err
}
return &mountsFile, nil
}
// writeConfig writes the mounts configuration file
func (m *Manager) writeConfig(mountsFile *MountsFile) error {
m.mu.Lock()
defer m.mu.Unlock()
if err := m.ensureDirs(); err != nil {
return err
}
data, err := json.MarshalIndent(mountsFile, "", " ")
if err != nil {
return err
}
return os.WriteFile(m.config.MountsFilePath(), data, 0644)
}
// IsMounted checks if a path is currently mounted
func (m *Manager) IsMounted(mountPath string) bool {
cmd := exec.Command("findmnt", "-n", mountPath)
output, err := cmd.CombinedOutput()
if err != nil {
return false
}
return strings.TrimSpace(string(output)) != ""
}
// ReadError reads the error file for a mount
func (m *Manager) ReadError(id string) (string, error) {
errorFile := filepath.Join(m.config.MountsErrorDir(), id+".error")
data, err := os.ReadFile(errorFile)
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) > 1 {
// Skip timestamp line, return rest
return strings.Join(lines[1:], "\n"), nil
}
return "", nil
}
// SanitizeName sanitizes a mount name for use as a directory name
func SanitizeName(name string) string {
// Convert to lowercase
result := strings.ToLower(name)
// Replace non-alphanumeric chars with hyphens
re := regexp.MustCompile(`[^a-z0-9-_]`)
result = re.ReplaceAllString(result, "-")
// Collapse multiple hyphens
re = regexp.MustCompile(`-+`)
result = re.ReplaceAllString(result, "-")
// Remove leading/trailing hyphens
result = strings.Trim(result, "-")
// Limit length
if len(result) > 64 {
result = result[:64]
}
return result
}
// ObscurePassword obscures a password using rclone
func ObscurePassword(password string) (string, error) {
// Escape special characters for shell
escapedPass := strings.ReplaceAll(password, `"`, `\"`)
cmd := exec.Command("rclone", "obscure", escapedPass)
output, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("failed to obscure password: %w", err)
}
return strings.TrimSpace(string(output)), nil
}
// ListMounts returns all mounts with their current status
func (m *Manager) ListMounts() ([]MountStatus, error) {
mountsFile, err := m.readConfig()
if err != nil {
return nil, err
}
statuses := make([]MountStatus, len(mountsFile.Mounts))
for i, mount := range mountsFile.Mounts {
mounted := m.IsMounted(mount.MountPath)
errMsg, _ := m.ReadError(mount.ID)
statuses[i] = MountStatus{
MountConfig: mount,
Mounted: mounted,
Error: errMsg,
LastChecked: NowMS(),
}
}
return statuses, nil
}
// GetMount returns a single mount by ID
func (m *Manager) GetMount(id string) (*MountStatus, error) {
mountsFile, err := m.readConfig()
if err != nil {
return nil, err
}
for _, mount := range mountsFile.Mounts {
if mount.ID == id {
mounted := m.IsMounted(mount.MountPath)
errMsg, _ := m.ReadError(mount.ID)
return &MountStatus{
MountConfig: mount,
Mounted: mounted,
Error: errMsg,
LastChecked: NowMS(),
}, nil
}
}
return nil, nil
}
// CreateMount creates a new mount configuration
func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
if req.Name == "" {
return nil, fmt.Errorf("mount name is required")
}
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
}
// Validate type-specific fields
switch req.Type {
case MountTypeNFS:
if req.NFSServer == "" || req.NFSPath == "" {
return nil, fmt.Errorf("NFS server and path are required")
}
case MountTypeSMB:
if req.SMBServer == "" || req.SMBShare == "" {
return nil, fmt.Errorf("SMB server and share are required")
}
case MountTypeRclone:
if req.RcloneRemote == "" {
return nil, fmt.Errorf("rclone remote is required")
}
}
// Generate ID and mount path
id := uuid.New().String()
safeName := SanitizeName(req.Name)
mountPath := filepath.Join(m.filesPath, safeName)
// Check if path already exists
mountsFile, err := m.readConfig()
if err != nil {
return nil, err
}
for _, mount := range mountsFile.Mounts {
if mount.MountPath == mountPath {
return nil, fmt.Errorf("mount path %s already configured", mountPath)
}
}
// Default enabled to true if not specified
enabled := true
if req.Enabled != nil {
enabled = *req.Enabled
}
mount := MountConfig{
ID: id,
Name: req.Name,
Type: req.Type,
Enabled: enabled,
DesiredMounted: enabled, // Auto-mount if enabled
MountPath: mountPath,
Options: req.Options,
}
// Type-specific fields
switch req.Type {
case MountTypeNFS:
mount.NFSServer = req.NFSServer
mount.NFSPath = req.NFSPath
case MountTypeSMB:
mount.SMBServer = req.SMBServer
mount.SMBShare = req.SMBShare
mount.SMBUsername = req.SMBUsername
mount.SMBDomain = req.SMBDomain
if req.SMBPassword != "" {
obscured, err := ObscurePassword(req.SMBPassword)
if err != nil {
return nil, fmt.Errorf("failed to secure password: %w", err)
}
mount.SMBPasswordObscured = obscured
}
case MountTypeRclone:
mount.RcloneRemote = req.RcloneRemote
mount.RclonePath = req.RclonePath
}
mountsFile.Mounts = append(mountsFile.Mounts, mount)
if err := m.writeConfig(mountsFile); err != nil {
return nil, err
}
log.Printf("[Mounts] Created mount config: %s (%s) -> %s", mount.Name, mount.Type, mount.MountPath)
return &MountStatus{
MountConfig: mount,
Mounted: false,
LastChecked: NowMS(),
}, nil
}
// RequestMount sets desiredMounted to true
func (m *Manager) RequestMount(id string) error {
mountsFile, err := m.readConfig()
if err != nil {
return err
}
for i, mount := range mountsFile.Mounts {
if mount.ID == id {
mountsFile.Mounts[i].DesiredMounted = true
if err := m.writeConfig(mountsFile); err != nil {
return err
}
log.Printf("[Mounts] Mount requested: %s", mount.Name)
return nil
}
}
return fmt.Errorf("mount not found")
}
// RequestUnmount sets desiredMounted to false
func (m *Manager) RequestUnmount(id string) error {
mountsFile, err := m.readConfig()
if err != nil {
return err
}
for i, mount := range mountsFile.Mounts {
if mount.ID == id {
mountsFile.Mounts[i].DesiredMounted = false
if err := m.writeConfig(mountsFile); err != nil {
return err
}
log.Printf("[Mounts] Unmount requested: %s", mount.Name)
return nil
}
}
return fmt.Errorf("mount not found")
}
// WaitForUnmount waits for a mount to be unmounted
func (m *Manager) WaitForUnmount(mountPath string, timeoutMS int) bool {
deadline := time.Now().Add(time.Duration(timeoutMS) * time.Millisecond)
for time.Now().Before(deadline) {
if !m.IsMounted(mountPath) {
return true
}
time.Sleep(500 * time.Millisecond)
}
return false
}
// DeleteMount removes a mount configuration
func (m *Manager) DeleteMount(id string) error {
mountsFile, err := m.readConfig()
if err != nil {
return err
}
var mount *MountConfig
index := -1
for i, mnt := range mountsFile.Mounts {
if mnt.ID == id {
mount = &mnt
index = i
break
}
}
if index == -1 {
return fmt.Errorf("mount not found")
}
// Request unmount first
mountsFile.Mounts[index].DesiredMounted = false
if err := m.writeConfig(mountsFile); err != nil {
return err
}
// Wait for unmount (15 seconds max)
m.WaitForUnmount(mount.MountPath, 15000)
// Remove from config
mountsFile.Mounts = append(mountsFile.Mounts[:index], mountsFile.Mounts[index+1:]...)
if err := m.writeConfig(mountsFile); err != nil {
return err
}
// Clean up error file
errorFile := filepath.Join(m.config.MountsErrorDir(), id+".error")
os.Remove(errorFile)
// Try to remove mount directory (will fail if not empty, which is fine)
os.Remove(mount.MountPath)
log.Printf("[Mounts] Deleted mount: %s", mount.Name)
return nil
}
// ListRcloneRemotes lists available rclone remotes
func (m *Manager) ListRcloneRemotes() ([]RcloneRemote, error) {
// Call rclone RC API
cmd := exec.Command("curl", "-s", "-X", "POST",
"-H", "Content-Type: application/json",
"-u", "admin:admin",
"http://127.0.0.1:5572/config/listremotes")
output, err := cmd.Output()
if err != nil {
log.Printf("[Mounts] Failed to list rclone remotes: %v", err)
return []RcloneRemote{}, nil
}
var response struct {
Remotes []string `json:"remotes"`
}
if err := json.Unmarshal(output, &response); err != nil {
return []RcloneRemote{}, nil
}
remotes := make([]RcloneRemote, 0, len(response.Remotes))
for _, name := range response.Remotes {
// Strip trailing colon if present
cleanName := strings.TrimSuffix(name, ":")
// Get type for this remote
remoteType := m.getRcloneRemoteType(cleanName)
remotes = append(remotes, RcloneRemote{
Name: cleanName,
Type: remoteType,
})
}
return remotes, nil
}
// getRcloneRemoteType gets the type of an rclone remote
func (m *Manager) getRcloneRemoteType(name string) string {
// Build request body
body := fmt.Sprintf(`{"name":"%s"}`, name)
cmd := exec.Command("curl", "-s", "-X", "POST",
"-H", "Content-Type: application/json",
"-u", "admin:admin",
"-d", body,
"http://127.0.0.1:5572/config/get")
output, err := cmd.Output()
if err != nil {
return "unknown"
}
var response map[string]interface{}
if err := json.Unmarshal(output, &response); err != nil {
return "unknown"
}
if typeStr, ok := response["type"].(string); ok {
return typeStr
}
return "unknown"
}
// Basic auth helper for rclone API
func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}
+109
View File
@@ -0,0 +1,109 @@
package mounts
import "time"
// MountType represents the type of mount
type MountType string
const (
MountTypeNFS MountType = "nfs"
MountTypeSMB MountType = "smb"
MountTypeRclone MountType = "rclone"
)
// MountConfig represents a mount configuration
type MountConfig struct {
// Common fields
ID string `json:"id"`
Name string `json:"name"`
Type MountType `json:"type"`
Enabled bool `json:"enabled"`
DesiredMounted bool `json:"desiredMounted"`
MountPath string `json:"mountPath"`
Options string `json:"options,omitempty"`
// NFS-specific fields
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
// SMB-specific fields
SMBServer string `json:"smbServer,omitempty"`
SMBShare string `json:"smbShare,omitempty"`
SMBUsername string `json:"smbUsername,omitempty"`
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
SMBDomain string `json:"smbDomain,omitempty"`
// rclone-specific fields
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
}
// MountStatus represents the runtime status of a mount
type MountStatus struct {
MountConfig
Mounted bool `json:"mounted"`
Error string `json:"error,omitempty"`
LastChecked int64 `json:"lastChecked"`
}
// MountsFile represents the mounts.json file structure
type MountsFile struct {
Version int `json:"version"`
Mounts []MountConfig `json:"mounts"`
}
// RcloneRemote represents an rclone remote
type RcloneRemote struct {
Name string `json:"name"`
Type string `json:"type"`
}
// CreateMountRequest is the request body for creating a mount
type CreateMountRequest struct {
Name string `json:"name"`
Type MountType `json:"type"`
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
Options string `json:"options,omitempty"`
// NFS
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
// SMB
SMBServer string `json:"smbServer,omitempty"`
SMBShare string `json:"smbShare,omitempty"`
SMBUsername string `json:"smbUsername,omitempty"`
SMBPassword string `json:"smbPassword,omitempty"` // Plain text, will be obscured
SMBDomain string `json:"smbDomain,omitempty"`
// rclone
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
}
// MountResponse is the response for mount operations
type MountResponse struct {
Mount *MountStatus `json:"mount,omitempty"`
}
// MountsListResponse is the response for listing mounts
type MountsListResponse struct {
Mounts []MountStatus `json:"mounts"`
}
// RcloneRemotesResponse is the response for listing rclone remotes
type RcloneRemotesResponse struct {
Remotes []RcloneRemote `json:"remotes"`
}
// StatusResponse is a generic status response
type StatusResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GateStatus interface{} `json:"gateStatus,omitempty"`
}
// Timestamp helpers
func NowMS() int64 {
return time.Now().UnixMilli()
}
+80
View File
@@ -490,3 +490,83 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
newValue := strings.Join(newValues, "|")
return true, c.client.HSet(ctx, hashKey, property, newValue).Err()
}
// GetMemoryInfo returns Redis memory usage information
func (c *Client) GetMemoryInfo() (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
info, err := c.client.Info(ctx, "memory").Result()
if err != nil {
return "", fmt.Errorf("info memory failed: %w", err)
}
// Extract used_memory_human from info string
for _, line := range strings.Split(info, "\r\n") {
if strings.HasPrefix(line, "used_memory_human:") {
return strings.TrimPrefix(line, "used_memory_human:"), nil
}
}
return "N/A", nil
}
// ClearAllMetadata deletes all file metadata and index
// Returns the number of files deleted
func (c *Client) ClearAllMetadata() (int64, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
// Get all hash IDs from index
indexKey := c.buildIndexKey()
hashIDs, err := c.client.SMembers(ctx, indexKey).Result()
if err != nil {
return 0, fmt.Errorf("smembers failed: %w", err)
}
var deletedCount int64
// Delete each file's hash
for _, hashID := range hashIDs {
hashKey := c.buildHashKey(hashID)
if err := c.client.Del(ctx, hashKey).Err(); err != nil {
log.Printf("[Storage] Warning: failed to delete %s: %v", hashKey, err)
continue
}
deletedCount++
}
// Delete the index set
if err := c.client.Del(ctx, indexKey).Err(); err != nil {
log.Printf("[Storage] Warning: failed to delete index: %v", err)
}
log.Printf("[Storage] Cleared %d file metadata entries", deletedCount)
return deletedCount, nil
}
// GetRedisClient returns the underlying redis client for advanced operations
func (c *Client) GetRedisClient() *redis.Client {
c.mu.RLock()
defer c.mu.RUnlock()
return c.client
}
// GetPrefix returns the key prefix used by this client
func (c *Client) GetPrefix() string {
return c.prefix
}
+138
View File
@@ -0,0 +1,138 @@
package watcher
import (
"sync"
"time"
)
// Debouncer delays events to ensure file stability
type Debouncer struct {
delay time.Duration
pending map[string]*PendingEvent
mu sync.Mutex
callback func(FileEvent)
stopped bool
}
// NewDebouncer creates a new debouncer with the specified delay
func NewDebouncer(delay time.Duration) *Debouncer {
return &Debouncer{
delay: delay,
pending: make(map[string]*PendingEvent),
}
}
// SetCallback sets the callback function for debounced events
func (d *Debouncer) SetCallback(cb func(FileEvent)) {
d.mu.Lock()
defer d.mu.Unlock()
d.callback = cb
}
// Add adds or updates an event in the debounce queue
func (d *Debouncer) Add(event FileEvent) {
d.mu.Lock()
defer d.mu.Unlock()
if d.stopped {
return
}
key := event.Path
if existing, ok := d.pending[key]; ok {
// Update existing pending event
existing.LastSeen = time.Now()
existing.Event = event // Update with latest event info
// Reset timer
existing.Timer.Reset(d.delay)
} else {
// Create new pending event
now := time.Now()
pending := &PendingEvent{
Event: event,
FirstSeen: now,
LastSeen: now,
}
// Create timer
pending.Timer = time.AfterFunc(d.delay, func() {
d.flush(key)
})
d.pending[key] = pending
}
}
// flush emits the event and removes it from pending
func (d *Debouncer) flush(key string) {
d.mu.Lock()
pending, ok := d.pending[key]
if !ok {
d.mu.Unlock()
return
}
// Check if enough time has passed since last activity
elapsed := time.Since(pending.LastSeen)
if elapsed < d.delay {
// Reset timer for remaining time
remaining := d.delay - elapsed
pending.Timer.Reset(remaining)
d.mu.Unlock()
return
}
// Remove from pending
delete(d.pending, key)
callback := d.callback
event := pending.Event
d.mu.Unlock()
// Call callback outside of lock
if callback != nil {
callback(event)
}
}
// Stop stops the debouncer and cancels all pending events
func (d *Debouncer) Stop() {
d.mu.Lock()
defer d.mu.Unlock()
d.stopped = true
// Cancel all timers
for _, pending := range d.pending {
pending.Timer.Stop()
}
// Clear pending
d.pending = make(map[string]*PendingEvent)
}
// PendingCount returns the number of pending events
func (d *Debouncer) PendingCount() int {
d.mu.Lock()
defer d.mu.Unlock()
return len(d.pending)
}
// FlushAll immediately flushes all pending events
func (d *Debouncer) FlushAll() {
d.mu.Lock()
keys := make([]string, 0, len(d.pending))
for key := range d.pending {
keys = append(keys, key)
}
d.mu.Unlock()
for _, key := range keys {
d.flush(key)
}
}
+223
View File
@@ -0,0 +1,223 @@
package watcher
import (
"bytes"
"encoding/json"
"log"
"net/http"
"sync"
"time"
)
const (
// MaxRetries is the maximum number of delivery retries
MaxRetries = 3
// RetryDelay is the delay between retries
RetryDelay = 5 * time.Second
// WebhookTimeout is the timeout for webhook requests
WebhookTimeout = 10 * time.Second
// MaxFailCount is the max failures before removing subscriber
MaxFailCount = 10
)
// Dispatcher sends file events to subscribers
type Dispatcher struct {
subscribers map[string]*Subscriber
sseClients map[chan FileEvent]bool
mu sync.RWMutex
httpClient *http.Client
}
// NewDispatcher creates a new event dispatcher
func NewDispatcher() *Dispatcher {
return &Dispatcher{
subscribers: make(map[string]*Subscriber),
sseClients: make(map[chan FileEvent]bool),
httpClient: &http.Client{
Timeout: WebhookTimeout,
},
}
}
// Subscribe registers a webhook subscriber
func (d *Dispatcher) Subscribe(url string, eventTypes []string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.subscribers[url] = &Subscriber{
URL: url,
RegisteredAt: NowMS(),
EventTypes: eventTypes,
FailCount: 0,
}
log.Printf("[Dispatcher] Subscribed webhook: %s", url)
return nil
}
// Unsubscribe removes a webhook subscriber
func (d *Dispatcher) Unsubscribe(url string) error {
d.mu.Lock()
defer d.mu.Unlock()
delete(d.subscribers, url)
log.Printf("[Dispatcher] Unsubscribed webhook: %s", url)
return nil
}
// ListSubscribers returns all subscribers
func (d *Dispatcher) ListSubscribers() []Subscriber {
d.mu.RLock()
defer d.mu.RUnlock()
result := make([]Subscriber, 0, len(d.subscribers))
for _, sub := range d.subscribers {
result = append(result, *sub)
}
return result
}
// AddSSEClient adds an SSE client channel
func (d *Dispatcher) AddSSEClient(ch chan FileEvent) {
d.mu.Lock()
defer d.mu.Unlock()
d.sseClients[ch] = true
}
// RemoveSSEClient removes an SSE client channel
func (d *Dispatcher) RemoveSSEClient(ch chan FileEvent) {
d.mu.Lock()
defer d.mu.Unlock()
delete(d.sseClients, ch)
close(ch)
}
// Dispatch sends an event to all subscribers
func (d *Dispatcher) Dispatch(event FileEvent) {
// Dispatch to webhooks
go d.dispatchToWebhooks(event)
// Dispatch to SSE clients
d.dispatchToSSE(event)
}
// dispatchToWebhooks sends event to webhook subscribers
func (d *Dispatcher) dispatchToWebhooks(event FileEvent) {
d.mu.RLock()
subscribers := make([]*Subscriber, 0, len(d.subscribers))
for _, sub := range d.subscribers {
subscribers = append(subscribers, sub)
}
d.mu.RUnlock()
for _, sub := range subscribers {
// Check event type filter
if len(sub.EventTypes) > 0 {
found := false
for _, et := range sub.EventTypes {
if et == string(event.Type) {
found = true
break
}
}
if !found {
continue
}
}
go d.deliverToWebhook(sub.URL, event)
}
}
// deliverToWebhook delivers an event to a webhook with retries
func (d *Dispatcher) deliverToWebhook(url string, event FileEvent) {
body, err := json.Marshal(event)
if err != nil {
log.Printf("[Dispatcher] Failed to marshal event: %v", err)
return
}
var lastErr error
for attempt := 0; attempt < MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(RetryDelay)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(body))
if err != nil {
lastErr = err
continue
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Event-Type", string(event.Type))
resp, err := d.httpClient.Do(req)
if err != nil {
lastErr = err
continue
}
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// Success
d.mu.Lock()
if sub, ok := d.subscribers[url]; ok {
sub.LastDelivery = NowMS()
sub.FailCount = 0
}
d.mu.Unlock()
return
}
lastErr = &httpError{StatusCode: resp.StatusCode}
}
// All retries failed
log.Printf("[Dispatcher] Failed to deliver to %s after %d retries: %v", url, MaxRetries, lastErr)
d.mu.Lock()
if sub, ok := d.subscribers[url]; ok {
sub.FailCount++
if sub.FailCount >= MaxFailCount {
delete(d.subscribers, url)
log.Printf("[Dispatcher] Removed webhook %s after %d failures", url, MaxFailCount)
}
}
d.mu.Unlock()
}
// dispatchToSSE sends event to SSE clients
func (d *Dispatcher) dispatchToSSE(event FileEvent) {
d.mu.RLock()
clients := make([]chan FileEvent, 0, len(d.sseClients))
for ch := range d.sseClients {
clients = append(clients, ch)
}
d.mu.RUnlock()
for _, ch := range clients {
select {
case ch <- event:
default:
// Channel full, skip
}
}
}
// SSEClientCount returns the number of connected SSE clients
func (d *Dispatcher) SSEClientCount() int {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.sseClients)
}
// httpError represents an HTTP error
type httpError struct {
StatusCode int
}
func (e *httpError) Error() string {
return http.StatusText(e.StatusCode)
}
+196
View File
@@ -0,0 +1,196 @@
package watcher
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"github.com/gorilla/mux"
)
// Handlers provides HTTP handlers for watcher operations
type Handlers struct {
watcher *Watcher
dispatcher *Dispatcher
}
// NewHandlers creates new watcher handlers
func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
return &Handlers{
watcher: watcher,
dispatcher: dispatcher,
}
}
// RegisterRoutes registers all watcher-related routes
func (h *Handlers) RegisterRoutes(r *mux.Router) {
// Event subscriptions
r.HandleFunc("/api/events/subscribe", h.handleSSESubscribe).Methods("GET")
r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
r.HandleFunc("/api/events/subscribers", h.handleListSubscribers).Methods("GET")
r.HandleFunc("/api/events/subscribers", h.handleAddSubscriber).Methods("POST")
r.HandleFunc("/api/events/subscribers/{url}", h.handleRemoveSubscriber).Methods("DELETE")
// Scan management
r.HandleFunc("/api/scan/trigger", h.handleTriggerScan).Methods("POST")
r.HandleFunc("/api/scan/status", h.handleScanStatus).Methods("GET")
}
// handleSSESubscribe handles GET /api/events/subscribe (Server-Sent Events)
func (h *Handlers) handleSSESubscribe(w http.ResponseWriter, r *http.Request) {
// Set SSE headers
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("Access-Control-Allow-Origin", "*")
// Create event channel
eventChan := make(chan FileEvent, 100)
h.dispatcher.AddSSEClient(eventChan)
// Cleanup on disconnect
defer h.dispatcher.RemoveSSEClient(eventChan)
// Get flusher
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "SSE not supported", http.StatusInternalServerError)
return
}
// Send initial connected message
fmt.Fprintf(w, "event: connected\ndata: {\"status\":\"connected\"}\n\n")
flusher.Flush()
// Stream events
for {
select {
case event := <-eventChan:
data, err := json.Marshal(event)
if err != nil {
continue
}
fmt.Fprintf(w, "event: file\ndata: %s\n\n", string(data))
flusher.Flush()
case <-r.Context().Done():
return
}
}
}
// handlePoll handles GET /api/events/poll
func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
// Get since parameter
sinceStr := r.URL.Query().Get("since")
sinceMS := int64(0)
if sinceStr != "" {
if parsed, err := strconv.ParseInt(sinceStr, 10, 64); err == nil {
sinceMS = parsed
}
}
// Get limit parameter
limitStr := r.URL.Query().Get("limit")
limit := 100
if limitStr != "" {
if parsed, err := strconv.Atoi(limitStr); err == nil {
limit = parsed
}
}
events := h.watcher.GetRecentEvents(sinceMS, limit)
writeJSON(w, http.StatusOK, EventsListResponse{
Events: events,
Count: len(events),
})
}
// handleListSubscribers handles GET /api/events/subscribers
func (h *Handlers) handleListSubscribers(w http.ResponseWriter, r *http.Request) {
subscribers := h.dispatcher.ListSubscribers()
writeJSON(w, http.StatusOK, SubscribersListResponse{
Subscribers: subscribers,
Count: len(subscribers),
})
}
// handleAddSubscriber handles POST /api/events/subscribers
func (h *Handlers) handleAddSubscriber(w http.ResponseWriter, r *http.Request) {
var req SubscribeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.URL == "" {
writeError(w, http.StatusBadRequest, "URL is required")
return
}
if err := h.dispatcher.Subscribe(req.URL, req.EventTypes); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, map[string]interface{}{
"status": "ok",
"message": "Subscribed",
"url": req.URL,
})
}
// handleRemoveSubscriber handles DELETE /api/events/subscribers/{url}
func (h *Handlers) handleRemoveSubscriber(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
url := vars["url"]
if url == "" {
writeError(w, http.StatusBadRequest, "URL is required")
return
}
if err := h.dispatcher.Unsubscribe(url); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Unsubscribed",
})
}
// handleTriggerScan handles POST /api/scan/trigger
func (h *Handlers) handleTriggerScan(w http.ResponseWriter, r *http.Request) {
go h.watcher.RunScan()
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Scan triggered",
})
}
// handleScanStatus handles GET /api/scan/status
func (h *Handlers) handleScanStatus(w http.ResponseWriter, r *http.Request) {
status := h.watcher.GetStatus()
writeJSON(w, http.StatusOK, status)
}
// Helper functions
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
func writeError(w http.ResponseWriter, status int, message string) {
writeJSON(w, status, map[string]string{
"error": http.StatusText(status),
"message": message,
})
}
+71
View File
@@ -0,0 +1,71 @@
package watcher
import "time"
// FileEventType represents the type of file event
type FileEventType string
const (
EventTypeAdd FileEventType = "add"
EventTypeChange FileEventType = "change"
EventTypeDelete FileEventType = "delete"
EventTypeRename FileEventType = "rename"
)
// FileEvent represents a file system event
type FileEvent struct {
Type FileEventType `json:"type"`
Path string `json:"path"` // Relative to FILES_PATH
Size int64 `json:"size,omitempty"`
Timestamp int64 `json:"timestamp"`
PartialHash string `json:"partialHash,omitempty"` // Hash of first 64KB
OldPath string `json:"oldPath,omitempty"` // For rename events
}
// Subscriber represents a webhook subscriber
type Subscriber struct {
URL string `json:"url"`
RegisteredAt int64 `json:"registeredAt"`
EventTypes []string `json:"eventTypes,omitempty"` // Empty means all
LastDelivery int64 `json:"lastDelivery,omitempty"`
FailCount int `json:"failCount"`
}
// PendingEvent tracks a file event that's being debounced
type PendingEvent struct {
Event FileEvent
FirstSeen time.Time
LastSeen time.Time
Timer *time.Timer
}
// SubscribeRequest is the request to register a webhook
type SubscribeRequest struct {
URL string `json:"url"`
EventTypes []string `json:"eventTypes,omitempty"`
}
// EventsListResponse is the response for listing events
type EventsListResponse struct {
Events []FileEvent `json:"events"`
Count int `json:"count"`
}
// SubscribersListResponse is the response for listing subscribers
type SubscribersListResponse struct {
Subscribers []Subscriber `json:"subscribers"`
Count int `json:"count"`
}
// ScanStatusResponse is the response for scan status
type ScanStatusResponse struct {
Status string `json:"status"`
Scanning bool `json:"scanning"`
LastScan int64 `json:"lastScan,omitempty"`
FileCount int `json:"fileCount,omitempty"`
}
// NowMS returns current time in milliseconds
func NowMS() int64 {
return time.Now().UnixMilli()
}
+364
View File
@@ -0,0 +1,364 @@
package watcher
import (
"crypto/sha256"
"encoding/hex"
"io"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/fsnotify/fsnotify"
"github.com/metazla/meta-core/internal/config"
)
const (
// PartialHashSize is the number of bytes to hash for file identification
PartialHashSize = 64 * 1024 // 64KB
)
// Watcher monitors directories for file changes
type Watcher struct {
config *config.Config
fsWatcher *fsnotify.Watcher
debouncer *Debouncer
dispatcher *Dispatcher
filesPath string
watchPaths []string
mu sync.RWMutex
isRunning bool
isScanning bool
lastScan int64
fileCount int
stopChan chan struct{}
eventBuffer []FileEvent
}
// NewWatcher creates a new file watcher
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
fsWatcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
debouncer := NewDebouncer(time.Duration(cfg.DebounceMS) * time.Millisecond)
w := &Watcher{
config: cfg,
fsWatcher: fsWatcher,
debouncer: debouncer,
dispatcher: dispatcher,
filesPath: cfg.FilesPath,
watchPaths: cfg.WatchFolderList,
stopChan: make(chan struct{}),
eventBuffer: make([]FileEvent, 0, 1000),
}
// Set debouncer callback
debouncer.SetCallback(func(event FileEvent) {
w.handleDebouncedEvent(event)
})
return w, nil
}
// Start begins watching for file changes
func (w *Watcher) Start() error {
w.mu.Lock()
if w.isRunning {
w.mu.Unlock()
return nil
}
w.isRunning = true
w.mu.Unlock()
// Add watch paths
for _, watchPath := range w.watchPaths {
if err := w.addWatchRecursive(watchPath); err != nil {
log.Printf("[Watcher] Warning: failed to watch %s: %v", watchPath, err)
}
}
// Start event processing goroutine
go w.processEvents()
// Start initial scan
go w.RunScan()
log.Printf("[Watcher] Started watching %d paths", len(w.watchPaths))
return nil
}
// Stop stops the watcher
func (w *Watcher) Stop() error {
w.mu.Lock()
if !w.isRunning {
w.mu.Unlock()
return nil
}
w.isRunning = false
w.mu.Unlock()
close(w.stopChan)
w.debouncer.Stop()
return w.fsWatcher.Close()
}
// addWatchRecursive adds a directory and all subdirectories to the watch list
func (w *Watcher) addWatchRecursive(root string) error {
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // Skip inaccessible paths
}
if info.IsDir() {
if err := w.fsWatcher.Add(path); err != nil {
log.Printf("[Watcher] Warning: cannot watch %s: %v", path, err)
}
}
return nil
})
}
// processEvents handles fsnotify events
func (w *Watcher) processEvents() {
for {
select {
case <-w.stopChan:
return
case event, ok := <-w.fsWatcher.Events:
if !ok {
return
}
w.handleFsEvent(event)
case err, ok := <-w.fsWatcher.Errors:
if !ok {
return
}
log.Printf("[Watcher] Error: %v", err)
}
}
}
// handleFsEvent converts an fsnotify event to a FileEvent
func (w *Watcher) handleFsEvent(event fsnotify.Event) {
// Get relative path
relPath, err := filepath.Rel(w.filesPath, event.Name)
if err != nil {
relPath = event.Name
}
// Clean up path
relPath = filepath.ToSlash(relPath)
// Determine event type
var eventType FileEventType
switch {
case event.Op&fsnotify.Create == fsnotify.Create:
eventType = EventTypeAdd
// If it's a new directory, add it to watch
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
w.fsWatcher.Add(event.Name)
}
case event.Op&fsnotify.Write == fsnotify.Write:
eventType = EventTypeChange
case event.Op&fsnotify.Remove == fsnotify.Remove:
eventType = EventTypeDelete
case event.Op&fsnotify.Rename == fsnotify.Rename:
eventType = EventTypeRename
default:
return // Ignore other events
}
// Skip directories for file events (except remove)
if eventType != EventTypeDelete {
info, err := os.Stat(event.Name)
if err == nil && info.IsDir() {
return
}
}
// Get file info if available
var size int64
if eventType != EventTypeDelete {
if info, err := os.Stat(event.Name); err == nil {
size = info.Size()
}
}
fileEvent := FileEvent{
Type: eventType,
Path: relPath,
Size: size,
Timestamp: NowMS(),
}
// Send to debouncer
w.debouncer.Add(fileEvent)
}
// handleDebouncedEvent processes a debounced event
func (w *Watcher) handleDebouncedEvent(event FileEvent) {
// Compute partial hash for add/change events
if event.Type == EventTypeAdd || event.Type == EventTypeChange {
fullPath := filepath.Join(w.filesPath, event.Path)
if hash, err := computePartialHash(fullPath); err == nil {
event.PartialHash = hash
}
}
// Add to buffer
w.mu.Lock()
w.eventBuffer = append(w.eventBuffer, event)
// Keep buffer size reasonable
if len(w.eventBuffer) > 10000 {
w.eventBuffer = w.eventBuffer[len(w.eventBuffer)-5000:]
}
w.mu.Unlock()
// Dispatch to subscribers
w.dispatcher.Dispatch(event)
log.Printf("[Watcher] Event: %s %s", event.Type, event.Path)
}
// RunScan performs a full directory scan
func (w *Watcher) RunScan() {
w.mu.Lock()
if w.isScanning {
w.mu.Unlock()
return
}
w.isScanning = true
w.mu.Unlock()
defer func() {
w.mu.Lock()
w.isScanning = false
w.lastScan = NowMS()
w.mu.Unlock()
}()
log.Println("[Watcher] Starting directory scan...")
fileCount := 0
for _, watchPath := range w.watchPaths {
count := w.scanDirectory(watchPath)
fileCount += count
}
w.mu.Lock()
w.fileCount = fileCount
w.mu.Unlock()
log.Printf("[Watcher] Scan complete: %d files found", fileCount)
}
// scanDirectory scans a directory and emits add events for files
func (w *Watcher) scanDirectory(root string) int {
count := 0
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
// Skip directories
if info.IsDir() {
return nil
}
// Skip hidden files
if strings.HasPrefix(filepath.Base(path), ".") {
return nil
}
// Get relative path
relPath, err := filepath.Rel(w.filesPath, path)
if err != nil {
relPath = path
}
relPath = filepath.ToSlash(relPath)
// Emit add event (through debouncer)
event := FileEvent{
Type: EventTypeAdd,
Path: relPath,
Size: info.Size(),
Timestamp: NowMS(),
}
// Compute partial hash
if hash, err := computePartialHash(path); err == nil {
event.PartialHash = hash
}
// Dispatch directly (skip debouncer for scan)
w.dispatcher.Dispatch(event)
count++
return nil
})
return count
}
// GetRecentEvents returns events since a given timestamp
func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
w.mu.RLock()
defer w.mu.RUnlock()
result := make([]FileEvent, 0)
for _, event := range w.eventBuffer {
if event.Timestamp > sinceMS {
result = append(result, event)
if limit > 0 && len(result) >= limit {
break
}
}
}
return result
}
// GetStatus returns watcher status
func (w *Watcher) GetStatus() ScanStatusResponse {
w.mu.RLock()
defer w.mu.RUnlock()
status := "running"
if !w.isRunning {
status = "stopped"
}
return ScanStatusResponse{
Status: status,
Scanning: w.isScanning,
LastScan: w.lastScan,
FileCount: w.fileCount,
}
}
// computePartialHash computes SHA-256 hash of first 64KB of a file
func computePartialHash(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
hasher := sha256.New()
buffer := make([]byte, PartialHashSize)
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "", err
}
hasher.Write(buffer[:n])
return hex.EncodeToString(hasher.Sum(nil)), nil
}