metastar/meta-core
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

leader discovery fix and code cleaning

Browse Source
This commit is contained in:
Pierre
2026-02-01 13:39:32 +01:00
parent d1e74e0b70
commit 9a60f0e231
14 changed files with 2095 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/handlers.go
+31
View File
@@ -134,6 +134,37 @@ func (s *Server) handleRole(w http.ResponseWriter, r *http.Request) {
})
}
// URLsResponse is the response for /urls
type URLsResponse struct {
Hostname string `json:"hostname"`
BaseUrl string `json:"baseUrl"`
ApiUrl string `json:"apiUrl"`
RedisUrl string `json:"redisUrl"`
WebdavUrl string `json:"webdavUrl"`
IsLeader bool `json:"isLeader"`
}
// handleURLs handles GET /urls
// Returns URLs for the current leader (or this instance if leader)
func (s *Server) handleURLs(w http.ResponseWriter, r *http.Request) {
info := s.election.LeaderInfo()
if info == nil {
writeError(w, http.StatusServiceUnavailable, "no leader available")
return
}
response := URLsResponse{
Hostname: info.Hostname,
BaseUrl: info.BaseUrl,
ApiUrl: info.ApiUrl,
RedisUrl: info.RedisUrl,
WebdavUrl: info.WebdavUrl,
IsLeader: s.election.IsLeader(),
}
writeJSON(w, http.StatusOK, response)
}
// handleGetMeta handles GET /meta/{hash}
func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
internal/api/server.go
+43 -20
View File
@@ -18,17 +18,18 @@ import (
// Server is the HTTP API server for meta-core
type Server struct {
config *config.Config
election *leader.Election
discovery *discovery.Service
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
config *config.Config
election *leader.Election
discovery *discovery.Service
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
mountPoller *mounts.Poller
watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
router *mux.Router
server *http.Server
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
router *mux.Router
server *http.Server
}
// NewServer creates a new API server
@@ -46,16 +47,8 @@ func NewServer(
router: mux.NewRouter(),
}
// Initialize mounts manager
mountsManager, err := mounts.NewManager(cfg)
if err != nil {
log.Printf("[API] Warning: failed to initialize mounts manager: %v", err)
} else {
s.mountsManager = mountsManager
s.mountsHandlers = mounts.NewHandlers(mountsManager)
}
// Initialize file watcher (if enabled)
// Initialize file watcher (if enabled) - before mounts so we can pass scanFunc
var scanFunc mounts.ScanFunc
if cfg.EnableFileWatcher && len(cfg.WatchFolderList) > 0 {
s.watcherDispatcher = watcher.NewDispatcher()
fileWatcher, err := watcher.NewWatcher(cfg, s.watcherDispatcher)
@@ -64,9 +57,24 @@ func NewServer(
} else {
s.fileWatcher = fileWatcher
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
// Create scan function from watcher
scanFunc = fileWatcher.ScanMountPath
}
}
// Initialize mounts manager
mountsManager, err := mounts.NewManager(cfg)
if err != nil {
log.Printf("[API] Warning: failed to initialize mounts manager: %v", err)
} else {
s.mountsManager = mountsManager
s.mountsHandlers = mounts.NewHandlers(mountsManager, scanFunc)
// Create and configure mount poller
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
mountsManager.SetPoller(s.mountPoller)
}
s.setupRoutes()
return s
}
@@ -78,6 +86,7 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/status", s.handleStatus).Methods("GET")
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
s.router.HandleFunc("/role", s.handleRole).Methods("GET")
s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
// Metadata Editor API routes (must be before /meta/{hash} routes)
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
@@ -170,11 +179,25 @@ func (s *Server) Start() error {
}
}
// Start mount poller (if initialized)
if s.mountPoller != nil {
if err := s.mountPoller.Start(); err != nil {
log.Printf("[API] Warning: failed to start mount poller: %v", err)
}
}
return nil
}
// Stop gracefully stops the HTTP server
func (s *Server) Stop() error {
// Stop mount poller
if s.mountPoller != nil {
if err := s.mountPoller.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop mount poller: %v", err)
}
}
// Stop file watcher
if s.fileWatcher != nil {
if err := s.fileWatcher.Stop(); err != nil {
internal/discovery/service.go
+56 -55
View File
@@ -13,18 +13,14 @@ import (
"github.com/metazla/meta-core/internal/config"
)
// ServiceInfo matches the TypeScript ServiceInfo interface
// ServiceInfo contains simplified service registration data
// URLs are obtained via the /urls API endpoint
type ServiceInfo struct {
Name string `json:"name"`
Version string `json:"version"`
API string `json:"api"`
Status string `json:"status"`
PID int `json:"pid"`
Hostname string `json:"hostname"`
StartedAt string `json:"startedAt"`
LastHeartbeat string `json:"lastHeartbeat"`
Capabilities []string `json:"capabilities"`
Endpoints map[string]string `json:"endpoints"`
Name string `json:"name"`
Hostname string `json:"hostname"`
BaseUrl string `json:"baseUrl"`
Status string `json:"status"`
LastHeartbeat string `json:"lastHeartbeat"`
}
// Service handles service registration and discovery
@@ -41,10 +37,11 @@ type Service struct {
// NewService creates a new service discovery instance
func NewService(cfg *config.Config) *Service {
hostname, _ := os.Hostname()
return &Service{
config: cfg,
servicesDir: cfg.ServicesDir(),
serviceFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+".json"),
serviceFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
stopChan: make(chan struct{}),
}
}
@@ -90,40 +87,20 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
hostname, _ := os.Hostname()
ip := getLocalIP()
// Use BASE_URL for external access (dashboard navigation), fall back to internal IP
var apiBase string
// Use BASE_URL for external access, fall back to internal IP
var baseUrl string
if s.config.BaseURL != "" {
apiBase = s.config.BaseURL
baseUrl = s.config.BaseURL
} else {
apiBase = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
}
// Internal URLs always use container IP for container-to-container communication
internalBase := fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
metaCoreBase := fmt.Sprintf("http://%s:%d", ip, s.config.HTTPPort)
return &ServiceInfo{
Name: s.config.ServiceName,
Version: s.config.ServiceVersion,
API: apiBase,
Status: "running",
PID: os.Getpid(),
Hostname: hostname,
StartedAt: time.Now().UTC().Format(time.RFC3339),
BaseUrl: baseUrl,
Status: "running",
LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
Capabilities: []string{"meta-core"},
Endpoints: map[string]string{
// meta-core sidecar endpoints (port 9000) - always internal
"health": metaCoreBase + "/health",
"meta": metaCoreBase + "/meta",
"leader": metaCoreBase + "/leader",
"services": metaCoreBase + "/services",
// main service endpoints - external for dashboard
"api": apiBase + "/api",
"webdav": apiBase + "/webdav",
// callback uses internal IP for container-to-container plugin communication
"callback": internalBase + "/api/plugins/callback",
},
}
}
@@ -198,23 +175,44 @@ func (s *Service) UpdateStatus(status string) error {
}
// Discover finds a service by name
// Looks for files matching pattern: name-*.json (hostname-based naming)
func (s *Service) Discover(name string) (*ServiceInfo, error) {
filePath := filepath.Join(s.servicesDir, name+".json")
data, err := os.ReadFile(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
// First try exact match for backward compatibility
exactPath := filepath.Join(s.servicesDir, name+".json")
if data, err := os.ReadFile(exactPath); err == nil {
var info ServiceInfo
if err := json.Unmarshal(data, &info); err == nil {
return s.checkStale(&info), nil
}
}
// Search for hostname-based files: name-*.json
pattern := filepath.Join(s.servicesDir, name+"-*.json")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
var info ServiceInfo
if err := json.Unmarshal(data, &info); err != nil {
return nil, err
// Return the first valid service found
for _, match := range matches {
data, err := os.ReadFile(match)
if err != nil {
continue
}
var info ServiceInfo
if err := json.Unmarshal(data, &info); err != nil {
continue
}
return s.checkStale(&info), nil
}
// Check if service is stale
return nil, nil
}
// checkStale marks a service as stale if heartbeat is too old
func (s *Service) checkStale(info *ServiceInfo) *ServiceInfo {
lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
if err == nil {
staleThreshold := time.Duration(s.config.StaleThresholdMS) * time.Millisecond
@@ -222,8 +220,7 @@ func (s *Service) Discover(name string) (*ServiceInfo, error) {
info.Status = "stale"
}
}
return &info, nil
return info
}
// DiscoverAll finds all registered services
@@ -242,16 +239,20 @@ func (s *Service) DiscoverAll() ([]*ServiceInfo, error) {
continue
}
name := entry.Name()[:len(entry.Name())-5] // Remove .json extension
info, err := s.Discover(name)
filePath := filepath.Join(s.servicesDir, entry.Name())
data, err := os.ReadFile(filePath)
if err != nil {
log.Printf("[Discovery] Failed to read service %s: %v", name, err)
log.Printf("[Discovery] Failed to read service file %s: %v", entry.Name(), err)
continue
}
if info != nil {
services = append(services, info)
var info ServiceInfo
if err := json.Unmarshal(data, &info); err != nil {
log.Printf("[Discovery] Failed to parse service file %s: %v", entry.Name(), err)
continue
}
services = append(services, s.checkStale(&info))
}
return services, nil
internal/leader/election.go
+20 -12
View File
@@ -14,12 +14,13 @@ import (
"github.com/metazla/meta-core/internal/config"
)
// LeaderLockInfo matches the TypeScript LeaderLockInfo interface
// LeaderLockInfo contains all URLs and metadata for the leader
type LeaderLockInfo struct {
Host string `json:"host"`
API string `json:"api"`
HTTP string `json:"http"`
BaseURL string `json:"baseUrl,omitempty"`
Hostname string `json:"hostname"`
BaseUrl string `json:"baseUrl"`
ApiUrl string `json:"apiUrl"` // meta-core API (port 9000)
RedisUrl string `json:"redisUrl"`
WebdavUrl string `json:"webdavUrl"`
Timestamp int64 `json:"timestamp"`
PID int `json:"pid"`
}
@@ -236,7 +237,7 @@ func (e *Election) transitionToLeader() error {
// Connect storage to local Redis
if e.storage != nil {
if err := e.storage.Connect(info.API); err != nil {
if err := e.storage.Connect(info.RedisUrl); err != nil {
log.Printf("[Election] Warning: failed to connect storage: %v", err)
}
}
@@ -269,11 +270,11 @@ func (e *Election) transitionToFollower() error {
e.mu.Unlock()
if info != nil {
log.Printf("[Election] Leader is at %s", info.API)
log.Printf("[Election] Leader is at %s (Redis: %s)", info.ApiUrl, info.RedisUrl)
// Connect storage to leader's Redis
if e.storage != nil {
if err := e.storage.Connect(info.API); err != nil {
if err := e.storage.Connect(info.RedisUrl); err != nil {
log.Printf("[Election] Warning: failed to connect to leader: %v", err)
}
}
@@ -293,11 +294,18 @@ func (e *Election) buildLeaderInfo() *LeaderLockInfo {
ip := getLocalIP()
hostname, _ := os.Hostname()
// Use BaseURL if configured, otherwise construct from IP
baseUrl := e.config.BaseURL
if baseUrl == "" {
baseUrl = fmt.Sprintf("http://%s:%d", ip, e.config.APIPort)
}
return &LeaderLockInfo{
Host: hostname,
API: fmt.Sprintf("redis://%s:%d", ip, e.config.RedisPort),
HTTP: fmt.Sprintf("http://%s:%d", ip, e.config.APIPort),
BaseURL: e.config.BaseURL,
Hostname: hostname,
BaseUrl: baseUrl,
ApiUrl: fmt.Sprintf("http://%s:%d", ip, e.config.HTTPPort),
RedisUrl: fmt.Sprintf("redis://%s:%d", ip, e.config.RedisPort),
WebdavUrl: baseUrl + "/webdav",
Timestamp: time.Now().UnixMilli(),
PID: os.Getpid(),
}
internal/mounts/handlers.go
+93 -3
View File
@@ -9,12 +9,16 @@ import (
// Handlers provides HTTP handlers for mount operations
type Handlers struct {
manager *Manager
manager *Manager
scanFunc ScanFunc
}
// NewHandlers creates new mount handlers
func NewHandlers(manager *Manager) *Handlers {
return &Handlers{manager: manager}
func NewHandlers(manager *Manager, scanFunc ScanFunc) *Handlers {
return &Handlers{
manager: manager,
scanFunc: scanFunc,
}
}
// RegisterRoutes registers all mount-related routes
@@ -24,10 +28,12 @@ func (h *Handlers) RegisterRoutes(r *mux.Router) {
r.HandleFunc("/api/mounts", h.handleCreateMount).Methods("POST")
r.HandleFunc("/api/mounts/rclone/remotes", h.handleListRcloneRemotes).Methods("GET")
r.HandleFunc("/api/mounts/{id}", h.handleGetMount).Methods("GET")
r.HandleFunc("/api/mounts/{id}", h.handleUpdateMount).Methods("PUT", "PATCH")
r.HandleFunc("/api/mounts/{id}", h.handleDeleteMount).Methods("DELETE")
r.HandleFunc("/api/mounts/{id}/mount", h.handleRequestMount).Methods("POST")
r.HandleFunc("/api/mounts/{id}/unmount", h.handleRequestUnmount).Methods("POST")
r.HandleFunc("/api/mounts/{id}/safe-unmount", h.handleSafeUnmount).Methods("POST")
r.HandleFunc("/api/mounts/{id}/scan", h.handleTriggerMountScan).Methods("POST")
}
// handleListMounts handles GET /api/mounts
@@ -194,6 +200,90 @@ func (h *Handlers) handleListRcloneRemotes(w http.ResponseWriter, r *http.Reques
writeJSON(w, http.StatusOK, RcloneRemotesResponse{Remotes: remotes})
}
// handleUpdateMount handles PUT/PATCH /api/mounts/{id}
func (h *Handlers) handleUpdateMount(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
var updates map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&updates); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
mount, err := h.manager.UpdateMount(id, updates)
if err != nil {
if err.Error() == "mount not found" {
writeError(w, http.StatusNotFound, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, MountResponse{Mount: mount})
}
// handleTriggerMountScan handles POST /api/mounts/{id}/scan
func (h *Handlers) handleTriggerMountScan(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
// Get mount info
mount, err := h.manager.GetMount(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if mount == nil {
writeError(w, http.StatusNotFound, "mount not found")
return
}
// Check if mount is mounted
if !mount.Mounted {
writeJSON(w, http.StatusBadRequest, MountScanResponse{
Status: "error",
Message: "mount is not currently mounted",
MountID: mount.ID,
MountPath: mount.MountPath,
})
return
}
// Check if scan function is available
if h.scanFunc == nil {
writeJSON(w, http.StatusServiceUnavailable, MountScanResponse{
Status: "error",
Message: "file watcher not available",
MountID: mount.ID,
MountPath: mount.MountPath,
})
return
}
// Trigger scan
fileCount, err := h.scanFunc(mount.MountPath)
if err != nil {
writeJSON(w, http.StatusInternalServerError, MountScanResponse{
Status: "error",
Message: err.Error(),
MountID: mount.ID,
MountPath: mount.MountPath,
})
return
}
writeJSON(w, http.StatusOK, MountScanResponse{
Status: "ok",
Message: "scan triggered successfully",
MountID: mount.ID,
MountPath: mount.MountPath,
FileCount: fileCount,
})
}
// Helper functions
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
internal/mounts/manager.go
+150 -13
View File
@@ -23,6 +23,7 @@ type Manager struct {
mu sync.RWMutex
filesPath string
mountsDir string
poller *Poller
}
// NewManager creates a new mount manager
@@ -41,6 +42,11 @@ func NewManager(cfg *config.Config) (*Manager, error) {
return m, nil
}
// SetPoller sets the poller for the manager
func (m *Manager) SetPoller(poller *Poller) {
m.poller = poller
}
// ensureDirs creates required directories
func (m *Manager) ensureDirs() error {
dirs := []string{
@@ -95,14 +101,30 @@ func (m *Manager) writeConfig(mountsFile *MountsFile) error {
return os.WriteFile(m.config.MountsFilePath(), data, 0644)
}
// IsMounted checks if a path is currently mounted
// IsMounted checks if a path is currently mounted by parsing /proc/mounts
// Note: Uses /proc/mounts instead of findmnt for Alpine compatibility
func (m *Manager) IsMounted(mountPath string) bool {
cmd := exec.Command("findmnt", "-n", mountPath)
output, err := cmd.CombinedOutput()
data, err := os.ReadFile("/proc/mounts")
if err != nil {
return false
}
return strings.TrimSpace(string(output)) != ""
// Clean the mount path (remove trailing slash)
cleanPath := strings.TrimSuffix(mountPath, "/")
// Parse each line of /proc/mounts
lines := strings.Split(string(data), "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) >= 2 {
// Field 1 is the mount point
mountPoint := strings.TrimSuffix(fields[1], "/")
if mountPoint == cleanPath {
return true
}
}
}
return false
}
// ReadError reads the error file for a mount
@@ -164,6 +186,26 @@ func ObscurePassword(password string) (string, error) {
// ListMounts returns all mounts with their current status
func (m *Manager) ListMounts() ([]MountStatus, error) {
statuses, err := m.listMountsWithoutPollingStatus()
if err != nil {
return nil, err
}
// Add polling status from poller (done separately to avoid deadlock)
if m.poller != nil {
for i := range statuses {
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
statuses[i].PollingActive = pollingStatus.Active
statuses[i].LastPolledScan = pollingStatus.LastScan
}
}
return statuses, nil
}
// listMountsWithoutPollingStatus returns mounts without querying the poller
// This is used internally by the poller to avoid deadlocks
func (m *Manager) listMountsWithoutPollingStatus() ([]MountStatus, error) {
mountsFile, err := m.readConfig()
if err != nil {
return nil, err
@@ -197,12 +239,21 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
mounted := m.IsMounted(mount.MountPath)
errMsg, _ := m.ReadError(mount.ID)
return &MountStatus{
status := &MountStatus{
MountConfig: mount,
Mounted: mounted,
Error: errMsg,
LastChecked: NowMS(),
}, nil
}
// Add polling status from poller (safe since GetMount isn't called from poller with lock held)
if m.poller != nil {
pollingStatus := m.poller.GetPollingStatus(mount.ID)
status.PollingActive = pollingStatus.Active
status.LastPolledScan = pollingStatus.LastScan
}
return status, nil
}
}
@@ -258,14 +309,26 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
enabled = *req.Enabled
}
// Default polling settings
pollingEnabled := false
if req.PollingEnabled != nil {
pollingEnabled = *req.PollingEnabled
}
pollingIntervalMs := DefaultPollingIntervalMs
if req.PollingIntervalMs != nil && *req.PollingIntervalMs >= MinPollingIntervalMs {
pollingIntervalMs = *req.PollingIntervalMs
}
mount := MountConfig{
ID: id,
Name: req.Name,
Type: req.Type,
Enabled: enabled,
DesiredMounted: enabled, // Auto-mount if enabled
MountPath: mountPath,
Options: req.Options,
ID: id,
Name: req.Name,
Type: req.Type,
Enabled: enabled,
DesiredMounted: enabled, // Auto-mount if enabled
MountPath: mountPath,
Options: req.Options,
PollingEnabled: pollingEnabled,
PollingIntervalMs: pollingIntervalMs,
}
// Type-specific fields
@@ -297,6 +360,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
log.Printf("[Mounts] Created mount config: %s (%s) -> %s", mount.Name, mount.Type, mount.MountPath)
// Notify poller about new mount
if m.poller != nil {
m.poller.NotifyMountChanged()
}
return &MountStatus{
MountConfig: mount,
Mounted: false,
@@ -304,6 +372,63 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
}, nil
}
// UpdateMount updates mount configuration (for polling settings)
func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*MountStatus, error) {
mountsFile, err := m.readConfig()
if err != nil {
return nil, err
}
index := -1
for i, mount := range mountsFile.Mounts {
if mount.ID == id {
index = i
break
}
}
if index == -1 {
return nil, fmt.Errorf("mount not found")
}
// Apply updates
mount := &mountsFile.Mounts[index]
if val, ok := updates["pollingEnabled"]; ok {
if enabled, ok := val.(bool); ok {
mount.PollingEnabled = enabled
}
}
if val, ok := updates["pollingIntervalMs"]; ok {
// Handle both int and float64 (JSON numbers are float64)
switch v := val.(type) {
case int:
if v >= MinPollingIntervalMs {
mount.PollingIntervalMs = v
}
case float64:
if int(v) >= MinPollingIntervalMs {
mount.PollingIntervalMs = int(v)
}
}
}
if err := m.writeConfig(mountsFile); err != nil {
return nil, err
}
log.Printf("[Mounts] Updated mount config: %s", mount.Name)
// Notify poller about config change
if m.poller != nil {
m.poller.NotifyMountChanged()
}
// Return updated status
return m.GetMount(id)
}
// RequestMount sets desiredMounted to true
func (m *Manager) RequestMount(id string) error {
mountsFile, err := m.readConfig()
@@ -318,6 +443,12 @@ func (m *Manager) RequestMount(id string) error {
return err
}
log.Printf("[Mounts] Mount requested: %s", mount.Name)
// Notify poller about state change
if m.poller != nil {
m.poller.NotifyMountChanged()
}
return nil
}
}
@@ -339,6 +470,12 @@ func (m *Manager) RequestUnmount(id string) error {
return err
}
log.Printf("[Mounts] Unmount requested: %s", mount.Name)
// Notify poller about state change
if m.poller != nil {
m.poller.NotifyMountChanged()
}
return nil
}
}
internal/mounts/poller.go
+269
View File
@@ -0,0 +1,269 @@
package mounts
import (
"log"
"sync"
"time"
)
// ScanFunc is a function type for scanning a mount path
type ScanFunc func(path string) (int, error)
// mountPoller manages polling for a single mount
type mountPoller struct {
mountID string
stopChan chan struct{}
stopped bool
mu sync.Mutex
}
// PollingStatus holds the polling state for a mount
type PollingStatus struct {
Active bool `json:"active"`
LastScan int64 `json:"lastScan,omitempty"`
}
// Poller manages per-mount polling goroutines
type Poller struct {
manager *Manager
scanFunc ScanFunc
pollers map[string]*mountPoller // key: mount ID
status map[string]*PollingStatus
running bool
mu sync.RWMutex
}
// NewPoller creates a new mount poller
func NewPoller(manager *Manager, scanFunc ScanFunc) *Poller {
return &Poller{
manager: manager,
scanFunc: scanFunc,
pollers: make(map[string]*mountPoller),
status: make(map[string]*PollingStatus),
}
}
// Start begins the polling service
func (p *Poller) Start() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.running {
return nil
}
p.running = true
log.Println("[Poller] Starting mount polling service")
// Initial sync with mount configurations
go p.syncWithMounts()
return nil
}
// Stop stops all polling goroutines
func (p *Poller) Stop() error {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
return nil
}
p.running = false
log.Println("[Poller] Stopping mount polling service")
// Stop all pollers
for id, poller := range p.pollers {
p.stopPoller(poller)
delete(p.pollers, id)
}
return nil
}
// SyncWithMounts synchronizes pollers with current mount configurations
func (p *Poller) SyncWithMounts() {
p.syncWithMounts()
}
// syncWithMounts is the internal implementation
func (p *Poller) syncWithMounts() {
p.mu.RLock()
running := p.running
p.mu.RUnlock()
if !running {
return
}
// Get current mounts BEFORE acquiring the write lock to avoid deadlock
// (ListMounts calls GetPollingStatus which needs RLock)
mounts, err := p.manager.listMountsWithoutPollingStatus()
if err != nil {
log.Printf("[Poller] Failed to list mounts: %v", err)
return
}
p.mu.Lock()
defer p.mu.Unlock()
// Re-check running state after acquiring lock
if !p.running {
return
}
// Track which mounts we've seen
seenMounts := make(map[string]bool)
for _, mount := range mounts {
seenMounts[mount.ID] = true
// Check if mount should be polled
shouldPoll := mount.PollingEnabled && mount.Mounted
existingPoller, exists := p.pollers[mount.ID]
if shouldPoll {
// Calculate interval
interval := mount.PollingIntervalMs
if interval < MinPollingIntervalMs {
interval = DefaultPollingIntervalMs
}
if !exists {
// Start new poller
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
p.startPollerForMount(mount.ID, mount.MountPath, interval)
}
} else if exists {
// Stop existing poller
log.Printf("[Poller] Stopping poller for mount %s", mount.Name)
p.stopPoller(existingPoller)
delete(p.pollers, mount.ID)
}
}
// Stop pollers for mounts that no longer exist
for id, poller := range p.pollers {
if !seenMounts[id] {
log.Printf("[Poller] Stopping poller for removed mount %s", id)
p.stopPoller(poller)
delete(p.pollers, id)
}
}
}
// startPollerForMount starts a polling goroutine for a mount
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
poller := &mountPoller{
mountID: mountID,
stopChan: make(chan struct{}),
}
p.pollers[mountID] = poller
// Initialize status
if _, exists := p.status[mountID]; !exists {
p.status[mountID] = &PollingStatus{}
}
p.status[mountID].Active = true
go func() {
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-poller.stopChan:
return
case <-ticker.C:
p.runPollScan(mountID, mountPath)
}
}
}()
}
// stopPoller stops a mount poller
func (p *Poller) stopPoller(poller *mountPoller) {
poller.mu.Lock()
defer poller.mu.Unlock()
if poller.stopped {
return
}
poller.stopped = true
close(poller.stopChan)
// Update status
if status, exists := p.status[poller.mountID]; exists {
status.Active = false
}
}
// runPollScan executes a scan for a mount
func (p *Poller) runPollScan(mountID, mountPath string) {
if p.scanFunc == nil {
return
}
fileCount, err := p.scanFunc(mountPath)
if err != nil {
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
return
}
// Update status
p.mu.Lock()
if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS()
}
p.mu.Unlock()
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
}
// TriggerScan triggers an immediate scan for a mount
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
if p.scanFunc == nil {
return 0, nil
}
fileCount, err := p.scanFunc(mountPath)
if err != nil {
return 0, err
}
// Update status
p.mu.Lock()
if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS()
} else {
p.status[mountID] = &PollingStatus{
LastScan: NowMS(),
}
}
p.mu.Unlock()
return fileCount, nil
}
// GetPollingStatus returns the polling status for a mount
func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
p.mu.RLock()
defer p.mu.RUnlock()
if status, exists := p.status[mountID]; exists {
return &PollingStatus{
Active: status.Active,
LastScan: status.LastScan,
}
}
return &PollingStatus{}
}
// NotifyMountChanged notifies the poller that a mount's state has changed
func (p *Poller) NotifyMountChanged() {
go p.syncWithMounts()
}
internal/mounts/types.go
+28 -3
View File
@@ -11,6 +11,12 @@ const (
MountTypeRclone MountType = "rclone"
)
// Polling constants
const (
DefaultPollingIntervalMs = 30000 // 30 seconds
MinPollingIntervalMs = 5000 // 5 seconds
)
// MountConfig represents a mount configuration
type MountConfig struct {
// Common fields
@@ -36,14 +42,20 @@ type MountConfig struct {
// rclone-specific fields
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
// Polling configuration
PollingEnabled bool `json:"pollingEnabled"`
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
}
// MountStatus represents the runtime status of a mount
type MountStatus struct {
MountConfig
Mounted bool `json:"mounted"`
Error string `json:"error,omitempty"`
LastChecked int64 `json:"lastChecked"`
Mounted bool `json:"mounted"`
Error string `json:"error,omitempty"`
LastChecked int64 `json:"lastChecked"`
PollingActive bool `json:"pollingActive,omitempty"`
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
}
// MountsFile represents the mounts.json file structure
@@ -79,6 +91,19 @@ type CreateMountRequest struct {
// rclone
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
// Polling
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
}
// MountScanResponse is the response for mount scan operations
type MountScanResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
MountID string `json:"mountId"`
MountPath string `json:"mountPath"`
FileCount int `json:"fileCount,omitempty"`
}
// MountResponse is the response for mount operations
internal/watcher/watcher.go
+30
View File
@@ -3,6 +3,7 @@ package watcher
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log"
"os"
@@ -308,6 +309,35 @@ func (w *Watcher) scanDirectory(root string) int {
return count
}
// ScanMountPath scans a specific mount path and returns file count
// This is used by the mount poller to scan individual mounts
func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
// Validate path is under filesPath
if !strings.HasPrefix(mountPath, w.filesPath) {
return 0, fmt.Errorf("mount path %s is not under files path %s", mountPath, w.filesPath)
}
// Check directory exists
info, err := os.Stat(mountPath)
if err != nil {
if os.IsNotExist(err) {
return 0, fmt.Errorf("mount path %s does not exist", mountPath)
}
return 0, fmt.Errorf("failed to stat mount path %s: %w", mountPath, err)
}
if !info.IsDir() {
return 0, fmt.Errorf("mount path %s is not a directory", mountPath)
}
// Scan the directory
log.Printf("[Watcher] Scanning mount path: %s", mountPath)
count := w.scanDirectory(mountPath)
log.Printf("[Watcher] Mount scan complete: %s (%d files)", mountPath, count)
return count, nil
}
// GetRecentEvents returns events since a given timestamp
func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
w.mu.RLock()