cleaning of the files event system

This commit is contained in:
Pierre
2026-02-08 15:30:31 +01:00
parent ce6995c388
commit a9cbbfacea
16 changed files with 1437 additions and 577 deletions
+192
View File
@@ -0,0 +1,192 @@
package watchers
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
)
// Handlers provides HTTP handlers for watchers management
type Handlers struct {
manager *Manager
poller *Poller
}
// NewHandlers creates new watchers handlers
func NewHandlers(manager *Manager, poller *Poller) *Handlers {
return &Handlers{
manager: manager,
poller: poller,
}
}
// RegisterRoutes registers all watchers-related routes
func (h *Handlers) RegisterRoutes(r *mux.Router) {
r.HandleFunc("/api/watchers", h.handleList).Methods("GET")
r.HandleFunc("/api/watchers", h.handleCreate).Methods("POST")
r.HandleFunc("/api/watchers/scan-all", h.handleScanAll).Methods("POST")
r.HandleFunc("/api/watchers/{id}", h.handleGet).Methods("GET")
r.HandleFunc("/api/watchers/{id}", h.handleUpdate).Methods("PUT")
r.HandleFunc("/api/watchers/{id}", h.handleDelete).Methods("DELETE")
r.HandleFunc("/api/watchers/{id}/scan", h.handleScan).Methods("POST")
}
// handleList handles GET /api/watchers
func (h *Handlers) handleList(w http.ResponseWriter, r *http.Request) {
statuses := h.poller.GetWatcherStatuses()
writeJSON(w, http.StatusOK, WatchersListResponse{
Watchers: statuses,
Count: len(statuses),
})
}
// handleGet handles GET /api/watchers/{id}
func (h *Handlers) handleGet(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
watcher, err := h.manager.Get(id)
if err != nil {
writeError(w, http.StatusNotFound, err.Error())
return
}
// Get runtime status
status := h.poller.GetStatus(id)
response := WatcherStatus{
WatcherConfig: *watcher,
Active: status.Active,
LastScan: status.LastScan,
FileCount: status.FileCount,
IsScanning: status.IsScanning,
}
writeJSON(w, http.StatusOK, response)
}
// handleCreate handles POST /api/watchers
func (h *Handlers) handleCreate(w http.ResponseWriter, r *http.Request) {
var req CreateWatcherRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
watcher, err := h.manager.Create(req)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusCreated, watcher)
}
// handleUpdate handles PUT /api/watchers/{id}
func (h *Handlers) handleUpdate(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
var req UpdateWatcherRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
watcher, err := h.manager.Update(id, req)
if err != nil {
if err.Error() == "watcher not found: "+id {
writeError(w, http.StatusNotFound, err.Error())
} else {
writeError(w, http.StatusBadRequest, err.Error())
}
return
}
// Get runtime status
status := h.poller.GetStatus(id)
response := WatcherStatus{
WatcherConfig: *watcher,
Active: status.Active,
LastScan: status.LastScan,
FileCount: status.FileCount,
IsScanning: status.IsScanning,
}
writeJSON(w, http.StatusOK, response)
}
// handleDelete handles DELETE /api/watchers/{id}
func (h *Handlers) handleDelete(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
if err := h.manager.Delete(id); err != nil {
if err.Error() == "watcher not found: "+id {
writeError(w, http.StatusNotFound, err.Error())
} else {
writeError(w, http.StatusInternalServerError, err.Error())
}
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Watcher deleted",
})
}
// handleScan handles POST /api/watchers/{id}/scan
func (h *Handlers) handleScan(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
fileCount, err := h.poller.TriggerScan(id)
if err != nil {
if err.Error() == "watcher not found: "+id {
writeError(w, http.StatusNotFound, err.Error())
} else {
writeError(w, http.StatusInternalServerError, err.Error())
}
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Scan triggered",
"fileCount": fileCount,
})
}
// handleScanAll handles POST /api/watchers/scan-all
func (h *Handlers) handleScanAll(w http.ResponseWriter, r *http.Request) {
totalCount, err := h.poller.TriggerScanAll()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "All watchers scanned",
"totalFiles": totalCount,
})
}
// Helper functions
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
func writeError(w http.ResponseWriter, status int, message string) {
writeJSON(w, status, map[string]string{
"error": http.StatusText(status),
"message": message,
})
}
+314
View File
@@ -0,0 +1,314 @@
package watchers
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sync"
"github.com/google/uuid"
"github.com/metazla/meta-core/internal/config"
)
// Manager handles CRUD operations for watchers configuration
type Manager struct {
config *config.Config
watchersFile string
mu sync.RWMutex
watchers map[string]*WatcherConfig
// Callback to notify poller of changes
onChanged func()
}
// NewManager creates a new watchers manager
func NewManager(cfg *config.Config) (*Manager, error) {
watchersFile := cfg.WatchersFilePath()
m := &Manager{
config: cfg,
watchersFile: watchersFile,
watchers: make(map[string]*WatcherConfig),
}
// Ensure directory exists
dir := filepath.Dir(watchersFile)
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create watchers directory: %w", err)
}
// Load or migrate
if err := m.loadOrMigrate(); err != nil {
return nil, err
}
return m, nil
}
// SetOnChanged sets the callback for watcher changes
func (m *Manager) SetOnChanged(cb func()) {
m.mu.Lock()
defer m.mu.Unlock()
m.onChanged = cb
}
// loadOrMigrate loads existing config or migrates from ENV
func (m *Manager) loadOrMigrate() error {
// Try to load existing file
if _, err := os.Stat(m.watchersFile); err == nil {
return m.load()
}
// File doesn't exist, check for ENV migration
if len(m.config.WatchFolderList) > 0 {
log.Printf("[Watchers] Migrating from WATCH_FOLDER_LIST env var")
return m.migrateFromEnv()
}
// No config, start empty
log.Printf("[Watchers] No existing config, starting with empty watchers list")
return nil
}
// migrateFromEnv creates watchers from WATCH_FOLDER_LIST env var
func (m *Manager) migrateFromEnv() error {
m.mu.Lock()
defer m.mu.Unlock()
for _, path := range m.config.WatchFolderList {
id := uuid.New().String()[:8]
watcher := &WatcherConfig{
ID: id,
Path: path,
IntervalMs: DefaultIntervalMs,
Enabled: true,
CreatedAt: NowMS(),
}
m.watchers[id] = watcher
log.Printf("[Watchers] Migrated watcher: %s -> %s", id, path)
}
// Save migrated config
return m.saveUnsafe()
}
// load loads watchers from file
func (m *Manager) load() error {
m.mu.Lock()
defer m.mu.Unlock()
data, err := os.ReadFile(m.watchersFile)
if err != nil {
return fmt.Errorf("failed to read watchers file: %w", err)
}
var file WatchersFile
if err := json.Unmarshal(data, &file); err != nil {
return fmt.Errorf("failed to parse watchers file: %w", err)
}
m.watchers = make(map[string]*WatcherConfig)
for i := range file.Watchers {
w := file.Watchers[i]
m.watchers[w.ID] = &w
}
log.Printf("[Watchers] Loaded %d watchers from file", len(m.watchers))
return nil
}
// save persists watchers to file (thread-safe)
func (m *Manager) save() error {
m.mu.Lock()
defer m.mu.Unlock()
return m.saveUnsafe()
}
// saveUnsafe persists watchers to file (must hold lock)
func (m *Manager) saveUnsafe() error {
watchers := make([]WatcherConfig, 0, len(m.watchers))
for _, w := range m.watchers {
watchers = append(watchers, *w)
}
file := WatchersFile{
Version: 1,
Watchers: watchers,
}
data, err := json.MarshalIndent(file, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal watchers: %w", err)
}
if err := os.WriteFile(m.watchersFile, data, 0644); err != nil {
return fmt.Errorf("failed to write watchers file: %w", err)
}
return nil
}
// notifyChanged calls the onChanged callback if set
func (m *Manager) notifyChanged() {
m.mu.RLock()
cb := m.onChanged
m.mu.RUnlock()
if cb != nil {
go cb()
}
}
// List returns all watchers
func (m *Manager) List() []WatcherConfig {
m.mu.RLock()
defer m.mu.RUnlock()
result := make([]WatcherConfig, 0, len(m.watchers))
for _, w := range m.watchers {
result = append(result, *w)
}
return result
}
// Get returns a watcher by ID
func (m *Manager) Get(id string) (*WatcherConfig, error) {
m.mu.RLock()
defer m.mu.RUnlock()
w, ok := m.watchers[id]
if !ok {
return nil, fmt.Errorf("watcher not found: %s", id)
}
return &WatcherConfig{
ID: w.ID,
Path: w.Path,
IntervalMs: w.IntervalMs,
Enabled: w.Enabled,
CreatedAt: w.CreatedAt,
}, nil
}
// Create adds a new watcher
func (m *Manager) Create(req CreateWatcherRequest) (*WatcherConfig, error) {
m.mu.Lock()
defer m.mu.Unlock()
// Validate path
if req.Path == "" {
return nil, fmt.Errorf("path is required")
}
// Set defaults
intervalMs := req.IntervalMs
if intervalMs < MinIntervalMs {
intervalMs = DefaultIntervalMs
}
id := uuid.New().String()[:8]
watcher := &WatcherConfig{
ID: id,
Path: req.Path,
IntervalMs: intervalMs,
Enabled: req.Enabled,
CreatedAt: NowMS(),
}
m.watchers[id] = watcher
if err := m.saveUnsafe(); err != nil {
delete(m.watchers, id)
return nil, err
}
log.Printf("[Watchers] Created watcher: %s -> %s", id, req.Path)
// Notify after releasing lock
go m.notifyChanged()
return &WatcherConfig{
ID: watcher.ID,
Path: watcher.Path,
IntervalMs: watcher.IntervalMs,
Enabled: watcher.Enabled,
CreatedAt: watcher.CreatedAt,
}, nil
}
// Update modifies an existing watcher
func (m *Manager) Update(id string, req UpdateWatcherRequest) (*WatcherConfig, error) {
m.mu.Lock()
defer m.mu.Unlock()
w, ok := m.watchers[id]
if !ok {
return nil, fmt.Errorf("watcher not found: %s", id)
}
if req.Path != nil {
if *req.Path == "" {
return nil, fmt.Errorf("path cannot be empty")
}
w.Path = *req.Path
}
if req.IntervalMs != nil {
interval := *req.IntervalMs
if interval < MinIntervalMs {
interval = MinIntervalMs
}
w.IntervalMs = interval
}
if req.Enabled != nil {
w.Enabled = *req.Enabled
}
if err := m.saveUnsafe(); err != nil {
return nil, err
}
log.Printf("[Watchers] Updated watcher: %s", id)
// Notify after releasing lock
go m.notifyChanged()
return &WatcherConfig{
ID: w.ID,
Path: w.Path,
IntervalMs: w.IntervalMs,
Enabled: w.Enabled,
CreatedAt: w.CreatedAt,
}, nil
}
// Delete removes a watcher
func (m *Manager) Delete(id string) error {
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.watchers[id]; !ok {
return fmt.Errorf("watcher not found: %s", id)
}
delete(m.watchers, id)
if err := m.saveUnsafe(); err != nil {
return err
}
log.Printf("[Watchers] Deleted watcher: %s", id)
// Notify after releasing lock
go m.notifyChanged()
return nil
}
// GetFilesPath returns the files path from config
func (m *Manager) GetFilesPath() string {
return m.config.FilesPath
}
+328
View File
@@ -0,0 +1,328 @@
package watchers
import (
"log"
"sync"
"time"
"github.com/metazla/meta-core/internal/watcher"
)
// ScanFunc is the function type for scanning a directory
type ScanFunc func(path string) int
// watcherPoller manages polling for a single watcher
type watcherPoller struct {
watcherID string
stopChan chan struct{}
stopped bool
mu sync.Mutex
}
// RuntimeStatus holds the runtime state for a watcher
type RuntimeStatus struct {
Active bool
LastScan int64
FileCount int
IsScanning bool
}
// Poller manages per-watcher polling goroutines
type Poller struct {
manager *Manager
scanner *watcher.Watcher
dispatcher *watcher.Dispatcher
pollers map[string]*watcherPoller // key: watcher ID
status map[string]*RuntimeStatus
running bool
mu sync.RWMutex
}
// NewPoller creates a new watchers poller
func NewPoller(manager *Manager, scanner *watcher.Watcher, dispatcher *watcher.Dispatcher) *Poller {
p := &Poller{
manager: manager,
scanner: scanner,
dispatcher: dispatcher,
pollers: make(map[string]*watcherPoller),
status: make(map[string]*RuntimeStatus),
}
// Register callback for watcher changes
manager.SetOnChanged(p.syncWithWatchers)
return p
}
// Start begins the polling service
func (p *Poller) Start() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.running {
return nil
}
p.running = true
log.Println("[WatcherPoller] Starting file watcher polling service")
// Initial sync with watcher configurations
go p.syncWithWatchers()
return nil
}
// Stop stops all polling goroutines
func (p *Poller) Stop() error {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
return nil
}
p.running = false
log.Println("[WatcherPoller] Stopping file watcher polling service")
// Stop all pollers
for id, poller := range p.pollers {
p.stopPoller(poller)
delete(p.pollers, id)
}
return nil
}
// SyncWithWatchers synchronizes pollers with current watcher configurations
func (p *Poller) syncWithWatchers() {
p.mu.RLock()
running := p.running
p.mu.RUnlock()
if !running {
return
}
// Get current watchers
watchers := p.manager.List()
p.mu.Lock()
defer p.mu.Unlock()
// Re-check running state after acquiring lock
if !p.running {
return
}
// Track which watchers we've seen
seenWatchers := make(map[string]bool)
for _, w := range watchers {
seenWatchers[w.ID] = true
existingPoller, exists := p.pollers[w.ID]
if w.Enabled {
// Calculate interval
interval := w.IntervalMs
if interval < MinIntervalMs {
interval = DefaultIntervalMs
}
if !exists {
// Start new poller
log.Printf("[WatcherPoller] Starting poller for watcher %s (path: %s, interval: %dms)", w.ID, w.Path, interval)
p.startPollerForWatcher(w.ID, w.Path, interval)
}
} else if exists {
// Stop existing poller
log.Printf("[WatcherPoller] Stopping poller for watcher %s", w.ID)
p.stopPoller(existingPoller)
delete(p.pollers, w.ID)
}
}
// Stop pollers for watchers that no longer exist
for id, poller := range p.pollers {
if !seenWatchers[id] {
log.Printf("[WatcherPoller] Stopping poller for removed watcher %s", id)
p.stopPoller(poller)
delete(p.pollers, id)
}
}
}
// startPollerForWatcher starts a polling goroutine for a watcher
func (p *Poller) startPollerForWatcher(watcherID, path string, intervalMs int) {
poller := &watcherPoller{
watcherID: watcherID,
stopChan: make(chan struct{}),
}
p.pollers[watcherID] = poller
// Initialize status
if _, exists := p.status[watcherID]; !exists {
p.status[watcherID] = &RuntimeStatus{}
}
p.status[watcherID].Active = true
go func() {
// Run initial scan immediately
p.runPollScan(watcherID, path)
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-poller.stopChan:
return
case <-ticker.C:
p.runPollScan(watcherID, path)
}
}
}()
}
// stopPoller stops a watcher poller
func (p *Poller) stopPoller(poller *watcherPoller) {
poller.mu.Lock()
defer poller.mu.Unlock()
if poller.stopped {
return
}
poller.stopped = true
close(poller.stopChan)
// Update status
if status, exists := p.status[poller.watcherID]; exists {
status.Active = false
}
}
// runPollScan executes a scan for a watcher
func (p *Poller) runPollScan(watcherID, path string) {
if p.scanner == nil {
return
}
// Mark as scanning
p.mu.Lock()
if status, exists := p.status[watcherID]; exists {
status.IsScanning = true
}
p.mu.Unlock()
// Perform scan
fileCount := p.scanner.ScanDirectory(path)
// Update status
p.mu.Lock()
if status, exists := p.status[watcherID]; exists {
status.LastScan = NowMS()
status.FileCount = fileCount
status.IsScanning = false
}
p.mu.Unlock()
log.Printf("[WatcherPoller] Poll scan complete for watcher %s: %d files", watcherID, fileCount)
}
// TriggerScan triggers an immediate scan for a watcher
func (p *Poller) TriggerScan(watcherID string) (int, error) {
w, err := p.manager.Get(watcherID)
if err != nil {
return 0, err
}
if p.scanner == nil {
return 0, nil
}
// Mark as scanning
p.mu.Lock()
if status, exists := p.status[watcherID]; exists {
status.IsScanning = true
} else {
p.status[watcherID] = &RuntimeStatus{IsScanning: true}
}
p.mu.Unlock()
// Perform scan
fileCount := p.scanner.ScanDirectory(w.Path)
// Update status
p.mu.Lock()
if status, exists := p.status[watcherID]; exists {
status.LastScan = NowMS()
status.FileCount = fileCount
status.IsScanning = false
}
p.mu.Unlock()
return fileCount, nil
}
// TriggerScanAll triggers scans for all enabled watchers
func (p *Poller) TriggerScanAll() (int, error) {
watchers := p.manager.List()
totalCount := 0
for _, w := range watchers {
if w.Enabled {
count, err := p.TriggerScan(w.ID)
if err != nil {
log.Printf("[WatcherPoller] Failed to scan watcher %s: %v", w.ID, err)
continue
}
totalCount += count
}
}
return totalCount, nil
}
// GetStatus returns the runtime status for a watcher
func (p *Poller) GetStatus(watcherID string) *RuntimeStatus {
p.mu.RLock()
defer p.mu.RUnlock()
if status, exists := p.status[watcherID]; exists {
return &RuntimeStatus{
Active: status.Active,
LastScan: status.LastScan,
FileCount: status.FileCount,
IsScanning: status.IsScanning,
}
}
return &RuntimeStatus{}
}
// GetWatcherStatuses returns all watchers with their runtime status
func (p *Poller) GetWatcherStatuses() []WatcherStatus {
watchers := p.manager.List()
result := make([]WatcherStatus, 0, len(watchers))
p.mu.RLock()
defer p.mu.RUnlock()
for _, w := range watchers {
status := WatcherStatus{
WatcherConfig: w,
}
if rs, exists := p.status[w.ID]; exists {
status.Active = rs.Active
status.LastScan = rs.LastScan
status.FileCount = rs.FileCount
status.IsScanning = rs.IsScanning
}
result = append(result, status)
}
return result
}
+59
View File
@@ -0,0 +1,59 @@
package watchers
import "time"
const (
// MinIntervalMs is the minimum polling interval (5 seconds)
MinIntervalMs = 5000
// DefaultIntervalMs is the default polling interval (30 seconds)
DefaultIntervalMs = 30000
)
// WatcherConfig represents a watcher configuration stored in watchers.json
type WatcherConfig struct {
ID string `json:"id"`
Path string `json:"path"` // e.g., "/files/watch", "/files/movies"
IntervalMs int `json:"intervalMs"` // Min: 5000 (5s), Default: 30000 (30s)
Enabled bool `json:"enabled"`
CreatedAt int64 `json:"createdAt"`
}
// WatcherStatus includes runtime status in API responses
type WatcherStatus struct {
WatcherConfig
Active bool `json:"active"` // Polling goroutine running
LastScan int64 `json:"lastScan"` // Unix ms timestamp
FileCount int `json:"fileCount"` // Files found in last scan
IsScanning bool `json:"isScanning"` // Currently scanning
}
// WatchersFile is the structure of watchers.json
type WatchersFile struct {
Version int `json:"version"`
Watchers []WatcherConfig `json:"watchers"`
}
// CreateWatcherRequest is the request body for creating a watcher
type CreateWatcherRequest struct {
Path string `json:"path"`
IntervalMs int `json:"intervalMs"`
Enabled bool `json:"enabled"`
}
// UpdateWatcherRequest is the request body for updating a watcher
type UpdateWatcherRequest struct {
Path *string `json:"path,omitempty"`
IntervalMs *int `json:"intervalMs,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
}
// WatchersListResponse is the API response for listing watchers
type WatchersListResponse struct {
Watchers []WatcherStatus `json:"watchers"`
Count int `json:"count"`
}
// NowMS returns current time in milliseconds
func NowMS() int64 {
return time.Now().UnixMilli()
}