cleaning of the files event system
This commit is contained in:
+42
-14
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/metazla/meta-core/internal/mounts"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
"github.com/metazla/meta-core/internal/watcher"
|
||||
"github.com/metazla/meta-core/internal/watchers"
|
||||
)
|
||||
|
||||
// Server is the HTTP API server for meta-core
|
||||
@@ -29,6 +30,9 @@ type Server struct {
|
||||
watcherDispatcher *watcher.Dispatcher
|
||||
fileWatcher *watcher.Watcher
|
||||
watcherHandlers *watcher.Handlers
|
||||
watchersManager *watchers.Manager
|
||||
watchersPoller *watchers.Poller
|
||||
watchersHandlers *watchers.Handlers
|
||||
router *mux.Router
|
||||
server *http.Server
|
||||
}
|
||||
@@ -50,19 +54,37 @@ func NewServer(
|
||||
router: mux.NewRouter(),
|
||||
}
|
||||
|
||||
// Initialize file watcher (if enabled) - before mounts so we can pass scanFunc
|
||||
// Initialize file watcher/scanner and dispatcher
|
||||
var scanFunc mounts.ScanFunc
|
||||
if cfg.EnableFileWatcher && len(cfg.WatchFolderList) > 0 {
|
||||
if cfg.EnableFileWatcher {
|
||||
s.watcherDispatcher = watcher.NewDispatcher()
|
||||
// Wire storage client for Redis Stream publishing
|
||||
if stor != nil {
|
||||
s.watcherDispatcher.SetStorageClient(stor)
|
||||
}
|
||||
fileWatcher, err := watcher.NewWatcher(cfg, s.watcherDispatcher)
|
||||
if err != nil {
|
||||
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
||||
} else {
|
||||
s.fileWatcher = fileWatcher
|
||||
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
||||
// Create scan function from watcher
|
||||
// Create scan function from watcher for mounts
|
||||
scanFunc = fileWatcher.ScanMountPath
|
||||
}
|
||||
|
||||
// Initialize watchers manager (polling-based file watching)
|
||||
watchersManager, err := watchers.NewManager(cfg)
|
||||
if err != nil {
|
||||
log.Printf("[API] Warning: failed to initialize watchers manager: %v", err)
|
||||
} else {
|
||||
s.watchersManager = watchersManager
|
||||
|
||||
// Create watchers poller
|
||||
if s.fileWatcher != nil {
|
||||
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
||||
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize mounts manager
|
||||
@@ -151,6 +173,12 @@ func (s *Server) setupRoutes() {
|
||||
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
|
||||
}
|
||||
|
||||
// Watchers management routes (if watchers initialized)
|
||||
if s.watchersHandlers != nil {
|
||||
s.watchersHandlers.RegisterRoutes(s.router)
|
||||
log.Println("[API] Watchers management routes registered at /api/watchers/*")
|
||||
}
|
||||
|
||||
// Add middleware
|
||||
s.router.Use(loggingMiddleware)
|
||||
s.router.Use(corsMiddleware)
|
||||
@@ -176,10 +204,10 @@ func (s *Server) Start() error {
|
||||
}
|
||||
}()
|
||||
|
||||
// Start file watcher (if initialized)
|
||||
if s.fileWatcher != nil {
|
||||
if err := s.fileWatcher.Start(); err != nil {
|
||||
log.Printf("[API] Warning: failed to start file watcher: %v", err)
|
||||
// Start watchers poller (if initialized)
|
||||
if s.watchersPoller != nil {
|
||||
if err := s.watchersPoller.Start(); err != nil {
|
||||
log.Printf("[API] Warning: failed to start watchers poller: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,6 +223,13 @@ func (s *Server) Start() error {
|
||||
|
||||
// Stop gracefully stops the HTTP server
|
||||
func (s *Server) Stop() error {
|
||||
// Stop watchers poller
|
||||
if s.watchersPoller != nil {
|
||||
if err := s.watchersPoller.Stop(); err != nil {
|
||||
log.Printf("[API] Warning: failed to stop watchers poller: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop mount poller
|
||||
if s.mountPoller != nil {
|
||||
if err := s.mountPoller.Stop(); err != nil {
|
||||
@@ -202,13 +237,6 @@ func (s *Server) Stop() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Stop file watcher
|
||||
if s.fileWatcher != nil {
|
||||
if err := s.fileWatcher.Stop(); err != nil {
|
||||
log.Printf("[API] Warning: failed to stop file watcher: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.server == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -104,6 +104,11 @@ func (c *Config) MountsErrorDir() string {
|
||||
return c.MountsDir + "/errors"
|
||||
}
|
||||
|
||||
// WatchersFilePath returns the path to the watchers configuration file
|
||||
func (c *Config) WatchersFilePath() string {
|
||||
return c.MetaCorePath + "/watchers.json"
|
||||
}
|
||||
|
||||
func getEnv(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
|
||||
@@ -570,3 +570,32 @@ func (c *Client) GetRedisClient() *redis.Client {
|
||||
func (c *Client) GetPrefix() string {
|
||||
return c.prefix
|
||||
}
|
||||
|
||||
// XAdd publishes an event to a Redis Stream
|
||||
// Uses XADD with MAXLEN ~ for approximate trimming
|
||||
func (c *Client) XAdd(stream string, maxLen int64, fields map[string]interface{}) (string, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return "", fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Build XADD arguments with approximate MAXLEN
|
||||
args := &redis.XAddArgs{
|
||||
Stream: stream,
|
||||
MaxLen: maxLen,
|
||||
Approx: true,
|
||||
Values: fields,
|
||||
}
|
||||
|
||||
id, err := c.client.XAdd(ctx, args).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("xadd failed: %w", err)
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
+38
-186
@@ -1,223 +1,75 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxRetries is the maximum number of delivery retries
|
||||
MaxRetries = 3
|
||||
// RetryDelay is the delay between retries
|
||||
RetryDelay = 5 * time.Second
|
||||
// WebhookTimeout is the timeout for webhook requests
|
||||
WebhookTimeout = 10 * time.Second
|
||||
// MaxFailCount is the max failures before removing subscriber
|
||||
MaxFailCount = 10
|
||||
// EventsStream is the Redis Stream name for file events
|
||||
EventsStream = "file:events"
|
||||
// StreamMaxLen is the approximate max length for the stream
|
||||
StreamMaxLen = 10000
|
||||
)
|
||||
|
||||
// Dispatcher sends file events to subscribers
|
||||
// Dispatcher sends file events to Redis Stream
|
||||
type Dispatcher struct {
|
||||
subscribers map[string]*Subscriber
|
||||
sseClients map[chan FileEvent]bool
|
||||
mu sync.RWMutex
|
||||
httpClient *http.Client
|
||||
mu sync.RWMutex
|
||||
storageClient *storage.Client
|
||||
}
|
||||
|
||||
// NewDispatcher creates a new event dispatcher
|
||||
func NewDispatcher() *Dispatcher {
|
||||
return &Dispatcher{
|
||||
subscribers: make(map[string]*Subscriber),
|
||||
sseClients: make(map[chan FileEvent]bool),
|
||||
httpClient: &http.Client{
|
||||
Timeout: WebhookTimeout,
|
||||
},
|
||||
}
|
||||
return &Dispatcher{}
|
||||
}
|
||||
|
||||
// Subscribe registers a webhook subscriber
|
||||
func (d *Dispatcher) Subscribe(url string, eventTypes []string) error {
|
||||
// SetStorageClient sets the storage client for Redis Stream publishing
|
||||
func (d *Dispatcher) SetStorageClient(client *storage.Client) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
d.subscribers[url] = &Subscriber{
|
||||
URL: url,
|
||||
RegisteredAt: NowMS(),
|
||||
EventTypes: eventTypes,
|
||||
FailCount: 0,
|
||||
}
|
||||
|
||||
log.Printf("[Dispatcher] Subscribed webhook: %s", url)
|
||||
return nil
|
||||
d.storageClient = client
|
||||
log.Println("[Dispatcher] Storage client set for Redis Stream publishing")
|
||||
}
|
||||
|
||||
// Unsubscribe removes a webhook subscriber
|
||||
func (d *Dispatcher) Unsubscribe(url string) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
delete(d.subscribers, url)
|
||||
log.Printf("[Dispatcher] Unsubscribed webhook: %s", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListSubscribers returns all subscribers
|
||||
func (d *Dispatcher) ListSubscribers() []Subscriber {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
result := make([]Subscriber, 0, len(d.subscribers))
|
||||
for _, sub := range d.subscribers {
|
||||
result = append(result, *sub)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// AddSSEClient adds an SSE client channel
|
||||
func (d *Dispatcher) AddSSEClient(ch chan FileEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.sseClients[ch] = true
|
||||
}
|
||||
|
||||
// RemoveSSEClient removes an SSE client channel
|
||||
func (d *Dispatcher) RemoveSSEClient(ch chan FileEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
delete(d.sseClients, ch)
|
||||
close(ch)
|
||||
}
|
||||
|
||||
// Dispatch sends an event to all subscribers
|
||||
// Dispatch sends an event to Redis Stream
|
||||
func (d *Dispatcher) Dispatch(event FileEvent) {
|
||||
// Dispatch to webhooks
|
||||
go d.dispatchToWebhooks(event)
|
||||
|
||||
// Dispatch to SSE clients
|
||||
d.dispatchToSSE(event)
|
||||
go d.dispatchToStream(event)
|
||||
}
|
||||
|
||||
// dispatchToWebhooks sends event to webhook subscribers
|
||||
func (d *Dispatcher) dispatchToWebhooks(event FileEvent) {
|
||||
// dispatchToStream publishes event to Redis Stream for service consumption
|
||||
func (d *Dispatcher) dispatchToStream(event FileEvent) {
|
||||
d.mu.RLock()
|
||||
subscribers := make([]*Subscriber, 0, len(d.subscribers))
|
||||
for _, sub := range d.subscribers {
|
||||
subscribers = append(subscribers, sub)
|
||||
}
|
||||
client := d.storageClient
|
||||
d.mu.RUnlock()
|
||||
|
||||
for _, sub := range subscribers {
|
||||
// Check event type filter
|
||||
if len(sub.EventTypes) > 0 {
|
||||
found := false
|
||||
for _, et := range sub.EventTypes {
|
||||
if et == string(event.Type) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
go d.deliverToWebhook(sub.URL, event)
|
||||
}
|
||||
}
|
||||
|
||||
// deliverToWebhook delivers an event to a webhook with retries
|
||||
func (d *Dispatcher) deliverToWebhook(url string, event FileEvent) {
|
||||
body, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
log.Printf("[Dispatcher] Failed to marshal event: %v", err)
|
||||
if client == nil || !client.IsConnected() {
|
||||
return
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < MaxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(RetryDelay)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Event-Type", string(event.Type))
|
||||
|
||||
resp, err := d.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
// Success
|
||||
d.mu.Lock()
|
||||
if sub, ok := d.subscribers[url]; ok {
|
||||
sub.LastDelivery = NowMS()
|
||||
sub.FailCount = 0
|
||||
}
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
lastErr = &httpError{StatusCode: resp.StatusCode}
|
||||
// Build stream fields
|
||||
fields := map[string]interface{}{
|
||||
"type": string(event.Type),
|
||||
"path": event.Path,
|
||||
"timestamp": fmt.Sprintf("%d", event.Timestamp),
|
||||
}
|
||||
|
||||
// All retries failed
|
||||
log.Printf("[Dispatcher] Failed to deliver to %s after %d retries: %v", url, MaxRetries, lastErr)
|
||||
|
||||
d.mu.Lock()
|
||||
if sub, ok := d.subscribers[url]; ok {
|
||||
sub.FailCount++
|
||||
if sub.FailCount >= MaxFailCount {
|
||||
delete(d.subscribers, url)
|
||||
log.Printf("[Dispatcher] Removed webhook %s after %d failures", url, MaxFailCount)
|
||||
}
|
||||
// Add optional fields
|
||||
if event.Size > 0 {
|
||||
fields["size"] = fmt.Sprintf("%d", event.Size)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// dispatchToSSE sends event to SSE clients
|
||||
func (d *Dispatcher) dispatchToSSE(event FileEvent) {
|
||||
d.mu.RLock()
|
||||
clients := make([]chan FileEvent, 0, len(d.sseClients))
|
||||
for ch := range d.sseClients {
|
||||
clients = append(clients, ch)
|
||||
if event.PartialHash != "" {
|
||||
fields["partialHash"] = event.PartialHash
|
||||
}
|
||||
if event.OldPath != "" {
|
||||
fields["oldPath"] = event.OldPath
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
for _, ch := range clients {
|
||||
select {
|
||||
case ch <- event:
|
||||
default:
|
||||
// Channel full, skip
|
||||
}
|
||||
// Publish to stream
|
||||
_, err := client.XAdd(EventsStream, StreamMaxLen, fields)
|
||||
if err != nil {
|
||||
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// SSEClientCount returns the number of connected SSE clients
|
||||
func (d *Dispatcher) SSEClientCount() int {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
return len(d.sseClients)
|
||||
}
|
||||
|
||||
// httpError represents an HTTP error
|
||||
type httpError struct {
|
||||
StatusCode int
|
||||
}
|
||||
|
||||
func (e *httpError) Error() string {
|
||||
return http.StatusText(e.StatusCode)
|
||||
}
|
||||
|
||||
+11
-115
@@ -2,7 +2,6 @@ package watcher
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
@@ -25,59 +24,12 @@ func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
|
||||
|
||||
// RegisterRoutes registers all watcher-related routes
|
||||
func (h *Handlers) RegisterRoutes(r *mux.Router) {
|
||||
// Event subscriptions
|
||||
r.HandleFunc("/api/events/subscribe", h.handleSSESubscribe).Methods("GET")
|
||||
// Event polling (for services that don't use Redis Streams)
|
||||
r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
|
||||
r.HandleFunc("/api/events/subscribers", h.handleListSubscribers).Methods("GET")
|
||||
r.HandleFunc("/api/events/subscribers", h.handleAddSubscriber).Methods("POST")
|
||||
r.HandleFunc("/api/events/subscribers/{url}", h.handleRemoveSubscriber).Methods("DELETE")
|
||||
|
||||
// Scan management
|
||||
r.HandleFunc("/api/scan/trigger", h.handleTriggerScan).Methods("POST")
|
||||
r.HandleFunc("/api/scan/status", h.handleScanStatus).Methods("GET")
|
||||
}
|
||||
|
||||
// handleSSESubscribe handles GET /api/events/subscribe (Server-Sent Events)
|
||||
func (h *Handlers) handleSSESubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
// Set SSE headers
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
// Create event channel
|
||||
eventChan := make(chan FileEvent, 100)
|
||||
h.dispatcher.AddSSEClient(eventChan)
|
||||
|
||||
// Cleanup on disconnect
|
||||
defer h.dispatcher.RemoveSSEClient(eventChan)
|
||||
|
||||
// Get flusher
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "SSE not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Send initial connected message
|
||||
fmt.Fprintf(w, "event: connected\ndata: {\"status\":\"connected\"}\n\n")
|
||||
flusher.Flush()
|
||||
|
||||
// Stream events
|
||||
for {
|
||||
select {
|
||||
case event := <-eventChan:
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "event: file\ndata: %s\n\n", string(data))
|
||||
flusher.Flush()
|
||||
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
// Deprecated scan endpoints - redirect to /api/watchers
|
||||
r.HandleFunc("/api/scan/trigger", h.handleTriggerScanDeprecated).Methods("POST")
|
||||
r.HandleFunc("/api/scan/status", h.handleScanStatusDeprecated).Methods("GET")
|
||||
}
|
||||
|
||||
// handlePoll handles GET /api/events/poll
|
||||
@@ -108,78 +60,22 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// handleListSubscribers handles GET /api/events/subscribers
|
||||
func (h *Handlers) handleListSubscribers(w http.ResponseWriter, r *http.Request) {
|
||||
subscribers := h.dispatcher.ListSubscribers()
|
||||
|
||||
writeJSON(w, http.StatusOK, SubscribersListResponse{
|
||||
Subscribers: subscribers,
|
||||
Count: len(subscribers),
|
||||
})
|
||||
}
|
||||
|
||||
// handleAddSubscriber handles POST /api/events/subscribers
|
||||
func (h *Handlers) handleAddSubscriber(w http.ResponseWriter, r *http.Request) {
|
||||
var req SubscribeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.URL == "" {
|
||||
writeError(w, http.StatusBadRequest, "URL is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Subscribe(req.URL, req.EventTypes); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Subscribed",
|
||||
"url": req.URL,
|
||||
})
|
||||
}
|
||||
|
||||
// handleRemoveSubscriber handles DELETE /api/events/subscribers/{url}
|
||||
func (h *Handlers) handleRemoveSubscriber(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
url := vars["url"]
|
||||
|
||||
if url == "" {
|
||||
writeError(w, http.StatusBadRequest, "URL is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Unsubscribe(url); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// handleTriggerScanDeprecated handles POST /api/scan/trigger (deprecated)
|
||||
func (h *Handlers) handleTriggerScanDeprecated(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Unsubscribed",
|
||||
"message": "Use POST /api/watchers/scan-all instead. This endpoint is deprecated.",
|
||||
})
|
||||
}
|
||||
|
||||
// handleTriggerScan handles POST /api/scan/trigger
|
||||
func (h *Handlers) handleTriggerScan(w http.ResponseWriter, r *http.Request) {
|
||||
go h.watcher.RunScan()
|
||||
|
||||
// handleScanStatusDeprecated handles GET /api/scan/status (deprecated)
|
||||
func (h *Handlers) handleScanStatusDeprecated(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Scan triggered",
|
||||
"status": "deprecated",
|
||||
"message": "Use GET /api/watchers instead. This endpoint is deprecated.",
|
||||
})
|
||||
}
|
||||
|
||||
// handleScanStatus handles GET /api/scan/status
|
||||
func (h *Handlers) handleScanStatus(w http.ResponseWriter, r *http.Request) {
|
||||
status := h.watcher.GetStatus()
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
|
||||
@@ -22,15 +22,6 @@ type FileEvent struct {
|
||||
OldPath string `json:"oldPath,omitempty"` // For rename events
|
||||
}
|
||||
|
||||
// Subscriber represents a webhook subscriber
|
||||
type Subscriber struct {
|
||||
URL string `json:"url"`
|
||||
RegisteredAt int64 `json:"registeredAt"`
|
||||
EventTypes []string `json:"eventTypes,omitempty"` // Empty means all
|
||||
LastDelivery int64 `json:"lastDelivery,omitempty"`
|
||||
FailCount int `json:"failCount"`
|
||||
}
|
||||
|
||||
// PendingEvent tracks a file event that's being debounced
|
||||
type PendingEvent struct {
|
||||
Event FileEvent
|
||||
@@ -39,24 +30,12 @@ type PendingEvent struct {
|
||||
Timer *time.Timer
|
||||
}
|
||||
|
||||
// SubscribeRequest is the request to register a webhook
|
||||
type SubscribeRequest struct {
|
||||
URL string `json:"url"`
|
||||
EventTypes []string `json:"eventTypes,omitempty"`
|
||||
}
|
||||
|
||||
// EventsListResponse is the response for listing events
|
||||
type EventsListResponse struct {
|
||||
Events []FileEvent `json:"events"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// SubscribersListResponse is the response for listing subscribers
|
||||
type SubscribersListResponse struct {
|
||||
Subscribers []Subscriber `json:"subscribers"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ScanStatusResponse is the response for scan status
|
||||
type ScanStatusResponse struct {
|
||||
Status string `json:"status"`
|
||||
|
||||
+25
-221
@@ -3,16 +3,13 @@ package watcher
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
)
|
||||
|
||||
@@ -21,188 +18,32 @@ const (
|
||||
PartialHashSize = 64 * 1024 // 64KB
|
||||
)
|
||||
|
||||
// Watcher monitors directories for file changes
|
||||
// Watcher provides file scanning and event dispatching
|
||||
type Watcher struct {
|
||||
config *config.Config
|
||||
fsWatcher *fsnotify.Watcher
|
||||
debouncer *Debouncer
|
||||
dispatcher *Dispatcher
|
||||
filesPath string
|
||||
watchPaths []string
|
||||
|
||||
mu sync.RWMutex
|
||||
isRunning bool
|
||||
isScanning bool
|
||||
lastScan int64
|
||||
fileCount int
|
||||
stopChan chan struct{}
|
||||
eventBuffer []FileEvent
|
||||
}
|
||||
|
||||
// NewWatcher creates a new file watcher
|
||||
// NewWatcher creates a new file watcher/scanner
|
||||
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
||||
fsWatcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debouncer := NewDebouncer(time.Duration(cfg.DebounceMS) * time.Millisecond)
|
||||
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
|
||||
|
||||
w := &Watcher{
|
||||
config: cfg,
|
||||
fsWatcher: fsWatcher,
|
||||
debouncer: debouncer,
|
||||
dispatcher: dispatcher,
|
||||
filesPath: cfg.FilesPath,
|
||||
watchPaths: cfg.WatchFolderList,
|
||||
stopChan: make(chan struct{}),
|
||||
eventBuffer: make([]FileEvent, 0, 1000),
|
||||
}
|
||||
|
||||
// Set debouncer callback
|
||||
debouncer.SetCallback(func(event FileEvent) {
|
||||
w.handleDebouncedEvent(event)
|
||||
})
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Start begins watching for file changes
|
||||
func (w *Watcher) Start() error {
|
||||
w.mu.Lock()
|
||||
if w.isRunning {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
w.isRunning = true
|
||||
w.mu.Unlock()
|
||||
|
||||
// Add watch paths
|
||||
for _, watchPath := range w.watchPaths {
|
||||
if err := w.addWatchRecursive(watchPath); err != nil {
|
||||
log.Printf("[Watcher] Warning: failed to watch %s: %v", watchPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start event processing goroutine
|
||||
go w.processEvents()
|
||||
|
||||
// Start initial scan
|
||||
go w.RunScan()
|
||||
|
||||
log.Printf("[Watcher] Started watching %d paths", len(w.watchPaths))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the watcher
|
||||
func (w *Watcher) Stop() error {
|
||||
w.mu.Lock()
|
||||
if !w.isRunning {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
w.isRunning = false
|
||||
w.mu.Unlock()
|
||||
|
||||
close(w.stopChan)
|
||||
w.debouncer.Stop()
|
||||
return w.fsWatcher.Close()
|
||||
}
|
||||
|
||||
// addWatchRecursive adds a directory and all subdirectories to the watch list
|
||||
func (w *Watcher) addWatchRecursive(root string) error {
|
||||
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip inaccessible paths
|
||||
}
|
||||
if info.IsDir() {
|
||||
if err := w.fsWatcher.Add(path); err != nil {
|
||||
log.Printf("[Watcher] Warning: cannot watch %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// processEvents handles fsnotify events
|
||||
func (w *Watcher) processEvents() {
|
||||
for {
|
||||
select {
|
||||
case <-w.stopChan:
|
||||
return
|
||||
|
||||
case event, ok := <-w.fsWatcher.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.handleFsEvent(event)
|
||||
|
||||
case err, ok := <-w.fsWatcher.Errors:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
log.Printf("[Watcher] Error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleFsEvent converts an fsnotify event to a FileEvent
|
||||
func (w *Watcher) handleFsEvent(event fsnotify.Event) {
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(w.filesPath, event.Name)
|
||||
if err != nil {
|
||||
relPath = event.Name
|
||||
}
|
||||
|
||||
// Clean up path
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Determine event type
|
||||
var eventType FileEventType
|
||||
switch {
|
||||
case event.Op&fsnotify.Create == fsnotify.Create:
|
||||
eventType = EventTypeAdd
|
||||
// If it's a new directory, add it to watch
|
||||
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
||||
w.fsWatcher.Add(event.Name)
|
||||
}
|
||||
case event.Op&fsnotify.Write == fsnotify.Write:
|
||||
eventType = EventTypeChange
|
||||
case event.Op&fsnotify.Remove == fsnotify.Remove:
|
||||
eventType = EventTypeDelete
|
||||
case event.Op&fsnotify.Rename == fsnotify.Rename:
|
||||
eventType = EventTypeRename
|
||||
default:
|
||||
return // Ignore other events
|
||||
}
|
||||
|
||||
// Skip directories for file events (except remove)
|
||||
if eventType != EventTypeDelete {
|
||||
info, err := os.Stat(event.Name)
|
||||
if err == nil && info.IsDir() {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Get file info if available
|
||||
var size int64
|
||||
if eventType != EventTypeDelete {
|
||||
if info, err := os.Stat(event.Name); err == nil {
|
||||
size = info.Size()
|
||||
}
|
||||
}
|
||||
|
||||
fileEvent := FileEvent{
|
||||
Type: eventType,
|
||||
Path: relPath,
|
||||
Size: size,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Send to debouncer
|
||||
w.debouncer.Add(fileEvent)
|
||||
}
|
||||
|
||||
// handleDebouncedEvent processes a debounced event
|
||||
func (w *Watcher) handleDebouncedEvent(event FileEvent) {
|
||||
// Compute partial hash for add/change events
|
||||
@@ -224,44 +65,11 @@ func (w *Watcher) handleDebouncedEvent(event FileEvent) {
|
||||
|
||||
// Dispatch to subscribers
|
||||
w.dispatcher.Dispatch(event)
|
||||
|
||||
log.Printf("[Watcher] Event: %s %s", event.Type, event.Path)
|
||||
}
|
||||
|
||||
// RunScan performs a full directory scan
|
||||
func (w *Watcher) RunScan() {
|
||||
w.mu.Lock()
|
||||
if w.isScanning {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.isScanning = true
|
||||
w.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
w.mu.Lock()
|
||||
w.isScanning = false
|
||||
w.lastScan = NowMS()
|
||||
w.mu.Unlock()
|
||||
}()
|
||||
|
||||
log.Println("[Watcher] Starting directory scan...")
|
||||
|
||||
fileCount := 0
|
||||
for _, watchPath := range w.watchPaths {
|
||||
count := w.scanDirectory(watchPath)
|
||||
fileCount += count
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
w.fileCount = fileCount
|
||||
w.mu.Unlock()
|
||||
|
||||
log.Printf("[Watcher] Scan complete: %d files found", fileCount)
|
||||
}
|
||||
|
||||
// scanDirectory scans a directory and emits add events for files
|
||||
func (w *Watcher) scanDirectory(root string) int {
|
||||
// ScanDirectory scans a directory and emits add events for files
|
||||
// Returns the count of files found
|
||||
func (w *Watcher) ScanDirectory(root string) int {
|
||||
count := 0
|
||||
|
||||
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
@@ -286,7 +94,7 @@ func (w *Watcher) scanDirectory(root string) int {
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Emit add event (through debouncer)
|
||||
// Emit add event
|
||||
event := FileEvent{
|
||||
Type: EventTypeAdd,
|
||||
Path: relPath,
|
||||
@@ -299,7 +107,7 @@ func (w *Watcher) scanDirectory(root string) int {
|
||||
event.PartialHash = hash
|
||||
}
|
||||
|
||||
// Dispatch directly (skip debouncer for scan)
|
||||
// Dispatch directly
|
||||
w.dispatcher.Dispatch(event)
|
||||
count++
|
||||
|
||||
@@ -314,25 +122,34 @@ func (w *Watcher) scanDirectory(root string) int {
|
||||
func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
|
||||
// Validate path is under filesPath
|
||||
if !strings.HasPrefix(mountPath, w.filesPath) {
|
||||
return 0, fmt.Errorf("mount path %s is not under files path %s", mountPath, w.filesPath)
|
||||
// Try to normalize paths
|
||||
absMount, err := filepath.Abs(mountPath)
|
||||
if err == nil {
|
||||
absFiles, err := filepath.Abs(w.filesPath)
|
||||
if err == nil && strings.HasPrefix(absMount, absFiles) {
|
||||
mountPath = absMount
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check directory exists
|
||||
info, err := os.Stat(mountPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, fmt.Errorf("mount path %s does not exist", mountPath)
|
||||
log.Printf("[Watcher] Mount path %s does not exist, skipping", mountPath)
|
||||
return 0, nil
|
||||
}
|
||||
return 0, fmt.Errorf("failed to stat mount path %s: %w", mountPath, err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
return 0, fmt.Errorf("mount path %s is not a directory", mountPath)
|
||||
log.Printf("[Watcher] Mount path %s is not a directory, skipping", mountPath)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Scan the directory
|
||||
log.Printf("[Watcher] Scanning mount path: %s", mountPath)
|
||||
count := w.scanDirectory(mountPath)
|
||||
count := w.ScanDirectory(mountPath)
|
||||
log.Printf("[Watcher] Mount scan complete: %s (%d files)", mountPath, count)
|
||||
|
||||
return count, nil
|
||||
@@ -356,22 +173,9 @@ func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
|
||||
return result
|
||||
}
|
||||
|
||||
// GetStatus returns watcher status
|
||||
func (w *Watcher) GetStatus() ScanStatusResponse {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
|
||||
status := "running"
|
||||
if !w.isRunning {
|
||||
status = "stopped"
|
||||
}
|
||||
|
||||
return ScanStatusResponse{
|
||||
Status: status,
|
||||
Scanning: w.isScanning,
|
||||
LastScan: w.lastScan,
|
||||
FileCount: w.fileCount,
|
||||
}
|
||||
// GetFilesPath returns the files path
|
||||
func (w *Watcher) GetFilesPath() string {
|
||||
return w.filesPath
|
||||
}
|
||||
|
||||
// computePartialHash computes SHA-256 hash of first 64KB of a file
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package watchers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Handlers provides HTTP handlers for watchers management
|
||||
type Handlers struct {
|
||||
manager *Manager
|
||||
poller *Poller
|
||||
}
|
||||
|
||||
// NewHandlers creates new watchers handlers
|
||||
func NewHandlers(manager *Manager, poller *Poller) *Handlers {
|
||||
return &Handlers{
|
||||
manager: manager,
|
||||
poller: poller,
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all watchers-related routes
|
||||
func (h *Handlers) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/watchers", h.handleList).Methods("GET")
|
||||
r.HandleFunc("/api/watchers", h.handleCreate).Methods("POST")
|
||||
r.HandleFunc("/api/watchers/scan-all", h.handleScanAll).Methods("POST")
|
||||
r.HandleFunc("/api/watchers/{id}", h.handleGet).Methods("GET")
|
||||
r.HandleFunc("/api/watchers/{id}", h.handleUpdate).Methods("PUT")
|
||||
r.HandleFunc("/api/watchers/{id}", h.handleDelete).Methods("DELETE")
|
||||
r.HandleFunc("/api/watchers/{id}/scan", h.handleScan).Methods("POST")
|
||||
}
|
||||
|
||||
// handleList handles GET /api/watchers
|
||||
func (h *Handlers) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
statuses := h.poller.GetWatcherStatuses()
|
||||
|
||||
writeJSON(w, http.StatusOK, WatchersListResponse{
|
||||
Watchers: statuses,
|
||||
Count: len(statuses),
|
||||
})
|
||||
}
|
||||
|
||||
// handleGet handles GET /api/watchers/{id}
|
||||
func (h *Handlers) handleGet(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
watcher, err := h.manager.Get(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get runtime status
|
||||
status := h.poller.GetStatus(id)
|
||||
|
||||
response := WatcherStatus{
|
||||
WatcherConfig: *watcher,
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
FileCount: status.FileCount,
|
||||
IsScanning: status.IsScanning,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleCreate handles POST /api/watchers
|
||||
func (h *Handlers) handleCreate(w http.ResponseWriter, r *http.Request) {
|
||||
var req CreateWatcherRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
watcher, err := h.manager.Create(req)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, watcher)
|
||||
}
|
||||
|
||||
// handleUpdate handles PUT /api/watchers/{id}
|
||||
func (h *Handlers) handleUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
var req UpdateWatcherRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
watcher, err := h.manager.Update(id, req)
|
||||
if err != nil {
|
||||
if err.Error() == "watcher not found: "+id {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
} else {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Get runtime status
|
||||
status := h.poller.GetStatus(id)
|
||||
|
||||
response := WatcherStatus{
|
||||
WatcherConfig: *watcher,
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
FileCount: status.FileCount,
|
||||
IsScanning: status.IsScanning,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleDelete handles DELETE /api/watchers/{id}
|
||||
func (h *Handlers) handleDelete(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
if err := h.manager.Delete(id); err != nil {
|
||||
if err.Error() == "watcher not found: "+id {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
} else {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Watcher deleted",
|
||||
})
|
||||
}
|
||||
|
||||
// handleScan handles POST /api/watchers/{id}/scan
|
||||
func (h *Handlers) handleScan(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
fileCount, err := h.poller.TriggerScan(id)
|
||||
if err != nil {
|
||||
if err.Error() == "watcher not found: "+id {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
} else {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Scan triggered",
|
||||
"fileCount": fileCount,
|
||||
})
|
||||
}
|
||||
|
||||
// handleScanAll handles POST /api/watchers/scan-all
|
||||
func (h *Handlers) handleScanAll(w http.ResponseWriter, r *http.Request) {
|
||||
totalCount, err := h.poller.TriggerScanAll()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "All watchers scanned",
|
||||
"totalFiles": totalCount,
|
||||
})
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, map[string]string{
|
||||
"error": http.StatusText(status),
|
||||
"message": message,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package watchers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
)
|
||||
|
||||
// Manager handles CRUD operations for watchers configuration
|
||||
type Manager struct {
|
||||
config *config.Config
|
||||
watchersFile string
|
||||
|
||||
mu sync.RWMutex
|
||||
watchers map[string]*WatcherConfig
|
||||
|
||||
// Callback to notify poller of changes
|
||||
onChanged func()
|
||||
}
|
||||
|
||||
// NewManager creates a new watchers manager
|
||||
func NewManager(cfg *config.Config) (*Manager, error) {
|
||||
watchersFile := cfg.WatchersFilePath()
|
||||
|
||||
m := &Manager{
|
||||
config: cfg,
|
||||
watchersFile: watchersFile,
|
||||
watchers: make(map[string]*WatcherConfig),
|
||||
}
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(watchersFile)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create watchers directory: %w", err)
|
||||
}
|
||||
|
||||
// Load or migrate
|
||||
if err := m.loadOrMigrate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// SetOnChanged sets the callback for watcher changes
|
||||
func (m *Manager) SetOnChanged(cb func()) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.onChanged = cb
|
||||
}
|
||||
|
||||
// loadOrMigrate loads existing config or migrates from ENV
|
||||
func (m *Manager) loadOrMigrate() error {
|
||||
// Try to load existing file
|
||||
if _, err := os.Stat(m.watchersFile); err == nil {
|
||||
return m.load()
|
||||
}
|
||||
|
||||
// File doesn't exist, check for ENV migration
|
||||
if len(m.config.WatchFolderList) > 0 {
|
||||
log.Printf("[Watchers] Migrating from WATCH_FOLDER_LIST env var")
|
||||
return m.migrateFromEnv()
|
||||
}
|
||||
|
||||
// No config, start empty
|
||||
log.Printf("[Watchers] No existing config, starting with empty watchers list")
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateFromEnv creates watchers from WATCH_FOLDER_LIST env var
|
||||
func (m *Manager) migrateFromEnv() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for _, path := range m.config.WatchFolderList {
|
||||
id := uuid.New().String()[:8]
|
||||
watcher := &WatcherConfig{
|
||||
ID: id,
|
||||
Path: path,
|
||||
IntervalMs: DefaultIntervalMs,
|
||||
Enabled: true,
|
||||
CreatedAt: NowMS(),
|
||||
}
|
||||
m.watchers[id] = watcher
|
||||
log.Printf("[Watchers] Migrated watcher: %s -> %s", id, path)
|
||||
}
|
||||
|
||||
// Save migrated config
|
||||
return m.saveUnsafe()
|
||||
}
|
||||
|
||||
// load loads watchers from file
|
||||
func (m *Manager) load() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
data, err := os.ReadFile(m.watchersFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read watchers file: %w", err)
|
||||
}
|
||||
|
||||
var file WatchersFile
|
||||
if err := json.Unmarshal(data, &file); err != nil {
|
||||
return fmt.Errorf("failed to parse watchers file: %w", err)
|
||||
}
|
||||
|
||||
m.watchers = make(map[string]*WatcherConfig)
|
||||
for i := range file.Watchers {
|
||||
w := file.Watchers[i]
|
||||
m.watchers[w.ID] = &w
|
||||
}
|
||||
|
||||
log.Printf("[Watchers] Loaded %d watchers from file", len(m.watchers))
|
||||
return nil
|
||||
}
|
||||
|
||||
// save persists watchers to file (thread-safe)
|
||||
func (m *Manager) save() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.saveUnsafe()
|
||||
}
|
||||
|
||||
// saveUnsafe persists watchers to file (must hold lock)
|
||||
func (m *Manager) saveUnsafe() error {
|
||||
watchers := make([]WatcherConfig, 0, len(m.watchers))
|
||||
for _, w := range m.watchers {
|
||||
watchers = append(watchers, *w)
|
||||
}
|
||||
|
||||
file := WatchersFile{
|
||||
Version: 1,
|
||||
Watchers: watchers,
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(file, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal watchers: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(m.watchersFile, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write watchers file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// notifyChanged calls the onChanged callback if set
|
||||
func (m *Manager) notifyChanged() {
|
||||
m.mu.RLock()
|
||||
cb := m.onChanged
|
||||
m.mu.RUnlock()
|
||||
|
||||
if cb != nil {
|
||||
go cb()
|
||||
}
|
||||
}
|
||||
|
||||
// List returns all watchers
|
||||
func (m *Manager) List() []WatcherConfig {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
result := make([]WatcherConfig, 0, len(m.watchers))
|
||||
for _, w := range m.watchers {
|
||||
result = append(result, *w)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Get returns a watcher by ID
|
||||
func (m *Manager) Get(id string) (*WatcherConfig, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
w, ok := m.watchers[id]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("watcher not found: %s", id)
|
||||
}
|
||||
return &WatcherConfig{
|
||||
ID: w.ID,
|
||||
Path: w.Path,
|
||||
IntervalMs: w.IntervalMs,
|
||||
Enabled: w.Enabled,
|
||||
CreatedAt: w.CreatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Create adds a new watcher
|
||||
func (m *Manager) Create(req CreateWatcherRequest) (*WatcherConfig, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Validate path
|
||||
if req.Path == "" {
|
||||
return nil, fmt.Errorf("path is required")
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
intervalMs := req.IntervalMs
|
||||
if intervalMs < MinIntervalMs {
|
||||
intervalMs = DefaultIntervalMs
|
||||
}
|
||||
|
||||
id := uuid.New().String()[:8]
|
||||
watcher := &WatcherConfig{
|
||||
ID: id,
|
||||
Path: req.Path,
|
||||
IntervalMs: intervalMs,
|
||||
Enabled: req.Enabled,
|
||||
CreatedAt: NowMS(),
|
||||
}
|
||||
|
||||
m.watchers[id] = watcher
|
||||
|
||||
if err := m.saveUnsafe(); err != nil {
|
||||
delete(m.watchers, id)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Printf("[Watchers] Created watcher: %s -> %s", id, req.Path)
|
||||
|
||||
// Notify after releasing lock
|
||||
go m.notifyChanged()
|
||||
|
||||
return &WatcherConfig{
|
||||
ID: watcher.ID,
|
||||
Path: watcher.Path,
|
||||
IntervalMs: watcher.IntervalMs,
|
||||
Enabled: watcher.Enabled,
|
||||
CreatedAt: watcher.CreatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Update modifies an existing watcher
|
||||
func (m *Manager) Update(id string, req UpdateWatcherRequest) (*WatcherConfig, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
w, ok := m.watchers[id]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("watcher not found: %s", id)
|
||||
}
|
||||
|
||||
if req.Path != nil {
|
||||
if *req.Path == "" {
|
||||
return nil, fmt.Errorf("path cannot be empty")
|
||||
}
|
||||
w.Path = *req.Path
|
||||
}
|
||||
|
||||
if req.IntervalMs != nil {
|
||||
interval := *req.IntervalMs
|
||||
if interval < MinIntervalMs {
|
||||
interval = MinIntervalMs
|
||||
}
|
||||
w.IntervalMs = interval
|
||||
}
|
||||
|
||||
if req.Enabled != nil {
|
||||
w.Enabled = *req.Enabled
|
||||
}
|
||||
|
||||
if err := m.saveUnsafe(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Printf("[Watchers] Updated watcher: %s", id)
|
||||
|
||||
// Notify after releasing lock
|
||||
go m.notifyChanged()
|
||||
|
||||
return &WatcherConfig{
|
||||
ID: w.ID,
|
||||
Path: w.Path,
|
||||
IntervalMs: w.IntervalMs,
|
||||
Enabled: w.Enabled,
|
||||
CreatedAt: w.CreatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Delete removes a watcher
|
||||
func (m *Manager) Delete(id string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if _, ok := m.watchers[id]; !ok {
|
||||
return fmt.Errorf("watcher not found: %s", id)
|
||||
}
|
||||
|
||||
delete(m.watchers, id)
|
||||
|
||||
if err := m.saveUnsafe(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[Watchers] Deleted watcher: %s", id)
|
||||
|
||||
// Notify after releasing lock
|
||||
go m.notifyChanged()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFilesPath returns the files path from config
|
||||
func (m *Manager) GetFilesPath() string {
|
||||
return m.config.FilesPath
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
package watchers
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/watcher"
|
||||
)
|
||||
|
||||
// ScanFunc is the function type for scanning a directory
|
||||
type ScanFunc func(path string) int
|
||||
|
||||
// watcherPoller manages polling for a single watcher
|
||||
type watcherPoller struct {
|
||||
watcherID string
|
||||
stopChan chan struct{}
|
||||
stopped bool
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// RuntimeStatus holds the runtime state for a watcher
|
||||
type RuntimeStatus struct {
|
||||
Active bool
|
||||
LastScan int64
|
||||
FileCount int
|
||||
IsScanning bool
|
||||
}
|
||||
|
||||
// Poller manages per-watcher polling goroutines
|
||||
type Poller struct {
|
||||
manager *Manager
|
||||
scanner *watcher.Watcher
|
||||
dispatcher *watcher.Dispatcher
|
||||
|
||||
pollers map[string]*watcherPoller // key: watcher ID
|
||||
status map[string]*RuntimeStatus
|
||||
|
||||
running bool
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewPoller creates a new watchers poller
|
||||
func NewPoller(manager *Manager, scanner *watcher.Watcher, dispatcher *watcher.Dispatcher) *Poller {
|
||||
p := &Poller{
|
||||
manager: manager,
|
||||
scanner: scanner,
|
||||
dispatcher: dispatcher,
|
||||
pollers: make(map[string]*watcherPoller),
|
||||
status: make(map[string]*RuntimeStatus),
|
||||
}
|
||||
|
||||
// Register callback for watcher changes
|
||||
manager.SetOnChanged(p.syncWithWatchers)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// Start begins the polling service
|
||||
func (p *Poller) Start() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if p.running {
|
||||
return nil
|
||||
}
|
||||
p.running = true
|
||||
|
||||
log.Println("[WatcherPoller] Starting file watcher polling service")
|
||||
|
||||
// Initial sync with watcher configurations
|
||||
go p.syncWithWatchers()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops all polling goroutines
|
||||
func (p *Poller) Stop() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if !p.running {
|
||||
return nil
|
||||
}
|
||||
p.running = false
|
||||
|
||||
log.Println("[WatcherPoller] Stopping file watcher polling service")
|
||||
|
||||
// Stop all pollers
|
||||
for id, poller := range p.pollers {
|
||||
p.stopPoller(poller)
|
||||
delete(p.pollers, id)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncWithWatchers synchronizes pollers with current watcher configurations
|
||||
func (p *Poller) syncWithWatchers() {
|
||||
p.mu.RLock()
|
||||
running := p.running
|
||||
p.mu.RUnlock()
|
||||
|
||||
if !running {
|
||||
return
|
||||
}
|
||||
|
||||
// Get current watchers
|
||||
watchers := p.manager.List()
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
// Re-check running state after acquiring lock
|
||||
if !p.running {
|
||||
return
|
||||
}
|
||||
|
||||
// Track which watchers we've seen
|
||||
seenWatchers := make(map[string]bool)
|
||||
|
||||
for _, w := range watchers {
|
||||
seenWatchers[w.ID] = true
|
||||
|
||||
existingPoller, exists := p.pollers[w.ID]
|
||||
|
||||
if w.Enabled {
|
||||
// Calculate interval
|
||||
interval := w.IntervalMs
|
||||
if interval < MinIntervalMs {
|
||||
interval = DefaultIntervalMs
|
||||
}
|
||||
|
||||
if !exists {
|
||||
// Start new poller
|
||||
log.Printf("[WatcherPoller] Starting poller for watcher %s (path: %s, interval: %dms)", w.ID, w.Path, interval)
|
||||
p.startPollerForWatcher(w.ID, w.Path, interval)
|
||||
}
|
||||
} else if exists {
|
||||
// Stop existing poller
|
||||
log.Printf("[WatcherPoller] Stopping poller for watcher %s", w.ID)
|
||||
p.stopPoller(existingPoller)
|
||||
delete(p.pollers, w.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop pollers for watchers that no longer exist
|
||||
for id, poller := range p.pollers {
|
||||
if !seenWatchers[id] {
|
||||
log.Printf("[WatcherPoller] Stopping poller for removed watcher %s", id)
|
||||
p.stopPoller(poller)
|
||||
delete(p.pollers, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startPollerForWatcher starts a polling goroutine for a watcher
|
||||
func (p *Poller) startPollerForWatcher(watcherID, path string, intervalMs int) {
|
||||
poller := &watcherPoller{
|
||||
watcherID: watcherID,
|
||||
stopChan: make(chan struct{}),
|
||||
}
|
||||
p.pollers[watcherID] = poller
|
||||
|
||||
// Initialize status
|
||||
if _, exists := p.status[watcherID]; !exists {
|
||||
p.status[watcherID] = &RuntimeStatus{}
|
||||
}
|
||||
p.status[watcherID].Active = true
|
||||
|
||||
go func() {
|
||||
// Run initial scan immediately
|
||||
p.runPollScan(watcherID, path)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-poller.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.runPollScan(watcherID, path)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// stopPoller stops a watcher poller
|
||||
func (p *Poller) stopPoller(poller *watcherPoller) {
|
||||
poller.mu.Lock()
|
||||
defer poller.mu.Unlock()
|
||||
|
||||
if poller.stopped {
|
||||
return
|
||||
}
|
||||
poller.stopped = true
|
||||
close(poller.stopChan)
|
||||
|
||||
// Update status
|
||||
if status, exists := p.status[poller.watcherID]; exists {
|
||||
status.Active = false
|
||||
}
|
||||
}
|
||||
|
||||
// runPollScan executes a scan for a watcher
|
||||
func (p *Poller) runPollScan(watcherID, path string) {
|
||||
if p.scanner == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Mark as scanning
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[watcherID]; exists {
|
||||
status.IsScanning = true
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
// Perform scan
|
||||
fileCount := p.scanner.ScanDirectory(path)
|
||||
|
||||
// Update status
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[watcherID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
status.FileCount = fileCount
|
||||
status.IsScanning = false
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
log.Printf("[WatcherPoller] Poll scan complete for watcher %s: %d files", watcherID, fileCount)
|
||||
}
|
||||
|
||||
// TriggerScan triggers an immediate scan for a watcher
|
||||
func (p *Poller) TriggerScan(watcherID string) (int, error) {
|
||||
w, err := p.manager.Get(watcherID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if p.scanner == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Mark as scanning
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[watcherID]; exists {
|
||||
status.IsScanning = true
|
||||
} else {
|
||||
p.status[watcherID] = &RuntimeStatus{IsScanning: true}
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
// Perform scan
|
||||
fileCount := p.scanner.ScanDirectory(w.Path)
|
||||
|
||||
// Update status
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[watcherID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
status.FileCount = fileCount
|
||||
status.IsScanning = false
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
return fileCount, nil
|
||||
}
|
||||
|
||||
// TriggerScanAll triggers scans for all enabled watchers
|
||||
func (p *Poller) TriggerScanAll() (int, error) {
|
||||
watchers := p.manager.List()
|
||||
totalCount := 0
|
||||
|
||||
for _, w := range watchers {
|
||||
if w.Enabled {
|
||||
count, err := p.TriggerScan(w.ID)
|
||||
if err != nil {
|
||||
log.Printf("[WatcherPoller] Failed to scan watcher %s: %v", w.ID, err)
|
||||
continue
|
||||
}
|
||||
totalCount += count
|
||||
}
|
||||
}
|
||||
|
||||
return totalCount, nil
|
||||
}
|
||||
|
||||
// GetStatus returns the runtime status for a watcher
|
||||
func (p *Poller) GetStatus(watcherID string) *RuntimeStatus {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
if status, exists := p.status[watcherID]; exists {
|
||||
return &RuntimeStatus{
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
FileCount: status.FileCount,
|
||||
IsScanning: status.IsScanning,
|
||||
}
|
||||
}
|
||||
return &RuntimeStatus{}
|
||||
}
|
||||
|
||||
// GetWatcherStatuses returns all watchers with their runtime status
|
||||
func (p *Poller) GetWatcherStatuses() []WatcherStatus {
|
||||
watchers := p.manager.List()
|
||||
result := make([]WatcherStatus, 0, len(watchers))
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
for _, w := range watchers {
|
||||
status := WatcherStatus{
|
||||
WatcherConfig: w,
|
||||
}
|
||||
|
||||
if rs, exists := p.status[w.ID]; exists {
|
||||
status.Active = rs.Active
|
||||
status.LastScan = rs.LastScan
|
||||
status.FileCount = rs.FileCount
|
||||
status.IsScanning = rs.IsScanning
|
||||
}
|
||||
|
||||
result = append(result, status)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package watchers
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
// MinIntervalMs is the minimum polling interval (5 seconds)
|
||||
MinIntervalMs = 5000
|
||||
// DefaultIntervalMs is the default polling interval (30 seconds)
|
||||
DefaultIntervalMs = 30000
|
||||
)
|
||||
|
||||
// WatcherConfig represents a watcher configuration stored in watchers.json
|
||||
type WatcherConfig struct {
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"` // e.g., "/files/watch", "/files/movies"
|
||||
IntervalMs int `json:"intervalMs"` // Min: 5000 (5s), Default: 30000 (30s)
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
}
|
||||
|
||||
// WatcherStatus includes runtime status in API responses
|
||||
type WatcherStatus struct {
|
||||
WatcherConfig
|
||||
Active bool `json:"active"` // Polling goroutine running
|
||||
LastScan int64 `json:"lastScan"` // Unix ms timestamp
|
||||
FileCount int `json:"fileCount"` // Files found in last scan
|
||||
IsScanning bool `json:"isScanning"` // Currently scanning
|
||||
}
|
||||
|
||||
// WatchersFile is the structure of watchers.json
|
||||
type WatchersFile struct {
|
||||
Version int `json:"version"`
|
||||
Watchers []WatcherConfig `json:"watchers"`
|
||||
}
|
||||
|
||||
// CreateWatcherRequest is the request body for creating a watcher
|
||||
type CreateWatcherRequest struct {
|
||||
Path string `json:"path"`
|
||||
IntervalMs int `json:"intervalMs"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// UpdateWatcherRequest is the request body for updating a watcher
|
||||
type UpdateWatcherRequest struct {
|
||||
Path *string `json:"path,omitempty"`
|
||||
IntervalMs *int `json:"intervalMs,omitempty"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
// WatchersListResponse is the API response for listing watchers
|
||||
type WatchersListResponse struct {
|
||||
Watchers []WatcherStatus `json:"watchers"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// NowMS returns current time in milliseconds
|
||||
func NowMS() int64 {
|
||||
return time.Now().UnixMilli()
|
||||
}
|
||||
Reference in New Issue
Block a user