cleaning of the files event system

This commit is contained in:
Pierre
2026-02-08 15:30:31 +01:00
parent ce6995c388
commit a9cbbfacea
16 changed files with 1437 additions and 577 deletions
+38 -186
View File
@@ -1,223 +1,75 @@
package watcher
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
"time"
"github.com/metazla/meta-core/internal/storage"
)
const (
// MaxRetries is the maximum number of delivery retries
MaxRetries = 3
// RetryDelay is the delay between retries
RetryDelay = 5 * time.Second
// WebhookTimeout is the timeout for webhook requests
WebhookTimeout = 10 * time.Second
// MaxFailCount is the max failures before removing subscriber
MaxFailCount = 10
// EventsStream is the Redis Stream name for file events
EventsStream = "file:events"
// StreamMaxLen is the approximate max length for the stream
StreamMaxLen = 10000
)
// Dispatcher sends file events to subscribers
// Dispatcher sends file events to Redis Stream
type Dispatcher struct {
subscribers map[string]*Subscriber
sseClients map[chan FileEvent]bool
mu sync.RWMutex
httpClient *http.Client
mu sync.RWMutex
storageClient *storage.Client
}
// NewDispatcher creates a new event dispatcher
func NewDispatcher() *Dispatcher {
return &Dispatcher{
subscribers: make(map[string]*Subscriber),
sseClients: make(map[chan FileEvent]bool),
httpClient: &http.Client{
Timeout: WebhookTimeout,
},
}
return &Dispatcher{}
}
// Subscribe registers a webhook subscriber
func (d *Dispatcher) Subscribe(url string, eventTypes []string) error {
// SetStorageClient sets the storage client for Redis Stream publishing
func (d *Dispatcher) SetStorageClient(client *storage.Client) {
d.mu.Lock()
defer d.mu.Unlock()
d.subscribers[url] = &Subscriber{
URL: url,
RegisteredAt: NowMS(),
EventTypes: eventTypes,
FailCount: 0,
}
log.Printf("[Dispatcher] Subscribed webhook: %s", url)
return nil
d.storageClient = client
log.Println("[Dispatcher] Storage client set for Redis Stream publishing")
}
// Unsubscribe removes a webhook subscriber
func (d *Dispatcher) Unsubscribe(url string) error {
d.mu.Lock()
defer d.mu.Unlock()
delete(d.subscribers, url)
log.Printf("[Dispatcher] Unsubscribed webhook: %s", url)
return nil
}
// ListSubscribers returns all subscribers
func (d *Dispatcher) ListSubscribers() []Subscriber {
d.mu.RLock()
defer d.mu.RUnlock()
result := make([]Subscriber, 0, len(d.subscribers))
for _, sub := range d.subscribers {
result = append(result, *sub)
}
return result
}
// AddSSEClient adds an SSE client channel
func (d *Dispatcher) AddSSEClient(ch chan FileEvent) {
d.mu.Lock()
defer d.mu.Unlock()
d.sseClients[ch] = true
}
// RemoveSSEClient removes an SSE client channel
func (d *Dispatcher) RemoveSSEClient(ch chan FileEvent) {
d.mu.Lock()
defer d.mu.Unlock()
delete(d.sseClients, ch)
close(ch)
}
// Dispatch sends an event to all subscribers
// Dispatch sends an event to Redis Stream
func (d *Dispatcher) Dispatch(event FileEvent) {
// Dispatch to webhooks
go d.dispatchToWebhooks(event)
// Dispatch to SSE clients
d.dispatchToSSE(event)
go d.dispatchToStream(event)
}
// dispatchToWebhooks sends event to webhook subscribers
func (d *Dispatcher) dispatchToWebhooks(event FileEvent) {
// dispatchToStream publishes event to Redis Stream for service consumption
func (d *Dispatcher) dispatchToStream(event FileEvent) {
d.mu.RLock()
subscribers := make([]*Subscriber, 0, len(d.subscribers))
for _, sub := range d.subscribers {
subscribers = append(subscribers, sub)
}
client := d.storageClient
d.mu.RUnlock()
for _, sub := range subscribers {
// Check event type filter
if len(sub.EventTypes) > 0 {
found := false
for _, et := range sub.EventTypes {
if et == string(event.Type) {
found = true
break
}
}
if !found {
continue
}
}
go d.deliverToWebhook(sub.URL, event)
}
}
// deliverToWebhook delivers an event to a webhook with retries
func (d *Dispatcher) deliverToWebhook(url string, event FileEvent) {
body, err := json.Marshal(event)
if err != nil {
log.Printf("[Dispatcher] Failed to marshal event: %v", err)
if client == nil || !client.IsConnected() {
return
}
var lastErr error
for attempt := 0; attempt < MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(RetryDelay)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(body))
if err != nil {
lastErr = err
continue
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Event-Type", string(event.Type))
resp, err := d.httpClient.Do(req)
if err != nil {
lastErr = err
continue
}
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// Success
d.mu.Lock()
if sub, ok := d.subscribers[url]; ok {
sub.LastDelivery = NowMS()
sub.FailCount = 0
}
d.mu.Unlock()
return
}
lastErr = &httpError{StatusCode: resp.StatusCode}
// Build stream fields
fields := map[string]interface{}{
"type": string(event.Type),
"path": event.Path,
"timestamp": fmt.Sprintf("%d", event.Timestamp),
}
// All retries failed
log.Printf("[Dispatcher] Failed to deliver to %s after %d retries: %v", url, MaxRetries, lastErr)
d.mu.Lock()
if sub, ok := d.subscribers[url]; ok {
sub.FailCount++
if sub.FailCount >= MaxFailCount {
delete(d.subscribers, url)
log.Printf("[Dispatcher] Removed webhook %s after %d failures", url, MaxFailCount)
}
// Add optional fields
if event.Size > 0 {
fields["size"] = fmt.Sprintf("%d", event.Size)
}
d.mu.Unlock()
}
// dispatchToSSE sends event to SSE clients
func (d *Dispatcher) dispatchToSSE(event FileEvent) {
d.mu.RLock()
clients := make([]chan FileEvent, 0, len(d.sseClients))
for ch := range d.sseClients {
clients = append(clients, ch)
if event.PartialHash != "" {
fields["partialHash"] = event.PartialHash
}
if event.OldPath != "" {
fields["oldPath"] = event.OldPath
}
d.mu.RUnlock()
for _, ch := range clients {
select {
case ch <- event:
default:
// Channel full, skip
}
// Publish to stream
_, err := client.XAdd(EventsStream, StreamMaxLen, fields)
if err != nil {
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
}
}
// SSEClientCount returns the number of connected SSE clients
func (d *Dispatcher) SSEClientCount() int {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.sseClients)
}
// httpError represents an HTTP error
type httpError struct {
StatusCode int
}
func (e *httpError) Error() string {
return http.StatusText(e.StatusCode)
}
+11 -115
View File
@@ -2,7 +2,6 @@ package watcher
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
@@ -25,59 +24,12 @@ func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
// RegisterRoutes registers all watcher-related routes
func (h *Handlers) RegisterRoutes(r *mux.Router) {
// Event subscriptions
r.HandleFunc("/api/events/subscribe", h.handleSSESubscribe).Methods("GET")
// Event polling (for services that don't use Redis Streams)
r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
r.HandleFunc("/api/events/subscribers", h.handleListSubscribers).Methods("GET")
r.HandleFunc("/api/events/subscribers", h.handleAddSubscriber).Methods("POST")
r.HandleFunc("/api/events/subscribers/{url}", h.handleRemoveSubscriber).Methods("DELETE")
// Scan management
r.HandleFunc("/api/scan/trigger", h.handleTriggerScan).Methods("POST")
r.HandleFunc("/api/scan/status", h.handleScanStatus).Methods("GET")
}
// handleSSESubscribe handles GET /api/events/subscribe (Server-Sent Events)
func (h *Handlers) handleSSESubscribe(w http.ResponseWriter, r *http.Request) {
// Set SSE headers
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("Access-Control-Allow-Origin", "*")
// Create event channel
eventChan := make(chan FileEvent, 100)
h.dispatcher.AddSSEClient(eventChan)
// Cleanup on disconnect
defer h.dispatcher.RemoveSSEClient(eventChan)
// Get flusher
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "SSE not supported", http.StatusInternalServerError)
return
}
// Send initial connected message
fmt.Fprintf(w, "event: connected\ndata: {\"status\":\"connected\"}\n\n")
flusher.Flush()
// Stream events
for {
select {
case event := <-eventChan:
data, err := json.Marshal(event)
if err != nil {
continue
}
fmt.Fprintf(w, "event: file\ndata: %s\n\n", string(data))
flusher.Flush()
case <-r.Context().Done():
return
}
}
// Deprecated scan endpoints - redirect to /api/watchers
r.HandleFunc("/api/scan/trigger", h.handleTriggerScanDeprecated).Methods("POST")
r.HandleFunc("/api/scan/status", h.handleScanStatusDeprecated).Methods("GET")
}
// handlePoll handles GET /api/events/poll
@@ -108,78 +60,22 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
})
}
// handleListSubscribers handles GET /api/events/subscribers
func (h *Handlers) handleListSubscribers(w http.ResponseWriter, r *http.Request) {
subscribers := h.dispatcher.ListSubscribers()
writeJSON(w, http.StatusOK, SubscribersListResponse{
Subscribers: subscribers,
Count: len(subscribers),
})
}
// handleAddSubscriber handles POST /api/events/subscribers
func (h *Handlers) handleAddSubscriber(w http.ResponseWriter, r *http.Request) {
var req SubscribeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.URL == "" {
writeError(w, http.StatusBadRequest, "URL is required")
return
}
if err := h.dispatcher.Subscribe(req.URL, req.EventTypes); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, map[string]interface{}{
"status": "ok",
"message": "Subscribed",
"url": req.URL,
})
}
// handleRemoveSubscriber handles DELETE /api/events/subscribers/{url}
func (h *Handlers) handleRemoveSubscriber(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
url := vars["url"]
if url == "" {
writeError(w, http.StatusBadRequest, "URL is required")
return
}
if err := h.dispatcher.Unsubscribe(url); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// handleTriggerScanDeprecated handles POST /api/scan/trigger (deprecated)
func (h *Handlers) handleTriggerScanDeprecated(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Unsubscribed",
"message": "Use POST /api/watchers/scan-all instead. This endpoint is deprecated.",
})
}
// handleTriggerScan handles POST /api/scan/trigger
func (h *Handlers) handleTriggerScan(w http.ResponseWriter, r *http.Request) {
go h.watcher.RunScan()
// handleScanStatusDeprecated handles GET /api/scan/status (deprecated)
func (h *Handlers) handleScanStatusDeprecated(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "Scan triggered",
"status": "deprecated",
"message": "Use GET /api/watchers instead. This endpoint is deprecated.",
})
}
// handleScanStatus handles GET /api/scan/status
func (h *Handlers) handleScanStatus(w http.ResponseWriter, r *http.Request) {
status := h.watcher.GetStatus()
writeJSON(w, http.StatusOK, status)
}
// Helper functions
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
-21
View File
@@ -22,15 +22,6 @@ type FileEvent struct {
OldPath string `json:"oldPath,omitempty"` // For rename events
}
// Subscriber represents a webhook subscriber
type Subscriber struct {
URL string `json:"url"`
RegisteredAt int64 `json:"registeredAt"`
EventTypes []string `json:"eventTypes,omitempty"` // Empty means all
LastDelivery int64 `json:"lastDelivery,omitempty"`
FailCount int `json:"failCount"`
}
// PendingEvent tracks a file event that's being debounced
type PendingEvent struct {
Event FileEvent
@@ -39,24 +30,12 @@ type PendingEvent struct {
Timer *time.Timer
}
// SubscribeRequest is the request to register a webhook
type SubscribeRequest struct {
URL string `json:"url"`
EventTypes []string `json:"eventTypes,omitempty"`
}
// EventsListResponse is the response for listing events
type EventsListResponse struct {
Events []FileEvent `json:"events"`
Count int `json:"count"`
}
// SubscribersListResponse is the response for listing subscribers
type SubscribersListResponse struct {
Subscribers []Subscriber `json:"subscribers"`
Count int `json:"count"`
}
// ScanStatusResponse is the response for scan status
type ScanStatusResponse struct {
Status string `json:"status"`
+25 -221
View File
@@ -3,16 +3,13 @@ package watcher
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/fsnotify/fsnotify"
"github.com/metazla/meta-core/internal/config"
)
@@ -21,188 +18,32 @@ const (
PartialHashSize = 64 * 1024 // 64KB
)
// Watcher monitors directories for file changes
// Watcher provides file scanning and event dispatching
type Watcher struct {
config *config.Config
fsWatcher *fsnotify.Watcher
debouncer *Debouncer
dispatcher *Dispatcher
filesPath string
watchPaths []string
mu sync.RWMutex
isRunning bool
isScanning bool
lastScan int64
fileCount int
stopChan chan struct{}
eventBuffer []FileEvent
}
// NewWatcher creates a new file watcher
// NewWatcher creates a new file watcher/scanner
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
fsWatcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
debouncer := NewDebouncer(time.Duration(cfg.DebounceMS) * time.Millisecond)
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
w := &Watcher{
config: cfg,
fsWatcher: fsWatcher,
debouncer: debouncer,
dispatcher: dispatcher,
filesPath: cfg.FilesPath,
watchPaths: cfg.WatchFolderList,
stopChan: make(chan struct{}),
eventBuffer: make([]FileEvent, 0, 1000),
}
// Set debouncer callback
debouncer.SetCallback(func(event FileEvent) {
w.handleDebouncedEvent(event)
})
return w, nil
}
// Start begins watching for file changes
func (w *Watcher) Start() error {
w.mu.Lock()
if w.isRunning {
w.mu.Unlock()
return nil
}
w.isRunning = true
w.mu.Unlock()
// Add watch paths
for _, watchPath := range w.watchPaths {
if err := w.addWatchRecursive(watchPath); err != nil {
log.Printf("[Watcher] Warning: failed to watch %s: %v", watchPath, err)
}
}
// Start event processing goroutine
go w.processEvents()
// Start initial scan
go w.RunScan()
log.Printf("[Watcher] Started watching %d paths", len(w.watchPaths))
return nil
}
// Stop stops the watcher
func (w *Watcher) Stop() error {
w.mu.Lock()
if !w.isRunning {
w.mu.Unlock()
return nil
}
w.isRunning = false
w.mu.Unlock()
close(w.stopChan)
w.debouncer.Stop()
return w.fsWatcher.Close()
}
// addWatchRecursive adds a directory and all subdirectories to the watch list
func (w *Watcher) addWatchRecursive(root string) error {
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // Skip inaccessible paths
}
if info.IsDir() {
if err := w.fsWatcher.Add(path); err != nil {
log.Printf("[Watcher] Warning: cannot watch %s: %v", path, err)
}
}
return nil
})
}
// processEvents handles fsnotify events
func (w *Watcher) processEvents() {
for {
select {
case <-w.stopChan:
return
case event, ok := <-w.fsWatcher.Events:
if !ok {
return
}
w.handleFsEvent(event)
case err, ok := <-w.fsWatcher.Errors:
if !ok {
return
}
log.Printf("[Watcher] Error: %v", err)
}
}
}
// handleFsEvent converts an fsnotify event to a FileEvent
func (w *Watcher) handleFsEvent(event fsnotify.Event) {
// Get relative path
relPath, err := filepath.Rel(w.filesPath, event.Name)
if err != nil {
relPath = event.Name
}
// Clean up path
relPath = filepath.ToSlash(relPath)
// Determine event type
var eventType FileEventType
switch {
case event.Op&fsnotify.Create == fsnotify.Create:
eventType = EventTypeAdd
// If it's a new directory, add it to watch
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
w.fsWatcher.Add(event.Name)
}
case event.Op&fsnotify.Write == fsnotify.Write:
eventType = EventTypeChange
case event.Op&fsnotify.Remove == fsnotify.Remove:
eventType = EventTypeDelete
case event.Op&fsnotify.Rename == fsnotify.Rename:
eventType = EventTypeRename
default:
return // Ignore other events
}
// Skip directories for file events (except remove)
if eventType != EventTypeDelete {
info, err := os.Stat(event.Name)
if err == nil && info.IsDir() {
return
}
}
// Get file info if available
var size int64
if eventType != EventTypeDelete {
if info, err := os.Stat(event.Name); err == nil {
size = info.Size()
}
}
fileEvent := FileEvent{
Type: eventType,
Path: relPath,
Size: size,
Timestamp: NowMS(),
}
// Send to debouncer
w.debouncer.Add(fileEvent)
}
// handleDebouncedEvent processes a debounced event
func (w *Watcher) handleDebouncedEvent(event FileEvent) {
// Compute partial hash for add/change events
@@ -224,44 +65,11 @@ func (w *Watcher) handleDebouncedEvent(event FileEvent) {
// Dispatch to subscribers
w.dispatcher.Dispatch(event)
log.Printf("[Watcher] Event: %s %s", event.Type, event.Path)
}
// RunScan performs a full directory scan
func (w *Watcher) RunScan() {
w.mu.Lock()
if w.isScanning {
w.mu.Unlock()
return
}
w.isScanning = true
w.mu.Unlock()
defer func() {
w.mu.Lock()
w.isScanning = false
w.lastScan = NowMS()
w.mu.Unlock()
}()
log.Println("[Watcher] Starting directory scan...")
fileCount := 0
for _, watchPath := range w.watchPaths {
count := w.scanDirectory(watchPath)
fileCount += count
}
w.mu.Lock()
w.fileCount = fileCount
w.mu.Unlock()
log.Printf("[Watcher] Scan complete: %d files found", fileCount)
}
// scanDirectory scans a directory and emits add events for files
func (w *Watcher) scanDirectory(root string) int {
// ScanDirectory scans a directory and emits add events for files
// Returns the count of files found
func (w *Watcher) ScanDirectory(root string) int {
count := 0
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
@@ -286,7 +94,7 @@ func (w *Watcher) scanDirectory(root string) int {
}
relPath = filepath.ToSlash(relPath)
// Emit add event (through debouncer)
// Emit add event
event := FileEvent{
Type: EventTypeAdd,
Path: relPath,
@@ -299,7 +107,7 @@ func (w *Watcher) scanDirectory(root string) int {
event.PartialHash = hash
}
// Dispatch directly (skip debouncer for scan)
// Dispatch directly
w.dispatcher.Dispatch(event)
count++
@@ -314,25 +122,34 @@ func (w *Watcher) scanDirectory(root string) int {
func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
// Validate path is under filesPath
if !strings.HasPrefix(mountPath, w.filesPath) {
return 0, fmt.Errorf("mount path %s is not under files path %s", mountPath, w.filesPath)
// Try to normalize paths
absMount, err := filepath.Abs(mountPath)
if err == nil {
absFiles, err := filepath.Abs(w.filesPath)
if err == nil && strings.HasPrefix(absMount, absFiles) {
mountPath = absMount
}
}
}
// Check directory exists
info, err := os.Stat(mountPath)
if err != nil {
if os.IsNotExist(err) {
return 0, fmt.Errorf("mount path %s does not exist", mountPath)
log.Printf("[Watcher] Mount path %s does not exist, skipping", mountPath)
return 0, nil
}
return 0, fmt.Errorf("failed to stat mount path %s: %w", mountPath, err)
return 0, err
}
if !info.IsDir() {
return 0, fmt.Errorf("mount path %s is not a directory", mountPath)
log.Printf("[Watcher] Mount path %s is not a directory, skipping", mountPath)
return 0, nil
}
// Scan the directory
log.Printf("[Watcher] Scanning mount path: %s", mountPath)
count := w.scanDirectory(mountPath)
count := w.ScanDirectory(mountPath)
log.Printf("[Watcher] Mount scan complete: %s (%d files)", mountPath, count)
return count, nil
@@ -356,22 +173,9 @@ func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
return result
}
// GetStatus returns watcher status
func (w *Watcher) GetStatus() ScanStatusResponse {
w.mu.RLock()
defer w.mu.RUnlock()
status := "running"
if !w.isRunning {
status = "stopped"
}
return ScanStatusResponse{
Status: status,
Scanning: w.isScanning,
LastScan: w.lastScan,
FileCount: w.fileCount,
}
// GetFilesPath returns the files path
func (w *Watcher) GetFilesPath() string {
return w.filesPath
}
// computePartialHash computes SHA-256 hash of first 64KB of a file