new flat KV structure
This commit is contained in:
Pierre
@@ -66,6 +66,9 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
|
||||
if event.OldPath != "" {
|
||||
fields["oldPath"] = event.OldPath
|
||||
}
|
||||
if event.WatcherID != "" {
|
||||
fields["watcherId"] = event.WatcherID
|
||||
}
|
||||
|
||||
// Publish to stream
|
||||
_, err := client.XAdd(EventsStream, StreamMaxLen, fields)
|
||||
@@ -73,3 +76,49 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
|
||||
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// EmitReset clears the stream and emits a reset event
|
||||
// This is called when triggering a full rescan
|
||||
func (d *Dispatcher) EmitReset(watcherID string) error {
|
||||
d.mu.RLock()
|
||||
client := d.storageClient
|
||||
d.mu.RUnlock()
|
||||
|
||||
if client == nil || !client.IsConnected() {
|
||||
return fmt.Errorf("storage client not connected")
|
||||
}
|
||||
|
||||
// Delete the stream to clear all historical events
|
||||
if err := client.ClearStream(EventsStream); err != nil {
|
||||
log.Printf("[Dispatcher] Warning: failed to clear stream: %v", err)
|
||||
// Continue anyway - we'll emit the reset event
|
||||
}
|
||||
|
||||
log.Printf("[Dispatcher] Cleared stream %s for reset", EventsStream)
|
||||
|
||||
// Emit reset event as the first message in the fresh stream
|
||||
event := FileEvent{
|
||||
Type: EventTypeReset,
|
||||
Path: "", // No specific path for reset
|
||||
WatcherID: watcherID,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Dispatch synchronously for reset
|
||||
d.dispatchToStream(event)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStreamLength returns the current stream length
|
||||
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
||||
d.mu.RLock()
|
||||
client := d.storageClient
|
||||
d.mu.RUnlock()
|
||||
|
||||
if client == nil || !client.IsConnected() {
|
||||
return 0, fmt.Errorf("storage client not connected")
|
||||
}
|
||||
|
||||
return client.GetStreamLength(EventsStream)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FileState tracks the known state of a file
|
||||
type FileState struct {
|
||||
Size int64
|
||||
MtimeNano int64
|
||||
MidHash string
|
||||
}
|
||||
|
||||
// WatcherState tracks all files for a single watcher (in-memory)
|
||||
type WatcherState struct {
|
||||
mu sync.RWMutex
|
||||
files map[string]*FileState // key: relative path
|
||||
}
|
||||
|
||||
// NewWatcherState creates a new watcher state tracker
|
||||
func NewWatcherState() *WatcherState {
|
||||
return &WatcherState{
|
||||
files: make(map[string]*FileState),
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns the file state for a given path, or nil if not tracked
|
||||
func (ws *WatcherState) Get(path string) *FileState {
|
||||
ws.mu.RLock()
|
||||
defer ws.mu.RUnlock()
|
||||
return ws.files[path]
|
||||
}
|
||||
|
||||
// Set updates or adds the file state for a given path
|
||||
func (ws *WatcherState) Set(path string, state *FileState) {
|
||||
ws.mu.Lock()
|
||||
defer ws.mu.Unlock()
|
||||
ws.files[path] = state
|
||||
}
|
||||
|
||||
// Delete removes a file from the tracked state
|
||||
func (ws *WatcherState) Delete(path string) {
|
||||
ws.mu.Lock()
|
||||
defer ws.mu.Unlock()
|
||||
delete(ws.files, path)
|
||||
}
|
||||
|
||||
// GetAll returns a snapshot of all tracked files
|
||||
// The returned map is a copy - safe to modify
|
||||
func (ws *WatcherState) GetAll() map[string]*FileState {
|
||||
ws.mu.RLock()
|
||||
defer ws.mu.RUnlock()
|
||||
|
||||
snapshot := make(map[string]*FileState, len(ws.files))
|
||||
for path, state := range ws.files {
|
||||
// Copy the state to avoid race conditions
|
||||
stateCopy := *state
|
||||
snapshot[path] = &stateCopy
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// Clear removes all tracked files
|
||||
func (ws *WatcherState) Clear() {
|
||||
ws.mu.Lock()
|
||||
defer ws.mu.Unlock()
|
||||
ws.files = make(map[string]*FileState)
|
||||
}
|
||||
|
||||
// Count returns the number of tracked files
|
||||
func (ws *WatcherState) Count() int {
|
||||
ws.mu.RLock()
|
||||
defer ws.mu.RUnlock()
|
||||
return len(ws.files)
|
||||
}
|
||||
|
||||
// StateRegistry manages per-watcher state
|
||||
type StateRegistry struct {
|
||||
mu sync.RWMutex
|
||||
states map[string]*WatcherState // key: watcher ID
|
||||
}
|
||||
|
||||
// NewStateRegistry creates a new state registry
|
||||
func NewStateRegistry() *StateRegistry {
|
||||
return &StateRegistry{
|
||||
states: make(map[string]*WatcherState),
|
||||
}
|
||||
}
|
||||
|
||||
// GetOrCreate returns the state for a watcher, creating one if needed
|
||||
func (sr *StateRegistry) GetOrCreate(watcherID string) *WatcherState {
|
||||
sr.mu.Lock()
|
||||
defer sr.mu.Unlock()
|
||||
|
||||
state, exists := sr.states[watcherID]
|
||||
if !exists {
|
||||
state = NewWatcherState()
|
||||
sr.states[watcherID] = state
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// Get returns the state for a watcher, or nil if not exists
|
||||
func (sr *StateRegistry) Get(watcherID string) *WatcherState {
|
||||
sr.mu.RLock()
|
||||
defer sr.mu.RUnlock()
|
||||
return sr.states[watcherID]
|
||||
}
|
||||
|
||||
// Clear removes the state for a watcher
|
||||
func (sr *StateRegistry) Clear(watcherID string) {
|
||||
sr.mu.Lock()
|
||||
defer sr.mu.Unlock()
|
||||
|
||||
if state, exists := sr.states[watcherID]; exists {
|
||||
state.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
// ClearAll removes all watcher states
|
||||
func (sr *StateRegistry) ClearAll() {
|
||||
sr.mu.Lock()
|
||||
defer sr.mu.Unlock()
|
||||
|
||||
for _, state := range sr.states {
|
||||
state.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Delete removes a watcher's state completely
|
||||
func (sr *StateRegistry) Delete(watcherID string) {
|
||||
sr.mu.Lock()
|
||||
defer sr.mu.Unlock()
|
||||
delete(sr.states, watcherID)
|
||||
}
|
||||
|
||||
// Count returns the number of files tracked for a watcher
|
||||
func (sr *StateRegistry) Count(watcherID string) int {
|
||||
sr.mu.RLock()
|
||||
defer sr.mu.RUnlock()
|
||||
|
||||
if state, exists := sr.states[watcherID]; exists {
|
||||
return state.Count()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -10,6 +10,7 @@ const (
|
||||
EventTypeChange FileEventType = "change"
|
||||
EventTypeDelete FileEventType = "delete"
|
||||
EventTypeRename FileEventType = "rename"
|
||||
EventTypeReset FileEventType = "reset"
|
||||
)
|
||||
|
||||
// FileEvent represents a file system event
|
||||
@@ -20,6 +21,7 @@ type FileEvent struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
MidHash256 string `json:"midhash256,omitempty"` // midhash256 CID (SHA-256 of middle 1MB + file size)
|
||||
OldPath string `json:"oldPath,omitempty"` // For rename events
|
||||
WatcherID string `json:"watcherId,omitempty"` // For reset events
|
||||
}
|
||||
|
||||
// PendingEvent tracks a file event that's being debounced
|
||||
@@ -44,6 +46,14 @@ type ScanStatusResponse struct {
|
||||
FileCount int `json:"fileCount,omitempty"`
|
||||
}
|
||||
|
||||
// DiffScanResult holds the results of a differential scan
|
||||
type DiffScanResult struct {
|
||||
Added int // Number of new files added
|
||||
Changed int // Number of files with changed size/mtime
|
||||
Deleted int // Number of files that no longer exist
|
||||
Total int // Total number of files currently tracked
|
||||
}
|
||||
|
||||
// NowMS returns current time in milliseconds
|
||||
func NowMS() int64 {
|
||||
return time.Now().UnixMilli()
|
||||
|
||||
+143
-9
@@ -13,10 +13,11 @@ import (
|
||||
|
||||
// Watcher provides file scanning and event dispatching
|
||||
type Watcher struct {
|
||||
config *config.Config
|
||||
debouncer *Debouncer
|
||||
dispatcher *Dispatcher
|
||||
filesPath string
|
||||
config *config.Config
|
||||
debouncer *Debouncer
|
||||
dispatcher *Dispatcher
|
||||
filesPath string
|
||||
stateRegistry *StateRegistry
|
||||
|
||||
mu sync.RWMutex
|
||||
eventBuffer []FileEvent
|
||||
@@ -27,11 +28,12 @@ func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
||||
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
|
||||
|
||||
w := &Watcher{
|
||||
config: cfg,
|
||||
debouncer: debouncer,
|
||||
dispatcher: dispatcher,
|
||||
filesPath: cfg.FilesPath,
|
||||
eventBuffer: make([]FileEvent, 0, 1000),
|
||||
config: cfg,
|
||||
debouncer: debouncer,
|
||||
dispatcher: dispatcher,
|
||||
filesPath: cfg.FilesPath,
|
||||
stateRegistry: NewStateRegistry(),
|
||||
eventBuffer: make([]FileEvent, 0, 1000),
|
||||
}
|
||||
|
||||
return w, nil
|
||||
@@ -110,6 +112,138 @@ func (w *Watcher) ScanDirectory(root string) int {
|
||||
return count
|
||||
}
|
||||
|
||||
// ScanDirectoryDiff scans a directory and only emits events for differences
|
||||
// Returns the count of added, changed, and deleted files
|
||||
func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResult {
|
||||
result := DiffScanResult{}
|
||||
state := w.stateRegistry.GetOrCreate(watcherID)
|
||||
|
||||
// Get snapshot of known files
|
||||
knownFiles := state.GetAll()
|
||||
seenFiles := make(map[string]bool)
|
||||
|
||||
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip hidden files
|
||||
if strings.HasPrefix(filepath.Base(path), ".") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(w.filesPath, path)
|
||||
if err != nil {
|
||||
relPath = path
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
seenFiles[relPath] = true
|
||||
|
||||
// Check current file state
|
||||
currentMtime := info.ModTime().UnixNano()
|
||||
currentSize := info.Size()
|
||||
|
||||
prevState := knownFiles[relPath]
|
||||
|
||||
if prevState == nil {
|
||||
// New file - emit add event
|
||||
event := FileEvent{
|
||||
Type: EventTypeAdd,
|
||||
Path: relPath,
|
||||
Size: currentSize,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute midhash256
|
||||
if hash, err := ComputeMidHash256(path); err == nil {
|
||||
event.MidHash256 = hash
|
||||
}
|
||||
|
||||
// Dispatch event
|
||||
w.dispatcher.Dispatch(event)
|
||||
|
||||
// Update state
|
||||
state.Set(relPath, &FileState{
|
||||
Size: currentSize,
|
||||
MtimeNano: currentMtime,
|
||||
MidHash: event.MidHash256,
|
||||
})
|
||||
|
||||
result.Added++
|
||||
} else if prevState.Size != currentSize || prevState.MtimeNano != currentMtime {
|
||||
// File changed - emit change event
|
||||
event := FileEvent{
|
||||
Type: EventTypeChange,
|
||||
Path: relPath,
|
||||
Size: currentSize,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute midhash256
|
||||
if hash, err := ComputeMidHash256(path); err == nil {
|
||||
event.MidHash256 = hash
|
||||
}
|
||||
|
||||
// Dispatch event
|
||||
w.dispatcher.Dispatch(event)
|
||||
|
||||
// Update state
|
||||
state.Set(relPath, &FileState{
|
||||
Size: currentSize,
|
||||
MtimeNano: currentMtime,
|
||||
MidHash: event.MidHash256,
|
||||
})
|
||||
|
||||
result.Changed++
|
||||
}
|
||||
// If file exists and hasn't changed, do nothing
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
// Check for deleted files (in known but not seen)
|
||||
for relPath := range knownFiles {
|
||||
if !seenFiles[relPath] {
|
||||
// File deleted - emit delete event
|
||||
event := FileEvent{
|
||||
Type: EventTypeDelete,
|
||||
Path: relPath,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
w.dispatcher.Dispatch(event)
|
||||
state.Delete(relPath)
|
||||
result.Deleted++
|
||||
}
|
||||
}
|
||||
|
||||
result.Total = state.Count()
|
||||
return result
|
||||
}
|
||||
|
||||
// ClearState clears the tracked state for a watcher
|
||||
func (w *Watcher) ClearState(watcherID string) {
|
||||
w.stateRegistry.Clear(watcherID)
|
||||
}
|
||||
|
||||
// ClearEventBuffer clears the in-memory event buffer
|
||||
func (w *Watcher) ClearEventBuffer() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.eventBuffer = make([]FileEvent, 0, 1000)
|
||||
}
|
||||
|
||||
// GetStateRegistry returns the state registry for external access
|
||||
func (w *Watcher) GetStateRegistry() *StateRegistry {
|
||||
return w.stateRegistry
|
||||
}
|
||||
|
||||
// ScanMountPath scans a specific mount path and returns file count
|
||||
// This is used by the mount poller to scan individual mounts
|
||||
func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
|
||||
|
||||
Reference in New Issue
Block a user