improved kv metacore
This commit is contained in:
+98
-17
@@ -19,11 +19,12 @@ import (
|
||||
|
||||
// Manager handles mount configuration and status
|
||||
type Manager struct {
|
||||
config *config.Config
|
||||
mu sync.RWMutex
|
||||
filesPath string
|
||||
mountsDir string
|
||||
poller *Poller
|
||||
config *config.Config
|
||||
mu sync.RWMutex
|
||||
filesPath string
|
||||
mountsDir string
|
||||
poller *Poller
|
||||
statsPoller *StatsPoller
|
||||
}
|
||||
|
||||
// NewManager creates a new mount manager
|
||||
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
|
||||
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
||||
}
|
||||
|
||||
// One-shot migration for legacy NFS entries — disable rather than fail to
|
||||
// start, so operators can fix their config at their own pace.
|
||||
if err := m.migrateLegacyMounts(); err != nil {
|
||||
log.Printf("[Mounts] legacy migration warning: %v", err)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// migrateLegacyMounts disables any mount entries with type="nfs". The native
|
||||
// NFS handler was removed when the project consolidated on rclone; an NFS
|
||||
// entry left enabled would loop the watcher with mount errors.
|
||||
func (m *Manager) migrateLegacyMounts() error {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dirty := false
|
||||
for i := range mountsFile.Mounts {
|
||||
mnt := &mountsFile.Mounts[i]
|
||||
if string(mnt.Type) == "nfs" {
|
||||
if mnt.Enabled || mnt.DesiredMounted {
|
||||
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
|
||||
mnt.Enabled = false
|
||||
mnt.DesiredMounted = false
|
||||
dirty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if dirty {
|
||||
return m.writeConfig(mountsFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetPoller sets the poller for the manager
|
||||
func (m *Manager) SetPoller(poller *Poller) {
|
||||
m.poller = poller
|
||||
}
|
||||
|
||||
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
|
||||
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
|
||||
// set; remains nil-safe when stats are disabled.
|
||||
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
|
||||
m.statsPoller = sp
|
||||
}
|
||||
|
||||
// ensureDirs creates required directories
|
||||
func (m *Manager) ensureDirs() error {
|
||||
dirs := []string{
|
||||
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
|
||||
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
||||
statuses[i].PollingActive = pollingStatus.Active
|
||||
statuses[i].LastPolledScan = pollingStatus.LastScan
|
||||
statuses[i].Scanning = pollingStatus.Scanning
|
||||
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||
statuses[i].NextScanAt = pollingStatus.NextScanAt
|
||||
}
|
||||
}
|
||||
if m.statsPoller != nil {
|
||||
for i := range statuses {
|
||||
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
|
||||
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
||||
status.PollingActive = pollingStatus.Active
|
||||
status.LastPolledScan = pollingStatus.LastScan
|
||||
status.Scanning = pollingStatus.Scanning
|
||||
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||
status.NextScanAt = pollingStatus.NextScanAt
|
||||
}
|
||||
if m.statsPoller != nil {
|
||||
status.IOStats = m.statsPoller.GetStats(mount.ID)
|
||||
}
|
||||
|
||||
return status, nil
|
||||
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
return nil, fmt.Errorf("mount name is required")
|
||||
}
|
||||
|
||||
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
|
||||
// "nfs" used to be valid; the native handler is gone. Reject loudly so the
|
||||
// caller knows their integration needs to migrate to a Samba re-export or
|
||||
// switch to a supported protocol.
|
||||
if req.Type == MountType("nfs") {
|
||||
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
|
||||
}
|
||||
|
||||
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
|
||||
}
|
||||
|
||||
// Validate type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
if req.NFSServer == "" || req.NFSPath == "" {
|
||||
return nil, fmt.Errorf("NFS server and path are required")
|
||||
}
|
||||
case MountTypeSMB:
|
||||
if req.SMBServer == "" || req.SMBShare == "" {
|
||||
return nil, fmt.Errorf("SMB server and share are required")
|
||||
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
enabled = *req.Enabled
|
||||
}
|
||||
|
||||
// Default polling settings
|
||||
pollingEnabled := false
|
||||
// Polling defaults on for both supported types — neither SMB-via-rclone nor
|
||||
// generic rclone delivers kernel inotify, so a periodic re-scan is the only
|
||||
// way to pick up upstream changes. Explicit PollingEnabled in the request
|
||||
// wins.
|
||||
pollingEnabled := true
|
||||
if req.PollingEnabled != nil {
|
||||
pollingEnabled = *req.PollingEnabled
|
||||
}
|
||||
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
Enabled: enabled,
|
||||
DesiredMounted: enabled, // Auto-mount if enabled
|
||||
MountPath: mountPath,
|
||||
Options: req.Options,
|
||||
CacheMaxSize: req.CacheMaxSize,
|
||||
CacheMaxAge: req.CacheMaxAge,
|
||||
DirCacheTime: req.DirCacheTime,
|
||||
PollingEnabled: pollingEnabled,
|
||||
PollingIntervalMs: pollingIntervalMs,
|
||||
}
|
||||
|
||||
// Type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
mount.NFSServer = req.NFSServer
|
||||
mount.NFSPath = req.NFSPath
|
||||
case MountTypeSMB:
|
||||
mount.SMBServer = req.SMBServer
|
||||
mount.SMBShare = req.SMBShare
|
||||
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
|
||||
}
|
||||
}
|
||||
|
||||
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
|
||||
// don't validate format here; rclone rejects bad values at mount time.
|
||||
if val, ok := updates["cacheMaxSize"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.CacheMaxSize = s
|
||||
}
|
||||
}
|
||||
if val, ok := updates["cacheMaxAge"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.CacheMaxAge = s
|
||||
}
|
||||
}
|
||||
if val, ok := updates["dirCacheTime"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.DirCacheTime = s
|
||||
}
|
||||
}
|
||||
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+171
-34
@@ -9,6 +9,25 @@ import (
|
||||
// ScanFunc is a function type for scanning a mount path
|
||||
type ScanFunc func(path string) (int, error)
|
||||
|
||||
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
|
||||
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
|
||||
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
|
||||
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
|
||||
// large remote shares (CORN, etc.) from monopolising the network/CPU.
|
||||
const IdleFactor = 9
|
||||
|
||||
// InitialScanDelay is how long a freshly-started poller waits before its first
|
||||
// scan. Avoids hammering the system at container startup when many mounts come
|
||||
// up at once.
|
||||
const InitialScanDelay = 2 * time.Second
|
||||
|
||||
// ReconcileInterval drives the periodic resync loop that keeps poller
|
||||
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
|
||||
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
|
||||
// — without periodic reconciliation, the poller would never observe the
|
||||
// transition from `mounted=false` to `mounted=true` and never start scanning.
|
||||
const ReconcileInterval = 10 * time.Second
|
||||
|
||||
// mountPoller manages polling for a single mount
|
||||
type mountPoller struct {
|
||||
mountID string
|
||||
@@ -19,8 +38,12 @@ type mountPoller struct {
|
||||
|
||||
// PollingStatus holds the polling state for a mount
|
||||
type PollingStatus struct {
|
||||
Active bool `json:"active"`
|
||||
LastScan int64 `json:"lastScan,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
|
||||
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
|
||||
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
|
||||
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
|
||||
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
|
||||
}
|
||||
|
||||
// Poller manages per-mount polling goroutines
|
||||
@@ -31,8 +54,9 @@ type Poller struct {
|
||||
pollers map[string]*mountPoller // key: mount ID
|
||||
status map[string]*PollingStatus
|
||||
|
||||
running bool
|
||||
mu sync.RWMutex
|
||||
running bool
|
||||
reconcileStop chan struct{}
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewPoller creates a new mount poller
|
||||
@@ -54,15 +78,35 @@ func (p *Poller) Start() error {
|
||||
return nil
|
||||
}
|
||||
p.running = true
|
||||
p.reconcileStop = make(chan struct{})
|
||||
|
||||
log.Println("[Poller] Starting mount polling service")
|
||||
|
||||
// Initial sync with mount configurations
|
||||
go p.syncWithMounts()
|
||||
|
||||
// Periodic reconciliation — picks up mount-watcher.sh transitions
|
||||
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
|
||||
go p.reconcileLoop(p.reconcileStop)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
|
||||
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
|
||||
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
|
||||
ticker := time.NewTicker(ReconcileInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.syncWithMounts()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops all polling goroutines
|
||||
func (p *Poller) Stop() error {
|
||||
p.mu.Lock()
|
||||
@@ -75,6 +119,11 @@ func (p *Poller) Stop() error {
|
||||
|
||||
log.Println("[Poller] Stopping mount polling service")
|
||||
|
||||
if p.reconcileStop != nil {
|
||||
close(p.reconcileStop)
|
||||
p.reconcileStop = nil
|
||||
}
|
||||
|
||||
// Stop all pollers
|
||||
for id, poller := range p.pollers {
|
||||
p.stopPoller(poller)
|
||||
@@ -127,7 +176,7 @@ func (p *Poller) syncWithMounts() {
|
||||
existingPoller, exists := p.pollers[mount.ID]
|
||||
|
||||
if shouldPoll {
|
||||
// Calculate interval
|
||||
// Calculate interval (used as the floor for adaptive scheduling)
|
||||
interval := mount.PollingIntervalMs
|
||||
if interval < MinPollingIntervalMs {
|
||||
interval = DefaultPollingIntervalMs
|
||||
@@ -135,7 +184,7 @@ func (p *Poller) syncWithMounts() {
|
||||
|
||||
if !exists {
|
||||
// Start new poller
|
||||
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
|
||||
log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
|
||||
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
||||
}
|
||||
} else if exists {
|
||||
@@ -156,7 +205,13 @@ func (p *Poller) syncWithMounts() {
|
||||
}
|
||||
}
|
||||
|
||||
// startPollerForMount starts a polling goroutine for a mount
|
||||
// startPollerForMount starts the adaptive polling goroutine for a mount.
|
||||
//
|
||||
// The loop runs scans sequentially: scan → measure duration → sleep
|
||||
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
|
||||
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
|
||||
// cycle on slow shares, while keeping fresh checks on small mounts via the
|
||||
// configured interval floor.
|
||||
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
||||
poller := &mountPoller{
|
||||
mountID: mountID,
|
||||
@@ -170,17 +225,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
|
||||
}
|
||||
p.status[mountID].Active = true
|
||||
|
||||
floor := time.Duration(intervalMs) * time.Millisecond
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
// Initial delay so a freshly-mounted share doesn't get hammered the
|
||||
// instant the poller starts.
|
||||
nextDelay := InitialScanDelay
|
||||
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-poller.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.runPollScan(mountID, mountPath)
|
||||
case <-time.After(nextDelay):
|
||||
}
|
||||
|
||||
// Re-check stop after the wake-up to avoid running a scan during shutdown.
|
||||
select {
|
||||
case <-poller.stopChan:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
duration, err := p.executeScan(mountID, mountPath)
|
||||
if err != nil {
|
||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||
}
|
||||
|
||||
// Adaptive backoff: idle for IdleFactor × last duration, with the
|
||||
// configured interval as a floor so tiny mounts still scan often.
|
||||
nextDelay = duration * IdleFactor
|
||||
if nextDelay < floor {
|
||||
nextDelay = floor
|
||||
}
|
||||
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -199,54 +277,109 @@ func (p *Poller) stopPoller(poller *mountPoller) {
|
||||
// Update status
|
||||
if status, exists := p.status[poller.mountID]; exists {
|
||||
status.Active = false
|
||||
status.NextScanAt = 0
|
||||
}
|
||||
}
|
||||
|
||||
// runPollScan executes a scan for a mount
|
||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||
// executeScan runs scanFunc with timing + status tracking. Both the adaptive
|
||||
// loop and manual TriggerScan funnel through here so the UI sees a consistent
|
||||
// "scanning / last duration / last scan" picture regardless of which path
|
||||
// kicked it off. Returns the scan duration so the caller can compute the next
|
||||
// adaptive delay.
|
||||
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
|
||||
if p.scanFunc == nil {
|
||||
return
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
startMs := start.UnixMilli()
|
||||
|
||||
p.mu.Lock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.Scanning = true
|
||||
status.CurrentScanStartedAt = startMs
|
||||
p.mu.Unlock()
|
||||
|
||||
fileCount, err := p.scanFunc(mountPath)
|
||||
if err != nil {
|
||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update status
|
||||
end := time.Now()
|
||||
duration := end.Sub(start)
|
||||
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
status.Scanning = false
|
||||
status.CurrentScanStartedAt = 0
|
||||
status.LastScan = end.UnixMilli()
|
||||
status.LastScanDurationMs = duration.Milliseconds()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
|
||||
if err == nil {
|
||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
return duration, err
|
||||
}
|
||||
|
||||
// TriggerScan triggers an immediate scan for a mount
|
||||
// setNextScanAt records the projected start time of the next adaptive scan
|
||||
// (used purely for the UI countdown — the loop's own timer is authoritative).
|
||||
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.NextScanAt = at.UnixMilli()
|
||||
}
|
||||
|
||||
// runPollScan is kept for backwards compatibility with any external caller —
|
||||
// new code should go through executeScan via the adaptive loop.
|
||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||
_, _ = p.executeScan(mountID, mountPath)
|
||||
}
|
||||
|
||||
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
|
||||
// Runs synchronously; concurrent calls with the adaptive loop are possible
|
||||
// but rare in practice — both paths update the same status atomically.
|
||||
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
||||
if p.scanFunc == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
startMs := start.UnixMilli()
|
||||
|
||||
p.mu.Lock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.Scanning = true
|
||||
status.CurrentScanStartedAt = startMs
|
||||
p.mu.Unlock()
|
||||
|
||||
fileCount, err := p.scanFunc(mountPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
end := time.Now()
|
||||
duration := end.Sub(start)
|
||||
|
||||
// Update status
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
} else {
|
||||
p.status[mountID] = &PollingStatus{
|
||||
LastScan: NowMS(),
|
||||
}
|
||||
status.Scanning = false
|
||||
status.CurrentScanStartedAt = 0
|
||||
status.LastScan = end.UnixMilli()
|
||||
status.LastScanDurationMs = duration.Milliseconds()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
return fileCount, nil
|
||||
return fileCount, err
|
||||
}
|
||||
|
||||
// GetPollingStatus returns the polling status for a mount
|
||||
@@ -256,8 +389,12 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
|
||||
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
return &PollingStatus{
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
Scanning: status.Scanning,
|
||||
CurrentScanStartedAt: status.CurrentScanStartedAt,
|
||||
LastScanDurationMs: status.LastScanDurationMs,
|
||||
NextScanAt: status.NextScanAt,
|
||||
}
|
||||
}
|
||||
return &PollingStatus{}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
|
||||
// time. The poller keeps the previous sample per mount and computes rates
|
||||
// (Bps, transfers/sec) from the delta to the latest sample.
|
||||
//
|
||||
// Post-consolidation: rclone is the only sample source, since all mount types
|
||||
// execute through the local rclone daemon.
|
||||
type StatsSample struct {
|
||||
TakenAt time.Time
|
||||
Source string // "rclone"
|
||||
BytesRead int64
|
||||
BytesWritten int64
|
||||
ReadOps int64
|
||||
WriteOps int64
|
||||
// File-level transfer count from rclone's RC API. Distinct from ReadOps
|
||||
// (which is unused for rclone — kernel ops counters are not exposed
|
||||
// through FUSE).
|
||||
Transfers int64
|
||||
}
|
||||
|
||||
// rcloneCoreStats matches the RC API response from POST /core/stats.
|
||||
type rcloneCoreStats struct {
|
||||
Bytes int64 `json:"bytes"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
Transfers int64 `json:"transfers"`
|
||||
TotalTransfers int64 `json:"totalTransfers"`
|
||||
}
|
||||
|
||||
// SampleRclone queries the local rclone RC API for the cumulative bytes +
|
||||
// transfers counters. The numbers are global across all active mounts on this
|
||||
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
|
||||
// sample for every mounted entry on a tick.
|
||||
func SampleRclone(ctx context.Context) (*StatsSample, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build rclone stats request: %w", err)
|
||||
}
|
||||
req.SetBasicAuth("admin", "admin")
|
||||
|
||||
client := &http.Client{Timeout: 3 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var stats rcloneCoreStats
|
||||
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||
return nil, fmt.Errorf("decode rclone stats: %w", err)
|
||||
}
|
||||
|
||||
return &StatsSample{
|
||||
Source: "rclone",
|
||||
TakenAt: time.Now(),
|
||||
BytesRead: stats.Bytes,
|
||||
Transfers: stats.Transfers,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
|
||||
// spot — short enough that throughput numbers feel live, long enough that
|
||||
// /proc reads aren't a load source themselves.
|
||||
const StatsInterval = 5 * time.Second
|
||||
|
||||
// StatsPoller samples cumulative counters from the local rclone RC API and
|
||||
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
|
||||
//
|
||||
// Post-consolidation note: every mount type executes through rclone, so
|
||||
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
|
||||
// rclone is the only counter source. core/stats is daemon-global across all
|
||||
// active mounts; the same sample is applied to each mount's status. If
|
||||
// per-mount byte counters become important, the path forward is rclone stats
|
||||
// groups (out of scope for this consolidation).
|
||||
type StatsPoller struct {
|
||||
manager *Manager
|
||||
|
||||
mu sync.RWMutex
|
||||
lastSamples map[string]*StatsSample // mountID -> previous raw sample
|
||||
latestStats map[string]*MountIOStats
|
||||
stopChan chan struct{}
|
||||
running bool
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
// NewStatsPoller constructs a StatsPoller backed by the given manager.
|
||||
func NewStatsPoller(manager *Manager) *StatsPoller {
|
||||
return &StatsPoller{
|
||||
manager: manager,
|
||||
lastSamples: make(map[string]*StatsSample),
|
||||
latestStats: make(map[string]*MountIOStats),
|
||||
pollInterval: StatsInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches the sampling loop. Idempotent.
|
||||
func (p *StatsPoller) Start() {
|
||||
p.mu.Lock()
|
||||
if p.running {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.running = true
|
||||
p.stopChan = make(chan struct{})
|
||||
stop := p.stopChan
|
||||
p.mu.Unlock()
|
||||
|
||||
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
|
||||
go p.loop(stop)
|
||||
}
|
||||
|
||||
// Stop terminates the sampling loop. Idempotent.
|
||||
func (p *StatsPoller) Stop() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.running {
|
||||
return
|
||||
}
|
||||
p.running = false
|
||||
if p.stopChan != nil {
|
||||
close(p.stopChan)
|
||||
p.stopChan = nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns the latest computed IO stats for a mount, or nil if no
|
||||
// stats have been observed yet.
|
||||
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
s, ok := p.latestStats[mountID]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
// Return a copy so the caller can't mutate our cache.
|
||||
cp := *s
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (p *StatsPoller) loop(stop <-chan struct{}) {
|
||||
// Take the first sample immediately so rates start populating on the
|
||||
// second tick instead of waiting two intervals.
|
||||
p.tick()
|
||||
|
||||
t := time.NewTicker(p.pollInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
p.tick()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StatsPoller) tick() {
|
||||
mounts, err := p.manager.listMountsWithoutPollingStatus()
|
||||
if err != nil {
|
||||
log.Printf("[StatsPoller] listMounts failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Sample rclone once per tick — its RC API gives global stats across all
|
||||
// active rclone mounts on this host. Both supported mount types (smb,
|
||||
// rclone) execute through the same rclone daemon, so the same sample is
|
||||
// applied to every mounted entry.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
sample, err := SampleRclone(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[StatsPoller] rclone sample failed: %v", err)
|
||||
return
|
||||
}
|
||||
if sample == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, m := range mounts {
|
||||
if !m.Mounted {
|
||||
continue
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
prev := p.lastSamples[m.ID]
|
||||
p.lastSamples[m.ID] = sample
|
||||
stats := computeStats(prev, sample)
|
||||
p.latestStats[m.ID] = stats
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// computeStats turns a (previous, current) raw-sample pair into the rate-
|
||||
// based MountIOStats the API exposes. With no previous sample the rates are
|
||||
// zero — operators will see cumulative counters but Bps/IOPS only on the
|
||||
// second poll.
|
||||
func computeStats(prev, cur *StatsSample) *MountIOStats {
|
||||
out := &MountIOStats{
|
||||
Source: cur.Source,
|
||||
BytesRead: cur.BytesRead,
|
||||
BytesWritten: cur.BytesWritten,
|
||||
ReadOps: cur.ReadOps,
|
||||
WriteOps: cur.WriteOps,
|
||||
LastSampleAt: cur.TakenAt.UnixMilli(),
|
||||
}
|
||||
if prev == nil {
|
||||
return out
|
||||
}
|
||||
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
|
||||
if dt <= 0 {
|
||||
return out
|
||||
}
|
||||
out.IntervalMs = int64(dt * 1000)
|
||||
|
||||
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
|
||||
// negative deltas as "no movement" rather than letting the rate go
|
||||
// negative or implausibly huge.
|
||||
if d := cur.BytesRead - prev.BytesRead; d > 0 {
|
||||
out.ReadBps = float64(d) / dt
|
||||
}
|
||||
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
|
||||
out.WriteBps = float64(d) / dt
|
||||
}
|
||||
if d := cur.ReadOps - prev.ReadOps; d > 0 {
|
||||
out.ReadIops = float64(d) / dt
|
||||
}
|
||||
if d := cur.WriteOps - prev.WriteOps; d > 0 {
|
||||
out.WriteIops = float64(d) / dt
|
||||
}
|
||||
if cur.Source == "rclone" {
|
||||
if d := cur.Transfers - prev.Transfers; d > 0 {
|
||||
out.TransfersPerSec = float64(d) / dt
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
+67
-17
@@ -2,11 +2,19 @@ package mounts
|
||||
|
||||
import "time"
|
||||
|
||||
// MountType represents the type of mount
|
||||
// MountType represents the type of mount.
|
||||
//
|
||||
// All types execute through the local rclone daemon. "smb" is a UX-level alias
|
||||
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
|
||||
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
|
||||
// the generic escape hatch for any pre-defined remote in rclone.conf.
|
||||
//
|
||||
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
|
||||
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
|
||||
// regardless.
|
||||
type MountType string
|
||||
|
||||
const (
|
||||
MountTypeNFS MountType = "nfs"
|
||||
MountTypeSMB MountType = "smb"
|
||||
MountTypeRclone MountType = "rclone"
|
||||
)
|
||||
@@ -17,6 +25,15 @@ const (
|
||||
MinPollingIntervalMs = 5000 // 5 seconds
|
||||
)
|
||||
|
||||
// VFS cache defaults — applied per-mount when the caller leaves the field
|
||||
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
|
||||
// Duration) untouched.
|
||||
const (
|
||||
DefaultCacheMaxSize = "50G"
|
||||
DefaultCacheMaxAge = "24h"
|
||||
DefaultDirCacheTime = "5m"
|
||||
)
|
||||
|
||||
// MountConfig represents a mount configuration
|
||||
type MountConfig struct {
|
||||
// Common fields
|
||||
@@ -26,23 +43,26 @@ type MountConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DesiredMounted bool `json:"desiredMounted"`
|
||||
MountPath string `json:"mountPath"`
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS-specific fields
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
|
||||
// SMB-specific fields
|
||||
// SMB-specific fields (also valid for type=smb routed through rclone-smb)
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
SMBShare string `json:"smbShare,omitempty"`
|
||||
SMBUsername string `json:"smbUsername,omitempty"`
|
||||
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
||||
SMBDomain string `json:"smbDomain,omitempty"`
|
||||
|
||||
// rclone-specific fields
|
||||
// rclone-specific fields (escape hatch — references a pre-defined remote)
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
|
||||
// VFS cache configuration — applied to the rclone mount call. All three
|
||||
// fields are optional; blanks fall back to Default* constants above.
|
||||
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
|
||||
// DirCacheTime use rclone's Duration ("24h", "30m").
|
||||
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||
|
||||
// Polling configuration
|
||||
PollingEnabled bool `json:"pollingEnabled"`
|
||||
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
||||
@@ -56,6 +76,36 @@ type MountStatus struct {
|
||||
LastChecked int64 `json:"lastChecked"`
|
||||
PollingActive bool `json:"pollingActive,omitempty"`
|
||||
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
||||
// Progress tracking — surface the poller's adaptive scan state to the UI.
|
||||
Scanning bool `json:"scanning,omitempty"`
|
||||
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
|
||||
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
|
||||
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
|
||||
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
|
||||
// /proc/self/mountstats, or rclone's RC API depending on mount type.
|
||||
IOStats *MountIOStats `json:"ioStats,omitempty"`
|
||||
}
|
||||
|
||||
// MountIOStats captures throughput + operation rate for a single mount.
|
||||
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
|
||||
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
|
||||
// the last two samples divided by the interval.
|
||||
type MountIOStats struct {
|
||||
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
|
||||
BytesRead int64 `json:"bytesRead"`
|
||||
BytesWritten int64 `json:"bytesWritten,omitempty"`
|
||||
ReadBps float64 `json:"readBps"`
|
||||
WriteBps float64 `json:"writeBps,omitempty"`
|
||||
ReadOps int64 `json:"readOps,omitempty"`
|
||||
WriteOps int64 `json:"writeOps,omitempty"`
|
||||
// ReadIops and WriteIops are op counters per second — number of read/write
|
||||
// requests issued by the kernel against the share. Distinct from
|
||||
// TransfersPerSec which is a file-level metric only relevant to rclone.
|
||||
ReadIops float64 `json:"readIops,omitempty"`
|
||||
WriteIops float64 `json:"writeIops,omitempty"`
|
||||
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
|
||||
LastSampleAt int64 `json:"lastSampleAt"` // ms
|
||||
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
|
||||
}
|
||||
|
||||
// MountsFile represents the mounts.json file structure
|
||||
@@ -72,14 +122,9 @@ type RcloneRemote struct {
|
||||
|
||||
// CreateMountRequest is the request body for creating a mount
|
||||
type CreateMountRequest struct {
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||
|
||||
// SMB
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
|
||||
// VFS cache (optional — falls back to Default* constants when unset)
|
||||
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||
|
||||
// Polling
|
||||
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
||||
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
||||
|
||||
Reference in New Issue
Block a user