improved kv metacore

This commit is contained in:
Pierre
2026-04-27 10:31:11 +02:00
parent 98da535335
commit bd833b7be4
33 changed files with 3091 additions and 318 deletions
+98 -17
View File
@@ -19,11 +19,12 @@ import (
// Manager handles mount configuration and status
type Manager struct {
config *config.Config
mu sync.RWMutex
filesPath string
mountsDir string
poller *Poller
config *config.Config
mu sync.RWMutex
filesPath string
mountsDir string
poller *Poller
statsPoller *StatsPoller
}
// NewManager creates a new mount manager
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
return nil, fmt.Errorf("failed to create mount directories: %w", err)
}
// One-shot migration for legacy NFS entries — disable rather than fail to
// start, so operators can fix their config at their own pace.
if err := m.migrateLegacyMounts(); err != nil {
log.Printf("[Mounts] legacy migration warning: %v", err)
}
return m, nil
}
// migrateLegacyMounts disables any mount entries with type="nfs". The native
// NFS handler was removed when the project consolidated on rclone; an NFS
// entry left enabled would loop the watcher with mount errors.
func (m *Manager) migrateLegacyMounts() error {
mountsFile, err := m.readConfig()
if err != nil {
return err
}
dirty := false
for i := range mountsFile.Mounts {
mnt := &mountsFile.Mounts[i]
if string(mnt.Type) == "nfs" {
if mnt.Enabled || mnt.DesiredMounted {
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
mnt.Enabled = false
mnt.DesiredMounted = false
dirty = true
}
}
}
if dirty {
return m.writeConfig(mountsFile)
}
return nil
}
// SetPoller sets the poller for the manager
func (m *Manager) SetPoller(poller *Poller) {
m.poller = poller
}
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
// set; remains nil-safe when stats are disabled.
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
m.statsPoller = sp
}
// ensureDirs creates required directories
func (m *Manager) ensureDirs() error {
dirs := []string{
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
statuses[i].PollingActive = pollingStatus.Active
statuses[i].LastPolledScan = pollingStatus.LastScan
statuses[i].Scanning = pollingStatus.Scanning
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
statuses[i].NextScanAt = pollingStatus.NextScanAt
}
}
if m.statsPoller != nil {
for i := range statuses {
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
}
}
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
pollingStatus := m.poller.GetPollingStatus(mount.ID)
status.PollingActive = pollingStatus.Active
status.LastPolledScan = pollingStatus.LastScan
status.Scanning = pollingStatus.Scanning
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
status.NextScanAt = pollingStatus.NextScanAt
}
if m.statsPoller != nil {
status.IOStats = m.statsPoller.GetStats(mount.ID)
}
return status, nil
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
return nil, fmt.Errorf("mount name is required")
}
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
// "nfs" used to be valid; the native handler is gone. Reject loudly so the
// caller knows their integration needs to migrate to a Samba re-export or
// switch to a supported protocol.
if req.Type == MountType("nfs") {
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
}
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
}
// Validate type-specific fields
switch req.Type {
case MountTypeNFS:
if req.NFSServer == "" || req.NFSPath == "" {
return nil, fmt.Errorf("NFS server and path are required")
}
case MountTypeSMB:
if req.SMBServer == "" || req.SMBShare == "" {
return nil, fmt.Errorf("SMB server and share are required")
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
enabled = *req.Enabled
}
// Default polling settings
pollingEnabled := false
// Polling defaults on for both supported types — neither SMB-via-rclone nor
// generic rclone delivers kernel inotify, so a periodic re-scan is the only
// way to pick up upstream changes. Explicit PollingEnabled in the request
// wins.
pollingEnabled := true
if req.PollingEnabled != nil {
pollingEnabled = *req.PollingEnabled
}
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
Enabled: enabled,
DesiredMounted: enabled, // Auto-mount if enabled
MountPath: mountPath,
Options: req.Options,
CacheMaxSize: req.CacheMaxSize,
CacheMaxAge: req.CacheMaxAge,
DirCacheTime: req.DirCacheTime,
PollingEnabled: pollingEnabled,
PollingIntervalMs: pollingIntervalMs,
}
// Type-specific fields
switch req.Type {
case MountTypeNFS:
mount.NFSServer = req.NFSServer
mount.NFSPath = req.NFSPath
case MountTypeSMB:
mount.SMBServer = req.SMBServer
mount.SMBShare = req.SMBShare
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
}
}
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
// don't validate format here; rclone rejects bad values at mount time.
if val, ok := updates["cacheMaxSize"]; ok {
if s, ok := val.(string); ok {
mount.CacheMaxSize = s
}
}
if val, ok := updates["cacheMaxAge"]; ok {
if s, ok := val.(string); ok {
mount.CacheMaxAge = s
}
}
if val, ok := updates["dirCacheTime"]; ok {
if s, ok := val.(string); ok {
mount.DirCacheTime = s
}
}
if err := m.writeConfig(mountsFile); err != nil {
return nil, err
}
+171 -34
View File
@@ -9,6 +9,25 @@ import (
// ScanFunc is a function type for scanning a mount path
type ScanFunc func(path string) (int, error)
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
// large remote shares (CORN, etc.) from monopolising the network/CPU.
const IdleFactor = 9
// InitialScanDelay is how long a freshly-started poller waits before its first
// scan. Avoids hammering the system at container startup when many mounts come
// up at once.
const InitialScanDelay = 2 * time.Second
// ReconcileInterval drives the periodic resync loop that keeps poller
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
// — without periodic reconciliation, the poller would never observe the
// transition from `mounted=false` to `mounted=true` and never start scanning.
const ReconcileInterval = 10 * time.Second
// mountPoller manages polling for a single mount
type mountPoller struct {
mountID string
@@ -19,8 +38,12 @@ type mountPoller struct {
// PollingStatus holds the polling state for a mount
type PollingStatus struct {
Active bool `json:"active"`
LastScan int64 `json:"lastScan,omitempty"`
Active bool `json:"active"`
LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
}
// Poller manages per-mount polling goroutines
@@ -31,8 +54,9 @@ type Poller struct {
pollers map[string]*mountPoller // key: mount ID
status map[string]*PollingStatus
running bool
mu sync.RWMutex
running bool
reconcileStop chan struct{}
mu sync.RWMutex
}
// NewPoller creates a new mount poller
@@ -54,15 +78,35 @@ func (p *Poller) Start() error {
return nil
}
p.running = true
p.reconcileStop = make(chan struct{})
log.Println("[Poller] Starting mount polling service")
// Initial sync with mount configurations
go p.syncWithMounts()
// Periodic reconciliation — picks up mount-watcher.sh transitions
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
go p.reconcileLoop(p.reconcileStop)
return nil
}
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
ticker := time.NewTicker(ReconcileInterval)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
p.syncWithMounts()
}
}
}
// Stop stops all polling goroutines
func (p *Poller) Stop() error {
p.mu.Lock()
@@ -75,6 +119,11 @@ func (p *Poller) Stop() error {
log.Println("[Poller] Stopping mount polling service")
if p.reconcileStop != nil {
close(p.reconcileStop)
p.reconcileStop = nil
}
// Stop all pollers
for id, poller := range p.pollers {
p.stopPoller(poller)
@@ -127,7 +176,7 @@ func (p *Poller) syncWithMounts() {
existingPoller, exists := p.pollers[mount.ID]
if shouldPoll {
// Calculate interval
// Calculate interval (used as the floor for adaptive scheduling)
interval := mount.PollingIntervalMs
if interval < MinPollingIntervalMs {
interval = DefaultPollingIntervalMs
@@ -135,7 +184,7 @@ func (p *Poller) syncWithMounts() {
if !exists {
// Start new poller
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
p.startPollerForMount(mount.ID, mount.MountPath, interval)
}
} else if exists {
@@ -156,7 +205,13 @@ func (p *Poller) syncWithMounts() {
}
}
// startPollerForMount starts a polling goroutine for a mount
// startPollerForMount starts the adaptive polling goroutine for a mount.
//
// The loop runs scans sequentially: scan → measure duration → sleep
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
// cycle on slow shares, while keeping fresh checks on small mounts via the
// configured interval floor.
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
poller := &mountPoller{
mountID: mountID,
@@ -170,17 +225,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
}
p.status[mountID].Active = true
floor := time.Duration(intervalMs) * time.Millisecond
go func() {
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
defer ticker.Stop()
// Initial delay so a freshly-mounted share doesn't get hammered the
// instant the poller starts.
nextDelay := InitialScanDelay
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
for {
select {
case <-poller.stopChan:
return
case <-ticker.C:
p.runPollScan(mountID, mountPath)
case <-time.After(nextDelay):
}
// Re-check stop after the wake-up to avoid running a scan during shutdown.
select {
case <-poller.stopChan:
return
default:
}
duration, err := p.executeScan(mountID, mountPath)
if err != nil {
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
}
// Adaptive backoff: idle for IdleFactor × last duration, with the
// configured interval as a floor so tiny mounts still scan often.
nextDelay = duration * IdleFactor
if nextDelay < floor {
nextDelay = floor
}
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
}
}()
}
@@ -199,54 +277,109 @@ func (p *Poller) stopPoller(poller *mountPoller) {
// Update status
if status, exists := p.status[poller.mountID]; exists {
status.Active = false
status.NextScanAt = 0
}
}
// runPollScan executes a scan for a mount
func (p *Poller) runPollScan(mountID, mountPath string) {
// executeScan runs scanFunc with timing + status tracking. Both the adaptive
// loop and manual TriggerScan funnel through here so the UI sees a consistent
// "scanning / last duration / last scan" picture regardless of which path
// kicked it off. Returns the scan duration so the caller can compute the next
// adaptive delay.
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
if p.scanFunc == nil {
return
return 0, nil
}
start := time.Now()
startMs := start.UnixMilli()
p.mu.Lock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.Scanning = true
status.CurrentScanStartedAt = startMs
p.mu.Unlock()
fileCount, err := p.scanFunc(mountPath)
if err != nil {
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
return
}
// Update status
end := time.Now()
duration := end.Sub(start)
p.mu.Lock()
if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS()
status.Scanning = false
status.CurrentScanStartedAt = 0
status.LastScan = end.UnixMilli()
status.LastScanDurationMs = duration.Milliseconds()
}
p.mu.Unlock()
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
if err == nil {
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
}
return duration, err
}
// TriggerScan triggers an immediate scan for a mount
// setNextScanAt records the projected start time of the next adaptive scan
// (used purely for the UI countdown — the loop's own timer is authoritative).
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
p.mu.Lock()
defer p.mu.Unlock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.NextScanAt = at.UnixMilli()
}
// runPollScan is kept for backwards compatibility with any external caller —
// new code should go through executeScan via the adaptive loop.
func (p *Poller) runPollScan(mountID, mountPath string) {
_, _ = p.executeScan(mountID, mountPath)
}
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
// Runs synchronously; concurrent calls with the adaptive loop are possible
// but rare in practice — both paths update the same status atomically.
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
if p.scanFunc == nil {
return 0, nil
}
start := time.Now()
startMs := start.UnixMilli()
p.mu.Lock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.Scanning = true
status.CurrentScanStartedAt = startMs
p.mu.Unlock()
fileCount, err := p.scanFunc(mountPath)
if err != nil {
return 0, err
}
end := time.Now()
duration := end.Sub(start)
// Update status
p.mu.Lock()
if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS()
} else {
p.status[mountID] = &PollingStatus{
LastScan: NowMS(),
}
status.Scanning = false
status.CurrentScanStartedAt = 0
status.LastScan = end.UnixMilli()
status.LastScanDurationMs = duration.Milliseconds()
}
p.mu.Unlock()
return fileCount, nil
return fileCount, err
}
// GetPollingStatus returns the polling status for a mount
@@ -256,8 +389,12 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
if status, exists := p.status[mountID]; exists {
return &PollingStatus{
Active: status.Active,
LastScan: status.LastScan,
Active: status.Active,
LastScan: status.LastScan,
Scanning: status.Scanning,
CurrentScanStartedAt: status.CurrentScanStartedAt,
LastScanDurationMs: status.LastScanDurationMs,
NextScanAt: status.NextScanAt,
}
}
return &PollingStatus{}
+70
View File
@@ -0,0 +1,70 @@
package mounts
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
)
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
// time. The poller keeps the previous sample per mount and computes rates
// (Bps, transfers/sec) from the delta to the latest sample.
//
// Post-consolidation: rclone is the only sample source, since all mount types
// execute through the local rclone daemon.
type StatsSample struct {
TakenAt time.Time
Source string // "rclone"
BytesRead int64
BytesWritten int64
ReadOps int64
WriteOps int64
// File-level transfer count from rclone's RC API. Distinct from ReadOps
// (which is unused for rclone — kernel ops counters are not exposed
// through FUSE).
Transfers int64
}
// rcloneCoreStats matches the RC API response from POST /core/stats.
type rcloneCoreStats struct {
Bytes int64 `json:"bytes"`
TotalBytes int64 `json:"totalBytes"`
Transfers int64 `json:"transfers"`
TotalTransfers int64 `json:"totalTransfers"`
}
// SampleRclone queries the local rclone RC API for the cumulative bytes +
// transfers counters. The numbers are global across all active mounts on this
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
// sample for every mounted entry on a tick.
func SampleRclone(ctx context.Context) (*StatsSample, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
if err != nil {
return nil, fmt.Errorf("build rclone stats request: %w", err)
}
req.SetBasicAuth("admin", "admin")
client := &http.Client{Timeout: 3 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
}
var stats rcloneCoreStats
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
return nil, fmt.Errorf("decode rclone stats: %w", err)
}
return &StatsSample{
Source: "rclone",
TakenAt: time.Now(),
BytesRead: stats.Bytes,
Transfers: stats.Transfers,
}, nil
}
+185
View File
@@ -0,0 +1,185 @@
package mounts
import (
"context"
"log"
"sync"
"time"
)
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
// spot — short enough that throughput numbers feel live, long enough that
// /proc reads aren't a load source themselves.
const StatsInterval = 5 * time.Second
// StatsPoller samples cumulative counters from the local rclone RC API and
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
//
// Post-consolidation note: every mount type executes through rclone, so
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
// rclone is the only counter source. core/stats is daemon-global across all
// active mounts; the same sample is applied to each mount's status. If
// per-mount byte counters become important, the path forward is rclone stats
// groups (out of scope for this consolidation).
type StatsPoller struct {
manager *Manager
mu sync.RWMutex
lastSamples map[string]*StatsSample // mountID -> previous raw sample
latestStats map[string]*MountIOStats
stopChan chan struct{}
running bool
pollInterval time.Duration
}
// NewStatsPoller constructs a StatsPoller backed by the given manager.
func NewStatsPoller(manager *Manager) *StatsPoller {
return &StatsPoller{
manager: manager,
lastSamples: make(map[string]*StatsSample),
latestStats: make(map[string]*MountIOStats),
pollInterval: StatsInterval,
}
}
// Start launches the sampling loop. Idempotent.
func (p *StatsPoller) Start() {
p.mu.Lock()
if p.running {
p.mu.Unlock()
return
}
p.running = true
p.stopChan = make(chan struct{})
stop := p.stopChan
p.mu.Unlock()
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
go p.loop(stop)
}
// Stop terminates the sampling loop. Idempotent.
func (p *StatsPoller) Stop() {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
return
}
p.running = false
if p.stopChan != nil {
close(p.stopChan)
p.stopChan = nil
}
}
// GetStats returns the latest computed IO stats for a mount, or nil if no
// stats have been observed yet.
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
p.mu.RLock()
defer p.mu.RUnlock()
s, ok := p.latestStats[mountID]
if !ok {
return nil
}
// Return a copy so the caller can't mutate our cache.
cp := *s
return &cp
}
func (p *StatsPoller) loop(stop <-chan struct{}) {
// Take the first sample immediately so rates start populating on the
// second tick instead of waiting two intervals.
p.tick()
t := time.NewTicker(p.pollInterval)
defer t.Stop()
for {
select {
case <-stop:
return
case <-t.C:
p.tick()
}
}
}
func (p *StatsPoller) tick() {
mounts, err := p.manager.listMountsWithoutPollingStatus()
if err != nil {
log.Printf("[StatsPoller] listMounts failed: %v", err)
return
}
// Sample rclone once per tick — its RC API gives global stats across all
// active rclone mounts on this host. Both supported mount types (smb,
// rclone) execute through the same rclone daemon, so the same sample is
// applied to every mounted entry.
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
sample, err := SampleRclone(ctx)
if err != nil {
log.Printf("[StatsPoller] rclone sample failed: %v", err)
return
}
if sample == nil {
return
}
for _, m := range mounts {
if !m.Mounted {
continue
}
p.mu.Lock()
prev := p.lastSamples[m.ID]
p.lastSamples[m.ID] = sample
stats := computeStats(prev, sample)
p.latestStats[m.ID] = stats
p.mu.Unlock()
}
}
// computeStats turns a (previous, current) raw-sample pair into the rate-
// based MountIOStats the API exposes. With no previous sample the rates are
// zero — operators will see cumulative counters but Bps/IOPS only on the
// second poll.
func computeStats(prev, cur *StatsSample) *MountIOStats {
out := &MountIOStats{
Source: cur.Source,
BytesRead: cur.BytesRead,
BytesWritten: cur.BytesWritten,
ReadOps: cur.ReadOps,
WriteOps: cur.WriteOps,
LastSampleAt: cur.TakenAt.UnixMilli(),
}
if prev == nil {
return out
}
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
if dt <= 0 {
return out
}
out.IntervalMs = int64(dt * 1000)
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
// negative deltas as "no movement" rather than letting the rate go
// negative or implausibly huge.
if d := cur.BytesRead - prev.BytesRead; d > 0 {
out.ReadBps = float64(d) / dt
}
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
out.WriteBps = float64(d) / dt
}
if d := cur.ReadOps - prev.ReadOps; d > 0 {
out.ReadIops = float64(d) / dt
}
if d := cur.WriteOps - prev.WriteOps; d > 0 {
out.WriteIops = float64(d) / dt
}
if cur.Source == "rclone" {
if d := cur.Transfers - prev.Transfers; d > 0 {
out.TransfersPerSec = float64(d) / dt
}
}
return out
}
+67 -17
View File
@@ -2,11 +2,19 @@ package mounts
import "time"
// MountType represents the type of mount
// MountType represents the type of mount.
//
// All types execute through the local rclone daemon. "smb" is a UX-level alias
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
// the generic escape hatch for any pre-defined remote in rclone.conf.
//
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
// regardless.
type MountType string
const (
MountTypeNFS MountType = "nfs"
MountTypeSMB MountType = "smb"
MountTypeRclone MountType = "rclone"
)
@@ -17,6 +25,15 @@ const (
MinPollingIntervalMs = 5000 // 5 seconds
)
// VFS cache defaults — applied per-mount when the caller leaves the field
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
// Duration) untouched.
const (
DefaultCacheMaxSize = "50G"
DefaultCacheMaxAge = "24h"
DefaultDirCacheTime = "5m"
)
// MountConfig represents a mount configuration
type MountConfig struct {
// Common fields
@@ -26,23 +43,26 @@ type MountConfig struct {
Enabled bool `json:"enabled"`
DesiredMounted bool `json:"desiredMounted"`
MountPath string `json:"mountPath"`
Options string `json:"options,omitempty"`
// NFS-specific fields
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
// SMB-specific fields
// SMB-specific fields (also valid for type=smb routed through rclone-smb)
SMBServer string `json:"smbServer,omitempty"`
SMBShare string `json:"smbShare,omitempty"`
SMBUsername string `json:"smbUsername,omitempty"`
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
SMBDomain string `json:"smbDomain,omitempty"`
// rclone-specific fields
// rclone-specific fields (escape hatch — references a pre-defined remote)
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
// VFS cache configuration — applied to the rclone mount call. All three
// fields are optional; blanks fall back to Default* constants above.
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
// DirCacheTime use rclone's Duration ("24h", "30m").
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
DirCacheTime string `json:"dirCacheTime,omitempty"`
// Polling configuration
PollingEnabled bool `json:"pollingEnabled"`
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
@@ -56,6 +76,36 @@ type MountStatus struct {
LastChecked int64 `json:"lastChecked"`
PollingActive bool `json:"pollingActive,omitempty"`
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
// Progress tracking — surface the poller's adaptive scan state to the UI.
Scanning bool `json:"scanning,omitempty"`
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
// /proc/self/mountstats, or rclone's RC API depending on mount type.
IOStats *MountIOStats `json:"ioStats,omitempty"`
}
// MountIOStats captures throughput + operation rate for a single mount.
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
// the last two samples divided by the interval.
type MountIOStats struct {
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
BytesRead int64 `json:"bytesRead"`
BytesWritten int64 `json:"bytesWritten,omitempty"`
ReadBps float64 `json:"readBps"`
WriteBps float64 `json:"writeBps,omitempty"`
ReadOps int64 `json:"readOps,omitempty"`
WriteOps int64 `json:"writeOps,omitempty"`
// ReadIops and WriteIops are op counters per second — number of read/write
// requests issued by the kernel against the share. Distinct from
// TransfersPerSec which is a file-level metric only relevant to rclone.
ReadIops float64 `json:"readIops,omitempty"`
WriteIops float64 `json:"writeIops,omitempty"`
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
LastSampleAt int64 `json:"lastSampleAt"` // ms
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
}
// MountsFile represents the mounts.json file structure
@@ -72,14 +122,9 @@ type RcloneRemote struct {
// CreateMountRequest is the request body for creating a mount
type CreateMountRequest struct {
Name string `json:"name"`
Type MountType `json:"type"`
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
Options string `json:"options,omitempty"`
// NFS
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
Name string `json:"name"`
Type MountType `json:"type"`
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
// SMB
SMBServer string `json:"smbServer,omitempty"`
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"`
// VFS cache (optional — falls back to Default* constants when unset)
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
DirCacheTime string `json:"dirCacheTime,omitempty"`
// Polling
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`