improved kv metacore
This commit is contained in:
+18
-9
@@ -7,7 +7,7 @@
|
|||||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||||
# - nginx (dashboard, WebDAV, API proxy)
|
# - nginx (dashboard, WebDAV, API proxy)
|
||||||
# - Redis (managed by supervisord on leader)
|
# - Redis (managed by supervisord on leader)
|
||||||
# - rclone (remote mount daemon)
|
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
|
||||||
# - supervisord (process management - runs on leader only)
|
# - supervisord (process management - runs on leader only)
|
||||||
#
|
#
|
||||||
# Architecture:
|
# Architecture:
|
||||||
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
|
|||||||
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
|
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
|
||||||
LABEL org.opencontainers.image.licenses=MIT
|
LABEL org.opencontainers.image.licenses=MIT
|
||||||
|
|
||||||
# Install runtime dependencies
|
# Pinned rclone release. Alpine's package can lag the latest by several
|
||||||
|
# minor versions and we depend on VFS cache fixes shipped post-1.65.
|
||||||
|
ARG RCLONE_VERSION=1.73.5
|
||||||
|
|
||||||
|
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
|
||||||
|
# all remote mounts go through rclone now.
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
# Core services
|
|
||||||
redis \
|
redis \
|
||||||
nginx \
|
nginx \
|
||||||
nginx-mod-http-dav-ext \
|
nginx-mod-http-dav-ext \
|
||||||
supervisor \
|
supervisor \
|
||||||
rclone \
|
|
||||||
fuse3 \
|
fuse3 \
|
||||||
# Mount utilities
|
|
||||||
nfs-utils \
|
|
||||||
cifs-utils \
|
|
||||||
# Tools
|
|
||||||
bash \
|
bash \
|
||||||
curl \
|
curl \
|
||||||
python3 \
|
python3 \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
apache2-utils \
|
apache2-utils \
|
||||||
# Process utilities
|
|
||||||
procps
|
procps
|
||||||
|
|
||||||
|
# Install rclone from the official binary release (pinned).
|
||||||
|
RUN set -eux; \
|
||||||
|
apk add --no-cache --virtual .rclone-deps unzip; \
|
||||||
|
cd /tmp; \
|
||||||
|
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
|
||||||
|
unzip -q rclone.zip; \
|
||||||
|
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
|
||||||
|
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
|
||||||
|
apk del .rclone-deps; \
|
||||||
|
/usr/local/bin/rclone version | head -2
|
||||||
|
|
||||||
# Copy Go binary from builder
|
# Copy Go binary from builder
|
||||||
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
|
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
|
||||||
RUN chmod +x /usr/local/bin/meta-core
|
RUN chmod +x /usr/local/bin/meta-core
|
||||||
|
|||||||
+220
-42
@@ -1,19 +1,47 @@
|
|||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
interface MountIOStats {
|
||||||
|
source: 'rclone';
|
||||||
|
bytesRead: number;
|
||||||
|
bytesWritten?: number;
|
||||||
|
readBps: number;
|
||||||
|
writeBps?: number;
|
||||||
|
readOps?: number;
|
||||||
|
writeOps?: number;
|
||||||
|
readIops?: number;
|
||||||
|
writeIops?: number;
|
||||||
|
transfersPerSec?: number;
|
||||||
|
lastSampleAt: number;
|
||||||
|
intervalMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
interface Mount {
|
interface Mount {
|
||||||
id: string;
|
id: string;
|
||||||
name: string;
|
name: string;
|
||||||
type: 'nfs' | 'smb' | 'rclone';
|
type: 'smb' | 'rclone';
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
desiredMounted: boolean;
|
desiredMounted: boolean;
|
||||||
mountPath: string;
|
mountPath: string;
|
||||||
mounted: boolean;
|
mounted: boolean;
|
||||||
error?: string;
|
error?: string;
|
||||||
lastChecked: number;
|
lastChecked: number;
|
||||||
|
// VFS cache configuration applied at mount time. Empty string means "use
|
||||||
|
// server default" (50G / 24h / 5m).
|
||||||
|
cacheMaxSize?: string;
|
||||||
|
cacheMaxAge?: string;
|
||||||
|
dirCacheTime?: string;
|
||||||
pollingEnabled?: boolean;
|
pollingEnabled?: boolean;
|
||||||
pollingIntervalMs?: number;
|
pollingIntervalMs?: number;
|
||||||
pollingActive?: boolean;
|
pollingActive?: boolean;
|
||||||
lastPolledScan?: number;
|
lastPolledScan?: number;
|
||||||
|
// Adaptive scan progress (set by the meta-core poller)
|
||||||
|
scanning?: boolean;
|
||||||
|
currentScanStartedAt?: number; // ms; only while scanning
|
||||||
|
lastScanDurationMs?: number;
|
||||||
|
nextScanAt?: number; // ms; planned start of next adaptive scan
|
||||||
|
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
|
||||||
|
// so the same numbers appear on every mount card.
|
||||||
|
ioStats?: MountIOStats;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface RcloneRemote {
|
interface RcloneRemote {
|
||||||
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
|
|||||||
background: '#7f1d1d',
|
background: '#7f1d1d',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
|
||||||
|
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
|
||||||
|
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
|
||||||
|
function formatBytes(n: number): string {
|
||||||
|
if (!isFinite(n) || n < 0) return '—';
|
||||||
|
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
let i = 0;
|
||||||
|
let v = n;
|
||||||
|
while (v >= 1000 && i < units.length - 1) {
|
||||||
|
v /= 1000;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRate(bps: number): string {
|
||||||
|
return `${formatBytes(bps)}/s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
|
||||||
|
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
|
||||||
|
// is just noise.
|
||||||
|
function formatDuration(ms: number): string {
|
||||||
|
if (ms < 0 || !isFinite(ms)) return '—';
|
||||||
|
const s = Math.floor(ms / 1000);
|
||||||
|
if (s < 60) return `${s}s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const remS = s % 60;
|
||||||
|
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
|
||||||
|
const h = Math.floor(m / 60);
|
||||||
|
const remM = m % 60;
|
||||||
|
return remM ? `${h}h ${remM}m` : `${h}h`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// IOStatsLine renders a single line of live throughput info beneath each
|
||||||
|
// mount card. Source is always rclone after the rclone-only consolidation,
|
||||||
|
// so we report file-level transfers/s rather than kernel IOPS.
|
||||||
|
function IOStatsLine({ stats }: { stats: MountIOStats }) {
|
||||||
|
const opsLabel = 'transfers/s';
|
||||||
|
const opsValue = stats.transfersPerSec;
|
||||||
|
|
||||||
|
const ageMs = Date.now() - stats.lastSampleAt;
|
||||||
|
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
|
||||||
|
|
||||||
|
return (
|
||||||
|
<p
|
||||||
|
style={{
|
||||||
|
color: stale ? '#6b7280' : '#9ca3af',
|
||||||
|
fontSize: '0.85rem',
|
||||||
|
margin: '0.25rem 0 0',
|
||||||
|
}}
|
||||||
|
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
|
||||||
|
>
|
||||||
|
↓ {formatRate(stats.readBps)}
|
||||||
|
{opsValue !== undefined && opsValue > 0 && (
|
||||||
|
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
|
||||||
|
)}
|
||||||
|
<> · {formatBytes(stats.bytesRead)} read total</>
|
||||||
|
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
|
||||||
|
<> · {formatBytes(stats.bytesWritten)} written</>
|
||||||
|
)}
|
||||||
|
</p>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ScanProgressProps {
|
||||||
|
mount: Mount;
|
||||||
|
now: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress bar shows scan ETA when running, an idle row with a countdown to
|
||||||
|
// the next adaptive scan when waiting. We can't show true % complete (we
|
||||||
|
// don't know the file count up-front), so the running bar is filled by
|
||||||
|
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
|
||||||
|
// it pins at 95% to signal "taking longer than last time".
|
||||||
|
function ScanProgress({ mount, now }: ScanProgressProps) {
|
||||||
|
if (!mount.pollingEnabled) return null;
|
||||||
|
|
||||||
|
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
|
||||||
|
const lastDuration = mount.lastScanDurationMs ?? 0;
|
||||||
|
|
||||||
|
let fill = 0;
|
||||||
|
let label = '';
|
||||||
|
|
||||||
|
if (isScanning) {
|
||||||
|
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
|
||||||
|
if (lastDuration > 0) {
|
||||||
|
fill = Math.min(0.95, elapsed / lastDuration);
|
||||||
|
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
|
||||||
|
} else {
|
||||||
|
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
|
||||||
|
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
|
||||||
|
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
|
||||||
|
if (lastDuration > 0 && mount.lastPolledScan) {
|
||||||
|
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
|
||||||
|
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
|
||||||
|
} else if (mount.nextScanAt) {
|
||||||
|
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
|
||||||
|
} else {
|
||||||
|
label = 'Polling enabled — waiting for first scan';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ marginTop: '0.5rem' }}>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: '6px',
|
||||||
|
background: '#0f3460',
|
||||||
|
borderRadius: '3px',
|
||||||
|
overflow: 'hidden',
|
||||||
|
position: 'relative',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width: `${Math.round(fill * 100)}%`,
|
||||||
|
height: '100%',
|
||||||
|
background: isScanning ? '#4ade80' : '#374151',
|
||||||
|
transition: 'width 0.5s linear',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export default function Mounts() {
|
export default function Mounts() {
|
||||||
const [mounts, setMounts] = useState<Mount[]>([]);
|
const [mounts, setMounts] = useState<Mount[]>([]);
|
||||||
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
|
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
|
||||||
@@ -50,19 +209,23 @@ export default function Mounts() {
|
|||||||
const [showForm, setShowForm] = useState(false);
|
const [showForm, setShowForm] = useState(false);
|
||||||
const [formData, setFormData] = useState({
|
const [formData, setFormData] = useState({
|
||||||
name: '',
|
name: '',
|
||||||
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
|
type: 'smb' as 'smb' | 'rclone',
|
||||||
nfsServer: '',
|
|
||||||
nfsPath: '',
|
|
||||||
smbServer: '',
|
smbServer: '',
|
||||||
smbShare: '',
|
smbShare: '',
|
||||||
smbUsername: '',
|
smbUsername: '',
|
||||||
smbPassword: '',
|
smbPassword: '',
|
||||||
rcloneRemote: '',
|
rcloneRemote: '',
|
||||||
rclonePath: '',
|
rclonePath: '',
|
||||||
pollingEnabled: false,
|
cacheMaxSize: '',
|
||||||
|
cacheMaxAge: '',
|
||||||
|
dirCacheTime: '',
|
||||||
|
pollingEnabled: true,
|
||||||
pollingIntervalMs: 30000,
|
pollingIntervalMs: 30000,
|
||||||
});
|
});
|
||||||
const [scanning, setScanning] = useState<string | null>(null);
|
const [scanning, setScanning] = useState<string | null>(null);
|
||||||
|
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
|
||||||
|
// refetches (which run every 5s).
|
||||||
|
const [now, setNow] = useState<number>(() => Date.now());
|
||||||
|
|
||||||
const fetchMounts = async () => {
|
const fetchMounts = async () => {
|
||||||
try {
|
try {
|
||||||
@@ -94,7 +257,11 @@ export default function Mounts() {
|
|||||||
fetchMounts();
|
fetchMounts();
|
||||||
fetchRemotes();
|
fetchRemotes();
|
||||||
const interval = setInterval(fetchMounts, 5000);
|
const interval = setInterval(fetchMounts, 5000);
|
||||||
return () => clearInterval(interval);
|
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||||
|
return () => {
|
||||||
|
clearInterval(interval);
|
||||||
|
clearInterval(tick);
|
||||||
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleMount = async (id: string) => {
|
const handleMount = async (id: string) => {
|
||||||
@@ -143,10 +310,7 @@ export default function Mounts() {
|
|||||||
type: formData.type,
|
type: formData.type,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (formData.type === 'nfs') {
|
if (formData.type === 'smb') {
|
||||||
body.nfsServer = formData.nfsServer;
|
|
||||||
body.nfsPath = formData.nfsPath;
|
|
||||||
} else if (formData.type === 'smb') {
|
|
||||||
body.smbServer = formData.smbServer;
|
body.smbServer = formData.smbServer;
|
||||||
body.smbShare = formData.smbShare;
|
body.smbShare = formData.smbShare;
|
||||||
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
|
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
|
||||||
@@ -156,6 +320,12 @@ export default function Mounts() {
|
|||||||
body.rclonePath = formData.rclonePath;
|
body.rclonePath = formData.rclonePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cache knobs — only forward non-empty values; the server applies defaults
|
||||||
|
// when fields are omitted.
|
||||||
|
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
|
||||||
|
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
|
||||||
|
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
|
||||||
|
|
||||||
// Add polling configuration
|
// Add polling configuration
|
||||||
body.pollingEnabled = formData.pollingEnabled;
|
body.pollingEnabled = formData.pollingEnabled;
|
||||||
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
|
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
|
||||||
@@ -171,16 +341,17 @@ export default function Mounts() {
|
|||||||
setShowForm(false);
|
setShowForm(false);
|
||||||
setFormData({
|
setFormData({
|
||||||
name: '',
|
name: '',
|
||||||
type: 'nfs',
|
type: 'smb',
|
||||||
nfsServer: '',
|
|
||||||
nfsPath: '',
|
|
||||||
smbServer: '',
|
smbServer: '',
|
||||||
smbShare: '',
|
smbShare: '',
|
||||||
smbUsername: '',
|
smbUsername: '',
|
||||||
smbPassword: '',
|
smbPassword: '',
|
||||||
rcloneRemote: '',
|
rcloneRemote: '',
|
||||||
rclonePath: '',
|
rclonePath: '',
|
||||||
pollingEnabled: false,
|
cacheMaxSize: '',
|
||||||
|
cacheMaxAge: '',
|
||||||
|
dirCacheTime: '',
|
||||||
|
pollingEnabled: true,
|
||||||
pollingIntervalMs: 30000,
|
pollingIntervalMs: 30000,
|
||||||
});
|
});
|
||||||
fetchMounts();
|
fetchMounts();
|
||||||
@@ -231,32 +402,15 @@ export default function Mounts() {
|
|||||||
<select
|
<select
|
||||||
style={inputStyle}
|
style={inputStyle}
|
||||||
value={formData.type}
|
value={formData.type}
|
||||||
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
|
onChange={(e) => {
|
||||||
|
const type = e.target.value as 'smb' | 'rclone';
|
||||||
|
setFormData({ ...formData, type });
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
<option value="nfs">NFS</option>
|
<option value="smb">SMB (via rclone)</option>
|
||||||
<option value="smb">SMB/CIFS</option>
|
<option value="rclone">rclone (pre-defined remote)</option>
|
||||||
<option value="rclone">rclone</option>
|
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
{formData.type === 'nfs' && (
|
|
||||||
<>
|
|
||||||
<input
|
|
||||||
style={inputStyle}
|
|
||||||
placeholder="NFS Server (e.g., 192.168.1.100)"
|
|
||||||
value={formData.nfsServer}
|
|
||||||
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
|
|
||||||
required
|
|
||||||
/>
|
|
||||||
<input
|
|
||||||
style={inputStyle}
|
|
||||||
placeholder="NFS Path (e.g., /export/media)"
|
|
||||||
value={formData.nfsPath}
|
|
||||||
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
|
|
||||||
required
|
|
||||||
/>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{formData.type === 'smb' && (
|
{formData.type === 'smb' && (
|
||||||
<>
|
<>
|
||||||
<input
|
<input
|
||||||
@@ -313,6 +467,30 @@ export default function Mounts() {
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||||
|
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
|
||||||
|
VFS cache (leave blank for defaults: 50G / 24h / 5m)
|
||||||
|
</p>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Cache max size (e.g., 50G)"
|
||||||
|
value={formData.cacheMaxSize}
|
||||||
|
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Cache max age (e.g., 24h)"
|
||||||
|
value={formData.cacheMaxAge}
|
||||||
|
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Dir cache time (e.g., 5m)"
|
||||||
|
value={formData.dirCacheTime}
|
||||||
|
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||||
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
|
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
|
||||||
<input
|
<input
|
||||||
@@ -362,17 +540,17 @@ export default function Mounts() {
|
|||||||
</span>
|
</span>
|
||||||
</p>
|
</p>
|
||||||
{mount.pollingEnabled && (
|
{mount.pollingEnabled && (
|
||||||
<p style={{ color: '#888', fontSize: '0.9rem' }}>
|
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
|
||||||
Polling:{' '}
|
Polling:{' '}
|
||||||
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
|
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
|
||||||
{mount.pollingActive ? 'Active' : 'Inactive'}
|
{mount.pollingActive ? 'Active' : 'Inactive'}
|
||||||
</span>
|
</span>
|
||||||
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
||||||
{mount.lastPolledScan && (
|
{' • Adaptive 10× idle'}
|
||||||
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
|
|
||||||
)}
|
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
|
||||||
|
<ScanProgress mount={mount} now={now} />
|
||||||
{mount.error && (
|
{mount.error && (
|
||||||
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
|
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
|
||||||
)}
|
)}
|
||||||
|
|||||||
+135
-167
@@ -2,11 +2,17 @@
|
|||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Mount Watcher Script
|
# Mount Watcher Script
|
||||||
#
|
#
|
||||||
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
|
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
|
||||||
# Runs as a supervisord program, polling every 5 seconds.
|
# rclone RC daemon. After the rclone-only consolidation, this script no
|
||||||
|
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
|
||||||
|
# through rclone's mount/mount endpoint.
|
||||||
#
|
#
|
||||||
# Uses findmnt to check actual mount status (source of truth).
|
# Mount types:
|
||||||
# Writes errors to /meta-core/mounts/errors/{id}.error
|
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
|
||||||
|
# - "rclone" → references a pre-defined remote in rclone.conf
|
||||||
|
#
|
||||||
|
# All mounts are read-only by construction (no API surface to override).
|
||||||
|
# Errors surface in /meta-core/mounts/errors/{id}.error
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
set -u
|
set -u
|
||||||
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
|
|||||||
FILES_PATH="${FILES_PATH:-/files}"
|
FILES_PATH="${FILES_PATH:-/files}"
|
||||||
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
||||||
|
|
||||||
|
# Fall-back VFS cache settings — applied when a mount entry leaves the field
|
||||||
|
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
|
||||||
|
# packages/meta-core/internal/mounts/types.go.
|
||||||
|
DEFAULT_CACHE_MAX_SIZE="50G"
|
||||||
|
DEFAULT_CACHE_MAX_AGE="24h"
|
||||||
|
DEFAULT_DIR_CACHE_TIME="5m"
|
||||||
|
|
||||||
|
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
|
||||||
|
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
|
||||||
|
|
||||||
# ANSI colors
|
# ANSI colors
|
||||||
RED='\033[0;31m'
|
RED='\033[0;31m'
|
||||||
GREEN='\033[0;32m'
|
GREEN='\033[0;32m'
|
||||||
@@ -54,9 +70,7 @@ init_dirs() {
|
|||||||
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
|
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
|
||||||
is_mounted() {
|
is_mounted() {
|
||||||
local mount_path="$1"
|
local mount_path="$1"
|
||||||
# Remove trailing slash for consistent matching
|
|
||||||
local clean_path="${mount_path%/}"
|
local clean_path="${mount_path%/}"
|
||||||
# Check if the path appears as a mount point in /proc/mounts (field 2)
|
|
||||||
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
|
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,145 +88,102 @@ write_error() {
|
|||||||
echo "$error" >> "$ERROR_DIR/$id.error"
|
echo "$error" >> "$ERROR_DIR/$id.error"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mount NFS share (always read-only)
|
# Build the rclone remote source string for a mount entry. Echoes the result.
|
||||||
do_mount_nfs() {
|
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
|
||||||
local id="$1"
|
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
|
||||||
local server="$2"
|
# the optional remote path.
|
||||||
local path="$3"
|
build_rclone_fs() {
|
||||||
local mount_path="$4"
|
local type="$1"
|
||||||
local options="${5:-}"
|
local smb_server="$2"
|
||||||
|
local smb_share="$3"
|
||||||
|
local smb_username="$4"
|
||||||
|
local smb_password_obscured="$5"
|
||||||
|
local smb_domain="$6"
|
||||||
|
local rclone_remote="$7"
|
||||||
|
local rclone_path="$8"
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
case "$type" in
|
||||||
|
smb)
|
||||||
# Always mount read-only for safety
|
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
|
||||||
local mount_opts="ro"
|
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
|
||||||
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
|
# embed without quoting.
|
||||||
|
local opts="host=${smb_server}"
|
||||||
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
|
if [ -n "$smb_username" ]; then
|
||||||
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
|
opts="${opts},user=${smb_username}"
|
||||||
local rc=$?
|
|
||||||
|
|
||||||
if [ $rc -eq 0 ]; then
|
|
||||||
clear_error "$id"
|
|
||||||
log_success "NFS mount successful: $mount_path"
|
|
||||||
else
|
|
||||||
write_error "$id" "$output"
|
|
||||||
log_error "NFS mount failed: $output"
|
|
||||||
fi
|
fi
|
||||||
return $rc
|
if [ -n "$smb_password_obscured" ]; then
|
||||||
|
opts="${opts},pass=${smb_password_obscured}"
|
||||||
|
fi
|
||||||
|
if [ -n "$smb_domain" ]; then
|
||||||
|
opts="${opts},domain=${smb_domain}"
|
||||||
|
fi
|
||||||
|
echo ":smb,${opts}:${smb_share}"
|
||||||
|
;;
|
||||||
|
rclone)
|
||||||
|
echo "${rclone_remote}:${rclone_path}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo ""
|
||||||
|
;;
|
||||||
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mount SMB/CIFS share (always read-only)
|
# Mount via rclone RC API (always read-only).
|
||||||
do_mount_smb() {
|
|
||||||
local id="$1"
|
|
||||||
local server="$2"
|
|
||||||
local share="$3"
|
|
||||||
local mount_path="$4"
|
|
||||||
local username="${5:-}"
|
|
||||||
local password_obscured="${6:-}"
|
|
||||||
local domain="${7:-}"
|
|
||||||
local options="${8:-}"
|
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
|
||||||
|
|
||||||
# Always mount read-only for safety
|
|
||||||
local mount_opts="ro"
|
|
||||||
|
|
||||||
# If we have credentials, reveal password and build options
|
|
||||||
if [ -n "$username" ]; then
|
|
||||||
mount_opts="${mount_opts},username=${username}"
|
|
||||||
|
|
||||||
if [ -n "$password_obscured" ]; then
|
|
||||||
# Reveal password using rclone (only exists in memory briefly)
|
|
||||||
local password
|
|
||||||
password=$(rclone reveal "$password_obscured" 2>/dev/null)
|
|
||||||
if [ -n "$password" ]; then
|
|
||||||
mount_opts="${mount_opts},password=${password}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$domain" ]; then
|
|
||||||
mount_opts="${mount_opts},domain=${domain}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add any additional options
|
|
||||||
if [ -n "$options" ]; then
|
|
||||||
if [ -n "$mount_opts" ]; then
|
|
||||||
mount_opts="${mount_opts},${options}"
|
|
||||||
else
|
|
||||||
mount_opts="$options"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
|
|
||||||
|
|
||||||
# Execute mount directly (not via eval) to handle special chars in password
|
|
||||||
local output
|
|
||||||
local rc
|
|
||||||
if [ -n "$mount_opts" ]; then
|
|
||||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
|
|
||||||
rc=$?
|
|
||||||
else
|
|
||||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
|
|
||||||
rc=$?
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Clear password from any variables (best effort)
|
|
||||||
password=""
|
|
||||||
mount_opts=""
|
|
||||||
|
|
||||||
if [ $rc -eq 0 ]; then
|
|
||||||
clear_error "$id"
|
|
||||||
log_success "SMB mount successful: $mount_path"
|
|
||||||
else
|
|
||||||
write_error "$id" "$output"
|
|
||||||
log_error "SMB mount failed: $output"
|
|
||||||
fi
|
|
||||||
return $rc
|
|
||||||
}
|
|
||||||
|
|
||||||
# Mount rclone remote using RC API (always read-only)
|
|
||||||
do_mount_rclone() {
|
do_mount_rclone() {
|
||||||
local id="$1"
|
local id="$1"
|
||||||
local remote="$2"
|
local fs="$2"
|
||||||
local remote_path="$3"
|
local mount_path="$3"
|
||||||
local mount_path="$4"
|
local cache_max_size="$4"
|
||||||
local options="${5:-}"
|
local cache_max_age="$5"
|
||||||
|
local dir_cache_time="$6"
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
mkdir -p "$mount_path"
|
||||||
|
|
||||||
# Build the remote source string
|
# Apply defaults for any blank cache fields.
|
||||||
local fs="${remote}:${remote_path}"
|
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
|
||||||
|
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
|
||||||
|
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
|
||||||
|
|
||||||
# Use rclone RC API to mount (always read-only for safety)
|
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
|
||||||
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
|
|
||||||
|
|
||||||
# Build JSON body for rclone mount with read-only option
|
# Build JSON body via python3 — embedding the obscured password and
|
||||||
|
# display fs string into a hand-rolled heredoc is asking for an injection
|
||||||
|
# bug the next time we add a field with a quote in it.
|
||||||
local json_body
|
local json_body
|
||||||
json_body=$(cat <<EOF
|
json_body=$(python3 -c '
|
||||||
{
|
import json, sys
|
||||||
"fs": "${fs}",
|
print(json.dumps({
|
||||||
"mountPoint": "${mount_path}",
|
"fs": sys.argv[1],
|
||||||
|
"mountPoint": sys.argv[2],
|
||||||
"mountOpt": {
|
"mountOpt": {
|
||||||
"AllowOther": true,
|
"AllowOther": True,
|
||||||
"ReadOnly": true
|
"ReadOnly": True,
|
||||||
},
|
},
|
||||||
"vfsOpt": {
|
"vfsOpt": {
|
||||||
"CacheMode": 2,
|
# CacheMode 3 = "full" — cache both reads and writes on local disk.
|
||||||
"ReadOnly": true
|
# CacheMode 2 ("writes") would silently bypass the cache for reads,
|
||||||
}
|
# which is the opposite of what we want for a read-only mount.
|
||||||
}
|
"CacheMode": 3,
|
||||||
EOF
|
"CacheMaxSize": sys.argv[3],
|
||||||
)
|
"CacheMaxAge": sys.argv[4],
|
||||||
|
"DirCacheTime": sys.argv[5],
|
||||||
|
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
|
||||||
|
# throughput in isolation but, under the 4+ concurrent reads from
|
||||||
|
# the processing pipeline, multiplies into 16+ simultaneous SMB
|
||||||
|
# requests and tanks aggregate throughput. Left at default — rely on
|
||||||
|
# file-level concurrency from the pipeline for parallelism instead.
|
||||||
|
"ReadOnly": True,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
|
||||||
|
|
||||||
|
local output
|
||||||
output=$(curl -s -X POST \
|
output=$(curl -s -X POST \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-u admin:admin \
|
-u "$RCLONE_RC_AUTH" \
|
||||||
-d "$json_body" \
|
-d "$json_body" \
|
||||||
"http://127.0.0.1:5572/mount/mount" 2>&1)
|
"${RCLONE_RC_URL}/mount/mount" 2>&1)
|
||||||
|
|
||||||
# Wait a moment for mount to establish
|
|
||||||
sleep 2
|
sleep 2
|
||||||
|
|
||||||
if is_mounted "$mount_path"; then
|
if is_mounted "$mount_path"; then
|
||||||
@@ -221,7 +192,6 @@ EOF
|
|||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
local error_msg="rclone mount failed"
|
local error_msg="rclone mount failed"
|
||||||
# Try to extract error from response
|
|
||||||
if echo "$output" | grep -q "error"; then
|
if echo "$output" | grep -q "error"; then
|
||||||
error_msg="$output"
|
error_msg="$output"
|
||||||
fi
|
fi
|
||||||
@@ -231,36 +201,28 @@ EOF
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Unmount a path
|
# Unmount a path. Always goes through the rclone RC API now — every mount we
|
||||||
|
# create lives behind rclone's FUSE driver.
|
||||||
do_unmount() {
|
do_unmount() {
|
||||||
local id="$1"
|
local id="$1"
|
||||||
local mount_path="$2"
|
local mount_path="$2"
|
||||||
local type="$3"
|
|
||||||
|
|
||||||
if ! is_mounted "$mount_path"; then
|
if ! is_mounted "$mount_path"; then
|
||||||
log "Already unmounted: $mount_path"
|
log "Already unmounted: $mount_path"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$type" = "rclone" ]; then
|
log "Unmounting: $mount_path"
|
||||||
# Use rclone RC API to unmount
|
|
||||||
log "Unmounting rclone: $mount_path"
|
|
||||||
curl -s -X POST \
|
curl -s -X POST \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-u admin:admin \
|
-u "$RCLONE_RC_AUTH" \
|
||||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||||
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
|
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
|
||||||
else
|
|
||||||
log "Unmounting: $mount_path"
|
|
||||||
umount "$mount_path" 2>&1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Wait a moment
|
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
||||||
# Check if still mounted
|
# Fallback to lazy umount in case the FUSE process is wedged.
|
||||||
if is_mounted "$mount_path"; then
|
if is_mounted "$mount_path"; then
|
||||||
# Force unmount with lazy option
|
|
||||||
log_warn "Force unmounting (lazy): $mount_path"
|
log_warn "Force unmounting (lazy): $mount_path"
|
||||||
umount -l "$mount_path" 2>&1
|
umount -l "$mount_path" 2>&1
|
||||||
sleep 1
|
sleep 1
|
||||||
@@ -277,12 +239,11 @@ do_unmount() {
|
|||||||
|
|
||||||
# Process all mounts from config
|
# Process all mounts from config
|
||||||
process_mounts() {
|
process_mounts() {
|
||||||
# Check if config file exists
|
|
||||||
if [ ! -f "$CONFIG_FILE" ]; then
|
if [ ! -f "$CONFIG_FILE" ]; then
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Parse JSON using python3
|
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
|
||||||
local mounts
|
local mounts
|
||||||
mounts=$(python3 -c '
|
mounts=$(python3 -c '
|
||||||
import sys, json
|
import sys, json
|
||||||
@@ -294,15 +255,12 @@ except:
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
for m in data.get("mounts", []):
|
for m in data.get("mounts", []):
|
||||||
# Output pipe-separated values
|
|
||||||
fields = [
|
fields = [
|
||||||
m.get("id", ""),
|
m.get("id", ""),
|
||||||
m.get("type", ""),
|
m.get("type", ""),
|
||||||
str(m.get("enabled", True)),
|
str(m.get("enabled", True)),
|
||||||
str(m.get("desiredMounted", False)),
|
str(m.get("desiredMounted", False)),
|
||||||
m.get("mountPath", ""),
|
m.get("mountPath", ""),
|
||||||
m.get("nfsServer", ""),
|
|
||||||
m.get("nfsPath", ""),
|
|
||||||
m.get("smbServer", ""),
|
m.get("smbServer", ""),
|
||||||
m.get("smbShare", ""),
|
m.get("smbShare", ""),
|
||||||
m.get("smbUsername", ""),
|
m.get("smbUsername", ""),
|
||||||
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
|
|||||||
m.get("smbDomain", ""),
|
m.get("smbDomain", ""),
|
||||||
m.get("rcloneRemote", ""),
|
m.get("rcloneRemote", ""),
|
||||||
m.get("rclonePath", ""),
|
m.get("rclonePath", ""),
|
||||||
m.get("options", "")
|
m.get("cacheMaxSize", ""),
|
||||||
|
m.get("cacheMaxAge", ""),
|
||||||
|
m.get("dirCacheTime", ""),
|
||||||
]
|
]
|
||||||
print("|".join(fields))
|
print("|".join(fields))
|
||||||
' 2>/dev/null)
|
' 2>/dev/null)
|
||||||
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
|
|||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
|
while IFS='|' read -r id type enabled desired mount_path \
|
||||||
|
smb_server smb_share smb_username smb_password_obscured smb_domain \
|
||||||
|
rclone_remote rclone_path \
|
||||||
|
cache_max_size cache_max_age dir_cache_time; do
|
||||||
# Skip empty lines
|
# Skip empty lines
|
||||||
[ -z "$id" ] && continue
|
[ -z "$id" ] && continue
|
||||||
|
|
||||||
# Check actual mount status
|
# Skip retired/legacy types — the Go-side migration disables NFS but a
|
||||||
|
# stale entry could still arrive here on a downgrade/upgrade race.
|
||||||
|
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
|
||||||
|
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
local is_mounted_now=false
|
local is_mounted_now=false
|
||||||
is_mounted "$mount_path" && is_mounted_now=true
|
is_mounted "$mount_path" && is_mounted_now=true
|
||||||
|
|
||||||
# Handle disabled mounts - unmount if currently mounted
|
# Disabled → unmount if currently up
|
||||||
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
||||||
if [ "$is_mounted_now" = true ]; then
|
if [ "$is_mounted_now" = true ]; then
|
||||||
log "Mount $id ($mount_path) disabled, unmounting..."
|
log "Mount $id ($mount_path) disabled, unmounting..."
|
||||||
do_unmount "$id" "$mount_path" "$type"
|
do_unmount "$id" "$mount_path"
|
||||||
fi
|
fi
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Handle desired state
|
|
||||||
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
||||||
# Should be mounted
|
# Should be mounted
|
||||||
if [ "$is_mounted_now" = false ]; then
|
if [ "$is_mounted_now" = false ]; then
|
||||||
|
local fs
|
||||||
|
fs=$(build_rclone_fs "$type" \
|
||||||
|
"$smb_server" "$smb_share" "$smb_username" \
|
||||||
|
"$smb_password_obscured" "$smb_domain" \
|
||||||
|
"$rclone_remote" "$rclone_path")
|
||||||
|
|
||||||
|
if [ -z "$fs" ]; then
|
||||||
|
log_error "Could not build rclone fs spec for mount $id (type=$type)"
|
||||||
|
write_error "$id" "Invalid mount config: missing required fields for type=$type"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
||||||
case "$type" in
|
do_mount_rclone "$id" "$fs" "$mount_path" \
|
||||||
nfs)
|
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
|
||||||
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
|
|
||||||
;;
|
|
||||||
smb)
|
|
||||||
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
|
|
||||||
;;
|
|
||||||
rclone)
|
|
||||||
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
log_error "Unknown mount type: $type"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
# Should be unmounted
|
# Should be unmounted
|
||||||
if [ "$is_mounted_now" = true ]; then
|
if [ "$is_mounted_now" = true ]; then
|
||||||
log "Mount $id ($mount_path) not desired, unmounting..."
|
log "Mount $id ($mount_path) not desired, unmounting..."
|
||||||
do_unmount "$id" "$mount_path" "$type"
|
do_unmount "$id" "$mount_path"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done <<< "$mounts"
|
done <<< "$mounts"
|
||||||
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
|
|||||||
|
|
||||||
# Main loop
|
# Main loop
|
||||||
main() {
|
main() {
|
||||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
|
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
|
||||||
|
|
||||||
init_dirs
|
init_dirs
|
||||||
|
|
||||||
|
|||||||
@@ -70,6 +70,9 @@ http {
|
|||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_cache off;
|
proxy_cache off;
|
||||||
proxy_read_timeout 86400s;
|
proxy_read_timeout 86400s;
|
||||||
|
|
||||||
|
# Snapshot import accepts multi-MB ZIP uploads.
|
||||||
|
client_max_body_size 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
@@ -192,6 +195,18 @@ http {
|
|||||||
# WebDAV HTML browser (for iframe embedding in dashboard)
|
# WebDAV HTML browser (for iframe embedding in dashboard)
|
||||||
# Returns HTML directory listings instead of JSON
|
# Returns HTML directory listings instead of JSON
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
|
# At the browser root, strip the "../" parent-dir row that autoindex
|
||||||
|
# otherwise emits — clicking it inside the dashboard iframe would
|
||||||
|
# navigate to "/" (the dashboard itself), nesting the page inside itself.
|
||||||
|
location = /webdav-browser/ {
|
||||||
|
alias /files/;
|
||||||
|
autoindex on;
|
||||||
|
charset utf-8;
|
||||||
|
sub_filter '<a href="../">../</a>' '';
|
||||||
|
sub_filter_once on;
|
||||||
|
}
|
||||||
|
|
||||||
location /webdav-browser/ {
|
location /webdav-browser/ {
|
||||||
alias /files/;
|
alias /files/;
|
||||||
autoindex on;
|
autoindex on;
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ startsecs=2
|
|||||||
# rclone: Remote storage mount daemon (RC API on port 5572)
|
# rclone: Remote storage mount daemon (RC API on port 5572)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
[program:rclone]
|
[program:rclone]
|
||||||
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
||||||
priority=30
|
priority=30
|
||||||
autostart=true
|
autostart=true
|
||||||
autorestart=true
|
autorestart=true
|
||||||
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
|
|||||||
startsecs=3
|
startsecs=3
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
|
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
[program:mount-watcher]
|
[program:mount-watcher]
|
||||||
command=/app/scripts/mount-watcher.sh
|
command=/app/scripts/mount-watcher.sh
|
||||||
|
|||||||
@@ -330,28 +330,39 @@ Events are published to the `meta:events` Redis stream.
|
|||||||
|
|
||||||
### Mount Management
|
### Mount Management
|
||||||
|
|
||||||
Meta-core manages SMB/rclone mount configurations:
|
Meta-core manages remote mounts via a single rclone backend. Two user-facing
|
||||||
|
mount types are supported: `smb` (synthesised on-the-fly into rclone's `:smb:`
|
||||||
|
backend) and `rclone` (references a pre-defined remote in `rclone.conf`).
|
||||||
|
Native `mount.cifs` and `mount.nfs` were removed in favour of one code path.
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────────────────────────────────────┐
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
│ Mount Manager │
|
│ Mount Manager │
|
||||||
│ │
|
│ │
|
||||||
│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │
|
│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │
|
||||||
│ │ Config │─►│ Validator │─►│ Health Monitor │ │
|
│ │ Config │─►│ Validator │─►│ Mount Watcher │ │
|
||||||
│ │ (mounts.json)│ │ │ │ │ │
|
│ │ (mounts.json)│ │ │ │ │ │
|
||||||
│ │ │ │ - SMB params │ │ - Accessibility check │ │
|
│ │ │ │ - SMB params │ │ - Calls rclone RC │ │
|
||||||
│ │ - id │ │ - rclone cfg │ │ - Error tracking │ │
|
│ │ - id │ │ - rclone cfg │ │ /mount/mount │ │
|
||||||
│ │ - type │ │ - paths │ │ - Status reporting │ │
|
│ │ - type │ │ - cache size │ │ - ReadOnly: true │ │
|
||||||
│ │ - path │ │ │ │ │ │
|
│ │ - path │ │ │ │ - Polls every 5s │ │
|
||||||
|
│ │ - cache* │ │ │ │ - Error tracking │ │
|
||||||
│ └──────────────┘ └──────────────┘ └────────────────────────┘ │
|
│ └──────────────┘ └──────────────┘ └────────────────────────┘ │
|
||||||
└─────────────────────────────────────────────────────────────────┘
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
Features:
|
Features:
|
||||||
- **Configuration**: Mount configs stored in `/meta-core/mounts/mounts.json`
|
- **Configuration**: Mount configs stored in `/meta-core/mounts/mounts.json`
|
||||||
- **Validation**: Validates mount commands and parameters
|
- **Read-only**: All mounts are mounted read-only by construction (no API knob)
|
||||||
- **Health Monitoring**: Polls mount points for accessibility
|
- **VFS cache**: Per-mount `cacheMaxSize` (default `50G`), `cacheMaxAge`
|
||||||
- **Error Handling**: Tracks mount errors in separate error files
|
(default `24h`), `dirCacheTime` (default `5m`) propagate to rclone's
|
||||||
|
`vfsOpt`. The cache directory itself is daemon-global — rclone keeps
|
||||||
|
per-remote subdirectories beneath it.
|
||||||
|
- **Health Monitoring**: Polls `/proc/mounts` every 5s for accessibility
|
||||||
|
- **Error Handling**: Tracks mount errors in `/meta-core/mounts/errors/{id}.error`
|
||||||
|
- **Stats**: `/api/mounts` returns daemon-global rclone counters (bytes/sec,
|
||||||
|
transfers/sec) — same numbers across all mounts since rclone tracks per-
|
||||||
|
daemon, not per-mount.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -183,3 +183,42 @@
|
|||||||
color: rgba(0, 0, 0, 0.5);
|
color: rgba(0, 0, 0, 0.5);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.app-nav {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-nav button {
|
||||||
|
background: transparent;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.2);
|
||||||
|
color: inherit;
|
||||||
|
padding: 0.4rem 1rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-nav button:hover {
|
||||||
|
background: rgba(255, 255, 255, 0.08);
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-nav button.active {
|
||||||
|
background: rgba(255, 255, 255, 0.15);
|
||||||
|
border-color: rgba(255, 255, 255, 0.4);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (prefers-color-scheme: light) {
|
||||||
|
.app-nav button {
|
||||||
|
border-color: rgba(0, 0, 0, 0.2);
|
||||||
|
}
|
||||||
|
.app-nav button:hover {
|
||||||
|
background: rgba(0, 0, 0, 0.05);
|
||||||
|
}
|
||||||
|
.app-nav button.active {
|
||||||
|
background: rgba(0, 0, 0, 0.08);
|
||||||
|
border-color: rgba(0, 0, 0, 0.4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ import { FileList } from './components/FileList';
|
|||||||
import { MetadataEditor } from './components/MetadataEditor';
|
import { MetadataEditor } from './components/MetadataEditor';
|
||||||
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
|
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
|
||||||
import { KVTreeView } from './components/KVTreeView';
|
import { KVTreeView } from './components/KVTreeView';
|
||||||
|
import { SnapshotPanel } from './components/SnapshotPanel';
|
||||||
import { MetadataAPI } from './api/metadataApi';
|
import { MetadataAPI } from './api/metadataApi';
|
||||||
import { SearchResult } from './types';
|
import { SearchResult } from './types';
|
||||||
import './App.css';
|
import './App.css';
|
||||||
|
|
||||||
|
type View = 'edit' | 'snapshot';
|
||||||
|
|
||||||
function App() {
|
function App() {
|
||||||
|
const [view, setView] = useState<View>('edit');
|
||||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||||
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
|
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
|
||||||
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
|
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
|
||||||
@@ -167,8 +171,27 @@ function App() {
|
|||||||
<p className="app-description">
|
<p className="app-description">
|
||||||
Search and edit metadata for your media files
|
Search and edit metadata for your media files
|
||||||
</p>
|
</p>
|
||||||
|
<nav className="app-nav">
|
||||||
|
<button
|
||||||
|
className={view === 'edit' ? 'active' : ''}
|
||||||
|
onClick={() => setView('edit')}
|
||||||
|
>
|
||||||
|
Edit
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className={view === 'snapshot' ? 'active' : ''}
|
||||||
|
onClick={() => setView('snapshot')}
|
||||||
|
>
|
||||||
|
Snapshot
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
{view === 'snapshot' ? (
|
||||||
|
<main className="app-main">
|
||||||
|
<SnapshotPanel />
|
||||||
|
</main>
|
||||||
|
) : (
|
||||||
<main className="app-main">
|
<main className="app-main">
|
||||||
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
|
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
|
||||||
|
|
||||||
@@ -230,6 +253,7 @@ function App() {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</main>
|
</main>
|
||||||
|
)}
|
||||||
|
|
||||||
<footer className="app-footer">
|
<footer className="app-footer">
|
||||||
<p>
|
<p>
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
const API_BASE = '/api/snapshot';
|
||||||
|
|
||||||
|
export type ImportMode = 'replace' | 'merge';
|
||||||
|
export type ImportConflict = 'mine' | 'source';
|
||||||
|
|
||||||
|
export interface FileImportResult {
|
||||||
|
hashId: string;
|
||||||
|
added: number;
|
||||||
|
updated: number;
|
||||||
|
unchanged: number;
|
||||||
|
kept: number;
|
||||||
|
deleted: number;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PluginOutputResult {
|
||||||
|
written: number;
|
||||||
|
skipped: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportResult {
|
||||||
|
mode: ImportMode;
|
||||||
|
conflict?: ImportConflict;
|
||||||
|
dryRun: boolean;
|
||||||
|
totalFiles: number;
|
||||||
|
filesOk: number;
|
||||||
|
filesFailed: number;
|
||||||
|
files?: FileImportResult[];
|
||||||
|
pluginOutput?: PluginOutputResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WipeResult {
|
||||||
|
metadataDeleted: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SnapshotAPI {
|
||||||
|
/**
|
||||||
|
* Trigger a download of a metadata snapshot ZIP.
|
||||||
|
* Browser handles the download via a synthetic <a> click.
|
||||||
|
*/
|
||||||
|
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
|
||||||
|
const parts: string[] = [];
|
||||||
|
if (includes.pluginOutput) parts.push('plugin-output');
|
||||||
|
const qs = parts.length ? `?include=${parts.join(',')}` : '';
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = `${API_BASE}/export${qs}`;
|
||||||
|
a.rel = 'noopener';
|
||||||
|
document.body.appendChild(a);
|
||||||
|
a.click();
|
||||||
|
a.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a snapshot ZIP and apply it.
|
||||||
|
* Mode replace deletes and rewrites each CID's properties.
|
||||||
|
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
|
||||||
|
*/
|
||||||
|
static async runImport(
|
||||||
|
file: File,
|
||||||
|
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
|
||||||
|
): Promise<ImportResult> {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
params.set('mode', opts.mode);
|
||||||
|
if (opts.mode === 'merge') {
|
||||||
|
params.set('conflict', opts.conflict ?? 'source');
|
||||||
|
}
|
||||||
|
if (opts.dryRun) {
|
||||||
|
params.set('dry_run', 'true');
|
||||||
|
}
|
||||||
|
const fd = new FormData();
|
||||||
|
fd.append('snapshot', file);
|
||||||
|
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: fd,
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const err = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(err.message || err.error || 'Import failed');
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wipe Redis metadata. Pass scope=metadata.
|
||||||
|
*/
|
||||||
|
static async wipe(scope: 'metadata'): Promise<WipeResult> {
|
||||||
|
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
|
||||||
|
method: 'POST',
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const err = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(err.message || err.error || 'Wipe failed');
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
.snapshot-panel {
|
||||||
|
max-width: 920px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-panel h2 {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-intro {
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 1rem 1.25rem;
|
||||||
|
margin-bottom: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section.danger {
|
||||||
|
border-color: #d04a4a;
|
||||||
|
background: rgba(208, 74, 74, 0.04);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section h3 {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-row {
|
||||||
|
display: flex;
|
||||||
|
gap: 1rem;
|
||||||
|
align-items: flex-start;
|
||||||
|
margin: 0.75rem 0;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-file {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-file span {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
min-width: 280px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset[disabled] {
|
||||||
|
opacity: 0.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset legend {
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
padding: 0 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset label {
|
||||||
|
display: block;
|
||||||
|
margin: 0.25rem 0;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset input[type='radio'] {
|
||||||
|
margin-right: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn {
|
||||||
|
background: var(--bg-button, #f0f0f0);
|
||||||
|
border: 1px solid var(--border, #c0c0c0);
|
||||||
|
color: var(--text, #222);
|
||||||
|
padding: 0.5rem 1rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn:hover:not(:disabled) {
|
||||||
|
background: var(--bg-button-hover, #e0e0e0);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn:disabled {
|
||||||
|
opacity: 0.5;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.primary {
|
||||||
|
background: #2a6df4;
|
||||||
|
border-color: #2a6df4;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.primary:hover:not(:disabled) {
|
||||||
|
background: #1a5de4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.danger {
|
||||||
|
background: #d04a4a;
|
||||||
|
border-color: #d04a4a;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.danger:hover:not(:disabled) {
|
||||||
|
background: #b83838;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-error {
|
||||||
|
background: rgba(208, 74, 74, 0.1);
|
||||||
|
color: #b83838;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-info {
|
||||||
|
background: rgba(42, 109, 244, 0.08);
|
||||||
|
color: #1a5de4;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result {
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-summary {
|
||||||
|
display: flex;
|
||||||
|
gap: 1rem;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table th,
|
||||||
|
.snapshot-result-table td {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
padding: 0.35rem 0.5rem;
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table th {
|
||||||
|
background: var(--bg-subtle, #f5f5f5);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors {
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors summary {
|
||||||
|
cursor: pointer;
|
||||||
|
color: #b83838;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors ul {
|
||||||
|
margin: 0.5rem 0 0;
|
||||||
|
padding-left: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors code {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-checkbox {
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin: 0.5rem 0 0.75rem;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-checkbox input {
|
||||||
|
margin-top: 0.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-hint {
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-aside {
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--text-muted, #555);
|
||||||
|
}
|
||||||
@@ -0,0 +1,271 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import {
|
||||||
|
SnapshotAPI,
|
||||||
|
ImportMode,
|
||||||
|
ImportConflict,
|
||||||
|
ImportResult,
|
||||||
|
} from '../api/snapshotApi';
|
||||||
|
import './SnapshotPanel.css';
|
||||||
|
|
||||||
|
export function SnapshotPanel() {
|
||||||
|
const [exportPluginOutput, setExportPluginOutput] = useState(false);
|
||||||
|
const [importFile, setImportFile] = useState<File | null>(null);
|
||||||
|
const [importMode, setImportMode] = useState<ImportMode>('merge');
|
||||||
|
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
|
||||||
|
const [importing, setImporting] = useState(false);
|
||||||
|
const [importResult, setImportResult] = useState<ImportResult | null>(null);
|
||||||
|
const [importError, setImportError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const [wipeBusy, setWipeBusy] = useState(false);
|
||||||
|
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const runImport = async (dryRun: boolean) => {
|
||||||
|
if (!importFile) {
|
||||||
|
setImportError('Pick a snapshot ZIP first.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setImporting(true);
|
||||||
|
setImportError(null);
|
||||||
|
setImportResult(null);
|
||||||
|
try {
|
||||||
|
const result = await SnapshotAPI.runImport(importFile, {
|
||||||
|
mode: importMode,
|
||||||
|
conflict: importMode === 'merge' ? importConflict : undefined,
|
||||||
|
dryRun,
|
||||||
|
});
|
||||||
|
setImportResult(result);
|
||||||
|
} catch (e: any) {
|
||||||
|
setImportError(e.message || String(e));
|
||||||
|
} finally {
|
||||||
|
setImporting(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const runWipe = async () => {
|
||||||
|
const ok = window.confirm(
|
||||||
|
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
|
||||||
|
'edit you made via this editor will be lost. Continue?'
|
||||||
|
);
|
||||||
|
if (!ok) return;
|
||||||
|
setWipeBusy(true);
|
||||||
|
setWipeMessage(null);
|
||||||
|
try {
|
||||||
|
const r = await SnapshotAPI.wipe('metadata');
|
||||||
|
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
|
||||||
|
} catch (e: any) {
|
||||||
|
setWipeMessage(`Failed: ${e.message || e}`);
|
||||||
|
} finally {
|
||||||
|
setWipeBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="snapshot-panel">
|
||||||
|
<h2>Snapshot</h2>
|
||||||
|
<p className="snapshot-intro">
|
||||||
|
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
|
||||||
|
archives — one nested-JSON file per CID, plus a manifest.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<section className="snapshot-section">
|
||||||
|
<h3>Export</h3>
|
||||||
|
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
|
||||||
|
<label className="snapshot-checkbox">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={exportPluginOutput}
|
||||||
|
onChange={(e) => setExportPluginOutput(e.target.checked)}
|
||||||
|
/>
|
||||||
|
<span>
|
||||||
|
Include plugin output
|
||||||
|
<span className="snapshot-hint">
|
||||||
|
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn primary"
|
||||||
|
onClick={() =>
|
||||||
|
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
|
||||||
|
}
|
||||||
|
>
|
||||||
|
Download snapshot.zip
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="snapshot-section">
|
||||||
|
<h3>Import</h3>
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<label className="snapshot-file">
|
||||||
|
<span>Snapshot ZIP</span>
|
||||||
|
<input
|
||||||
|
type="file"
|
||||||
|
accept=".zip,application/zip"
|
||||||
|
onChange={(e) => {
|
||||||
|
setImportFile(e.target.files?.[0] ?? null);
|
||||||
|
setImportResult(null);
|
||||||
|
setImportError(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<fieldset className="snapshot-fieldset">
|
||||||
|
<legend>Mode</legend>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="mode"
|
||||||
|
value="merge"
|
||||||
|
checked={importMode === 'merge'}
|
||||||
|
onChange={() => setImportMode('merge')}
|
||||||
|
/>
|
||||||
|
Merge — keep existing fields not in the snapshot
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="mode"
|
||||||
|
value="replace"
|
||||||
|
checked={importMode === 'replace'}
|
||||||
|
onChange={() => setImportMode('replace')}
|
||||||
|
/>
|
||||||
|
Replace — wipe each imported CID and rewrite from snapshot
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
|
||||||
|
<legend>Conflict</legend>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="conflict"
|
||||||
|
value="source"
|
||||||
|
checked={importConflict === 'source'}
|
||||||
|
onChange={() => setImportConflict('source')}
|
||||||
|
/>
|
||||||
|
Source wins (overwrite my edits)
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="conflict"
|
||||||
|
value="mine"
|
||||||
|
checked={importConflict === 'mine'}
|
||||||
|
onChange={() => setImportConflict('mine')}
|
||||||
|
/>
|
||||||
|
Mine wins (keep my edits, fill in only missing fields)
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<button
|
||||||
|
className="snapshot-btn"
|
||||||
|
disabled={!importFile || importing}
|
||||||
|
onClick={() => runImport(true)}
|
||||||
|
>
|
||||||
|
Dry run
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn primary"
|
||||||
|
disabled={!importFile || importing}
|
||||||
|
onClick={() => runImport(false)}
|
||||||
|
>
|
||||||
|
{importing ? 'Importing…' : 'Apply import'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{importError && <div className="snapshot-error">{importError}</div>}
|
||||||
|
{importResult && <ImportResultView result={importResult} />}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="snapshot-section danger">
|
||||||
|
<h3>Wipe metadata</h3>
|
||||||
|
<p>
|
||||||
|
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
|
||||||
|
Files on disk are untouched.
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn danger"
|
||||||
|
disabled={wipeBusy}
|
||||||
|
onClick={runWipe}
|
||||||
|
>
|
||||||
|
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
|
||||||
|
</button>
|
||||||
|
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ImportResultView({ result }: { result: ImportResult }) {
|
||||||
|
const totals = (result.files ?? []).reduce(
|
||||||
|
(acc, f) => {
|
||||||
|
acc.added += f.added;
|
||||||
|
acc.updated += f.updated;
|
||||||
|
acc.unchanged += f.unchanged;
|
||||||
|
acc.kept += f.kept;
|
||||||
|
acc.deleted += f.deleted;
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
<div className="snapshot-result">
|
||||||
|
<div className="snapshot-result-summary">
|
||||||
|
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
|
||||||
|
<span>
|
||||||
|
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
|
||||||
|
</span>
|
||||||
|
<span>
|
||||||
|
mode={result.mode}
|
||||||
|
{result.conflict ? `, conflict=${result.conflict}` : ''}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<table className="snapshot-result-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>added</th>
|
||||||
|
<th>updated</th>
|
||||||
|
<th>unchanged</th>
|
||||||
|
<th>kept (mine)</th>
|
||||||
|
<th>deleted (replace)</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>{totals.added}</td>
|
||||||
|
<td>{totals.updated}</td>
|
||||||
|
<td>{totals.unchanged}</td>
|
||||||
|
<td>{totals.kept}</td>
|
||||||
|
<td>{totals.deleted}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{result.pluginOutput && (
|
||||||
|
<div className="snapshot-result-aside">
|
||||||
|
plugin-output: {result.pluginOutput.written} written
|
||||||
|
{result.pluginOutput.skipped > 0
|
||||||
|
? `, ${result.pluginOutput.skipped} skipped (existed)`
|
||||||
|
: ''}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{result.files && result.files.some((f) => f.error) && (
|
||||||
|
<details className="snapshot-result-errors">
|
||||||
|
<summary>Per-file errors</summary>
|
||||||
|
<ul>
|
||||||
|
{result.files
|
||||||
|
.filter((f) => f.error)
|
||||||
|
.map((f) => (
|
||||||
|
<li key={f.hashId}>
|
||||||
|
<code>{f.hashId}</code>: {f.error}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -522,3 +522,54 @@
|
|||||||
max-height: 160px;
|
max-height: 160px;
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.kv-app-gear {
|
||||||
|
margin-left: 0.25rem;
|
||||||
|
font-size: 1.05rem;
|
||||||
|
opacity: 0.6;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-gear:hover {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-backdrop {
|
||||||
|
position: fixed;
|
||||||
|
inset: 0;
|
||||||
|
background: rgba(0, 0, 0, 0.45);
|
||||||
|
z-index: 1000;
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 4vh 1rem;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal {
|
||||||
|
position: relative;
|
||||||
|
background: white;
|
||||||
|
color: inherit;
|
||||||
|
border-radius: 8px;
|
||||||
|
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
|
||||||
|
width: min(960px, 100%);
|
||||||
|
max-height: 92vh;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-close {
|
||||||
|
position: absolute;
|
||||||
|
top: 0.5rem;
|
||||||
|
right: 0.75rem;
|
||||||
|
background: transparent;
|
||||||
|
border: none;
|
||||||
|
font-size: 1.6rem;
|
||||||
|
line-height: 1;
|
||||||
|
cursor: pointer;
|
||||||
|
color: inherit;
|
||||||
|
opacity: 0.6;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-close:hover {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { KVBranchEditor } from './KVBranchEditor';
|
|||||||
import { SearchFieldChips } from './SearchFieldChips';
|
import { SearchFieldChips } from './SearchFieldChips';
|
||||||
import { ToastStack, makeToast } from './Toast';
|
import { ToastStack, makeToast } from './Toast';
|
||||||
import { ToastEntry } from './types';
|
import { ToastEntry } from './types';
|
||||||
|
import { SnapshotPanel } from '../components/SnapshotPanel';
|
||||||
import './KVApp.css';
|
import './KVApp.css';
|
||||||
|
|
||||||
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
|
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
|
||||||
@@ -28,6 +29,7 @@ function loadSearchFields(): string[] {
|
|||||||
//
|
//
|
||||||
// A sticky header carries the search bar, the breadcrumb, and basic stats.
|
// A sticky header carries the search bar, the breadcrumb, and basic stats.
|
||||||
export default function KVApp() {
|
export default function KVApp() {
|
||||||
|
const [snapshotOpen, setSnapshotOpen] = useState(false);
|
||||||
// Selected node. branchPath is set when a branch is selected; selectedKey
|
// Selected node. branchPath is set when a branch is selected; selectedKey
|
||||||
// when a leaf is. Mutually exclusive.
|
// when a leaf is. Mutually exclusive.
|
||||||
const [selectedKey, setSelectedKey] = useState<string | null>(null);
|
const [selectedKey, setSelectedKey] = useState<string | null>(null);
|
||||||
@@ -149,6 +151,14 @@ export default function KVApp() {
|
|||||||
onClick={() => setReloadToken((n) => n + 1)}
|
onClick={() => setReloadToken((n) => n + 1)}
|
||||||
title="refresh tree"
|
title="refresh tree"
|
||||||
>Refresh</button>
|
>Refresh</button>
|
||||||
|
<button
|
||||||
|
className="kv-btn-icon kv-app-gear"
|
||||||
|
onClick={() => setSnapshotOpen(true)}
|
||||||
|
title="Snapshot — export / import / wipe"
|
||||||
|
aria-label="Open snapshot settings"
|
||||||
|
>
|
||||||
|
⚙
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
|
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
|
||||||
{breadcrumb && breadcrumb.length > 0 && (
|
{breadcrumb && breadcrumb.length > 0 && (
|
||||||
@@ -237,6 +247,28 @@ export default function KVApp() {
|
|||||||
</main>
|
</main>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{snapshotOpen && (
|
||||||
|
<div
|
||||||
|
className="kv-modal-backdrop"
|
||||||
|
onClick={() => setSnapshotOpen(false)}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className="kv-modal"
|
||||||
|
onClick={(e) => e.stopPropagation()}
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
className="kv-modal-close"
|
||||||
|
onClick={() => setSnapshotOpen(false)}
|
||||||
|
aria-label="Close"
|
||||||
|
title="Close"
|
||||||
|
>×</button>
|
||||||
|
<SnapshotPanel />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<ToastStack toasts={toasts} onDismiss={dismissToast} />
|
<ToastStack toasts={toasts} onDismiss={dismissToast} />
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/metadataApi.ts","./src/components/BulkMetadataEditor.tsx","./src/components/DynamicFieldRenderer.tsx","./src/components/DynamicPluginSection.tsx","./src/components/FileList.tsx","./src/components/KVTreeView.tsx","./src/components/LanguageCombobox.tsx","./src/components/MetadataEditor.tsx","./src/components/SearchBar.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/typeHeuristics.ts","./src/kv/types.ts","./src/types/index.ts","./src/utils/formatting.ts"],"version":"5.9.3"}
|
{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/metadataApi.ts","./src/api/snapshotApi.ts","./src/components/BulkMetadataEditor.tsx","./src/components/DynamicFieldRenderer.tsx","./src/components/DynamicPluginSection.tsx","./src/components/FileList.tsx","./src/components/KVTreeView.tsx","./src/components/LanguageCombobox.tsx","./src/components/MetadataEditor.tsx","./src/components/SearchBar.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/typeHeuristics.ts","./src/kv/types.ts","./src/types/index.ts","./src/utils/formatting.ts"],"version":"5.9.3"}
|
||||||
@@ -29,6 +29,7 @@ type Server struct {
|
|||||||
mountsManager *mounts.Manager
|
mountsManager *mounts.Manager
|
||||||
mountsHandlers *mounts.Handlers
|
mountsHandlers *mounts.Handlers
|
||||||
mountPoller *mounts.Poller
|
mountPoller *mounts.Poller
|
||||||
|
mountStatsPoller *mounts.StatsPoller
|
||||||
watcherDispatcher *watcher.Dispatcher
|
watcherDispatcher *watcher.Dispatcher
|
||||||
fileWatcher *watcher.Watcher
|
fileWatcher *watcher.Watcher
|
||||||
watcherHandlers *watcher.Handlers
|
watcherHandlers *watcher.Handlers
|
||||||
@@ -71,6 +72,11 @@ func NewServer(
|
|||||||
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
||||||
} else {
|
} else {
|
||||||
s.fileWatcher = fileWatcher
|
s.fileWatcher = fileWatcher
|
||||||
|
// Wire storage so the watcher can persist (path,size,mtime,cid)
|
||||||
|
// tuples to Redis as it computes midhashes.
|
||||||
|
if stor != nil {
|
||||||
|
fileWatcher.SetStorageClient(stor)
|
||||||
|
}
|
||||||
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
||||||
// Create scan function from watcher for mounts
|
// Create scan function from watcher for mounts
|
||||||
scanFunc = fileWatcher.ScanMountPath
|
scanFunc = fileWatcher.ScanMountPath
|
||||||
@@ -88,6 +94,20 @@ func NewServer(
|
|||||||
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
||||||
// Pass republish callback to handlers (handles nil metaPublisher case)
|
// Pass republish callback to handlers (handles nil metaPublisher case)
|
||||||
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
|
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
|
||||||
|
|
||||||
|
// Hydrate the in-memory state registry from Redis tuples so
|
||||||
|
// the first scan tick after boot can skip ComputeMidHash256
|
||||||
|
// for files whose (size, mtime) match.
|
||||||
|
configs := watchersManager.List()
|
||||||
|
roots := make([]watcher.WatcherRoot, 0, len(configs))
|
||||||
|
for _, c := range configs {
|
||||||
|
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
|
||||||
|
}
|
||||||
|
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
|
||||||
|
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
|
||||||
|
} else {
|
||||||
|
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,6 +123,12 @@ func NewServer(
|
|||||||
// Create and configure mount poller
|
// Create and configure mount poller
|
||||||
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
|
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
|
||||||
mountsManager.SetPoller(s.mountPoller)
|
mountsManager.SetPoller(s.mountPoller)
|
||||||
|
|
||||||
|
// IO stats poller — queries the local rclone RC API every few seconds
|
||||||
|
// and exposes live throughput per mount via the same status JSON.
|
||||||
|
// Stats are daemon-global (rclone limitation); see stats_poller.go.
|
||||||
|
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
|
||||||
|
mountsManager.SetStatsPoller(s.mountStatsPoller)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
|
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
|
||||||
@@ -126,6 +152,11 @@ func (s *Server) setupRoutes() {
|
|||||||
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
||||||
|
|
||||||
|
// Snapshot (export / import / wipe)
|
||||||
|
s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
|
||||||
|
s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
||||||
@@ -240,6 +271,13 @@ func (s *Server) Start() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
|
||||||
|
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
|
||||||
|
// mounts), so always-on rather than gated by config.
|
||||||
|
if s.mountStatsPoller != nil {
|
||||||
|
s.mountStatsPoller.Start()
|
||||||
|
}
|
||||||
|
|
||||||
// Start meta publisher (if storage is connected)
|
// Start meta publisher (if storage is connected)
|
||||||
// Publishes keyspace notifications to meta:events stream
|
// Publishes keyspace notifications to meta:events stream
|
||||||
if s.storage != nil && s.storage.IsConnected() {
|
if s.storage != nil && s.storage.IsConnected() {
|
||||||
@@ -278,6 +316,11 @@ func (s *Server) Stop() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop IO stats poller
|
||||||
|
if s.mountStatsPoller != nil {
|
||||||
|
s.mountStatsPoller.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
// Stop meta publisher
|
// Stop meta publisher
|
||||||
if s.metaPublisher != nil {
|
if s.metaPublisher != nil {
|
||||||
if err := s.metaPublisher.Stop(); err != nil {
|
if err := s.metaPublisher.Stop(); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/snapshot"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bound the upload size so a malicious client can't OOM the server.
|
||||||
|
// 2 GiB is well above any realistic export.
|
||||||
|
const maxImportBytes = 2 << 30
|
||||||
|
|
||||||
|
// pluginOutputDir is where plugin-produced artifacts live on disk
|
||||||
|
// (subtitles, posters, etc). Mounted from the host into the meta-core
|
||||||
|
// container at $FILES_PATH/plugin.
|
||||||
|
func (s *Server) pluginOutputDir() string {
|
||||||
|
return filepath.Join(s.config.FilesPath, "plugin")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotExport streams a ZIP snapshot of all metadata.
|
||||||
|
// GET /api/snapshot/export?include=hash-index,plugin-output
|
||||||
|
func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The server has a 30s WriteTimeout (server.go:222). Long exports
|
||||||
|
// blow past that, leaving a truncated ZIP without a central directory.
|
||||||
|
// Clear the deadline for this handler so the export can run as long as
|
||||||
|
// needed. ResponseController is the standard way to do this.
|
||||||
|
rc := http.NewResponseController(w)
|
||||||
|
_ = rc.SetWriteDeadline(time.Time{})
|
||||||
|
|
||||||
|
include := splitCSVParam(r.URL.Query().Get("include"))
|
||||||
|
opts := snapshot.ExportOptions{
|
||||||
|
IncludePluginOutput: include["plugin-output"],
|
||||||
|
PluginOutputDir: s.pluginOutputDir(),
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
|
||||||
|
w.Header().Set("Content-Type", "application/zip")
|
||||||
|
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
|
||||||
|
// We stream — no Content-Length.
|
||||||
|
|
||||||
|
exporter := snapshot.NewExporter(s.storage)
|
||||||
|
if _, err := exporter.Export(w, opts); err != nil {
|
||||||
|
// Headers may already be sent; log and bail.
|
||||||
|
// Client will see a truncated zip, which is the best signal we can give.
|
||||||
|
fmt.Printf("[Snapshot] export failed: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotImport applies an uploaded ZIP snapshot.
|
||||||
|
// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
|
||||||
|
// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
|
||||||
|
func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
|
||||||
|
// when the snapshot is large. Clear both for this handler.
|
||||||
|
rc := http.NewResponseController(w)
|
||||||
|
_ = rc.SetReadDeadline(time.Time{})
|
||||||
|
_ = rc.SetWriteDeadline(time.Time{})
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
opts := snapshot.ImportOptions{
|
||||||
|
Mode: snapshot.Mode(q.Get("mode")),
|
||||||
|
Conflict: snapshot.Conflict(q.Get("conflict")),
|
||||||
|
DryRun: strings.EqualFold(q.Get("dry_run"), "true"),
|
||||||
|
PluginOutputDir: s.pluginOutputDir(),
|
||||||
|
}
|
||||||
|
if opts.Mode == "" {
|
||||||
|
opts.Mode = snapshot.ModeMerge
|
||||||
|
}
|
||||||
|
if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
|
||||||
|
opts.Conflict = snapshot.ConflictSource
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := readImportBody(r)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
importer := snapshot.NewImporter(s.storage)
|
||||||
|
result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotWipe clears Redis metadata.
|
||||||
|
// POST /api/snapshot/wipe?scope=metadata
|
||||||
|
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
scopes := splitCSVParam(r.URL.Query().Get("scope"))
|
||||||
|
if len(scopes) == 0 {
|
||||||
|
writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
scope := snapshot.WipeScope{
|
||||||
|
Metadata: scopes["metadata"],
|
||||||
|
}
|
||||||
|
if !scope.Metadata {
|
||||||
|
writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
wiper := snapshot.NewWiper(s.storage)
|
||||||
|
res, err := wiper.Wipe(scope)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readImportBody accepts either a raw application/zip body or a multipart
|
||||||
|
// upload with a field named "snapshot". Buffers up to maxImportBytes.
|
||||||
|
func readImportBody(r *http.Request) ([]byte, error) {
|
||||||
|
r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
|
||||||
|
|
||||||
|
ct := r.Header.Get("Content-Type")
|
||||||
|
if strings.HasPrefix(ct, "multipart/form-data") {
|
||||||
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse multipart: %w", err)
|
||||||
|
}
|
||||||
|
f, _, err := r.FormFile("snapshot")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return io.ReadAll(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
return io.ReadAll(r.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
|
||||||
|
func splitCSVParam(v string) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, part := range strings.Split(v, ",") {
|
||||||
|
p := strings.TrimSpace(part)
|
||||||
|
if p != "" {
|
||||||
|
out[p] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
+93
-12
@@ -24,6 +24,7 @@ type Manager struct {
|
|||||||
filesPath string
|
filesPath string
|
||||||
mountsDir string
|
mountsDir string
|
||||||
poller *Poller
|
poller *Poller
|
||||||
|
statsPoller *StatsPoller
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManager creates a new mount manager
|
// NewManager creates a new mount manager
|
||||||
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
|
|||||||
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One-shot migration for legacy NFS entries — disable rather than fail to
|
||||||
|
// start, so operators can fix their config at their own pace.
|
||||||
|
if err := m.migrateLegacyMounts(); err != nil {
|
||||||
|
log.Printf("[Mounts] legacy migration warning: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// migrateLegacyMounts disables any mount entries with type="nfs". The native
|
||||||
|
// NFS handler was removed when the project consolidated on rclone; an NFS
|
||||||
|
// entry left enabled would loop the watcher with mount errors.
|
||||||
|
func (m *Manager) migrateLegacyMounts() error {
|
||||||
|
mountsFile, err := m.readConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
dirty := false
|
||||||
|
for i := range mountsFile.Mounts {
|
||||||
|
mnt := &mountsFile.Mounts[i]
|
||||||
|
if string(mnt.Type) == "nfs" {
|
||||||
|
if mnt.Enabled || mnt.DesiredMounted {
|
||||||
|
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
|
||||||
|
mnt.Enabled = false
|
||||||
|
mnt.DesiredMounted = false
|
||||||
|
dirty = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if dirty {
|
||||||
|
return m.writeConfig(mountsFile)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// SetPoller sets the poller for the manager
|
// SetPoller sets the poller for the manager
|
||||||
func (m *Manager) SetPoller(poller *Poller) {
|
func (m *Manager) SetPoller(poller *Poller) {
|
||||||
m.poller = poller
|
m.poller = poller
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
|
||||||
|
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
|
||||||
|
// set; remains nil-safe when stats are disabled.
|
||||||
|
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
|
||||||
|
m.statsPoller = sp
|
||||||
|
}
|
||||||
|
|
||||||
// ensureDirs creates required directories
|
// ensureDirs creates required directories
|
||||||
func (m *Manager) ensureDirs() error {
|
func (m *Manager) ensureDirs() error {
|
||||||
dirs := []string{
|
dirs := []string{
|
||||||
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
|
|||||||
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
||||||
statuses[i].PollingActive = pollingStatus.Active
|
statuses[i].PollingActive = pollingStatus.Active
|
||||||
statuses[i].LastPolledScan = pollingStatus.LastScan
|
statuses[i].LastPolledScan = pollingStatus.LastScan
|
||||||
|
statuses[i].Scanning = pollingStatus.Scanning
|
||||||
|
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||||
|
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||||
|
statuses[i].NextScanAt = pollingStatus.NextScanAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.statsPoller != nil {
|
||||||
|
for i := range statuses {
|
||||||
|
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
|
|||||||
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
||||||
status.PollingActive = pollingStatus.Active
|
status.PollingActive = pollingStatus.Active
|
||||||
status.LastPolledScan = pollingStatus.LastScan
|
status.LastPolledScan = pollingStatus.LastScan
|
||||||
|
status.Scanning = pollingStatus.Scanning
|
||||||
|
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||||
|
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||||
|
status.NextScanAt = pollingStatus.NextScanAt
|
||||||
|
}
|
||||||
|
if m.statsPoller != nil {
|
||||||
|
status.IOStats = m.statsPoller.GetStats(mount.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return status, nil
|
return status, nil
|
||||||
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
return nil, fmt.Errorf("mount name is required")
|
return nil, fmt.Errorf("mount name is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
// "nfs" used to be valid; the native handler is gone. Reject loudly so the
|
||||||
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
|
// caller knows their integration needs to migrate to a Samba re-export or
|
||||||
|
// switch to a supported protocol.
|
||||||
|
if req.Type == MountType("nfs") {
|
||||||
|
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||||
|
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate type-specific fields
|
// Validate type-specific fields
|
||||||
switch req.Type {
|
switch req.Type {
|
||||||
case MountTypeNFS:
|
|
||||||
if req.NFSServer == "" || req.NFSPath == "" {
|
|
||||||
return nil, fmt.Errorf("NFS server and path are required")
|
|
||||||
}
|
|
||||||
case MountTypeSMB:
|
case MountTypeSMB:
|
||||||
if req.SMBServer == "" || req.SMBShare == "" {
|
if req.SMBServer == "" || req.SMBShare == "" {
|
||||||
return nil, fmt.Errorf("SMB server and share are required")
|
return nil, fmt.Errorf("SMB server and share are required")
|
||||||
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
enabled = *req.Enabled
|
enabled = *req.Enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default polling settings
|
// Polling defaults on for both supported types — neither SMB-via-rclone nor
|
||||||
pollingEnabled := false
|
// generic rclone delivers kernel inotify, so a periodic re-scan is the only
|
||||||
|
// way to pick up upstream changes. Explicit PollingEnabled in the request
|
||||||
|
// wins.
|
||||||
|
pollingEnabled := true
|
||||||
if req.PollingEnabled != nil {
|
if req.PollingEnabled != nil {
|
||||||
pollingEnabled = *req.PollingEnabled
|
pollingEnabled = *req.PollingEnabled
|
||||||
}
|
}
|
||||||
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
Enabled: enabled,
|
Enabled: enabled,
|
||||||
DesiredMounted: enabled, // Auto-mount if enabled
|
DesiredMounted: enabled, // Auto-mount if enabled
|
||||||
MountPath: mountPath,
|
MountPath: mountPath,
|
||||||
Options: req.Options,
|
CacheMaxSize: req.CacheMaxSize,
|
||||||
|
CacheMaxAge: req.CacheMaxAge,
|
||||||
|
DirCacheTime: req.DirCacheTime,
|
||||||
PollingEnabled: pollingEnabled,
|
PollingEnabled: pollingEnabled,
|
||||||
PollingIntervalMs: pollingIntervalMs,
|
PollingIntervalMs: pollingIntervalMs,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type-specific fields
|
// Type-specific fields
|
||||||
switch req.Type {
|
switch req.Type {
|
||||||
case MountTypeNFS:
|
|
||||||
mount.NFSServer = req.NFSServer
|
|
||||||
mount.NFSPath = req.NFSPath
|
|
||||||
case MountTypeSMB:
|
case MountTypeSMB:
|
||||||
mount.SMBServer = req.SMBServer
|
mount.SMBServer = req.SMBServer
|
||||||
mount.SMBShare = req.SMBShare
|
mount.SMBShare = req.SMBShare
|
||||||
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
|
||||||
|
// don't validate format here; rclone rejects bad values at mount time.
|
||||||
|
if val, ok := updates["cacheMaxSize"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.CacheMaxSize = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if val, ok := updates["cacheMaxAge"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.CacheMaxAge = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if val, ok := updates["dirCacheTime"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.DirCacheTime = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := m.writeConfig(mountsFile); err != nil {
|
if err := m.writeConfig(mountsFile); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
+166
-29
@@ -9,6 +9,25 @@ import (
|
|||||||
// ScanFunc is a function type for scanning a mount path
|
// ScanFunc is a function type for scanning a mount path
|
||||||
type ScanFunc func(path string) (int, error)
|
type ScanFunc func(path string) (int, error)
|
||||||
|
|
||||||
|
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
|
||||||
|
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
|
||||||
|
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
|
||||||
|
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
|
||||||
|
// large remote shares (CORN, etc.) from monopolising the network/CPU.
|
||||||
|
const IdleFactor = 9
|
||||||
|
|
||||||
|
// InitialScanDelay is how long a freshly-started poller waits before its first
|
||||||
|
// scan. Avoids hammering the system at container startup when many mounts come
|
||||||
|
// up at once.
|
||||||
|
const InitialScanDelay = 2 * time.Second
|
||||||
|
|
||||||
|
// ReconcileInterval drives the periodic resync loop that keeps poller
|
||||||
|
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
|
||||||
|
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
|
||||||
|
// — without periodic reconciliation, the poller would never observe the
|
||||||
|
// transition from `mounted=false` to `mounted=true` and never start scanning.
|
||||||
|
const ReconcileInterval = 10 * time.Second
|
||||||
|
|
||||||
// mountPoller manages polling for a single mount
|
// mountPoller manages polling for a single mount
|
||||||
type mountPoller struct {
|
type mountPoller struct {
|
||||||
mountID string
|
mountID string
|
||||||
@@ -20,7 +39,11 @@ type mountPoller struct {
|
|||||||
// PollingStatus holds the polling state for a mount
|
// PollingStatus holds the polling state for a mount
|
||||||
type PollingStatus struct {
|
type PollingStatus struct {
|
||||||
Active bool `json:"active"`
|
Active bool `json:"active"`
|
||||||
LastScan int64 `json:"lastScan,omitempty"`
|
LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
|
||||||
|
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
|
||||||
|
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
|
||||||
|
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
|
||||||
|
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
|
||||||
}
|
}
|
||||||
|
|
||||||
// Poller manages per-mount polling goroutines
|
// Poller manages per-mount polling goroutines
|
||||||
@@ -32,6 +55,7 @@ type Poller struct {
|
|||||||
status map[string]*PollingStatus
|
status map[string]*PollingStatus
|
||||||
|
|
||||||
running bool
|
running bool
|
||||||
|
reconcileStop chan struct{}
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,15 +78,35 @@ func (p *Poller) Start() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
p.running = true
|
p.running = true
|
||||||
|
p.reconcileStop = make(chan struct{})
|
||||||
|
|
||||||
log.Println("[Poller] Starting mount polling service")
|
log.Println("[Poller] Starting mount polling service")
|
||||||
|
|
||||||
// Initial sync with mount configurations
|
// Initial sync with mount configurations
|
||||||
go p.syncWithMounts()
|
go p.syncWithMounts()
|
||||||
|
|
||||||
|
// Periodic reconciliation — picks up mount-watcher.sh transitions
|
||||||
|
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
|
||||||
|
go p.reconcileLoop(p.reconcileStop)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
|
||||||
|
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
|
||||||
|
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
|
||||||
|
ticker := time.NewTicker(ReconcileInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
p.syncWithMounts()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Stop stops all polling goroutines
|
// Stop stops all polling goroutines
|
||||||
func (p *Poller) Stop() error {
|
func (p *Poller) Stop() error {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
@@ -75,6 +119,11 @@ func (p *Poller) Stop() error {
|
|||||||
|
|
||||||
log.Println("[Poller] Stopping mount polling service")
|
log.Println("[Poller] Stopping mount polling service")
|
||||||
|
|
||||||
|
if p.reconcileStop != nil {
|
||||||
|
close(p.reconcileStop)
|
||||||
|
p.reconcileStop = nil
|
||||||
|
}
|
||||||
|
|
||||||
// Stop all pollers
|
// Stop all pollers
|
||||||
for id, poller := range p.pollers {
|
for id, poller := range p.pollers {
|
||||||
p.stopPoller(poller)
|
p.stopPoller(poller)
|
||||||
@@ -127,7 +176,7 @@ func (p *Poller) syncWithMounts() {
|
|||||||
existingPoller, exists := p.pollers[mount.ID]
|
existingPoller, exists := p.pollers[mount.ID]
|
||||||
|
|
||||||
if shouldPoll {
|
if shouldPoll {
|
||||||
// Calculate interval
|
// Calculate interval (used as the floor for adaptive scheduling)
|
||||||
interval := mount.PollingIntervalMs
|
interval := mount.PollingIntervalMs
|
||||||
if interval < MinPollingIntervalMs {
|
if interval < MinPollingIntervalMs {
|
||||||
interval = DefaultPollingIntervalMs
|
interval = DefaultPollingIntervalMs
|
||||||
@@ -135,7 +184,7 @@ func (p *Poller) syncWithMounts() {
|
|||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
// Start new poller
|
// Start new poller
|
||||||
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
|
log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
|
||||||
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
||||||
}
|
}
|
||||||
} else if exists {
|
} else if exists {
|
||||||
@@ -156,7 +205,13 @@ func (p *Poller) syncWithMounts() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startPollerForMount starts a polling goroutine for a mount
|
// startPollerForMount starts the adaptive polling goroutine for a mount.
|
||||||
|
//
|
||||||
|
// The loop runs scans sequentially: scan → measure duration → sleep
|
||||||
|
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
|
||||||
|
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
|
||||||
|
// cycle on slow shares, while keeping fresh checks on small mounts via the
|
||||||
|
// configured interval floor.
|
||||||
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
||||||
poller := &mountPoller{
|
poller := &mountPoller{
|
||||||
mountID: mountID,
|
mountID: mountID,
|
||||||
@@ -170,17 +225,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
|
|||||||
}
|
}
|
||||||
p.status[mountID].Active = true
|
p.status[mountID].Active = true
|
||||||
|
|
||||||
|
floor := time.Duration(intervalMs) * time.Millisecond
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
|
// Initial delay so a freshly-mounted share doesn't get hammered the
|
||||||
defer ticker.Stop()
|
// instant the poller starts.
|
||||||
|
nextDelay := InitialScanDelay
|
||||||
|
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-poller.stopChan:
|
case <-poller.stopChan:
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-time.After(nextDelay):
|
||||||
p.runPollScan(mountID, mountPath)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Re-check stop after the wake-up to avoid running a scan during shutdown.
|
||||||
|
select {
|
||||||
|
case <-poller.stopChan:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
duration, err := p.executeScan(mountID, mountPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adaptive backoff: idle for IdleFactor × last duration, with the
|
||||||
|
// configured interval as a floor so tiny mounts still scan often.
|
||||||
|
nextDelay = duration * IdleFactor
|
||||||
|
if nextDelay < floor {
|
||||||
|
nextDelay = floor
|
||||||
|
}
|
||||||
|
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -199,54 +277,109 @@ func (p *Poller) stopPoller(poller *mountPoller) {
|
|||||||
// Update status
|
// Update status
|
||||||
if status, exists := p.status[poller.mountID]; exists {
|
if status, exists := p.status[poller.mountID]; exists {
|
||||||
status.Active = false
|
status.Active = false
|
||||||
|
status.NextScanAt = 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runPollScan executes a scan for a mount
|
// executeScan runs scanFunc with timing + status tracking. Both the adaptive
|
||||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
// loop and manual TriggerScan funnel through here so the UI sees a consistent
|
||||||
|
// "scanning / last duration / last scan" picture regardless of which path
|
||||||
|
// kicked it off. Returns the scan duration so the caller can compute the next
|
||||||
|
// adaptive delay.
|
||||||
|
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
|
||||||
if p.scanFunc == nil {
|
if p.scanFunc == nil {
|
||||||
return
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
startMs := start.UnixMilli()
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.Scanning = true
|
||||||
|
status.CurrentScanStartedAt = startMs
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
fileCount, err := p.scanFunc(mountPath)
|
fileCount, err := p.scanFunc(mountPath)
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update status
|
end := time.Now()
|
||||||
|
duration := end.Sub(start)
|
||||||
|
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
if status, exists := p.status[mountID]; exists {
|
if status, exists := p.status[mountID]; exists {
|
||||||
status.LastScan = NowMS()
|
status.Scanning = false
|
||||||
|
status.CurrentScanStartedAt = 0
|
||||||
|
status.LastScan = end.UnixMilli()
|
||||||
|
status.LastScanDurationMs = duration.Milliseconds()
|
||||||
}
|
}
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
|
if err == nil {
|
||||||
|
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TriggerScan triggers an immediate scan for a mount
|
return duration, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// setNextScanAt records the projected start time of the next adaptive scan
|
||||||
|
// (used purely for the UI countdown — the loop's own timer is authoritative).
|
||||||
|
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.NextScanAt = at.UnixMilli()
|
||||||
|
}
|
||||||
|
|
||||||
|
// runPollScan is kept for backwards compatibility with any external caller —
|
||||||
|
// new code should go through executeScan via the adaptive loop.
|
||||||
|
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||||
|
_, _ = p.executeScan(mountID, mountPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
|
||||||
|
// Runs synchronously; concurrent calls with the adaptive loop are possible
|
||||||
|
// but rare in practice — both paths update the same status atomically.
|
||||||
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
||||||
if p.scanFunc == nil {
|
if p.scanFunc == nil {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
startMs := start.UnixMilli()
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.Scanning = true
|
||||||
|
status.CurrentScanStartedAt = startMs
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
fileCount, err := p.scanFunc(mountPath)
|
fileCount, err := p.scanFunc(mountPath)
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
end := time.Now()
|
||||||
}
|
duration := end.Sub(start)
|
||||||
|
|
||||||
// Update status
|
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
if status, exists := p.status[mountID]; exists {
|
if status, exists := p.status[mountID]; exists {
|
||||||
status.LastScan = NowMS()
|
status.Scanning = false
|
||||||
} else {
|
status.CurrentScanStartedAt = 0
|
||||||
p.status[mountID] = &PollingStatus{
|
status.LastScan = end.UnixMilli()
|
||||||
LastScan: NowMS(),
|
status.LastScanDurationMs = duration.Milliseconds()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
return fileCount, nil
|
return fileCount, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPollingStatus returns the polling status for a mount
|
// GetPollingStatus returns the polling status for a mount
|
||||||
@@ -258,6 +391,10 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
|
|||||||
return &PollingStatus{
|
return &PollingStatus{
|
||||||
Active: status.Active,
|
Active: status.Active,
|
||||||
LastScan: status.LastScan,
|
LastScan: status.LastScan,
|
||||||
|
Scanning: status.Scanning,
|
||||||
|
CurrentScanStartedAt: status.CurrentScanStartedAt,
|
||||||
|
LastScanDurationMs: status.LastScanDurationMs,
|
||||||
|
NextScanAt: status.NextScanAt,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &PollingStatus{}
|
return &PollingStatus{}
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package mounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
|
||||||
|
// time. The poller keeps the previous sample per mount and computes rates
|
||||||
|
// (Bps, transfers/sec) from the delta to the latest sample.
|
||||||
|
//
|
||||||
|
// Post-consolidation: rclone is the only sample source, since all mount types
|
||||||
|
// execute through the local rclone daemon.
|
||||||
|
type StatsSample struct {
|
||||||
|
TakenAt time.Time
|
||||||
|
Source string // "rclone"
|
||||||
|
BytesRead int64
|
||||||
|
BytesWritten int64
|
||||||
|
ReadOps int64
|
||||||
|
WriteOps int64
|
||||||
|
// File-level transfer count from rclone's RC API. Distinct from ReadOps
|
||||||
|
// (which is unused for rclone — kernel ops counters are not exposed
|
||||||
|
// through FUSE).
|
||||||
|
Transfers int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// rcloneCoreStats matches the RC API response from POST /core/stats.
|
||||||
|
type rcloneCoreStats struct {
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
TotalBytes int64 `json:"totalBytes"`
|
||||||
|
Transfers int64 `json:"transfers"`
|
||||||
|
TotalTransfers int64 `json:"totalTransfers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleRclone queries the local rclone RC API for the cumulative bytes +
|
||||||
|
// transfers counters. The numbers are global across all active mounts on this
|
||||||
|
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
|
||||||
|
// sample for every mounted entry on a tick.
|
||||||
|
func SampleRclone(ctx context.Context) (*StatsSample, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build rclone stats request: %w", err)
|
||||||
|
}
|
||||||
|
req.SetBasicAuth("admin", "admin")
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 3 * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats rcloneCoreStats
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode rclone stats: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StatsSample{
|
||||||
|
Source: "rclone",
|
||||||
|
TakenAt: time.Now(),
|
||||||
|
BytesRead: stats.Bytes,
|
||||||
|
Transfers: stats.Transfers,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
package mounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
|
||||||
|
// spot — short enough that throughput numbers feel live, long enough that
|
||||||
|
// /proc reads aren't a load source themselves.
|
||||||
|
const StatsInterval = 5 * time.Second
|
||||||
|
|
||||||
|
// StatsPoller samples cumulative counters from the local rclone RC API and
|
||||||
|
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
|
||||||
|
//
|
||||||
|
// Post-consolidation note: every mount type executes through rclone, so
|
||||||
|
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
|
||||||
|
// rclone is the only counter source. core/stats is daemon-global across all
|
||||||
|
// active mounts; the same sample is applied to each mount's status. If
|
||||||
|
// per-mount byte counters become important, the path forward is rclone stats
|
||||||
|
// groups (out of scope for this consolidation).
|
||||||
|
type StatsPoller struct {
|
||||||
|
manager *Manager
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
lastSamples map[string]*StatsSample // mountID -> previous raw sample
|
||||||
|
latestStats map[string]*MountIOStats
|
||||||
|
stopChan chan struct{}
|
||||||
|
running bool
|
||||||
|
pollInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStatsPoller constructs a StatsPoller backed by the given manager.
|
||||||
|
func NewStatsPoller(manager *Manager) *StatsPoller {
|
||||||
|
return &StatsPoller{
|
||||||
|
manager: manager,
|
||||||
|
lastSamples: make(map[string]*StatsSample),
|
||||||
|
latestStats: make(map[string]*MountIOStats),
|
||||||
|
pollInterval: StatsInterval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start launches the sampling loop. Idempotent.
|
||||||
|
func (p *StatsPoller) Start() {
|
||||||
|
p.mu.Lock()
|
||||||
|
if p.running {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.running = true
|
||||||
|
p.stopChan = make(chan struct{})
|
||||||
|
stop := p.stopChan
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
|
||||||
|
go p.loop(stop)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop terminates the sampling loop. Idempotent.
|
||||||
|
func (p *StatsPoller) Stop() {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if !p.running {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.running = false
|
||||||
|
if p.stopChan != nil {
|
||||||
|
close(p.stopChan)
|
||||||
|
p.stopChan = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns the latest computed IO stats for a mount, or nil if no
|
||||||
|
// stats have been observed yet.
|
||||||
|
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
s, ok := p.latestStats[mountID]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Return a copy so the caller can't mutate our cache.
|
||||||
|
cp := *s
|
||||||
|
return &cp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *StatsPoller) loop(stop <-chan struct{}) {
|
||||||
|
// Take the first sample immediately so rates start populating on the
|
||||||
|
// second tick instead of waiting two intervals.
|
||||||
|
p.tick()
|
||||||
|
|
||||||
|
t := time.NewTicker(p.pollInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
p.tick()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *StatsPoller) tick() {
|
||||||
|
mounts, err := p.manager.listMountsWithoutPollingStatus()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[StatsPoller] listMounts failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample rclone once per tick — its RC API gives global stats across all
|
||||||
|
// active rclone mounts on this host. Both supported mount types (smb,
|
||||||
|
// rclone) execute through the same rclone daemon, so the same sample is
|
||||||
|
// applied to every mounted entry.
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
sample, err := SampleRclone(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[StatsPoller] rclone sample failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if sample == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, m := range mounts {
|
||||||
|
if !m.Mounted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
prev := p.lastSamples[m.ID]
|
||||||
|
p.lastSamples[m.ID] = sample
|
||||||
|
stats := computeStats(prev, sample)
|
||||||
|
p.latestStats[m.ID] = stats
|
||||||
|
p.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeStats turns a (previous, current) raw-sample pair into the rate-
|
||||||
|
// based MountIOStats the API exposes. With no previous sample the rates are
|
||||||
|
// zero — operators will see cumulative counters but Bps/IOPS only on the
|
||||||
|
// second poll.
|
||||||
|
func computeStats(prev, cur *StatsSample) *MountIOStats {
|
||||||
|
out := &MountIOStats{
|
||||||
|
Source: cur.Source,
|
||||||
|
BytesRead: cur.BytesRead,
|
||||||
|
BytesWritten: cur.BytesWritten,
|
||||||
|
ReadOps: cur.ReadOps,
|
||||||
|
WriteOps: cur.WriteOps,
|
||||||
|
LastSampleAt: cur.TakenAt.UnixMilli(),
|
||||||
|
}
|
||||||
|
if prev == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
|
||||||
|
if dt <= 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
out.IntervalMs = int64(dt * 1000)
|
||||||
|
|
||||||
|
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
|
||||||
|
// negative deltas as "no movement" rather than letting the rate go
|
||||||
|
// negative or implausibly huge.
|
||||||
|
if d := cur.BytesRead - prev.BytesRead; d > 0 {
|
||||||
|
out.ReadBps = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
|
||||||
|
out.WriteBps = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.ReadOps - prev.ReadOps; d > 0 {
|
||||||
|
out.ReadIops = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.WriteOps - prev.WriteOps; d > 0 {
|
||||||
|
out.WriteIops = float64(d) / dt
|
||||||
|
}
|
||||||
|
if cur.Source == "rclone" {
|
||||||
|
if d := cur.Transfers - prev.Transfers; d > 0 {
|
||||||
|
out.TransfersPerSec = float64(d) / dt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
+64
-14
@@ -2,11 +2,19 @@ package mounts
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
// MountType represents the type of mount
|
// MountType represents the type of mount.
|
||||||
|
//
|
||||||
|
// All types execute through the local rclone daemon. "smb" is a UX-level alias
|
||||||
|
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
|
||||||
|
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
|
||||||
|
// the generic escape hatch for any pre-defined remote in rclone.conf.
|
||||||
|
//
|
||||||
|
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
|
||||||
|
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
|
||||||
|
// regardless.
|
||||||
type MountType string
|
type MountType string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
MountTypeNFS MountType = "nfs"
|
|
||||||
MountTypeSMB MountType = "smb"
|
MountTypeSMB MountType = "smb"
|
||||||
MountTypeRclone MountType = "rclone"
|
MountTypeRclone MountType = "rclone"
|
||||||
)
|
)
|
||||||
@@ -17,6 +25,15 @@ const (
|
|||||||
MinPollingIntervalMs = 5000 // 5 seconds
|
MinPollingIntervalMs = 5000 // 5 seconds
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// VFS cache defaults — applied per-mount when the caller leaves the field
|
||||||
|
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
|
||||||
|
// Duration) untouched.
|
||||||
|
const (
|
||||||
|
DefaultCacheMaxSize = "50G"
|
||||||
|
DefaultCacheMaxAge = "24h"
|
||||||
|
DefaultDirCacheTime = "5m"
|
||||||
|
)
|
||||||
|
|
||||||
// MountConfig represents a mount configuration
|
// MountConfig represents a mount configuration
|
||||||
type MountConfig struct {
|
type MountConfig struct {
|
||||||
// Common fields
|
// Common fields
|
||||||
@@ -26,23 +43,26 @@ type MountConfig struct {
|
|||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
DesiredMounted bool `json:"desiredMounted"`
|
DesiredMounted bool `json:"desiredMounted"`
|
||||||
MountPath string `json:"mountPath"`
|
MountPath string `json:"mountPath"`
|
||||||
Options string `json:"options,omitempty"`
|
|
||||||
|
|
||||||
// NFS-specific fields
|
// SMB-specific fields (also valid for type=smb routed through rclone-smb)
|
||||||
NFSServer string `json:"nfsServer,omitempty"`
|
|
||||||
NFSPath string `json:"nfsPath,omitempty"`
|
|
||||||
|
|
||||||
// SMB-specific fields
|
|
||||||
SMBServer string `json:"smbServer,omitempty"`
|
SMBServer string `json:"smbServer,omitempty"`
|
||||||
SMBShare string `json:"smbShare,omitempty"`
|
SMBShare string `json:"smbShare,omitempty"`
|
||||||
SMBUsername string `json:"smbUsername,omitempty"`
|
SMBUsername string `json:"smbUsername,omitempty"`
|
||||||
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
||||||
SMBDomain string `json:"smbDomain,omitempty"`
|
SMBDomain string `json:"smbDomain,omitempty"`
|
||||||
|
|
||||||
// rclone-specific fields
|
// rclone-specific fields (escape hatch — references a pre-defined remote)
|
||||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||||
RclonePath string `json:"rclonePath,omitempty"`
|
RclonePath string `json:"rclonePath,omitempty"`
|
||||||
|
|
||||||
|
// VFS cache configuration — applied to the rclone mount call. All three
|
||||||
|
// fields are optional; blanks fall back to Default* constants above.
|
||||||
|
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
|
||||||
|
// DirCacheTime use rclone's Duration ("24h", "30m").
|
||||||
|
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||||
|
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||||
|
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||||
|
|
||||||
// Polling configuration
|
// Polling configuration
|
||||||
PollingEnabled bool `json:"pollingEnabled"`
|
PollingEnabled bool `json:"pollingEnabled"`
|
||||||
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
||||||
@@ -56,6 +76,36 @@ type MountStatus struct {
|
|||||||
LastChecked int64 `json:"lastChecked"`
|
LastChecked int64 `json:"lastChecked"`
|
||||||
PollingActive bool `json:"pollingActive,omitempty"`
|
PollingActive bool `json:"pollingActive,omitempty"`
|
||||||
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
||||||
|
// Progress tracking — surface the poller's adaptive scan state to the UI.
|
||||||
|
Scanning bool `json:"scanning,omitempty"`
|
||||||
|
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
|
||||||
|
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
|
||||||
|
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
|
||||||
|
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
|
||||||
|
// /proc/self/mountstats, or rclone's RC API depending on mount type.
|
||||||
|
IOStats *MountIOStats `json:"ioStats,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MountIOStats captures throughput + operation rate for a single mount.
|
||||||
|
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
|
||||||
|
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
|
||||||
|
// the last two samples divided by the interval.
|
||||||
|
type MountIOStats struct {
|
||||||
|
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
|
||||||
|
BytesRead int64 `json:"bytesRead"`
|
||||||
|
BytesWritten int64 `json:"bytesWritten,omitempty"`
|
||||||
|
ReadBps float64 `json:"readBps"`
|
||||||
|
WriteBps float64 `json:"writeBps,omitempty"`
|
||||||
|
ReadOps int64 `json:"readOps,omitempty"`
|
||||||
|
WriteOps int64 `json:"writeOps,omitempty"`
|
||||||
|
// ReadIops and WriteIops are op counters per second — number of read/write
|
||||||
|
// requests issued by the kernel against the share. Distinct from
|
||||||
|
// TransfersPerSec which is a file-level metric only relevant to rclone.
|
||||||
|
ReadIops float64 `json:"readIops,omitempty"`
|
||||||
|
WriteIops float64 `json:"writeIops,omitempty"`
|
||||||
|
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
|
||||||
|
LastSampleAt int64 `json:"lastSampleAt"` // ms
|
||||||
|
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
|
||||||
}
|
}
|
||||||
|
|
||||||
// MountsFile represents the mounts.json file structure
|
// MountsFile represents the mounts.json file structure
|
||||||
@@ -75,11 +125,6 @@ type CreateMountRequest struct {
|
|||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Type MountType `json:"type"`
|
Type MountType `json:"type"`
|
||||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||||
Options string `json:"options,omitempty"`
|
|
||||||
|
|
||||||
// NFS
|
|
||||||
NFSServer string `json:"nfsServer,omitempty"`
|
|
||||||
NFSPath string `json:"nfsPath,omitempty"`
|
|
||||||
|
|
||||||
// SMB
|
// SMB
|
||||||
SMBServer string `json:"smbServer,omitempty"`
|
SMBServer string `json:"smbServer,omitempty"`
|
||||||
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
|
|||||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||||
RclonePath string `json:"rclonePath,omitempty"`
|
RclonePath string `json:"rclonePath,omitempty"`
|
||||||
|
|
||||||
|
// VFS cache (optional — falls back to Default* constants when unset)
|
||||||
|
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||||
|
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||||
|
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||||
|
|
||||||
// Polling
|
// Polling
|
||||||
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
||||||
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
|
||||||
|
// Trailing slash because ZIP entry names are forward-slash relative paths.
|
||||||
|
const PluginOutputArchivePrefix = "cache/plugin-output/"
|
||||||
|
|
||||||
|
// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
|
||||||
|
// file under it into zw at PluginOutputArchivePrefix + relPath.
|
||||||
|
//
|
||||||
|
// Returns the number of files added. A missing srcDir is not an error: it
|
||||||
|
// just means no plugin has produced output yet, so the section stays empty.
|
||||||
|
func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
|
||||||
|
info, err := os.Stat(srcDir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("stat %s: %w", srcDir, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return 0, fmt.Errorf("%s is not a directory", srcDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(srcDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
|
||||||
|
|
||||||
|
w, err := zw.Create(entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if _, err := io.Copy(w, f); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return count, fmt.Errorf("walk %s: %w", srcDir, err)
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
|
||||||
|
// the ZIP and writes it under dstDir. Existing files are overwritten when
|
||||||
|
// overwrite=true; otherwise they're left alone (treats existing files as the
|
||||||
|
// "mine wins" choice — same toggle the metadata import uses).
|
||||||
|
//
|
||||||
|
// Path traversal is rejected: any entry whose relative path resolves outside
|
||||||
|
// dstDir is skipped with a warning.
|
||||||
|
func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
|
||||||
|
if dstDir == "" {
|
||||||
|
return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dstDir, 0o755); err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanRoot, err := filepath.Abs(dstDir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(f.Name, "/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
|
||||||
|
dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
|
||||||
|
|
||||||
|
// Reject anything that escapes the dst root.
|
||||||
|
if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
|
||||||
|
log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !overwrite {
|
||||||
|
if _, err := os.Stat(dst); err == nil {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||||
|
return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeZipFile(f, dst); err != nil {
|
||||||
|
return written, skipped, err
|
||||||
|
}
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
return written, skipped, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeZipFile(f *zip.File, dst string) error {
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open zip entry %s: %w", f.Name, err)
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
|
||||||
|
out, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
defer out.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(out, rc); err != nil {
|
||||||
|
return fmt.Errorf("write %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExportOptions controls what an export includes.
|
||||||
|
type ExportOptions struct {
|
||||||
|
IncludePluginOutput bool
|
||||||
|
// PluginOutputDir is the absolute path on disk that holds plugin-produced
|
||||||
|
// artifacts (typically $FILES_PATH/plugin). Required when
|
||||||
|
// IncludePluginOutput is true.
|
||||||
|
PluginOutputDir string
|
||||||
|
// SourceHost is recorded in the manifest so imports can attribute
|
||||||
|
// where the snapshot came from (defaults to OS hostname).
|
||||||
|
SourceHost string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exporter writes snapshot ZIPs from a live Redis store.
|
||||||
|
type Exporter struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewExporter(stor *storage.Client) *Exporter {
|
||||||
|
return &Exporter{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export streams a ZIP archive to w. The archive layout is:
|
||||||
|
//
|
||||||
|
// manifest.json
|
||||||
|
// index.json
|
||||||
|
// metadata/{cid}.json (one per known CID)
|
||||||
|
//
|
||||||
|
// Optional sections (hash-index, plugin-output) are gated by ExportOptions
|
||||||
|
// but are not yet implemented — they live on filesystems meta-core does not
|
||||||
|
// own, and need cross-service collection. The flag is plumbed for forward
|
||||||
|
// compatibility but does nothing today.
|
||||||
|
func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
|
||||||
|
if e.storage == nil || !e.storage.IsConnected() {
|
||||||
|
return Manifest{}, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
hashIDs, err := e.storage.GetAllHashIDs()
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
zw := zip.NewWriter(w)
|
||||||
|
defer zw.Close()
|
||||||
|
|
||||||
|
// 1. manifest.json — written last after we know FileCount, but we need
|
||||||
|
// the entry order to put it first in the archive for readers. Solve
|
||||||
|
// by buffering in memory: it's tiny.
|
||||||
|
host := opts.SourceHost
|
||||||
|
if host == "" {
|
||||||
|
if h, err := os.Hostname(); err == nil {
|
||||||
|
host = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
manifest := Manifest{
|
||||||
|
SchemaVersion: ManifestSchemaVersion,
|
||||||
|
GeneratedAt: time.Now().UTC(),
|
||||||
|
SourceHost: host,
|
||||||
|
FileCount: len(hashIDs),
|
||||||
|
Includes: Includes{
|
||||||
|
Metadata: true,
|
||||||
|
PluginOutput: opts.IncludePluginOutput,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := writeJSON(zw, ManifestFile, manifest); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. index.json — flat list of CIDs. Redundant with metadata/ but
|
||||||
|
// cheap and useful for readers that only need the keyset.
|
||||||
|
if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. metadata/{cid}.json — one nested document per CID.
|
||||||
|
for _, hashID := range hashIDs {
|
||||||
|
flat, err := e.storage.GetMetadataFlat(hashID)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(flat) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nested := Reconstruct(flat)
|
||||||
|
entry := MetadataDir + hashID + ".json"
|
||||||
|
if err := writeJSON(zw, entry, nested); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
|
||||||
|
// streams each file into the ZIP. Heavy (~120 MB on a populated dev
|
||||||
|
// stack), so it's gated by the ExportOptions toggle.
|
||||||
|
if opts.IncludePluginOutput {
|
||||||
|
if opts.PluginOutputDir == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
|
||||||
|
}
|
||||||
|
n, err := addPluginOutput(zw, opts.PluginOutputDir)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
|
||||||
|
}
|
||||||
|
log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
|
||||||
|
len(hashIDs), host, opts.IncludePluginOutput)
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSON(zw *zip.Writer, name string, body interface{}) error {
|
||||||
|
w, err := zw.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(body)
|
||||||
|
}
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Flat keys use "/" as the separator. Property paths inside Redis are
|
||||||
|
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
|
||||||
|
const flatSep = "/"
|
||||||
|
|
||||||
|
// numericKey matches a path segment that is a non-negative decimal integer.
|
||||||
|
// Used to detect array slots (".../video/0/codec").
|
||||||
|
var numericKey = regexp.MustCompile(`^\d+$`)
|
||||||
|
|
||||||
|
// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
|
||||||
|
// Keys are property paths (no leading slash, no file:{cid}/ prefix).
|
||||||
|
//
|
||||||
|
// Sibling keys whose path segments are all sequential integers starting at 0
|
||||||
|
// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
|
||||||
|
// are coerced from string to int/float/bool/null when they parse cleanly so
|
||||||
|
// the exported JSON is human-readable; on import we round-trip back to strings.
|
||||||
|
func Reconstruct(flat map[string]string) interface{} {
|
||||||
|
root := map[string]interface{}{}
|
||||||
|
for path, value := range flat {
|
||||||
|
parts := strings.Split(path, flatSep)
|
||||||
|
setPath(root, parts, parseValue(value))
|
||||||
|
}
|
||||||
|
return convertNumericMapsToArrays(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
|
||||||
|
// flat property map whose values are all strings (Redis-ready).
|
||||||
|
//
|
||||||
|
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
|
||||||
|
// values are stored as "" to preserve the field's presence — this matches
|
||||||
|
// what flattenMetadata in the TS implementation does.
|
||||||
|
func Flatten(value interface{}) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
flattenInto(value, "", out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cur := root
|
||||||
|
for i, part := range parts {
|
||||||
|
if i == len(parts)-1 {
|
||||||
|
cur[part] = leaf
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next, ok := cur[part].(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
next = map[string]interface{}{}
|
||||||
|
cur[part] = next
|
||||||
|
}
|
||||||
|
cur = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertNumericMapsToArrays(v interface{}) interface{} {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case map[string]interface{}:
|
||||||
|
// Recurse first so child arrays are detected before parent.
|
||||||
|
for k, child := range t {
|
||||||
|
t[k] = convertNumericMapsToArrays(child)
|
||||||
|
}
|
||||||
|
if !looksLikeArray(t) {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
// Sort numerically and emit as []interface{}.
|
||||||
|
idx := make([]int, 0, len(t))
|
||||||
|
for k := range t {
|
||||||
|
n, _ := strconv.Atoi(k)
|
||||||
|
idx = append(idx, n)
|
||||||
|
}
|
||||||
|
sort.Ints(idx)
|
||||||
|
arr := make([]interface{}, len(idx))
|
||||||
|
for i, n := range idx {
|
||||||
|
arr[i] = t[strconv.Itoa(n)]
|
||||||
|
}
|
||||||
|
return arr
|
||||||
|
case []interface{}:
|
||||||
|
for i := range t {
|
||||||
|
t[i] = convertNumericMapsToArrays(t[i])
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
default:
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// looksLikeArray returns true when every key in m is a non-negative integer
|
||||||
|
// and the keys form a contiguous range starting at 0. Empty maps return false
|
||||||
|
// — we can't tell whether the original was {} or [], so prefer {}.
|
||||||
|
func looksLikeArray(m map[string]interface{}) bool {
|
||||||
|
if len(m) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen := make(map[int]bool, len(m))
|
||||||
|
for k := range m {
|
||||||
|
if !numericKey.MatchString(k) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(k)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen[n] = true
|
||||||
|
}
|
||||||
|
for i := 0; i < len(m); i++ {
|
||||||
|
if !seen[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func flattenInto(v interface{}, path string, out map[string]string) {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case nil:
|
||||||
|
out[path] = ""
|
||||||
|
case map[string]interface{}:
|
||||||
|
if len(t) == 0 && path != "" {
|
||||||
|
// Empty objects don't have a Redis representation; skip.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for k, child := range t {
|
||||||
|
next := k
|
||||||
|
if path != "" {
|
||||||
|
next = path + flatSep + k
|
||||||
|
}
|
||||||
|
flattenInto(child, next, out)
|
||||||
|
}
|
||||||
|
case []interface{}:
|
||||||
|
for i, child := range t {
|
||||||
|
next := strconv.Itoa(i)
|
||||||
|
if path != "" {
|
||||||
|
next = path + flatSep + next
|
||||||
|
}
|
||||||
|
flattenInto(child, next, out)
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
out[path] = t
|
||||||
|
case bool:
|
||||||
|
if t {
|
||||||
|
out[path] = "true"
|
||||||
|
} else {
|
||||||
|
out[path] = "false"
|
||||||
|
}
|
||||||
|
case float64:
|
||||||
|
// JSON numbers always decode to float64; emit ints without decimals.
|
||||||
|
if t == float64(int64(t)) {
|
||||||
|
out[path] = strconv.FormatInt(int64(t), 10)
|
||||||
|
} else {
|
||||||
|
out[path] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
out[path] = fmt.Sprintf("%v", t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseValue mirrors the TS reconstruct logic: convert obvious primitives
|
||||||
|
// back to their typed form for nicer exported JSON, leave anything ambiguous
|
||||||
|
// as a string so we don't lose data (e.g. "5.1" channel layouts).
|
||||||
|
func parseValue(s string) interface{} {
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch s {
|
||||||
|
case "true":
|
||||||
|
return true
|
||||||
|
case "false":
|
||||||
|
return false
|
||||||
|
case "null":
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Only treat as number when the string is a clean integer or decimal,
|
||||||
|
// no leading zeros / trailing junk. Matches the TS regex.
|
||||||
|
if isCleanNumber(s) {
|
||||||
|
if n, err := strconv.ParseFloat(s, 64); err == nil {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||||
|
|
||||||
|
func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Mode controls how an import reconciles existing CIDs.
|
||||||
|
type Mode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ModeReplace deletes every existing property of an imported CID and
|
||||||
|
// writes the snapshot's values verbatim. Untouched CIDs are left alone.
|
||||||
|
ModeReplace Mode = "replace"
|
||||||
|
// ModeMerge keeps unmentioned existing properties and only touches the
|
||||||
|
// fields present in the snapshot. Conflict resolution within merge is
|
||||||
|
// controlled by Conflict.
|
||||||
|
ModeMerge Mode = "merge"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Conflict picks a winner for fields present on both sides during merge.
|
||||||
|
type Conflict string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ConflictMine keeps the existing (target) value when both sides have one.
|
||||||
|
ConflictMine Conflict = "mine"
|
||||||
|
// ConflictSource overwrites the existing value with the snapshot's value.
|
||||||
|
ConflictSource Conflict = "source"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ImportOptions controls how Import applies a snapshot.
|
||||||
|
type ImportOptions struct {
|
||||||
|
Mode Mode
|
||||||
|
Conflict Conflict
|
||||||
|
DryRun bool
|
||||||
|
// PluginOutputDir is the on-disk destination for cache/plugin-output/*
|
||||||
|
// entries. When empty, plugin-output is silently skipped on import.
|
||||||
|
PluginOutputDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
|
||||||
|
type FileResult struct {
|
||||||
|
HashID string `json:"hashId"`
|
||||||
|
Added int `json:"added"` // new property keys written
|
||||||
|
Updated int `json:"updated"` // existing property keys whose value changed
|
||||||
|
Unchanged int `json:"unchanged"` // existing property keys whose value matched
|
||||||
|
Kept int `json:"kept"` // existing property keys preserved by ConflictMine
|
||||||
|
Deleted int `json:"deleted"` // existing property keys removed (replace mode)
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PluginOutputResult summarizes what happened to cache/plugin-output entries.
|
||||||
|
type PluginOutputResult struct {
|
||||||
|
Written int `json:"written"`
|
||||||
|
Skipped int `json:"skipped"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImportResult is the response body for the import endpoint.
|
||||||
|
type ImportResult struct {
|
||||||
|
Mode Mode `json:"mode"`
|
||||||
|
Conflict Conflict `json:"conflict,omitempty"`
|
||||||
|
DryRun bool `json:"dryRun"`
|
||||||
|
TotalFiles int `json:"totalFiles"`
|
||||||
|
FilesOK int `json:"filesOk"`
|
||||||
|
FilesFailed int `json:"filesFailed"`
|
||||||
|
Files []FileResult `json:"files,omitempty"`
|
||||||
|
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Importer applies snapshot ZIPs to a live Redis store.
|
||||||
|
type Importer struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewImporter(stor *storage.Client) *Importer {
|
||||||
|
return &Importer{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
|
||||||
|
// *bytes.Reader after buffering the upload) and applies it to Redis.
|
||||||
|
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
|
||||||
|
if i.storage == nil || !i.storage.IsConnected() {
|
||||||
|
return nil, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
|
||||||
|
return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
|
||||||
|
}
|
||||||
|
if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
|
||||||
|
return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
|
||||||
|
}
|
||||||
|
|
||||||
|
zr, err := zip.NewReader(r, size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open zip: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest Manifest
|
||||||
|
if err := readJSON(zr, ManifestFile, &manifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("read manifest: %w", err)
|
||||||
|
}
|
||||||
|
if manifest.SchemaVersion > ManifestSchemaVersion {
|
||||||
|
return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
|
||||||
|
manifest.SchemaVersion, ManifestSchemaVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := &ImportResult{
|
||||||
|
Mode: opts.Mode,
|
||||||
|
Conflict: opts.Conflict,
|
||||||
|
DryRun: opts.DryRun,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
|
||||||
|
if hashID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fr := i.applyOne(f, hashID, opts)
|
||||||
|
result.TotalFiles++
|
||||||
|
if fr.Error != "" {
|
||||||
|
result.FilesFailed++
|
||||||
|
} else {
|
||||||
|
result.FilesOK++
|
||||||
|
}
|
||||||
|
result.Files = append(result.Files, fr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plugin-output extraction. Only applied on real imports (not dry-run).
|
||||||
|
// If the snapshot doesn't contain a cache/plugin-output/ section, this is
|
||||||
|
// a no-op. If the destination is unset, we skip silently — the manifest's
|
||||||
|
// pluginOutput flag tells the caller it was on the source side either way.
|
||||||
|
if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
|
||||||
|
// Existing files are kept under ConflictMine / replace-doesn't-apply,
|
||||||
|
// overwritten under ConflictSource. Replace mode behaves like
|
||||||
|
// "overwrite" for cache files (it's the destructive choice for
|
||||||
|
// metadata; mirror that intent here).
|
||||||
|
overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
|
||||||
|
written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("extract plugin-output: %w", err)
|
||||||
|
}
|
||||||
|
result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
|
||||||
|
log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
|
||||||
|
opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
|
||||||
|
fr := FileResult{HashID: hashID}
|
||||||
|
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("open entry: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
|
||||||
|
var nested interface{}
|
||||||
|
if err := json.NewDecoder(rc).Decode(&nested); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("decode json: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
incoming := Flatten(nested)
|
||||||
|
|
||||||
|
existing, err := i.storage.GetMetadataFlat(hashID)
|
||||||
|
if err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("read existing: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
if existing == nil {
|
||||||
|
existing = map[string]string{}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch opts.Mode {
|
||||||
|
case ModeReplace:
|
||||||
|
// Diff for reporting.
|
||||||
|
toWrite := map[string]string{}
|
||||||
|
for k, v := range incoming {
|
||||||
|
toWrite[k] = v
|
||||||
|
if cur, ok := existing[k]; ok {
|
||||||
|
if cur == v {
|
||||||
|
fr.Unchanged++
|
||||||
|
} else {
|
||||||
|
fr.Updated++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fr.Added++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k := range existing {
|
||||||
|
if _, ok := incoming[k]; !ok {
|
||||||
|
fr.Deleted++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !opts.DryRun {
|
||||||
|
if _, err := i.storage.DeleteMetadata(hashID); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("delete existing: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("write: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case ModeMerge:
|
||||||
|
toWrite := map[string]string{}
|
||||||
|
for k, v := range incoming {
|
||||||
|
cur, present := existing[k]
|
||||||
|
switch {
|
||||||
|
case !present:
|
||||||
|
toWrite[k] = v
|
||||||
|
fr.Added++
|
||||||
|
case cur == v:
|
||||||
|
fr.Unchanged++
|
||||||
|
case opts.Conflict == ConflictSource:
|
||||||
|
toWrite[k] = v
|
||||||
|
fr.Updated++
|
||||||
|
case opts.Conflict == ConflictMine:
|
||||||
|
fr.Kept++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !opts.DryRun && len(toWrite) > 0 {
|
||||||
|
if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("merge: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
|
||||||
|
func readJSON(zr *zip.Reader, name string, out interface{}) error {
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if f.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
return json.NewDecoder(rc).Decode(out)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("entry %q not found", name)
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
// Package snapshot implements export/import/wipe of MetaMesh metadata.
|
||||||
|
//
|
||||||
|
// The on-disk format is a ZIP archive containing one nested-JSON file per
|
||||||
|
// known CID (cid_midhash256). Schema is defined by ManifestSchemaVersion;
|
||||||
|
// readers must reject archives whose schemaVersion exceeds what they know.
|
||||||
|
package snapshot
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// ManifestSchemaVersion is the on-disk format version. Bump when the layout
|
||||||
|
// of metadata/{cid}.json or the manifest changes in a non-additive way.
|
||||||
|
const ManifestSchemaVersion = 1
|
||||||
|
|
||||||
|
// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
|
||||||
|
const MetadataDir = "metadata/"
|
||||||
|
|
||||||
|
// ManifestFile is the path inside the ZIP that holds the manifest JSON.
|
||||||
|
const ManifestFile = "manifest.json"
|
||||||
|
|
||||||
|
// IndexFile is the path inside the ZIP that holds the file index (CID list).
|
||||||
|
const IndexFile = "index.json"
|
||||||
|
|
||||||
|
// Manifest is the metadata header stored at the root of every snapshot ZIP.
|
||||||
|
type Manifest struct {
|
||||||
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
|
GeneratedAt time.Time `json:"generatedAt"`
|
||||||
|
SourceHost string `json:"sourceHost,omitempty"`
|
||||||
|
MetaCoreVer string `json:"metaCoreVersion,omitempty"`
|
||||||
|
FileCount int `json:"fileCount"`
|
||||||
|
Includes Includes `json:"includes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Includes records which optional sections are present in a snapshot.
|
||||||
|
type Includes struct {
|
||||||
|
Metadata bool `json:"metadata"`
|
||||||
|
PluginOutput bool `json:"pluginOutput,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexFileBody is the structure of index.json inside the ZIP.
|
||||||
|
type IndexFileBody struct {
|
||||||
|
HashIDs []string `json:"hashIds"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WipeScope picks which classes of data to clear.
|
||||||
|
type WipeScope struct {
|
||||||
|
Metadata bool // file:* keys + file:__index__
|
||||||
|
}
|
||||||
|
|
||||||
|
// WipeResult reports what was cleared.
|
||||||
|
type WipeResult struct {
|
||||||
|
MetadataDeleted int64 `json:"metadataDeleted"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wiper clears metadata from Redis.
|
||||||
|
type Wiper struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWiper(stor *storage.Client) *Wiper {
|
||||||
|
return &Wiper{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wipe applies the requested scope. Returns the count of files removed.
|
||||||
|
func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
|
||||||
|
if w.storage == nil || !w.storage.IsConnected() {
|
||||||
|
return nil, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
res := &WipeResult{}
|
||||||
|
if scope.Metadata {
|
||||||
|
n, err := w.storage.ClearAllMetadata()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clear metadata: %w", err)
|
||||||
|
}
|
||||||
|
res.MetadataDeleted = n
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
@@ -537,6 +537,93 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
|
|||||||
return true, c.client.Set(ctx, key, newValue, 0).Err()
|
return true, c.client.Set(ctx, key, newValue, 0).Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FileTuple holds the (path, size, mtime) record for one file. Used by the
|
||||||
|
// watcher hydrator at startup to avoid re-mid-hashing files whose tuple
|
||||||
|
// matches what's already in Redis.
|
||||||
|
type FileTuple struct {
|
||||||
|
HashID string
|
||||||
|
FilePath string
|
||||||
|
Size int64
|
||||||
|
MtimeNano int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTuplesForAllFiles returns one FileTuple per indexed CID, fetched in
|
||||||
|
// batched MGETs (no SCAN per CID). Skips files missing any of the three
|
||||||
|
// fields. Sub-second on local Redis for ~10K files.
|
||||||
|
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("smembers index: %w", err)
|
||||||
|
}
|
||||||
|
if len(hashIDs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the key list: 3 keys per CID, in a fixed order so we can decode
|
||||||
|
// the MGET response by position. fields[i] for hashIDs[j] sits at
|
||||||
|
// keys[j*3+i].
|
||||||
|
const fieldsPerFile = 3
|
||||||
|
fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"}
|
||||||
|
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
|
||||||
|
for _, h := range hashIDs {
|
||||||
|
prefix := c.buildKeyPrefix(h)
|
||||||
|
for _, f := range fields {
|
||||||
|
keys = append(keys, prefix+f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunked MGET — Redis caps args per command; 1000 is a safe batch.
|
||||||
|
const batch = 1000
|
||||||
|
values := make([]interface{}, 0, len(keys))
|
||||||
|
for start := 0; start < len(keys); start += batch {
|
||||||
|
end := start + batch
|
||||||
|
if end > len(keys) {
|
||||||
|
end = len(keys)
|
||||||
|
}
|
||||||
|
chunk, err := c.client.MGet(ctx, keys[start:end]...).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("mget chunk [%d:%d]: %w", start, end, err)
|
||||||
|
}
|
||||||
|
values = append(values, chunk...)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]FileTuple, 0, len(hashIDs))
|
||||||
|
for i, h := range hashIDs {
|
||||||
|
base := i * fieldsPerFile
|
||||||
|
filePath, _ := values[base].(string)
|
||||||
|
sizeStr, _ := values[base+1].(string)
|
||||||
|
mtimeStr, _ := values[base+2].(string)
|
||||||
|
if filePath == "" || sizeStr == "" || mtimeStr == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseInt(sizeStr, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, FileTuple{
|
||||||
|
HashID: h,
|
||||||
|
FilePath: filePath,
|
||||||
|
Size: size,
|
||||||
|
MtimeNano: mtime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetMemoryInfo returns Redis memory usage information
|
// GetMemoryInfo returns Redis memory usage information
|
||||||
func (c *Client) GetMemoryInfo() (string, error) {
|
func (c *Client) GetMemoryInfo() (string, error) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
@@ -1059,6 +1146,51 @@ func (c *Client) ClearStream(stream string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StreamEntry represents one entry from a Redis stream.
|
||||||
|
type StreamEntry struct {
|
||||||
|
ID string
|
||||||
|
Values map[string]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// XRevRange reads stream entries from newest to oldest.
|
||||||
|
// start defaults to "+" (newest) and stop defaults to "-" (oldest) when empty.
|
||||||
|
// If count > 0, at most that many entries are returned.
|
||||||
|
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
if start == "" {
|
||||||
|
start = "+"
|
||||||
|
}
|
||||||
|
if stop == "" {
|
||||||
|
stop = "-"
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var msgs []redis.XMessage
|
||||||
|
var err error
|
||||||
|
if count > 0 {
|
||||||
|
msgs, err = c.client.XRevRangeN(ctx, stream, start, stop, count).Result()
|
||||||
|
} else {
|
||||||
|
msgs, err = c.client.XRevRange(ctx, stream, start, stop).Result()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("xrevrange failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := make([]StreamEntry, 0, len(msgs))
|
||||||
|
for _, m := range msgs {
|
||||||
|
entries = append(entries, StreamEntry{ID: m.ID, Values: m.Values})
|
||||||
|
}
|
||||||
|
return entries, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetStreamLength returns the length of a Redis stream
|
// GetStreamLength returns the length of a Redis stream
|
||||||
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package watcher
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/metazla/meta-core/internal/storage"
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
@@ -108,6 +110,85 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetRecentEvents reads recent events from the Redis stream, newest first.
|
||||||
|
// If sinceMS > 0, only events strictly newer than that timestamp are returned.
|
||||||
|
// limit caps the number of returned events (defaults to 100 when <= 0).
|
||||||
|
func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
|
||||||
|
d.mu.RLock()
|
||||||
|
client := d.storageClient
|
||||||
|
d.mu.RUnlock()
|
||||||
|
|
||||||
|
if client == nil || !client.IsConnected() {
|
||||||
|
return []FileEvent{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
start := "+"
|
||||||
|
stop := "-"
|
||||||
|
if sinceMS > 0 {
|
||||||
|
// Exclusive lower bound: skip entries at or before sinceMS.
|
||||||
|
stop = fmt.Sprintf("(%d-0", sinceMS)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := client.XRevRange(EventsStream, start, stop, int64(limit))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
events := make([]FileEvent, 0, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
events = append(events, parseStreamEntry(entry))
|
||||||
|
}
|
||||||
|
return events, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseStreamEntry converts a Redis stream entry back into a FileEvent.
|
||||||
|
// All fields are stored as strings by XADD, so numerics need parsing.
|
||||||
|
func parseStreamEntry(entry storage.StreamEntry) FileEvent {
|
||||||
|
event := FileEvent{}
|
||||||
|
|
||||||
|
if v, ok := entry.Values["type"].(string); ok {
|
||||||
|
event.Type = FileEventType(v)
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["path"].(string); ok {
|
||||||
|
event.Path = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["timestamp"].(string); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
event.Timestamp = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["size"].(string); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
event.Size = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["midhash256"].(string); ok {
|
||||||
|
event.MidHash256 = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["oldPath"].(string); ok {
|
||||||
|
event.OldPath = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["watcherId"].(string); ok {
|
||||||
|
event.WatcherID = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to the stream ID's millisecond prefix if the timestamp
|
||||||
|
// field is missing or unparseable — every Redis stream ID starts with ms.
|
||||||
|
if event.Timestamp == 0 && entry.ID != "" {
|
||||||
|
if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
|
||||||
|
if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {
|
||||||
|
event.Timestamp = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return event
|
||||||
|
}
|
||||||
|
|
||||||
// GetStreamLength returns the current stream length
|
// GetStreamLength returns the current stream length
|
||||||
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
||||||
d.mu.RLock()
|
d.mu.RLock()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package watcher
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
@@ -52,7 +53,14 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
events := h.watcher.GetRecentEvents(sinceMS, limit)
|
// Read events from the Redis `file:events` stream rather than the
|
||||||
|
// in-memory buffer. The buffer is fed by the unused debouncer path,
|
||||||
|
// so it is always empty in practice — the stream is the source of truth.
|
||||||
|
events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[watcher.handlePoll] failed to read stream: %v", err)
|
||||||
|
events = []FileEvent{}
|
||||||
|
}
|
||||||
|
|
||||||
writeJSON(w, http.StatusOK, EventsListResponse{
|
writeJSON(w, http.StatusOK, EventsListResponse{
|
||||||
Events: events,
|
Events: events,
|
||||||
|
|||||||
@@ -26,15 +26,13 @@ const (
|
|||||||
// The size prefix (8-byte big-endian) ensures that files with identical
|
// The size prefix (8-byte big-endian) ensures that files with identical
|
||||||
// middle content but different sizes produce different hashes.
|
// middle content but different sizes produce different hashes.
|
||||||
//
|
//
|
||||||
|
// fileSize must be passed in by the caller (typically from a FileInfo it
|
||||||
|
// already has from filepath.Walk or a previous os.Stat). This function
|
||||||
|
// deliberately does NOT call os.Stat itself — re-statting every file during
|
||||||
|
// a scan doubles the syscall count, which matters on slow / network drives.
|
||||||
|
//
|
||||||
// Returns: CID v1 string in base32lower format (prefixed with "b")
|
// Returns: CID v1 string in base32lower format (prefixed with "b")
|
||||||
func ComputeMidHash256(filePath string) (string, error) {
|
func ComputeMidHash256(filePath string, fileSize int64) (string, error) {
|
||||||
// Get file info
|
|
||||||
fileInfo, err := os.Stat(filePath)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
fileSize := fileInfo.Size()
|
|
||||||
|
|
||||||
// Open file
|
// Open file
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+114
-8
@@ -4,10 +4,12 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/metazla/meta-core/internal/config"
|
"github.com/metazla/meta-core/internal/config"
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -18,11 +20,39 @@ type Watcher struct {
|
|||||||
dispatcher *Dispatcher
|
dispatcher *Dispatcher
|
||||||
filesPath string
|
filesPath string
|
||||||
stateRegistry *StateRegistry
|
stateRegistry *StateRegistry
|
||||||
|
storage *storage.Client
|
||||||
|
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
eventBuffer []FileEvent
|
eventBuffer []FileEvent
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
|
||||||
|
// tuple to Redis whenever it computes a midhash. Without this, only the
|
||||||
|
// in-memory state registry knows about the file — Redis would only learn
|
||||||
|
// about it via downstream consumers of the event stream.
|
||||||
|
func (w *Watcher) SetStorageClient(s *storage.Client) {
|
||||||
|
w.storage = s
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
|
||||||
|
// Caller already has all four values from its existing os.Stat / FileInfo —
|
||||||
|
// no extra IO is performed here beyond the Redis write itself.
|
||||||
|
//
|
||||||
|
// This is the inverse-index that lets a fresh meta-core boot skip
|
||||||
|
// re-mid-hashing files whose (path, size, mtime) match what's in Redis.
|
||||||
|
func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
|
||||||
|
if w.storage == nil || midhash == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := w.storage.MergeMetadataFlat(midhash, map[string]string{
|
||||||
|
"filePath": absPath,
|
||||||
|
"sizeByte": strconv.FormatInt(size, 10),
|
||||||
|
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||||
|
}); err != nil {
|
||||||
|
log.Printf("[Watcher] failed to write tuple for %s: %v", midhash, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewWatcher creates a new file watcher/scanner
|
// NewWatcher creates a new file watcher/scanner
|
||||||
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
||||||
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
|
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
|
||||||
@@ -44,7 +74,7 @@ func (w *Watcher) handleDebouncedEvent(event FileEvent) {
|
|||||||
// Compute midhash256 for add/change events
|
// Compute midhash256 for add/change events
|
||||||
if event.Type == EventTypeAdd || event.Type == EventTypeChange {
|
if event.Type == EventTypeAdd || event.Type == EventTypeChange {
|
||||||
fullPath := filepath.Join(w.filesPath, event.Path)
|
fullPath := filepath.Join(w.filesPath, event.Path)
|
||||||
if hash, err := ComputeMidHash256(fullPath); err == nil {
|
if hash, err := ComputeMidHash256(fullPath, event.Size); err == nil {
|
||||||
event.MidHash256 = hash
|
event.MidHash256 = hash
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -90,16 +120,20 @@ func (w *Watcher) ScanDirectory(root string) int {
|
|||||||
relPath = filepath.ToSlash(relPath)
|
relPath = filepath.ToSlash(relPath)
|
||||||
|
|
||||||
// Emit add event
|
// Emit add event
|
||||||
|
curSize := info.Size()
|
||||||
|
curMtime := info.ModTime().UnixNano()
|
||||||
event := FileEvent{
|
event := FileEvent{
|
||||||
Type: EventTypeAdd,
|
Type: EventTypeAdd,
|
||||||
Path: relPath,
|
Path: relPath,
|
||||||
Size: info.Size(),
|
Size: curSize,
|
||||||
Timestamp: NowMS(),
|
Timestamp: NowMS(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compute midhash256
|
// Compute midhash256 — uses the size we already have from Walk's
|
||||||
if hash, err := ComputeMidHash256(path); err == nil {
|
// FileInfo, so no extra os.Stat is needed.
|
||||||
|
if hash, err := ComputeMidHash256(path, curSize); err == nil {
|
||||||
event.MidHash256 = hash
|
event.MidHash256 = hash
|
||||||
|
w.writeFileTuple(path, curSize, curMtime, hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dispatch directly
|
// Dispatch directly
|
||||||
@@ -160,9 +194,11 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
|||||||
Timestamp: NowMS(),
|
Timestamp: NowMS(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compute midhash256
|
// Compute midhash256 — size is already known from the walk's
|
||||||
if hash, err := ComputeMidHash256(path); err == nil {
|
// FileInfo, no extra stat needed.
|
||||||
|
if hash, err := ComputeMidHash256(path, currentSize); err == nil {
|
||||||
event.MidHash256 = hash
|
event.MidHash256 = hash
|
||||||
|
w.writeFileTuple(path, currentSize, currentMtime, hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dispatch event
|
// Dispatch event
|
||||||
@@ -185,9 +221,10 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
|||||||
Timestamp: NowMS(),
|
Timestamp: NowMS(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compute midhash256
|
// Compute midhash256 — reuse the size we already have.
|
||||||
if hash, err := ComputeMidHash256(path); err == nil {
|
if hash, err := ComputeMidHash256(path, currentSize); err == nil {
|
||||||
event.MidHash256 = hash
|
event.MidHash256 = hash
|
||||||
|
w.writeFileTuple(path, currentSize, currentMtime, hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dispatch event
|
// Dispatch event
|
||||||
@@ -227,6 +264,75 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WatcherRoot pairs a watcher's id with the absolute path it watches. The
|
||||||
|
// hydrator uses Root to decide which Redis-tracked file belongs to which
|
||||||
|
// in-memory WatcherState.
|
||||||
|
type WatcherRoot struct {
|
||||||
|
ID string
|
||||||
|
Root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// HydrateStateFromStorage repopulates the per-watcher in-memory state from
|
||||||
|
// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
|
||||||
|
// next scan tick will see existing files as "known" and skip
|
||||||
|
// ComputeMidHash256 for any whose size+mtime match — turning a cold-start
|
||||||
|
// rescan from minutes to seconds on a populated library.
|
||||||
|
//
|
||||||
|
// Files whose filePath doesn't fall under any watcher's root are silently
|
||||||
|
// skipped; files missing any tuple field are skipped too (a partial record
|
||||||
|
// can't seed valid state).
|
||||||
|
//
|
||||||
|
// Returns counts so the caller can log / surface them.
|
||||||
|
func (w *Watcher) HydrateStateFromStorage(roots []WatcherRoot) (loaded, skipped int, err error) {
|
||||||
|
if w.storage == nil || !w.storage.IsConnected() {
|
||||||
|
return 0, 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tuples, err := w.storage.GetTuplesForAllFiles()
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort roots by length descending so longest-prefix wins when paths
|
||||||
|
// nest (e.g. /files/watch/foo and /files/watch).
|
||||||
|
sortedRoots := make([]WatcherRoot, len(roots))
|
||||||
|
copy(sortedRoots, roots)
|
||||||
|
for i := 1; i < len(sortedRoots); i++ {
|
||||||
|
for j := i; j > 0 && len(sortedRoots[j].Root) > len(sortedRoots[j-1].Root); j-- {
|
||||||
|
sortedRoots[j], sortedRoots[j-1] = sortedRoots[j-1], sortedRoots[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// State keys are relative to w.filesPath (matching ScanDirectoryDiff's
|
||||||
|
// Rel(w.filesPath, path) convention). The watcher root only decides
|
||||||
|
// *which* per-watcher state map this file lives in.
|
||||||
|
for _, t := range tuples {
|
||||||
|
matched := false
|
||||||
|
for _, r := range sortedRoots {
|
||||||
|
if t.FilePath == r.Root || strings.HasPrefix(t.FilePath, r.Root+"/") {
|
||||||
|
rel, err := filepath.Rel(w.filesPath, t.FilePath)
|
||||||
|
if err != nil {
|
||||||
|
rel = t.FilePath
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
ws := w.stateRegistry.GetOrCreate(r.ID)
|
||||||
|
ws.Set(rel, &FileState{
|
||||||
|
Size: t.Size,
|
||||||
|
MtimeNano: t.MtimeNano,
|
||||||
|
MidHash: t.HashID,
|
||||||
|
})
|
||||||
|
loaded++
|
||||||
|
matched = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return loaded, skipped, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ClearState clears the tracked state for a watcher
|
// ClearState clears the tracked state for a watcher
|
||||||
func (w *Watcher) ClearState(watcherID string) {
|
func (w *Watcher) ClearState(watcherID string) {
|
||||||
w.stateRegistry.Clear(watcherID)
|
w.stateRegistry.Clear(watcherID)
|
||||||
|
|||||||
Binary file not shown.
Reference in New Issue
Block a user