improved kv metacore
This commit is contained in:
Pierre
+18
-9
@@ -7,7 +7,7 @@
|
||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||
# - nginx (dashboard, WebDAV, API proxy)
|
||||
# - Redis (managed by supervisord on leader)
|
||||
# - rclone (remote mount daemon)
|
||||
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
|
||||
# - supervisord (process management - runs on leader only)
|
||||
#
|
||||
# Architecture:
|
||||
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
|
||||
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
|
||||
LABEL org.opencontainers.image.licenses=MIT
|
||||
|
||||
# Install runtime dependencies
|
||||
# Pinned rclone release. Alpine's package can lag the latest by several
|
||||
# minor versions and we depend on VFS cache fixes shipped post-1.65.
|
||||
ARG RCLONE_VERSION=1.73.5
|
||||
|
||||
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
|
||||
# all remote mounts go through rclone now.
|
||||
RUN apk add --no-cache \
|
||||
# Core services
|
||||
redis \
|
||||
nginx \
|
||||
nginx-mod-http-dav-ext \
|
||||
supervisor \
|
||||
rclone \
|
||||
fuse3 \
|
||||
# Mount utilities
|
||||
nfs-utils \
|
||||
cifs-utils \
|
||||
# Tools
|
||||
bash \
|
||||
curl \
|
||||
python3 \
|
||||
ca-certificates \
|
||||
apache2-utils \
|
||||
# Process utilities
|
||||
procps
|
||||
|
||||
# Install rclone from the official binary release (pinned).
|
||||
RUN set -eux; \
|
||||
apk add --no-cache --virtual .rclone-deps unzip; \
|
||||
cd /tmp; \
|
||||
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
|
||||
unzip -q rclone.zip; \
|
||||
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
|
||||
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
|
||||
apk del .rclone-deps; \
|
||||
/usr/local/bin/rclone version | head -2
|
||||
|
||||
# Copy Go binary from builder
|
||||
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
|
||||
RUN chmod +x /usr/local/bin/meta-core
|
||||
|
||||
+220
-42
@@ -1,19 +1,47 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
interface MountIOStats {
|
||||
source: 'rclone';
|
||||
bytesRead: number;
|
||||
bytesWritten?: number;
|
||||
readBps: number;
|
||||
writeBps?: number;
|
||||
readOps?: number;
|
||||
writeOps?: number;
|
||||
readIops?: number;
|
||||
writeIops?: number;
|
||||
transfersPerSec?: number;
|
||||
lastSampleAt: number;
|
||||
intervalMs?: number;
|
||||
}
|
||||
|
||||
interface Mount {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'nfs' | 'smb' | 'rclone';
|
||||
type: 'smb' | 'rclone';
|
||||
enabled: boolean;
|
||||
desiredMounted: boolean;
|
||||
mountPath: string;
|
||||
mounted: boolean;
|
||||
error?: string;
|
||||
lastChecked: number;
|
||||
// VFS cache configuration applied at mount time. Empty string means "use
|
||||
// server default" (50G / 24h / 5m).
|
||||
cacheMaxSize?: string;
|
||||
cacheMaxAge?: string;
|
||||
dirCacheTime?: string;
|
||||
pollingEnabled?: boolean;
|
||||
pollingIntervalMs?: number;
|
||||
pollingActive?: boolean;
|
||||
lastPolledScan?: number;
|
||||
// Adaptive scan progress (set by the meta-core poller)
|
||||
scanning?: boolean;
|
||||
currentScanStartedAt?: number; // ms; only while scanning
|
||||
lastScanDurationMs?: number;
|
||||
nextScanAt?: number; // ms; planned start of next adaptive scan
|
||||
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
|
||||
// so the same numbers appear on every mount card.
|
||||
ioStats?: MountIOStats;
|
||||
}
|
||||
|
||||
interface RcloneRemote {
|
||||
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
|
||||
background: '#7f1d1d',
|
||||
};
|
||||
|
||||
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
|
||||
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
|
||||
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
|
||||
function formatBytes(n: number): string {
|
||||
if (!isFinite(n) || n < 0) return '—';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let i = 0;
|
||||
let v = n;
|
||||
while (v >= 1000 && i < units.length - 1) {
|
||||
v /= 1000;
|
||||
i++;
|
||||
}
|
||||
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
|
||||
}
|
||||
|
||||
function formatRate(bps: number): string {
|
||||
return `${formatBytes(bps)}/s`;
|
||||
}
|
||||
|
||||
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
|
||||
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
|
||||
// is just noise.
|
||||
function formatDuration(ms: number): string {
|
||||
if (ms < 0 || !isFinite(ms)) return '—';
|
||||
const s = Math.floor(ms / 1000);
|
||||
if (s < 60) return `${s}s`;
|
||||
const m = Math.floor(s / 60);
|
||||
const remS = s % 60;
|
||||
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
|
||||
const h = Math.floor(m / 60);
|
||||
const remM = m % 60;
|
||||
return remM ? `${h}h ${remM}m` : `${h}h`;
|
||||
}
|
||||
|
||||
// IOStatsLine renders a single line of live throughput info beneath each
|
||||
// mount card. Source is always rclone after the rclone-only consolidation,
|
||||
// so we report file-level transfers/s rather than kernel IOPS.
|
||||
function IOStatsLine({ stats }: { stats: MountIOStats }) {
|
||||
const opsLabel = 'transfers/s';
|
||||
const opsValue = stats.transfersPerSec;
|
||||
|
||||
const ageMs = Date.now() - stats.lastSampleAt;
|
||||
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
|
||||
|
||||
return (
|
||||
<p
|
||||
style={{
|
||||
color: stale ? '#6b7280' : '#9ca3af',
|
||||
fontSize: '0.85rem',
|
||||
margin: '0.25rem 0 0',
|
||||
}}
|
||||
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
|
||||
>
|
||||
↓ {formatRate(stats.readBps)}
|
||||
{opsValue !== undefined && opsValue > 0 && (
|
||||
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
|
||||
)}
|
||||
<> · {formatBytes(stats.bytesRead)} read total</>
|
||||
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
|
||||
<> · {formatBytes(stats.bytesWritten)} written</>
|
||||
)}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
interface ScanProgressProps {
|
||||
mount: Mount;
|
||||
now: number;
|
||||
}
|
||||
|
||||
// Progress bar shows scan ETA when running, an idle row with a countdown to
|
||||
// the next adaptive scan when waiting. We can't show true % complete (we
|
||||
// don't know the file count up-front), so the running bar is filled by
|
||||
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
|
||||
// it pins at 95% to signal "taking longer than last time".
|
||||
function ScanProgress({ mount, now }: ScanProgressProps) {
|
||||
if (!mount.pollingEnabled) return null;
|
||||
|
||||
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
|
||||
const lastDuration = mount.lastScanDurationMs ?? 0;
|
||||
|
||||
let fill = 0;
|
||||
let label = '';
|
||||
|
||||
if (isScanning) {
|
||||
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
|
||||
if (lastDuration > 0) {
|
||||
fill = Math.min(0.95, elapsed / lastDuration);
|
||||
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
|
||||
} else {
|
||||
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
|
||||
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
|
||||
}
|
||||
} else {
|
||||
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
|
||||
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
|
||||
if (lastDuration > 0 && mount.lastPolledScan) {
|
||||
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
|
||||
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
|
||||
} else if (mount.nextScanAt) {
|
||||
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
|
||||
} else {
|
||||
label = 'Polling enabled — waiting for first scan';
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div style={{ marginTop: '0.5rem' }}>
|
||||
<div
|
||||
style={{
|
||||
height: '6px',
|
||||
background: '#0f3460',
|
||||
borderRadius: '3px',
|
||||
overflow: 'hidden',
|
||||
position: 'relative',
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: `${Math.round(fill * 100)}%`,
|
||||
height: '100%',
|
||||
background: isScanning ? '#4ade80' : '#374151',
|
||||
transition: 'width 0.5s linear',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Mounts() {
|
||||
const [mounts, setMounts] = useState<Mount[]>([]);
|
||||
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
|
||||
@@ -50,19 +209,23 @@ export default function Mounts() {
|
||||
const [showForm, setShowForm] = useState(false);
|
||||
const [formData, setFormData] = useState({
|
||||
name: '',
|
||||
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
|
||||
nfsServer: '',
|
||||
nfsPath: '',
|
||||
type: 'smb' as 'smb' | 'rclone',
|
||||
smbServer: '',
|
||||
smbShare: '',
|
||||
smbUsername: '',
|
||||
smbPassword: '',
|
||||
rcloneRemote: '',
|
||||
rclonePath: '',
|
||||
pollingEnabled: false,
|
||||
cacheMaxSize: '',
|
||||
cacheMaxAge: '',
|
||||
dirCacheTime: '',
|
||||
pollingEnabled: true,
|
||||
pollingIntervalMs: 30000,
|
||||
});
|
||||
const [scanning, setScanning] = useState<string | null>(null);
|
||||
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
|
||||
// refetches (which run every 5s).
|
||||
const [now, setNow] = useState<number>(() => Date.now());
|
||||
|
||||
const fetchMounts = async () => {
|
||||
try {
|
||||
@@ -94,7 +257,11 @@ export default function Mounts() {
|
||||
fetchMounts();
|
||||
fetchRemotes();
|
||||
const interval = setInterval(fetchMounts, 5000);
|
||||
return () => clearInterval(interval);
|
||||
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => {
|
||||
clearInterval(interval);
|
||||
clearInterval(tick);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleMount = async (id: string) => {
|
||||
@@ -143,10 +310,7 @@ export default function Mounts() {
|
||||
type: formData.type,
|
||||
};
|
||||
|
||||
if (formData.type === 'nfs') {
|
||||
body.nfsServer = formData.nfsServer;
|
||||
body.nfsPath = formData.nfsPath;
|
||||
} else if (formData.type === 'smb') {
|
||||
if (formData.type === 'smb') {
|
||||
body.smbServer = formData.smbServer;
|
||||
body.smbShare = formData.smbShare;
|
||||
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
|
||||
@@ -156,6 +320,12 @@ export default function Mounts() {
|
||||
body.rclonePath = formData.rclonePath;
|
||||
}
|
||||
|
||||
// Cache knobs — only forward non-empty values; the server applies defaults
|
||||
// when fields are omitted.
|
||||
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
|
||||
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
|
||||
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
|
||||
|
||||
// Add polling configuration
|
||||
body.pollingEnabled = formData.pollingEnabled;
|
||||
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
|
||||
@@ -171,16 +341,17 @@ export default function Mounts() {
|
||||
setShowForm(false);
|
||||
setFormData({
|
||||
name: '',
|
||||
type: 'nfs',
|
||||
nfsServer: '',
|
||||
nfsPath: '',
|
||||
type: 'smb',
|
||||
smbServer: '',
|
||||
smbShare: '',
|
||||
smbUsername: '',
|
||||
smbPassword: '',
|
||||
rcloneRemote: '',
|
||||
rclonePath: '',
|
||||
pollingEnabled: false,
|
||||
cacheMaxSize: '',
|
||||
cacheMaxAge: '',
|
||||
dirCacheTime: '',
|
||||
pollingEnabled: true,
|
||||
pollingIntervalMs: 30000,
|
||||
});
|
||||
fetchMounts();
|
||||
@@ -231,32 +402,15 @@ export default function Mounts() {
|
||||
<select
|
||||
style={inputStyle}
|
||||
value={formData.type}
|
||||
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
|
||||
onChange={(e) => {
|
||||
const type = e.target.value as 'smb' | 'rclone';
|
||||
setFormData({ ...formData, type });
|
||||
}}
|
||||
>
|
||||
<option value="nfs">NFS</option>
|
||||
<option value="smb">SMB/CIFS</option>
|
||||
<option value="rclone">rclone</option>
|
||||
<option value="smb">SMB (via rclone)</option>
|
||||
<option value="rclone">rclone (pre-defined remote)</option>
|
||||
</select>
|
||||
|
||||
{formData.type === 'nfs' && (
|
||||
<>
|
||||
<input
|
||||
style={inputStyle}
|
||||
placeholder="NFS Server (e.g., 192.168.1.100)"
|
||||
value={formData.nfsServer}
|
||||
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
|
||||
required
|
||||
/>
|
||||
<input
|
||||
style={inputStyle}
|
||||
placeholder="NFS Path (e.g., /export/media)"
|
||||
value={formData.nfsPath}
|
||||
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
|
||||
required
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{formData.type === 'smb' && (
|
||||
<>
|
||||
<input
|
||||
@@ -313,6 +467,30 @@ export default function Mounts() {
|
||||
</>
|
||||
)}
|
||||
|
||||
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
|
||||
VFS cache (leave blank for defaults: 50G / 24h / 5m)
|
||||
</p>
|
||||
<input
|
||||
style={inputStyle}
|
||||
placeholder="Cache max size (e.g., 50G)"
|
||||
value={formData.cacheMaxSize}
|
||||
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
|
||||
/>
|
||||
<input
|
||||
style={inputStyle}
|
||||
placeholder="Cache max age (e.g., 24h)"
|
||||
value={formData.cacheMaxAge}
|
||||
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
|
||||
/>
|
||||
<input
|
||||
style={inputStyle}
|
||||
placeholder="Dir cache time (e.g., 5m)"
|
||||
value={formData.dirCacheTime}
|
||||
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
|
||||
<input
|
||||
@@ -362,17 +540,17 @@ export default function Mounts() {
|
||||
</span>
|
||||
</p>
|
||||
{mount.pollingEnabled && (
|
||||
<p style={{ color: '#888', fontSize: '0.9rem' }}>
|
||||
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
|
||||
Polling:{' '}
|
||||
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
|
||||
{mount.pollingActive ? 'Active' : 'Inactive'}
|
||||
</span>
|
||||
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
||||
{mount.lastPolledScan && (
|
||||
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
|
||||
)}
|
||||
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
||||
{' • Adaptive 10× idle'}
|
||||
</p>
|
||||
)}
|
||||
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
|
||||
<ScanProgress mount={mount} now={now} />
|
||||
{mount.error && (
|
||||
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
|
||||
)}
|
||||
|
||||
+138
-170
@@ -2,11 +2,17 @@
|
||||
# =============================================================================
|
||||
# Mount Watcher Script
|
||||
#
|
||||
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
|
||||
# Runs as a supervisord program, polling every 5 seconds.
|
||||
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
|
||||
# rclone RC daemon. After the rclone-only consolidation, this script no
|
||||
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
|
||||
# through rclone's mount/mount endpoint.
|
||||
#
|
||||
# Uses findmnt to check actual mount status (source of truth).
|
||||
# Writes errors to /meta-core/mounts/errors/{id}.error
|
||||
# Mount types:
|
||||
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
|
||||
# - "rclone" → references a pre-defined remote in rclone.conf
|
||||
#
|
||||
# All mounts are read-only by construction (no API surface to override).
|
||||
# Errors surface in /meta-core/mounts/errors/{id}.error
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
|
||||
FILES_PATH="${FILES_PATH:-/files}"
|
||||
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
||||
|
||||
# Fall-back VFS cache settings — applied when a mount entry leaves the field
|
||||
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
|
||||
# packages/meta-core/internal/mounts/types.go.
|
||||
DEFAULT_CACHE_MAX_SIZE="50G"
|
||||
DEFAULT_CACHE_MAX_AGE="24h"
|
||||
DEFAULT_DIR_CACHE_TIME="5m"
|
||||
|
||||
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
|
||||
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
|
||||
|
||||
# ANSI colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@@ -54,9 +70,7 @@ init_dirs() {
|
||||
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
|
||||
is_mounted() {
|
||||
local mount_path="$1"
|
||||
# Remove trailing slash for consistent matching
|
||||
local clean_path="${mount_path%/}"
|
||||
# Check if the path appears as a mount point in /proc/mounts (field 2)
|
||||
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
|
||||
}
|
||||
|
||||
@@ -74,145 +88,102 @@ write_error() {
|
||||
echo "$error" >> "$ERROR_DIR/$id.error"
|
||||
}
|
||||
|
||||
# Mount NFS share (always read-only)
|
||||
do_mount_nfs() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
# Build the rclone remote source string for a mount entry. Echoes the result.
|
||||
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
|
||||
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
|
||||
# the optional remote path.
|
||||
build_rclone_fs() {
|
||||
local type="$1"
|
||||
local smb_server="$2"
|
||||
local smb_share="$3"
|
||||
local smb_username="$4"
|
||||
local smb_password_obscured="$5"
|
||||
local smb_domain="$6"
|
||||
local rclone_remote="$7"
|
||||
local rclone_path="$8"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
|
||||
|
||||
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
|
||||
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
|
||||
local rc=$?
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "NFS mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "NFS mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Mount SMB/CIFS share (always read-only)
|
||||
do_mount_smb() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local share="$3"
|
||||
local mount_path="$4"
|
||||
local username="${5:-}"
|
||||
local password_obscured="${6:-}"
|
||||
local domain="${7:-}"
|
||||
local options="${8:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
|
||||
# If we have credentials, reveal password and build options
|
||||
if [ -n "$username" ]; then
|
||||
mount_opts="${mount_opts},username=${username}"
|
||||
|
||||
if [ -n "$password_obscured" ]; then
|
||||
# Reveal password using rclone (only exists in memory briefly)
|
||||
local password
|
||||
password=$(rclone reveal "$password_obscured" 2>/dev/null)
|
||||
if [ -n "$password" ]; then
|
||||
mount_opts="${mount_opts},password=${password}"
|
||||
case "$type" in
|
||||
smb)
|
||||
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
|
||||
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
|
||||
# embed without quoting.
|
||||
local opts="host=${smb_server}"
|
||||
if [ -n "$smb_username" ]; then
|
||||
opts="${opts},user=${smb_username}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$domain" ]; then
|
||||
mount_opts="${mount_opts},domain=${domain}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Add any additional options
|
||||
if [ -n "$options" ]; then
|
||||
if [ -n "$mount_opts" ]; then
|
||||
mount_opts="${mount_opts},${options}"
|
||||
else
|
||||
mount_opts="$options"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
|
||||
|
||||
# Execute mount directly (not via eval) to handle special chars in password
|
||||
local output
|
||||
local rc
|
||||
if [ -n "$mount_opts" ]; then
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
|
||||
rc=$?
|
||||
else
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
# Clear password from any variables (best effort)
|
||||
password=""
|
||||
mount_opts=""
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "SMB mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "SMB mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
if [ -n "$smb_password_obscured" ]; then
|
||||
opts="${opts},pass=${smb_password_obscured}"
|
||||
fi
|
||||
if [ -n "$smb_domain" ]; then
|
||||
opts="${opts},domain=${smb_domain}"
|
||||
fi
|
||||
echo ":smb,${opts}:${smb_share}"
|
||||
;;
|
||||
rclone)
|
||||
echo "${rclone_remote}:${rclone_path}"
|
||||
;;
|
||||
*)
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Mount rclone remote using RC API (always read-only)
|
||||
# Mount via rclone RC API (always read-only).
|
||||
do_mount_rclone() {
|
||||
local id="$1"
|
||||
local remote="$2"
|
||||
local remote_path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
local fs="$2"
|
||||
local mount_path="$3"
|
||||
local cache_max_size="$4"
|
||||
local cache_max_age="$5"
|
||||
local dir_cache_time="$6"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Build the remote source string
|
||||
local fs="${remote}:${remote_path}"
|
||||
# Apply defaults for any blank cache fields.
|
||||
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
|
||||
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
|
||||
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
|
||||
|
||||
# Use rclone RC API to mount (always read-only for safety)
|
||||
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
|
||||
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
|
||||
|
||||
# Build JSON body for rclone mount with read-only option
|
||||
# Build JSON body via python3 — embedding the obscured password and
|
||||
# display fs string into a hand-rolled heredoc is asking for an injection
|
||||
# bug the next time we add a field with a quote in it.
|
||||
local json_body
|
||||
json_body=$(cat <<EOF
|
||||
{
|
||||
"fs": "${fs}",
|
||||
"mountPoint": "${mount_path}",
|
||||
json_body=$(python3 -c '
|
||||
import json, sys
|
||||
print(json.dumps({
|
||||
"fs": sys.argv[1],
|
||||
"mountPoint": sys.argv[2],
|
||||
"mountOpt": {
|
||||
"AllowOther": true,
|
||||
"ReadOnly": true
|
||||
"AllowOther": True,
|
||||
"ReadOnly": True,
|
||||
},
|
||||
"vfsOpt": {
|
||||
"CacheMode": 2,
|
||||
"ReadOnly": true
|
||||
}
|
||||
}
|
||||
EOF
|
||||
)
|
||||
# CacheMode 3 = "full" — cache both reads and writes on local disk.
|
||||
# CacheMode 2 ("writes") would silently bypass the cache for reads,
|
||||
# which is the opposite of what we want for a read-only mount.
|
||||
"CacheMode": 3,
|
||||
"CacheMaxSize": sys.argv[3],
|
||||
"CacheMaxAge": sys.argv[4],
|
||||
"DirCacheTime": sys.argv[5],
|
||||
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
|
||||
# throughput in isolation but, under the 4+ concurrent reads from
|
||||
# the processing pipeline, multiplies into 16+ simultaneous SMB
|
||||
# requests and tanks aggregate throughput. Left at default — rely on
|
||||
# file-level concurrency from the pipeline for parallelism instead.
|
||||
"ReadOnly": True,
|
||||
},
|
||||
}))
|
||||
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
|
||||
|
||||
local output
|
||||
output=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-u "$RCLONE_RC_AUTH" \
|
||||
-d "$json_body" \
|
||||
"http://127.0.0.1:5572/mount/mount" 2>&1)
|
||||
"${RCLONE_RC_URL}/mount/mount" 2>&1)
|
||||
|
||||
# Wait a moment for mount to establish
|
||||
sleep 2
|
||||
|
||||
if is_mounted "$mount_path"; then
|
||||
@@ -221,7 +192,6 @@ EOF
|
||||
return 0
|
||||
else
|
||||
local error_msg="rclone mount failed"
|
||||
# Try to extract error from response
|
||||
if echo "$output" | grep -q "error"; then
|
||||
error_msg="$output"
|
||||
fi
|
||||
@@ -231,36 +201,28 @@ EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Unmount a path
|
||||
# Unmount a path. Always goes through the rclone RC API now — every mount we
|
||||
# create lives behind rclone's FUSE driver.
|
||||
do_unmount() {
|
||||
local id="$1"
|
||||
local mount_path="$2"
|
||||
local type="$3"
|
||||
|
||||
if ! is_mounted "$mount_path"; then
|
||||
log "Already unmounted: $mount_path"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$type" = "rclone" ]; then
|
||||
# Use rclone RC API to unmount
|
||||
log "Unmounting rclone: $mount_path"
|
||||
curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
|
||||
else
|
||||
log "Unmounting: $mount_path"
|
||||
umount "$mount_path" 2>&1
|
||||
fi
|
||||
log "Unmounting: $mount_path"
|
||||
curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u "$RCLONE_RC_AUTH" \
|
||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
|
||||
|
||||
# Wait a moment
|
||||
sleep 1
|
||||
|
||||
# Check if still mounted
|
||||
# Fallback to lazy umount in case the FUSE process is wedged.
|
||||
if is_mounted "$mount_path"; then
|
||||
# Force unmount with lazy option
|
||||
log_warn "Force unmounting (lazy): $mount_path"
|
||||
umount -l "$mount_path" 2>&1
|
||||
sleep 1
|
||||
@@ -277,12 +239,11 @@ do_unmount() {
|
||||
|
||||
# Process all mounts from config
|
||||
process_mounts() {
|
||||
# Check if config file exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Parse JSON using python3
|
||||
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
|
||||
local mounts
|
||||
mounts=$(python3 -c '
|
||||
import sys, json
|
||||
@@ -294,15 +255,12 @@ except:
|
||||
sys.exit(0)
|
||||
|
||||
for m in data.get("mounts", []):
|
||||
# Output pipe-separated values
|
||||
fields = [
|
||||
m.get("id", ""),
|
||||
m.get("type", ""),
|
||||
str(m.get("enabled", True)),
|
||||
str(m.get("desiredMounted", False)),
|
||||
m.get("mountPath", ""),
|
||||
m.get("nfsServer", ""),
|
||||
m.get("nfsPath", ""),
|
||||
m.get("smbServer", ""),
|
||||
m.get("smbShare", ""),
|
||||
m.get("smbUsername", ""),
|
||||
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
|
||||
m.get("smbDomain", ""),
|
||||
m.get("rcloneRemote", ""),
|
||||
m.get("rclonePath", ""),
|
||||
m.get("options", "")
|
||||
m.get("cacheMaxSize", ""),
|
||||
m.get("cacheMaxAge", ""),
|
||||
m.get("dirCacheTime", ""),
|
||||
]
|
||||
print("|".join(fields))
|
||||
' 2>/dev/null)
|
||||
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
|
||||
while IFS='|' read -r id type enabled desired mount_path \
|
||||
smb_server smb_share smb_username smb_password_obscured smb_domain \
|
||||
rclone_remote rclone_path \
|
||||
cache_max_size cache_max_age dir_cache_time; do
|
||||
# Skip empty lines
|
||||
[ -z "$id" ] && continue
|
||||
|
||||
# Check actual mount status
|
||||
# Skip retired/legacy types — the Go-side migration disables NFS but a
|
||||
# stale entry could still arrive here on a downgrade/upgrade race.
|
||||
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
|
||||
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
|
||||
continue
|
||||
fi
|
||||
|
||||
local is_mounted_now=false
|
||||
is_mounted "$mount_path" && is_mounted_now=true
|
||||
|
||||
# Handle disabled mounts - unmount if currently mounted
|
||||
# Disabled → unmount if currently up
|
||||
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) disabled, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
do_unmount "$id" "$mount_path"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Handle desired state
|
||||
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
||||
# Should be mounted
|
||||
if [ "$is_mounted_now" = false ]; then
|
||||
local fs
|
||||
fs=$(build_rclone_fs "$type" \
|
||||
"$smb_server" "$smb_share" "$smb_username" \
|
||||
"$smb_password_obscured" "$smb_domain" \
|
||||
"$rclone_remote" "$rclone_path")
|
||||
|
||||
if [ -z "$fs" ]; then
|
||||
log_error "Could not build rclone fs spec for mount $id (type=$type)"
|
||||
write_error "$id" "Invalid mount config: missing required fields for type=$type"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
||||
case "$type" in
|
||||
nfs)
|
||||
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
|
||||
;;
|
||||
smb)
|
||||
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
|
||||
;;
|
||||
rclone)
|
||||
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown mount type: $type"
|
||||
;;
|
||||
esac
|
||||
do_mount_rclone "$id" "$fs" "$mount_path" \
|
||||
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
|
||||
fi
|
||||
else
|
||||
# Should be unmounted
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) not desired, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
do_unmount "$id" "$mount_path"
|
||||
fi
|
||||
fi
|
||||
done <<< "$mounts"
|
||||
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
|
||||
|
||||
# Main loop
|
||||
main() {
|
||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
|
||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
|
||||
|
||||
init_dirs
|
||||
|
||||
|
||||
@@ -70,6 +70,9 @@ http {
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 86400s;
|
||||
|
||||
# Snapshot import accepts multi-MB ZIP uploads.
|
||||
client_max_body_size 0;
|
||||
}
|
||||
|
||||
# =========================================================================
|
||||
@@ -192,6 +195,18 @@ http {
|
||||
# WebDAV HTML browser (for iframe embedding in dashboard)
|
||||
# Returns HTML directory listings instead of JSON
|
||||
# =========================================================================
|
||||
|
||||
# At the browser root, strip the "../" parent-dir row that autoindex
|
||||
# otherwise emits — clicking it inside the dashboard iframe would
|
||||
# navigate to "/" (the dashboard itself), nesting the page inside itself.
|
||||
location = /webdav-browser/ {
|
||||
alias /files/;
|
||||
autoindex on;
|
||||
charset utf-8;
|
||||
sub_filter '<a href="../">../</a>' '';
|
||||
sub_filter_once on;
|
||||
}
|
||||
|
||||
location /webdav-browser/ {
|
||||
alias /files/;
|
||||
autoindex on;
|
||||
|
||||
@@ -71,7 +71,7 @@ startsecs=2
|
||||
# rclone: Remote storage mount daemon (RC API on port 5572)
|
||||
# =============================================================================
|
||||
[program:rclone]
|
||||
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
||||
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
||||
priority=30
|
||||
autostart=true
|
||||
autorestart=true
|
||||
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
|
||||
startsecs=3
|
||||
|
||||
# =============================================================================
|
||||
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
|
||||
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
|
||||
# =============================================================================
|
||||
[program:mount-watcher]
|
||||
command=/app/scripts/mount-watcher.sh
|
||||
|
||||
@@ -330,28 +330,39 @@ Events are published to the `meta:events` Redis stream.
|
||||
|
||||
### Mount Management
|
||||
|
||||
Meta-core manages SMB/rclone mount configurations:
|
||||
Meta-core manages remote mounts via a single rclone backend. Two user-facing
|
||||
mount types are supported: `smb` (synthesised on-the-fly into rclone's `:smb:`
|
||||
backend) and `rclone` (references a pre-defined remote in `rclone.conf`).
|
||||
Native `mount.cifs` and `mount.nfs` were removed in favour of one code path.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Mount Manager │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │
|
||||
│ │ Config │─►│ Validator │─►│ Health Monitor │ │
|
||||
│ │ Config │─►│ Validator │─►│ Mount Watcher │ │
|
||||
│ │ (mounts.json)│ │ │ │ │ │
|
||||
│ │ │ │ - SMB params │ │ - Accessibility check │ │
|
||||
│ │ - id │ │ - rclone cfg │ │ - Error tracking │ │
|
||||
│ │ - type │ │ - paths │ │ - Status reporting │ │
|
||||
│ │ - path │ │ │ │ │ │
|
||||
│ │ │ │ - SMB params │ │ - Calls rclone RC │ │
|
||||
│ │ - id │ │ - rclone cfg │ │ /mount/mount │ │
|
||||
│ │ - type │ │ - cache size │ │ - ReadOnly: true │ │
|
||||
│ │ - path │ │ │ │ - Polls every 5s │ │
|
||||
│ │ - cache* │ │ │ │ - Error tracking │ │
|
||||
│ └──────────────┘ └──────────────┘ └────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Features:
|
||||
- **Configuration**: Mount configs stored in `/meta-core/mounts/mounts.json`
|
||||
- **Validation**: Validates mount commands and parameters
|
||||
- **Health Monitoring**: Polls mount points for accessibility
|
||||
- **Error Handling**: Tracks mount errors in separate error files
|
||||
- **Read-only**: All mounts are mounted read-only by construction (no API knob)
|
||||
- **VFS cache**: Per-mount `cacheMaxSize` (default `50G`), `cacheMaxAge`
|
||||
(default `24h`), `dirCacheTime` (default `5m`) propagate to rclone's
|
||||
`vfsOpt`. The cache directory itself is daemon-global — rclone keeps
|
||||
per-remote subdirectories beneath it.
|
||||
- **Health Monitoring**: Polls `/proc/mounts` every 5s for accessibility
|
||||
- **Error Handling**: Tracks mount errors in `/meta-core/mounts/errors/{id}.error`
|
||||
- **Stats**: `/api/mounts` returns daemon-global rclone counters (bytes/sec,
|
||||
transfers/sec) — same numbers across all mounts since rclone tracks per-
|
||||
daemon, not per-mount.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -183,3 +183,42 @@
|
||||
color: rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
}
|
||||
|
||||
.app-nav {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.app-nav button {
|
||||
background: transparent;
|
||||
border: 1px solid rgba(255, 255, 255, 0.2);
|
||||
color: inherit;
|
||||
padding: 0.4rem 1rem;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
.app-nav button:hover {
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.app-nav button.active {
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
border-color: rgba(255, 255, 255, 0.4);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
.app-nav button {
|
||||
border-color: rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
.app-nav button:hover {
|
||||
background: rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
.app-nav button.active {
|
||||
background: rgba(0, 0, 0, 0.08);
|
||||
border-color: rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,11 +4,15 @@ import { FileList } from './components/FileList';
|
||||
import { MetadataEditor } from './components/MetadataEditor';
|
||||
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
|
||||
import { KVTreeView } from './components/KVTreeView';
|
||||
import { SnapshotPanel } from './components/SnapshotPanel';
|
||||
import { MetadataAPI } from './api/metadataApi';
|
||||
import { SearchResult } from './types';
|
||||
import './App.css';
|
||||
|
||||
type View = 'edit' | 'snapshot';
|
||||
|
||||
function App() {
|
||||
const [view, setView] = useState<View>('edit');
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
|
||||
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
|
||||
@@ -167,8 +171,27 @@ function App() {
|
||||
<p className="app-description">
|
||||
Search and edit metadata for your media files
|
||||
</p>
|
||||
<nav className="app-nav">
|
||||
<button
|
||||
className={view === 'edit' ? 'active' : ''}
|
||||
onClick={() => setView('edit')}
|
||||
>
|
||||
Edit
|
||||
</button>
|
||||
<button
|
||||
className={view === 'snapshot' ? 'active' : ''}
|
||||
onClick={() => setView('snapshot')}
|
||||
>
|
||||
Snapshot
|
||||
</button>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
{view === 'snapshot' ? (
|
||||
<main className="app-main">
|
||||
<SnapshotPanel />
|
||||
</main>
|
||||
) : (
|
||||
<main className="app-main">
|
||||
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
|
||||
|
||||
@@ -230,6 +253,7 @@ function App() {
|
||||
</div>
|
||||
)}
|
||||
</main>
|
||||
)}
|
||||
|
||||
<footer className="app-footer">
|
||||
<p>
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
const API_BASE = '/api/snapshot';
|
||||
|
||||
export type ImportMode = 'replace' | 'merge';
|
||||
export type ImportConflict = 'mine' | 'source';
|
||||
|
||||
export interface FileImportResult {
|
||||
hashId: string;
|
||||
added: number;
|
||||
updated: number;
|
||||
unchanged: number;
|
||||
kept: number;
|
||||
deleted: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface PluginOutputResult {
|
||||
written: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
mode: ImportMode;
|
||||
conflict?: ImportConflict;
|
||||
dryRun: boolean;
|
||||
totalFiles: number;
|
||||
filesOk: number;
|
||||
filesFailed: number;
|
||||
files?: FileImportResult[];
|
||||
pluginOutput?: PluginOutputResult;
|
||||
}
|
||||
|
||||
export interface WipeResult {
|
||||
metadataDeleted: number;
|
||||
}
|
||||
|
||||
export class SnapshotAPI {
|
||||
/**
|
||||
* Trigger a download of a metadata snapshot ZIP.
|
||||
* Browser handles the download via a synthetic <a> click.
|
||||
*/
|
||||
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
|
||||
const parts: string[] = [];
|
||||
if (includes.pluginOutput) parts.push('plugin-output');
|
||||
const qs = parts.length ? `?include=${parts.join(',')}` : '';
|
||||
const a = document.createElement('a');
|
||||
a.href = `${API_BASE}/export${qs}`;
|
||||
a.rel = 'noopener';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a snapshot ZIP and apply it.
|
||||
* Mode replace deletes and rewrites each CID's properties.
|
||||
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
|
||||
*/
|
||||
static async runImport(
|
||||
file: File,
|
||||
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
|
||||
): Promise<ImportResult> {
|
||||
const params = new URLSearchParams();
|
||||
params.set('mode', opts.mode);
|
||||
if (opts.mode === 'merge') {
|
||||
params.set('conflict', opts.conflict ?? 'source');
|
||||
}
|
||||
if (opts.dryRun) {
|
||||
params.set('dry_run', 'true');
|
||||
}
|
||||
const fd = new FormData();
|
||||
fd.append('snapshot', file);
|
||||
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
|
||||
method: 'POST',
|
||||
body: fd,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({}));
|
||||
throw new Error(err.message || err.error || 'Import failed');
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wipe Redis metadata. Pass scope=metadata.
|
||||
*/
|
||||
static async wipe(scope: 'metadata'): Promise<WipeResult> {
|
||||
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
|
||||
method: 'POST',
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({}));
|
||||
throw new Error(err.message || err.error || 'Wipe failed');
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
.snapshot-panel {
|
||||
max-width: 920px;
|
||||
margin: 0 auto;
|
||||
padding: 1.5rem;
|
||||
}
|
||||
|
||||
.snapshot-panel h2 {
|
||||
margin: 0 0 0.5rem;
|
||||
}
|
||||
|
||||
.snapshot-intro {
|
||||
color: var(--text-muted, #888);
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.snapshot-section {
|
||||
border: 1px solid var(--border, #e0e0e0);
|
||||
border-radius: 6px;
|
||||
padding: 1rem 1.25rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
|
||||
.snapshot-section.danger {
|
||||
border-color: #d04a4a;
|
||||
background: rgba(208, 74, 74, 0.04);
|
||||
}
|
||||
|
||||
.snapshot-section h3 {
|
||||
margin: 0 0 0.5rem;
|
||||
}
|
||||
|
||||
.snapshot-row {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
align-items: flex-start;
|
||||
margin: 0.75rem 0;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.snapshot-file {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.snapshot-file span {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted, #888);
|
||||
}
|
||||
|
||||
.snapshot-fieldset {
|
||||
border: 1px solid var(--border, #e0e0e0);
|
||||
border-radius: 4px;
|
||||
padding: 0.5rem 0.75rem;
|
||||
min-width: 280px;
|
||||
}
|
||||
|
||||
.snapshot-fieldset[disabled] {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.snapshot-fieldset legend {
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
padding: 0 0.25rem;
|
||||
}
|
||||
|
||||
.snapshot-fieldset label {
|
||||
display: block;
|
||||
margin: 0.25rem 0;
|
||||
font-size: 0.9rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.snapshot-fieldset input[type='radio'] {
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
.snapshot-btn {
|
||||
background: var(--bg-button, #f0f0f0);
|
||||
border: 1px solid var(--border, #c0c0c0);
|
||||
color: var(--text, #222);
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
.snapshot-btn:hover:not(:disabled) {
|
||||
background: var(--bg-button-hover, #e0e0e0);
|
||||
}
|
||||
|
||||
.snapshot-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.snapshot-btn.primary {
|
||||
background: #2a6df4;
|
||||
border-color: #2a6df4;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.snapshot-btn.primary:hover:not(:disabled) {
|
||||
background: #1a5de4;
|
||||
}
|
||||
|
||||
.snapshot-btn.danger {
|
||||
background: #d04a4a;
|
||||
border-color: #d04a4a;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.snapshot-btn.danger:hover:not(:disabled) {
|
||||
background: #b83838;
|
||||
}
|
||||
|
||||
.snapshot-error {
|
||||
background: rgba(208, 74, 74, 0.1);
|
||||
color: #b83838;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.snapshot-info {
|
||||
background: rgba(42, 109, 244, 0.08);
|
||||
color: #1a5de4;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border-radius: 4px;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.snapshot-result {
|
||||
margin-top: 0.75rem;
|
||||
border: 1px solid var(--border, #e0e0e0);
|
||||
border-radius: 4px;
|
||||
padding: 0.75rem;
|
||||
}
|
||||
|
||||
.snapshot-result-summary {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 0.5rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.snapshot-result-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.snapshot-result-table th,
|
||||
.snapshot-result-table td {
|
||||
border: 1px solid var(--border, #e0e0e0);
|
||||
padding: 0.35rem 0.5rem;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.snapshot-result-table th {
|
||||
background: var(--bg-subtle, #f5f5f5);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.snapshot-result-errors {
|
||||
margin-top: 0.5rem;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.snapshot-result-errors summary {
|
||||
cursor: pointer;
|
||||
color: #b83838;
|
||||
}
|
||||
|
||||
.snapshot-result-errors ul {
|
||||
margin: 0.5rem 0 0;
|
||||
padding-left: 1.25rem;
|
||||
}
|
||||
|
||||
.snapshot-result-errors code {
|
||||
font-size: 0.8rem;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.snapshot-checkbox {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.5rem;
|
||||
margin: 0.5rem 0 0.75rem;
|
||||
cursor: pointer;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
.snapshot-checkbox input {
|
||||
margin-top: 0.2rem;
|
||||
}
|
||||
|
||||
.snapshot-hint {
|
||||
color: var(--text-muted, #888);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.snapshot-result-aside {
|
||||
margin-top: 0.5rem;
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted, #555);
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
SnapshotAPI,
|
||||
ImportMode,
|
||||
ImportConflict,
|
||||
ImportResult,
|
||||
} from '../api/snapshotApi';
|
||||
import './SnapshotPanel.css';
|
||||
|
||||
export function SnapshotPanel() {
|
||||
const [exportPluginOutput, setExportPluginOutput] = useState(false);
|
||||
const [importFile, setImportFile] = useState<File | null>(null);
|
||||
const [importMode, setImportMode] = useState<ImportMode>('merge');
|
||||
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
|
||||
const [importing, setImporting] = useState(false);
|
||||
const [importResult, setImportResult] = useState<ImportResult | null>(null);
|
||||
const [importError, setImportError] = useState<string | null>(null);
|
||||
|
||||
const [wipeBusy, setWipeBusy] = useState(false);
|
||||
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
|
||||
|
||||
const runImport = async (dryRun: boolean) => {
|
||||
if (!importFile) {
|
||||
setImportError('Pick a snapshot ZIP first.');
|
||||
return;
|
||||
}
|
||||
setImporting(true);
|
||||
setImportError(null);
|
||||
setImportResult(null);
|
||||
try {
|
||||
const result = await SnapshotAPI.runImport(importFile, {
|
||||
mode: importMode,
|
||||
conflict: importMode === 'merge' ? importConflict : undefined,
|
||||
dryRun,
|
||||
});
|
||||
setImportResult(result);
|
||||
} catch (e: any) {
|
||||
setImportError(e.message || String(e));
|
||||
} finally {
|
||||
setImporting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const runWipe = async () => {
|
||||
const ok = window.confirm(
|
||||
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
|
||||
'edit you made via this editor will be lost. Continue?'
|
||||
);
|
||||
if (!ok) return;
|
||||
setWipeBusy(true);
|
||||
setWipeMessage(null);
|
||||
try {
|
||||
const r = await SnapshotAPI.wipe('metadata');
|
||||
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
|
||||
} catch (e: any) {
|
||||
setWipeMessage(`Failed: ${e.message || e}`);
|
||||
} finally {
|
||||
setWipeBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="snapshot-panel">
|
||||
<h2>Snapshot</h2>
|
||||
<p className="snapshot-intro">
|
||||
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
|
||||
archives — one nested-JSON file per CID, plus a manifest.
|
||||
</p>
|
||||
|
||||
<section className="snapshot-section">
|
||||
<h3>Export</h3>
|
||||
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
|
||||
<label className="snapshot-checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={exportPluginOutput}
|
||||
onChange={(e) => setExportPluginOutput(e.target.checked)}
|
||||
/>
|
||||
<span>
|
||||
Include plugin output
|
||||
<span className="snapshot-hint">
|
||||
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
<button
|
||||
className="snapshot-btn primary"
|
||||
onClick={() =>
|
||||
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
|
||||
}
|
||||
>
|
||||
Download snapshot.zip
|
||||
</button>
|
||||
</section>
|
||||
|
||||
<section className="snapshot-section">
|
||||
<h3>Import</h3>
|
||||
<div className="snapshot-row">
|
||||
<label className="snapshot-file">
|
||||
<span>Snapshot ZIP</span>
|
||||
<input
|
||||
type="file"
|
||||
accept=".zip,application/zip"
|
||||
onChange={(e) => {
|
||||
setImportFile(e.target.files?.[0] ?? null);
|
||||
setImportResult(null);
|
||||
setImportError(null);
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="snapshot-row">
|
||||
<fieldset className="snapshot-fieldset">
|
||||
<legend>Mode</legend>
|
||||
<label>
|
||||
<input
|
||||
type="radio"
|
||||
name="mode"
|
||||
value="merge"
|
||||
checked={importMode === 'merge'}
|
||||
onChange={() => setImportMode('merge')}
|
||||
/>
|
||||
Merge — keep existing fields not in the snapshot
|
||||
</label>
|
||||
<label>
|
||||
<input
|
||||
type="radio"
|
||||
name="mode"
|
||||
value="replace"
|
||||
checked={importMode === 'replace'}
|
||||
onChange={() => setImportMode('replace')}
|
||||
/>
|
||||
Replace — wipe each imported CID and rewrite from snapshot
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
|
||||
<legend>Conflict</legend>
|
||||
<label>
|
||||
<input
|
||||
type="radio"
|
||||
name="conflict"
|
||||
value="source"
|
||||
checked={importConflict === 'source'}
|
||||
onChange={() => setImportConflict('source')}
|
||||
/>
|
||||
Source wins (overwrite my edits)
|
||||
</label>
|
||||
<label>
|
||||
<input
|
||||
type="radio"
|
||||
name="conflict"
|
||||
value="mine"
|
||||
checked={importConflict === 'mine'}
|
||||
onChange={() => setImportConflict('mine')}
|
||||
/>
|
||||
Mine wins (keep my edits, fill in only missing fields)
|
||||
</label>
|
||||
</fieldset>
|
||||
</div>
|
||||
|
||||
<div className="snapshot-row">
|
||||
<button
|
||||
className="snapshot-btn"
|
||||
disabled={!importFile || importing}
|
||||
onClick={() => runImport(true)}
|
||||
>
|
||||
Dry run
|
||||
</button>
|
||||
<button
|
||||
className="snapshot-btn primary"
|
||||
disabled={!importFile || importing}
|
||||
onClick={() => runImport(false)}
|
||||
>
|
||||
{importing ? 'Importing…' : 'Apply import'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{importError && <div className="snapshot-error">{importError}</div>}
|
||||
{importResult && <ImportResultView result={importResult} />}
|
||||
</section>
|
||||
|
||||
<section className="snapshot-section danger">
|
||||
<h3>Wipe metadata</h3>
|
||||
<p>
|
||||
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
|
||||
Files on disk are untouched.
|
||||
</p>
|
||||
<button
|
||||
className="snapshot-btn danger"
|
||||
disabled={wipeBusy}
|
||||
onClick={runWipe}
|
||||
>
|
||||
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
|
||||
</button>
|
||||
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ImportResultView({ result }: { result: ImportResult }) {
|
||||
const totals = (result.files ?? []).reduce(
|
||||
(acc, f) => {
|
||||
acc.added += f.added;
|
||||
acc.updated += f.updated;
|
||||
acc.unchanged += f.unchanged;
|
||||
acc.kept += f.kept;
|
||||
acc.deleted += f.deleted;
|
||||
return acc;
|
||||
},
|
||||
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
|
||||
);
|
||||
return (
|
||||
<div className="snapshot-result">
|
||||
<div className="snapshot-result-summary">
|
||||
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
|
||||
<span>
|
||||
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
|
||||
</span>
|
||||
<span>
|
||||
mode={result.mode}
|
||||
{result.conflict ? `, conflict=${result.conflict}` : ''}
|
||||
</span>
|
||||
</div>
|
||||
<table className="snapshot-result-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>added</th>
|
||||
<th>updated</th>
|
||||
<th>unchanged</th>
|
||||
<th>kept (mine)</th>
|
||||
<th>deleted (replace)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>{totals.added}</td>
|
||||
<td>{totals.updated}</td>
|
||||
<td>{totals.unchanged}</td>
|
||||
<td>{totals.kept}</td>
|
||||
<td>{totals.deleted}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
{result.pluginOutput && (
|
||||
<div className="snapshot-result-aside">
|
||||
plugin-output: {result.pluginOutput.written} written
|
||||
{result.pluginOutput.skipped > 0
|
||||
? `, ${result.pluginOutput.skipped} skipped (existed)`
|
||||
: ''}
|
||||
</div>
|
||||
)}
|
||||
{result.files && result.files.some((f) => f.error) && (
|
||||
<details className="snapshot-result-errors">
|
||||
<summary>Per-file errors</summary>
|
||||
<ul>
|
||||
{result.files
|
||||
.filter((f) => f.error)
|
||||
.map((f) => (
|
||||
<li key={f.hashId}>
|
||||
<code>{f.hashId}</code>: {f.error}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -522,3 +522,54 @@
|
||||
max-height: 160px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.kv-app-gear {
|
||||
margin-left: 0.25rem;
|
||||
font-size: 1.05rem;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.kv-app-gear:hover {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.kv-modal-backdrop {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.45);
|
||||
z-index: 1000;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
padding: 4vh 1rem;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.kv-modal {
|
||||
position: relative;
|
||||
background: white;
|
||||
color: inherit;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
|
||||
width: min(960px, 100%);
|
||||
max-height: 92vh;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.kv-modal-close {
|
||||
position: absolute;
|
||||
top: 0.5rem;
|
||||
right: 0.75rem;
|
||||
background: transparent;
|
||||
border: none;
|
||||
font-size: 1.6rem;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
color: inherit;
|
||||
opacity: 0.6;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.kv-modal-close:hover {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { KVBranchEditor } from './KVBranchEditor';
|
||||
import { SearchFieldChips } from './SearchFieldChips';
|
||||
import { ToastStack, makeToast } from './Toast';
|
||||
import { ToastEntry } from './types';
|
||||
import { SnapshotPanel } from '../components/SnapshotPanel';
|
||||
import './KVApp.css';
|
||||
|
||||
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
|
||||
@@ -28,6 +29,7 @@ function loadSearchFields(): string[] {
|
||||
//
|
||||
// A sticky header carries the search bar, the breadcrumb, and basic stats.
|
||||
export default function KVApp() {
|
||||
const [snapshotOpen, setSnapshotOpen] = useState(false);
|
||||
// Selected node. branchPath is set when a branch is selected; selectedKey
|
||||
// when a leaf is. Mutually exclusive.
|
||||
const [selectedKey, setSelectedKey] = useState<string | null>(null);
|
||||
@@ -149,6 +151,14 @@ export default function KVApp() {
|
||||
onClick={() => setReloadToken((n) => n + 1)}
|
||||
title="refresh tree"
|
||||
>Refresh</button>
|
||||
<button
|
||||
className="kv-btn-icon kv-app-gear"
|
||||
onClick={() => setSnapshotOpen(true)}
|
||||
title="Snapshot — export / import / wipe"
|
||||
aria-label="Open snapshot settings"
|
||||
>
|
||||
⚙
|
||||
</button>
|
||||
</div>
|
||||
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
|
||||
{breadcrumb && breadcrumb.length > 0 && (
|
||||
@@ -237,6 +247,28 @@ export default function KVApp() {
|
||||
</main>
|
||||
</div>
|
||||
|
||||
{snapshotOpen && (
|
||||
<div
|
||||
className="kv-modal-backdrop"
|
||||
onClick={() => setSnapshotOpen(false)}
|
||||
>
|
||||
<div
|
||||
className="kv-modal"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
>
|
||||
<button
|
||||
className="kv-modal-close"
|
||||
onClick={() => setSnapshotOpen(false)}
|
||||
aria-label="Close"
|
||||
title="Close"
|
||||
>×</button>
|
||||
<SnapshotPanel />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ToastStack toasts={toasts} onDismiss={dismissToast} />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/metadataApi.ts","./src/components/BulkMetadataEditor.tsx","./src/components/DynamicFieldRenderer.tsx","./src/components/DynamicPluginSection.tsx","./src/components/FileList.tsx","./src/components/KVTreeView.tsx","./src/components/LanguageCombobox.tsx","./src/components/MetadataEditor.tsx","./src/components/SearchBar.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/typeHeuristics.ts","./src/kv/types.ts","./src/types/index.ts","./src/utils/formatting.ts"],"version":"5.9.3"}
|
||||
{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/metadataApi.ts","./src/api/snapshotApi.ts","./src/components/BulkMetadataEditor.tsx","./src/components/DynamicFieldRenderer.tsx","./src/components/DynamicPluginSection.tsx","./src/components/FileList.tsx","./src/components/KVTreeView.tsx","./src/components/LanguageCombobox.tsx","./src/components/MetadataEditor.tsx","./src/components/SearchBar.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/typeHeuristics.ts","./src/kv/types.ts","./src/types/index.ts","./src/utils/formatting.ts"],"version":"5.9.3"}
|
||||
@@ -29,6 +29,7 @@ type Server struct {
|
||||
mountsManager *mounts.Manager
|
||||
mountsHandlers *mounts.Handlers
|
||||
mountPoller *mounts.Poller
|
||||
mountStatsPoller *mounts.StatsPoller
|
||||
watcherDispatcher *watcher.Dispatcher
|
||||
fileWatcher *watcher.Watcher
|
||||
watcherHandlers *watcher.Handlers
|
||||
@@ -71,6 +72,11 @@ func NewServer(
|
||||
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
||||
} else {
|
||||
s.fileWatcher = fileWatcher
|
||||
// Wire storage so the watcher can persist (path,size,mtime,cid)
|
||||
// tuples to Redis as it computes midhashes.
|
||||
if stor != nil {
|
||||
fileWatcher.SetStorageClient(stor)
|
||||
}
|
||||
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
||||
// Create scan function from watcher for mounts
|
||||
scanFunc = fileWatcher.ScanMountPath
|
||||
@@ -88,6 +94,20 @@ func NewServer(
|
||||
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
||||
// Pass republish callback to handlers (handles nil metaPublisher case)
|
||||
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
|
||||
|
||||
// Hydrate the in-memory state registry from Redis tuples so
|
||||
// the first scan tick after boot can skip ComputeMidHash256
|
||||
// for files whose (size, mtime) match.
|
||||
configs := watchersManager.List()
|
||||
roots := make([]watcher.WatcherRoot, 0, len(configs))
|
||||
for _, c := range configs {
|
||||
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
|
||||
}
|
||||
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
|
||||
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
|
||||
} else {
|
||||
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -103,6 +123,12 @@ func NewServer(
|
||||
// Create and configure mount poller
|
||||
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
|
||||
mountsManager.SetPoller(s.mountPoller)
|
||||
|
||||
// IO stats poller — queries the local rclone RC API every few seconds
|
||||
// and exposes live throughput per mount via the same status JSON.
|
||||
// Stats are daemon-global (rclone limitation); see stats_poller.go.
|
||||
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
|
||||
mountsManager.SetStatsPoller(s.mountStatsPoller)
|
||||
}
|
||||
|
||||
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
|
||||
@@ -126,6 +152,11 @@ func (s *Server) setupRoutes() {
|
||||
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
||||
|
||||
// Snapshot (export / import / wipe)
|
||||
s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
|
||||
s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
|
||||
s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
||||
@@ -240,6 +271,13 @@ func (s *Server) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
|
||||
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
|
||||
// mounts), so always-on rather than gated by config.
|
||||
if s.mountStatsPoller != nil {
|
||||
s.mountStatsPoller.Start()
|
||||
}
|
||||
|
||||
// Start meta publisher (if storage is connected)
|
||||
// Publishes keyspace notifications to meta:events stream
|
||||
if s.storage != nil && s.storage.IsConnected() {
|
||||
@@ -278,6 +316,11 @@ func (s *Server) Stop() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Stop IO stats poller
|
||||
if s.mountStatsPoller != nil {
|
||||
s.mountStatsPoller.Stop()
|
||||
}
|
||||
|
||||
// Stop meta publisher
|
||||
if s.metaPublisher != nil {
|
||||
if err := s.metaPublisher.Stop(); err != nil {
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/snapshot"
|
||||
)
|
||||
|
||||
// Bound the upload size so a malicious client can't OOM the server.
|
||||
// 2 GiB is well above any realistic export.
|
||||
const maxImportBytes = 2 << 30
|
||||
|
||||
// pluginOutputDir is where plugin-produced artifacts live on disk
|
||||
// (subtitles, posters, etc). Mounted from the host into the meta-core
|
||||
// container at $FILES_PATH/plugin.
|
||||
func (s *Server) pluginOutputDir() string {
|
||||
return filepath.Join(s.config.FilesPath, "plugin")
|
||||
}
|
||||
|
||||
// handleSnapshotExport streams a ZIP snapshot of all metadata.
|
||||
// GET /api/snapshot/export?include=hash-index,plugin-output
|
||||
func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// The server has a 30s WriteTimeout (server.go:222). Long exports
|
||||
// blow past that, leaving a truncated ZIP without a central directory.
|
||||
// Clear the deadline for this handler so the export can run as long as
|
||||
// needed. ResponseController is the standard way to do this.
|
||||
rc := http.NewResponseController(w)
|
||||
_ = rc.SetWriteDeadline(time.Time{})
|
||||
|
||||
include := splitCSVParam(r.URL.Query().Get("include"))
|
||||
opts := snapshot.ExportOptions{
|
||||
IncludePluginOutput: include["plugin-output"],
|
||||
PluginOutputDir: s.pluginOutputDir(),
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
|
||||
// We stream — no Content-Length.
|
||||
|
||||
exporter := snapshot.NewExporter(s.storage)
|
||||
if _, err := exporter.Export(w, opts); err != nil {
|
||||
// Headers may already be sent; log and bail.
|
||||
// Client will see a truncated zip, which is the best signal we can give.
|
||||
fmt.Printf("[Snapshot] export failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleSnapshotImport applies an uploaded ZIP snapshot.
|
||||
// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
|
||||
// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
|
||||
func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
|
||||
// when the snapshot is large. Clear both for this handler.
|
||||
rc := http.NewResponseController(w)
|
||||
_ = rc.SetReadDeadline(time.Time{})
|
||||
_ = rc.SetWriteDeadline(time.Time{})
|
||||
|
||||
q := r.URL.Query()
|
||||
opts := snapshot.ImportOptions{
|
||||
Mode: snapshot.Mode(q.Get("mode")),
|
||||
Conflict: snapshot.Conflict(q.Get("conflict")),
|
||||
DryRun: strings.EqualFold(q.Get("dry_run"), "true"),
|
||||
PluginOutputDir: s.pluginOutputDir(),
|
||||
}
|
||||
if opts.Mode == "" {
|
||||
opts.Mode = snapshot.ModeMerge
|
||||
}
|
||||
if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
|
||||
opts.Conflict = snapshot.ConflictSource
|
||||
}
|
||||
|
||||
body, err := readImportBody(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
importer := snapshot.NewImporter(s.storage)
|
||||
result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
// handleSnapshotWipe clears Redis metadata.
|
||||
// POST /api/snapshot/wipe?scope=metadata
|
||||
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
scopes := splitCSVParam(r.URL.Query().Get("scope"))
|
||||
if len(scopes) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
|
||||
return
|
||||
}
|
||||
scope := snapshot.WipeScope{
|
||||
Metadata: scopes["metadata"],
|
||||
}
|
||||
if !scope.Metadata {
|
||||
writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
|
||||
return
|
||||
}
|
||||
|
||||
wiper := snapshot.NewWiper(s.storage)
|
||||
res, err := wiper.Wipe(scope)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, res)
|
||||
}
|
||||
|
||||
// readImportBody accepts either a raw application/zip body or a multipart
|
||||
// upload with a field named "snapshot". Buffers up to maxImportBytes.
|
||||
func readImportBody(r *http.Request) ([]byte, error) {
|
||||
r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
|
||||
|
||||
ct := r.Header.Get("Content-Type")
|
||||
if strings.HasPrefix(ct, "multipart/form-data") {
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
return nil, fmt.Errorf("parse multipart: %w", err)
|
||||
}
|
||||
f, _, err := r.FormFile("snapshot")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
return io.ReadAll(f)
|
||||
}
|
||||
|
||||
return io.ReadAll(r.Body)
|
||||
}
|
||||
|
||||
// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
|
||||
func splitCSVParam(v string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, part := range strings.Split(v, ",") {
|
||||
p := strings.TrimSpace(part)
|
||||
if p != "" {
|
||||
out[p] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
+98
-17
@@ -19,11 +19,12 @@ import (
|
||||
|
||||
// Manager handles mount configuration and status
|
||||
type Manager struct {
|
||||
config *config.Config
|
||||
mu sync.RWMutex
|
||||
filesPath string
|
||||
mountsDir string
|
||||
poller *Poller
|
||||
config *config.Config
|
||||
mu sync.RWMutex
|
||||
filesPath string
|
||||
mountsDir string
|
||||
poller *Poller
|
||||
statsPoller *StatsPoller
|
||||
}
|
||||
|
||||
// NewManager creates a new mount manager
|
||||
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
|
||||
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
||||
}
|
||||
|
||||
// One-shot migration for legacy NFS entries — disable rather than fail to
|
||||
// start, so operators can fix their config at their own pace.
|
||||
if err := m.migrateLegacyMounts(); err != nil {
|
||||
log.Printf("[Mounts] legacy migration warning: %v", err)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// migrateLegacyMounts disables any mount entries with type="nfs". The native
|
||||
// NFS handler was removed when the project consolidated on rclone; an NFS
|
||||
// entry left enabled would loop the watcher with mount errors.
|
||||
func (m *Manager) migrateLegacyMounts() error {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dirty := false
|
||||
for i := range mountsFile.Mounts {
|
||||
mnt := &mountsFile.Mounts[i]
|
||||
if string(mnt.Type) == "nfs" {
|
||||
if mnt.Enabled || mnt.DesiredMounted {
|
||||
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
|
||||
mnt.Enabled = false
|
||||
mnt.DesiredMounted = false
|
||||
dirty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if dirty {
|
||||
return m.writeConfig(mountsFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetPoller sets the poller for the manager
|
||||
func (m *Manager) SetPoller(poller *Poller) {
|
||||
m.poller = poller
|
||||
}
|
||||
|
||||
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
|
||||
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
|
||||
// set; remains nil-safe when stats are disabled.
|
||||
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
|
||||
m.statsPoller = sp
|
||||
}
|
||||
|
||||
// ensureDirs creates required directories
|
||||
func (m *Manager) ensureDirs() error {
|
||||
dirs := []string{
|
||||
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
|
||||
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
||||
statuses[i].PollingActive = pollingStatus.Active
|
||||
statuses[i].LastPolledScan = pollingStatus.LastScan
|
||||
statuses[i].Scanning = pollingStatus.Scanning
|
||||
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||
statuses[i].NextScanAt = pollingStatus.NextScanAt
|
||||
}
|
||||
}
|
||||
if m.statsPoller != nil {
|
||||
for i := range statuses {
|
||||
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
|
||||
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
||||
status.PollingActive = pollingStatus.Active
|
||||
status.LastPolledScan = pollingStatus.LastScan
|
||||
status.Scanning = pollingStatus.Scanning
|
||||
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||
status.NextScanAt = pollingStatus.NextScanAt
|
||||
}
|
||||
if m.statsPoller != nil {
|
||||
status.IOStats = m.statsPoller.GetStats(mount.ID)
|
||||
}
|
||||
|
||||
return status, nil
|
||||
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
return nil, fmt.Errorf("mount name is required")
|
||||
}
|
||||
|
||||
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
|
||||
// "nfs" used to be valid; the native handler is gone. Reject loudly so the
|
||||
// caller knows their integration needs to migrate to a Samba re-export or
|
||||
// switch to a supported protocol.
|
||||
if req.Type == MountType("nfs") {
|
||||
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
|
||||
}
|
||||
|
||||
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
|
||||
}
|
||||
|
||||
// Validate type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
if req.NFSServer == "" || req.NFSPath == "" {
|
||||
return nil, fmt.Errorf("NFS server and path are required")
|
||||
}
|
||||
case MountTypeSMB:
|
||||
if req.SMBServer == "" || req.SMBShare == "" {
|
||||
return nil, fmt.Errorf("SMB server and share are required")
|
||||
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
enabled = *req.Enabled
|
||||
}
|
||||
|
||||
// Default polling settings
|
||||
pollingEnabled := false
|
||||
// Polling defaults on for both supported types — neither SMB-via-rclone nor
|
||||
// generic rclone delivers kernel inotify, so a periodic re-scan is the only
|
||||
// way to pick up upstream changes. Explicit PollingEnabled in the request
|
||||
// wins.
|
||||
pollingEnabled := true
|
||||
if req.PollingEnabled != nil {
|
||||
pollingEnabled = *req.PollingEnabled
|
||||
}
|
||||
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
Enabled: enabled,
|
||||
DesiredMounted: enabled, // Auto-mount if enabled
|
||||
MountPath: mountPath,
|
||||
Options: req.Options,
|
||||
CacheMaxSize: req.CacheMaxSize,
|
||||
CacheMaxAge: req.CacheMaxAge,
|
||||
DirCacheTime: req.DirCacheTime,
|
||||
PollingEnabled: pollingEnabled,
|
||||
PollingIntervalMs: pollingIntervalMs,
|
||||
}
|
||||
|
||||
// Type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
mount.NFSServer = req.NFSServer
|
||||
mount.NFSPath = req.NFSPath
|
||||
case MountTypeSMB:
|
||||
mount.SMBServer = req.SMBServer
|
||||
mount.SMBShare = req.SMBShare
|
||||
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
|
||||
}
|
||||
}
|
||||
|
||||
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
|
||||
// don't validate format here; rclone rejects bad values at mount time.
|
||||
if val, ok := updates["cacheMaxSize"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.CacheMaxSize = s
|
||||
}
|
||||
}
|
||||
if val, ok := updates["cacheMaxAge"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.CacheMaxAge = s
|
||||
}
|
||||
}
|
||||
if val, ok := updates["dirCacheTime"]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
mount.DirCacheTime = s
|
||||
}
|
||||
}
|
||||
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+171
-34
@@ -9,6 +9,25 @@ import (
|
||||
// ScanFunc is a function type for scanning a mount path
|
||||
type ScanFunc func(path string) (int, error)
|
||||
|
||||
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
|
||||
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
|
||||
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
|
||||
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
|
||||
// large remote shares (CORN, etc.) from monopolising the network/CPU.
|
||||
const IdleFactor = 9
|
||||
|
||||
// InitialScanDelay is how long a freshly-started poller waits before its first
|
||||
// scan. Avoids hammering the system at container startup when many mounts come
|
||||
// up at once.
|
||||
const InitialScanDelay = 2 * time.Second
|
||||
|
||||
// ReconcileInterval drives the periodic resync loop that keeps poller
|
||||
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
|
||||
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
|
||||
// — without periodic reconciliation, the poller would never observe the
|
||||
// transition from `mounted=false` to `mounted=true` and never start scanning.
|
||||
const ReconcileInterval = 10 * time.Second
|
||||
|
||||
// mountPoller manages polling for a single mount
|
||||
type mountPoller struct {
|
||||
mountID string
|
||||
@@ -19,8 +38,12 @@ type mountPoller struct {
|
||||
|
||||
// PollingStatus holds the polling state for a mount
|
||||
type PollingStatus struct {
|
||||
Active bool `json:"active"`
|
||||
LastScan int64 `json:"lastScan,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
|
||||
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
|
||||
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
|
||||
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
|
||||
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
|
||||
}
|
||||
|
||||
// Poller manages per-mount polling goroutines
|
||||
@@ -31,8 +54,9 @@ type Poller struct {
|
||||
pollers map[string]*mountPoller // key: mount ID
|
||||
status map[string]*PollingStatus
|
||||
|
||||
running bool
|
||||
mu sync.RWMutex
|
||||
running bool
|
||||
reconcileStop chan struct{}
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewPoller creates a new mount poller
|
||||
@@ -54,15 +78,35 @@ func (p *Poller) Start() error {
|
||||
return nil
|
||||
}
|
||||
p.running = true
|
||||
p.reconcileStop = make(chan struct{})
|
||||
|
||||
log.Println("[Poller] Starting mount polling service")
|
||||
|
||||
// Initial sync with mount configurations
|
||||
go p.syncWithMounts()
|
||||
|
||||
// Periodic reconciliation — picks up mount-watcher.sh transitions
|
||||
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
|
||||
go p.reconcileLoop(p.reconcileStop)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
|
||||
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
|
||||
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
|
||||
ticker := time.NewTicker(ReconcileInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.syncWithMounts()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops all polling goroutines
|
||||
func (p *Poller) Stop() error {
|
||||
p.mu.Lock()
|
||||
@@ -75,6 +119,11 @@ func (p *Poller) Stop() error {
|
||||
|
||||
log.Println("[Poller] Stopping mount polling service")
|
||||
|
||||
if p.reconcileStop != nil {
|
||||
close(p.reconcileStop)
|
||||
p.reconcileStop = nil
|
||||
}
|
||||
|
||||
// Stop all pollers
|
||||
for id, poller := range p.pollers {
|
||||
p.stopPoller(poller)
|
||||
@@ -127,7 +176,7 @@ func (p *Poller) syncWithMounts() {
|
||||
existingPoller, exists := p.pollers[mount.ID]
|
||||
|
||||
if shouldPoll {
|
||||
// Calculate interval
|
||||
// Calculate interval (used as the floor for adaptive scheduling)
|
||||
interval := mount.PollingIntervalMs
|
||||
if interval < MinPollingIntervalMs {
|
||||
interval = DefaultPollingIntervalMs
|
||||
@@ -135,7 +184,7 @@ func (p *Poller) syncWithMounts() {
|
||||
|
||||
if !exists {
|
||||
// Start new poller
|
||||
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
|
||||
log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
|
||||
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
||||
}
|
||||
} else if exists {
|
||||
@@ -156,7 +205,13 @@ func (p *Poller) syncWithMounts() {
|
||||
}
|
||||
}
|
||||
|
||||
// startPollerForMount starts a polling goroutine for a mount
|
||||
// startPollerForMount starts the adaptive polling goroutine for a mount.
|
||||
//
|
||||
// The loop runs scans sequentially: scan → measure duration → sleep
|
||||
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
|
||||
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
|
||||
// cycle on slow shares, while keeping fresh checks on small mounts via the
|
||||
// configured interval floor.
|
||||
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
||||
poller := &mountPoller{
|
||||
mountID: mountID,
|
||||
@@ -170,17 +225,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
|
||||
}
|
||||
p.status[mountID].Active = true
|
||||
|
||||
floor := time.Duration(intervalMs) * time.Millisecond
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
// Initial delay so a freshly-mounted share doesn't get hammered the
|
||||
// instant the poller starts.
|
||||
nextDelay := InitialScanDelay
|
||||
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-poller.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.runPollScan(mountID, mountPath)
|
||||
case <-time.After(nextDelay):
|
||||
}
|
||||
|
||||
// Re-check stop after the wake-up to avoid running a scan during shutdown.
|
||||
select {
|
||||
case <-poller.stopChan:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
duration, err := p.executeScan(mountID, mountPath)
|
||||
if err != nil {
|
||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||
}
|
||||
|
||||
// Adaptive backoff: idle for IdleFactor × last duration, with the
|
||||
// configured interval as a floor so tiny mounts still scan often.
|
||||
nextDelay = duration * IdleFactor
|
||||
if nextDelay < floor {
|
||||
nextDelay = floor
|
||||
}
|
||||
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -199,54 +277,109 @@ func (p *Poller) stopPoller(poller *mountPoller) {
|
||||
// Update status
|
||||
if status, exists := p.status[poller.mountID]; exists {
|
||||
status.Active = false
|
||||
status.NextScanAt = 0
|
||||
}
|
||||
}
|
||||
|
||||
// runPollScan executes a scan for a mount
|
||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||
// executeScan runs scanFunc with timing + status tracking. Both the adaptive
|
||||
// loop and manual TriggerScan funnel through here so the UI sees a consistent
|
||||
// "scanning / last duration / last scan" picture regardless of which path
|
||||
// kicked it off. Returns the scan duration so the caller can compute the next
|
||||
// adaptive delay.
|
||||
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
|
||||
if p.scanFunc == nil {
|
||||
return
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
startMs := start.UnixMilli()
|
||||
|
||||
p.mu.Lock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.Scanning = true
|
||||
status.CurrentScanStartedAt = startMs
|
||||
p.mu.Unlock()
|
||||
|
||||
fileCount, err := p.scanFunc(mountPath)
|
||||
if err != nil {
|
||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update status
|
||||
end := time.Now()
|
||||
duration := end.Sub(start)
|
||||
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
status.Scanning = false
|
||||
status.CurrentScanStartedAt = 0
|
||||
status.LastScan = end.UnixMilli()
|
||||
status.LastScanDurationMs = duration.Milliseconds()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
|
||||
if err == nil {
|
||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
return duration, err
|
||||
}
|
||||
|
||||
// TriggerScan triggers an immediate scan for a mount
|
||||
// setNextScanAt records the projected start time of the next adaptive scan
|
||||
// (used purely for the UI countdown — the loop's own timer is authoritative).
|
||||
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.NextScanAt = at.UnixMilli()
|
||||
}
|
||||
|
||||
// runPollScan is kept for backwards compatibility with any external caller —
|
||||
// new code should go through executeScan via the adaptive loop.
|
||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||
_, _ = p.executeScan(mountID, mountPath)
|
||||
}
|
||||
|
||||
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
|
||||
// Runs synchronously; concurrent calls with the adaptive loop are possible
|
||||
// but rare in practice — both paths update the same status atomically.
|
||||
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
||||
if p.scanFunc == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
startMs := start.UnixMilli()
|
||||
|
||||
p.mu.Lock()
|
||||
status, exists := p.status[mountID]
|
||||
if !exists {
|
||||
status = &PollingStatus{}
|
||||
p.status[mountID] = status
|
||||
}
|
||||
status.Scanning = true
|
||||
status.CurrentScanStartedAt = startMs
|
||||
p.mu.Unlock()
|
||||
|
||||
fileCount, err := p.scanFunc(mountPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
end := time.Now()
|
||||
duration := end.Sub(start)
|
||||
|
||||
// Update status
|
||||
p.mu.Lock()
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
status.LastScan = NowMS()
|
||||
} else {
|
||||
p.status[mountID] = &PollingStatus{
|
||||
LastScan: NowMS(),
|
||||
}
|
||||
status.Scanning = false
|
||||
status.CurrentScanStartedAt = 0
|
||||
status.LastScan = end.UnixMilli()
|
||||
status.LastScanDurationMs = duration.Milliseconds()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
return fileCount, nil
|
||||
return fileCount, err
|
||||
}
|
||||
|
||||
// GetPollingStatus returns the polling status for a mount
|
||||
@@ -256,8 +389,12 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
|
||||
|
||||
if status, exists := p.status[mountID]; exists {
|
||||
return &PollingStatus{
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
Active: status.Active,
|
||||
LastScan: status.LastScan,
|
||||
Scanning: status.Scanning,
|
||||
CurrentScanStartedAt: status.CurrentScanStartedAt,
|
||||
LastScanDurationMs: status.LastScanDurationMs,
|
||||
NextScanAt: status.NextScanAt,
|
||||
}
|
||||
}
|
||||
return &PollingStatus{}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
|
||||
// time. The poller keeps the previous sample per mount and computes rates
|
||||
// (Bps, transfers/sec) from the delta to the latest sample.
|
||||
//
|
||||
// Post-consolidation: rclone is the only sample source, since all mount types
|
||||
// execute through the local rclone daemon.
|
||||
type StatsSample struct {
|
||||
TakenAt time.Time
|
||||
Source string // "rclone"
|
||||
BytesRead int64
|
||||
BytesWritten int64
|
||||
ReadOps int64
|
||||
WriteOps int64
|
||||
// File-level transfer count from rclone's RC API. Distinct from ReadOps
|
||||
// (which is unused for rclone — kernel ops counters are not exposed
|
||||
// through FUSE).
|
||||
Transfers int64
|
||||
}
|
||||
|
||||
// rcloneCoreStats matches the RC API response from POST /core/stats.
|
||||
type rcloneCoreStats struct {
|
||||
Bytes int64 `json:"bytes"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
Transfers int64 `json:"transfers"`
|
||||
TotalTransfers int64 `json:"totalTransfers"`
|
||||
}
|
||||
|
||||
// SampleRclone queries the local rclone RC API for the cumulative bytes +
|
||||
// transfers counters. The numbers are global across all active mounts on this
|
||||
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
|
||||
// sample for every mounted entry on a tick.
|
||||
func SampleRclone(ctx context.Context) (*StatsSample, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build rclone stats request: %w", err)
|
||||
}
|
||||
req.SetBasicAuth("admin", "admin")
|
||||
|
||||
client := &http.Client{Timeout: 3 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var stats rcloneCoreStats
|
||||
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||
return nil, fmt.Errorf("decode rclone stats: %w", err)
|
||||
}
|
||||
|
||||
return &StatsSample{
|
||||
Source: "rclone",
|
||||
TakenAt: time.Now(),
|
||||
BytesRead: stats.Bytes,
|
||||
Transfers: stats.Transfers,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
|
||||
// spot — short enough that throughput numbers feel live, long enough that
|
||||
// /proc reads aren't a load source themselves.
|
||||
const StatsInterval = 5 * time.Second
|
||||
|
||||
// StatsPoller samples cumulative counters from the local rclone RC API and
|
||||
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
|
||||
//
|
||||
// Post-consolidation note: every mount type executes through rclone, so
|
||||
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
|
||||
// rclone is the only counter source. core/stats is daemon-global across all
|
||||
// active mounts; the same sample is applied to each mount's status. If
|
||||
// per-mount byte counters become important, the path forward is rclone stats
|
||||
// groups (out of scope for this consolidation).
|
||||
type StatsPoller struct {
|
||||
manager *Manager
|
||||
|
||||
mu sync.RWMutex
|
||||
lastSamples map[string]*StatsSample // mountID -> previous raw sample
|
||||
latestStats map[string]*MountIOStats
|
||||
stopChan chan struct{}
|
||||
running bool
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
// NewStatsPoller constructs a StatsPoller backed by the given manager.
|
||||
func NewStatsPoller(manager *Manager) *StatsPoller {
|
||||
return &StatsPoller{
|
||||
manager: manager,
|
||||
lastSamples: make(map[string]*StatsSample),
|
||||
latestStats: make(map[string]*MountIOStats),
|
||||
pollInterval: StatsInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches the sampling loop. Idempotent.
|
||||
func (p *StatsPoller) Start() {
|
||||
p.mu.Lock()
|
||||
if p.running {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.running = true
|
||||
p.stopChan = make(chan struct{})
|
||||
stop := p.stopChan
|
||||
p.mu.Unlock()
|
||||
|
||||
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
|
||||
go p.loop(stop)
|
||||
}
|
||||
|
||||
// Stop terminates the sampling loop. Idempotent.
|
||||
func (p *StatsPoller) Stop() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.running {
|
||||
return
|
||||
}
|
||||
p.running = false
|
||||
if p.stopChan != nil {
|
||||
close(p.stopChan)
|
||||
p.stopChan = nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns the latest computed IO stats for a mount, or nil if no
|
||||
// stats have been observed yet.
|
||||
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
s, ok := p.latestStats[mountID]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
// Return a copy so the caller can't mutate our cache.
|
||||
cp := *s
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (p *StatsPoller) loop(stop <-chan struct{}) {
|
||||
// Take the first sample immediately so rates start populating on the
|
||||
// second tick instead of waiting two intervals.
|
||||
p.tick()
|
||||
|
||||
t := time.NewTicker(p.pollInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
p.tick()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StatsPoller) tick() {
|
||||
mounts, err := p.manager.listMountsWithoutPollingStatus()
|
||||
if err != nil {
|
||||
log.Printf("[StatsPoller] listMounts failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Sample rclone once per tick — its RC API gives global stats across all
|
||||
// active rclone mounts on this host. Both supported mount types (smb,
|
||||
// rclone) execute through the same rclone daemon, so the same sample is
|
||||
// applied to every mounted entry.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
sample, err := SampleRclone(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[StatsPoller] rclone sample failed: %v", err)
|
||||
return
|
||||
}
|
||||
if sample == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, m := range mounts {
|
||||
if !m.Mounted {
|
||||
continue
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
prev := p.lastSamples[m.ID]
|
||||
p.lastSamples[m.ID] = sample
|
||||
stats := computeStats(prev, sample)
|
||||
p.latestStats[m.ID] = stats
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// computeStats turns a (previous, current) raw-sample pair into the rate-
|
||||
// based MountIOStats the API exposes. With no previous sample the rates are
|
||||
// zero — operators will see cumulative counters but Bps/IOPS only on the
|
||||
// second poll.
|
||||
func computeStats(prev, cur *StatsSample) *MountIOStats {
|
||||
out := &MountIOStats{
|
||||
Source: cur.Source,
|
||||
BytesRead: cur.BytesRead,
|
||||
BytesWritten: cur.BytesWritten,
|
||||
ReadOps: cur.ReadOps,
|
||||
WriteOps: cur.WriteOps,
|
||||
LastSampleAt: cur.TakenAt.UnixMilli(),
|
||||
}
|
||||
if prev == nil {
|
||||
return out
|
||||
}
|
||||
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
|
||||
if dt <= 0 {
|
||||
return out
|
||||
}
|
||||
out.IntervalMs = int64(dt * 1000)
|
||||
|
||||
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
|
||||
// negative deltas as "no movement" rather than letting the rate go
|
||||
// negative or implausibly huge.
|
||||
if d := cur.BytesRead - prev.BytesRead; d > 0 {
|
||||
out.ReadBps = float64(d) / dt
|
||||
}
|
||||
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
|
||||
out.WriteBps = float64(d) / dt
|
||||
}
|
||||
if d := cur.ReadOps - prev.ReadOps; d > 0 {
|
||||
out.ReadIops = float64(d) / dt
|
||||
}
|
||||
if d := cur.WriteOps - prev.WriteOps; d > 0 {
|
||||
out.WriteIops = float64(d) / dt
|
||||
}
|
||||
if cur.Source == "rclone" {
|
||||
if d := cur.Transfers - prev.Transfers; d > 0 {
|
||||
out.TransfersPerSec = float64(d) / dt
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
+67
-17
@@ -2,11 +2,19 @@ package mounts
|
||||
|
||||
import "time"
|
||||
|
||||
// MountType represents the type of mount
|
||||
// MountType represents the type of mount.
|
||||
//
|
||||
// All types execute through the local rclone daemon. "smb" is a UX-level alias
|
||||
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
|
||||
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
|
||||
// the generic escape hatch for any pre-defined remote in rclone.conf.
|
||||
//
|
||||
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
|
||||
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
|
||||
// regardless.
|
||||
type MountType string
|
||||
|
||||
const (
|
||||
MountTypeNFS MountType = "nfs"
|
||||
MountTypeSMB MountType = "smb"
|
||||
MountTypeRclone MountType = "rclone"
|
||||
)
|
||||
@@ -17,6 +25,15 @@ const (
|
||||
MinPollingIntervalMs = 5000 // 5 seconds
|
||||
)
|
||||
|
||||
// VFS cache defaults — applied per-mount when the caller leaves the field
|
||||
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
|
||||
// Duration) untouched.
|
||||
const (
|
||||
DefaultCacheMaxSize = "50G"
|
||||
DefaultCacheMaxAge = "24h"
|
||||
DefaultDirCacheTime = "5m"
|
||||
)
|
||||
|
||||
// MountConfig represents a mount configuration
|
||||
type MountConfig struct {
|
||||
// Common fields
|
||||
@@ -26,23 +43,26 @@ type MountConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DesiredMounted bool `json:"desiredMounted"`
|
||||
MountPath string `json:"mountPath"`
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS-specific fields
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
|
||||
// SMB-specific fields
|
||||
// SMB-specific fields (also valid for type=smb routed through rclone-smb)
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
SMBShare string `json:"smbShare,omitempty"`
|
||||
SMBUsername string `json:"smbUsername,omitempty"`
|
||||
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
||||
SMBDomain string `json:"smbDomain,omitempty"`
|
||||
|
||||
// rclone-specific fields
|
||||
// rclone-specific fields (escape hatch — references a pre-defined remote)
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
|
||||
// VFS cache configuration — applied to the rclone mount call. All three
|
||||
// fields are optional; blanks fall back to Default* constants above.
|
||||
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
|
||||
// DirCacheTime use rclone's Duration ("24h", "30m").
|
||||
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||
|
||||
// Polling configuration
|
||||
PollingEnabled bool `json:"pollingEnabled"`
|
||||
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
||||
@@ -56,6 +76,36 @@ type MountStatus struct {
|
||||
LastChecked int64 `json:"lastChecked"`
|
||||
PollingActive bool `json:"pollingActive,omitempty"`
|
||||
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
||||
// Progress tracking — surface the poller's adaptive scan state to the UI.
|
||||
Scanning bool `json:"scanning,omitempty"`
|
||||
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
|
||||
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
|
||||
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
|
||||
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
|
||||
// /proc/self/mountstats, or rclone's RC API depending on mount type.
|
||||
IOStats *MountIOStats `json:"ioStats,omitempty"`
|
||||
}
|
||||
|
||||
// MountIOStats captures throughput + operation rate for a single mount.
|
||||
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
|
||||
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
|
||||
// the last two samples divided by the interval.
|
||||
type MountIOStats struct {
|
||||
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
|
||||
BytesRead int64 `json:"bytesRead"`
|
||||
BytesWritten int64 `json:"bytesWritten,omitempty"`
|
||||
ReadBps float64 `json:"readBps"`
|
||||
WriteBps float64 `json:"writeBps,omitempty"`
|
||||
ReadOps int64 `json:"readOps,omitempty"`
|
||||
WriteOps int64 `json:"writeOps,omitempty"`
|
||||
// ReadIops and WriteIops are op counters per second — number of read/write
|
||||
// requests issued by the kernel against the share. Distinct from
|
||||
// TransfersPerSec which is a file-level metric only relevant to rclone.
|
||||
ReadIops float64 `json:"readIops,omitempty"`
|
||||
WriteIops float64 `json:"writeIops,omitempty"`
|
||||
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
|
||||
LastSampleAt int64 `json:"lastSampleAt"` // ms
|
||||
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
|
||||
}
|
||||
|
||||
// MountsFile represents the mounts.json file structure
|
||||
@@ -72,14 +122,9 @@ type RcloneRemote struct {
|
||||
|
||||
// CreateMountRequest is the request body for creating a mount
|
||||
type CreateMountRequest struct {
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||
|
||||
// SMB
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
|
||||
// VFS cache (optional — falls back to Default* constants when unset)
|
||||
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||
|
||||
// Polling
|
||||
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
||||
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
|
||||
// Trailing slash because ZIP entry names are forward-slash relative paths.
|
||||
const PluginOutputArchivePrefix = "cache/plugin-output/"
|
||||
|
||||
// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
|
||||
// file under it into zw at PluginOutputArchivePrefix + relPath.
|
||||
//
|
||||
// Returns the number of files added. A missing srcDir is not an error: it
|
||||
// just means no plugin has produced output yet, so the section stays empty.
|
||||
func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
|
||||
info, err := os.Stat(srcDir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stat %s: %w", srcDir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return 0, fmt.Errorf("%s is not a directory", srcDir)
|
||||
}
|
||||
|
||||
count := 0
|
||||
err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(srcDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
|
||||
|
||||
w, err := zw.Create(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
return err
|
||||
}
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("walk %s: %w", srcDir, err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
|
||||
// the ZIP and writes it under dstDir. Existing files are overwritten when
|
||||
// overwrite=true; otherwise they're left alone (treats existing files as the
|
||||
// "mine wins" choice — same toggle the metadata import uses).
|
||||
//
|
||||
// Path traversal is rejected: any entry whose relative path resolves outside
|
||||
// dstDir is skipped with a warning.
|
||||
func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
|
||||
if dstDir == "" {
|
||||
return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
|
||||
}
|
||||
if err := os.MkdirAll(dstDir, 0o755); err != nil {
|
||||
return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
|
||||
}
|
||||
|
||||
cleanRoot, err := filepath.Abs(dstDir)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
|
||||
}
|
||||
|
||||
for _, f := range zr.File {
|
||||
if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(f.Name, "/") {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
|
||||
dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
|
||||
|
||||
// Reject anything that escapes the dst root.
|
||||
if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
|
||||
log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
if !overwrite {
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
|
||||
}
|
||||
|
||||
if err := writeZipFile(f, dst); err != nil {
|
||||
return written, skipped, err
|
||||
}
|
||||
written++
|
||||
}
|
||||
return written, skipped, nil
|
||||
}
|
||||
|
||||
func writeZipFile(f *zip.File, dst string) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open zip entry %s: %w", f.Name, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", dst, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err := io.Copy(out, rc); err != nil {
|
||||
return fmt.Errorf("write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// ExportOptions controls what an export includes.
|
||||
type ExportOptions struct {
|
||||
IncludePluginOutput bool
|
||||
// PluginOutputDir is the absolute path on disk that holds plugin-produced
|
||||
// artifacts (typically $FILES_PATH/plugin). Required when
|
||||
// IncludePluginOutput is true.
|
||||
PluginOutputDir string
|
||||
// SourceHost is recorded in the manifest so imports can attribute
|
||||
// where the snapshot came from (defaults to OS hostname).
|
||||
SourceHost string
|
||||
}
|
||||
|
||||
// Exporter writes snapshot ZIPs from a live Redis store.
|
||||
type Exporter struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewExporter(stor *storage.Client) *Exporter {
|
||||
return &Exporter{storage: stor}
|
||||
}
|
||||
|
||||
// Export streams a ZIP archive to w. The archive layout is:
|
||||
//
|
||||
// manifest.json
|
||||
// index.json
|
||||
// metadata/{cid}.json (one per known CID)
|
||||
//
|
||||
// Optional sections (hash-index, plugin-output) are gated by ExportOptions
|
||||
// but are not yet implemented — they live on filesystems meta-core does not
|
||||
// own, and need cross-service collection. The flag is plumbed for forward
|
||||
// compatibility but does nothing today.
|
||||
func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
|
||||
if e.storage == nil || !e.storage.IsConnected() {
|
||||
return Manifest{}, fmt.Errorf("storage not connected")
|
||||
}
|
||||
|
||||
hashIDs, err := e.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
|
||||
}
|
||||
|
||||
zw := zip.NewWriter(w)
|
||||
defer zw.Close()
|
||||
|
||||
// 1. manifest.json — written last after we know FileCount, but we need
|
||||
// the entry order to put it first in the archive for readers. Solve
|
||||
// by buffering in memory: it's tiny.
|
||||
host := opts.SourceHost
|
||||
if host == "" {
|
||||
if h, err := os.Hostname(); err == nil {
|
||||
host = h
|
||||
}
|
||||
}
|
||||
manifest := Manifest{
|
||||
SchemaVersion: ManifestSchemaVersion,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
SourceHost: host,
|
||||
FileCount: len(hashIDs),
|
||||
Includes: Includes{
|
||||
Metadata: true,
|
||||
PluginOutput: opts.IncludePluginOutput,
|
||||
},
|
||||
}
|
||||
if err := writeJSON(zw, ManifestFile, manifest); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
|
||||
// 2. index.json — flat list of CIDs. Redundant with metadata/ but
|
||||
// cheap and useful for readers that only need the keyset.
|
||||
if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write index: %w", err)
|
||||
}
|
||||
|
||||
// 3. metadata/{cid}.json — one nested document per CID.
|
||||
for _, hashID := range hashIDs {
|
||||
flat, err := e.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
|
||||
continue
|
||||
}
|
||||
if len(flat) == 0 {
|
||||
continue
|
||||
}
|
||||
nested := Reconstruct(flat)
|
||||
entry := MetadataDir + hashID + ".json"
|
||||
if err := writeJSON(zw, entry, nested); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
|
||||
// streams each file into the ZIP. Heavy (~120 MB on a populated dev
|
||||
// stack), so it's gated by the ExportOptions toggle.
|
||||
if opts.IncludePluginOutput {
|
||||
if opts.PluginOutputDir == "" {
|
||||
return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
|
||||
}
|
||||
n, err := addPluginOutput(zw, opts.PluginOutputDir)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
|
||||
}
|
||||
log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
|
||||
}
|
||||
|
||||
log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
|
||||
len(hashIDs), host, opts.IncludePluginOutput)
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func writeJSON(zw *zip.Writer, name string, body interface{}) error {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(body)
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Flat keys use "/" as the separator. Property paths inside Redis are
|
||||
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
|
||||
const flatSep = "/"
|
||||
|
||||
// numericKey matches a path segment that is a non-negative decimal integer.
|
||||
// Used to detect array slots (".../video/0/codec").
|
||||
var numericKey = regexp.MustCompile(`^\d+$`)
|
||||
|
||||
// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
|
||||
// Keys are property paths (no leading slash, no file:{cid}/ prefix).
|
||||
//
|
||||
// Sibling keys whose path segments are all sequential integers starting at 0
|
||||
// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
|
||||
// are coerced from string to int/float/bool/null when they parse cleanly so
|
||||
// the exported JSON is human-readable; on import we round-trip back to strings.
|
||||
func Reconstruct(flat map[string]string) interface{} {
|
||||
root := map[string]interface{}{}
|
||||
for path, value := range flat {
|
||||
parts := strings.Split(path, flatSep)
|
||||
setPath(root, parts, parseValue(value))
|
||||
}
|
||||
return convertNumericMapsToArrays(root)
|
||||
}
|
||||
|
||||
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
|
||||
// flat property map whose values are all strings (Redis-ready).
|
||||
//
|
||||
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
|
||||
// values are stored as "" to preserve the field's presence — this matches
|
||||
// what flattenMetadata in the TS implementation does.
|
||||
func Flatten(value interface{}) map[string]string {
|
||||
out := map[string]string{}
|
||||
flattenInto(value, "", out)
|
||||
return out
|
||||
}
|
||||
|
||||
func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
|
||||
if len(parts) == 0 {
|
||||
return
|
||||
}
|
||||
cur := root
|
||||
for i, part := range parts {
|
||||
if i == len(parts)-1 {
|
||||
cur[part] = leaf
|
||||
return
|
||||
}
|
||||
next, ok := cur[part].(map[string]interface{})
|
||||
if !ok {
|
||||
next = map[string]interface{}{}
|
||||
cur[part] = next
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
}
|
||||
|
||||
func convertNumericMapsToArrays(v interface{}) interface{} {
|
||||
switch t := v.(type) {
|
||||
case map[string]interface{}:
|
||||
// Recurse first so child arrays are detected before parent.
|
||||
for k, child := range t {
|
||||
t[k] = convertNumericMapsToArrays(child)
|
||||
}
|
||||
if !looksLikeArray(t) {
|
||||
return t
|
||||
}
|
||||
// Sort numerically and emit as []interface{}.
|
||||
idx := make([]int, 0, len(t))
|
||||
for k := range t {
|
||||
n, _ := strconv.Atoi(k)
|
||||
idx = append(idx, n)
|
||||
}
|
||||
sort.Ints(idx)
|
||||
arr := make([]interface{}, len(idx))
|
||||
for i, n := range idx {
|
||||
arr[i] = t[strconv.Itoa(n)]
|
||||
}
|
||||
return arr
|
||||
case []interface{}:
|
||||
for i := range t {
|
||||
t[i] = convertNumericMapsToArrays(t[i])
|
||||
}
|
||||
return t
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// looksLikeArray returns true when every key in m is a non-negative integer
|
||||
// and the keys form a contiguous range starting at 0. Empty maps return false
|
||||
// — we can't tell whether the original was {} or [], so prefer {}.
|
||||
func looksLikeArray(m map[string]interface{}) bool {
|
||||
if len(m) == 0 {
|
||||
return false
|
||||
}
|
||||
seen := make(map[int]bool, len(m))
|
||||
for k := range m {
|
||||
if !numericKey.MatchString(k) {
|
||||
return false
|
||||
}
|
||||
n, err := strconv.Atoi(k)
|
||||
if err != nil || n < 0 {
|
||||
return false
|
||||
}
|
||||
seen[n] = true
|
||||
}
|
||||
for i := 0; i < len(m); i++ {
|
||||
if !seen[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func flattenInto(v interface{}, path string, out map[string]string) {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
out[path] = ""
|
||||
case map[string]interface{}:
|
||||
if len(t) == 0 && path != "" {
|
||||
// Empty objects don't have a Redis representation; skip.
|
||||
return
|
||||
}
|
||||
for k, child := range t {
|
||||
next := k
|
||||
if path != "" {
|
||||
next = path + flatSep + k
|
||||
}
|
||||
flattenInto(child, next, out)
|
||||
}
|
||||
case []interface{}:
|
||||
for i, child := range t {
|
||||
next := strconv.Itoa(i)
|
||||
if path != "" {
|
||||
next = path + flatSep + next
|
||||
}
|
||||
flattenInto(child, next, out)
|
||||
}
|
||||
case string:
|
||||
out[path] = t
|
||||
case bool:
|
||||
if t {
|
||||
out[path] = "true"
|
||||
} else {
|
||||
out[path] = "false"
|
||||
}
|
||||
case float64:
|
||||
// JSON numbers always decode to float64; emit ints without decimals.
|
||||
if t == float64(int64(t)) {
|
||||
out[path] = strconv.FormatInt(int64(t), 10)
|
||||
} else {
|
||||
out[path] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||
}
|
||||
default:
|
||||
out[path] = fmt.Sprintf("%v", t)
|
||||
}
|
||||
}
|
||||
|
||||
// parseValue mirrors the TS reconstruct logic: convert obvious primitives
|
||||
// back to their typed form for nicer exported JSON, leave anything ambiguous
|
||||
// as a string so we don't lose data (e.g. "5.1" channel layouts).
|
||||
func parseValue(s string) interface{} {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
switch s {
|
||||
case "true":
|
||||
return true
|
||||
case "false":
|
||||
return false
|
||||
case "null":
|
||||
return nil
|
||||
}
|
||||
// Only treat as number when the string is a clean integer or decimal,
|
||||
// no leading zeros / trailing junk. Matches the TS regex.
|
||||
if isCleanNumber(s) {
|
||||
if n, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
|
||||
func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }
|
||||
@@ -0,0 +1,258 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// Mode controls how an import reconciles existing CIDs.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
// ModeReplace deletes every existing property of an imported CID and
|
||||
// writes the snapshot's values verbatim. Untouched CIDs are left alone.
|
||||
ModeReplace Mode = "replace"
|
||||
// ModeMerge keeps unmentioned existing properties and only touches the
|
||||
// fields present in the snapshot. Conflict resolution within merge is
|
||||
// controlled by Conflict.
|
||||
ModeMerge Mode = "merge"
|
||||
)
|
||||
|
||||
// Conflict picks a winner for fields present on both sides during merge.
|
||||
type Conflict string
|
||||
|
||||
const (
|
||||
// ConflictMine keeps the existing (target) value when both sides have one.
|
||||
ConflictMine Conflict = "mine"
|
||||
// ConflictSource overwrites the existing value with the snapshot's value.
|
||||
ConflictSource Conflict = "source"
|
||||
)
|
||||
|
||||
// ImportOptions controls how Import applies a snapshot.
|
||||
type ImportOptions struct {
|
||||
Mode Mode
|
||||
Conflict Conflict
|
||||
DryRun bool
|
||||
// PluginOutputDir is the on-disk destination for cache/plugin-output/*
|
||||
// entries. When empty, plugin-output is silently skipped on import.
|
||||
PluginOutputDir string
|
||||
}
|
||||
|
||||
// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
|
||||
type FileResult struct {
|
||||
HashID string `json:"hashId"`
|
||||
Added int `json:"added"` // new property keys written
|
||||
Updated int `json:"updated"` // existing property keys whose value changed
|
||||
Unchanged int `json:"unchanged"` // existing property keys whose value matched
|
||||
Kept int `json:"kept"` // existing property keys preserved by ConflictMine
|
||||
Deleted int `json:"deleted"` // existing property keys removed (replace mode)
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// PluginOutputResult summarizes what happened to cache/plugin-output entries.
|
||||
type PluginOutputResult struct {
|
||||
Written int `json:"written"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
// ImportResult is the response body for the import endpoint.
|
||||
type ImportResult struct {
|
||||
Mode Mode `json:"mode"`
|
||||
Conflict Conflict `json:"conflict,omitempty"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
TotalFiles int `json:"totalFiles"`
|
||||
FilesOK int `json:"filesOk"`
|
||||
FilesFailed int `json:"filesFailed"`
|
||||
Files []FileResult `json:"files,omitempty"`
|
||||
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
|
||||
}
|
||||
|
||||
// Importer applies snapshot ZIPs to a live Redis store.
|
||||
type Importer struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewImporter(stor *storage.Client) *Importer {
|
||||
return &Importer{storage: stor}
|
||||
}
|
||||
|
||||
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
|
||||
// *bytes.Reader after buffering the upload) and applies it to Redis.
|
||||
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
|
||||
if i.storage == nil || !i.storage.IsConnected() {
|
||||
return nil, fmt.Errorf("storage not connected")
|
||||
}
|
||||
if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
|
||||
return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
|
||||
}
|
||||
if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
|
||||
return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
|
||||
}
|
||||
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open zip: %w", err)
|
||||
}
|
||||
|
||||
var manifest Manifest
|
||||
if err := readJSON(zr, ManifestFile, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
if manifest.SchemaVersion > ManifestSchemaVersion {
|
||||
return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
|
||||
manifest.SchemaVersion, ManifestSchemaVersion)
|
||||
}
|
||||
|
||||
result := &ImportResult{
|
||||
Mode: opts.Mode,
|
||||
Conflict: opts.Conflict,
|
||||
DryRun: opts.DryRun,
|
||||
}
|
||||
|
||||
for _, f := range zr.File {
|
||||
if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
|
||||
continue
|
||||
}
|
||||
hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
|
||||
if hashID == "" {
|
||||
continue
|
||||
}
|
||||
fr := i.applyOne(f, hashID, opts)
|
||||
result.TotalFiles++
|
||||
if fr.Error != "" {
|
||||
result.FilesFailed++
|
||||
} else {
|
||||
result.FilesOK++
|
||||
}
|
||||
result.Files = append(result.Files, fr)
|
||||
}
|
||||
|
||||
// Plugin-output extraction. Only applied on real imports (not dry-run).
|
||||
// If the snapshot doesn't contain a cache/plugin-output/ section, this is
|
||||
// a no-op. If the destination is unset, we skip silently — the manifest's
|
||||
// pluginOutput flag tells the caller it was on the source side either way.
|
||||
if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
|
||||
// Existing files are kept under ConflictMine / replace-doesn't-apply,
|
||||
// overwritten under ConflictSource. Replace mode behaves like
|
||||
// "overwrite" for cache files (it's the destructive choice for
|
||||
// metadata; mirror that intent here).
|
||||
overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
|
||||
written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("extract plugin-output: %w", err)
|
||||
}
|
||||
result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
|
||||
log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
|
||||
}
|
||||
|
||||
log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
|
||||
opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
|
||||
fr := FileResult{HashID: hashID}
|
||||
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
fr.Error = fmt.Sprintf("open entry: %v", err)
|
||||
return fr
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
var nested interface{}
|
||||
if err := json.NewDecoder(rc).Decode(&nested); err != nil {
|
||||
fr.Error = fmt.Sprintf("decode json: %v", err)
|
||||
return fr
|
||||
}
|
||||
incoming := Flatten(nested)
|
||||
|
||||
existing, err := i.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
fr.Error = fmt.Sprintf("read existing: %v", err)
|
||||
return fr
|
||||
}
|
||||
if existing == nil {
|
||||
existing = map[string]string{}
|
||||
}
|
||||
|
||||
switch opts.Mode {
|
||||
case ModeReplace:
|
||||
// Diff for reporting.
|
||||
toWrite := map[string]string{}
|
||||
for k, v := range incoming {
|
||||
toWrite[k] = v
|
||||
if cur, ok := existing[k]; ok {
|
||||
if cur == v {
|
||||
fr.Unchanged++
|
||||
} else {
|
||||
fr.Updated++
|
||||
}
|
||||
} else {
|
||||
fr.Added++
|
||||
}
|
||||
}
|
||||
for k := range existing {
|
||||
if _, ok := incoming[k]; !ok {
|
||||
fr.Deleted++
|
||||
}
|
||||
}
|
||||
if !opts.DryRun {
|
||||
if _, err := i.storage.DeleteMetadata(hashID); err != nil {
|
||||
fr.Error = fmt.Sprintf("delete existing: %v", err)
|
||||
return fr
|
||||
}
|
||||
if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
|
||||
fr.Error = fmt.Sprintf("write: %v", err)
|
||||
return fr
|
||||
}
|
||||
}
|
||||
|
||||
case ModeMerge:
|
||||
toWrite := map[string]string{}
|
||||
for k, v := range incoming {
|
||||
cur, present := existing[k]
|
||||
switch {
|
||||
case !present:
|
||||
toWrite[k] = v
|
||||
fr.Added++
|
||||
case cur == v:
|
||||
fr.Unchanged++
|
||||
case opts.Conflict == ConflictSource:
|
||||
toWrite[k] = v
|
||||
fr.Updated++
|
||||
case opts.Conflict == ConflictMine:
|
||||
fr.Kept++
|
||||
}
|
||||
}
|
||||
if !opts.DryRun && len(toWrite) > 0 {
|
||||
if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
|
||||
fr.Error = fmt.Sprintf("merge: %v", err)
|
||||
return fr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fr
|
||||
}
|
||||
|
||||
func readJSON(zr *zip.Reader, name string, out interface{}) error {
|
||||
for _, f := range zr.File {
|
||||
if f.Name != name {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
return json.NewDecoder(rc).Decode(out)
|
||||
}
|
||||
return fmt.Errorf("entry %q not found", name)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Package snapshot implements export/import/wipe of MetaMesh metadata.
|
||||
//
|
||||
// The on-disk format is a ZIP archive containing one nested-JSON file per
|
||||
// known CID (cid_midhash256). Schema is defined by ManifestSchemaVersion;
|
||||
// readers must reject archives whose schemaVersion exceeds what they know.
|
||||
package snapshot
|
||||
|
||||
import "time"
|
||||
|
||||
// ManifestSchemaVersion is the on-disk format version. Bump when the layout
|
||||
// of metadata/{cid}.json or the manifest changes in a non-additive way.
|
||||
const ManifestSchemaVersion = 1
|
||||
|
||||
// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
|
||||
const MetadataDir = "metadata/"
|
||||
|
||||
// ManifestFile is the path inside the ZIP that holds the manifest JSON.
|
||||
const ManifestFile = "manifest.json"
|
||||
|
||||
// IndexFile is the path inside the ZIP that holds the file index (CID list).
|
||||
const IndexFile = "index.json"
|
||||
|
||||
// Manifest is the metadata header stored at the root of every snapshot ZIP.
|
||||
type Manifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
GeneratedAt time.Time `json:"generatedAt"`
|
||||
SourceHost string `json:"sourceHost,omitempty"`
|
||||
MetaCoreVer string `json:"metaCoreVersion,omitempty"`
|
||||
FileCount int `json:"fileCount"`
|
||||
Includes Includes `json:"includes"`
|
||||
}
|
||||
|
||||
// Includes records which optional sections are present in a snapshot.
|
||||
type Includes struct {
|
||||
Metadata bool `json:"metadata"`
|
||||
PluginOutput bool `json:"pluginOutput,omitempty"`
|
||||
}
|
||||
|
||||
// IndexFileBody is the structure of index.json inside the ZIP.
|
||||
type IndexFileBody struct {
|
||||
HashIDs []string `json:"hashIds"`
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// WipeScope picks which classes of data to clear.
|
||||
type WipeScope struct {
|
||||
Metadata bool // file:* keys + file:__index__
|
||||
}
|
||||
|
||||
// WipeResult reports what was cleared.
|
||||
type WipeResult struct {
|
||||
MetadataDeleted int64 `json:"metadataDeleted"`
|
||||
}
|
||||
|
||||
// Wiper clears metadata from Redis.
|
||||
type Wiper struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewWiper(stor *storage.Client) *Wiper {
|
||||
return &Wiper{storage: stor}
|
||||
}
|
||||
|
||||
// Wipe applies the requested scope. Returns the count of files removed.
|
||||
func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
|
||||
if w.storage == nil || !w.storage.IsConnected() {
|
||||
return nil, fmt.Errorf("storage not connected")
|
||||
}
|
||||
res := &WipeResult{}
|
||||
if scope.Metadata {
|
||||
n, err := w.storage.ClearAllMetadata()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clear metadata: %w", err)
|
||||
}
|
||||
res.MetadataDeleted = n
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -537,6 +537,93 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
|
||||
return true, c.client.Set(ctx, key, newValue, 0).Err()
|
||||
}
|
||||
|
||||
// FileTuple holds the (path, size, mtime) record for one file. Used by the
|
||||
// watcher hydrator at startup to avoid re-mid-hashing files whose tuple
|
||||
// matches what's already in Redis.
|
||||
type FileTuple struct {
|
||||
HashID string
|
||||
FilePath string
|
||||
Size int64
|
||||
MtimeNano int64
|
||||
}
|
||||
|
||||
// GetTuplesForAllFiles returns one FileTuple per indexed CID, fetched in
|
||||
// batched MGETs (no SCAN per CID). Skips files missing any of the three
|
||||
// fields. Sub-second on local Redis for ~10K files.
|
||||
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return nil, fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("smembers index: %w", err)
|
||||
}
|
||||
if len(hashIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build the key list: 3 keys per CID, in a fixed order so we can decode
|
||||
// the MGET response by position. fields[i] for hashIDs[j] sits at
|
||||
// keys[j*3+i].
|
||||
const fieldsPerFile = 3
|
||||
fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"}
|
||||
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
|
||||
for _, h := range hashIDs {
|
||||
prefix := c.buildKeyPrefix(h)
|
||||
for _, f := range fields {
|
||||
keys = append(keys, prefix+f)
|
||||
}
|
||||
}
|
||||
|
||||
// Chunked MGET — Redis caps args per command; 1000 is a safe batch.
|
||||
const batch = 1000
|
||||
values := make([]interface{}, 0, len(keys))
|
||||
for start := 0; start < len(keys); start += batch {
|
||||
end := start + batch
|
||||
if end > len(keys) {
|
||||
end = len(keys)
|
||||
}
|
||||
chunk, err := c.client.MGet(ctx, keys[start:end]...).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mget chunk [%d:%d]: %w", start, end, err)
|
||||
}
|
||||
values = append(values, chunk...)
|
||||
}
|
||||
|
||||
out := make([]FileTuple, 0, len(hashIDs))
|
||||
for i, h := range hashIDs {
|
||||
base := i * fieldsPerFile
|
||||
filePath, _ := values[base].(string)
|
||||
sizeStr, _ := values[base+1].(string)
|
||||
mtimeStr, _ := values[base+2].(string)
|
||||
if filePath == "" || sizeStr == "" || mtimeStr == "" {
|
||||
continue
|
||||
}
|
||||
size, err := strconv.ParseInt(sizeStr, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, FileTuple{
|
||||
HashID: h,
|
||||
FilePath: filePath,
|
||||
Size: size,
|
||||
MtimeNano: mtime,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetMemoryInfo returns Redis memory usage information
|
||||
func (c *Client) GetMemoryInfo() (string, error) {
|
||||
c.mu.RLock()
|
||||
@@ -1059,6 +1146,51 @@ func (c *Client) ClearStream(stream string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// StreamEntry represents one entry from a Redis stream.
|
||||
type StreamEntry struct {
|
||||
ID string
|
||||
Values map[string]interface{}
|
||||
}
|
||||
|
||||
// XRevRange reads stream entries from newest to oldest.
|
||||
// start defaults to "+" (newest) and stop defaults to "-" (oldest) when empty.
|
||||
// If count > 0, at most that many entries are returned.
|
||||
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return nil, fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
if start == "" {
|
||||
start = "+"
|
||||
}
|
||||
if stop == "" {
|
||||
stop = "-"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var msgs []redis.XMessage
|
||||
var err error
|
||||
if count > 0 {
|
||||
msgs, err = c.client.XRevRangeN(ctx, stream, start, stop, count).Result()
|
||||
} else {
|
||||
msgs, err = c.client.XRevRange(ctx, stream, start, stop).Result()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("xrevrange failed: %w", err)
|
||||
}
|
||||
|
||||
entries := make([]StreamEntry, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
entries = append(entries, StreamEntry{ID: m.ID, Values: m.Values})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// GetStreamLength returns the length of a Redis stream
|
||||
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
||||
c.mu.RLock()
|
||||
|
||||
@@ -3,6 +3,8 @@ package watcher
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
@@ -108,6 +110,85 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRecentEvents reads recent events from the Redis stream, newest first.
|
||||
// If sinceMS > 0, only events strictly newer than that timestamp are returned.
|
||||
// limit caps the number of returned events (defaults to 100 when <= 0).
|
||||
func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
|
||||
d.mu.RLock()
|
||||
client := d.storageClient
|
||||
d.mu.RUnlock()
|
||||
|
||||
if client == nil || !client.IsConnected() {
|
||||
return []FileEvent{}, nil
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
start := "+"
|
||||
stop := "-"
|
||||
if sinceMS > 0 {
|
||||
// Exclusive lower bound: skip entries at or before sinceMS.
|
||||
stop = fmt.Sprintf("(%d-0", sinceMS)
|
||||
}
|
||||
|
||||
entries, err := client.XRevRange(EventsStream, start, stop, int64(limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
events := make([]FileEvent, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
events = append(events, parseStreamEntry(entry))
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// parseStreamEntry converts a Redis stream entry back into a FileEvent.
|
||||
// All fields are stored as strings by XADD, so numerics need parsing.
|
||||
func parseStreamEntry(entry storage.StreamEntry) FileEvent {
|
||||
event := FileEvent{}
|
||||
|
||||
if v, ok := entry.Values["type"].(string); ok {
|
||||
event.Type = FileEventType(v)
|
||||
}
|
||||
if v, ok := entry.Values["path"].(string); ok {
|
||||
event.Path = v
|
||||
}
|
||||
if v, ok := entry.Values["timestamp"].(string); ok {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
event.Timestamp = n
|
||||
}
|
||||
}
|
||||
if v, ok := entry.Values["size"].(string); ok {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
event.Size = n
|
||||
}
|
||||
}
|
||||
if v, ok := entry.Values["midhash256"].(string); ok {
|
||||
event.MidHash256 = v
|
||||
}
|
||||
if v, ok := entry.Values["oldPath"].(string); ok {
|
||||
event.OldPath = v
|
||||
}
|
||||
if v, ok := entry.Values["watcherId"].(string); ok {
|
||||
event.WatcherID = v
|
||||
}
|
||||
|
||||
// Fall back to the stream ID's millisecond prefix if the timestamp
|
||||
// field is missing or unparseable — every Redis stream ID starts with ms.
|
||||
if event.Timestamp == 0 && entry.ID != "" {
|
||||
if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
|
||||
if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {
|
||||
event.Timestamp = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return event
|
||||
}
|
||||
|
||||
// GetStreamLength returns the current stream length
|
||||
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
||||
d.mu.RLock()
|
||||
|
||||
@@ -2,6 +2,7 @@ package watcher
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
@@ -52,7 +53,14 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
events := h.watcher.GetRecentEvents(sinceMS, limit)
|
||||
// Read events from the Redis `file:events` stream rather than the
|
||||
// in-memory buffer. The buffer is fed by the unused debouncer path,
|
||||
// so it is always empty in practice — the stream is the source of truth.
|
||||
events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)
|
||||
if err != nil {
|
||||
log.Printf("[watcher.handlePoll] failed to read stream: %v", err)
|
||||
events = []FileEvent{}
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, EventsListResponse{
|
||||
Events: events,
|
||||
|
||||
@@ -26,15 +26,13 @@ const (
|
||||
// The size prefix (8-byte big-endian) ensures that files with identical
|
||||
// middle content but different sizes produce different hashes.
|
||||
//
|
||||
// fileSize must be passed in by the caller (typically from a FileInfo it
|
||||
// already has from filepath.Walk or a previous os.Stat). This function
|
||||
// deliberately does NOT call os.Stat itself — re-statting every file during
|
||||
// a scan doubles the syscall count, which matters on slow / network drives.
|
||||
//
|
||||
// Returns: CID v1 string in base32lower format (prefixed with "b")
|
||||
func ComputeMidHash256(filePath string) (string, error) {
|
||||
// Get file info
|
||||
fileInfo, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fileSize := fileInfo.Size()
|
||||
|
||||
func ComputeMidHash256(filePath string, fileSize int64) (string, error) {
|
||||
// Open file
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
|
||||
+114
-8
@@ -4,10 +4,12 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
|
||||
@@ -18,11 +20,39 @@ type Watcher struct {
|
||||
dispatcher *Dispatcher
|
||||
filesPath string
|
||||
stateRegistry *StateRegistry
|
||||
storage *storage.Client
|
||||
|
||||
mu sync.RWMutex
|
||||
eventBuffer []FileEvent
|
||||
}
|
||||
|
||||
// SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
|
||||
// tuple to Redis whenever it computes a midhash. Without this, only the
|
||||
// in-memory state registry knows about the file — Redis would only learn
|
||||
// about it via downstream consumers of the event stream.
|
||||
func (w *Watcher) SetStorageClient(s *storage.Client) {
|
||||
w.storage = s
|
||||
}
|
||||
|
||||
// writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
|
||||
// Caller already has all four values from its existing os.Stat / FileInfo —
|
||||
// no extra IO is performed here beyond the Redis write itself.
|
||||
//
|
||||
// This is the inverse-index that lets a fresh meta-core boot skip
|
||||
// re-mid-hashing files whose (path, size, mtime) match what's in Redis.
|
||||
func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
|
||||
if w.storage == nil || midhash == "" {
|
||||
return
|
||||
}
|
||||
if _, err := w.storage.MergeMetadataFlat(midhash, map[string]string{
|
||||
"filePath": absPath,
|
||||
"sizeByte": strconv.FormatInt(size, 10),
|
||||
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
|
||||
}); err != nil {
|
||||
log.Printf("[Watcher] failed to write tuple for %s: %v", midhash, err)
|
||||
}
|
||||
}
|
||||
|
||||
// NewWatcher creates a new file watcher/scanner
|
||||
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
||||
debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
|
||||
@@ -44,7 +74,7 @@ func (w *Watcher) handleDebouncedEvent(event FileEvent) {
|
||||
// Compute midhash256 for add/change events
|
||||
if event.Type == EventTypeAdd || event.Type == EventTypeChange {
|
||||
fullPath := filepath.Join(w.filesPath, event.Path)
|
||||
if hash, err := ComputeMidHash256(fullPath); err == nil {
|
||||
if hash, err := ComputeMidHash256(fullPath, event.Size); err == nil {
|
||||
event.MidHash256 = hash
|
||||
}
|
||||
}
|
||||
@@ -90,16 +120,20 @@ func (w *Watcher) ScanDirectory(root string) int {
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Emit add event
|
||||
curSize := info.Size()
|
||||
curMtime := info.ModTime().UnixNano()
|
||||
event := FileEvent{
|
||||
Type: EventTypeAdd,
|
||||
Path: relPath,
|
||||
Size: info.Size(),
|
||||
Size: curSize,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute midhash256
|
||||
if hash, err := ComputeMidHash256(path); err == nil {
|
||||
// Compute midhash256 — uses the size we already have from Walk's
|
||||
// FileInfo, so no extra os.Stat is needed.
|
||||
if hash, err := ComputeMidHash256(path, curSize); err == nil {
|
||||
event.MidHash256 = hash
|
||||
w.writeFileTuple(path, curSize, curMtime, hash)
|
||||
}
|
||||
|
||||
// Dispatch directly
|
||||
@@ -160,9 +194,11 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute midhash256
|
||||
if hash, err := ComputeMidHash256(path); err == nil {
|
||||
// Compute midhash256 — size is already known from the walk's
|
||||
// FileInfo, no extra stat needed.
|
||||
if hash, err := ComputeMidHash256(path, currentSize); err == nil {
|
||||
event.MidHash256 = hash
|
||||
w.writeFileTuple(path, currentSize, currentMtime, hash)
|
||||
}
|
||||
|
||||
// Dispatch event
|
||||
@@ -185,9 +221,10 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute midhash256
|
||||
if hash, err := ComputeMidHash256(path); err == nil {
|
||||
// Compute midhash256 — reuse the size we already have.
|
||||
if hash, err := ComputeMidHash256(path, currentSize); err == nil {
|
||||
event.MidHash256 = hash
|
||||
w.writeFileTuple(path, currentSize, currentMtime, hash)
|
||||
}
|
||||
|
||||
// Dispatch event
|
||||
@@ -227,6 +264,75 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
|
||||
return result
|
||||
}
|
||||
|
||||
// WatcherRoot pairs a watcher's id with the absolute path it watches. The
|
||||
// hydrator uses Root to decide which Redis-tracked file belongs to which
|
||||
// in-memory WatcherState.
|
||||
type WatcherRoot struct {
|
||||
ID string
|
||||
Root string
|
||||
}
|
||||
|
||||
// HydrateStateFromStorage repopulates the per-watcher in-memory state from
|
||||
// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
|
||||
// next scan tick will see existing files as "known" and skip
|
||||
// ComputeMidHash256 for any whose size+mtime match — turning a cold-start
|
||||
// rescan from minutes to seconds on a populated library.
|
||||
//
|
||||
// Files whose filePath doesn't fall under any watcher's root are silently
|
||||
// skipped; files missing any tuple field are skipped too (a partial record
|
||||
// can't seed valid state).
|
||||
//
|
||||
// Returns counts so the caller can log / surface them.
|
||||
func (w *Watcher) HydrateStateFromStorage(roots []WatcherRoot) (loaded, skipped int, err error) {
|
||||
if w.storage == nil || !w.storage.IsConnected() {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
tuples, err := w.storage.GetTuplesForAllFiles()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Sort roots by length descending so longest-prefix wins when paths
|
||||
// nest (e.g. /files/watch/foo and /files/watch).
|
||||
sortedRoots := make([]WatcherRoot, len(roots))
|
||||
copy(sortedRoots, roots)
|
||||
for i := 1; i < len(sortedRoots); i++ {
|
||||
for j := i; j > 0 && len(sortedRoots[j].Root) > len(sortedRoots[j-1].Root); j-- {
|
||||
sortedRoots[j], sortedRoots[j-1] = sortedRoots[j-1], sortedRoots[j]
|
||||
}
|
||||
}
|
||||
|
||||
// State keys are relative to w.filesPath (matching ScanDirectoryDiff's
|
||||
// Rel(w.filesPath, path) convention). The watcher root only decides
|
||||
// *which* per-watcher state map this file lives in.
|
||||
for _, t := range tuples {
|
||||
matched := false
|
||||
for _, r := range sortedRoots {
|
||||
if t.FilePath == r.Root || strings.HasPrefix(t.FilePath, r.Root+"/") {
|
||||
rel, err := filepath.Rel(w.filesPath, t.FilePath)
|
||||
if err != nil {
|
||||
rel = t.FilePath
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
ws := w.stateRegistry.GetOrCreate(r.ID)
|
||||
ws.Set(rel, &FileState{
|
||||
Size: t.Size,
|
||||
MtimeNano: t.MtimeNano,
|
||||
MidHash: t.HashID,
|
||||
})
|
||||
loaded++
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
return loaded, skipped, nil
|
||||
}
|
||||
|
||||
// ClearState clears the tracked state for a watcher
|
||||
func (w *Watcher) ClearState(watcherID string) {
|
||||
w.stateRegistry.Clear(watcherID)
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user