change election leader mechanism
This commit is contained in:
Pierre
+12
-5
@@ -3,11 +3,17 @@
|
||||
# Multi-stage build for meta-core as a standalone service
|
||||
#
|
||||
# Includes:
|
||||
# - meta-core Go binary (leader election, Redis, HTTP API, Mount API, Watcher)
|
||||
# - Bash-based leader election script (flock on shared volume)
|
||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||
# - nginx (dashboard, WebDAV, API proxy)
|
||||
# - Redis (leader-managed)
|
||||
# - Redis (managed by supervisord on leader)
|
||||
# - rclone (remote mount daemon)
|
||||
# - supervisord (process management)
|
||||
# - supervisord (process management - runs on leader only)
|
||||
#
|
||||
# Architecture:
|
||||
# - leader-election.sh handles flock, starts supervisord on leader
|
||||
# - Followers remain dormant, retry flock every 5s
|
||||
# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -103,6 +109,7 @@ RUN mkdir -p \
|
||||
COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
COPY docker/nginx.conf /etc/nginx/nginx.conf
|
||||
COPY docker/mount-watcher.sh /app/scripts/mount-watcher.sh
|
||||
COPY docker/leader-election.sh /app/scripts/leader-election.sh
|
||||
COPY docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Copy dashboard static files from builder
|
||||
@@ -112,7 +119,7 @@ COPY --from=dashboard-builder /app/dist /app/dashboard/dist
|
||||
COPY --from=editor-builder /app/dist /app/editor/dist
|
||||
|
||||
# Set permissions
|
||||
RUN chmod +x /app/scripts/mount-watcher.sh /entrypoint.sh
|
||||
RUN chmod +x /app/scripts/mount-watcher.sh /app/scripts/leader-election.sh /entrypoint.sh
|
||||
|
||||
# Create htpasswd for rclone basic auth (admin:admin)
|
||||
RUN htpasswd -bc /etc/nginx/.htpasswd admin admin
|
||||
@@ -137,4 +144,4 @@ HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD curl -sf http://localhost/health || exit 1
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
||||
CMD ["/app/scripts/leader-election.sh"]
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Leader Election Script for meta-core
|
||||
#
|
||||
# Uses flock for leader election. Only the leader runs the full application
|
||||
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
|
||||
# Followers remain dormant, periodically retrying to acquire the lock.
|
||||
#
|
||||
# Environment variables:
|
||||
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
|
||||
# META_CORE_HTTP_PORT - Port for meta-core HTTP API (default: 9000)
|
||||
# ELECTION_RETRY_SECS - Seconds between flock retry attempts (default: 5)
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
META_CORE_PATH="${META_CORE_PATH:-/meta-core}"
|
||||
META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
|
||||
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
|
||||
|
||||
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
|
||||
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
|
||||
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
|
||||
|
||||
# Get the container's IP address
|
||||
# Note: Alpine uses BusyBox hostname which uses -i (lowercase) instead of -I
|
||||
get_local_ip() {
|
||||
# Try hostname -i first (Alpine/BusyBox), then fallback to hostname -I (GNU)
|
||||
local ip
|
||||
ip=$(hostname -i 2>/dev/null | awk '{print $1}')
|
||||
if [ -z "$ip" ] || [ "$ip" = "127.0.0.1" ]; then
|
||||
# Fallback: parse /etc/hosts or use hostname
|
||||
ip=$(getent hosts "$(hostname)" 2>/dev/null | awk '{print $1}' | head -1)
|
||||
fi
|
||||
if [ -z "$ip" ]; then
|
||||
# Last resort: use hostname
|
||||
ip=$(hostname)
|
||||
fi
|
||||
echo "$ip"
|
||||
}
|
||||
|
||||
# Write leader info file (plain text API URL)
|
||||
write_leader_info() {
|
||||
local ip=$(get_local_ip)
|
||||
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
|
||||
|
||||
# Write atomically using temp file + rename
|
||||
local temp_file="${INFO_FILE}.tmp"
|
||||
echo "$api_url" > "$temp_file"
|
||||
mv "$temp_file" "$INFO_FILE"
|
||||
|
||||
echo "[election] Wrote leader info: $api_url"
|
||||
}
|
||||
|
||||
# Clean up leader info file on shutdown
|
||||
cleanup() {
|
||||
echo "[election] Cleaning up..."
|
||||
rm -f "$INFO_FILE"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Set up signal handlers for graceful shutdown
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# Ensure lock directory exists
|
||||
mkdir -p "$(dirname "$LOCK_FILE")"
|
||||
|
||||
echo "[election] Starting leader election..."
|
||||
echo "[election] Lock file: $LOCK_FILE"
|
||||
echo "[election] Info file: $INFO_FILE"
|
||||
|
||||
# Open lock file for flock (file descriptor 200)
|
||||
exec 200>"$LOCK_FILE"
|
||||
|
||||
# Main election loop
|
||||
while true; do
|
||||
# Try non-blocking exclusive flock
|
||||
if flock -n 200; then
|
||||
echo "[election] Acquired flock - transitioning to LEADER"
|
||||
|
||||
# Write leader info before starting services
|
||||
write_leader_info
|
||||
|
||||
# Export role for meta-core Go binary to read
|
||||
export META_CORE_ROLE="leader"
|
||||
|
||||
# exec supervisord - replaces this process
|
||||
# When supervisord exits, the flock is released automatically
|
||||
echo "[election] Starting supervisord..."
|
||||
exec /usr/bin/supervisord -c "$SUPERVISORD_CONF"
|
||||
else
|
||||
echo "[election] Lock held by another process - acting as FOLLOWER"
|
||||
echo "[election] Waiting ${ELECTION_RETRY_SECS}s before retry..."
|
||||
sleep "$ELECTION_RETRY_SECS"
|
||||
fi
|
||||
done
|
||||
@@ -139,6 +139,18 @@ http {
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location /urls {
|
||||
proxy_pass http://meta-core-api/urls;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location /status {
|
||||
proxy_pass http://meta-core-api/status;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# =========================================================================
|
||||
# rclone Web UI (proxied with basic auth)
|
||||
# =========================================================================
|
||||
|
||||
+17
-2
@@ -16,11 +16,26 @@ serverurl=unix:///var/run/supervisor.sock
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
# =============================================================================
|
||||
# meta-core: Leader election, Redis management, HTTP API, Mount API, File Watcher
|
||||
# Redis: Key-value store (managed by supervisord, started before meta-core)
|
||||
# Previously spawned by Go code, now managed by bash-based leader election
|
||||
# =============================================================================
|
||||
[program:redis]
|
||||
command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
|
||||
priority=1
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
startsecs=2
|
||||
|
||||
# =============================================================================
|
||||
# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
|
||||
# =============================================================================
|
||||
[program:meta-core]
|
||||
command=/usr/local/bin/meta-core
|
||||
priority=5
|
||||
priority=10
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
|
||||
Reference in New Issue
Block a user