change election leader mechanism
This commit is contained in:
+17
-46
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
@@ -24,36 +25,23 @@ func main() {
|
|||||||
cfg := config.Load()
|
cfg := config.Load()
|
||||||
log.Printf("[meta-core] Service: %s, HTTP Port: %d", cfg.ServiceName, cfg.HTTPPort)
|
log.Printf("[meta-core] Service: %s, HTTP Port: %d", cfg.ServiceName, cfg.HTTPPort)
|
||||||
|
|
||||||
// Create storage client
|
// Create role provider (reads META_CORE_ROLE from environment, set by leader-election.sh)
|
||||||
|
roleProvider := leader.NewLocalRoleProvider(cfg)
|
||||||
|
log.Printf("[meta-core] Role: %s", roleProvider.Role())
|
||||||
|
|
||||||
|
// Connect to local Redis (supervisord starts Redis before meta-core)
|
||||||
|
redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
|
||||||
|
log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
|
||||||
|
|
||||||
storageClient := storage.NewClient("")
|
storageClient := storage.NewClient("")
|
||||||
|
if err := storageClient.Connect(redisURL); err != nil {
|
||||||
// Create leader election
|
log.Fatalf("[meta-core] Failed to connect to Redis: %v", err)
|
||||||
election := leader.NewElection(cfg)
|
|
||||||
|
|
||||||
// Wire up storage connector to election
|
|
||||||
election.SetStorageConnector(&storageConnectorAdapter{client: storageClient})
|
|
||||||
|
|
||||||
// Set up leader/follower callbacks
|
|
||||||
election.OnBecomeLeader(func() {
|
|
||||||
log.Println("[meta-core] Became LEADER - Redis is running")
|
|
||||||
})
|
|
||||||
|
|
||||||
election.OnBecomeFollower(func(info *leader.LeaderLockInfo) {
|
|
||||||
log.Printf("[meta-core] Became FOLLOWER - Leader at %s (Redis: %s)", info.BaseUrl, info.RedisUrl)
|
|
||||||
})
|
|
||||||
|
|
||||||
election.OnLeaderLost(func() {
|
|
||||||
log.Println("[meta-core] Lost leadership")
|
|
||||||
})
|
|
||||||
|
|
||||||
// Start leader election
|
|
||||||
if err := election.Start(); err != nil {
|
|
||||||
log.Fatalf("[meta-core] Failed to start leader election: %v", err)
|
|
||||||
}
|
}
|
||||||
|
log.Println("[meta-core] Connected to Redis")
|
||||||
|
|
||||||
// Create and start service discovery
|
// Create and start service discovery
|
||||||
disc := discovery.NewService(cfg)
|
disc := discovery.NewService(cfg)
|
||||||
disc.SetRoleProvider(&roleProviderAdapter{election: election})
|
disc.SetRoleProvider(&roleProviderAdapter{provider: roleProvider})
|
||||||
if err := disc.Start(); err != nil {
|
if err := disc.Start(); err != nil {
|
||||||
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
|
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
|
||||||
}
|
}
|
||||||
@@ -65,7 +53,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create and start API server
|
// Create and start API server
|
||||||
apiServer := api.NewServer(cfg, election, disc, cleaner, storageClient)
|
apiServer := api.NewServer(cfg, roleProvider, disc, cleaner, storageClient)
|
||||||
if err := apiServer.Start(); err != nil {
|
if err := apiServer.Start(); err != nil {
|
||||||
log.Fatalf("[meta-core] Failed to start API server: %v", err)
|
log.Fatalf("[meta-core] Failed to start API server: %v", err)
|
||||||
}
|
}
|
||||||
@@ -90,10 +78,6 @@ func main() {
|
|||||||
log.Printf("[meta-core] Error stopping service discovery: %v", err)
|
log.Printf("[meta-core] Error stopping service discovery: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := election.Stop(); err != nil {
|
|
||||||
log.Printf("[meta-core] Error stopping leader election: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := storageClient.Close(); err != nil {
|
if err := storageClient.Close(); err != nil {
|
||||||
log.Printf("[meta-core] Error closing storage client: %v", err)
|
log.Printf("[meta-core] Error closing storage client: %v", err)
|
||||||
}
|
}
|
||||||
@@ -108,24 +92,11 @@ func waitForShutdown() {
|
|||||||
<-sigChan
|
<-sigChan
|
||||||
}
|
}
|
||||||
|
|
||||||
// storageConnectorAdapter adapts storage.Client to leader.StorageConnector
|
// roleProviderAdapter adapts leader.RoleProvider to discovery.RoleProvider
|
||||||
type storageConnectorAdapter struct {
|
|
||||||
client *storage.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *storageConnectorAdapter) Connect(url string) error {
|
|
||||||
return a.client.Connect(url)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *storageConnectorAdapter) Close() error {
|
|
||||||
return a.client.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// roleProviderAdapter adapts leader.Election to discovery.RoleProvider
|
|
||||||
type roleProviderAdapter struct {
|
type roleProviderAdapter struct {
|
||||||
election *leader.Election
|
provider leader.RoleProvider
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *roleProviderAdapter) Role() string {
|
func (a *roleProviderAdapter) Role() string {
|
||||||
return string(a.election.Role())
|
return string(a.provider.Role())
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-5
@@ -3,11 +3,17 @@
|
|||||||
# Multi-stage build for meta-core as a standalone service
|
# Multi-stage build for meta-core as a standalone service
|
||||||
#
|
#
|
||||||
# Includes:
|
# Includes:
|
||||||
# - meta-core Go binary (leader election, Redis, HTTP API, Mount API, Watcher)
|
# - Bash-based leader election script (flock on shared volume)
|
||||||
|
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||||
# - nginx (dashboard, WebDAV, API proxy)
|
# - nginx (dashboard, WebDAV, API proxy)
|
||||||
# - Redis (leader-managed)
|
# - Redis (managed by supervisord on leader)
|
||||||
# - rclone (remote mount daemon)
|
# - rclone (remote mount daemon)
|
||||||
# - supervisord (process management)
|
# - supervisord (process management - runs on leader only)
|
||||||
|
#
|
||||||
|
# Architecture:
|
||||||
|
# - leader-election.sh handles flock, starts supervisord on leader
|
||||||
|
# - Followers remain dormant, retry flock every 5s
|
||||||
|
# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
@@ -103,6 +109,7 @@ RUN mkdir -p \
|
|||||||
COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||||
COPY docker/nginx.conf /etc/nginx/nginx.conf
|
COPY docker/nginx.conf /etc/nginx/nginx.conf
|
||||||
COPY docker/mount-watcher.sh /app/scripts/mount-watcher.sh
|
COPY docker/mount-watcher.sh /app/scripts/mount-watcher.sh
|
||||||
|
COPY docker/leader-election.sh /app/scripts/leader-election.sh
|
||||||
COPY docker/entrypoint.sh /entrypoint.sh
|
COPY docker/entrypoint.sh /entrypoint.sh
|
||||||
|
|
||||||
# Copy dashboard static files from builder
|
# Copy dashboard static files from builder
|
||||||
@@ -112,7 +119,7 @@ COPY --from=dashboard-builder /app/dist /app/dashboard/dist
|
|||||||
COPY --from=editor-builder /app/dist /app/editor/dist
|
COPY --from=editor-builder /app/dist /app/editor/dist
|
||||||
|
|
||||||
# Set permissions
|
# Set permissions
|
||||||
RUN chmod +x /app/scripts/mount-watcher.sh /entrypoint.sh
|
RUN chmod +x /app/scripts/mount-watcher.sh /app/scripts/leader-election.sh /entrypoint.sh
|
||||||
|
|
||||||
# Create htpasswd for rclone basic auth (admin:admin)
|
# Create htpasswd for rclone basic auth (admin:admin)
|
||||||
RUN htpasswd -bc /etc/nginx/.htpasswd admin admin
|
RUN htpasswd -bc /etc/nginx/.htpasswd admin admin
|
||||||
@@ -137,4 +144,4 @@ HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
|
|||||||
CMD curl -sf http://localhost/health || exit 1
|
CMD curl -sf http://localhost/health || exit 1
|
||||||
|
|
||||||
ENTRYPOINT ["/entrypoint.sh"]
|
ENTRYPOINT ["/entrypoint.sh"]
|
||||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
CMD ["/app/scripts/leader-election.sh"]
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# =============================================================================
|
||||||
|
# Leader Election Script for meta-core
|
||||||
|
#
|
||||||
|
# Uses flock for leader election. Only the leader runs the full application
|
||||||
|
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
|
||||||
|
# Followers remain dormant, periodically retrying to acquire the lock.
|
||||||
|
#
|
||||||
|
# Environment variables:
|
||||||
|
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
|
||||||
|
# META_CORE_HTTP_PORT - Port for meta-core HTTP API (default: 9000)
|
||||||
|
# ELECTION_RETRY_SECS - Seconds between flock retry attempts (default: 5)
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
META_CORE_PATH="${META_CORE_PATH:-/meta-core}"
|
||||||
|
META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
|
||||||
|
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
|
||||||
|
|
||||||
|
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
|
||||||
|
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
|
||||||
|
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
|
||||||
|
|
||||||
|
# Get the container's IP address
|
||||||
|
# Note: Alpine uses BusyBox hostname which uses -i (lowercase) instead of -I
|
||||||
|
get_local_ip() {
|
||||||
|
# Try hostname -i first (Alpine/BusyBox), then fallback to hostname -I (GNU)
|
||||||
|
local ip
|
||||||
|
ip=$(hostname -i 2>/dev/null | awk '{print $1}')
|
||||||
|
if [ -z "$ip" ] || [ "$ip" = "127.0.0.1" ]; then
|
||||||
|
# Fallback: parse /etc/hosts or use hostname
|
||||||
|
ip=$(getent hosts "$(hostname)" 2>/dev/null | awk '{print $1}' | head -1)
|
||||||
|
fi
|
||||||
|
if [ -z "$ip" ]; then
|
||||||
|
# Last resort: use hostname
|
||||||
|
ip=$(hostname)
|
||||||
|
fi
|
||||||
|
echo "$ip"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write leader info file (plain text API URL)
|
||||||
|
write_leader_info() {
|
||||||
|
local ip=$(get_local_ip)
|
||||||
|
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
|
||||||
|
|
||||||
|
# Write atomically using temp file + rename
|
||||||
|
local temp_file="${INFO_FILE}.tmp"
|
||||||
|
echo "$api_url" > "$temp_file"
|
||||||
|
mv "$temp_file" "$INFO_FILE"
|
||||||
|
|
||||||
|
echo "[election] Wrote leader info: $api_url"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Clean up leader info file on shutdown
|
||||||
|
cleanup() {
|
||||||
|
echo "[election] Cleaning up..."
|
||||||
|
rm -f "$INFO_FILE"
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Set up signal handlers for graceful shutdown
|
||||||
|
trap cleanup SIGTERM SIGINT
|
||||||
|
|
||||||
|
# Ensure lock directory exists
|
||||||
|
mkdir -p "$(dirname "$LOCK_FILE")"
|
||||||
|
|
||||||
|
echo "[election] Starting leader election..."
|
||||||
|
echo "[election] Lock file: $LOCK_FILE"
|
||||||
|
echo "[election] Info file: $INFO_FILE"
|
||||||
|
|
||||||
|
# Open lock file for flock (file descriptor 200)
|
||||||
|
exec 200>"$LOCK_FILE"
|
||||||
|
|
||||||
|
# Main election loop
|
||||||
|
while true; do
|
||||||
|
# Try non-blocking exclusive flock
|
||||||
|
if flock -n 200; then
|
||||||
|
echo "[election] Acquired flock - transitioning to LEADER"
|
||||||
|
|
||||||
|
# Write leader info before starting services
|
||||||
|
write_leader_info
|
||||||
|
|
||||||
|
# Export role for meta-core Go binary to read
|
||||||
|
export META_CORE_ROLE="leader"
|
||||||
|
|
||||||
|
# exec supervisord - replaces this process
|
||||||
|
# When supervisord exits, the flock is released automatically
|
||||||
|
echo "[election] Starting supervisord..."
|
||||||
|
exec /usr/bin/supervisord -c "$SUPERVISORD_CONF"
|
||||||
|
else
|
||||||
|
echo "[election] Lock held by another process - acting as FOLLOWER"
|
||||||
|
echo "[election] Waiting ${ELECTION_RETRY_SECS}s before retry..."
|
||||||
|
sleep "$ELECTION_RETRY_SECS"
|
||||||
|
fi
|
||||||
|
done
|
||||||
@@ -139,6 +139,18 @@ http {
|
|||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location /urls {
|
||||||
|
proxy_pass http://meta-core-api/urls;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /status {
|
||||||
|
proxy_pass http://meta-core-api/status;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# rclone Web UI (proxied with basic auth)
|
# rclone Web UI (proxied with basic auth)
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|||||||
+17
-2
@@ -16,11 +16,26 @@ serverurl=unix:///var/run/supervisor.sock
|
|||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# meta-core: Leader election, Redis management, HTTP API, Mount API, File Watcher
|
# Redis: Key-value store (managed by supervisord, started before meta-core)
|
||||||
|
# Previously spawned by Go code, now managed by bash-based leader election
|
||||||
|
# =============================================================================
|
||||||
|
[program:redis]
|
||||||
|
command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
|
||||||
|
priority=1
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
stderr_logfile=/dev/stderr
|
||||||
|
stderr_logfile_maxbytes=0
|
||||||
|
startsecs=2
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
[program:meta-core]
|
[program:meta-core]
|
||||||
command=/usr/local/bin/meta-core
|
command=/usr/local/bin/meta-core
|
||||||
priority=5
|
priority=10
|
||||||
autostart=true
|
autostart=true
|
||||||
autorestart=true
|
autorestart=true
|
||||||
stdout_logfile=/dev/stdout
|
stdout_logfile=/dev/stdout
|
||||||
|
|||||||
@@ -77,10 +77,10 @@ var startTime = time.Now()
|
|||||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||||
response := HealthResponse{
|
response := HealthResponse{
|
||||||
Status: "ok",
|
Status: "ok",
|
||||||
Role: string(s.election.Role()),
|
Role: string(s.roleProvider.Role()),
|
||||||
Redis: s.storage.Health(),
|
Redis: s.storage.Health(),
|
||||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||||
Leader: s.election.LeaderInfo(),
|
Leader: s.roleProvider.LeaderInfo(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if !response.Redis {
|
if !response.Redis {
|
||||||
@@ -100,13 +100,13 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
response := StatusResponse{
|
response := StatusResponse{
|
||||||
Status: "ok",
|
Status: "ok",
|
||||||
Role: string(s.election.Role()),
|
Role: string(s.roleProvider.Role()),
|
||||||
Redis: s.storage.Health(),
|
Redis: s.storage.Health(),
|
||||||
ServiceName: s.config.ServiceName,
|
ServiceName: s.config.ServiceName,
|
||||||
Version: s.config.ServiceVersion,
|
Version: s.config.ServiceVersion,
|
||||||
Uptime: int64(time.Since(startTime).Seconds()),
|
Uptime: int64(time.Since(startTime).Seconds()),
|
||||||
FileCount: fileCount,
|
FileCount: fileCount,
|
||||||
Leader: s.election.LeaderInfo(),
|
Leader: s.roleProvider.LeaderInfo(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if !response.Redis {
|
if !response.Redis {
|
||||||
@@ -118,7 +118,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// handleLeader handles GET /leader
|
// handleLeader handles GET /leader
|
||||||
func (s *Server) handleLeader(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleLeader(w http.ResponseWriter, r *http.Request) {
|
||||||
info := s.election.LeaderInfo()
|
info := s.roleProvider.LeaderInfo()
|
||||||
if info == nil {
|
if info == nil {
|
||||||
writeError(w, http.StatusServiceUnavailable, "no leader available")
|
writeError(w, http.StatusServiceUnavailable, "no leader available")
|
||||||
return
|
return
|
||||||
@@ -130,7 +130,7 @@ func (s *Server) handleLeader(w http.ResponseWriter, r *http.Request) {
|
|||||||
// handleRole handles GET /role
|
// handleRole handles GET /role
|
||||||
func (s *Server) handleRole(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleRole(w http.ResponseWriter, r *http.Request) {
|
||||||
writeJSON(w, http.StatusOK, map[string]string{
|
writeJSON(w, http.StatusOK, map[string]string{
|
||||||
"role": string(s.election.Role()),
|
"role": string(s.roleProvider.Role()),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,7 +147,7 @@ type URLsResponse struct {
|
|||||||
// handleURLs handles GET /urls
|
// handleURLs handles GET /urls
|
||||||
// Returns URLs for the current leader (or this instance if leader)
|
// Returns URLs for the current leader (or this instance if leader)
|
||||||
func (s *Server) handleURLs(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleURLs(w http.ResponseWriter, r *http.Request) {
|
||||||
info := s.election.LeaderInfo()
|
info := s.roleProvider.LeaderInfo()
|
||||||
if info == nil {
|
if info == nil {
|
||||||
writeError(w, http.StatusServiceUnavailable, "no leader available")
|
writeError(w, http.StatusServiceUnavailable, "no leader available")
|
||||||
return
|
return
|
||||||
@@ -159,7 +159,7 @@ func (s *Server) handleURLs(w http.ResponseWriter, r *http.Request) {
|
|||||||
ApiUrl: info.ApiUrl,
|
ApiUrl: info.ApiUrl,
|
||||||
RedisUrl: info.RedisUrl,
|
RedisUrl: info.RedisUrl,
|
||||||
WebdavUrl: info.WebdavUrl,
|
WebdavUrl: info.WebdavUrl,
|
||||||
IsLeader: s.election.IsLeader(),
|
IsLeader: s.roleProvider.IsLeader(),
|
||||||
}
|
}
|
||||||
|
|
||||||
writeJSON(w, http.StatusOK, response)
|
writeJSON(w, http.StatusOK, response)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
// Server is the HTTP API server for meta-core
|
// Server is the HTTP API server for meta-core
|
||||||
type Server struct {
|
type Server struct {
|
||||||
config *config.Config
|
config *config.Config
|
||||||
election *leader.Election
|
roleProvider leader.RoleProvider
|
||||||
discovery *discovery.Service
|
discovery *discovery.Service
|
||||||
cleaner *discovery.Cleaner
|
cleaner *discovery.Cleaner
|
||||||
storage *storage.Client
|
storage *storage.Client
|
||||||
@@ -48,15 +48,15 @@ type Server struct {
|
|||||||
// NewServer creates a new API server
|
// NewServer creates a new API server
|
||||||
func NewServer(
|
func NewServer(
|
||||||
cfg *config.Config,
|
cfg *config.Config,
|
||||||
election *leader.Election,
|
roleProvider leader.RoleProvider,
|
||||||
disc *discovery.Service,
|
disc *discovery.Service,
|
||||||
cleaner *discovery.Cleaner,
|
cleaner *discovery.Cleaner,
|
||||||
stor *storage.Client,
|
stor *storage.Client,
|
||||||
) *Server {
|
) *Server {
|
||||||
s := &Server{
|
s := &Server{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
election: election,
|
roleProvider: roleProvider,
|
||||||
discovery: disc,
|
discovery: disc,
|
||||||
cleaner: cleaner,
|
cleaner: cleaner,
|
||||||
storage: stor,
|
storage: stor,
|
||||||
router: mux.NewRouter(),
|
router: mux.NewRouter(),
|
||||||
|
|||||||
+49
-413
@@ -1,18 +1,9 @@
|
|||||||
package leader
|
package leader
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/metazla/meta-core/internal/config"
|
"github.com/metazla/meta-core/internal/config"
|
||||||
@@ -38,449 +29,94 @@ const (
|
|||||||
RoleFollower Role = "follower"
|
RoleFollower Role = "follower"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Election handles leader election using flock on shared filesystem
|
// RoleProvider provides role and leader info
|
||||||
type Election struct {
|
// This interface is used by the API server to get role information
|
||||||
config *config.Config
|
// without needing the full election logic (which is now in bash)
|
||||||
|
type RoleProvider interface {
|
||||||
|
Role() Role
|
||||||
|
LeaderInfo() *LeaderLockInfo
|
||||||
|
IsLeader() bool
|
||||||
|
}
|
||||||
|
|
||||||
lockFile *os.File
|
// LocalRoleProvider provides role info for this instance
|
||||||
|
// It reads the role from environment and builds leader info locally
|
||||||
|
// This is used when running as the leader (started by leader-election.sh)
|
||||||
|
type LocalRoleProvider struct {
|
||||||
|
config *config.Config
|
||||||
role Role
|
role Role
|
||||||
leaderInfo *LeaderLockInfo
|
leaderInfo *LeaderLockInfo
|
||||||
mu sync.RWMutex
|
|
||||||
|
|
||||||
redisManager *RedisManager
|
|
||||||
storage StorageConnector
|
|
||||||
|
|
||||||
// Callbacks
|
|
||||||
onBecomeLeader func()
|
|
||||||
onBecomeFollower func(info *LeaderLockInfo)
|
|
||||||
onLeaderLost func()
|
|
||||||
|
|
||||||
// Lifecycle
|
|
||||||
stopChan chan struct{}
|
|
||||||
isShuttingDown bool
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// StorageConnector interface for connecting to Redis
|
// NewLocalRoleProvider creates a RoleProvider for the local instance
|
||||||
type StorageConnector interface {
|
// It reads META_CORE_ROLE from environment (set by leader-election.sh)
|
||||||
Connect(url string) error
|
func NewLocalRoleProvider(cfg *config.Config) *LocalRoleProvider {
|
||||||
Close() error
|
roleStr := os.Getenv("META_CORE_ROLE")
|
||||||
}
|
var role Role
|
||||||
|
switch roleStr {
|
||||||
// NewElection creates a new leader election instance
|
case "leader":
|
||||||
func NewElection(cfg *config.Config) *Election {
|
role = RoleLeader
|
||||||
return &Election{
|
case "follower":
|
||||||
config: cfg,
|
role = RoleFollower
|
||||||
role: RoleUnknown,
|
default:
|
||||||
redisManager: NewRedisManager(cfg),
|
// Default to leader if not set (for backward compatibility)
|
||||||
stopChan: make(chan struct{}),
|
role = RoleLeader
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetStorageConnector sets the storage connector for Redis connections
|
|
||||||
func (e *Election) SetStorageConnector(s StorageConnector) {
|
|
||||||
e.storage = s
|
|
||||||
}
|
|
||||||
|
|
||||||
// OnBecomeLeader sets callback for when this instance becomes leader
|
|
||||||
func (e *Election) OnBecomeLeader(fn func()) {
|
|
||||||
e.onBecomeLeader = fn
|
|
||||||
}
|
|
||||||
|
|
||||||
// OnBecomeFollower sets callback for when this instance becomes follower
|
|
||||||
func (e *Election) OnBecomeFollower(fn func(info *LeaderLockInfo)) {
|
|
||||||
e.onBecomeFollower = fn
|
|
||||||
}
|
|
||||||
|
|
||||||
// OnLeaderLost sets callback for when leadership is lost
|
|
||||||
func (e *Election) OnLeaderLost(fn func()) {
|
|
||||||
e.onLeaderLost = fn
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start begins the leader election process
|
|
||||||
func (e *Election) Start() error {
|
|
||||||
log.Println("[Election] Starting leader election...")
|
|
||||||
|
|
||||||
// Ensure lock directory exists
|
|
||||||
lockDir := filepath.Dir(e.config.LockFilePath())
|
|
||||||
if err := os.MkdirAll(lockDir, 0755); err != nil {
|
|
||||||
return fmt.Errorf("failed to create lock directory: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to acquire the lock
|
provider := &LocalRoleProvider{
|
||||||
acquired, err := e.tryAcquireLock()
|
config: cfg,
|
||||||
if err != nil {
|
role: role,
|
||||||
return fmt.Errorf("failed to acquire lock: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if acquired {
|
// Build leader info (always, since we're the leader when running)
|
||||||
if err := e.transitionToLeader(); err != nil {
|
provider.leaderInfo = provider.buildLeaderInfo()
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := e.transitionToFollower(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start health check loop
|
return provider
|
||||||
e.wg.Add(1)
|
|
||||||
go e.healthCheckLoop()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop gracefully shuts down the election
|
// Role returns the current role
|
||||||
func (e *Election) Stop() error {
|
func (p *LocalRoleProvider) Role() Role {
|
||||||
log.Println("[Election] Stopping leader election...")
|
return p.role
|
||||||
e.isShuttingDown = true
|
|
||||||
close(e.stopChan)
|
|
||||||
|
|
||||||
e.wg.Wait()
|
|
||||||
|
|
||||||
// Stop Redis if we were leader
|
|
||||||
if e.Role() == RoleLeader {
|
|
||||||
if err := e.redisManager.Stop(); err != nil {
|
|
||||||
log.Printf("[Election] Error stopping Redis: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release the lock
|
|
||||||
e.releaseLock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Role returns the current role (thread-safe)
|
// LeaderInfo returns the leader info
|
||||||
func (e *Election) Role() Role {
|
func (p *LocalRoleProvider) LeaderInfo() *LeaderLockInfo {
|
||||||
e.mu.RLock()
|
if p.leaderInfo == nil {
|
||||||
defer e.mu.RUnlock()
|
|
||||||
return e.role
|
|
||||||
}
|
|
||||||
|
|
||||||
// LeaderInfo returns the current leader info (thread-safe)
|
|
||||||
func (e *Election) LeaderInfo() *LeaderLockInfo {
|
|
||||||
e.mu.RLock()
|
|
||||||
defer e.mu.RUnlock()
|
|
||||||
if e.leaderInfo == nil {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Return a copy
|
// Return a copy
|
||||||
info := *e.leaderInfo
|
info := *p.leaderInfo
|
||||||
|
// Update timestamp on each call
|
||||||
|
info.Timestamp = time.Now().UnixMilli()
|
||||||
return &info
|
return &info
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsLeader returns true if this instance is the leader
|
// IsLeader returns true if this instance is the leader
|
||||||
func (e *Election) IsLeader() bool {
|
func (p *LocalRoleProvider) IsLeader() bool {
|
||||||
return e.Role() == RoleLeader
|
return p.role == RoleLeader
|
||||||
}
|
|
||||||
|
|
||||||
// tryAcquireLock attempts to acquire exclusive flock on the lock file
|
|
||||||
func (e *Election) tryAcquireLock() (bool, error) {
|
|
||||||
lockPath := e.config.LockFilePath()
|
|
||||||
|
|
||||||
// Open or create the lock file
|
|
||||||
f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0666)
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("failed to open lock file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try non-blocking exclusive flock
|
|
||||||
err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
|
||||||
if err != nil {
|
|
||||||
f.Close()
|
|
||||||
if err == syscall.EWOULDBLOCK {
|
|
||||||
log.Println("[Election] Lock is held by another process")
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("flock failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
e.lockFile = f
|
|
||||||
log.Printf("[Election] Acquired flock on %s", lockPath)
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// releaseLock releases the flock
|
|
||||||
func (e *Election) releaseLock() {
|
|
||||||
if e.lockFile != nil {
|
|
||||||
syscall.Flock(int(e.lockFile.Fd()), syscall.LOCK_UN)
|
|
||||||
e.lockFile.Close()
|
|
||||||
e.lockFile = nil
|
|
||||||
log.Println("[Election] Released flock")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// transitionToLeader handles becoming the leader
|
|
||||||
func (e *Election) transitionToLeader() error {
|
|
||||||
log.Println("[Election] Transitioning to LEADER role")
|
|
||||||
|
|
||||||
e.mu.Lock()
|
|
||||||
e.role = RoleLeader
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
// Start Redis
|
|
||||||
if err := e.redisManager.Start(); err != nil {
|
|
||||||
return fmt.Errorf("failed to start Redis: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for Redis to be ready
|
|
||||||
if err := e.redisManager.WaitForReady(30 * time.Second); err != nil {
|
|
||||||
return fmt.Errorf("Redis not ready: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build and write leader info
|
|
||||||
info := e.buildLeaderInfo()
|
|
||||||
e.mu.Lock()
|
|
||||||
e.leaderInfo = info
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
if err := e.writeLeaderInfo(info); err != nil {
|
|
||||||
return fmt.Errorf("failed to write leader info: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connect storage to local Redis
|
|
||||||
if e.storage != nil {
|
|
||||||
if err := e.storage.Connect(info.RedisUrl); err != nil {
|
|
||||||
log.Printf("[Election] Warning: failed to connect storage: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Notify callback
|
|
||||||
if e.onBecomeLeader != nil {
|
|
||||||
e.onBecomeLeader()
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("[Election] Now acting as LEADER")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// transitionToFollower handles becoming a follower
|
|
||||||
func (e *Election) transitionToFollower() error {
|
|
||||||
log.Println("[Election] Transitioning to FOLLOWER role")
|
|
||||||
|
|
||||||
e.mu.Lock()
|
|
||||||
e.role = RoleFollower
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
// Read leader info
|
|
||||||
info, err := e.readLeaderInfo()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to read leader info: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.Lock()
|
|
||||||
e.leaderInfo = info
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
if info != nil {
|
|
||||||
log.Printf("[Election] Leader is at %s (Redis: %s)", info.ApiUrl, info.RedisUrl)
|
|
||||||
|
|
||||||
// Connect storage to leader's Redis
|
|
||||||
if e.storage != nil {
|
|
||||||
if err := e.storage.Connect(info.RedisUrl); err != nil {
|
|
||||||
log.Printf("[Election] Warning: failed to connect to leader: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Notify callback
|
|
||||||
if e.onBecomeFollower != nil {
|
|
||||||
e.onBecomeFollower(info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("[Election] Now acting as FOLLOWER")
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildLeaderInfo creates leader info for this instance
|
// buildLeaderInfo creates leader info for this instance
|
||||||
func (e *Election) buildLeaderInfo() *LeaderLockInfo {
|
func (p *LocalRoleProvider) buildLeaderInfo() *LeaderLockInfo {
|
||||||
ip := getLocalIP()
|
ip := getLocalIP()
|
||||||
hostname, _ := os.Hostname()
|
hostname, _ := os.Hostname()
|
||||||
|
|
||||||
// Use BaseURL if configured, otherwise construct from IP
|
// Use BaseURL if configured, otherwise construct from IP
|
||||||
baseUrl := e.config.BaseURL
|
baseUrl := p.config.BaseURL
|
||||||
if baseUrl == "" {
|
if baseUrl == "" {
|
||||||
baseUrl = fmt.Sprintf("http://%s:%d", ip, e.config.APIPort)
|
baseUrl = fmt.Sprintf("http://%s:%d", ip, p.config.APIPort)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &LeaderLockInfo{
|
return &LeaderLockInfo{
|
||||||
Hostname: hostname,
|
Hostname: hostname,
|
||||||
BaseUrl: baseUrl,
|
BaseUrl: baseUrl,
|
||||||
ApiUrl: fmt.Sprintf("http://%s:%d", ip, e.config.HTTPPort),
|
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
|
||||||
RedisUrl: fmt.Sprintf("redis://%s:%d", ip, e.config.RedisPort),
|
RedisUrl: fmt.Sprintf("redis://%s:%d", ip, p.config.RedisPort),
|
||||||
WebdavUrl: fmt.Sprintf("http://%s:%d/webdav", ip, e.config.APIPort),
|
WebdavUrl: fmt.Sprintf("http://%s:%d/webdav", ip, p.config.APIPort),
|
||||||
Timestamp: time.Now().UnixMilli(),
|
Timestamp: time.Now().UnixMilli(),
|
||||||
PID: os.Getpid(),
|
PID: os.Getpid(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeLeaderInfo atomically writes leader API URL to file (plain text)
|
|
||||||
// Only writes if the content has changed to avoid triggering file watchers unnecessarily
|
|
||||||
func (e *Election) writeLeaderInfo(info *LeaderLockInfo) error {
|
|
||||||
infoPath := e.config.InfoFilePath()
|
|
||||||
|
|
||||||
// Write just the API URL as plain text
|
|
||||||
newData := []byte(info.ApiUrl)
|
|
||||||
|
|
||||||
// Check if file already has the same content
|
|
||||||
existingData, err := os.ReadFile(infoPath)
|
|
||||||
if err == nil && string(existingData) == string(newData) {
|
|
||||||
// Content unchanged, skip write to avoid triggering file watchers
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Content changed or file doesn't exist, write atomically
|
|
||||||
tempPath := infoPath + ".tmp"
|
|
||||||
if err := os.WriteFile(tempPath, newData, 0644); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.Rename(tempPath, infoPath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("[Election] Wrote leader API URL to %s: %s", infoPath, info.ApiUrl)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readLeaderInfo reads leader API URL from file and fetches full info via /urls API
|
|
||||||
func (e *Election) readLeaderInfo() (*LeaderLockInfo, error) {
|
|
||||||
infoPath := e.config.InfoFilePath()
|
|
||||||
|
|
||||||
data, err := os.ReadFile(infoPath)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// File contains plain text API URL
|
|
||||||
apiUrl := strings.TrimSpace(string(data))
|
|
||||||
if apiUrl == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fetch full info from /urls API
|
|
||||||
return e.fetchLeaderInfoFromAPI(apiUrl)
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchLeaderInfoFromAPI calls the /urls API to get full leader info
|
|
||||||
func (e *Election) fetchLeaderInfoFromAPI(apiUrl string) (*LeaderLockInfo, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", apiUrl+"/urls", nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Accept", "application/json")
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to call /urls API: %w", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("/urls API returned status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse URLsResponse format
|
|
||||||
var urlsResp struct {
|
|
||||||
Hostname string `json:"hostname"`
|
|
||||||
BaseUrl string `json:"baseUrl"`
|
|
||||||
ApiUrl string `json:"apiUrl"`
|
|
||||||
RedisUrl string `json:"redisUrl"`
|
|
||||||
WebdavUrl string `json:"webdavUrl"`
|
|
||||||
IsLeader bool `json:"isLeader"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &urlsResp); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to parse /urls response: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert to LeaderLockInfo
|
|
||||||
return &LeaderLockInfo{
|
|
||||||
Hostname: urlsResp.Hostname,
|
|
||||||
BaseUrl: urlsResp.BaseUrl,
|
|
||||||
ApiUrl: urlsResp.ApiUrl,
|
|
||||||
RedisUrl: urlsResp.RedisUrl,
|
|
||||||
WebdavUrl: urlsResp.WebdavUrl,
|
|
||||||
Timestamp: time.Now().UnixMilli(),
|
|
||||||
PID: 0, // Unknown for remote leader
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// healthCheckLoop periodically checks health and updates timestamps
|
|
||||||
func (e *Election) healthCheckLoop() {
|
|
||||||
defer e.wg.Done()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(time.Duration(e.config.HealthCheckIntervalMS) * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-e.stopChan:
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
e.performHealthCheck()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// performHealthCheck checks health based on current role
|
|
||||||
func (e *Election) performHealthCheck() {
|
|
||||||
switch e.Role() {
|
|
||||||
case RoleLeader:
|
|
||||||
// Update timestamp in leader info
|
|
||||||
e.updateLeaderTimestamp()
|
|
||||||
|
|
||||||
// Check if Redis is still running
|
|
||||||
if !e.redisManager.IsRunning() {
|
|
||||||
log.Println("[Election] Redis not running, attempting restart...")
|
|
||||||
if err := e.redisManager.Start(); err != nil {
|
|
||||||
log.Printf("[Election] Failed to restart Redis: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case RoleFollower:
|
|
||||||
// Re-read leader info in case it changed
|
|
||||||
info, err := e.readLeaderInfo()
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Election] Failed to read leader info: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if info != nil {
|
|
||||||
e.mu.Lock()
|
|
||||||
e.leaderInfo = info
|
|
||||||
e.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// updateLeaderTimestamp updates the timestamp in leader info
|
|
||||||
func (e *Election) updateLeaderTimestamp() {
|
|
||||||
e.mu.Lock()
|
|
||||||
if e.leaderInfo != nil {
|
|
||||||
e.leaderInfo.Timestamp = time.Now().UnixMilli()
|
|
||||||
info := *e.leaderInfo
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
if err := e.writeLeaderInfo(&info); err != nil {
|
|
||||||
log.Printf("[Election] Failed to update timestamp: %v", err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
e.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// getLocalIP returns the local IP address
|
// getLocalIP returns the local IP address
|
||||||
func getLocalIP() string {
|
func getLocalIP() string {
|
||||||
addrs, err := net.InterfaceAddrs()
|
addrs, err := net.InterfaceAddrs()
|
||||||
|
|||||||
@@ -1,185 +0,0 @@
|
|||||||
package leader
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/metazla/meta-core/internal/config"
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RedisManager handles spawning and managing the Redis process
|
|
||||||
type RedisManager struct {
|
|
||||||
config *config.Config
|
|
||||||
cmd *exec.Cmd
|
|
||||||
running bool
|
|
||||||
mu sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewRedisManager creates a new Redis manager
|
|
||||||
func NewRedisManager(cfg *config.Config) *RedisManager {
|
|
||||||
return &RedisManager{
|
|
||||||
config: cfg,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start spawns the Redis server process
|
|
||||||
func (rm *RedisManager) Start() error {
|
|
||||||
rm.mu.Lock()
|
|
||||||
defer rm.mu.Unlock()
|
|
||||||
|
|
||||||
if rm.running {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure Redis data directory exists
|
|
||||||
dataDir := rm.config.RedisDataDir()
|
|
||||||
if err := os.MkdirAll(dataDir, 0755); err != nil {
|
|
||||||
return fmt.Errorf("failed to create Redis data directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("[Redis] Spawning Redis on port %d...", rm.config.RedisPort)
|
|
||||||
|
|
||||||
// Build Redis arguments matching TypeScript implementation
|
|
||||||
args := []string{
|
|
||||||
"--port", fmt.Sprintf("%d", rm.config.RedisPort),
|
|
||||||
"--bind", "0.0.0.0",
|
|
||||||
"--protected-mode", "no",
|
|
||||||
"--dir", dataDir,
|
|
||||||
"--appendonly", "yes",
|
|
||||||
"--appendfilename", "appendonly.aof",
|
|
||||||
"--dbfilename", "dump.rdb",
|
|
||||||
"--save", "60", "1", // Save after 60 seconds if at least 1 key changed
|
|
||||||
"--loglevel", "warning",
|
|
||||||
// Enable keyspace notifications for meta:events stream
|
|
||||||
// K = Keyspace events (publish to __keyspace@0__:keyname)
|
|
||||||
// g = Generic commands (DEL, EXPIRE, RENAME)
|
|
||||||
// $ = String commands (SET, MSET, etc.)
|
|
||||||
"--notify-keyspace-events", "Kg$",
|
|
||||||
}
|
|
||||||
|
|
||||||
rm.cmd = exec.Command("redis-server", args...)
|
|
||||||
rm.cmd.Stdout = os.Stdout
|
|
||||||
rm.cmd.Stderr = os.Stderr
|
|
||||||
|
|
||||||
if err := rm.cmd.Start(); err != nil {
|
|
||||||
return fmt.Errorf("failed to start Redis: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rm.running = true
|
|
||||||
log.Printf("[Redis] Started with PID %d", rm.cmd.Process.Pid)
|
|
||||||
|
|
||||||
// Monitor the process in background
|
|
||||||
go rm.monitorProcess()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop gracefully stops the Redis server
|
|
||||||
func (rm *RedisManager) Stop() error {
|
|
||||||
rm.mu.Lock()
|
|
||||||
defer rm.mu.Unlock()
|
|
||||||
|
|
||||||
if !rm.running || rm.cmd == nil || rm.cmd.Process == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("[Redis] Stopping Redis...")
|
|
||||||
|
|
||||||
// Try graceful shutdown first (SIGTERM)
|
|
||||||
if err := rm.cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
|
||||||
log.Printf("[Redis] Failed to send SIGTERM: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait up to 10 seconds for graceful shutdown
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- rm.cmd.Wait()
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
log.Println("[Redis] Stopped gracefully")
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
log.Println("[Redis] Timeout, sending SIGKILL...")
|
|
||||||
rm.cmd.Process.Kill()
|
|
||||||
<-done
|
|
||||||
}
|
|
||||||
|
|
||||||
rm.running = false
|
|
||||||
rm.cmd = nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsRunning returns true if Redis is currently running
|
|
||||||
func (rm *RedisManager) IsRunning() bool {
|
|
||||||
rm.mu.RLock()
|
|
||||||
defer rm.mu.RUnlock()
|
|
||||||
return rm.running
|
|
||||||
}
|
|
||||||
|
|
||||||
// WaitForReady waits for Redis to be ready to accept connections
|
|
||||||
func (rm *RedisManager) WaitForReady(timeout time.Duration) error {
|
|
||||||
log.Println("[Redis] Waiting for Redis to be ready...")
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
client := redis.NewClient(&redis.Options{
|
|
||||||
Addr: fmt.Sprintf("localhost:%d", rm.config.RedisPort),
|
|
||||||
})
|
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(100 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return fmt.Errorf("timeout waiting for Redis")
|
|
||||||
case <-ticker.C:
|
|
||||||
if err := client.Ping(ctx).Err(); err == nil {
|
|
||||||
log.Println("[Redis] Ready to accept connections")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ping checks if Redis is responding
|
|
||||||
func (rm *RedisManager) Ping() error {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
client := redis.NewClient(&redis.Options{
|
|
||||||
Addr: fmt.Sprintf("localhost:%d", rm.config.RedisPort),
|
|
||||||
})
|
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
return client.Ping(ctx).Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorProcess monitors the Redis process and updates state when it exits
|
|
||||||
func (rm *RedisManager) monitorProcess() {
|
|
||||||
if rm.cmd == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err := rm.cmd.Wait()
|
|
||||||
|
|
||||||
rm.mu.Lock()
|
|
||||||
rm.running = false
|
|
||||||
rm.mu.Unlock()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Redis] Process exited with error: %v", err)
|
|
||||||
} else {
|
|
||||||
log.Println("[Redis] Process exited normally")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -68,6 +68,15 @@ func (p *Poller) Start() error {
|
|||||||
|
|
||||||
log.Println("[WatcherPoller] Starting file watcher polling service")
|
log.Println("[WatcherPoller] Starting file watcher polling service")
|
||||||
|
|
||||||
|
// Clear file:events stream on startup to prevent duplicate events from accumulating
|
||||||
|
// across container restarts. Since in-memory state is empty, all files will be
|
||||||
|
// re-emitted as "add" events - we need a fresh stream to match.
|
||||||
|
if p.dispatcher != nil {
|
||||||
|
if err := p.dispatcher.EmitReset("startup"); err != nil {
|
||||||
|
log.Printf("[WatcherPoller] Warning: failed to clear stream on startup: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Initial sync with watcher configurations
|
// Initial sync with watcher configurations
|
||||||
go p.syncWithWatchers()
|
go p.syncWithWatchers()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user