diff --git a/README.md b/README.md index f0280a8..ae945d5 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,15 @@ meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-strem | Feature | Description | |---------|-------------| -| **Leader Election** | POSIX flock-based distributed consensus on shared filesystem | +| **Leader Election** | flock-based leader election (Go binary only runs as leader) | | **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash | | **HTTP API** | Language-agnostic REST interface for metadata and service discovery | | **Service Discovery** | File-based registry with heartbeats and stale detection | | **Metadata Storage** | Flat key-value schema with connection pooling and batch operations | +| **File Watching** | Directory scanning, MidHash256 computation, event dispatch | +| **Mount Management** | SMB/rclone configuration, health monitoring | +| **WebDAV Server** | File access with LRU caching and cache invalidation | +| **Event Publishing** | Redis keyspace notifications to meta:events stream | ### Design Characteristics @@ -26,24 +30,36 @@ meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-strem ## Architecture +meta-core runs as a **standalone container** that other services connect to via HTTP API: + ``` -┌─────────────────────────────────────────────────────────────┐ -│ Container (meta-sort / meta-fuse / meta-stremio) │ -│ │ -│ ┌──────────────┐ HTTP API ┌──────────────────┐ │ -│ │ Main Service │ ◄────────────────► │ meta-core │ │ -│ │ (TS/Python) │ localhost:9000 │ (Go binary) │ │ -│ └──────────────┘ └────────┬─────────┘ │ -│ │ │ -└───────────────────────────────────────────────┼─────────────┘ - │ - ┌───────────────────────┴───────────────┐ - │ /meta-core (shared volume) │ - │ ├── locks/kv-leader.lock │ - │ ├── locks/kv-leader.info │ - │ ├── db/redis/ │ - │ └── services/*.json │ - └───────────────────────────────────────┘ +┌───────────────────────────────────────────────────────────────────────┐ +│ meta-core container (standalone) │ +│ │ +│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │ +│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │ +│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │ +│ │ │ │ │ │ │ │ /health, /services │ │ +│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │ +│ │ +│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │ +└──────────────────────────────────────────────┬────────────────────────┘ + │ + ┌──────────────────────────────────────┴──────────────────────────┐ + │ /meta-core (shared volume) │ + │ ├── locks/kv-leader.info (leader URLs) │ + │ ├── db/redis/ (Redis data) │ + │ ├── services/ (hostname-based .json) │ + │ └── mounts/ (mount configurations) │ + └─────────────────────────────────────────────────────────────────┘ + ▲ ▲ ▲ + ┌────────────┘ │ └────────────┐ + │ │ │ +┌───────────────┐ ┌───────────────┐ ┌───────────────┐ +│ meta-sort │ │ meta-fuse │ │ meta-stremio │ +│ (reads info, │ │ (reads info, │ │ (reads info, │ +│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │ +└───────────────┘ └───────────────┘ └───────────────┘ ``` ## Quick Start @@ -95,6 +111,10 @@ docker run -v meta-core:/meta-core -v files:/files meta-core | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval | | `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval | | `STALE_THRESHOLD_MS` | `60000` | Stale service threshold | +| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning | +| `CACHE_ENABLED` | `true` | Enable WebDAV caching | +| `CACHE_MAX_SIZE_GB` | `100` | Maximum cache size in GB | +| `CACHE_TTL_SECONDS` | `3600` | Cache entry TTL | ## API Reference @@ -103,19 +123,15 @@ docker run -v meta-core:/meta-core -v files:/files meta-core ```bash # Health check curl http://localhost:9000/health -# {"status":"ok","role":"leader","redis":true,"timestamp":"..."} +# {"status":"ok","redis":true,"timestamp":"..."} # Detailed status curl http://localhost:9000/status -# {"status":"ok","role":"leader","serviceName":"meta-core","version":"1.0.0",...} +# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...} # Current leader curl http://localhost:9000/leader # {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...} - -# This instance's role -curl http://localhost:9000/role -# {"role":"leader"} ``` ### Metadata Operations @@ -177,6 +193,111 @@ curl http://localhost:9000/services/meta-sort # {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...} ``` +### Mount Management + +```bash +# List all mounts +curl http://localhost:9000/api/mounts/list +# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]} + +# Add a new mount +curl -X POST http://localhost:9000/api/mounts/add \ + -H "Content-Type: application/json" \ + -d '{"type":"smb","path":"/files/corn/nas","config":{...}}' + +# Remove a mount +curl -X DELETE http://localhost:9000/api/mounts/smb-nas +``` + +### File Watching + +```bash +# Get scan status +curl http://localhost:9000/api/scan/status +# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847} + +# Trigger rescan +curl -X POST http://localhost:9000/api/scan/trigger + +# Stream file events (SSE) +curl http://localhost:9000/api/events/stream +# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."} +``` + +### Cache Management + +```bash +# Get cache status +curl http://localhost:9000/api/cache/status +# {"enabled":true,"size":"2.5GB","entries":150,"maxSize":"100GB"} + +# Clear cache +curl -X POST http://localhost:9000/api/cache/clear + +# Detailed cache stats +curl http://localhost:9000/api/cache/stats +``` + +### KV Browser + +```bash +# Storage statistics +curl http://localhost:9000/api/kv/info +# {"totalKeys":50000,"memoryUsed":"256MB",...} + +# List keys with cursor pagination +curl "http://localhost:9000/api/kv/keys?cursor=0&count=100" + +# Get specific key value +curl http://localhost:9000/api/kv/key/file:bafkr.../title +``` + +### Metadata Editor + +```bash +# Get all hash IDs +curl http://localhost:9000/api/metadata/hash-ids +# {"hashIds":["bafkr...","bafks..."]} + +# Paginated metadata listing +curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50" + +# Search metadata +curl -X POST http://localhost:9000/api/metadata/search \ + -H "Content-Type: application/json" \ + -d '{"query":"inception","fields":["title"]}' + +# Batch update +curl -X POST http://localhost:9000/api/metadata/batch \ + -H "Content-Type: application/json" \ + -d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}' + +# Clear all metadata +curl -X POST http://localhost:9000/api/metadata/clear +``` + +### WebDAV Access + +```bash +# List directory (returns JSON) +curl http://localhost:9000/webdav/ +# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}] + +# Get file +curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv + +# Upload file +curl -X PUT http://localhost:9000/webdav/plugin/output.txt \ + -H "Content-Type: application/octet-stream" \ + --data-binary @output.txt + +# Create directory +curl -X MKCOL http://localhost:9000/webdav/newdir/ + +# Delete file/directory +curl -X DELETE http://localhost:9000/webdav/oldfile.txt +``` + ## Leader Election meta-core uses POSIX file locking (flock) for distributed leader election: @@ -223,16 +344,13 @@ election.OnLeaderLost(func() { ### Lock Info Format -```json -{ - "host": "meta-sort-dev", - "api": "redis://10.0.1.50:6379", - "http": "http://10.0.1.50:8180", - "baseUrl": "http://localhost:8180", - "timestamp": 1704067200000, - "pid": 12345 -} +The `kv-leader.info` file contains a plain text API URL: + ``` +http://10.0.1.50:9000 +``` + +Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.). ## Redis Management @@ -347,6 +465,12 @@ make clean | `internal/storage` | Redis client wrapper with flat key-value operations | | `internal/discovery` | Service registration, heartbeat loop, discovery | | `internal/api` | HTTP server, router, and all endpoint handlers | +| `internal/cache` | WebDAV file caching with LRU eviction and invalidation | +| `internal/events` | Redis keyspace notification publisher to meta:events | +| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch | +| `internal/watchers` | Polling-based watcher configuration management | +| `internal/mounts` | SMB/rclone mount configuration and health monitoring | +| `internal/webdav` | WebDAV protocol handler with caching integration | ### Startup Sequence @@ -357,9 +481,13 @@ make clean 4. Register role transition callbacks 5. Start election (acquire lock or become follower) 6. Start service discovery (register + heartbeat loop) -7. Start HTTP API server -8. Wait for SIGINT/SIGTERM -9. Shutdown in reverse order +7. Initialize dead service cleaner +8. Initialize cache manager (if enabled) +9. Initialize file watcher (if enabled) +10. Initialize mount manager +11. Start HTTP API server +12. Wait for SIGINT/SIGTERM +13. Shutdown in reverse order ``` ## Integration @@ -397,6 +525,8 @@ metadata = response.json()['metadata'] |------------|---------|---------| | `github.com/gorilla/mux` | v1.8.1 | HTTP router | | `github.com/redis/go-redis/v9` | v9.4.0 | Redis client | +| `golang.org/x/net` | v0.20.0 | WebDAV support | +| `github.com/google/uuid` | v1.6.0 | UUID generation | | Go stdlib | - | context, encoding/json, net, os, sync, syscall | ### Runtime Requirements diff --git a/cmd/meta-core/main.go b/cmd/meta-core/main.go index 3cedc6c..0ed0c39 100644 --- a/cmd/meta-core/main.go +++ b/cmd/meta-core/main.go @@ -6,6 +6,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/metazla/meta-core/internal/api" "github.com/metazla/meta-core/internal/config" @@ -25,23 +26,33 @@ func main() { cfg := config.Load() log.Printf("[meta-core] Service: %s, HTTP Port: %d", cfg.ServiceName, cfg.HTTPPort) - // Create role provider (reads META_CORE_ROLE from environment, set by leader-election.sh) - roleProvider := leader.NewLocalRoleProvider(cfg) - log.Printf("[meta-core] Role: %s", roleProvider.Role()) + // Create leader info provider (Go binary only runs as leader - followers loop in bash) + leaderProvider := leader.NewLeaderInfoProvider(cfg) // Connect to local Redis (supervisord starts Redis before meta-core) + // Retry connection since Redis may still be loading its dataset redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort) log.Printf("[meta-core] Connecting to Redis at %s", redisURL) storageClient := storage.NewClient("") - if err := storageClient.Connect(redisURL); err != nil { - log.Fatalf("[meta-core] Failed to connect to Redis: %v", err) + maxRetries := 30 + retryDelay := 1 // seconds + var err error + for i := 0; i < maxRetries; i++ { + err = storageClient.Connect(redisURL) + if err == nil { + break + } + log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err) + time.Sleep(time.Duration(retryDelay) * time.Second) + } + if err != nil { + log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err) } log.Println("[meta-core] Connected to Redis") // Create and start service discovery disc := discovery.NewService(cfg) - disc.SetRoleProvider(&roleProviderAdapter{provider: roleProvider}) if err := disc.Start(); err != nil { log.Fatalf("[meta-core] Failed to start service discovery: %v", err) } @@ -53,7 +64,7 @@ func main() { } // Create and start API server - apiServer := api.NewServer(cfg, roleProvider, disc, cleaner, storageClient) + apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient) if err := apiServer.Start(); err != nil { log.Fatalf("[meta-core] Failed to start API server: %v", err) } @@ -92,11 +103,3 @@ func waitForShutdown() { <-sigChan } -// roleProviderAdapter adapts leader.RoleProvider to discovery.RoleProvider -type roleProviderAdapter struct { - provider leader.RoleProvider -} - -func (a *roleProviderAdapter) Role() string { - return string(a.provider.Role()) -} diff --git a/dashboard/src/pages/Dashboard.tsx b/dashboard/src/pages/Dashboard.tsx index c76bbc0..7b68e22 100644 --- a/dashboard/src/pages/Dashboard.tsx +++ b/dashboard/src/pages/Dashboard.tsx @@ -2,7 +2,6 @@ import { useEffect, useState } from 'react'; interface HealthStatus { status: string; - role: string; redis: boolean; timestamp: string; } @@ -121,7 +120,6 @@ export default function Dashboard() {
{health.status === 'ok' ? 'Healthy' : health.status}
-

Role: {health.role}

Redis: {health.redis ? 'Connected' : 'Disconnected'}

) : ( diff --git a/docker/leader-election.sh b/docker/leader-election.sh index 85f10b7..f92a819 100644 --- a/docker/leader-election.sh +++ b/docker/leader-election.sh @@ -82,9 +82,6 @@ while true; do # Write leader info before starting services write_leader_info - # Export role for meta-core Go binary to read - export META_CORE_ROLE="leader" - # exec supervisord - replaces this process # When supervisord exits, the flock is released automatically echo "[election] Starting supervisord..." diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index cf9424b..99712f5 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -15,6 +15,7 @@ MetaMesh consists of three services that require shared access to metadata and f | **meta-sort** | TypeScript | Watches folders, extracts metadata, writes to KV | | **meta-fuse** | TypeScript/Rust | Serves virtual filesystem from KV metadata | | **meta-stremio** | Python | Stremio addon for HLS streaming | +| **meta-dup** | TypeScript | Duplicate detection service | ### Problem Statement @@ -44,71 +45,91 @@ This design enables: ## Architecture Design -### Sidecar Pattern +### Standalone Container Pattern -Meta-core runs as a **sidecar process** in each container, alongside the main service: +Meta-core runs as a **standalone container** that other services connect to via HTTP API: ``` -┌─────────────────────────────────────────────────────────────────┐ -│ Container (meta-sort / meta-fuse / meta-stremio) │ -│ │ -│ ┌──────────────────┐ HTTP API ┌──────────────────────┐│ -│ │ Main Service │◄────────────────►│ meta-core ││ -│ │ │ localhost:9000 │ (Go binary) ││ -│ │ - TypeScript │ │ ││ -│ │ - Python │ │ - Leader election ││ -│ │ - Any language │ │ - Redis management ││ -│ │ │ │ - Service discovery ││ -│ └──────────────────┘ │ - File access API ││ -│ └──────────┬───────────┘│ -│ │ │ -└───────────────────────────────────────────────────┼─────────────┘ - │ - ┌───────────────────────────────┼───────────────┐ - │ /meta-core (shared volume) │ - │ ▼ │ - │ ┌─────────────────────────────────────────┐ │ - │ │ locks/ │ │ - │ │ kv-leader.lock (flock marker) │ │ - │ │ kv-leader.info (leader JSON) │ │ - │ │ db/ │ │ - │ │ redis/ (RDB + AOF) │ │ - │ │ services/ │ │ - │ │ meta-sort.json (service registry) │ │ - │ │ meta-fuse.json │ │ - │ │ meta-stremio.json │ │ - │ └─────────────────────────────────────────┘ │ - └──────────────────────────────────────────────┘ +┌───────────────────────────────────────────────────────────────────────┐ +│ meta-core container (standalone) │ +│ │ +│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │ +│ │ Leader │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │ +│ │ Election │ │ Server │ │ Server │ │ /meta, /urls, │ │ +│ │ (flock) │ │ │ │ │ │ /health, /services │ │ +│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │ +│ │ +│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │ +└──────────────────────────────────────────────┬────────────────────────┘ + │ + ┌──────────────────────────┼────────────────────────┐ + │ /meta-core (shared volume) │ + │ ▼ │ + │ ┌─────────────────────────────────────────────┐ │ + │ │ locks/ │ │ + │ │ kv-leader.lock (flock marker) │ │ + │ │ kv-leader.info (leader JSON) │ │ + │ │ db/ │ │ + │ │ redis/ (RDB + AOF) │ │ + │ │ services/ │ │ + │ │ meta-sort-*.json (service registry) │ │ + │ │ meta-fuse-*.json │ │ + │ │ meta-stremio-*.json │ │ + │ │ mounts/ │ │ + │ │ mounts.json (mount configurations) │ │ + │ └─────────────────────────────────────────────┘ │ + └──────────────────────────────────────────────────┘ + ▲ ▲ + ┌────────────────────┘ └────────────────────┐ + │ │ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ meta-sort │ │ meta-fuse │ │ meta-stremio │ │ meta-dup │ +│ container │ │ container │ │ container │ │ container │ +│ │ │ │ │ │ │ │ +│ Reads leader │ │ Reads leader │ │ Reads leader │ │ Reads leader │ +│ .info, calls │ │ .info, calls │ │ .info, calls │ │ .info, calls │ +│ HTTP API │ │ HTTP API │ │ HTTP API │ │ HTTP API │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ ┌──────────────────────────────────────────────┐ │ /files (shared volume) │ │ │ - │ media/ │ - │ movies/ │ - │ Movie.2024.mkv │ - │ series/ │ - │ Show.S01E01.mkv │ + │ watch/ (host media, read-only) │ + │ test/ (test media) │ + │ plugin/ (plugin output, read-write) │ + │ corn/ (SMB/rclone mounts) │ └──────────────────────────────────────────────┘ ``` ### Multi-Container Topology -Across all containers, exactly **one meta-core instance** is the leader: +Meta-core runs as a **standalone container**. Other services connect to it via HTTP API: ``` + ┌─────────────────────────────────────┐ + │ meta-core container │ + │ ★ LEADER │ + │ │ + │ ┌─────────┐ ┌──────────────────┐ │ + │ │ Redis │ │ HTTP API (:9000) │ │ + │ │ Server │ │ WebDAV Server │ │ + │ └─────────┘ └──────────────────┘ │ + └────────────────────┬────────────────┘ + │ + ┌────────────────────────────┼────────────────────────────┐ + │ │ │ + ▼ ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ │ meta-sort │ │ meta-fuse │ │ meta-stremio │ │ container │ │ container │ │ container │ │ │ │ │ │ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ -│ │ meta-core │ │ │ │ meta-core │ │ │ │ meta-core │ │ -│ │ ★ LEADER │ │ │ │ follower │ │ │ │ follower │ │ -│ │ │ │ │ │ │ │ │ │ │ │ -│ │ [Redis]──────┼──┼──┼──┼───────────────┼──┼──┼──┼───────────────│ │ -│ │ │ │ │ │ │ │ │ │ │ │ +│ │ LeaderClient │ │ │ │ LeaderClient │ │ │ │ LeaderClient │ │ +│ │ (reads info, │ │ │ │ (reads info, │ │ │ │ (reads info, │ │ +│ │ calls HTTP) │ │ │ │ calls HTTP) │ │ │ │ calls HTTP) │ │ │ └───────────────┘ │ │ └───────────────┘ │ │ └───────────────┘ │ -│ ▲ │ │ │ │ │ │ │ -│ │ HTTP │ │ │ HTTP │ │ │ HTTP │ +│ ▲ │ │ ▲ │ │ ▲ │ +│ │ │ │ │ │ │ │ │ │ ▼ │ │ ▼ │ │ ▼ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ │ │ meta-sort │ │ │ │ meta-fuse │ │ │ │ meta-stremio │ │ @@ -124,24 +145,40 @@ Across all containers, exactly **one meta-core instance** is the leader: └──────────────────────────────┘ ``` +For high availability, multiple meta-core instances can run with flock-based failover: + +``` +┌─────────────────────┐ ┌─────────────────────┐ +│ meta-core │ │ meta-core-2 │ +│ ★ LEADER │ │ follower │ +│ │ │ │ +│ [Redis] ───────────┼──────┼─► Connects to │ +│ │ │ leader's Redis │ +└─────────────────────┘ └─────────────────────┘ + │ + ▼ + /meta-core/locks/kv-leader.lock (flock held by leader) +``` + --- ## Leader Election ### Mechanism: flock-based Consensus -Meta-core uses **POSIX file locking (flock)** on a shared filesystem for leader election. This approach requires no external consensus system (etcd, Consul, Zookeeper). +Meta-core uses **POSIX file locking (flock)** on a shared filesystem for leader election. The election is performed by `leader-election.sh` which starts supervisord when the lock is acquired. The Go binary only ever runs as leader (followers loop in bash and never start the binary). This approach requires no external consensus system (etcd, Consul, Zookeeper). ``` Leader Election Flow: - meta-core startup + Container startup │ ▼ ┌──────────────────┐ + │ leader-election │ + │ .sh │ │ Try flock() on │ │ kv-leader.lock │ - │ (non-blocking) │ └────────┬─────────┘ │ ┌───────┴───────┐ @@ -149,19 +186,22 @@ Leader Election Flow: acquired blocked │ │ ▼ ▼ -┌─────────┐ ┌──────────┐ -│ LEADER │ │ FOLLOWER │ -└────┬────┘ └────┬─────┘ - │ │ - ▼ ▼ -┌──────────┐ ┌────────────┐ -│ Spawn │ │ Read │ -│ Redis │ │ leader.info│ -│ │ │ │ -│ Write │ │ Connect to │ -│ leader │ │ leader │ -│ .info │ │ Redis │ -└──────────┘ └────────────┘ +┌─────────────┐ ┌──────────────┐ +│ Write │ │ Sleep and │ +│ leader.info │ │ retry flock │ +│ │ │ │ +│ exec │ └──────────────┘ +│ supervisord │ +└──────┬──────┘ + │ + ▼ +┌──────────────┐ +│ meta-core │ +│ (Go binary) │ +│ │ +│ Spawns Redis │ +│ Starts API │ +└──────────────┘ ``` ### Lock File Structure @@ -172,16 +212,11 @@ Leader Election Flow: - Automatically released on process death (kernel handles cleanup) **`/meta-core/locks/kv-leader.info`** -```json -{ - "hostname": "meta-sort-dev", - "redis_url": "redis://10.0.1.50:6379", - "http_url": "http://10.0.1.50:9000", - "started_at": "2024-01-15T10:30:00Z", - "pid": 12345, - "version": "1.0.0" -} ``` +http://10.0.1.50:9000 +``` + +Plain text file containing the leader's API URL. Services read this to discover the leader, then call the `/urls` API endpoint for full connection details (Redis URL, WebDAV URL, etc.). ### Failover Behavior @@ -225,6 +260,96 @@ Follower meta-core instances: 3. **Cache metadata locally** for fast reads (optional) 4. **Attempt leader acquisition** on leader failure +### WebDAV Caching Architecture + +Meta-core provides a WebDAV server at `/webdav/` with integrated caching: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ WebDAV Server │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │ HTTP Handler │─►│ Cache Layer │─►│ File System (/files) │ │ +│ │ │ │ │ │ │ │ +│ │ GET/PUT/ │ │ LRU Index │ │ - watch/ │ │ +│ │ DELETE/MKCOL │ │ TTL Expiry │ │ - plugin/ │ │ +│ │ │ │ Invalidation │ │ - corn/ │ │ +│ └──────────────┘ └──────────────┘ └────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ Redis │ │ +│ │ Keyspace │ │ +│ │ Notifications│ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Cache features: +- **LRU Index**: Tracks file access times and sizes for eviction +- **Eviction**: Removes least-recently-used entries when limit exceeded +- **TTL**: Expires entries after configured time (`CACHE_TTL_SECONDS`) +- **Invalidation**: Cache invalidator listens to Redis keyspace changes +- **Directory Listing**: Returns JSON for programmatic access + +### File Watching System + +Meta-core scans directories and emits events for file changes: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ File Watcher │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │ Directory │─►│ State │─►│ Event Dispatcher │ │ +│ │ Scanner │ │ Registry │ │ │ │ +│ │ │ │ │ │ - Debouncing │ │ +│ │ - Recursive │ │ - File hash │ │ - Buffering │ │ +│ │ - MidHash256 │ │ - Timestamps │ │ - SSE streaming │ │ +│ └──────────────┘ └──────────────┘ └────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────┐ │ +│ │ meta:events stream │ │ +│ │ (add/change/delete/ │ │ +│ │ rename/reset) │ │ +│ └──────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Components: +- **Directory Scanner**: Recursively scans file trees, computes MidHash256 +- **State Registry**: Tracks file states for change detection +- **Debouncer**: Batches rapid events for efficiency +- **Event Dispatcher**: Routes events to subscribers with buffering + +Events are published to the `meta:events` Redis stream. + +### Mount Management + +Meta-core manages SMB/rclone mount configurations: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Mount Manager │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │ Config │─►│ Validator │─►│ Health Monitor │ │ +│ │ (mounts.json)│ │ │ │ │ │ +│ │ │ │ - SMB params │ │ - Accessibility check │ │ +│ │ - id │ │ - rclone cfg │ │ - Error tracking │ │ +│ │ - type │ │ - paths │ │ - Status reporting │ │ +│ │ - path │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Features: +- **Configuration**: Mount configs stored in `/meta-core/mounts/mounts.json` +- **Validation**: Validates mount commands and parameters +- **Health Monitoring**: Polls mount points for accessibility +- **Error Handling**: Tracks mount errors in separate error files + --- ## Data Model @@ -308,6 +433,38 @@ Service Discovery GET /services → List registered services POST /services/{name} → Register/heartbeat service DELETE /services/{name} → Deregister service + +Mount Management + GET /api/mounts/list → List all mounts + POST /api/mounts/add → Add new mount + DELETE /api/mounts/{id} → Remove mount + +File Watching + GET /api/scan/status → Scan status + POST /api/scan/trigger → Trigger rescan + GET /api/events/stream → Stream file events (SSE) + +Cache Management + GET /api/cache/status → Cache statistics + POST /api/cache/clear → Clear cache + GET /api/cache/stats → Detailed metrics + +KV Browser + GET /api/kv/info → Storage statistics + GET /api/kv/keys → Key listing with cursor + GET /api/kv/key/{key} → Get key value + +Metadata Editor + GET /api/metadata/hash-ids → All hash IDs + GET /api/metadata/list → Paginated listing + POST /api/metadata/search → Advanced search + POST /api/metadata/batch → Batch updates + POST /api/metadata/clear → Clear all metadata + +WebDAV + GET/PUT/DELETE /webdav/* → File operations + MKCOL/COPY/MOVE /webdav/* → Directory operations + PROPFIND /webdav/* → Property queries ``` ### Health Endpoint @@ -318,7 +475,6 @@ GET /health Response: { "status": "healthy", - "role": "leader", // or "follower" "leader": { "hostname": "meta-sort-dev", "http_url": "http://10.0.1.50:9000", @@ -478,24 +634,47 @@ Meta-core is implemented in Go for the following reasons: meta-core ├── cmd/ │ └── meta-core/ -│ └── main.go # Entry point +│ └── main.go # Entry point ├── internal/ -│ ├── leader/ -│ │ ├── election.go # flock-based election -│ │ └── redis.go # Redis process management -│ ├── api/ -│ │ ├── server.go # HTTP server setup -│ │ ├── health.go # Health endpoints -│ │ ├── meta.go # Metadata endpoints -│ │ ├── data.go # Data endpoints -│ │ └── services.go # Service discovery endpoints -│ ├── storage/ -│ │ ├── redis.go # Redis client wrapper -│ │ └── cache.go # Optional local cache -│ └── config/ -│ └── config.go # Configuration loading -├── go.mod -└── go.sum +│ ├── api/ # HTTP server and handlers +│ │ ├── server.go # HTTP server setup +│ │ ├── health.go # Health endpoints +│ │ ├── meta.go # Metadata endpoints +│ │ ├── data.go # Data endpoints +│ │ ├── services.go # Service discovery endpoints +│ │ ├── mounts.go # Mount management endpoints +│ │ ├── cache.go # Cache management endpoints +│ │ └── kv.go # KV browser endpoints +│ ├── cache/ # WebDAV caching with LRU +│ │ ├── lru.go # LRU index implementation +│ │ ├── invalidator.go # Keyspace notification listener +│ │ └── manager.go # Cache lifecycle management +│ ├── config/ # Configuration loading +│ │ └── config.go +│ ├── discovery/ # Service registration +│ │ └── discovery.go +│ ├── events/ # Keyspace notification publisher +│ │ └── publisher.go +│ ├── leader/ # Role provider and leader info +│ │ ├── election.go # flock-based election +│ │ └── redis.go # Redis process management +│ ├── mounts/ # SMB/rclone management +│ │ ├── config.go # Mount configuration +│ │ └── health.go # Mount health monitoring +│ ├── storage/ # Redis client wrapper +│ │ └── client.go +│ ├── watcher/ # File scanning and events +│ │ ├── scanner.go # Directory scanner +│ │ ├── dispatcher.go # Event dispatcher +│ │ └── debouncer.go # Event debouncing +│ ├── watchers/ # Polling-based watchers +│ │ └── config.go +│ └── webdav/ # WebDAV protocol handler +│ └── handler.go +├── test/ +├── docs/ +├── Makefile +└── go.mod ``` ### Process Management @@ -813,6 +992,31 @@ healthcheck: --- +## WebDAV URL Configuration + +Meta-core exposes two WebDAV URLs via the `/urls` API to support both internal and external file access: + +| Field | Purpose | Example | +|-------|---------|---------| +| `webdavUrl` | External access (via nginx/HTTPS) | `https://media.example.com/webdav` | +| `webdavUrlInternal` | Internal container-to-container | `http://meta-core-1:9000/webdav` | + +### URL Composition + +- **External** (`webdavUrl`): `{BASE_URL}/webdav` + - Uses `BASE_URL` env var if set, otherwise falls back to `http://{ip}:{API_PORT}` + +- **Internal** (`webdavUrlInternal`): `http://{hostname}:{META_CORE_HTTP_PORT}/webdav` + - Uses container hostname (Docker DNS resolvable) + - Port 9000 is the direct Go WebDAV server + +### Usage Patterns + +- **Container plugins** should use `webdavUrlInternal` for file access (direct connection, lower latency) +- **External clients** should use `webdavUrl` for browser/remote access (may include SSL termination) + +--- + ## Configuration Reference ### Environment Variables @@ -829,6 +1033,10 @@ healthcheck: | `META_CORE_LOG_LEVEL` | `info` | Log verbosity | | `META_CORE_CACHE_ENABLED` | `false` | Enable local read cache | | `META_CORE_CACHE_TTL` | `60` | Cache TTL (seconds) | +| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning | +| `CACHE_ENABLED` | `true` | Enable WebDAV caching | +| `CACHE_MAX_SIZE_GB` | `100` | Max cache size | +| `CACHE_TTL_SECONDS` | `3600` | Cache entry TTL | ### File Paths @@ -839,6 +1047,9 @@ healthcheck: | `/meta-core/db/redis/dump.rdb` | Redis RDB snapshot | | `/meta-core/db/redis/appendonly.aof` | Redis AOF log | | `/meta-core/services/*.json` | Service registry files | +| `/meta-core/mounts/mounts.json` | Mount configurations | +| `/meta-core/cache/` | Cached files directory | +| `/meta-core/watchers.json` | Watcher configurations | --- diff --git a/docs/metadata-storage-structure.md b/docs/metadata-storage-structure.md index 0f68bfa..577f567 100644 --- a/docs/metadata-storage-structure.md +++ b/docs/metadata-storage-structure.md @@ -376,6 +376,38 @@ PSUBSCRIBE '__keyspace@0__:file:*' PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*' ``` +### Cache Invalidation + +The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Cache Invalidation Flow │ +│ │ +│ Redis Keyspace Cache │ +│ ┌──────────────┐ Invalidator │ +│ │ SET file:*/ │ ──────────► ┌──────────────┐ │ +│ │ DEL file:*/ │ subscribe │ Listens to │ │ +│ └──────────────┘ │ __keyspace@ │ │ +│ │ 0__:file:* │ │ +│ └──────┬───────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ Remove from │ │ +│ │ LRU cache │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Flow: +1. Cache invalidator subscribes to `__keyspace@0__:file:*` +2. When metadata changes (SET/DEL), invalidator receives notification +3. Invalidator removes the corresponding file from the WebDAV cache +4. Next access fetches fresh copy from source filesystem + +This ensures cached files stay synchronized with metadata changes. + ### 3. Pub/Sub Channels (Legacy) Batch notification channels are still supported: @@ -492,6 +524,9 @@ redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*' | File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` | | Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` | | Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` | +| meta-core Storage | `packages/meta-core/internal/storage/client.go` | +| meta-core Events | `packages/meta-core/internal/events/publisher.go` | +| meta-core Cache | `packages/meta-core/internal/cache/` | ## Debugging @@ -523,6 +558,22 @@ docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0 # Monitor keyspace notifications (requires notify-keyspace-events enabled) docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*' + +# meta-core API debugging +# Cache status +curl http://localhost:9000/api/cache/status + +# Clear cache +curl -X POST http://localhost:9000/api/cache/clear + +# Watch events stream (SSE) +curl http://localhost:9000/api/events/stream + +# KV browser - storage info +curl http://localhost:9000/api/kv/info + +# Scan status +curl http://localhost:9000/api/scan/status ``` ## meta:events Stream diff --git a/internal/api/handlers.go b/internal/api/handlers.go index e34fa4e..ec21cac 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -34,7 +34,6 @@ var contentTypeByExt = map[string]string{ // HealthResponse is the response for /health type HealthResponse struct { Status string `json:"status"` - Role string `json:"role"` Redis bool `json:"redis"` Timestamp string `json:"timestamp"` Leader *leader.LeaderLockInfo `json:"leader,omitempty"` @@ -43,7 +42,6 @@ type HealthResponse struct { // StatusResponse is the response for /status type StatusResponse struct { Status string `json:"status"` - Role string `json:"role"` Redis bool `json:"redis"` ServiceName string `json:"serviceName"` Version string `json:"version"` @@ -77,10 +75,9 @@ var startTime = time.Now() func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { response := HealthResponse{ Status: "ok", - Role: string(s.roleProvider.Role()), Redis: s.storage.Health(), Timestamp: time.Now().UTC().Format(time.RFC3339), - Leader: s.roleProvider.LeaderInfo(), + Leader: s.leaderProvider.LeaderInfo(), } if !response.Redis { @@ -100,13 +97,12 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { response := StatusResponse{ Status: "ok", - Role: string(s.roleProvider.Role()), Redis: s.storage.Health(), ServiceName: s.config.ServiceName, Version: s.config.ServiceVersion, Uptime: int64(time.Since(startTime).Seconds()), FileCount: fileCount, - Leader: s.roleProvider.LeaderInfo(), + Leader: s.leaderProvider.LeaderInfo(), } if !response.Redis { @@ -118,7 +114,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { // handleLeader handles GET /leader func (s *Server) handleLeader(w http.ResponseWriter, r *http.Request) { - info := s.roleProvider.LeaderInfo() + info := s.leaderProvider.LeaderInfo() if info == nil { writeError(w, http.StatusServiceUnavailable, "no leader available") return @@ -127,39 +123,32 @@ func (s *Server) handleLeader(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, info) } -// handleRole handles GET /role -func (s *Server) handleRole(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{ - "role": string(s.roleProvider.Role()), - }) -} - // URLsResponse is the response for /urls type URLsResponse struct { - Hostname string `json:"hostname"` - BaseUrl string `json:"baseUrl"` - ApiUrl string `json:"apiUrl"` - RedisUrl string `json:"redisUrl"` - WebdavUrl string `json:"webdavUrl"` - IsLeader bool `json:"isLeader"` + Hostname string `json:"hostname"` + BaseUrl string `json:"baseUrl"` + ApiUrl string `json:"apiUrl"` + RedisUrl string `json:"redisUrl"` + WebdavUrl string `json:"webdavUrl"` + WebdavUrlInternal string `json:"webdavUrlInternal"` } // handleURLs handles GET /urls -// Returns URLs for the current leader (or this instance if leader) +// Returns URLs for this instance (always leader since Go binary only runs as leader) func (s *Server) handleURLs(w http.ResponseWriter, r *http.Request) { - info := s.roleProvider.LeaderInfo() + info := s.leaderProvider.LeaderInfo() if info == nil { writeError(w, http.StatusServiceUnavailable, "no leader available") return } response := URLsResponse{ - Hostname: info.Hostname, - BaseUrl: info.BaseUrl, - ApiUrl: info.ApiUrl, - RedisUrl: info.RedisUrl, - WebdavUrl: info.WebdavUrl, - IsLeader: s.roleProvider.IsLeader(), + Hostname: info.Hostname, + BaseUrl: info.BaseUrl, + ApiUrl: info.ApiUrl, + RedisUrl: info.RedisUrl, + WebdavUrl: info.WebdavUrl, + WebdavUrlInternal: info.WebdavUrlInternal, } writeJSON(w, http.StatusOK, response) diff --git a/internal/api/server.go b/internal/api/server.go index 089cccc..75de929 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -23,7 +23,7 @@ import ( // Server is the HTTP API server for meta-core type Server struct { config *config.Config - roleProvider leader.RoleProvider + leaderProvider *leader.LeaderInfoProvider discovery *discovery.Service cleaner *discovery.Cleaner storage *storage.Client @@ -48,18 +48,18 @@ type Server struct { // NewServer creates a new API server func NewServer( cfg *config.Config, - roleProvider leader.RoleProvider, + leaderProvider *leader.LeaderInfoProvider, disc *discovery.Service, cleaner *discovery.Cleaner, stor *storage.Client, ) *Server { s := &Server{ - config: cfg, - roleProvider: roleProvider, - discovery: disc, - cleaner: cleaner, - storage: stor, - router: mux.NewRouter(), + config: cfg, + leaderProvider: leaderProvider, + discovery: disc, + cleaner: cleaner, + storage: stor, + router: mux.NewRouter(), } // Initialize file watcher/scanner and dispatcher @@ -130,7 +130,6 @@ func (s *Server) setupRoutes() { s.router.HandleFunc("/health", s.handleHealth).Methods("GET") s.router.HandleFunc("/status", s.handleStatus).Methods("GET") s.router.HandleFunc("/leader", s.handleLeader).Methods("GET") - s.router.HandleFunc("/role", s.handleRole).Methods("GET") s.router.HandleFunc("/urls", s.handleURLs).Methods("GET") // Metadata Editor API routes (must be before /meta/{hash} routes) diff --git a/internal/discovery/service.go b/internal/discovery/service.go index c951994..90c854a 100644 --- a/internal/discovery/service.go +++ b/internal/discovery/service.go @@ -21,22 +21,15 @@ type ServiceInfo struct { BaseUrl string `json:"baseUrl"` Status string `json:"status"` LastHeartbeat string `json:"lastHeartbeat"` - Role string `json:"role,omitempty"` // "leader", "follower", or empty (for non-meta-core services) -} - -// RoleProvider interface for getting the current role -type RoleProvider interface { - Role() string } // Service handles service registration and discovery type Service struct { - config *config.Config - servicesDir string - serviceFile string - info *ServiceInfo - roleProvider RoleProvider - mu sync.RWMutex + config *config.Config + servicesDir string + serviceFile string + info *ServiceInfo + mu sync.RWMutex stopChan chan struct{} wg sync.WaitGroup @@ -53,11 +46,6 @@ func NewService(cfg *config.Config) *Service { } } -// SetRoleProvider sets the role provider for this service -func (s *Service) SetRoleProvider(rp RoleProvider) { - s.roleProvider = rp -} - // Start begins service registration and heartbeat func (s *Service) Start() error { log.Printf("[Discovery] Starting service discovery for %s", s.config.ServiceName) @@ -115,11 +103,6 @@ func (s *Service) buildServiceInfo() *ServiceInfo { LastHeartbeat: time.Now().UTC().Format(time.RFC3339), } - // Add role if role provider is set (e.g., for meta-core) - if s.roleProvider != nil { - info.Role = s.roleProvider.Role() - } - return info } @@ -164,7 +147,7 @@ func (s *Service) heartbeatLoop() { } } -// heartbeat updates the last heartbeat timestamp and role +// heartbeat updates the last heartbeat timestamp func (s *Service) heartbeat() { s.mu.Lock() defer s.mu.Unlock() @@ -175,11 +158,6 @@ func (s *Service) heartbeat() { s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339) - // Update role in case it changed (e.g., leader election) - if s.roleProvider != nil { - s.info.Role = s.roleProvider.Role() - } - if err := s.writeServiceInfo(s.info); err != nil { log.Printf("[Discovery] Failed to update heartbeat: %v", err) } diff --git a/internal/leader/election.go b/internal/leader/election.go index 07dee54..b693969 100644 --- a/internal/leader/election.go +++ b/internal/leader/election.go @@ -11,60 +11,27 @@ import ( // LeaderLockInfo contains all URLs and metadata for the leader type LeaderLockInfo struct { - Hostname string `json:"hostname"` - BaseUrl string `json:"baseUrl"` - ApiUrl string `json:"apiUrl"` // meta-core API (port 9000) - RedisUrl string `json:"redisUrl"` - WebdavUrl string `json:"webdavUrl"` - Timestamp int64 `json:"timestamp"` - PID int `json:"pid"` + Hostname string `json:"hostname"` + BaseUrl string `json:"baseUrl"` + ApiUrl string `json:"apiUrl"` // meta-core API (port 9000) + RedisUrl string `json:"redisUrl"` + WebdavUrl string `json:"webdavUrl"` // External WebDAV URL (via baseUrl) + WebdavUrlInternal string `json:"webdavUrlInternal"` // Internal WebDAV URL (direct to port 9000) + Timestamp int64 `json:"timestamp"` + PID int `json:"pid"` } -// Role represents the current role of this instance -type Role string - -const ( - RoleUnknown Role = "unknown" - RoleLeader Role = "leader" - RoleFollower Role = "follower" -) - -// RoleProvider provides role and leader info -// This interface is used by the API server to get role information -// without needing the full election logic (which is now in bash) -type RoleProvider interface { - Role() Role - LeaderInfo() *LeaderLockInfo - IsLeader() bool -} - -// LocalRoleProvider provides role info for this instance -// It reads the role from environment and builds leader info locally -// This is used when running as the leader (started by leader-election.sh) -type LocalRoleProvider struct { +// LeaderInfoProvider provides leader info for this instance +// The Go binary only ever runs as leader (followers loop in bash and never start the binary) +type LeaderInfoProvider struct { config *config.Config - role Role leaderInfo *LeaderLockInfo } -// NewLocalRoleProvider creates a RoleProvider for the local instance -// It reads META_CORE_ROLE from environment (set by leader-election.sh) -func NewLocalRoleProvider(cfg *config.Config) *LocalRoleProvider { - roleStr := os.Getenv("META_CORE_ROLE") - var role Role - switch roleStr { - case "leader": - role = RoleLeader - case "follower": - role = RoleFollower - default: - // Default to leader if not set (for backward compatibility) - role = RoleLeader - } - - provider := &LocalRoleProvider{ +// NewLeaderInfoProvider creates a LeaderInfoProvider for the local instance +func NewLeaderInfoProvider(cfg *config.Config) *LeaderInfoProvider { + provider := &LeaderInfoProvider{ config: cfg, - role: role, } // Build leader info (always, since we're the leader when running) @@ -73,13 +40,8 @@ func NewLocalRoleProvider(cfg *config.Config) *LocalRoleProvider { return provider } -// Role returns the current role -func (p *LocalRoleProvider) Role() Role { - return p.role -} - // LeaderInfo returns the leader info -func (p *LocalRoleProvider) LeaderInfo() *LeaderLockInfo { +func (p *LeaderInfoProvider) LeaderInfo() *LeaderLockInfo { if p.leaderInfo == nil { return nil } @@ -90,13 +52,8 @@ func (p *LocalRoleProvider) LeaderInfo() *LeaderLockInfo { return &info } -// IsLeader returns true if this instance is the leader -func (p *LocalRoleProvider) IsLeader() bool { - return p.role == RoleLeader -} - // buildLeaderInfo creates leader info for this instance -func (p *LocalRoleProvider) buildLeaderInfo() *LeaderLockInfo { +func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo { ip := getLocalIP() hostname, _ := os.Hostname() @@ -106,14 +63,23 @@ func (p *LocalRoleProvider) buildLeaderInfo() *LeaderLockInfo { baseUrl = fmt.Sprintf("http://%s:%d", ip, p.config.APIPort) } + // External WebDAV URL: for clients outside Docker (via nginx, potentially HTTPS) + // Uses baseUrl which may include custom scheme/port + webdavUrl := baseUrl + "/webdav" + + // Internal WebDAV URL: direct to Go WebDAV server (no nginx overhead) + // Uses hostname (Docker DNS resolvable) and HTTPPort (9000) + webdavUrlInternal := fmt.Sprintf("http://%s:%d/webdav", hostname, p.config.HTTPPort) + return &LeaderLockInfo{ - Hostname: hostname, - BaseUrl: baseUrl, - ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort), - RedisUrl: fmt.Sprintf("redis://%s:%d", ip, p.config.RedisPort), - WebdavUrl: fmt.Sprintf("http://%s:%d/webdav", ip, p.config.APIPort), - Timestamp: time.Now().UnixMilli(), - PID: os.Getpid(), + Hostname: hostname, + BaseUrl: baseUrl, + ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort), + RedisUrl: fmt.Sprintf("redis://%s:%d", ip, p.config.RedisPort), + WebdavUrl: webdavUrl, + WebdavUrlInternal: webdavUrlInternal, + Timestamp: time.Now().UnixMilli(), + PID: os.Getpid(), } } diff --git a/test/api_test.go b/test/api_test.go index 434ce3c..29d16f9 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -12,7 +12,6 @@ import ( // MockHealthResponse represents the expected health response structure type MockHealthResponse struct { Status string `json:"status"` - Role string `json:"role"` Redis bool `json:"redis"` Timestamp string `json:"timestamp"` } @@ -21,7 +20,6 @@ func TestHealthResponseFormat(t *testing.T) { // Test that the health response has the expected format response := MockHealthResponse{ Status: "ok", - Role: "leader", Redis: true, Timestamp: "2024-01-01T00:00:00Z", } @@ -37,7 +35,7 @@ func TestHealthResponseFormat(t *testing.T) { } // Check required fields - requiredFields := []string{"status", "role", "redis", "timestamp"} + requiredFields := []string{"status", "redis", "timestamp"} for _, field := range requiredFields { if _, exists := parsed[field]; !exists { t.Errorf("Missing required field: %s", field) @@ -56,7 +54,6 @@ func TestRouterSetup(t *testing.T) { {"/health", "GET"}, {"/status", "GET"}, {"/leader", "GET"}, - {"/role", "GET"}, {"/meta/{hash}", "GET"}, {"/meta/{hash}", "PUT"}, {"/meta/{hash}", "DELETE"}, @@ -83,7 +80,6 @@ func TestRouterSetup(t *testing.T) { {"GET", "/health"}, {"GET", "/status"}, {"GET", "/leader"}, - {"GET", "/role"}, {"GET", "/meta/testhash123"}, {"PUT", "/meta/testhash123"}, {"DELETE", "/meta/testhash123"},