Files
meta-core/internal/api/server.go
T
2026-11-04 11:43:01 +00:00

492 lines
19 KiB
Go

package api
import (
"context"
"fmt"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/mounts"
"github.com/metazla/meta-core/internal/reports"
"github.com/metazla/meta-core/internal/schema"
"github.com/metazla/meta-core/internal/storage"
"github.com/metazla/meta-core/internal/watcher"
"github.com/metazla/meta-core/internal/watchers"
"github.com/metazla/meta-core/internal/webdav"
)
// Server is the HTTP API server for meta-core
type Server struct {
config *config.Config
leaderProvider *leader.LeaderInfoProvider
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
mountPoller *mounts.Poller
mountStatsPoller *mounts.StatsPoller
watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
watchersManager *watchers.Manager
watchersPoller *watchers.Poller
watchersHandlers *watchers.Handlers
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler
// challenges backs proof-of-possession on reveal/delete. In-process, so a
// restart invalidates the ones in flight — see identity/challenge.go.
challenges *identity.ChallengeStore
// mesh is the UDP discovery node (beacon v2). nil when discovery
// is disabled or failed to bind, in which case /api/neighbors reports an
// empty list rather than erroring — a dashboard nav is decoration and
// must not take the API down with it.
mesh *meshdisco.Node
router *mux.Router
server *http.Server
// reports is the bug-report inbox (reports.go); nil = routes answer 503.
reports *reports.Store
// stats caches the keyspace-walk half of GET /api/stats so a polling
// dashboard never puts a multi-second store walk on the request path.
stats *statsCache
}
// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
// server's constructor signature (and its tests) stay untouched.
func (s *Server) SetMeshNode(n *meshdisco.Node) {
s.mesh = n
}
// NewServer creates a new API server
func NewServer(
cfg *config.Config,
leaderProvider *leader.LeaderInfoProvider,
stor *storage.Client,
) *Server {
s := &Server{
config: cfg,
leaderProvider: leaderProvider,
storage: stor,
router: mux.NewRouter(),
stats: &statsCache{},
}
// Initialize file watcher/scanner and dispatcher
var scanFunc mounts.ScanFunc
if cfg.EnableFileWatcher {
s.watcherDispatcher = watcher.NewDispatcher()
// Wire storage client for file:events publishing
if stor != nil {
s.watcherDispatcher.SetStorageClient(stor)
}
fileWatcher, err := watcher.NewWatcher(cfg, s.watcherDispatcher)
if err != nil {
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
} else {
s.fileWatcher = fileWatcher
// Wire storage so the watcher can persist (path,size,mtime,cid)
// tuples to the store as it computes midhashes.
if stor != nil {
fileWatcher.SetStorageClient(stor)
}
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
// Create scan function from watcher for mounts
scanFunc = fileWatcher.ScanMountPath
}
// Initialize watchers manager (polling-based file watching)
watchersManager, err := watchers.NewManager(cfg)
if err != nil {
log.Printf("[API] Warning: failed to initialize watchers manager: %v", err)
} else {
s.watchersManager = watchersManager
// Create watchers poller
if s.fileWatcher != nil {
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller)
// Hydrate the in-memory state registry from stored tuples so
// the first scan tick after boot can skip ComputeMidHash256
// for files whose (size, mtime) match.
configs := watchersManager.List()
roots := make([]watcher.WatcherRoot, 0, len(configs))
for _, c := range configs {
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
}
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
} else {
log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
}
}
}
}
// Initialize mounts manager
mountsManager, err := mounts.NewManager(cfg)
if err != nil {
log.Printf("[API] Warning: failed to initialize mounts manager: %v", err)
} else {
s.mountsManager = mountsManager
s.mountsHandlers = mounts.NewHandlers(mountsManager, scanFunc)
// Create and configure mount poller
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
mountsManager.SetPoller(s.mountPoller)
// IO stats poller — queries the local rclone RC API every few seconds
// and exposes live throughput per mount via the same status JSON.
// Stats are daemon-global (rclone limitation); see stats_poller.go.
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
mountsManager.SetStatsPoller(s.mountStatsPoller)
}
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
s.webdavHandler = webdav.NewHandler(cfg)
s.challenges = identity.NewChallengeStore()
// One-time migration: fold a pre-multi-account identity.json into the
// per-account keystore so existing single-identity hosts keep their uid.
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
log.Printf("[API] Warning: identity migration failed: %v", err)
} else if migrated {
log.Printf("[API] Migrated legacy identity.json into multi-account keystore")
}
s.setupRoutes()
return s
}
// setupRoutes configures all API routes
func (s *Server) setupRoutes() {
// Health and status
s.router.HandleFunc("/health", s.handleHealth).Methods("GET")
s.router.HandleFunc("/status", s.handleStatus).Methods("GET")
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
// Dashboard headline counters in one request. Cheap half live, keyspace
// walks served from a background-refreshed cache — see stats.go.
s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
// Metadata Editor API routes (must be before /meta/{hash} routes)
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleGone).Methods("POST")
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
// User Data Layer signing identity (see internal/identity). Auth-gated
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
// Signed claims (v2.0.0): the only metadata write path, the raw claims
// behind a record, and the stateless merge clients use for search hits.
s.router.HandleFunc("/api/claims", s.handlePostClaims).Methods("POST")
s.router.HandleFunc("/api/merge", s.handleMerge).Methods("POST")
s.router.HandleFunc("/api/meta/{cid}/claims", s.handleClaimsByCID).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/claims", s.handleClaimsByRID).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleGone).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("DELETE")
// Schema inference
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
// KV Browser API routes
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
s.router.HandleFunc("/api/kv/tree", s.handleKVTree).Methods("GET")
s.router.HandleFunc("/api/kv/search", s.handleKVSearch).Methods("GET")
s.router.HandleFunc("/api/kv/find", s.handleKVFind).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValueGet).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValuePut).Methods("PUT")
s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
// User Data Layer (UDL) routes — per-user signed records (My List,
// Continue Watching, ratings, profile). See internal/api/udl.go.
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordGet).Methods("GET")
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordPut).Methods("PUT")
s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
// Metadata operations - base endpoints
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
s.router.HandleFunc("/meta/{hash}", s.handleGone).Methods("PUT", "PATCH", "DELETE")
s.router.HandleFunc("/meta", s.handleListMeta).Methods("GET")
// Metadata operations - set operations (must be before property routes)
s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleGone).Methods("POST")
// Metadata operations - property-level (key can contain slashes)
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGetProperty).Methods("GET")
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGone).Methods("PUT", "DELETE")
// Data operations
s.router.HandleFunc("/data/{hash}/path", s.handleGetDataPath).Methods("GET")
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
// File operations (by CID)
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
// Register one file under /files/plugin by midhash — the watcher's
// mint + alias + file:events path for a single file. Registered
// unconditionally: it exists for boxes whose watcher is OFF.
s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST")
// File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched
// server-side, so clients answer "how many files, how big" without one
// read per hashId. ?summary=1 returns only {count, totalSize}.
s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET")
// Public metadata-by-CID resolver. Reverse-index lookup → full document.
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
// without going through Authelia.
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
// SSE event streams. The only way consumers see meta-core's change
// streams — they never touch the database. Auth-bypass
// applies but exposure should be inside-only (do NOT proxy through
// Caddy publicly). See docs/api-mediated-access.md "Auth".
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
// beacon v2: neighbours heard over UDP. Supersedes /api/services,
// which is still served above while consumers migrate.
s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
// Report inbox: bug reports / feedback forwarded by the client apps,
// read by the dashboard's Reports page. Perimeter-gated (reports.go).
s.setupReportRoutes()
// Mount management routes (if manager initialized)
if s.mountsHandlers != nil {
s.mountsHandlers.RegisterRoutes(s.router)
}
// File watcher routes (if watcher initialized)
if s.watcherHandlers != nil {
s.watcherHandlers.RegisterRoutes(s.router)
}
// Watchers management routes (if watchers initialized)
if s.watchersHandlers != nil {
s.watchersHandlers.RegisterRoutes(s.router)
}
// WebDAV handler (caching is handled by nginx proxy_cache)
if s.webdavHandler != nil {
s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
}
// Add middleware
s.router.Use(loggingMiddleware)
s.router.Use(corsMiddleware)
}
// Start starts the HTTP server
func (s *Server) Start() error {
addr := fmt.Sprintf("%s:%d", s.config.HTTPHost, s.config.HTTPPort)
s.server = &http.Server{
Addr: addr,
Handler: s.router,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
log.Printf("[API] Starting HTTP server on %s", addr)
go func() {
if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("[API] HTTP server error: %v", err)
}
}()
// Start watchers poller (if initialized)
if s.watchersPoller != nil {
if err := s.watchersPoller.Start(); err != nil {
log.Printf("[API] Warning: failed to start watchers poller: %v", err)
}
}
// Start mount poller (if initialized)
if s.mountPoller != nil {
if err := s.mountPoller.Start(); err != nil {
log.Printf("[API] Warning: failed to start mount poller: %v", err)
}
}
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
// mounts), so always-on rather than gated by config.
if s.mountStatsPoller != nil {
s.mountStatsPoller.Start()
}
// Schema: served from counters the write path keeps exact; nothing runs.
// Change events need no publisher either — the storage engine appends them
// in the same transaction as each change, and a consumer that wants the
// whole state asks /api/events/meta for "0" and gets a snapshot (see
// streamSnapshot). The boot republish of every field is gone with it.
if s.storage != nil && s.storage.IsConnected() {
s.schemaIndexer = schema.NewIndexer(s.storage.Store())
go func() {
started := time.Now()
if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
log.Printf("[API] Warning: search row rebuild failed: %v", err)
} else if n > 0 {
log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
}
}()
go s.logWriteStats()
}
if s.reports != nil {
go s.pruneReports()
}
return nil
}
// Stop gracefully stops the HTTP server
func (s *Server) Stop() error {
// Stop watchers poller
if s.watchersPoller != nil {
if err := s.watchersPoller.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop watchers poller: %v", err)
}
}
// Stop mount poller
if s.mountPoller != nil {
if err := s.mountPoller.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop mount poller: %v", err)
}
}
// Stop IO stats poller
if s.mountStatsPoller != nil {
s.mountStatsPoller.Stop()
}
if s.server == nil {
return nil
}
log.Println("[API] Stopping HTTP server...")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return s.server.Shutdown(ctx)
}
// logWriteStats reports, every 15 minutes, how many writes changed a value
// and how many found it already in place (kv skips those). The ratio is what
// the no-op skip buys on a permanent store that re-persists repeated hits.
func (s *Server) logWriteStats() {
ticker := time.NewTicker(15 * time.Minute)
defer ticker.Stop()
for range ticker.C {
store := s.storage.Store()
if store == nil {
return
}
changed, unchanged := store.WriteStats()
if total := changed + unchanged; total > 0 {
log.Printf("[Storage] writes since start: %d changed, %d unchanged skipped (%.0f%% no-op)", changed, unchanged, 100*float64(unchanged)/float64(total))
}
}
}
// statusRecorder captures the response status so loggingMiddleware can stay
// silent on success and only report failures. It forwards Flush/Unwrap so SSE
// and other streaming handlers keep working through the wrapper.
type statusRecorder struct {
http.ResponseWriter
status int
written bool
}
func (r *statusRecorder) WriteHeader(code int) {
if !r.written {
r.status = code
r.written = true
}
r.ResponseWriter.WriteHeader(code)
}
func (r *statusRecorder) Write(b []byte) (int, error) {
if !r.written {
r.status = http.StatusOK
r.written = true
}
return r.ResponseWriter.Write(b)
}
func (r *statusRecorder) Flush() {
if f, ok := r.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
// loggingMiddleware logs a request only when it fails (status >= 400), keeping
// the nominal case quiet. Successful requests are not logged.
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(rec, r)
if rec.status >= 400 {
log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
}
})
}
// corsMiddleware adds CORS headers
func corsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next.ServeHTTP(w, r)
})
}