new flat KV structure
This commit is contained in:
Pierre
Vendored
+73
@@ -0,0 +1,73 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Handlers provides HTTP handlers for cache management
|
||||
type Handlers struct {
|
||||
manager *Manager
|
||||
}
|
||||
|
||||
// NewHandlers creates new cache handlers
|
||||
func NewHandlers(manager *Manager) *Handlers {
|
||||
return &Handlers{
|
||||
manager: manager,
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterRoutes registers cache management routes
|
||||
func (h *Handlers) RegisterRoutes(router *mux.Router) {
|
||||
router.HandleFunc("/api/cache/stats", h.handleStats).Methods("GET")
|
||||
router.HandleFunc("/api/cache/clear", h.handleClear).Methods("POST")
|
||||
router.HandleFunc("/api/cache/invalidate", h.handleInvalidate).Methods("DELETE")
|
||||
}
|
||||
|
||||
// handleStats returns cache statistics
|
||||
func (h *Handlers) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
stats := h.manager.Stats()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(stats)
|
||||
}
|
||||
|
||||
// handleClear clears the entire cache
|
||||
func (h *Handlers) handleClear(w http.ResponseWriter, r *http.Request) {
|
||||
if err := h.manager.Clear(); err != nil {
|
||||
log.Printf("[CacheHandlers] Error clearing cache: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Cache cleared",
|
||||
})
|
||||
}
|
||||
|
||||
// handleInvalidate invalidates a specific path
|
||||
func (h *Handlers) handleInvalidate(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Query().Get("path")
|
||||
if path == "" {
|
||||
http.Error(w, "path query parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.manager.Invalidate(path); err != nil {
|
||||
log.Printf("[CacheHandlers] Error invalidating path %s: %v", path, err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Path invalidated",
|
||||
"path": path,
|
||||
})
|
||||
}
|
||||
Vendored
+242
@@ -0,0 +1,242 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LRUIndex maintains an in-memory LRU index of cache entries
|
||||
type LRUIndex struct {
|
||||
mu sync.RWMutex
|
||||
entries map[string]*list.Element // path -> list element
|
||||
lruList *list.List // LRU order (front = most recent)
|
||||
maxSize int64 // Maximum total size in bytes
|
||||
currSize int64 // Current total size in bytes
|
||||
|
||||
// Stats
|
||||
hitCount int64
|
||||
missCount int64
|
||||
evictionCount int64
|
||||
}
|
||||
|
||||
// lruEntry wraps CacheEntry for the linked list
|
||||
type lruEntry struct {
|
||||
entry *CacheEntry
|
||||
}
|
||||
|
||||
// NewLRUIndex creates a new LRU index
|
||||
func NewLRUIndex(maxSizeGB int) *LRUIndex {
|
||||
return &LRUIndex{
|
||||
entries: make(map[string]*list.Element),
|
||||
lruList: list.New(),
|
||||
maxSize: int64(maxSizeGB) * 1024 * 1024 * 1024, // GB to bytes
|
||||
}
|
||||
}
|
||||
|
||||
// Get retrieves a cache entry and updates its LRU position
|
||||
// Returns nil if not found
|
||||
func (idx *LRUIndex) Get(path string) *CacheEntry {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
elem, exists := idx.entries[path]
|
||||
if !exists {
|
||||
idx.missCount++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move to front (most recently used)
|
||||
idx.lruList.MoveToFront(elem)
|
||||
|
||||
// Update last accessed time
|
||||
entry := elem.Value.(*lruEntry).entry
|
||||
entry.LastAccessed = time.Now()
|
||||
|
||||
idx.hitCount++
|
||||
return entry
|
||||
}
|
||||
|
||||
// Put adds or updates a cache entry
|
||||
func (idx *LRUIndex) Put(entry *CacheEntry) {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
if elem, exists := idx.entries[entry.OriginalPath]; exists {
|
||||
// Update existing entry
|
||||
oldEntry := elem.Value.(*lruEntry).entry
|
||||
idx.currSize -= oldEntry.Size
|
||||
idx.currSize += entry.Size
|
||||
|
||||
elem.Value.(*lruEntry).entry = entry
|
||||
idx.lruList.MoveToFront(elem)
|
||||
} else {
|
||||
// Add new entry
|
||||
le := &lruEntry{entry: entry}
|
||||
elem := idx.lruList.PushFront(le)
|
||||
idx.entries[entry.OriginalPath] = elem
|
||||
idx.currSize += entry.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Remove removes a cache entry by path
|
||||
// Returns the removed entry or nil if not found
|
||||
func (idx *LRUIndex) Remove(path string) *CacheEntry {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
elem, exists := idx.entries[path]
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry := elem.Value.(*lruEntry).entry
|
||||
idx.lruList.Remove(elem)
|
||||
delete(idx.entries, path)
|
||||
idx.currSize -= entry.Size
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
// GetLRUEntry returns the least recently used entry without removing it
|
||||
// Returns nil if the index is empty
|
||||
func (idx *LRUIndex) GetLRUEntry() *CacheEntry {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
back := idx.lruList.Back()
|
||||
if back == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return back.Value.(*lruEntry).entry
|
||||
}
|
||||
|
||||
// PopLRUEntry removes and returns the least recently used entry
|
||||
// Returns nil if the index is empty
|
||||
func (idx *LRUIndex) PopLRUEntry() *CacheEntry {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
back := idx.lruList.Back()
|
||||
if back == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry := back.Value.(*lruEntry).entry
|
||||
idx.lruList.Remove(back)
|
||||
delete(idx.entries, entry.OriginalPath)
|
||||
idx.currSize -= entry.Size
|
||||
idx.evictionCount++
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
// GetExpiredEntries returns all entries that have exceeded the TTL
|
||||
func (idx *LRUIndex) GetExpiredEntries(ttl time.Duration) []*CacheEntry {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
var expired []*CacheEntry
|
||||
cutoff := time.Now().Add(-ttl)
|
||||
|
||||
for elem := idx.lruList.Back(); elem != nil; elem = elem.Prev() {
|
||||
entry := elem.Value.(*lruEntry).entry
|
||||
if entry.CachedAt.Before(cutoff) {
|
||||
expired = append(expired, entry)
|
||||
}
|
||||
}
|
||||
|
||||
return expired
|
||||
}
|
||||
|
||||
// Size returns the current total size of cached files
|
||||
func (idx *LRUIndex) Size() int64 {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
return idx.currSize
|
||||
}
|
||||
|
||||
// Count returns the number of entries in the index
|
||||
func (idx *LRUIndex) Count() int {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
return len(idx.entries)
|
||||
}
|
||||
|
||||
// MaxSize returns the maximum cache size
|
||||
func (idx *LRUIndex) MaxSize() int64 {
|
||||
return idx.maxSize
|
||||
}
|
||||
|
||||
// NeedsEviction returns true if the cache size exceeds the maximum
|
||||
func (idx *LRUIndex) NeedsEviction() bool {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
return idx.currSize > idx.maxSize
|
||||
}
|
||||
|
||||
// Stats returns cache statistics
|
||||
func (idx *LRUIndex) Stats() *CacheStats {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
total := idx.hitCount + idx.missCount
|
||||
var hitRate float64
|
||||
if total > 0 {
|
||||
hitRate = float64(idx.hitCount) / float64(total) * 100
|
||||
}
|
||||
|
||||
return &CacheStats{
|
||||
Enabled: true,
|
||||
TotalSize: idx.currSize,
|
||||
MaxSize: idx.maxSize,
|
||||
EntryCount: len(idx.entries),
|
||||
HitCount: idx.hitCount,
|
||||
MissCount: idx.missCount,
|
||||
EvictionCount: idx.evictionCount,
|
||||
HitRate: hitRate,
|
||||
}
|
||||
}
|
||||
|
||||
// GetAllEntries returns all cache entries (for persistence)
|
||||
func (idx *LRUIndex) GetAllEntries() map[string]*CacheEntry {
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
result := make(map[string]*CacheEntry, len(idx.entries))
|
||||
for path, elem := range idx.entries {
|
||||
result[path] = elem.Value.(*lruEntry).entry
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// LoadFromMap loads entries from a map (for loading persisted index)
|
||||
func (idx *LRUIndex) LoadFromMap(entries map[string]*CacheEntry) {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
// Clear existing entries
|
||||
idx.lruList.Init()
|
||||
idx.entries = make(map[string]*list.Element)
|
||||
idx.currSize = 0
|
||||
|
||||
// Load entries sorted by LastAccessed (oldest first for proper LRU order)
|
||||
// For simplicity, we just add them in arbitrary order - the access pattern will sort them
|
||||
for _, entry := range entries {
|
||||
le := &lruEntry{entry: entry}
|
||||
elem := idx.lruList.PushBack(le) // Add to back (oldest)
|
||||
idx.entries[entry.OriginalPath] = elem
|
||||
idx.currSize += entry.Size
|
||||
}
|
||||
}
|
||||
|
||||
// Clear removes all entries from the index
|
||||
func (idx *LRUIndex) Clear() {
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
idx.lruList.Init()
|
||||
idx.entries = make(map[string]*list.Element)
|
||||
idx.currSize = 0
|
||||
}
|
||||
Vendored
+166
@@ -0,0 +1,166 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
// EventsStream is the Redis Stream name for file events
|
||||
EventsStream = "file:events"
|
||||
|
||||
// ConsumerGroup is the consumer group name for cache invalidation
|
||||
ConsumerGroup = "cache-invalidator"
|
||||
|
||||
// ConsumerName identifies this consumer in the group
|
||||
ConsumerName = "meta-core-cache"
|
||||
)
|
||||
|
||||
// Invalidator subscribes to Redis Stream for file events and invalidates cache
|
||||
type Invalidator struct {
|
||||
manager *Manager
|
||||
redisClient *redis.Client
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
lastID string
|
||||
}
|
||||
|
||||
// NewInvalidator creates a new cache invalidator
|
||||
func NewInvalidator(manager *Manager, redisClient *redis.Client) *Invalidator {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Invalidator{
|
||||
manager: manager,
|
||||
redisClient: redisClient,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
lastID: "$", // Start from new messages
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts listening to the Redis Stream
|
||||
func (inv *Invalidator) Start() error {
|
||||
if inv.manager == nil || !inv.manager.Enabled() {
|
||||
log.Println("[CacheInvalidator] Cache disabled, not starting invalidator")
|
||||
return nil
|
||||
}
|
||||
|
||||
if inv.redisClient == nil {
|
||||
log.Println("[CacheInvalidator] Redis client not available, not starting invalidator")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to create consumer group (ignore error if already exists)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := inv.redisClient.XGroupCreateMkStream(ctx, EventsStream, ConsumerGroup, "$").Err()
|
||||
if err != nil && err.Error() != "BUSYGROUP Consumer Group name already exists" {
|
||||
// Log but don't fail - we can still use XREAD
|
||||
log.Printf("[CacheInvalidator] Note: consumer group creation: %v (using XREAD fallback)", err)
|
||||
}
|
||||
|
||||
inv.wg.Add(1)
|
||||
go inv.listen()
|
||||
|
||||
log.Println("[CacheInvalidator] Started listening on file:events stream")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the invalidator
|
||||
func (inv *Invalidator) Stop() error {
|
||||
log.Println("[CacheInvalidator] Stopping...")
|
||||
inv.cancel()
|
||||
inv.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// listen reads from the Redis Stream and processes events
|
||||
func (inv *Invalidator) listen() {
|
||||
defer inv.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-inv.ctx.Done():
|
||||
return
|
||||
default:
|
||||
inv.readEvents()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readEvents reads and processes a batch of events
|
||||
func (inv *Invalidator) readEvents() {
|
||||
ctx, cancel := context.WithTimeout(inv.ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Use XREAD with BLOCK for efficient waiting
|
||||
streams, err := inv.redisClient.XRead(ctx, &redis.XReadArgs{
|
||||
Streams: []string{EventsStream, inv.lastID},
|
||||
Count: 100,
|
||||
Block: 2 * time.Second,
|
||||
}).Result()
|
||||
|
||||
if err != nil {
|
||||
if err == context.Canceled || err == context.DeadlineExceeded || err == redis.Nil {
|
||||
return // Normal timeout or shutdown
|
||||
}
|
||||
log.Printf("[CacheInvalidator] Error reading stream: %v", err)
|
||||
time.Sleep(1 * time.Second) // Backoff on error
|
||||
return
|
||||
}
|
||||
|
||||
for _, stream := range streams {
|
||||
for _, msg := range stream.Messages {
|
||||
inv.processEvent(msg)
|
||||
inv.lastID = msg.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processEvent processes a single file event
|
||||
func (inv *Invalidator) processEvent(msg redis.XMessage) {
|
||||
eventType, _ := msg.Values["type"].(string)
|
||||
path, _ := msg.Values["path"].(string)
|
||||
|
||||
switch eventType {
|
||||
case "delete":
|
||||
// File deleted - invalidate cache
|
||||
if path != "" {
|
||||
log.Printf("[CacheInvalidator] Invalidating deleted file: %s", path)
|
||||
inv.manager.Invalidate(path)
|
||||
}
|
||||
|
||||
case "change":
|
||||
// File changed - invalidate cache
|
||||
if path != "" {
|
||||
log.Printf("[CacheInvalidator] Invalidating changed file: %s", path)
|
||||
inv.manager.Invalidate(path)
|
||||
}
|
||||
|
||||
case "rename":
|
||||
// File renamed - invalidate old path
|
||||
oldPath, _ := msg.Values["oldPath"].(string)
|
||||
if oldPath != "" {
|
||||
log.Printf("[CacheInvalidator] Invalidating renamed file (old): %s", oldPath)
|
||||
inv.manager.Invalidate(oldPath)
|
||||
}
|
||||
// Also invalidate new path in case it was cached before
|
||||
if path != "" {
|
||||
inv.manager.Invalidate(path)
|
||||
}
|
||||
|
||||
case "reset":
|
||||
// Full reset - clear entire cache
|
||||
log.Println("[CacheInvalidator] Reset event received - clearing cache")
|
||||
inv.manager.Clear()
|
||||
|
||||
case "add":
|
||||
// New file - nothing to invalidate
|
||||
// (file wouldn't be in cache yet)
|
||||
}
|
||||
}
|
||||
Vendored
+494
@@ -0,0 +1,494 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
)
|
||||
|
||||
// Manager handles caching operations for WebDAV files
|
||||
type Manager struct {
|
||||
config *config.Config
|
||||
index *LRUIndex
|
||||
filesPath string
|
||||
cachePath string
|
||||
ttl time.Duration
|
||||
enabled bool
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
cleanupWg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewManager creates a new cache manager
|
||||
func NewManager(cfg *config.Config) (*Manager, error) {
|
||||
if !cfg.CacheEnabled {
|
||||
log.Println("[Cache] Caching is disabled")
|
||||
return &Manager{
|
||||
config: cfg,
|
||||
enabled: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Ensure cache directory exists
|
||||
if err := os.MkdirAll(cfg.CacheDir(), 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create cache directory: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
m := &Manager{
|
||||
config: cfg,
|
||||
index: NewLRUIndex(cfg.CacheMaxSizeGB),
|
||||
filesPath: cfg.FilesPath,
|
||||
cachePath: cfg.CacheDir(),
|
||||
ttl: time.Duration(cfg.CacheTTLSeconds) * time.Second,
|
||||
enabled: true,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
|
||||
// Load existing index from disk
|
||||
if err := m.loadIndex(); err != nil {
|
||||
log.Printf("[Cache] Warning: failed to load index: %v (starting fresh)", err)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Start starts the cache manager (cleanup goroutine)
|
||||
func (m *Manager) Start() error {
|
||||
if !m.enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("[Cache] Starting cache manager (max: %dGB, ttl: %v)",
|
||||
m.config.CacheMaxSizeGB, m.ttl)
|
||||
|
||||
// Start cleanup goroutine
|
||||
m.cleanupWg.Add(1)
|
||||
go m.cleanupLoop()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the cache manager
|
||||
func (m *Manager) Stop() error {
|
||||
if !m.enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[Cache] Stopping cache manager...")
|
||||
m.cancel()
|
||||
m.cleanupWg.Wait()
|
||||
|
||||
// Persist index
|
||||
if err := m.saveIndex(); err != nil {
|
||||
log.Printf("[Cache] Warning: failed to save index: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enabled returns whether caching is enabled
|
||||
func (m *Manager) Enabled() bool {
|
||||
return m.enabled
|
||||
}
|
||||
|
||||
// Get retrieves a file from cache or fetches it from origin
|
||||
// Returns the cached file path and content type, or an error
|
||||
func (m *Manager) Get(path string) (string, string, error) {
|
||||
if !m.enabled {
|
||||
// Return original path if caching disabled
|
||||
return filepath.Join(m.filesPath, path), "", nil
|
||||
}
|
||||
|
||||
// Normalize path (remove leading slash if present)
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
|
||||
// Check cache
|
||||
entry := m.index.Get(path)
|
||||
if entry != nil {
|
||||
// Verify cached file still exists
|
||||
if _, err := os.Stat(entry.CachePath); err == nil {
|
||||
log.Printf("[Cache] HIT: %s", path)
|
||||
return entry.CachePath, entry.ContentType, nil
|
||||
}
|
||||
// Cached file missing, remove from index
|
||||
m.index.Remove(path)
|
||||
}
|
||||
|
||||
log.Printf("[Cache] MISS: %s", path)
|
||||
|
||||
// Fetch from origin and cache
|
||||
cachedPath, contentType, err := m.fetchAndCache(path)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return cachedPath, contentType, nil
|
||||
}
|
||||
|
||||
// GetOriginalPath returns the original file path (for when caching is bypassed)
|
||||
func (m *Manager) GetOriginalPath(path string) string {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
return filepath.Join(m.filesPath, path)
|
||||
}
|
||||
|
||||
// fetchAndCache fetches a file from origin and caches it
|
||||
func (m *Manager) fetchAndCache(path string) (string, string, error) {
|
||||
originPath := filepath.Join(m.filesPath, path)
|
||||
|
||||
// Check if origin file exists
|
||||
info, err := os.Stat(originPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Skip caching directories
|
||||
if info.IsDir() {
|
||||
return originPath, "", nil
|
||||
}
|
||||
|
||||
// Generate cache path using SHA256 of original path with 2-char sharding
|
||||
cachePath := m.getCachePath(path)
|
||||
|
||||
// Ensure cache subdirectory exists
|
||||
cacheDir := filepath.Dir(cachePath)
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return "", "", fmt.Errorf("failed to create cache dir: %w", err)
|
||||
}
|
||||
|
||||
// Copy file to cache
|
||||
src, err := os.Open(originPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := os.Create(cachePath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
_, err = io.Copy(dst, src)
|
||||
if err != nil {
|
||||
os.Remove(cachePath)
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Detect content type
|
||||
contentType := detectContentType(originPath)
|
||||
|
||||
// Create cache entry
|
||||
entry := &CacheEntry{
|
||||
OriginalPath: path,
|
||||
CachePath: cachePath,
|
||||
Size: info.Size(),
|
||||
CachedAt: time.Now(),
|
||||
LastAccessed: time.Now(),
|
||||
ContentType: contentType,
|
||||
}
|
||||
|
||||
// Add to index
|
||||
m.index.Put(entry)
|
||||
|
||||
// Trigger eviction if needed
|
||||
m.evictIfNeeded()
|
||||
|
||||
log.Printf("[Cache] Cached: %s (%d bytes)", path, info.Size())
|
||||
|
||||
return cachePath, contentType, nil
|
||||
}
|
||||
|
||||
// getCachePath generates a cache path using SHA256 with 2-char directory sharding
|
||||
func (m *Manager) getCachePath(path string) string {
|
||||
hash := sha256.Sum256([]byte(path))
|
||||
hexHash := hex.EncodeToString(hash[:])
|
||||
|
||||
// Use first 2 characters for sharding
|
||||
shard := hexHash[:2]
|
||||
filename := hexHash[2:]
|
||||
|
||||
return filepath.Join(m.cachePath, shard, filename)
|
||||
}
|
||||
|
||||
// Invalidate removes a specific path from the cache
|
||||
func (m *Manager) Invalidate(path string) error {
|
||||
if !m.enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
|
||||
entry := m.index.Remove(path)
|
||||
if entry == nil {
|
||||
return nil // Not in cache
|
||||
}
|
||||
|
||||
// Remove cached file
|
||||
if err := os.Remove(entry.CachePath); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[Cache] Warning: failed to remove cached file: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("[Cache] Invalidated: %s", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear removes all entries from the cache
|
||||
func (m *Manager) Clear() error {
|
||||
if !m.enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Get all entries before clearing
|
||||
entries := m.index.GetAllEntries()
|
||||
|
||||
// Clear index
|
||||
m.index.Clear()
|
||||
|
||||
// Remove all cached files
|
||||
for _, entry := range entries {
|
||||
if err := os.Remove(entry.CachePath); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[Cache] Warning: failed to remove cached file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove empty shard directories
|
||||
files, err := os.ReadDir(m.cachePath)
|
||||
if err == nil {
|
||||
for _, f := range files {
|
||||
if f.IsDir() && len(f.Name()) == 2 {
|
||||
dirPath := filepath.Join(m.cachePath, f.Name())
|
||||
os.Remove(dirPath) // Only removes if empty
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[Cache] Cleared %d entries", len(entries))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stats returns cache statistics
|
||||
func (m *Manager) Stats() *CacheStats {
|
||||
if !m.enabled {
|
||||
return &CacheStats{Enabled: false}
|
||||
}
|
||||
|
||||
stats := m.index.Stats()
|
||||
stats.TTLSeconds = m.config.CacheTTLSeconds
|
||||
return stats
|
||||
}
|
||||
|
||||
// evictIfNeeded evicts LRU entries until cache is under max size
|
||||
func (m *Manager) evictIfNeeded() {
|
||||
for m.index.NeedsEviction() {
|
||||
entry := m.index.PopLRUEntry()
|
||||
if entry == nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Remove cached file
|
||||
if err := os.Remove(entry.CachePath); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[Cache] Warning: failed to remove evicted file: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("[Cache] Evicted (LRU): %s", entry.OriginalPath)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupLoop periodically cleans up expired entries
|
||||
func (m *Manager) cleanupLoop() {
|
||||
defer m.cleanupWg.Done()
|
||||
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.cleanupExpired()
|
||||
m.evictIfNeeded()
|
||||
|
||||
// Periodically save index
|
||||
if err := m.saveIndex(); err != nil {
|
||||
log.Printf("[Cache] Warning: failed to save index: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupExpired removes entries that have exceeded TTL
|
||||
func (m *Manager) cleanupExpired() {
|
||||
expired := m.index.GetExpiredEntries(m.ttl)
|
||||
|
||||
for _, entry := range expired {
|
||||
m.index.Remove(entry.OriginalPath)
|
||||
|
||||
if err := os.Remove(entry.CachePath); err != nil && !os.IsNotExist(err) {
|
||||
log.Printf("[Cache] Warning: failed to remove expired file: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("[Cache] Expired (TTL): %s", entry.OriginalPath)
|
||||
}
|
||||
|
||||
if len(expired) > 0 {
|
||||
log.Printf("[Cache] Cleaned up %d expired entries", len(expired))
|
||||
}
|
||||
}
|
||||
|
||||
// loadIndex loads the cache index from disk
|
||||
func (m *Manager) loadIndex() error {
|
||||
indexPath := m.config.CacheIndexPath()
|
||||
|
||||
data, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil // No index file yet
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var idx CacheIndex
|
||||
if err := json.Unmarshal(data, &idx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate and load entries
|
||||
validEntries := make(map[string]*CacheEntry)
|
||||
for path, entry := range idx.Entries {
|
||||
// Verify cached file still exists
|
||||
if _, err := os.Stat(entry.CachePath); err == nil {
|
||||
validEntries[path] = entry
|
||||
}
|
||||
}
|
||||
|
||||
m.index.LoadFromMap(validEntries)
|
||||
|
||||
log.Printf("[Cache] Loaded %d entries from index", len(validEntries))
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveIndex persists the cache index to disk
|
||||
func (m *Manager) saveIndex() error {
|
||||
indexPath := m.config.CacheIndexPath()
|
||||
|
||||
idx := &CacheIndex{
|
||||
Version: 1,
|
||||
Entries: m.index.GetAllEntries(),
|
||||
Stats: m.index.Stats(),
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(idx, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write atomically using temp file
|
||||
tmpPath := indexPath + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpPath, indexPath)
|
||||
}
|
||||
|
||||
// detectContentType detects the content type based on file extension
|
||||
func detectContentType(path string) string {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
|
||||
// Common media types
|
||||
switch ext {
|
||||
case ".mp4":
|
||||
return "video/mp4"
|
||||
case ".mkv":
|
||||
return "video/x-matroska"
|
||||
case ".avi":
|
||||
return "video/x-msvideo"
|
||||
case ".webm":
|
||||
return "video/webm"
|
||||
case ".mov":
|
||||
return "video/quicktime"
|
||||
case ".mp3":
|
||||
return "audio/mpeg"
|
||||
case ".flac":
|
||||
return "audio/flac"
|
||||
case ".wav":
|
||||
return "audio/wav"
|
||||
case ".aac":
|
||||
return "audio/aac"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
case ".srt":
|
||||
return "text/plain"
|
||||
case ".vtt":
|
||||
return "text/vtt"
|
||||
case ".ass", ".ssa":
|
||||
return "text/x-ssa"
|
||||
case ".nfo":
|
||||
return "text/plain"
|
||||
case ".txt":
|
||||
return "text/plain"
|
||||
case ".json":
|
||||
return "application/json"
|
||||
case ".xml":
|
||||
return "application/xml"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
// ServeFile serves a file through the cache, handling HTTP response
|
||||
func (m *Manager) ServeFile(w http.ResponseWriter, r *http.Request, path string) error {
|
||||
cachedPath, contentType, err := m.Get(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open the file (cached or original)
|
||||
file, err := os.Open(cachedPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set content type if detected
|
||||
if contentType != "" {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
|
||||
// Use http.ServeContent for proper range support
|
||||
http.ServeContent(w, r, filepath.Base(path), stat.ModTime(), file)
|
||||
return nil
|
||||
}
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// CacheEntry represents a cached file
|
||||
type CacheEntry struct {
|
||||
OriginalPath string `json:"originalPath"` // Relative to /files/
|
||||
CachePath string `json:"cachePath"` // Path in /meta-core/cache/
|
||||
Size int64 `json:"size"`
|
||||
CachedAt time.Time `json:"cachedAt"`
|
||||
LastAccessed time.Time `json:"lastAccessed"`
|
||||
ContentType string `json:"contentType"`
|
||||
}
|
||||
|
||||
// CacheStats provides cache statistics
|
||||
type CacheStats struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
TotalSize int64 `json:"totalSize"` // Current cache size in bytes
|
||||
MaxSize int64 `json:"maxSize"` // Maximum cache size in bytes
|
||||
EntryCount int `json:"entryCount"` // Number of cached entries
|
||||
HitCount int64 `json:"hitCount"` // Number of cache hits
|
||||
MissCount int64 `json:"missCount"` // Number of cache misses
|
||||
EvictionCount int64 `json:"evictionCount"` // Number of evictions
|
||||
HitRate float64 `json:"hitRate"` // Hit rate percentage
|
||||
TTLSeconds int `json:"ttlSeconds"` // TTL in seconds
|
||||
}
|
||||
|
||||
// CacheIndex is the persistent index structure
|
||||
type CacheIndex struct {
|
||||
Version int `json:"version"`
|
||||
Entries map[string]*CacheEntry `json:"entries"` // keyed by OriginalPath
|
||||
Stats *CacheStats `json:"stats"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// NewCacheIndex creates a new empty cache index
|
||||
func NewCacheIndex() *CacheIndex {
|
||||
return &CacheIndex{
|
||||
Version: 1,
|
||||
Entries: make(map[string]*CacheEntry),
|
||||
Stats: &CacheStats{
|
||||
Enabled: true,
|
||||
},
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user