move feature from meta-sort to meta-core
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// KVInfoResponse is the response for GET /api/kv/info
|
||||
type KVInfoResponse struct {
|
||||
Prefix string `json:"prefix"`
|
||||
FileCount int `json:"fileCount"`
|
||||
KeyCount int `json:"keyCount"`
|
||||
TotalSize int64 `json:"totalSize"`
|
||||
MemoryUsage string `json:"memoryUsage"`
|
||||
}
|
||||
|
||||
// KVKeysResponse is the response for GET /api/kv/keys
|
||||
type KVKeysResponse struct {
|
||||
Keys []string `json:"keys"`
|
||||
Cursor string `json:"cursor"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
// KVKeyValueResponse is the response for GET /api/kv/key/{key}
|
||||
type KVKeyValueResponse struct {
|
||||
Key string `json:"key"`
|
||||
Type string `json:"type"`
|
||||
Value map[string]string `json:"value"`
|
||||
}
|
||||
|
||||
// handleKVInfo handles GET /api/kv/info
|
||||
func (s *Server) handleKVInfo(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// Get all hash IDs to count files
|
||||
hashIDs, err := s.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get memory usage
|
||||
memoryUsage, err := s.storage.GetMemoryInfo()
|
||||
if err != nil {
|
||||
memoryUsage = "N/A"
|
||||
}
|
||||
|
||||
// Calculate total size from sizeByte metadata (if available)
|
||||
var totalSize int64
|
||||
for _, hashID := range hashIDs {
|
||||
sizeStr, err := s.storage.GetProperty(hashID, "sizeByte")
|
||||
if err == nil && sizeStr != "" {
|
||||
if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil {
|
||||
totalSize += size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response := KVInfoResponse{
|
||||
Prefix: "file:",
|
||||
FileCount: len(hashIDs),
|
||||
KeyCount: len(hashIDs),
|
||||
TotalSize: totalSize,
|
||||
MemoryUsage: memoryUsage,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleKVKeys handles GET /api/kv/keys
|
||||
func (s *Server) handleKVKeys(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse query parameters
|
||||
cursorStr := r.URL.Query().Get("cursor")
|
||||
countStr := r.URL.Query().Get("count")
|
||||
|
||||
cursor := 0
|
||||
if cursorStr != "" {
|
||||
if c, err := strconv.Atoi(cursorStr); err == nil {
|
||||
cursor = c
|
||||
}
|
||||
}
|
||||
|
||||
count := 50
|
||||
if countStr != "" {
|
||||
if c, err := strconv.Atoi(countStr); err == nil && c > 0 {
|
||||
count = c
|
||||
}
|
||||
}
|
||||
|
||||
// Get all hash IDs
|
||||
allHashIDs, err := s.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Simple offset-based pagination
|
||||
var keys []string
|
||||
end := cursor + count
|
||||
if end > len(allHashIDs) {
|
||||
end = len(allHashIDs)
|
||||
}
|
||||
|
||||
if cursor < len(allHashIDs) {
|
||||
for _, id := range allHashIDs[cursor:end] {
|
||||
keys = append(keys, "file:"+id)
|
||||
}
|
||||
}
|
||||
|
||||
nextCursor := cursor + len(keys)
|
||||
hasMore := nextCursor < len(allHashIDs)
|
||||
|
||||
response := KVKeysResponse{
|
||||
Keys: keys,
|
||||
Cursor: strconv.Itoa(nextCursor),
|
||||
HasMore: hasMore,
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
response.Cursor = "0"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleKVGetKey handles GET /api/kv/key/{key}
|
||||
func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
key := vars["key"]
|
||||
|
||||
if key == "" {
|
||||
writeError(w, http.StatusBadRequest, "key is required")
|
||||
return
|
||||
}
|
||||
|
||||
// URL decode the key (in case it contains encoded characters)
|
||||
decodedKey, err := url.PathUnescape(key)
|
||||
if err != nil {
|
||||
decodedKey = key
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// Extract hash ID from key (format: file:{hashId})
|
||||
if !strings.HasPrefix(decodedKey, "file:") {
|
||||
writeError(w, http.StatusBadRequest, "invalid key format, expected: file:{hashId}")
|
||||
return
|
||||
}
|
||||
|
||||
hashID := strings.TrimPrefix(decodedKey, "file:")
|
||||
|
||||
// Get metadata for this hash ID
|
||||
metadata, err := s.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
writeError(w, http.StatusNotFound, "key not found")
|
||||
return
|
||||
}
|
||||
|
||||
response := KVKeyValueResponse{
|
||||
Key: decodedKey,
|
||||
Type: "hash",
|
||||
Value: metadata,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// HashIDsResponse is the response for GET /api/metadata/hash-ids
|
||||
type HashIDsResponse struct {
|
||||
HashIds []string `json:"hashIds"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// MetadataListItem represents a file in the metadata list
|
||||
type MetadataListItem struct {
|
||||
HashID string `json:"hashId"`
|
||||
Title string `json:"title,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
FilePath string `json:"filePath,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Year string `json:"year,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataListResponse is the response for GET /api/metadata/list
|
||||
type MetadataListResponse struct {
|
||||
Items []MetadataListItem `json:"items"`
|
||||
Count int `json:"count"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// MetadataSearchRequest is the request body for POST /api/metadata/search
|
||||
type MetadataSearchRequest struct {
|
||||
Query string `json:"query,omitempty"`
|
||||
HashID string `json:"hashId,omitempty"`
|
||||
Property string `json:"property,omitempty"`
|
||||
PropertyValue string `json:"propertyValue,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataSearchResult is a single search result
|
||||
type MetadataSearchResult struct {
|
||||
HashID string `json:"hashId"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
|
||||
// MetadataSearchResponse is the response for POST /api/metadata/search
|
||||
type MetadataSearchResponse struct {
|
||||
Results []MetadataSearchResult `json:"results"`
|
||||
Count int `json:"count"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// MetadataBatchUpdateRequest is the request body for POST /api/metadata/batch
|
||||
type MetadataBatchUpdateRequest struct {
|
||||
Updates []MetadataBatchItem `json:"updates"`
|
||||
}
|
||||
|
||||
// MetadataBatchItem is a single item in a batch update
|
||||
type MetadataBatchItem struct {
|
||||
HashID string `json:"hashId"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataBatchResult is a single result from a batch update
|
||||
type MetadataBatchResult struct {
|
||||
HashID string `json:"hashId"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataBatchResponse is the response for POST /api/metadata/batch
|
||||
type MetadataBatchResponse struct {
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Success int `json:"success"`
|
||||
Errors int `json:"errors"`
|
||||
Results []MetadataBatchResult `json:"results"`
|
||||
}
|
||||
|
||||
// PropertyUpdateRequest is the request body for PUT /api/metadata/{hashId}/property
|
||||
type PropertyUpdateRequest struct {
|
||||
Property string `json:"property"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// handleGetHashIds handles GET /api/metadata/hash-ids
|
||||
func (s *Server) handleGetHashIds(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
hashIDs, err := s.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
response := HashIDsResponse{
|
||||
HashIds: hashIDs,
|
||||
Count: len(hashIDs),
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleListMetadata handles GET /api/metadata/list
|
||||
func (s *Server) handleListMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse pagination parameters
|
||||
offsetStr := r.URL.Query().Get("offset")
|
||||
limitStr := r.URL.Query().Get("limit")
|
||||
|
||||
offset := 0
|
||||
if offsetStr != "" {
|
||||
if o, err := strconv.Atoi(offsetStr); err == nil {
|
||||
offset = o
|
||||
}
|
||||
}
|
||||
|
||||
limit := 100
|
||||
if limitStr != "" {
|
||||
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
|
||||
// Get all hash IDs
|
||||
allHashIDs, err := s.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
total := len(allHashIDs)
|
||||
|
||||
// Apply pagination
|
||||
end := offset + limit
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
|
||||
var items []MetadataListItem
|
||||
if offset < total {
|
||||
for _, hashID := range allHashIDs[offset:end] {
|
||||
metadata, err := s.storage.GetMetadataFlat(hashID)
|
||||
if err != nil || metadata == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
item := MetadataListItem{
|
||||
HashID: hashID,
|
||||
Title: metadata["title"],
|
||||
FileName: metadata["fileName"],
|
||||
FilePath: metadata["filePath"],
|
||||
Type: metadata["type"],
|
||||
Year: metadata["year"],
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
|
||||
response := MetadataListResponse{
|
||||
Items: items,
|
||||
Count: len(items),
|
||||
Total: total,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleGetMetadataByHashId handles GET /api/metadata/{hashId}
|
||||
func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
hashID := vars["hashId"]
|
||||
|
||||
if hashID == "" {
|
||||
writeError(w, http.StatusBadRequest, "hash ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
metadata, err := s.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
writeError(w, http.StatusNotFound, "metadata not found")
|
||||
return
|
||||
}
|
||||
|
||||
response := MetadataResponse{
|
||||
HashID: hashID,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId}
|
||||
func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
hashID := vars["hashId"]
|
||||
|
||||
if hashID == "" {
|
||||
writeError(w, http.StatusBadRequest, "hash ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
var metadata map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
// Exclude internal processing status field
|
||||
delete(metadata, "processingStatus")
|
||||
|
||||
if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"hashId": hashID,
|
||||
"message": "Metadata updated successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId}
|
||||
func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
hashID := vars["hashId"]
|
||||
|
||||
if hashID == "" {
|
||||
writeError(w, http.StatusBadRequest, "hash ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
deletedCount, err := s.storage.DeleteMetadata(hashID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"hashId": hashID,
|
||||
"deletedKeys": deletedCount,
|
||||
})
|
||||
}
|
||||
|
||||
// handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X
|
||||
func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
hashID := vars["hashId"]
|
||||
property := r.URL.Query().Get("property")
|
||||
|
||||
if hashID == "" {
|
||||
writeError(w, http.StatusBadRequest, "hash ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if property == "" {
|
||||
writeError(w, http.StatusBadRequest, "property parameter is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
value, err := s.storage.GetProperty(hashID, property)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if value == "" {
|
||||
writeError(w, http.StatusNotFound, "property not found")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"hashId": hashID,
|
||||
"property": property,
|
||||
"value": value,
|
||||
})
|
||||
}
|
||||
|
||||
// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property
|
||||
func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
hashID := vars["hashId"]
|
||||
|
||||
if hashID == "" {
|
||||
writeError(w, http.StatusBadRequest, "hash ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
var req PropertyUpdateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Property == "" {
|
||||
writeError(w, http.StatusBadRequest, "property is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"hashId": hashID,
|
||||
"property": req.Property,
|
||||
"value": req.Value,
|
||||
"message": "Property updated successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// handleSearchMetadata handles POST /api/metadata/search
|
||||
func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
var req MetadataSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Limit == 0 {
|
||||
req.Limit = 100
|
||||
}
|
||||
|
||||
// If hashId is provided, do exact match
|
||||
if req.HashID != "" {
|
||||
metadata, err := s.storage.GetMetadataFlat(req.HashID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if metadata != nil {
|
||||
writeJSON(w, http.StatusOK, MetadataSearchResponse{
|
||||
Results: []MetadataSearchResult{{HashID: req.HashID, Metadata: metadata}},
|
||||
Count: 1,
|
||||
Total: 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, MetadataSearchResponse{
|
||||
Results: []MetadataSearchResult{},
|
||||
Count: 0,
|
||||
Total: 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Get all hash IDs
|
||||
allHashIDs, err := s.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var results []MetadataSearchResult
|
||||
total := len(allHashIDs)
|
||||
|
||||
// Search fields for query
|
||||
searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
|
||||
|
||||
for _, hashID := range allHashIDs {
|
||||
if len(results) >= req.Limit {
|
||||
break
|
||||
}
|
||||
|
||||
metadata, err := s.storage.GetMetadataFlat(hashID)
|
||||
if err != nil || metadata == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
matches := false
|
||||
|
||||
// Property search
|
||||
if req.Property != "" && req.PropertyValue != "" {
|
||||
value, ok := metadata[req.Property]
|
||||
if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) {
|
||||
matches = true
|
||||
}
|
||||
}
|
||||
|
||||
// General query search
|
||||
if req.Query != "" && req.Property == "" {
|
||||
queryLower := strings.ToLower(req.Query)
|
||||
|
||||
// Check if query matches hashId
|
||||
if strings.Contains(strings.ToLower(hashID), queryLower) {
|
||||
matches = true
|
||||
}
|
||||
|
||||
// Check search fields
|
||||
if !matches {
|
||||
for _, field := range searchFields {
|
||||
if value, ok := metadata[field]; ok {
|
||||
if strings.Contains(strings.ToLower(value), queryLower) {
|
||||
matches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no query or property filter, return all
|
||||
if req.Query == "" && req.Property == "" {
|
||||
matches = true
|
||||
}
|
||||
|
||||
if matches {
|
||||
results = append(results, MetadataSearchResult{
|
||||
HashID: hashID,
|
||||
Metadata: metadata,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, MetadataSearchResponse{
|
||||
Results: results,
|
||||
Count: len(results),
|
||||
Total: total,
|
||||
})
|
||||
}
|
||||
|
||||
// handleBatchUpdate handles POST /api/metadata/batch
|
||||
func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
var req MetadataBatchUpdateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.Updates) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "updates array is required")
|
||||
return
|
||||
}
|
||||
|
||||
var results []MetadataBatchResult
|
||||
successCount := 0
|
||||
errorCount := 0
|
||||
|
||||
for _, update := range req.Updates {
|
||||
if update.HashID == "" {
|
||||
results = append(results, MetadataBatchResult{
|
||||
HashID: "unknown",
|
||||
Status: "error",
|
||||
Error: "Missing hash ID",
|
||||
})
|
||||
errorCount++
|
||||
continue
|
||||
}
|
||||
|
||||
var updateErr error
|
||||
|
||||
// Update complete metadata
|
||||
if update.Metadata != nil {
|
||||
// Exclude internal processing status
|
||||
delete(update.Metadata, "processingStatus")
|
||||
updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata)
|
||||
}
|
||||
|
||||
// Update individual properties
|
||||
if updateErr == nil && update.Properties != nil {
|
||||
for property, value := range update.Properties {
|
||||
if err := s.storage.SetProperty(update.HashID, property, value); err != nil {
|
||||
updateErr = err
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if updateErr != nil {
|
||||
results = append(results, MetadataBatchResult{
|
||||
HashID: update.HashID,
|
||||
Status: "error",
|
||||
Error: updateErr.Error(),
|
||||
})
|
||||
errorCount++
|
||||
} else {
|
||||
results = append(results, MetadataBatchResult{
|
||||
HashID: update.HashID,
|
||||
Status: "ok",
|
||||
})
|
||||
successCount++
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, MetadataBatchResponse{
|
||||
Status: "completed",
|
||||
Total: len(results),
|
||||
Success: successCount,
|
||||
Errors: errorCount,
|
||||
Results: results,
|
||||
})
|
||||
}
|
||||
|
||||
// handleClearMetadata handles POST /api/metadata/clear
|
||||
func (s *Server) handleClearMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
deletedCount, err := s.storage.ClearAllMetadata()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Metadata cleared",
|
||||
"deletedCount": deletedCount,
|
||||
})
|
||||
}
|
||||
+80
-6
@@ -11,17 +11,24 @@ import (
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
"github.com/metazla/meta-core/internal/discovery"
|
||||
"github.com/metazla/meta-core/internal/leader"
|
||||
"github.com/metazla/meta-core/internal/mounts"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
"github.com/metazla/meta-core/internal/watcher"
|
||||
)
|
||||
|
||||
// Server is the HTTP API server for meta-core
|
||||
type Server struct {
|
||||
config *config.Config
|
||||
election *leader.Election
|
||||
discovery *discovery.Service
|
||||
storage *storage.Client
|
||||
router *mux.Router
|
||||
server *http.Server
|
||||
config *config.Config
|
||||
election *leader.Election
|
||||
discovery *discovery.Service
|
||||
storage *storage.Client
|
||||
mountsManager *mounts.Manager
|
||||
mountsHandlers *mounts.Handlers
|
||||
watcherDispatcher *watcher.Dispatcher
|
||||
fileWatcher *watcher.Watcher
|
||||
watcherHandlers *watcher.Handlers
|
||||
router *mux.Router
|
||||
server *http.Server
|
||||
}
|
||||
|
||||
// NewServer creates a new API server
|
||||
@@ -39,6 +46,27 @@ func NewServer(
|
||||
router: mux.NewRouter(),
|
||||
}
|
||||
|
||||
// Initialize mounts manager
|
||||
mountsManager, err := mounts.NewManager(cfg)
|
||||
if err != nil {
|
||||
log.Printf("[API] Warning: failed to initialize mounts manager: %v", err)
|
||||
} else {
|
||||
s.mountsManager = mountsManager
|
||||
s.mountsHandlers = mounts.NewHandlers(mountsManager)
|
||||
}
|
||||
|
||||
// Initialize file watcher (if enabled)
|
||||
if cfg.EnableFileWatcher && len(cfg.WatchFolderList) > 0 {
|
||||
s.watcherDispatcher = watcher.NewDispatcher()
|
||||
fileWatcher, err := watcher.NewWatcher(cfg, s.watcherDispatcher)
|
||||
if err != nil {
|
||||
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
||||
} else {
|
||||
s.fileWatcher = fileWatcher
|
||||
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
||||
}
|
||||
}
|
||||
|
||||
s.setupRoutes()
|
||||
return s
|
||||
}
|
||||
@@ -51,6 +79,23 @@ func (s *Server) setupRoutes() {
|
||||
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
|
||||
s.router.HandleFunc("/role", s.handleRole).Methods("GET")
|
||||
|
||||
// Metadata Editor API routes (must be before /meta/{hash} routes)
|
||||
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
|
||||
s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
|
||||
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
|
||||
|
||||
// KV Browser API routes
|
||||
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
|
||||
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
|
||||
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
|
||||
|
||||
// Metadata operations - base endpoints
|
||||
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
|
||||
s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
|
||||
@@ -78,6 +123,21 @@ func (s *Server) setupRoutes() {
|
||||
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
|
||||
s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
|
||||
|
||||
// Mount management routes (if manager initialized)
|
||||
if s.mountsHandlers != nil {
|
||||
s.mountsHandlers.RegisterRoutes(s.router)
|
||||
log.Println("[API] Mount management routes registered at /api/mounts/*")
|
||||
}
|
||||
|
||||
log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
|
||||
log.Println("[API] KV Browser routes registered at /api/kv/*")
|
||||
|
||||
// File watcher routes (if watcher initialized)
|
||||
if s.watcherHandlers != nil {
|
||||
s.watcherHandlers.RegisterRoutes(s.router)
|
||||
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
|
||||
}
|
||||
|
||||
// Add middleware
|
||||
s.router.Use(loggingMiddleware)
|
||||
s.router.Use(corsMiddleware)
|
||||
@@ -103,11 +163,25 @@ func (s *Server) Start() error {
|
||||
}
|
||||
}()
|
||||
|
||||
// Start file watcher (if initialized)
|
||||
if s.fileWatcher != nil {
|
||||
if err := s.fileWatcher.Start(); err != nil {
|
||||
log.Printf("[API] Warning: failed to start file watcher: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully stops the HTTP server
|
||||
func (s *Server) Stop() error {
|
||||
// Stop file watcher
|
||||
if s.fileWatcher != nil {
|
||||
if err := s.fileWatcher.Stop(); err != nil {
|
||||
log.Printf("[API] Warning: failed to stop file watcher: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.server == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Config holds all configuration for meta-core
|
||||
@@ -28,11 +29,20 @@ type Config struct {
|
||||
HealthCheckIntervalMS int // Health check interval in ms (default: 5000)
|
||||
HeartbeatIntervalMS int // Service heartbeat interval in ms (default: 30000)
|
||||
StaleThresholdMS int // Stale service threshold in ms (default: 60000)
|
||||
|
||||
// File watcher configuration
|
||||
WatchFolderList []string // List of folders to watch for file changes
|
||||
WatchIntervalMS int // Polling interval for network mounts (default: 1000)
|
||||
DebounceMS int // File change debounce time (default: 30000)
|
||||
EnableFileWatcher bool // Enable file watcher (default: true)
|
||||
|
||||
// Mount configuration
|
||||
MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
|
||||
}
|
||||
|
||||
// Load creates a Config from environment variables
|
||||
func Load() *Config {
|
||||
return &Config{
|
||||
cfg := &Config{
|
||||
MetaCorePath: getEnv("META_CORE_PATH", "/meta-core"),
|
||||
FilesPath: getEnv("FILES_PATH", "/files"),
|
||||
ServiceName: getEnv("SERVICE_NAME", "meta-core"),
|
||||
@@ -45,7 +55,19 @@ func Load() *Config {
|
||||
HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
|
||||
HeartbeatIntervalMS: getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
|
||||
StaleThresholdMS: getEnvInt("STALE_THRESHOLD_MS", 60000),
|
||||
WatchIntervalMS: getEnvInt("WATCH_INTERVAL_MS", 1000),
|
||||
DebounceMS: getEnvInt("DEBOUNCE_MS", 30000),
|
||||
EnableFileWatcher: getEnvBool("ENABLE_FILE_WATCHER", true),
|
||||
}
|
||||
|
||||
// Parse watch folder list (comma-separated)
|
||||
watchFolders := getEnv("WATCH_FOLDER_LIST", "/files/")
|
||||
cfg.WatchFolderList = parseCommaSeparated(watchFolders)
|
||||
|
||||
// Set mounts directory
|
||||
cfg.MountsDir = cfg.MetaCorePath + "/mounts"
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
// LockFilePath returns the path to the leader lock file
|
||||
@@ -68,6 +90,16 @@ func (c *Config) ServicesDir() string {
|
||||
return c.MetaCorePath + "/services"
|
||||
}
|
||||
|
||||
// MountsFilePath returns the path to the mounts configuration file
|
||||
func (c *Config) MountsFilePath() string {
|
||||
return c.MountsDir + "/mounts.json"
|
||||
}
|
||||
|
||||
// MountsErrorDir returns the path to mount error files
|
||||
func (c *Config) MountsErrorDir() string {
|
||||
return c.MountsDir + "/errors"
|
||||
}
|
||||
|
||||
func getEnv(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
@@ -83,3 +115,26 @@ func getEnvInt(key string, defaultValue int) int {
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getEnvBool(key string, defaultValue bool) bool {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
lower := strings.ToLower(value)
|
||||
return lower == "true" || lower == "1" || lower == "yes"
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func parseCommaSeparated(s string) []string {
|
||||
if s == "" {
|
||||
return []string{}
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
result := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed != "" {
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -90,7 +90,16 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
|
||||
hostname, _ := os.Hostname()
|
||||
ip := getLocalIP()
|
||||
|
||||
apiBase := fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
|
||||
// Use BASE_URL for external access (dashboard navigation), fall back to internal IP
|
||||
var apiBase string
|
||||
if s.config.BaseURL != "" {
|
||||
apiBase = s.config.BaseURL
|
||||
} else {
|
||||
apiBase = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
|
||||
}
|
||||
|
||||
// Internal URLs always use container IP for container-to-container communication
|
||||
internalBase := fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
|
||||
metaCoreBase := fmt.Sprintf("http://%s:%d", ip, s.config.HTTPPort)
|
||||
|
||||
return &ServiceInfo{
|
||||
@@ -104,15 +113,16 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
|
||||
LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
|
||||
Capabilities: []string{"meta-core"},
|
||||
Endpoints: map[string]string{
|
||||
// meta-core sidecar endpoints (port 9000)
|
||||
// meta-core sidecar endpoints (port 9000) - always internal
|
||||
"health": metaCoreBase + "/health",
|
||||
"meta": metaCoreBase + "/meta",
|
||||
"leader": metaCoreBase + "/leader",
|
||||
"services": metaCoreBase + "/services",
|
||||
// main service endpoints (port 80)
|
||||
"api": apiBase + "/api",
|
||||
"webdav": apiBase + "/webdav",
|
||||
"callback": apiBase + "/api/plugins/callback",
|
||||
// main service endpoints - external for dashboard
|
||||
"api": apiBase + "/api",
|
||||
"webdav": apiBase + "/webdav",
|
||||
// callback uses internal IP for container-to-container plugin communication
|
||||
"callback": internalBase + "/api/plugins/callback",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Handlers provides HTTP handlers for mount operations
|
||||
type Handlers struct {
|
||||
manager *Manager
|
||||
}
|
||||
|
||||
// NewHandlers creates new mount handlers
|
||||
func NewHandlers(manager *Manager) *Handlers {
|
||||
return &Handlers{manager: manager}
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all mount-related routes
|
||||
func (h *Handlers) RegisterRoutes(r *mux.Router) {
|
||||
// Mount management
|
||||
r.HandleFunc("/api/mounts", h.handleListMounts).Methods("GET")
|
||||
r.HandleFunc("/api/mounts", h.handleCreateMount).Methods("POST")
|
||||
r.HandleFunc("/api/mounts/rclone/remotes", h.handleListRcloneRemotes).Methods("GET")
|
||||
r.HandleFunc("/api/mounts/{id}", h.handleGetMount).Methods("GET")
|
||||
r.HandleFunc("/api/mounts/{id}", h.handleDeleteMount).Methods("DELETE")
|
||||
r.HandleFunc("/api/mounts/{id}/mount", h.handleRequestMount).Methods("POST")
|
||||
r.HandleFunc("/api/mounts/{id}/unmount", h.handleRequestUnmount).Methods("POST")
|
||||
r.HandleFunc("/api/mounts/{id}/safe-unmount", h.handleSafeUnmount).Methods("POST")
|
||||
}
|
||||
|
||||
// handleListMounts handles GET /api/mounts
|
||||
func (h *Handlers) handleListMounts(w http.ResponseWriter, r *http.Request) {
|
||||
mounts, err := h.manager.ListMounts()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, MountsListResponse{Mounts: mounts})
|
||||
}
|
||||
|
||||
// handleGetMount handles GET /api/mounts/{id}
|
||||
func (h *Handlers) handleGetMount(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
mount, err := h.manager.GetMount(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if mount == nil {
|
||||
writeError(w, http.StatusNotFound, "mount not found")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, MountResponse{Mount: mount})
|
||||
}
|
||||
|
||||
// handleCreateMount handles POST /api/mounts
|
||||
func (h *Handlers) handleCreateMount(w http.ResponseWriter, r *http.Request) {
|
||||
var req CreateMountRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
mount, err := h.manager.CreateMount(&req)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, MountResponse{Mount: mount})
|
||||
}
|
||||
|
||||
// handleDeleteMount handles DELETE /api/mounts/{id}
|
||||
func (h *Handlers) handleDeleteMount(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
if err := h.manager.DeleteMount(id); err != nil {
|
||||
if err.Error() == "mount not found" {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, StatusResponse{Status: "ok"})
|
||||
}
|
||||
|
||||
// handleRequestMount handles POST /api/mounts/{id}/mount
|
||||
func (h *Handlers) handleRequestMount(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
if err := h.manager.RequestMount(id); err != nil {
|
||||
if err.Error() == "mount not found" {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, StatusResponse{
|
||||
Status: "ok",
|
||||
Message: "Mount requested",
|
||||
})
|
||||
}
|
||||
|
||||
// handleRequestUnmount handles POST /api/mounts/{id}/unmount
|
||||
func (h *Handlers) handleRequestUnmount(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
if err := h.manager.RequestUnmount(id); err != nil {
|
||||
if err.Error() == "mount not found" {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, StatusResponse{
|
||||
Status: "ok",
|
||||
Message: "Unmount requested",
|
||||
})
|
||||
}
|
||||
|
||||
// handleSafeUnmount handles POST /api/mounts/{id}/safe-unmount
|
||||
func (h *Handlers) handleSafeUnmount(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
// Get timeout from query param (default 60000ms)
|
||||
timeoutStr := r.URL.Query().Get("timeout")
|
||||
timeoutMS := 60000
|
||||
if timeoutStr != "" {
|
||||
if parsed, err := parseIntOrDefault(timeoutStr, 60000); err == nil {
|
||||
timeoutMS = parsed
|
||||
}
|
||||
}
|
||||
|
||||
// Get mount info
|
||||
mount, err := h.manager.GetMount(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if mount == nil {
|
||||
writeError(w, http.StatusNotFound, "mount not found")
|
||||
return
|
||||
}
|
||||
|
||||
// Request unmount
|
||||
if err := h.manager.RequestUnmount(id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for unmount
|
||||
success := h.manager.WaitForUnmount(mount.MountPath, timeoutMS)
|
||||
|
||||
if !success {
|
||||
writeJSON(w, http.StatusOK, StatusResponse{
|
||||
Status: "warning",
|
||||
Message: "Unmount requested but mount may still be attached",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, StatusResponse{
|
||||
Status: "ok",
|
||||
Message: "Safe unmount completed",
|
||||
})
|
||||
}
|
||||
|
||||
// handleListRcloneRemotes handles GET /api/mounts/rclone/remotes
|
||||
func (h *Handlers) handleListRcloneRemotes(w http.ResponseWriter, r *http.Request) {
|
||||
remotes, err := h.manager.ListRcloneRemotes()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, RcloneRemotesResponse{Remotes: remotes})
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, map[string]string{
|
||||
"error": http.StatusText(status),
|
||||
"message": message,
|
||||
})
|
||||
}
|
||||
|
||||
func parseIntOrDefault(s string, defaultVal int) (int, error) {
|
||||
var result int
|
||||
if s == "" {
|
||||
return defaultVal, nil
|
||||
}
|
||||
err := json.Unmarshal([]byte(s), &result)
|
||||
if err != nil {
|
||||
return defaultVal, nil
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
package mounts
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
)
|
||||
|
||||
// Manager handles mount configuration and status
|
||||
type Manager struct {
|
||||
config *config.Config
|
||||
mu sync.RWMutex
|
||||
filesPath string
|
||||
mountsDir string
|
||||
}
|
||||
|
||||
// NewManager creates a new mount manager
|
||||
func NewManager(cfg *config.Config) (*Manager, error) {
|
||||
m := &Manager{
|
||||
config: cfg,
|
||||
filesPath: cfg.FilesPath,
|
||||
mountsDir: cfg.MountsDir,
|
||||
}
|
||||
|
||||
// Ensure directories exist
|
||||
if err := m.ensureDirs(); err != nil {
|
||||
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ensureDirs creates required directories
|
||||
func (m *Manager) ensureDirs() error {
|
||||
dirs := []string{
|
||||
m.mountsDir,
|
||||
m.config.MountsErrorDir(),
|
||||
}
|
||||
|
||||
for _, dir := range dirs {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readConfig reads the mounts configuration file
|
||||
func (m *Manager) readConfig() (*MountsFile, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
data, err := os.ReadFile(m.config.MountsFilePath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &MountsFile{Version: 1, Mounts: []MountConfig{}}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var mountsFile MountsFile
|
||||
if err := json.Unmarshal(data, &mountsFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &mountsFile, nil
|
||||
}
|
||||
|
||||
// writeConfig writes the mounts configuration file
|
||||
func (m *Manager) writeConfig(mountsFile *MountsFile) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if err := m.ensureDirs(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(mountsFile, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(m.config.MountsFilePath(), data, 0644)
|
||||
}
|
||||
|
||||
// IsMounted checks if a path is currently mounted
|
||||
func (m *Manager) IsMounted(mountPath string) bool {
|
||||
cmd := exec.Command("findmnt", "-n", mountPath)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(string(output)) != ""
|
||||
}
|
||||
|
||||
// ReadError reads the error file for a mount
|
||||
func (m *Manager) ReadError(id string) (string, error) {
|
||||
errorFile := filepath.Join(m.config.MountsErrorDir(), id+".error")
|
||||
data, err := os.ReadFile(errorFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
|
||||
if len(lines) > 1 {
|
||||
// Skip timestamp line, return rest
|
||||
return strings.Join(lines[1:], "\n"), nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// SanitizeName sanitizes a mount name for use as a directory name
|
||||
func SanitizeName(name string) string {
|
||||
// Convert to lowercase
|
||||
result := strings.ToLower(name)
|
||||
|
||||
// Replace non-alphanumeric chars with hyphens
|
||||
re := regexp.MustCompile(`[^a-z0-9-_]`)
|
||||
result = re.ReplaceAllString(result, "-")
|
||||
|
||||
// Collapse multiple hyphens
|
||||
re = regexp.MustCompile(`-+`)
|
||||
result = re.ReplaceAllString(result, "-")
|
||||
|
||||
// Remove leading/trailing hyphens
|
||||
result = strings.Trim(result, "-")
|
||||
|
||||
// Limit length
|
||||
if len(result) > 64 {
|
||||
result = result[:64]
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// ObscurePassword obscures a password using rclone
|
||||
func ObscurePassword(password string) (string, error) {
|
||||
// Escape special characters for shell
|
||||
escapedPass := strings.ReplaceAll(password, `"`, `\"`)
|
||||
|
||||
cmd := exec.Command("rclone", "obscure", escapedPass)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to obscure password: %w", err)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(string(output)), nil
|
||||
}
|
||||
|
||||
// ListMounts returns all mounts with their current status
|
||||
func (m *Manager) ListMounts() ([]MountStatus, error) {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
statuses := make([]MountStatus, len(mountsFile.Mounts))
|
||||
for i, mount := range mountsFile.Mounts {
|
||||
mounted := m.IsMounted(mount.MountPath)
|
||||
errMsg, _ := m.ReadError(mount.ID)
|
||||
|
||||
statuses[i] = MountStatus{
|
||||
MountConfig: mount,
|
||||
Mounted: mounted,
|
||||
Error: errMsg,
|
||||
LastChecked: NowMS(),
|
||||
}
|
||||
}
|
||||
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
// GetMount returns a single mount by ID
|
||||
func (m *Manager) GetMount(id string) (*MountStatus, error) {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, mount := range mountsFile.Mounts {
|
||||
if mount.ID == id {
|
||||
mounted := m.IsMounted(mount.MountPath)
|
||||
errMsg, _ := m.ReadError(mount.ID)
|
||||
|
||||
return &MountStatus{
|
||||
MountConfig: mount,
|
||||
Mounted: mounted,
|
||||
Error: errMsg,
|
||||
LastChecked: NowMS(),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// CreateMount creates a new mount configuration
|
||||
func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
||||
if req.Name == "" {
|
||||
return nil, fmt.Errorf("mount name is required")
|
||||
}
|
||||
|
||||
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
|
||||
}
|
||||
|
||||
// Validate type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
if req.NFSServer == "" || req.NFSPath == "" {
|
||||
return nil, fmt.Errorf("NFS server and path are required")
|
||||
}
|
||||
case MountTypeSMB:
|
||||
if req.SMBServer == "" || req.SMBShare == "" {
|
||||
return nil, fmt.Errorf("SMB server and share are required")
|
||||
}
|
||||
case MountTypeRclone:
|
||||
if req.RcloneRemote == "" {
|
||||
return nil, fmt.Errorf("rclone remote is required")
|
||||
}
|
||||
}
|
||||
|
||||
// Generate ID and mount path
|
||||
id := uuid.New().String()
|
||||
safeName := SanitizeName(req.Name)
|
||||
mountPath := filepath.Join(m.filesPath, safeName)
|
||||
|
||||
// Check if path already exists
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, mount := range mountsFile.Mounts {
|
||||
if mount.MountPath == mountPath {
|
||||
return nil, fmt.Errorf("mount path %s already configured", mountPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Default enabled to true if not specified
|
||||
enabled := true
|
||||
if req.Enabled != nil {
|
||||
enabled = *req.Enabled
|
||||
}
|
||||
|
||||
mount := MountConfig{
|
||||
ID: id,
|
||||
Name: req.Name,
|
||||
Type: req.Type,
|
||||
Enabled: enabled,
|
||||
DesiredMounted: enabled, // Auto-mount if enabled
|
||||
MountPath: mountPath,
|
||||
Options: req.Options,
|
||||
}
|
||||
|
||||
// Type-specific fields
|
||||
switch req.Type {
|
||||
case MountTypeNFS:
|
||||
mount.NFSServer = req.NFSServer
|
||||
mount.NFSPath = req.NFSPath
|
||||
case MountTypeSMB:
|
||||
mount.SMBServer = req.SMBServer
|
||||
mount.SMBShare = req.SMBShare
|
||||
mount.SMBUsername = req.SMBUsername
|
||||
mount.SMBDomain = req.SMBDomain
|
||||
if req.SMBPassword != "" {
|
||||
obscured, err := ObscurePassword(req.SMBPassword)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to secure password: %w", err)
|
||||
}
|
||||
mount.SMBPasswordObscured = obscured
|
||||
}
|
||||
case MountTypeRclone:
|
||||
mount.RcloneRemote = req.RcloneRemote
|
||||
mount.RclonePath = req.RclonePath
|
||||
}
|
||||
|
||||
mountsFile.Mounts = append(mountsFile.Mounts, mount)
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Printf("[Mounts] Created mount config: %s (%s) -> %s", mount.Name, mount.Type, mount.MountPath)
|
||||
|
||||
return &MountStatus{
|
||||
MountConfig: mount,
|
||||
Mounted: false,
|
||||
LastChecked: NowMS(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RequestMount sets desiredMounted to true
|
||||
func (m *Manager) RequestMount(id string) error {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, mount := range mountsFile.Mounts {
|
||||
if mount.ID == id {
|
||||
mountsFile.Mounts[i].DesiredMounted = true
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[Mounts] Mount requested: %s", mount.Name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("mount not found")
|
||||
}
|
||||
|
||||
// RequestUnmount sets desiredMounted to false
|
||||
func (m *Manager) RequestUnmount(id string) error {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, mount := range mountsFile.Mounts {
|
||||
if mount.ID == id {
|
||||
mountsFile.Mounts[i].DesiredMounted = false
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[Mounts] Unmount requested: %s", mount.Name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("mount not found")
|
||||
}
|
||||
|
||||
// WaitForUnmount waits for a mount to be unmounted
|
||||
func (m *Manager) WaitForUnmount(mountPath string, timeoutMS int) bool {
|
||||
deadline := time.Now().Add(time.Duration(timeoutMS) * time.Millisecond)
|
||||
for time.Now().Before(deadline) {
|
||||
if !m.IsMounted(mountPath) {
|
||||
return true
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DeleteMount removes a mount configuration
|
||||
func (m *Manager) DeleteMount(id string) error {
|
||||
mountsFile, err := m.readConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var mount *MountConfig
|
||||
index := -1
|
||||
for i, mnt := range mountsFile.Mounts {
|
||||
if mnt.ID == id {
|
||||
mount = &mnt
|
||||
index = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if index == -1 {
|
||||
return fmt.Errorf("mount not found")
|
||||
}
|
||||
|
||||
// Request unmount first
|
||||
mountsFile.Mounts[index].DesiredMounted = false
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Wait for unmount (15 seconds max)
|
||||
m.WaitForUnmount(mount.MountPath, 15000)
|
||||
|
||||
// Remove from config
|
||||
mountsFile.Mounts = append(mountsFile.Mounts[:index], mountsFile.Mounts[index+1:]...)
|
||||
if err := m.writeConfig(mountsFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clean up error file
|
||||
errorFile := filepath.Join(m.config.MountsErrorDir(), id+".error")
|
||||
os.Remove(errorFile)
|
||||
|
||||
// Try to remove mount directory (will fail if not empty, which is fine)
|
||||
os.Remove(mount.MountPath)
|
||||
|
||||
log.Printf("[Mounts] Deleted mount: %s", mount.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListRcloneRemotes lists available rclone remotes
|
||||
func (m *Manager) ListRcloneRemotes() ([]RcloneRemote, error) {
|
||||
// Call rclone RC API
|
||||
cmd := exec.Command("curl", "-s", "-X", "POST",
|
||||
"-H", "Content-Type: application/json",
|
||||
"-u", "admin:admin",
|
||||
"http://127.0.0.1:5572/config/listremotes")
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
log.Printf("[Mounts] Failed to list rclone remotes: %v", err)
|
||||
return []RcloneRemote{}, nil
|
||||
}
|
||||
|
||||
var response struct {
|
||||
Remotes []string `json:"remotes"`
|
||||
}
|
||||
if err := json.Unmarshal(output, &response); err != nil {
|
||||
return []RcloneRemote{}, nil
|
||||
}
|
||||
|
||||
remotes := make([]RcloneRemote, 0, len(response.Remotes))
|
||||
for _, name := range response.Remotes {
|
||||
// Strip trailing colon if present
|
||||
cleanName := strings.TrimSuffix(name, ":")
|
||||
|
||||
// Get type for this remote
|
||||
remoteType := m.getRcloneRemoteType(cleanName)
|
||||
|
||||
remotes = append(remotes, RcloneRemote{
|
||||
Name: cleanName,
|
||||
Type: remoteType,
|
||||
})
|
||||
}
|
||||
|
||||
return remotes, nil
|
||||
}
|
||||
|
||||
// getRcloneRemoteType gets the type of an rclone remote
|
||||
func (m *Manager) getRcloneRemoteType(name string) string {
|
||||
// Build request body
|
||||
body := fmt.Sprintf(`{"name":"%s"}`, name)
|
||||
|
||||
cmd := exec.Command("curl", "-s", "-X", "POST",
|
||||
"-H", "Content-Type: application/json",
|
||||
"-u", "admin:admin",
|
||||
"-d", body,
|
||||
"http://127.0.0.1:5572/config/get")
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
var response map[string]interface{}
|
||||
if err := json.Unmarshal(output, &response); err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
if typeStr, ok := response["type"].(string); ok {
|
||||
return typeStr
|
||||
}
|
||||
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
// Basic auth helper for rclone API
|
||||
func basicAuth(username, password string) string {
|
||||
auth := username + ":" + password
|
||||
return base64.StdEncoding.EncodeToString([]byte(auth))
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package mounts
|
||||
|
||||
import "time"
|
||||
|
||||
// MountType represents the type of mount
|
||||
type MountType string
|
||||
|
||||
const (
|
||||
MountTypeNFS MountType = "nfs"
|
||||
MountTypeSMB MountType = "smb"
|
||||
MountTypeRclone MountType = "rclone"
|
||||
)
|
||||
|
||||
// MountConfig represents a mount configuration
|
||||
type MountConfig struct {
|
||||
// Common fields
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled bool `json:"enabled"`
|
||||
DesiredMounted bool `json:"desiredMounted"`
|
||||
MountPath string `json:"mountPath"`
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS-specific fields
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
|
||||
// SMB-specific fields
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
SMBShare string `json:"smbShare,omitempty"`
|
||||
SMBUsername string `json:"smbUsername,omitempty"`
|
||||
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
||||
SMBDomain string `json:"smbDomain,omitempty"`
|
||||
|
||||
// rclone-specific fields
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
}
|
||||
|
||||
// MountStatus represents the runtime status of a mount
|
||||
type MountStatus struct {
|
||||
MountConfig
|
||||
Mounted bool `json:"mounted"`
|
||||
Error string `json:"error,omitempty"`
|
||||
LastChecked int64 `json:"lastChecked"`
|
||||
}
|
||||
|
||||
// MountsFile represents the mounts.json file structure
|
||||
type MountsFile struct {
|
||||
Version int `json:"version"`
|
||||
Mounts []MountConfig `json:"mounts"`
|
||||
}
|
||||
|
||||
// RcloneRemote represents an rclone remote
|
||||
type RcloneRemote struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// CreateMountRequest is the request body for creating a mount
|
||||
type CreateMountRequest struct {
|
||||
Name string `json:"name"`
|
||||
Type MountType `json:"type"`
|
||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||
Options string `json:"options,omitempty"`
|
||||
|
||||
// NFS
|
||||
NFSServer string `json:"nfsServer,omitempty"`
|
||||
NFSPath string `json:"nfsPath,omitempty"`
|
||||
|
||||
// SMB
|
||||
SMBServer string `json:"smbServer,omitempty"`
|
||||
SMBShare string `json:"smbShare,omitempty"`
|
||||
SMBUsername string `json:"smbUsername,omitempty"`
|
||||
SMBPassword string `json:"smbPassword,omitempty"` // Plain text, will be obscured
|
||||
SMBDomain string `json:"smbDomain,omitempty"`
|
||||
|
||||
// rclone
|
||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||
RclonePath string `json:"rclonePath,omitempty"`
|
||||
}
|
||||
|
||||
// MountResponse is the response for mount operations
|
||||
type MountResponse struct {
|
||||
Mount *MountStatus `json:"mount,omitempty"`
|
||||
}
|
||||
|
||||
// MountsListResponse is the response for listing mounts
|
||||
type MountsListResponse struct {
|
||||
Mounts []MountStatus `json:"mounts"`
|
||||
}
|
||||
|
||||
// RcloneRemotesResponse is the response for listing rclone remotes
|
||||
type RcloneRemotesResponse struct {
|
||||
Remotes []RcloneRemote `json:"remotes"`
|
||||
}
|
||||
|
||||
// StatusResponse is a generic status response
|
||||
type StatusResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
GateStatus interface{} `json:"gateStatus,omitempty"`
|
||||
}
|
||||
|
||||
// Timestamp helpers
|
||||
func NowMS() int64 {
|
||||
return time.Now().UnixMilli()
|
||||
}
|
||||
@@ -490,3 +490,83 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
|
||||
newValue := strings.Join(newValues, "|")
|
||||
return true, c.client.HSet(ctx, hashKey, property, newValue).Err()
|
||||
}
|
||||
|
||||
// GetMemoryInfo returns Redis memory usage information
|
||||
func (c *Client) GetMemoryInfo() (string, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return "", fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
info, err := c.client.Info(ctx, "memory").Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("info memory failed: %w", err)
|
||||
}
|
||||
|
||||
// Extract used_memory_human from info string
|
||||
for _, line := range strings.Split(info, "\r\n") {
|
||||
if strings.HasPrefix(line, "used_memory_human:") {
|
||||
return strings.TrimPrefix(line, "used_memory_human:"), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "N/A", nil
|
||||
}
|
||||
|
||||
// ClearAllMetadata deletes all file metadata and index
|
||||
// Returns the number of files deleted
|
||||
func (c *Client) ClearAllMetadata() (int64, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.client == nil {
|
||||
return 0, fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Get all hash IDs from index
|
||||
indexKey := c.buildIndexKey()
|
||||
hashIDs, err := c.client.SMembers(ctx, indexKey).Result()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("smembers failed: %w", err)
|
||||
}
|
||||
|
||||
var deletedCount int64
|
||||
|
||||
// Delete each file's hash
|
||||
for _, hashID := range hashIDs {
|
||||
hashKey := c.buildHashKey(hashID)
|
||||
if err := c.client.Del(ctx, hashKey).Err(); err != nil {
|
||||
log.Printf("[Storage] Warning: failed to delete %s: %v", hashKey, err)
|
||||
continue
|
||||
}
|
||||
deletedCount++
|
||||
}
|
||||
|
||||
// Delete the index set
|
||||
if err := c.client.Del(ctx, indexKey).Err(); err != nil {
|
||||
log.Printf("[Storage] Warning: failed to delete index: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("[Storage] Cleared %d file metadata entries", deletedCount)
|
||||
return deletedCount, nil
|
||||
}
|
||||
|
||||
// GetRedisClient returns the underlying redis client for advanced operations
|
||||
func (c *Client) GetRedisClient() *redis.Client {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.client
|
||||
}
|
||||
|
||||
// GetPrefix returns the key prefix used by this client
|
||||
func (c *Client) GetPrefix() string {
|
||||
return c.prefix
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Debouncer delays events to ensure file stability
|
||||
type Debouncer struct {
|
||||
delay time.Duration
|
||||
pending map[string]*PendingEvent
|
||||
mu sync.Mutex
|
||||
callback func(FileEvent)
|
||||
stopped bool
|
||||
}
|
||||
|
||||
// NewDebouncer creates a new debouncer with the specified delay
|
||||
func NewDebouncer(delay time.Duration) *Debouncer {
|
||||
return &Debouncer{
|
||||
delay: delay,
|
||||
pending: make(map[string]*PendingEvent),
|
||||
}
|
||||
}
|
||||
|
||||
// SetCallback sets the callback function for debounced events
|
||||
func (d *Debouncer) SetCallback(cb func(FileEvent)) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.callback = cb
|
||||
}
|
||||
|
||||
// Add adds or updates an event in the debounce queue
|
||||
func (d *Debouncer) Add(event FileEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
if d.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
key := event.Path
|
||||
|
||||
if existing, ok := d.pending[key]; ok {
|
||||
// Update existing pending event
|
||||
existing.LastSeen = time.Now()
|
||||
existing.Event = event // Update with latest event info
|
||||
|
||||
// Reset timer
|
||||
existing.Timer.Reset(d.delay)
|
||||
} else {
|
||||
// Create new pending event
|
||||
now := time.Now()
|
||||
pending := &PendingEvent{
|
||||
Event: event,
|
||||
FirstSeen: now,
|
||||
LastSeen: now,
|
||||
}
|
||||
|
||||
// Create timer
|
||||
pending.Timer = time.AfterFunc(d.delay, func() {
|
||||
d.flush(key)
|
||||
})
|
||||
|
||||
d.pending[key] = pending
|
||||
}
|
||||
}
|
||||
|
||||
// flush emits the event and removes it from pending
|
||||
func (d *Debouncer) flush(key string) {
|
||||
d.mu.Lock()
|
||||
|
||||
pending, ok := d.pending[key]
|
||||
if !ok {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Check if enough time has passed since last activity
|
||||
elapsed := time.Since(pending.LastSeen)
|
||||
if elapsed < d.delay {
|
||||
// Reset timer for remaining time
|
||||
remaining := d.delay - elapsed
|
||||
pending.Timer.Reset(remaining)
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Remove from pending
|
||||
delete(d.pending, key)
|
||||
callback := d.callback
|
||||
event := pending.Event
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
// Call callback outside of lock
|
||||
if callback != nil {
|
||||
callback(event)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the debouncer and cancels all pending events
|
||||
func (d *Debouncer) Stop() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
d.stopped = true
|
||||
|
||||
// Cancel all timers
|
||||
for _, pending := range d.pending {
|
||||
pending.Timer.Stop()
|
||||
}
|
||||
|
||||
// Clear pending
|
||||
d.pending = make(map[string]*PendingEvent)
|
||||
}
|
||||
|
||||
// PendingCount returns the number of pending events
|
||||
func (d *Debouncer) PendingCount() int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return len(d.pending)
|
||||
}
|
||||
|
||||
// FlushAll immediately flushes all pending events
|
||||
func (d *Debouncer) FlushAll() {
|
||||
d.mu.Lock()
|
||||
|
||||
keys := make([]string, 0, len(d.pending))
|
||||
for key := range d.pending {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
for _, key := range keys {
|
||||
d.flush(key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxRetries is the maximum number of delivery retries
|
||||
MaxRetries = 3
|
||||
// RetryDelay is the delay between retries
|
||||
RetryDelay = 5 * time.Second
|
||||
// WebhookTimeout is the timeout for webhook requests
|
||||
WebhookTimeout = 10 * time.Second
|
||||
// MaxFailCount is the max failures before removing subscriber
|
||||
MaxFailCount = 10
|
||||
)
|
||||
|
||||
// Dispatcher sends file events to subscribers
|
||||
type Dispatcher struct {
|
||||
subscribers map[string]*Subscriber
|
||||
sseClients map[chan FileEvent]bool
|
||||
mu sync.RWMutex
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewDispatcher creates a new event dispatcher
|
||||
func NewDispatcher() *Dispatcher {
|
||||
return &Dispatcher{
|
||||
subscribers: make(map[string]*Subscriber),
|
||||
sseClients: make(map[chan FileEvent]bool),
|
||||
httpClient: &http.Client{
|
||||
Timeout: WebhookTimeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe registers a webhook subscriber
|
||||
func (d *Dispatcher) Subscribe(url string, eventTypes []string) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
d.subscribers[url] = &Subscriber{
|
||||
URL: url,
|
||||
RegisteredAt: NowMS(),
|
||||
EventTypes: eventTypes,
|
||||
FailCount: 0,
|
||||
}
|
||||
|
||||
log.Printf("[Dispatcher] Subscribed webhook: %s", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unsubscribe removes a webhook subscriber
|
||||
func (d *Dispatcher) Unsubscribe(url string) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
delete(d.subscribers, url)
|
||||
log.Printf("[Dispatcher] Unsubscribed webhook: %s", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListSubscribers returns all subscribers
|
||||
func (d *Dispatcher) ListSubscribers() []Subscriber {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
result := make([]Subscriber, 0, len(d.subscribers))
|
||||
for _, sub := range d.subscribers {
|
||||
result = append(result, *sub)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// AddSSEClient adds an SSE client channel
|
||||
func (d *Dispatcher) AddSSEClient(ch chan FileEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.sseClients[ch] = true
|
||||
}
|
||||
|
||||
// RemoveSSEClient removes an SSE client channel
|
||||
func (d *Dispatcher) RemoveSSEClient(ch chan FileEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
delete(d.sseClients, ch)
|
||||
close(ch)
|
||||
}
|
||||
|
||||
// Dispatch sends an event to all subscribers
|
||||
func (d *Dispatcher) Dispatch(event FileEvent) {
|
||||
// Dispatch to webhooks
|
||||
go d.dispatchToWebhooks(event)
|
||||
|
||||
// Dispatch to SSE clients
|
||||
d.dispatchToSSE(event)
|
||||
}
|
||||
|
||||
// dispatchToWebhooks sends event to webhook subscribers
|
||||
func (d *Dispatcher) dispatchToWebhooks(event FileEvent) {
|
||||
d.mu.RLock()
|
||||
subscribers := make([]*Subscriber, 0, len(d.subscribers))
|
||||
for _, sub := range d.subscribers {
|
||||
subscribers = append(subscribers, sub)
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
for _, sub := range subscribers {
|
||||
// Check event type filter
|
||||
if len(sub.EventTypes) > 0 {
|
||||
found := false
|
||||
for _, et := range sub.EventTypes {
|
||||
if et == string(event.Type) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
go d.deliverToWebhook(sub.URL, event)
|
||||
}
|
||||
}
|
||||
|
||||
// deliverToWebhook delivers an event to a webhook with retries
|
||||
func (d *Dispatcher) deliverToWebhook(url string, event FileEvent) {
|
||||
body, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
log.Printf("[Dispatcher] Failed to marshal event: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < MaxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(RetryDelay)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Event-Type", string(event.Type))
|
||||
|
||||
resp, err := d.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
// Success
|
||||
d.mu.Lock()
|
||||
if sub, ok := d.subscribers[url]; ok {
|
||||
sub.LastDelivery = NowMS()
|
||||
sub.FailCount = 0
|
||||
}
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
lastErr = &httpError{StatusCode: resp.StatusCode}
|
||||
}
|
||||
|
||||
// All retries failed
|
||||
log.Printf("[Dispatcher] Failed to deliver to %s after %d retries: %v", url, MaxRetries, lastErr)
|
||||
|
||||
d.mu.Lock()
|
||||
if sub, ok := d.subscribers[url]; ok {
|
||||
sub.FailCount++
|
||||
if sub.FailCount >= MaxFailCount {
|
||||
delete(d.subscribers, url)
|
||||
log.Printf("[Dispatcher] Removed webhook %s after %d failures", url, MaxFailCount)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// dispatchToSSE sends event to SSE clients
|
||||
func (d *Dispatcher) dispatchToSSE(event FileEvent) {
|
||||
d.mu.RLock()
|
||||
clients := make([]chan FileEvent, 0, len(d.sseClients))
|
||||
for ch := range d.sseClients {
|
||||
clients = append(clients, ch)
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
for _, ch := range clients {
|
||||
select {
|
||||
case ch <- event:
|
||||
default:
|
||||
// Channel full, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SSEClientCount returns the number of connected SSE clients
|
||||
func (d *Dispatcher) SSEClientCount() int {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
return len(d.sseClients)
|
||||
}
|
||||
|
||||
// httpError represents an HTTP error
|
||||
type httpError struct {
|
||||
StatusCode int
|
||||
}
|
||||
|
||||
func (e *httpError) Error() string {
|
||||
return http.StatusText(e.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Handlers provides HTTP handlers for watcher operations
|
||||
type Handlers struct {
|
||||
watcher *Watcher
|
||||
dispatcher *Dispatcher
|
||||
}
|
||||
|
||||
// NewHandlers creates new watcher handlers
|
||||
func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
|
||||
return &Handlers{
|
||||
watcher: watcher,
|
||||
dispatcher: dispatcher,
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all watcher-related routes
|
||||
func (h *Handlers) RegisterRoutes(r *mux.Router) {
|
||||
// Event subscriptions
|
||||
r.HandleFunc("/api/events/subscribe", h.handleSSESubscribe).Methods("GET")
|
||||
r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
|
||||
r.HandleFunc("/api/events/subscribers", h.handleListSubscribers).Methods("GET")
|
||||
r.HandleFunc("/api/events/subscribers", h.handleAddSubscriber).Methods("POST")
|
||||
r.HandleFunc("/api/events/subscribers/{url}", h.handleRemoveSubscriber).Methods("DELETE")
|
||||
|
||||
// Scan management
|
||||
r.HandleFunc("/api/scan/trigger", h.handleTriggerScan).Methods("POST")
|
||||
r.HandleFunc("/api/scan/status", h.handleScanStatus).Methods("GET")
|
||||
}
|
||||
|
||||
// handleSSESubscribe handles GET /api/events/subscribe (Server-Sent Events)
|
||||
func (h *Handlers) handleSSESubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
// Set SSE headers
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
// Create event channel
|
||||
eventChan := make(chan FileEvent, 100)
|
||||
h.dispatcher.AddSSEClient(eventChan)
|
||||
|
||||
// Cleanup on disconnect
|
||||
defer h.dispatcher.RemoveSSEClient(eventChan)
|
||||
|
||||
// Get flusher
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "SSE not supported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Send initial connected message
|
||||
fmt.Fprintf(w, "event: connected\ndata: {\"status\":\"connected\"}\n\n")
|
||||
flusher.Flush()
|
||||
|
||||
// Stream events
|
||||
for {
|
||||
select {
|
||||
case event := <-eventChan:
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "event: file\ndata: %s\n\n", string(data))
|
||||
flusher.Flush()
|
||||
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handlePoll handles GET /api/events/poll
|
||||
func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
||||
// Get since parameter
|
||||
sinceStr := r.URL.Query().Get("since")
|
||||
sinceMS := int64(0)
|
||||
if sinceStr != "" {
|
||||
if parsed, err := strconv.ParseInt(sinceStr, 10, 64); err == nil {
|
||||
sinceMS = parsed
|
||||
}
|
||||
}
|
||||
|
||||
// Get limit parameter
|
||||
limitStr := r.URL.Query().Get("limit")
|
||||
limit := 100
|
||||
if limitStr != "" {
|
||||
if parsed, err := strconv.Atoi(limitStr); err == nil {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
events := h.watcher.GetRecentEvents(sinceMS, limit)
|
||||
|
||||
writeJSON(w, http.StatusOK, EventsListResponse{
|
||||
Events: events,
|
||||
Count: len(events),
|
||||
})
|
||||
}
|
||||
|
||||
// handleListSubscribers handles GET /api/events/subscribers
|
||||
func (h *Handlers) handleListSubscribers(w http.ResponseWriter, r *http.Request) {
|
||||
subscribers := h.dispatcher.ListSubscribers()
|
||||
|
||||
writeJSON(w, http.StatusOK, SubscribersListResponse{
|
||||
Subscribers: subscribers,
|
||||
Count: len(subscribers),
|
||||
})
|
||||
}
|
||||
|
||||
// handleAddSubscriber handles POST /api/events/subscribers
|
||||
func (h *Handlers) handleAddSubscriber(w http.ResponseWriter, r *http.Request) {
|
||||
var req SubscribeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.URL == "" {
|
||||
writeError(w, http.StatusBadRequest, "URL is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Subscribe(req.URL, req.EventTypes); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Subscribed",
|
||||
"url": req.URL,
|
||||
})
|
||||
}
|
||||
|
||||
// handleRemoveSubscriber handles DELETE /api/events/subscribers/{url}
|
||||
func (h *Handlers) handleRemoveSubscriber(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
url := vars["url"]
|
||||
|
||||
if url == "" {
|
||||
writeError(w, http.StatusBadRequest, "URL is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Unsubscribe(url); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Unsubscribed",
|
||||
})
|
||||
}
|
||||
|
||||
// handleTriggerScan handles POST /api/scan/trigger
|
||||
func (h *Handlers) handleTriggerScan(w http.ResponseWriter, r *http.Request) {
|
||||
go h.watcher.RunScan()
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"message": "Scan triggered",
|
||||
})
|
||||
}
|
||||
|
||||
// handleScanStatus handles GET /api/scan/status
|
||||
func (h *Handlers) handleScanStatus(w http.ResponseWriter, r *http.Request) {
|
||||
status := h.watcher.GetStatus()
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, map[string]string{
|
||||
"error": http.StatusText(status),
|
||||
"message": message,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package watcher
|
||||
|
||||
import "time"
|
||||
|
||||
// FileEventType represents the type of file event
|
||||
type FileEventType string
|
||||
|
||||
const (
|
||||
EventTypeAdd FileEventType = "add"
|
||||
EventTypeChange FileEventType = "change"
|
||||
EventTypeDelete FileEventType = "delete"
|
||||
EventTypeRename FileEventType = "rename"
|
||||
)
|
||||
|
||||
// FileEvent represents a file system event
|
||||
type FileEvent struct {
|
||||
Type FileEventType `json:"type"`
|
||||
Path string `json:"path"` // Relative to FILES_PATH
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
PartialHash string `json:"partialHash,omitempty"` // Hash of first 64KB
|
||||
OldPath string `json:"oldPath,omitempty"` // For rename events
|
||||
}
|
||||
|
||||
// Subscriber represents a webhook subscriber
|
||||
type Subscriber struct {
|
||||
URL string `json:"url"`
|
||||
RegisteredAt int64 `json:"registeredAt"`
|
||||
EventTypes []string `json:"eventTypes,omitempty"` // Empty means all
|
||||
LastDelivery int64 `json:"lastDelivery,omitempty"`
|
||||
FailCount int `json:"failCount"`
|
||||
}
|
||||
|
||||
// PendingEvent tracks a file event that's being debounced
|
||||
type PendingEvent struct {
|
||||
Event FileEvent
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
Timer *time.Timer
|
||||
}
|
||||
|
||||
// SubscribeRequest is the request to register a webhook
|
||||
type SubscribeRequest struct {
|
||||
URL string `json:"url"`
|
||||
EventTypes []string `json:"eventTypes,omitempty"`
|
||||
}
|
||||
|
||||
// EventsListResponse is the response for listing events
|
||||
type EventsListResponse struct {
|
||||
Events []FileEvent `json:"events"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// SubscribersListResponse is the response for listing subscribers
|
||||
type SubscribersListResponse struct {
|
||||
Subscribers []Subscriber `json:"subscribers"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ScanStatusResponse is the response for scan status
|
||||
type ScanStatusResponse struct {
|
||||
Status string `json:"status"`
|
||||
Scanning bool `json:"scanning"`
|
||||
LastScan int64 `json:"lastScan,omitempty"`
|
||||
FileCount int `json:"fileCount,omitempty"`
|
||||
}
|
||||
|
||||
// NowMS returns current time in milliseconds
|
||||
func NowMS() int64 {
|
||||
return time.Now().UnixMilli()
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package watcher
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
// PartialHashSize is the number of bytes to hash for file identification
|
||||
PartialHashSize = 64 * 1024 // 64KB
|
||||
)
|
||||
|
||||
// Watcher monitors directories for file changes
|
||||
type Watcher struct {
|
||||
config *config.Config
|
||||
fsWatcher *fsnotify.Watcher
|
||||
debouncer *Debouncer
|
||||
dispatcher *Dispatcher
|
||||
filesPath string
|
||||
watchPaths []string
|
||||
|
||||
mu sync.RWMutex
|
||||
isRunning bool
|
||||
isScanning bool
|
||||
lastScan int64
|
||||
fileCount int
|
||||
stopChan chan struct{}
|
||||
eventBuffer []FileEvent
|
||||
}
|
||||
|
||||
// NewWatcher creates a new file watcher
|
||||
func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
|
||||
fsWatcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debouncer := NewDebouncer(time.Duration(cfg.DebounceMS) * time.Millisecond)
|
||||
|
||||
w := &Watcher{
|
||||
config: cfg,
|
||||
fsWatcher: fsWatcher,
|
||||
debouncer: debouncer,
|
||||
dispatcher: dispatcher,
|
||||
filesPath: cfg.FilesPath,
|
||||
watchPaths: cfg.WatchFolderList,
|
||||
stopChan: make(chan struct{}),
|
||||
eventBuffer: make([]FileEvent, 0, 1000),
|
||||
}
|
||||
|
||||
// Set debouncer callback
|
||||
debouncer.SetCallback(func(event FileEvent) {
|
||||
w.handleDebouncedEvent(event)
|
||||
})
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Start begins watching for file changes
|
||||
func (w *Watcher) Start() error {
|
||||
w.mu.Lock()
|
||||
if w.isRunning {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
w.isRunning = true
|
||||
w.mu.Unlock()
|
||||
|
||||
// Add watch paths
|
||||
for _, watchPath := range w.watchPaths {
|
||||
if err := w.addWatchRecursive(watchPath); err != nil {
|
||||
log.Printf("[Watcher] Warning: failed to watch %s: %v", watchPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start event processing goroutine
|
||||
go w.processEvents()
|
||||
|
||||
// Start initial scan
|
||||
go w.RunScan()
|
||||
|
||||
log.Printf("[Watcher] Started watching %d paths", len(w.watchPaths))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the watcher
|
||||
func (w *Watcher) Stop() error {
|
||||
w.mu.Lock()
|
||||
if !w.isRunning {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
w.isRunning = false
|
||||
w.mu.Unlock()
|
||||
|
||||
close(w.stopChan)
|
||||
w.debouncer.Stop()
|
||||
return w.fsWatcher.Close()
|
||||
}
|
||||
|
||||
// addWatchRecursive adds a directory and all subdirectories to the watch list
|
||||
func (w *Watcher) addWatchRecursive(root string) error {
|
||||
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip inaccessible paths
|
||||
}
|
||||
if info.IsDir() {
|
||||
if err := w.fsWatcher.Add(path); err != nil {
|
||||
log.Printf("[Watcher] Warning: cannot watch %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// processEvents handles fsnotify events
|
||||
func (w *Watcher) processEvents() {
|
||||
for {
|
||||
select {
|
||||
case <-w.stopChan:
|
||||
return
|
||||
|
||||
case event, ok := <-w.fsWatcher.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.handleFsEvent(event)
|
||||
|
||||
case err, ok := <-w.fsWatcher.Errors:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
log.Printf("[Watcher] Error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleFsEvent converts an fsnotify event to a FileEvent
|
||||
func (w *Watcher) handleFsEvent(event fsnotify.Event) {
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(w.filesPath, event.Name)
|
||||
if err != nil {
|
||||
relPath = event.Name
|
||||
}
|
||||
|
||||
// Clean up path
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Determine event type
|
||||
var eventType FileEventType
|
||||
switch {
|
||||
case event.Op&fsnotify.Create == fsnotify.Create:
|
||||
eventType = EventTypeAdd
|
||||
// If it's a new directory, add it to watch
|
||||
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
||||
w.fsWatcher.Add(event.Name)
|
||||
}
|
||||
case event.Op&fsnotify.Write == fsnotify.Write:
|
||||
eventType = EventTypeChange
|
||||
case event.Op&fsnotify.Remove == fsnotify.Remove:
|
||||
eventType = EventTypeDelete
|
||||
case event.Op&fsnotify.Rename == fsnotify.Rename:
|
||||
eventType = EventTypeRename
|
||||
default:
|
||||
return // Ignore other events
|
||||
}
|
||||
|
||||
// Skip directories for file events (except remove)
|
||||
if eventType != EventTypeDelete {
|
||||
info, err := os.Stat(event.Name)
|
||||
if err == nil && info.IsDir() {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Get file info if available
|
||||
var size int64
|
||||
if eventType != EventTypeDelete {
|
||||
if info, err := os.Stat(event.Name); err == nil {
|
||||
size = info.Size()
|
||||
}
|
||||
}
|
||||
|
||||
fileEvent := FileEvent{
|
||||
Type: eventType,
|
||||
Path: relPath,
|
||||
Size: size,
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Send to debouncer
|
||||
w.debouncer.Add(fileEvent)
|
||||
}
|
||||
|
||||
// handleDebouncedEvent processes a debounced event
|
||||
func (w *Watcher) handleDebouncedEvent(event FileEvent) {
|
||||
// Compute partial hash for add/change events
|
||||
if event.Type == EventTypeAdd || event.Type == EventTypeChange {
|
||||
fullPath := filepath.Join(w.filesPath, event.Path)
|
||||
if hash, err := computePartialHash(fullPath); err == nil {
|
||||
event.PartialHash = hash
|
||||
}
|
||||
}
|
||||
|
||||
// Add to buffer
|
||||
w.mu.Lock()
|
||||
w.eventBuffer = append(w.eventBuffer, event)
|
||||
// Keep buffer size reasonable
|
||||
if len(w.eventBuffer) > 10000 {
|
||||
w.eventBuffer = w.eventBuffer[len(w.eventBuffer)-5000:]
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
// Dispatch to subscribers
|
||||
w.dispatcher.Dispatch(event)
|
||||
|
||||
log.Printf("[Watcher] Event: %s %s", event.Type, event.Path)
|
||||
}
|
||||
|
||||
// RunScan performs a full directory scan
|
||||
func (w *Watcher) RunScan() {
|
||||
w.mu.Lock()
|
||||
if w.isScanning {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.isScanning = true
|
||||
w.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
w.mu.Lock()
|
||||
w.isScanning = false
|
||||
w.lastScan = NowMS()
|
||||
w.mu.Unlock()
|
||||
}()
|
||||
|
||||
log.Println("[Watcher] Starting directory scan...")
|
||||
|
||||
fileCount := 0
|
||||
for _, watchPath := range w.watchPaths {
|
||||
count := w.scanDirectory(watchPath)
|
||||
fileCount += count
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
w.fileCount = fileCount
|
||||
w.mu.Unlock()
|
||||
|
||||
log.Printf("[Watcher] Scan complete: %d files found", fileCount)
|
||||
}
|
||||
|
||||
// scanDirectory scans a directory and emits add events for files
|
||||
func (w *Watcher) scanDirectory(root string) int {
|
||||
count := 0
|
||||
|
||||
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip hidden files
|
||||
if strings.HasPrefix(filepath.Base(path), ".") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get relative path
|
||||
relPath, err := filepath.Rel(w.filesPath, path)
|
||||
if err != nil {
|
||||
relPath = path
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
// Emit add event (through debouncer)
|
||||
event := FileEvent{
|
||||
Type: EventTypeAdd,
|
||||
Path: relPath,
|
||||
Size: info.Size(),
|
||||
Timestamp: NowMS(),
|
||||
}
|
||||
|
||||
// Compute partial hash
|
||||
if hash, err := computePartialHash(path); err == nil {
|
||||
event.PartialHash = hash
|
||||
}
|
||||
|
||||
// Dispatch directly (skip debouncer for scan)
|
||||
w.dispatcher.Dispatch(event)
|
||||
count++
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
// GetRecentEvents returns events since a given timestamp
|
||||
func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
|
||||
result := make([]FileEvent, 0)
|
||||
for _, event := range w.eventBuffer {
|
||||
if event.Timestamp > sinceMS {
|
||||
result = append(result, event)
|
||||
if limit > 0 && len(result) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetStatus returns watcher status
|
||||
func (w *Watcher) GetStatus() ScanStatusResponse {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
|
||||
status := "running"
|
||||
if !w.isRunning {
|
||||
status = "stopped"
|
||||
}
|
||||
|
||||
return ScanStatusResponse{
|
||||
Status: status,
|
||||
Scanning: w.isScanning,
|
||||
LastScan: w.lastScan,
|
||||
FileCount: w.fileCount,
|
||||
}
|
||||
}
|
||||
|
||||
// computePartialHash computes SHA-256 hash of first 64KB of a file
|
||||
func computePartialHash(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
buffer := make([]byte, PartialHashSize)
|
||||
n, err := file.Read(buffer)
|
||||
if err != nil && err != io.EOF {
|
||||
return "", err
|
||||
}
|
||||
|
||||
hasher.Write(buffer[:n])
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
Reference in New Issue
Block a user