improve editor and add type detection
This commit is contained in:
Pierre
@@ -340,6 +340,63 @@ func (s *Server) handleHeadData(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// handleGetFileByCIDInfo handles GET /api/file/{cid}/info.
|
||||
// Returns lightweight metadata for the file (existence, content type, size,
|
||||
// relative path) without streaming any bytes. Used by the editor to decide
|
||||
// whether to render a preview (image/video) or a download-only button.
|
||||
func (s *Server) handleGetFileByCIDInfo(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
cid := vars["cid"]
|
||||
|
||||
type infoResp struct {
|
||||
Exists bool `json:"exists"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
FilePath string `json:"filePath,omitempty"`
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
if cid == "" {
|
||||
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
|
||||
return
|
||||
}
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
relPath, err := s.storage.LookupPathByCID(cid)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if relPath == "" {
|
||||
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
|
||||
return
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(s.config.FilesPath, relPath)
|
||||
fi, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
_ = json.NewEncoder(w).Encode(infoResp{Exists: false, FilePath: relPath})
|
||||
return
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(fullPath))
|
||||
contentType, ok := contentTypeByExt[ext]
|
||||
if !ok {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
_ = json.NewEncoder(w).Encode(infoResp{
|
||||
Exists: true,
|
||||
ContentType: contentType,
|
||||
Size: fi.Size(),
|
||||
FilePath: relPath,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetFileByCID handles GET /file/{cid}
|
||||
// Serves a file by looking up its CID in poster/backdrop metadata
|
||||
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// handleSchemaGet returns the live per-field schema derived from meta:events.
|
||||
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
|
||||
if s.schemaIndexer == nil {
|
||||
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
snap := s.schemaIndexer.Snapshot()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(snap)
|
||||
}
|
||||
|
||||
// handleSchemaRescan re-emits a set event for every existing property via the
|
||||
// meta publisher, waits for the indexer to drain, and returns the refreshed
|
||||
// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
|
||||
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
|
||||
if s.schemaIndexer == nil {
|
||||
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
count, err := s.RepublishMetadata()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
s.schemaIndexer.WaitForDrain(10 * time.Second)
|
||||
|
||||
snap := s.schemaIndexer.Snapshot()
|
||||
snap.Source = "rescan"
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"published_events": count,
|
||||
"schema": snap,
|
||||
})
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/metazla/meta-core/internal/events"
|
||||
"github.com/metazla/meta-core/internal/leader"
|
||||
"github.com/metazla/meta-core/internal/mounts"
|
||||
"github.com/metazla/meta-core/internal/schema"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
"github.com/metazla/meta-core/internal/watcher"
|
||||
"github.com/metazla/meta-core/internal/watchers"
|
||||
@@ -37,6 +38,7 @@ type Server struct {
|
||||
watchersPoller *watchers.Poller
|
||||
watchersHandlers *watchers.Handlers
|
||||
metaPublisher *events.MetaPublisher
|
||||
schemaIndexer *schema.Indexer
|
||||
webdavHandler *webdav.Handler
|
||||
router *mux.Router
|
||||
server *http.Server
|
||||
@@ -163,6 +165,10 @@ func (s *Server) setupRoutes() {
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
|
||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
|
||||
|
||||
// Schema inference
|
||||
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
|
||||
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
|
||||
|
||||
// KV Browser API routes
|
||||
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
|
||||
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
|
||||
@@ -194,6 +200,7 @@ func (s *Server) setupRoutes() {
|
||||
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
|
||||
|
||||
// File operations (by CID)
|
||||
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
|
||||
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
|
||||
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
|
||||
|
||||
@@ -285,6 +292,15 @@ func (s *Server) Start() error {
|
||||
if err := s.metaPublisher.Start(); err != nil {
|
||||
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
|
||||
} else {
|
||||
// Schema indexer consumes the same stream to derive a live
|
||||
// per-field schema (type/hint/breakdown). Start it before the
|
||||
// republish flood so it captures the bootstrap events.
|
||||
s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
|
||||
if err := s.schemaIndexer.Start(); err != nil {
|
||||
log.Printf("[API] Warning: failed to start schema indexer: %v", err)
|
||||
s.schemaIndexer = nil
|
||||
}
|
||||
|
||||
// Republish events for all existing metadata
|
||||
go func() {
|
||||
count, err := s.RepublishMetadata()
|
||||
@@ -321,6 +337,13 @@ func (s *Server) Stop() error {
|
||||
s.mountStatsPoller.Stop()
|
||||
}
|
||||
|
||||
// Stop schema indexer before the publisher so the consumer drains cleanly
|
||||
if s.schemaIndexer != nil {
|
||||
if err := s.schemaIndexer.Stop(); err != nil {
|
||||
log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop meta publisher
|
||||
if s.metaPublisher != nil {
|
||||
if err := s.metaPublisher.Stop(); err != nil {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cidRegex matches the CID shapes produced by @metazla/meta-hash:
|
||||
// - CIDv1 multibase-base32: every algorithm (crc32, md5, sha1, sha2-256,
|
||||
// sha3-256, sha3-384, midhash256, btih-v2, …) is serialized via
|
||||
// CID.createV1(code, digest).toString(), which always yields a lowercase
|
||||
// base32 string starting with "ba" (the "b" is the multibase tag, the
|
||||
// "a" is the CIDv1 version byte under base32-encoding-of-0x01).
|
||||
// - CIDv0 base58btc: "Qm" + 44 chars, kept for compatibility with other
|
||||
// services that may still emit v0.
|
||||
//
|
||||
// Minimum CIDv1 length 16 chars matches the shortest real output we observed
|
||||
// (crc32: "bafk3eaqerjhiwhy").
|
||||
var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
|
||||
|
||||
// ClassifyValue returns the (primitive, hint) for a Redis value.
|
||||
// exists=false (key missing) and JSON null literal both map to PrimUndefined.
|
||||
// Empty string is a valid string.
|
||||
func ClassifyValue(value string, exists bool) (Primitive, Hint) {
|
||||
if !exists {
|
||||
return PrimUndefined, ""
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "null" {
|
||||
return PrimUndefined, ""
|
||||
}
|
||||
if trimmed == "true" || trimmed == "false" {
|
||||
return PrimBool, ""
|
||||
}
|
||||
if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
|
||||
if isTimestampInt(trimmed) {
|
||||
return PrimInt, HintTimestamp
|
||||
}
|
||||
return PrimInt, ""
|
||||
}
|
||||
if _, err := strconv.ParseFloat(trimmed, 64); err == nil {
|
||||
return PrimFloat, ""
|
||||
}
|
||||
if len(trimmed) > 0 {
|
||||
switch trimmed[0] {
|
||||
case '{':
|
||||
var v map[string]any
|
||||
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||
return PrimJSONObj, ""
|
||||
}
|
||||
case '[':
|
||||
var v []any
|
||||
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||
return PrimJSONArr, ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if cidRegex.MatchString(value) {
|
||||
return PrimString, HintCID
|
||||
}
|
||||
if isTimestampString(value) {
|
||||
return PrimString, HintTimestamp
|
||||
}
|
||||
return PrimString, ""
|
||||
}
|
||||
|
||||
// isTimestampInt accepts unix-seconds in [2001-09, 2100] or unix-millis in the
|
||||
// same range. Anything else is treated as a plain int.
|
||||
func isTimestampInt(s string) bool {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
return false
|
||||
}
|
||||
if n >= 1_000_000_000 && n <= 4_102_444_800 {
|
||||
return true
|
||||
}
|
||||
if n >= 1_000_000_000_000 && n <= 4_102_444_800_000 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isTimestampString(s string) bool {
|
||||
if _, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return true
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", s); err == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// breakdownLabel returns the most specific label for a classified value.
|
||||
// Hints take precedence over primitives.
|
||||
func breakdownLabel(prim Primitive, hint Hint) string {
|
||||
if hint != "" {
|
||||
return string(hint)
|
||||
}
|
||||
return string(prim)
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/events"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
|
||||
|
||||
// fieldState is the per-field accumulator. perKey lets us undo a previous
|
||||
// contribution when the same Redis key is re-SET or DELed.
|
||||
type fieldState struct {
|
||||
counts map[string]int
|
||||
primitives map[Primitive]int
|
||||
hintCounts map[Hint]int
|
||||
typedCount int
|
||||
undefinedCount int
|
||||
languageKeyed bool
|
||||
perKey map[string]labelPair
|
||||
}
|
||||
|
||||
type labelPair struct {
|
||||
prim Primitive
|
||||
hint Hint
|
||||
}
|
||||
|
||||
func newFieldState() *fieldState {
|
||||
return &fieldState{
|
||||
counts: make(map[string]int),
|
||||
primitives: make(map[Primitive]int),
|
||||
hintCounts: make(map[Hint]int),
|
||||
perKey: make(map[string]labelPair),
|
||||
}
|
||||
}
|
||||
|
||||
// Indexer maintains a live field-schema by consuming meta:events.
|
||||
type Indexer struct {
|
||||
client *redis.Client
|
||||
keyPrefix string
|
||||
|
||||
mu sync.RWMutex
|
||||
fields map[string]*fieldState
|
||||
|
||||
processed uint64 // total events processed since start
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
runMu sync.Mutex
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
|
||||
// so the indexer can build absolute Redis keys for GET calls.
|
||||
func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Indexer{
|
||||
client: client,
|
||||
keyPrefix: keyPrefix,
|
||||
fields: make(map[string]*fieldState),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins consuming the meta:events stream from the earliest available ID.
|
||||
func (i *Indexer) Start() error {
|
||||
i.runMu.Lock()
|
||||
if i.running {
|
||||
i.runMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
i.running = true
|
||||
i.runMu.Unlock()
|
||||
|
||||
i.wg.Add(1)
|
||||
go i.consumeLoop()
|
||||
|
||||
log.Println("[SchemaIndexer] Started - consuming meta:events stream")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the consumer goroutine.
|
||||
func (i *Indexer) Stop() error {
|
||||
i.runMu.Lock()
|
||||
if !i.running {
|
||||
i.runMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
i.running = false
|
||||
i.runMu.Unlock()
|
||||
|
||||
i.cancel()
|
||||
i.wg.Wait()
|
||||
log.Println("[SchemaIndexer] Stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProcessedCount returns the total number of stream events processed.
|
||||
// Used by WaitForDrain to detect when a rescan flood has been absorbed.
|
||||
func (i *Indexer) ProcessedCount() uint64 {
|
||||
return atomic.LoadUint64(&i.processed)
|
||||
}
|
||||
|
||||
// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
|
||||
// the timeout elapses. Suitable for use right after RepublishMetadata.
|
||||
func (i *Indexer) WaitForDrain(timeout time.Duration) {
|
||||
deadline := time.Now().Add(timeout)
|
||||
last := i.ProcessedCount()
|
||||
stableSince := time.Now()
|
||||
for time.Now().Before(deadline) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
now := i.ProcessedCount()
|
||||
if now != last {
|
||||
last = now
|
||||
stableSince = time.Now()
|
||||
continue
|
||||
}
|
||||
if time.Since(stableSince) >= 300*time.Millisecond {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Indexer) consumeLoop() {
|
||||
defer i.wg.Done()
|
||||
lastID := "0"
|
||||
for {
|
||||
select {
|
||||
case <-i.ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
|
||||
Streams: []string{events.MetaEventsStream, lastID},
|
||||
Block: 2 * time.Second,
|
||||
Count: 500,
|
||||
}).Result()
|
||||
|
||||
if err != nil {
|
||||
if err == redis.Nil || i.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
log.Printf("[SchemaIndexer] XRead error: %v", err)
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, stream := range res {
|
||||
for _, msg := range stream.Messages {
|
||||
lastID = msg.ID
|
||||
i.processEvent(msg.Values)
|
||||
atomic.AddUint64(&i.processed, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Indexer) processEvent(values map[string]any) {
|
||||
op, _ := values["type"].(string)
|
||||
key, _ := values["key"].(string)
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
fieldPath, keyHint, ok := parseFieldPath(key)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
switch op {
|
||||
case "set":
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
|
||||
cancel()
|
||||
exists := err != redis.Nil
|
||||
if err != nil && err != redis.Nil {
|
||||
// Treat transient errors as "value disappeared" — nothing to record.
|
||||
return
|
||||
}
|
||||
prim, hint := ClassifyValue(val, exists)
|
||||
i.upsert(fieldPath, key, keyHint, prim, hint)
|
||||
case "del", "expire":
|
||||
i.remove(fieldPath, key)
|
||||
}
|
||||
}
|
||||
|
||||
// parseFieldPath turns "file:<hashID>/<propertyPath>" into the logical field
|
||||
// path used as the map key. hashID may itself contain ':' (e.g.
|
||||
// "midhash256:abc123"); the split is on the first '/'.
|
||||
//
|
||||
// When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as
|
||||
// a language code: the field name collapses to the prefix and key_hint is set.
|
||||
func parseFieldPath(key string) (string, KeyHint, bool) {
|
||||
if !strings.HasPrefix(key, "file:") {
|
||||
return "", "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(key, "file:")
|
||||
slash := strings.Index(rest, "/")
|
||||
if slash < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
property := rest[slash+1:]
|
||||
if property == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if last := strings.LastIndex(property, "/"); last >= 0 {
|
||||
tail := property[last+1:]
|
||||
if langCodeRegex.MatchString(tail) {
|
||||
return property[:last], KeyHintLanguageCode, true
|
||||
}
|
||||
}
|
||||
return property, "", true
|
||||
}
|
||||
|
||||
func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
fs, ok := i.fields[fieldPath]
|
||||
if !ok {
|
||||
fs = newFieldState()
|
||||
i.fields[fieldPath] = fs
|
||||
}
|
||||
if keyHint != "" {
|
||||
fs.languageKeyed = true
|
||||
}
|
||||
|
||||
if old, ok := fs.perKey[redisKey]; ok {
|
||||
decContribution(fs, old)
|
||||
}
|
||||
|
||||
fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
|
||||
incContribution(fs, labelPair{prim: prim, hint: hint})
|
||||
}
|
||||
|
||||
func (i *Indexer) remove(fieldPath, redisKey string) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
fs, ok := i.fields[fieldPath]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
old, ok := fs.perKey[redisKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(fs.perKey, redisKey)
|
||||
decContribution(fs, old)
|
||||
if len(fs.perKey) == 0 {
|
||||
delete(i.fields, fieldPath)
|
||||
}
|
||||
}
|
||||
|
||||
func incContribution(fs *fieldState, lp labelPair) {
|
||||
label := breakdownLabel(lp.prim, lp.hint)
|
||||
fs.counts[label]++
|
||||
if lp.prim == PrimUndefined {
|
||||
fs.undefinedCount++
|
||||
return
|
||||
}
|
||||
fs.typedCount++
|
||||
fs.primitives[lp.prim]++
|
||||
if lp.hint != "" {
|
||||
fs.hintCounts[lp.hint]++
|
||||
}
|
||||
}
|
||||
|
||||
func decContribution(fs *fieldState, lp labelPair) {
|
||||
label := breakdownLabel(lp.prim, lp.hint)
|
||||
fs.counts[label]--
|
||||
if fs.counts[label] <= 0 {
|
||||
delete(fs.counts, label)
|
||||
}
|
||||
if lp.prim == PrimUndefined {
|
||||
fs.undefinedCount--
|
||||
return
|
||||
}
|
||||
fs.typedCount--
|
||||
fs.primitives[lp.prim]--
|
||||
if fs.primitives[lp.prim] <= 0 {
|
||||
delete(fs.primitives, lp.prim)
|
||||
}
|
||||
if lp.hint != "" {
|
||||
fs.hintCounts[lp.hint]--
|
||||
if fs.hintCounts[lp.hint] <= 0 {
|
||||
delete(fs.hintCounts, lp.hint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot returns the current schema. Safe to call concurrently with consumption.
|
||||
func (i *Indexer) Snapshot() *SchemaResponse {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
out := &SchemaResponse{
|
||||
Fields: make(map[string]*FieldSchema, len(i.fields)),
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
Source: "live",
|
||||
}
|
||||
for fp, fs := range i.fields {
|
||||
out.Fields[fp] = computeFieldSchema(fs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func computeFieldSchema(fs *fieldState) *FieldSchema {
|
||||
out := &FieldSchema{
|
||||
Breakdown: make(map[string]int, len(fs.counts)),
|
||||
}
|
||||
for k, v := range fs.counts {
|
||||
out.Breakdown[k] = v
|
||||
}
|
||||
|
||||
nPrim := len(fs.primitives)
|
||||
hasUndef := fs.undefinedCount > 0
|
||||
|
||||
switch {
|
||||
case nPrim == 0 && hasUndef:
|
||||
out.Type = PrimUndefined
|
||||
case nPrim == 1 && !hasUndef:
|
||||
for p := range fs.primitives {
|
||||
out.Type = p
|
||||
}
|
||||
default:
|
||||
out.Type = PrimMixed
|
||||
}
|
||||
|
||||
// Strict hint promotion: exactly one hint observed, and it covers every
|
||||
// typed (non-undefined) value. Hints are never set on mixed fields.
|
||||
if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 {
|
||||
for h, c := range fs.hintCounts {
|
||||
if c == fs.typedCount {
|
||||
out.Hint = h
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fs.languageKeyed {
|
||||
out.KeyHint = KeyHintLanguageCode
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package schema
|
||||
|
||||
import "time"
|
||||
|
||||
// Primitive is the coarse value type used by the schema indexer.
|
||||
// PrimMixed appears only as an aggregate label on a field, never per value.
|
||||
type Primitive string
|
||||
|
||||
const (
|
||||
PrimString Primitive = "string"
|
||||
PrimInt Primitive = "int"
|
||||
PrimFloat Primitive = "float"
|
||||
PrimBool Primitive = "bool"
|
||||
PrimJSONObj Primitive = "json-object"
|
||||
PrimJSONArr Primitive = "json-array"
|
||||
PrimUndefined Primitive = "undefined"
|
||||
PrimMixed Primitive = "mixed"
|
||||
)
|
||||
|
||||
// Hint is a semantic refinement on top of a primitive (e.g. cid, timestamp).
|
||||
// A hint is promoted onto a field only when every non-undefined value matches it.
|
||||
type Hint string
|
||||
|
||||
const (
|
||||
HintCID Hint = "cid"
|
||||
HintTimestamp Hint = "timestamp"
|
||||
)
|
||||
|
||||
// KeyHint describes the shape of the subkey dimension when a field is keyed
|
||||
// by a structured discriminator (e.g. title/eng, title/fre → language-code).
|
||||
type KeyHint string
|
||||
|
||||
const KeyHintLanguageCode KeyHint = "language-code"
|
||||
|
||||
// FieldSchema is the per-field entry returned by GET /api/schema.
|
||||
type FieldSchema struct {
|
||||
Type Primitive `json:"type"`
|
||||
Hint Hint `json:"hint,omitempty"`
|
||||
KeyHint KeyHint `json:"key_hint,omitempty"`
|
||||
Breakdown map[string]int `json:"breakdown"`
|
||||
}
|
||||
|
||||
// SchemaResponse is the top-level payload for GET /api/schema.
|
||||
type SchemaResponse struct {
|
||||
Fields map[string]*FieldSchema `json:"fields"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
+79
-17
@@ -321,8 +321,15 @@ func (c *Client) CountFiles() (int, error) {
|
||||
return len(hashIDs), nil
|
||||
}
|
||||
|
||||
// LookupPathByCID searches all file metadata for a matching CID
|
||||
// Returns the file path if found in poster/posterPath or backdrop/backdropPath
|
||||
// LookupPathByCID resolves a CID to its file path on disk.
|
||||
//
|
||||
// Resolution covers three cases:
|
||||
// - midhash256 hashID itself: the CID identifies a file's content, return its filePath.
|
||||
// - poster/backdrop fields: the CID is for the image asset, return posterPath/backdropPath.
|
||||
// - any cid_* field: the CID is another digest of the same file, return its filePath.
|
||||
//
|
||||
// Walks every indexed hash; acceptable for admin/UI use, but would benefit from
|
||||
// a reverse index (cid → hashID/field) if call volume grows.
|
||||
func (c *Client) LookupPathByCID(cid string) (string, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
@@ -338,31 +345,45 @@ func (c *Client) LookupPathByCID(cid string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Get all hash IDs first
|
||||
hashIDs, err := c.getAllHashIDsInternal(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// For each hash, check poster and backdrop CIDs using flat keys
|
||||
for _, hashID := range hashIDs {
|
||||
prefix := c.buildKeyPrefix(hashID)
|
||||
meta, err := c.getMetadataFlatInternal(ctx, hashID)
|
||||
if err != nil || len(meta) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check poster
|
||||
posterCID, err := c.client.Get(ctx, prefix+"poster").Result()
|
||||
if err == nil && posterCID == cid {
|
||||
posterPath, err := c.client.Get(ctx, prefix+"posterPath").Result()
|
||||
if err == nil && posterPath != "" {
|
||||
return posterPath, nil
|
||||
// midhash256 == hashID: the file itself.
|
||||
if hashID == cid {
|
||||
if fp := meta["filePath"]; fp != "" {
|
||||
return normalizeFilesRelativePath(fp), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Check backdrop
|
||||
backdropCID, err := c.client.Get(ctx, prefix+"backdrop").Result()
|
||||
if err == nil && backdropCID == cid {
|
||||
backdropPath, err := c.client.Get(ctx, prefix+"backdropPath").Result()
|
||||
if err == nil && backdropPath != "" {
|
||||
return backdropPath, nil
|
||||
// Poster / backdrop refer to a sidecar image asset.
|
||||
if meta["poster"] == cid {
|
||||
if p := meta["posterPath"]; p != "" {
|
||||
return normalizeFilesRelativePath(p), nil
|
||||
}
|
||||
}
|
||||
if meta["backdrop"] == cid {
|
||||
if p := meta["backdropPath"]; p != "" {
|
||||
return normalizeFilesRelativePath(p), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Any other cid_* field is an additional digest of the file itself.
|
||||
for field, value := range meta {
|
||||
if value != cid {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(field, "cid_") {
|
||||
if fp := meta["filePath"]; fp != "" {
|
||||
return normalizeFilesRelativePath(fp), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -370,6 +391,47 @@ func (c *Client) LookupPathByCID(cid string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
|
||||
// safely filepath.Join(FilesPath, …) without double-prefixing. Different
|
||||
// plugins store paths inconsistently — poster/backdrop use FilesPath-relative
|
||||
// ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…").
|
||||
func normalizeFilesRelativePath(p string) string {
|
||||
return strings.TrimPrefix(p, "/files")
|
||||
}
|
||||
|
||||
// getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock.
|
||||
// Callers must already hold c.mu.
|
||||
func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) {
|
||||
prefix := c.buildKeyPrefix(hashID)
|
||||
result := make(map[string]string)
|
||||
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan failed: %w", err)
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
values, err := c.client.MGet(ctx, keys...).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mget failed: %w", err)
|
||||
}
|
||||
for i, key := range keys {
|
||||
if values[i] == nil {
|
||||
continue
|
||||
}
|
||||
field := strings.TrimPrefix(key, prefix)
|
||||
result[field] = values[i].(string)
|
||||
}
|
||||
}
|
||||
cursor = nextCursor
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// getAllHashIDsInternal is an internal version that doesn't acquire locks
|
||||
// (caller must hold the lock)
|
||||
func (c *Client) getAllHashIDsInternal(ctx context.Context) ([]string, error) {
|
||||
|
||||
Reference in New Issue
Block a user