Compare commits

..
2 Commits
Author SHA1 Message Date
Worph 4894810fd4 new file tuple API
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 12:01:33 +02:00
Worph ac21315255 improved cid resolution
Build and Push Docker Image / build (push) Failing after 13s
2026-09-12 21:37:38 +02:00
10 changed files with 535 additions and 41 deletions
+1
View File
@@ -209,6 +209,7 @@ the `/file/{cid}` reverse-lookup path.)
```bash ```bash
GET /api/metadata/hash-ids GET /api/metadata/hash-ids
GET /api/files/tuples
GET /api/metadata/list GET /api/metadata/list
POST /api/metadata/search POST /api/metadata/search
POST /api/metadata/batch POST /api/metadata/batch
+1
View File
@@ -301,6 +301,7 @@ CID-addressed (public, auth-bypassed at the perimeter)
Editor / KV / schema / snapshot Editor / KV / schema / snapshot
GET /api/metadata/hash-ids GET /api/metadata/hash-ids
GET /api/files/tuples
GET /api/metadata/list GET /api/metadata/list
POST /api/metadata/search POST /api/metadata/search
POST /api/metadata/batch POST /api/metadata/batch
+132
View File
@@ -0,0 +1,132 @@
package api
import (
"encoding/json"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/metazla/meta-core/internal/watcher"
)
// registerFileRequest is the body of POST /api/files/register.
type registerFileRequest struct {
// Path relative to FILES_PATH, e.g. "plugin/subtitles/ab12….eng.srt".
Path string `json:"path"`
}
type registerFileResponse struct {
Root string `json:"root"`
CID string `json:"cid"`
Created bool `json:"created"`
Path string `json:"path"`
Size int64 `json:"size"`
}
// pluginRelPath normalises a FILES_PATH-relative path and refuses anything
// outside plugin/. Plugin output is the only tree a service may register this
// way: /files/watch is the user's library and belongs to the watcher.
func pluginRelPath(raw string) (string, bool) {
p := strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(raw)), "/")
if p == "" {
return "", false
}
for _, seg := range strings.Split(p, "/") {
if seg == ".." {
return "", false
}
}
clean := path.Clean(p)
if !strings.HasPrefix(clean, "plugin/") || clean == "plugin/" {
return "", false
}
return clean, true
}
// handleRegisterFile handles POST /api/files/register.
//
// Does for ONE file what the watcher does for every file it scans: midhash256
// it, mint a root carrying filePath/sizeByte/mtimeNano, register the
// cid:<midhash> alias, and append a file:events entry so meta-share's
// ipfs_seed chunks, seeds and provides it.
//
// It exists for cores that run with ENABLE_FILE_WATCHER=false (a meta-watch
// client box). Without it, bytes written under /files/plugin over WebDAV are
// never resolvable by CID and never reach the swarm. Idempotent: the same
// content at the same path returns the existing root with created=false.
func (s *Server) handleRegisterFile(w http.ResponseWriter, r *http.Request) {
if s.storage == nil || !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req registerFileRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
rel, ok := pluginRelPath(req.Path)
if !ok {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
fullPath := filepath.Join(s.config.FilesPath, filepath.FromSlash(rel))
absFiles, _ := filepath.Abs(filepath.Join(s.config.FilesPath, "plugin"))
absFull, _ := filepath.Abs(fullPath)
if !strings.HasPrefix(absFull, absFiles+string(filepath.Separator)) {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
info, err := os.Stat(fullPath)
if err != nil {
writeError(w, http.StatusNotFound, "file not found")
return
}
if info.IsDir() {
writeError(w, http.StatusBadRequest, "path is a directory, not a file")
return
}
midhash, err := watcher.ComputeMidHash256(fullPath, info.Size())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to hash file")
return
}
root, created, err := s.storage.RegisterFileTuple(fullPath, info.Size(), info.ModTime().UnixNano(), midhash)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if created {
ev := watcher.FileEvent{
Type: watcher.EventTypeAdd,
Path: rel,
Size: info.Size(),
MidHash256: midhash,
Timestamp: watcher.NowMS(),
}
if err := watcher.PublishEvent(s.storage, ev); err != nil {
// The record is written and resolvable; only seeding is late.
writeError(w, http.StatusInternalServerError, "registered, but file event not published: "+err.Error())
return
}
}
status := http.StatusOK
if created {
status = http.StatusCreated
}
writeJSON(w, status, registerFileResponse{
Root: root,
CID: midhash,
Created: created,
Path: rel,
Size: info.Size(),
})
}
+144
View File
@@ -0,0 +1,144 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/storage"
)
// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
// which is exactly the configuration this route exists for.
func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
files := t.TempDir()
return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
}
func writeFixture(t *testing.T, files, rel, body string) string {
t.Helper()
full := filepath.Join(files, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return full
}
func postRegister(t *testing.T, s *Server, path string) (int, registerFileResponse) {
t.Helper()
body, _ := json.Marshal(registerFileRequest{Path: path})
req := httptest.NewRequest(http.MethodPost, "/api/files/register", strings.NewReader(string(body)))
rr := httptest.NewRecorder()
s.handleRegisterFile(rr, req)
var resp registerFileResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
t.Helper()
entries, err := mr.Stream("file:events")
if err != nil {
return 0
}
return len(entries)
}
// The whole point of the route: with the watcher off, a file written under
// plugin/ becomes a record resolvable by its midhash, and meta-share hears
// about it on file:events.
func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
rel := "plugin/subtitles/ab12cd34.eng.srt"
full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
code, resp := postRegister(t, s, rel)
if code != http.StatusCreated || !resp.Created {
t.Fatalf("want 201 created, got %d %+v", code, resp)
}
if !strings.HasPrefix(resp.CID, "b") || resp.Root == "" {
t.Fatalf("want a multibase cid and a root, got %+v", resp)
}
root, err := s.storage.GetByCID(resp.CID)
if err != nil || root != resp.Root {
t.Fatalf("cid alias must resolve to the root: got %q err=%v, want %q", root, err, resp.Root)
}
if fp, _ := s.storage.GetProperty(resp.Root, "filePath"); fp != full {
t.Fatalf("filePath = %q, want %q", fp, full)
}
entries, err := mr.Stream("file:events")
if err != nil || len(entries) != 1 {
t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
}
vals := strings.Join(entries[0].Values, "|")
if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
}
}
// Re-uploading the same subtitle is a no-op: same root, no second event (which
// would make meta-share re-chunk and re-provide the file).
func TestRegisterFile_IsIdempotent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
rel := "plugin/subtitles/same.eng.srt"
writeFixture(t, files, rel, "same bytes")
_, first := postRegister(t, s, rel)
code, second := postRegister(t, s, rel)
if code != http.StatusOK || second.Created {
t.Fatalf("second register must be 200 created=false, got %d %+v", code, second)
}
if second.Root != first.Root || second.CID != first.CID {
t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
}
if n := streamLen(t, mr); n != 1 {
t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
}
}
// Only plugin output may be registered here. /files/watch is the user's
// library and belongs to the watcher; traversal must not escape either.
func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
writeFixture(t, files, "watch/movie.mkv", "x")
for _, p := range []string{
"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
"plugin/../watch/movie.mkv", "../etc/passwd", "plugin/../../etc/passwd",
} {
if code, _ := postRegister(t, s, p); code != http.StatusBadRequest {
t.Errorf("path %q: want 400, got %d", p, code)
}
}
}
func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
t.Errorf("missing file: want 404, got %d", code)
}
if err := os.MkdirAll(filepath.Join(files, "plugin", "subtitles", "dir"), 0o755); err != nil {
t.Fatal(err)
}
if code, _ := postRegister(t, s, "plugin/subtitles/dir"); code != http.StatusBadRequest {
t.Errorf("directory: want 400, got %d", code)
}
}
+74
View File
@@ -0,0 +1,74 @@
package api
import "net/http"
// fileTupleJSON is one file-backed root on the wire.
type fileTupleJSON struct {
HashID string `json:"hashId"`
FilePath string `json:"filePath"`
SizeByte int64 `json:"sizeByte"`
MtimeNano int64 `json:"mtimeNano"`
}
// FileTuplesResponse is the body of GET /api/files/tuples.
//
// Files is a pointer so the summary form can omit it while the full form
// always carries it — an empty library is `"files": []`, never an absent key,
// which a client could mistake for a meta-core that lacks the rows.
type FileTuplesResponse struct {
Files *[]fileTupleJSON `json:"files,omitempty"`
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
}
// handleGetFileTuples handles GET /api/files/tuples.
//
// Lists the FILE-BACKED roots — records carrying filePath, sizeByte and
// mtimeNano, i.e. what the watcher or /api/files/register minted — with their
// sizes, plus the count and total. Gateway / cid-rooted records carry no file
// and are not included; neither is a locator record that only merged a cache
// file's metadata (it has no mtimeNano).
//
// It exists so a client never walks every hashId to answer "how many files,
// how big": GetTuplesForAllFiles reads the three fields for every root in
// chunked MGETs server-side (~250 round-trips for 82k roots, no SCAN), where
// the client-side equivalent was one HTTP request per record.
//
// ?summary=1 omits the rows and returns only {count, totalSize}.
func (s *Server) handleGetFileTuples(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
tuples, err := s.storage.GetTuplesForAllFiles()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
summary := r.URL.Query().Get("summary")
withRows := summary == "" || summary == "0" || summary == "false"
resp := FileTuplesResponse{Count: len(tuples)}
var rows []fileTupleJSON
if withRows {
rows = make([]fileTupleJSON, 0, len(tuples))
}
for _, t := range tuples {
resp.TotalSize += t.Size
if withRows {
rows = append(rows, fileTupleJSON{
HashID: t.HashID,
FilePath: t.FilePath,
SizeByte: t.Size,
MtimeNano: t.MtimeNano,
})
}
}
if withRows {
resp.Files = &rows
}
writeJSON(w, http.StatusOK, resp)
}
+114
View File
@@ -0,0 +1,114 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sort"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return &Server{storage: c}, c
}
func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesResponse) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/api/files/tuples"+query, nil)
rr := httptest.NewRecorder()
s.handleGetFileTuples(rr, req)
var resp FileTuplesResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
t.Helper()
id, err := c.Mint(path, size, mtime)
if err != nil {
t.Fatalf("mint %s: %v", path, err)
}
return id
}
// Only records that are actually files count: a gateway record and a record
// that merely carries a path + size (no mtimeNano) are both left out.
func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
s, c := newFileTuplesTestServer(t)
a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil {
t.Fatal(err)
}
if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil {
t.Fatal(err)
}
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
if resp.Files == nil || len(*resp.Files) != 2 {
t.Fatalf("want 2 rows, got %+v", resp.Files)
}
got := []string{(*resp.Files)[0].HashID, (*resp.Files)[1].HashID}
want := []string{a, b}
sort.Strings(got)
sort.Strings(want)
if got[0] != want[0] || got[1] != want[1] {
t.Fatalf("want roots %v, got %v", want, got)
}
}
func TestFileTuples_SummaryOmitsRows(t *testing.T) {
s, c := newFileTuplesTestServer(t)
mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
code, resp := getFileTuples(t, s, "?summary=1")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files != nil {
t.Fatalf("summary must omit rows, got %d", len(*resp.Files))
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
}
// An empty library still carries "files": [] — clients treat an absent key as
// "this meta-core cannot list files", so it must not disappear when empty.
func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
s, _ := newFileTuplesTestServer(t)
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files == nil || len(*resp.Files) != 0 || resp.Count != 0 {
t.Fatalf("want an empty files array and count 0, got %+v", resp)
}
}
func TestFileTuples_StorageNotConnected(t *testing.T) {
s := &Server{storage: storage.NewClient("")}
code, _ := getFileTuples(t, s, "")
if code != http.StatusServiceUnavailable {
t.Fatalf("want 503, got %d", code)
}
}
+8
View File
@@ -243,6 +243,14 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET") s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET") s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST") s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
// Register one file under /files/plugin by midhash — the watcher's
// mint + alias + file:events path for a single file. Registered
// unconditionally: it exists for boxes whose watcher is OFF.
s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST")
// File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched
// server-side, so clients answer "how many files, how big" without one
// read per hashId. ?summary=1 returns only {count, totalSize}.
s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET")
// Public metadata-by-CID resolver. Reverse-index lookup → full document. // Public metadata-by-CID resolver. Reverse-index lookup → full document.
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query // Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
+47
View File
@@ -422,3 +422,50 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
Duplicates: dups, Duplicates: dups,
}, nil }, nil
} }
// RegisterFileTuple records one physical file under its midhash256 CID, the
// same way the file watcher does for every file it scans: mint a UUID root
// (filePath/sizeByte/mtimeNano) and write `cids/<midhash>`, whose hook
// registers the cid:<midhash> reverse alias. Content already known under
// another path is recorded as a duplicate; the same path just refreshes
// size/mtime.
//
// Returns the root and whether it was newly minted. Shared by the watcher and
// POST /api/files/register, which is how a box running with the watcher off
// (a meta-watch client core) still gets files it writes under /files/plugin
// resolvable by CID.
func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
if midhash == "" {
return "", false, fmt.Errorf("midhash is required")
}
uuid, err := c.GetByCID(midhash)
if err != nil {
return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if uuid == "" {
uuid, err = c.Mint(absPath, size, mtimeNano)
if err != nil {
return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
}
if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
}
return uuid, true, nil
}
existing, _ := c.GetProperty(uuid, "filePath")
if existing != absPath {
if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
}
return uuid, false, nil
}
if _, err := c.MergeMetadataFlat(uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return uuid, false, nil
}
+12 -4
View File
@@ -50,8 +50,18 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
client := d.storageClient client := d.storageClient
d.mu.RUnlock() d.mu.RUnlock()
if err := PublishEvent(client, event); err != nil {
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
}
}
// PublishEvent appends one file event to the file:events stream. Exported so a
// caller with no Dispatcher (POST /api/files/register on a box whose watcher is
// off) emits exactly the entry the watcher would — meta-share's ipfs_seed
// consumes this stream to chunk, seed and provide new files.
func PublishEvent(client *storage.Client, event FileEvent) error {
if client == nil || !client.IsConnected() { if client == nil || !client.IsConnected() {
return return fmt.Errorf("storage client not connected")
} }
// Build stream fields // Build stream fields
@@ -79,9 +89,7 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
// `EmitReset → ClearStream` path is still the only way the stream gets // `EmitReset → ClearStream` path is still the only way the stream gets
// truncated to zero (signalled to consumers via the `reset` event). // truncated to zero (signalled to consumers via the `reset` event).
_, err := client.XAdd(EventsStream, EventsStreamMaxLen, fields) _, err := client.XAdd(EventsStream, EventsStreamMaxLen, fields)
if err != nil { return err
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
}
} }
// EmitReset clears the stream and emits a reset event // EmitReset clears the stream and emits a reset event
+2 -37
View File
@@ -4,7 +4,6 @@ import (
"log" "log"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"sync" "sync"
@@ -54,42 +53,8 @@ func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash
return return
} }
uuid, err := w.storage.GetByCID(midhash) if _, _, err := w.storage.RegisterFileTuple(absPath, size, mtimeNano, midhash); err != nil {
if err != nil { log.Printf("[Watcher] register %s: %v", absPath, err)
log.Printf("[Watcher] GetByCID(%s): %v", midhash, err)
return
}
if uuid == "" {
// New content. Mint sets filePath/sizeByte/mtimeNano; SetProperty
// then writes the midhash as a cids/<cid> key-set member — and the
// cid_resolution.go hook registers the reverse-index alias for us.
uuid, err = w.storage.Mint(absPath, size, mtimeNano)
if err != nil {
log.Printf("[Watcher] Mint(%s): %v", absPath, err)
return
}
if err := w.storage.SetProperty(uuid, storage.CIDsKeyPrefix+midhash, "true"); err != nil {
log.Printf("[Watcher] SetProperty cids/%s for %s: %v", midhash, uuid, err)
}
return
}
// Known content. If this is a new physical path for the same content,
// record it as a duplicate. Otherwise refresh size/mtime — same content,
// same path, but the file may have been rewritten with a fresh mtime.
existing, _ := w.storage.GetProperty(uuid, "filePath")
if existing != absPath {
if _, err := w.storage.AddDuplicatePath(uuid, absPath); err != nil {
log.Printf("[Watcher] AddDuplicatePath %s @ %s: %v", uuid, absPath, err)
}
return
}
if _, err := w.storage.MergeMetadataFlat(uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
log.Printf("[Watcher] refresh tuple for %s: %v", uuid, err)
} }
} }