diff --git a/README.md b/README.md index f426c7b..a495da2 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ the `/file/{cid}` reverse-lookup path.) ```bash GET /api/metadata/hash-ids +GET /api/files/tuples GET /api/metadata/list POST /api/metadata/search POST /api/metadata/batch diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index 7a8bd7d..b57dbac 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -301,6 +301,7 @@ CID-addressed (public, auth-bypassed at the perimeter) Editor / KV / schema / snapshot GET /api/metadata/hash-ids + GET /api/files/tuples GET /api/metadata/list POST /api/metadata/search POST /api/metadata/batch diff --git a/internal/api/files_tuples.go b/internal/api/files_tuples.go new file mode 100644 index 0000000..d2a5b35 --- /dev/null +++ b/internal/api/files_tuples.go @@ -0,0 +1,74 @@ +package api + +import "net/http" + +// fileTupleJSON is one file-backed root on the wire. +type fileTupleJSON struct { + HashID string `json:"hashId"` + FilePath string `json:"filePath"` + SizeByte int64 `json:"sizeByte"` + MtimeNano int64 `json:"mtimeNano"` +} + +// FileTuplesResponse is the body of GET /api/files/tuples. +// +// Files is a pointer so the summary form can omit it while the full form +// always carries it — an empty library is `"files": []`, never an absent key, +// which a client could mistake for a meta-core that lacks the rows. +type FileTuplesResponse struct { + Files *[]fileTupleJSON `json:"files,omitempty"` + Count int `json:"count"` + TotalSize int64 `json:"totalSize"` +} + +// handleGetFileTuples handles GET /api/files/tuples. +// +// Lists the FILE-BACKED roots — records carrying filePath, sizeByte and +// mtimeNano, i.e. what the watcher or /api/files/register minted — with their +// sizes, plus the count and total. Gateway / cid-rooted records carry no file +// and are not included; neither is a locator record that only merged a cache +// file's metadata (it has no mtimeNano). +// +// It exists so a client never walks every hashId to answer "how many files, +// how big": GetTuplesForAllFiles reads the three fields for every root in +// chunked MGETs server-side (~250 round-trips for 82k roots, no SCAN), where +// the client-side equivalent was one HTTP request per record. +// +// ?summary=1 omits the rows and returns only {count, totalSize}. +func (s *Server) handleGetFileTuples(w http.ResponseWriter, r *http.Request) { + if !s.storage.IsConnected() { + writeError(w, http.StatusServiceUnavailable, "storage not connected") + return + } + + tuples, err := s.storage.GetTuplesForAllFiles() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + summary := r.URL.Query().Get("summary") + withRows := summary == "" || summary == "0" || summary == "false" + + resp := FileTuplesResponse{Count: len(tuples)} + var rows []fileTupleJSON + if withRows { + rows = make([]fileTupleJSON, 0, len(tuples)) + } + for _, t := range tuples { + resp.TotalSize += t.Size + if withRows { + rows = append(rows, fileTupleJSON{ + HashID: t.HashID, + FilePath: t.FilePath, + SizeByte: t.Size, + MtimeNano: t.MtimeNano, + }) + } + } + if withRows { + resp.Files = &rows + } + + writeJSON(w, http.StatusOK, resp) +} diff --git a/internal/api/files_tuples_test.go b/internal/api/files_tuples_test.go new file mode 100644 index 0000000..d38c91d --- /dev/null +++ b/internal/api/files_tuples_test.go @@ -0,0 +1,114 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sort" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/metazla/meta-core/internal/storage" +) + +func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) { + t.Helper() + mr := miniredis.RunT(t) + c := storage.NewClient("") + if err := c.Connect("redis://" + mr.Addr()); err != nil { + t.Fatalf("connect to miniredis: %v", err) + } + t.Cleanup(func() { _ = c.Close() }) + return &Server{storage: c}, c +} + +func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesResponse) { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/files/tuples"+query, nil) + rr := httptest.NewRecorder() + s.handleGetFileTuples(rr, req) + var resp FileTuplesResponse + _ = json.Unmarshal(rr.Body.Bytes(), &resp) + return rr.Code, resp +} + +func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string { + t.Helper() + id, err := c.Mint(path, size, mtime) + if err != nil { + t.Fatalf("mint %s: %v", path, err) + } + return id +} + +// Only records that are actually files count: a gateway record and a record +// that merely carries a path + size (no mtimeNano) are both left out. +func TestFileTuples_OnlyFileBackedRoots(t *testing.T) { + s, c := newFileTuplesTestServer(t) + a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1) + b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2) + if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil { + t.Fatal(err) + } + if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil { + t.Fatal(err) + } + + code, resp := getFileTuples(t, s, "") + if code != http.StatusOK { + t.Fatalf("want 200, got %d", code) + } + if resp.Count != 2 || resp.TotalSize != 3500 { + t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize) + } + if resp.Files == nil || len(*resp.Files) != 2 { + t.Fatalf("want 2 rows, got %+v", resp.Files) + } + got := []string{(*resp.Files)[0].HashID, (*resp.Files)[1].HashID} + want := []string{a, b} + sort.Strings(got) + sort.Strings(want) + if got[0] != want[0] || got[1] != want[1] { + t.Fatalf("want roots %v, got %v", want, got) + } +} + +func TestFileTuples_SummaryOmitsRows(t *testing.T) { + s, c := newFileTuplesTestServer(t) + mustMint(t, c, "/files/watch/a.mkv", 1000, 1) + mustMint(t, c, "/files/watch/b.mkv", 2500, 2) + + code, resp := getFileTuples(t, s, "?summary=1") + if code != http.StatusOK { + t.Fatalf("want 200, got %d", code) + } + if resp.Files != nil { + t.Fatalf("summary must omit rows, got %d", len(*resp.Files)) + } + if resp.Count != 2 || resp.TotalSize != 3500 { + t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize) + } +} + +// An empty library still carries "files": [] — clients treat an absent key as +// "this meta-core cannot list files", so it must not disappear when empty. +func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) { + s, _ := newFileTuplesTestServer(t) + + code, resp := getFileTuples(t, s, "") + if code != http.StatusOK { + t.Fatalf("want 200, got %d", code) + } + if resp.Files == nil || len(*resp.Files) != 0 || resp.Count != 0 { + t.Fatalf("want an empty files array and count 0, got %+v", resp) + } +} + +func TestFileTuples_StorageNotConnected(t *testing.T) { + s := &Server{storage: storage.NewClient("")} + + code, _ := getFileTuples(t, s, "") + if code != http.StatusServiceUnavailable { + t.Fatalf("want 503, got %d", code) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 6b0c75a..a1e3b17 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -247,6 +247,10 @@ func (s *Server) setupRoutes() { // mint + alias + file:events path for a single file. Registered // unconditionally: it exists for boxes whose watcher is OFF. s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST") + // File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched + // server-side, so clients answer "how many files, how big" without one + // read per hashId. ?summary=1 returns only {count, totalSize}. + s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET") // Public metadata-by-CID resolver. Reverse-index lookup → full document. // Auth-bypassed alongside /api/file/{cid} so meta-share peers can query