initial commit of meta-dup

This commit is contained in:
Pierre
2026-02-08 16:29:39 +01:00
parent b3a9adbf3c
commit 2baa9a6925
27 changed files with 6735 additions and 0 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+13
View File
@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>meta-dup - Duplicate Detection</title>
<script type="module" crossorigin src="/assets/index-GT6exvk4.js"></script>
</head>
<body>
<div id="root"></div>
</body>
</html>
+13
View File
@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>meta-dup - Duplicate Detection</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
+24
View File
@@ -0,0 +1,24 @@
{
"name": "@meta-dup/ui",
"version": "1.0.0",
"description": "Dashboard UI for meta-dup",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview",
"lint": "eslint src --ext ts,tsx"
},
"license": "MIT",
"dependencies": {
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
"devDependencies": {
"@types/react": "^18.2.55",
"@types/react-dom": "^18.2.19",
"@vitejs/plugin-react": "^4.2.1",
"typescript": "^5.3.3",
"vite": "^5.0.12"
}
}
+653
View File
@@ -0,0 +1,653 @@
import { useState, useEffect, useCallback } from 'react';
import type { DuplicateData, DuplicateGroup, HealthStatus, FileInfo } from './types';
import { formatNumber, formatDate, getFilename, truncateHash, formatBytes } from './utils/format';
import ServiceNav from './components/ServiceNav';
function App() {
const [duplicateData, setDuplicateData] = useState<DuplicateData | null>(null);
const [health, setHealth] = useState<HealthStatus | null>(null);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [activeTab, setActiveTab] = useState<'hash' | 'title'>('hash');
const [expandedGroups, setExpandedGroups] = useState<Set<string>>(new Set());
const [rebuilding, setRebuilding] = useState(false);
const fetchDuplicates = useCallback(async () => {
try {
const res = await fetch('/api/duplicates');
if (res.ok) {
const data = await res.json();
setDuplicateData(data);
setError(null);
} else {
setError('Failed to fetch duplicate data');
}
} catch (err) {
console.error('Failed to fetch duplicates:', err);
setError('Failed to fetch duplicate data');
}
}, []);
const fetchHealth = useCallback(async () => {
try {
const res = await fetch('/health');
if (res.ok) {
const data = await res.json();
setHealth(data);
}
} catch (err) {
console.error('Failed to fetch health:', err);
}
}, []);
const rebuildIndex = async () => {
setRebuilding(true);
try {
const res = await fetch('/api/duplicates/rebuild', { method: 'POST' });
if (res.ok) {
await fetchDuplicates();
} else {
setError('Failed to rebuild index');
}
} catch (err) {
console.error('Failed to rebuild:', err);
setError('Failed to rebuild index');
} finally {
setRebuilding(false);
}
};
useEffect(() => {
fetchDuplicates();
fetchHealth();
const interval = setInterval(() => {
fetchDuplicates();
fetchHealth();
}, 10000); // Refresh every 10s
return () => clearInterval(interval);
}, [fetchDuplicates, fetchHealth]);
const toggleGroup = (key: string) => {
setExpandedGroups(prev => {
const next = new Set(prev);
if (next.has(key)) {
next.delete(key);
} else {
next.add(key);
}
return next;
});
};
const expandAll = () => {
const groups = activeTab === 'hash'
? duplicateData?.hashDuplicates
: duplicateData?.titleDuplicates;
if (groups) {
setExpandedGroups(new Set(groups.map(g => g.key)));
}
};
const collapseAll = () => {
setExpandedGroups(new Set());
};
const renderFileInfo = (file: FileInfo, idx: number) => (
<div key={file.hashId} className="file-item">
<span className="file-index">{idx + 1}.</span>
<span className="file-name" title={file.filePath}>{getFilename(file.filePath)}</span>
<span className="file-path" title={file.filePath}>{file.filePath}</span>
{file.sizeByte && (
<span className="file-size">{formatBytes(file.sizeByte)}</span>
)}
</div>
);
const renderDuplicateGroup = (group: DuplicateGroup, type: 'hash' | 'title') => {
const isExpanded = expandedGroups.has(group.key);
const displayKey = type === 'hash'
? truncateHash(group.key, 16)
: group.key;
return (
<div key={group.key} className={`duplicate-group ${isExpanded ? 'expanded' : ''}`}>
<div
className="group-header"
onClick={() => toggleGroup(group.key)}
>
<span className="expand-icon">{isExpanded ? '▼' : '▶'}</span>
<span className="group-key" title={group.key}>{displayKey}</span>
<span className="group-count">{group.files.length} files</span>
</div>
{isExpanded && (
<div className="group-files">
{group.files.map((file, idx) => renderFileInfo(file, idx))}
</div>
)}
</div>
);
};
const hashDuplicates = duplicateData?.hashDuplicates || [];
const titleDuplicates = duplicateData?.titleDuplicates || [];
const stats = duplicateData?.stats;
return (
<div className="app">
<header className="header">
<h1>meta-dup</h1>
<span className="subtitle">Duplicate Detection Service</span>
{health && (
<span className={`status-badge ${health.status}`}>
{health.status === 'ok' ? '● Connected' : '○ Degraded'}
</span>
)}
</header>
<main className="main">
<ServiceNav />
{error && (
<div className="error-banner">{error}</div>
)}
{/* Stats Cards */}
{stats && (
<div className="stats-cards">
<div className="stat-card hash">
<div className="stat-icon">#</div>
<div className="stat-content">
<div className="stat-value">{formatNumber(stats.hashGroupCount)}</div>
<div className="stat-label">Hash Duplicate Groups</div>
<div className="stat-detail">{formatNumber(stats.hashFileCount)} total files</div>
</div>
</div>
<div className="stat-card title">
<div className="stat-icon">T</div>
<div className="stat-content">
<div className="stat-value">{formatNumber(stats.titleGroupCount)}</div>
<div className="stat-label">Title Duplicate Groups</div>
<div className="stat-detail">{formatNumber(stats.titleFileCount)} total files</div>
</div>
</div>
<div className="stat-card info">
<div className="stat-icon">i</div>
<div className="stat-content">
<div className="stat-value">{formatNumber(stats.totalFilesTracked)}</div>
<div className="stat-label">Total Files Tracked</div>
<div className="stat-detail">
Last updated: {formatDate(stats.lastUpdated)}
</div>
</div>
</div>
</div>
)}
{/* Action Bar */}
<div className="action-bar">
<button
className="btn btn-primary"
onClick={rebuildIndex}
disabled={rebuilding}
>
{rebuilding ? 'Rebuilding...' : 'Rebuild Index'}
</button>
</div>
{/* Tabs */}
<div className="card duplicates-card">
<div className="tabs-header">
<div className="tabs">
<button
className={`tab ${activeTab === 'hash' ? 'active' : ''}`}
onClick={() => setActiveTab('hash')}
>
Hash Duplicates ({hashDuplicates.length})
</button>
<button
className={`tab ${activeTab === 'title' ? 'active' : ''}`}
onClick={() => setActiveTab('title')}
>
Title Duplicates ({titleDuplicates.length})
</button>
</div>
<div className="tab-actions">
<button className="btn btn-small btn-secondary" onClick={expandAll}>
Expand All
</button>
<button className="btn btn-small btn-secondary" onClick={collapseAll}>
Collapse All
</button>
</div>
</div>
<div className="tab-content">
{activeTab === 'hash' && (
<div className="duplicates-list">
{hashDuplicates.length === 0 ? (
<div className="empty-state">
<p>No hash duplicates found</p>
<p className="hint">Files with identical content (same SHA-256 hash)</p>
</div>
) : (
<>
<p className="list-description">
Files with identical content (exact byte-for-byte matches)
</p>
{hashDuplicates.map(group => renderDuplicateGroup(group, 'hash'))}
</>
)}
</div>
)}
{activeTab === 'title' && (
<div className="duplicates-list">
{titleDuplicates.length === 0 ? (
<div className="empty-state">
<p>No title duplicates found</p>
<p className="hint">Files with similar parsed titles</p>
</div>
) : (
<>
<p className="list-description">
Files with matching parsed titles (may be different quality/releases)
</p>
{titleDuplicates.map(group => renderDuplicateGroup(group, 'title'))}
</>
)}
</div>
)}
</div>
</div>
</main>
<style>{`
:root {
--bg-primary: #0d1117;
--bg-secondary: #161b22;
--bg-tertiary: #21262d;
--text-primary: #f0f6fc;
--text-secondary: #8b949e;
--border-color: #30363d;
--accent-primary: #58a6ff;
--accent-green: #3fb950;
--error: #f85149;
}
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: var(--bg-primary);
color: var(--text-primary);
min-height: 100vh;
}
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.header {
background: var(--bg-secondary);
border-bottom: 1px solid var(--border-color);
padding: 16px 24px;
display: flex;
align-items: center;
gap: 16px;
}
.header h1 {
font-size: 1.5rem;
font-weight: 600;
}
.subtitle {
color: var(--text-secondary);
font-size: 0.9rem;
}
.status-badge {
margin-left: auto;
padding: 4px 12px;
border-radius: 12px;
font-size: 0.85rem;
}
.status-badge.ok {
background: rgba(63, 185, 80, 0.1);
color: var(--accent-green);
}
.status-badge.degraded {
background: rgba(248, 81, 73, 0.1);
color: var(--error);
}
.main {
flex: 1;
padding: 24px;
max-width: 1400px;
margin: 0 auto;
width: 100%;
}
.error-banner {
background: rgba(248, 81, 73, 0.1);
border: 1px solid var(--error);
color: var(--error);
padding: 12px 16px;
border-radius: 8px;
margin-bottom: 20px;
}
/* Stats Cards */
.stats-cards {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 16px;
margin-bottom: 24px;
}
.stat-card {
background: var(--bg-secondary);
border-radius: 12px;
padding: 20px;
display: flex;
gap: 16px;
align-items: center;
border: 1px solid var(--border-color);
}
.stat-card.hash {
border-left: 4px solid #a78bfa;
}
.stat-card.title {
border-left: 4px solid #fbbf24;
}
.stat-card.info {
border-left: 4px solid #60a5fa;
}
.stat-icon {
width: 48px;
height: 48px;
background: var(--bg-tertiary);
border-radius: 12px;
display: flex;
align-items: center;
justify-content: center;
font-size: 1.5rem;
font-weight: bold;
color: var(--text-secondary);
}
.stat-card.hash .stat-icon { color: #a78bfa; }
.stat-card.title .stat-icon { color: #fbbf24; }
.stat-card.info .stat-icon { color: #60a5fa; }
.stat-content { flex: 1; }
.stat-value {
font-size: 1.8rem;
font-weight: 700;
color: var(--text-primary);
}
.stat-label {
color: var(--text-secondary);
font-size: 0.9rem;
}
.stat-detail {
color: var(--text-secondary);
font-size: 0.8rem;
margin-top: 4px;
opacity: 0.7;
}
/* Action Bar */
.action-bar {
display: flex;
gap: 12px;
margin-bottom: 20px;
}
/* Buttons */
.btn {
padding: 10px 20px;
border: none;
border-radius: 8px;
font-size: 1rem;
cursor: pointer;
transition: all 0.2s;
}
.btn-primary {
background: var(--accent-primary);
color: white;
}
.btn-primary:hover:not(:disabled) {
background: #4393e6;
}
.btn-primary:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.btn-secondary {
background: var(--bg-tertiary);
color: var(--text-primary);
border: 1px solid var(--border-color);
}
.btn-secondary:hover {
background: var(--bg-secondary);
}
.btn-small {
padding: 6px 12px;
font-size: 0.85rem;
}
/* Card */
.card {
background: var(--bg-secondary);
border-radius: 12px;
border: 1px solid var(--border-color);
padding: 20px;
}
/* Tabs */
.duplicates-card {
min-height: 400px;
}
.tabs-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
border-bottom: 1px solid var(--border-color);
padding-bottom: 12px;
}
.tabs {
display: flex;
gap: 8px;
}
.tab {
padding: 10px 20px;
background: transparent;
border: none;
color: var(--text-secondary);
font-size: 1rem;
cursor: pointer;
border-radius: 8px 8px 0 0;
transition: all 0.2s;
}
.tab:hover {
color: var(--text-primary);
background: var(--bg-tertiary);
}
.tab.active {
color: var(--accent-primary);
background: var(--bg-tertiary);
font-weight: 600;
}
.tab-actions {
display: flex;
gap: 8px;
}
/* Duplicates List */
.list-description {
color: var(--text-secondary);
font-size: 0.9rem;
margin-bottom: 16px;
}
.duplicates-list {
display: flex;
flex-direction: column;
gap: 8px;
}
.duplicate-group {
background: var(--bg-tertiary);
border-radius: 8px;
overflow: hidden;
}
.group-header {
display: flex;
align-items: center;
gap: 12px;
padding: 12px 16px;
cursor: pointer;
transition: background 0.2s;
}
.group-header:hover {
background: var(--bg-secondary);
}
.expand-icon {
color: var(--text-secondary);
font-size: 0.8rem;
width: 16px;
}
.group-key {
flex: 1;
font-family: monospace;
font-size: 0.9rem;
color: var(--accent-primary);
}
.group-count {
background: var(--bg-secondary);
padding: 4px 10px;
border-radius: 12px;
font-size: 0.8rem;
color: var(--text-secondary);
}
.group-files {
background: var(--bg-secondary);
padding: 8px 16px;
border-top: 1px solid var(--border-color);
}
.file-item {
display: grid;
grid-template-columns: 30px 200px 1fr auto;
gap: 12px;
padding: 8px 0;
border-bottom: 1px solid var(--border-color);
font-size: 0.85rem;
align-items: center;
}
.file-item:last-child {
border-bottom: none;
}
.file-index {
color: var(--text-secondary);
text-align: right;
}
.file-name {
font-weight: 500;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.file-path {
color: var(--text-secondary);
font-family: monospace;
font-size: 0.8rem;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.file-size {
color: var(--text-secondary);
font-size: 0.8rem;
white-space: nowrap;
}
/* Empty State */
.empty-state {
text-align: center;
padding: 60px 20px;
color: var(--text-secondary);
}
.empty-state p {
margin: 0;
}
.empty-state .hint {
font-size: 0.9rem;
margin-top: 8px;
opacity: 0.7;
}
@media (max-width: 800px) {
.tabs-header {
flex-direction: column;
gap: 12px;
align-items: flex-start;
}
.file-item {
grid-template-columns: 30px 1fr;
}
.file-path, .file-size {
grid-column: 1 / -1;
padding-left: 42px;
}
}
`}</style>
</div>
);
}
export default App;
@@ -0,0 +1,170 @@
import { useState, useEffect } from 'react';
interface ServiceInfo {
name: string;
url: string;
api: string;
status: string;
capabilities: string[];
version: string;
role?: string; // "leader", "follower", or undefined (for non-meta-core services)
}
interface ServicesResponse {
services: ServiceInfo[];
current: string;
}
const serviceIcons: Record<string, string> = {
'meta-sort': '📁',
'meta-fuse': '🗂️',
'meta-stremio': '🎬',
'meta-dup': '🔍',
'meta-orbit': '🌐',
'default': '📦'
};
function formatServiceName(name: string): string {
return name.split('-').map(word =>
word.charAt(0).toUpperCase() + word.slice(1)
).join(' ');
}
function ServiceNav() {
const [services, setServices] = useState<ServiceInfo[]>([]);
const [loading, setLoading] = useState(true);
const currentService = 'meta-dup';
useEffect(() => {
const fetchServices = async () => {
try {
const response = await fetch('/api/services');
if (response.ok) {
const data: ServicesResponse = await response.json();
setServices(data.services || []);
}
} catch (error) {
console.error('Failed to fetch services:', error);
} finally {
setLoading(false);
}
};
fetchServices();
const interval = setInterval(fetchServices, 30000);
return () => clearInterval(interval);
}, []);
if (loading) return null;
// Filter out follower meta-core instances (only show leader)
const filteredServices = services.filter(
s => s.name !== 'meta-core' || s.role === 'leader'
);
const sortedServices = [...filteredServices].sort((a, b) => a.name.localeCompare(b.name));
if (sortedServices.length === 0) return null;
return (
<nav className="services-nav">
<span className="services-nav-label">Services:</span>
<div className="services-nav-items">
{sortedServices.map(service => {
const icon = serviceIcons[service.name] || serviceIcons.default;
const isActive = service.name === currentService;
return (
<a
key={service.name}
href={isActive ? '#' : service.url}
className={`service-link${isActive ? ' active' : ''}`}
onClick={(e) => {
e.preventDefault();
if (!isActive) {
window.location.href = service.url;
}
}}
>
<span className="service-icon">{icon}</span>
<span>{formatServiceName(service.name)}</span>
<span className="service-status"></span>
</a>
);
})}
</div>
<style>{`
.services-nav {
background: var(--bg-secondary);
border-radius: 12px;
padding: 0.5rem;
margin-bottom: 1.5rem;
display: flex;
align-items: center;
gap: 0.5rem;
flex-wrap: wrap;
border: 1px solid var(--border-color);
}
.services-nav-label {
color: var(--text-secondary);
font-size: 0.85rem;
padding: 0.5rem 0.75rem;
white-space: nowrap;
}
.services-nav-items {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.service-link {
display: inline-flex;
align-items: center;
gap: 0.4rem;
padding: 0.5rem 1rem;
border-radius: 8px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: all 0.2s;
color: var(--text-primary);
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
}
.service-link:hover {
background: var(--border-color);
border-color: rgba(78, 205, 196, 0.5);
text-decoration: none;
}
.service-link.active {
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
border-color: rgba(78, 205, 196, 0.5);
}
.service-link .service-icon {
font-size: 1.1rem;
}
.service-link .service-status {
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--accent-primary);
margin-left: 0.25rem;
}
@media (max-width: 600px) {
.services-nav {
flex-direction: column;
align-items: flex-start;
}
}
`}</style>
</nav>
);
}
export default ServiceNav;
+9
View File
@@ -0,0 +1,9 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<App />
</React.StrictMode>
);
+48
View File
@@ -0,0 +1,48 @@
/**
* Type definitions for meta-dup-ui
*/
export interface FileInfo {
hashId: string;
filePath: string;
title?: string;
sha256?: string;
sizeByte?: number;
year?: number;
season?: number;
episode?: number;
fileType?: string;
}
export interface DuplicateGroup {
key: string;
files: FileInfo[];
}
export interface DuplicateStats {
hashGroupCount: number;
hashFileCount: number;
titleGroupCount: number;
titleFileCount: number;
totalFilesTracked: number;
lastUpdated: string;
}
export interface DuplicateData {
hashDuplicates: DuplicateGroup[];
titleDuplicates: DuplicateGroup[];
stats: DuplicateStats;
computedAt: string;
}
export interface HealthStatus {
status: 'ok' | 'degraded';
service: string;
redis: boolean;
stats: {
filesTracked: number;
hashDuplicates: number;
titleDuplicates: number;
};
timestamp: string;
}
+28
View File
@@ -0,0 +1,28 @@
/**
* Formatting utilities
*/
export function formatNumber(num: number): string {
return num.toLocaleString();
}
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
export function formatDate(dateStr: string): string {
return new Date(dateStr).toLocaleString();
}
export function getFilename(path: string): string {
return path.split('/').pop() || path;
}
export function truncateHash(hash: string, length: number = 16): string {
if (hash.length <= length) return hash;
return `${hash.substring(0, length)}...`;
}
+23
View File
@@ -0,0 +1,23 @@
{
"compilerOptions": {
"target": "ES2022",
"useDefineForClassFields": true,
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"module": "ESNext",
"skipLibCheck": true,
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true
},
"include": ["src"],
"references": [{ "path": "./tsconfig.node.json" }]
}
+10
View File
@@ -0,0 +1,10 @@
{
"compilerOptions": {
"composite": true,
"skipLibCheck": true,
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true
},
"include": ["vite.config.ts"]
}
+23
View File
@@ -0,0 +1,23 @@
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
export default defineConfig({
plugins: [react()],
server: {
port: 5175,
proxy: {
'/api': {
target: 'http://localhost:3000',
changeOrigin: true,
},
'/health': {
target: 'http://localhost:3000',
changeOrigin: true,
},
},
},
build: {
outDir: 'dist',
sourcemap: true,
},
});