file event improvement using midhash
This commit is contained in:
Pierre
@@ -0,0 +1,68 @@
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Tag for the image (optional, defaults to sha)'
|
||||
required: false
|
||||
type: string
|
||||
meta_core_version:
|
||||
description: 'meta-core version to use (default: latest)'
|
||||
required: false
|
||||
type: string
|
||||
default: 'latest'
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
META_CORE_IMAGE=ghcr.io/worph/meta-core:${{ inputs.meta_core_version || 'latest' }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -0,0 +1,2 @@
|
||||
node_modules
|
||||
packages/meta-dup-core/dist
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
# meta-dup standalone Docker image
|
||||
# Includes: Redis (via meta-core), nginx, Node.js backend, React UI
|
||||
#
|
||||
# Build standalone:
|
||||
# docker build -t meta-dup .
|
||||
#
|
||||
# Build with custom meta-core (for development):
|
||||
# docker build --build-arg META_CORE_IMAGE=meta-core:local -t meta-dup .
|
||||
|
||||
# Stage 0: Get meta-core binary from published image
|
||||
ARG META_CORE_IMAGE=ghcr.io/worph/meta-core:latest
|
||||
FROM ${META_CORE_IMAGE} AS meta-core
|
||||
|
||||
# Stage 1: Build UI
|
||||
FROM node:21-alpine AS ui-builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy UI package and install
|
||||
COPY packages/meta-dup-ui/package.json ./
|
||||
RUN npm install
|
||||
|
||||
# Copy UI source and build
|
||||
COPY packages/meta-dup-ui/ ./
|
||||
RUN npm run build
|
||||
|
||||
# Stage 2: Build Backend
|
||||
FROM node:21-alpine AS backend-builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy backend package and install
|
||||
COPY packages/meta-dup-core/package.json ./
|
||||
RUN npm install
|
||||
|
||||
# Copy backend source and build
|
||||
COPY packages/meta-dup-core/tsconfig.json ./
|
||||
COPY packages/meta-dup-core/src/ ./src/
|
||||
RUN npm run build
|
||||
|
||||
# Stage 3: Runtime
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# Container registry metadata
|
||||
LABEL org.opencontainers.image.source=https://github.com/worph/meta-dup
|
||||
LABEL org.opencontainers.image.description="MetaMesh duplicate detection service"
|
||||
LABEL org.opencontainers.image.licenses=MIT
|
||||
|
||||
# Avoid prompts during install
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
nginx \
|
||||
supervisor \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Node.js 21
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - \
|
||||
&& apt-get install -y nodejs
|
||||
|
||||
# Create directories
|
||||
RUN mkdir -p \
|
||||
/app/backend \
|
||||
/app/ui \
|
||||
/var/log/supervisor \
|
||||
/var/log/nginx
|
||||
|
||||
# Copy built UI
|
||||
COPY --from=ui-builder /build/dist /app/ui
|
||||
RUN chmod -R 755 /app/ui
|
||||
|
||||
# Copy built backend
|
||||
COPY --from=backend-builder /build/dist /app/backend/dist
|
||||
COPY --from=backend-builder /build/node_modules /app/backend/node_modules
|
||||
COPY --from=backend-builder /build/package.json /app/backend/
|
||||
|
||||
# Copy meta-core sidecar binary
|
||||
COPY --from=meta-core /usr/local/bin/meta-core /usr/local/bin/meta-core
|
||||
RUN chmod +x /usr/local/bin/meta-core
|
||||
|
||||
# Copy configuration files
|
||||
COPY docker/nginx.conf /etc/nginx/nginx.conf
|
||||
COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
|
||||
# Environment variables
|
||||
ENV NODE_ENV=production \
|
||||
REDIS_PREFIX="" \
|
||||
API_HOST=0.0.0.0 \
|
||||
API_PORT=3000
|
||||
|
||||
# Expose port 80 (nginx)
|
||||
EXPOSE 80
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD curl -f http://localhost/health || exit 1
|
||||
|
||||
# Start supervisord (manages meta-core, backend, nginx)
|
||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
||||
@@ -0,0 +1,65 @@
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
keepalive_timeout 65;
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
|
||||
|
||||
# Upstream servers
|
||||
upstream api {
|
||||
server 127.0.0.1:3000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
# API routes
|
||||
location /api/ {
|
||||
proxy_pass http://api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Health check (direct to API)
|
||||
location /health {
|
||||
proxy_pass http://api/health;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# Static assets
|
||||
location /assets/ {
|
||||
alias /app/ui/assets/;
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# React SPA at root (catch-all)
|
||||
location / {
|
||||
alias /app/ui/;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
user=root
|
||||
logfile=/var/log/supervisor/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
loglevel=info
|
||||
|
||||
[program:meta-core]
|
||||
command=/usr/local/bin/meta-core
|
||||
priority=5
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
startsecs=3
|
||||
environment=
|
||||
META_CORE_PATH="/meta-core",
|
||||
FILES_PATH="/files",
|
||||
SERVICE_NAME="meta-dup",
|
||||
SERVICE_VERSION="1.0.0",
|
||||
META_CORE_HTTP_PORT="9000"
|
||||
|
||||
[program:nginx]
|
||||
command=/usr/sbin/nginx -g "daemon off;"
|
||||
priority=20
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/var/log/supervisor/nginx.log
|
||||
stderr_logfile=/var/log/supervisor/nginx_error.log
|
||||
startsecs=2
|
||||
|
||||
[program:meta-dup]
|
||||
command=/usr/bin/node /app/backend/dist/index.js
|
||||
directory=/app/backend
|
||||
priority=30
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
startsecs=5
|
||||
environment=
|
||||
NODE_ENV="production",
|
||||
REDIS_PREFIX="",
|
||||
API_HOST="0.0.0.0",
|
||||
API_PORT="3000"
|
||||
@@ -27,7 +27,7 @@ export interface StreamMessage {
|
||||
// Direct file event fields (from meta-core)
|
||||
path?: string;
|
||||
size?: string;
|
||||
partialHash?: string;
|
||||
midhash256?: string; // midhash256 CID computed by meta-core
|
||||
oldPath?: string;
|
||||
// Legacy payload field (for backward compatibility)
|
||||
payload?: string;
|
||||
@@ -657,7 +657,7 @@ export class RedisClient implements Partial<IKVClient> {
|
||||
type: fieldMap.type as StreamMessage['type'],
|
||||
path: fieldMap.path,
|
||||
size: fieldMap.size,
|
||||
partialHash: fieldMap.partialHash,
|
||||
midhash256: fieldMap.midhash256,
|
||||
oldPath: fieldMap.oldPath,
|
||||
payload: fieldMap.payload,
|
||||
timestamp: fieldMap.timestamp || '0',
|
||||
|
||||
-660
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Vendored
-13
@@ -1,13 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>meta-dup - Duplicate Detection</title>
|
||||
<script type="module" crossorigin src="/assets/index-GT6exvk4.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user