From d0417e0056c57cccd9660adc66caaad94df5789b Mon Sep 17 00:00:00 2001 From: dextmorgn Date: Mon, 27 Oct 2025 11:15:47 +0100 Subject: [PATCH] feat: upgrade deploy to use docker only --- .dockerignore | 73 ++++++++ .env.example | 1 + Makefile | 47 ++++-- docker-compose.dev.yml | 159 ++++++++++++++++++ docker-compose.prod.yml | 148 ++++++++++++++++ flowsint-api/Dockerfile | 55 ++++-- flowsint-api/entrypoint.sh | 16 +- flowsint-app/Dockerfile | 24 +-- flowsint-app/vite.config.ts | 6 + .../src/flowsint_core/core/config.py | 4 +- .../src/flowsint_core/core/events.py | 2 +- .../src/flowsint_core/core/graph_db.py | 1 - .../src/flowsint_core/tasks/event.py | 4 +- flowsint-core/src/flowsint_core/tasks/flow.py | 1 - .../src/flowsint_core/tasks/transform.py | 1 - 15 files changed, 484 insertions(+), 58 deletions(-) create mode 100644 .dockerignore create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.prod.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b8594be --- /dev/null +++ b/.dockerignore @@ -0,0 +1,73 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ +*.egg +.eggs/ + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Git +.git/ +.gitignore +.gitattributes + +# Documentation +*.md +!README.md +docs/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.mypy_cache/ +.hypothesis/ + +# Node (for frontend) +flowsint-app/node_modules/ +flowsint-app/dist/ +flowsint-app/.next/ +flowsint-app/out/ +flowsint-app/build/ + +# Environment files +.env.local +.env.*.local + +# Logs +*.log +logs/ + +# Docker +docker-compose*.yml +Dockerfile.dev +.dockerignore + +# CI/CD +.github/ +.gitlab-ci.yml + +# Misc +*.bak +*.tmp +temp/ +tmp/ diff --git a/.env.example b/.env.example index aa6db3d..cc6fc2b 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,7 @@ NEO4J_USERNAME=neo4j NEO4J_PASSWORD=password VITE_API_URL=http://127.0.0.1:5001 DATABASE_URL=postgresql://flowsint:flowsint@localhost:5433/flowsint +REDIS_URL=redis://redis:6379/0 HIBP_API_KEY= ETHERSCAN_API_KEY= WHOXY_API_KEY= diff --git a/Makefile b/Makefile index 55b023a..dbab9d9 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,34 @@ PROJECT_ROOT := $(shell pwd) -.PHONY: install run stop infra api frontend celery clean dev check-env +.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser ENV_DIRS := . flowsint-api flowsint-core flowsint-app +open-browser: + @echo "⏳ Waiting for frontend to be ready..." + @bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done' + @echo "🌐 Opening browser..." + @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ Flowsint ready at http://localhost:5173" dev: + @echo "🚀 Starting Flowsint in DEVELOPMENT mode..." $(MAKE) check-env - $(MAKE) install - $(MAKE) run + docker compose -f docker-compose.dev.yml up --build -d + $(MAKE) open-browser + docker compose -f docker-compose.dev.yml logs -f prod: - # temporary + @echo "🚀 Starting Flowsint in PRODUCTION mode..." $(MAKE) check-env - $(MAKE) install - $(MAKE) run + docker compose -f docker-compose.prod.yml up --build -d + $(MAKE) open-browser + +build-dev: + @echo "🔨 Building development images..." + docker compose -f docker-compose.dev.yml build + +build-prod: + @echo "🔨 Building production images..." + docker compose -f docker-compose.prod.yml build check-env: @echo "🔎 Checking .env files..." @@ -68,18 +83,30 @@ run: @echo "⏳ Waiting for frontend to be ready..." @bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done' @echo "🌐 Opening browser..." - @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ All services ready! Frontend at http://localhost:5173" + @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ All services ready! Flowsint at http://localhost:5173" $(MAKE) -j2 api celery stop: @echo "🛑 Stopping all services..." + -docker compose -f docker-compose.dev.yml down + -docker compose -f docker-compose.prod.yml down -docker compose down -# --- Nettoyage complet --- +stop-dev: + @echo "🛑 Stopping development services..." + docker compose -f docker-compose.dev.yml down + +stop-prod: + @echo "🛑 Stopping production services..." + docker compose -f docker-compose.prod.yml down + clean: - @echo "Removing containers, volumes and venvs..." - docker compose down -v --remove-orphans + @echo "🧹 Removing containers, volumes and venvs..." + -docker compose -f docker-compose.dev.yml down -v --remove-orphans + -docker compose -f docker-compose.prod.yml down -v --remove-orphans + -docker compose down -v --remove-orphans rm -rf $(PROJECT_ROOT)/flowsint-app/node_modules rm -rf $(PROJECT_ROOT)/flowsint-core/.venv rm -rf $(PROJECT_ROOT)/flowsint-transforms/.venv rm -rf $(PROJECT_ROOT)/flowsint-api/.venv + @echo "✅ Cleanup complete!" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..f4481cd --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,159 @@ +name: flowsint-dev + +services: + # PostgreSQL database + postgres: + image: postgres:15 + container_name: flowsint-postgres-dev + restart: always + environment: + POSTGRES_USER: flowsint + POSTGRES_PASSWORD: flowsint + POSTGRES_DB: flowsint + ports: + - "5433:5432" + volumes: + - pg_data_dev:/var/lib/postgresql/data + networks: + - flowsint_network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U flowsint"] + interval: 10s + timeout: 5s + retries: 5 + + # Redis for Celery & cache + redis: + image: redis:alpine + container_name: flowsint-redis-dev + ports: + - "6379:6379" + networks: + - flowsint_network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # Neo4j graph database + neo4j: + image: neo4j:5 + container_name: flowsint-neo4j-dev + ports: + - "7474:7474" # Web UI + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_apoc_export_file_enabled=true + - NEO4J_apoc_import_file_enabled=true + - NEO4J_apoc_import_file_use__neo4j__config=true + volumes: + - neo4j_data_dev:/data + - neo4j_logs_dev:/logs + - neo4j_import_dev:/var/lib/neo4j/import + - neo4j_plugins_dev:/plugins + restart: unless-stopped + networks: + - flowsint_network + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + + # FastAPI Backend + api: + build: + context: . + dockerfile: flowsint-api/Dockerfile + target: dev + container_name: flowsint-api-dev + restart: unless-stopped + ports: + - "5001:5001" + volumes: + - ./flowsint-api:/app/flowsint-api + - ./flowsint-core:/app/flowsint-core + - ./flowsint-types:/app/flowsint-types + - ./flowsint-transforms:/app/flowsint-transforms + environment: + - DATABASE_URL=postgresql://flowsint:flowsint@postgres:5432/flowsint + - NEO4J_URI_BOLT=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME} + - NEO4J_PASSWORD=${NEO4J_PASSWORD} + - AUTH_SECRET=${AUTH_SECRET} + - HIBP_API_KEY=${HIBP_API_KEY} + - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY} + - WHOXY_API_KEY=${WHOXY_API_KEY} + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + neo4j: + condition: service_healthy + networks: + - flowsint_network + + # Celery Worker + celery: + build: + context: . + dockerfile: flowsint-api/Dockerfile + target: dev + container_name: flowsint-celery-dev + restart: unless-stopped + command: celery -A flowsint_core.core.celery worker --loglevel=info --pool=solo + volumes: + - ./flowsint-api:/app/flowsint-api + - ./flowsint-core:/app/flowsint-core + - ./flowsint-types:/app/flowsint-types + - ./flowsint-transforms:/app/flowsint-transforms + environment: + - DATABASE_URL=postgresql://flowsint:flowsint@postgres:5432/flowsint + - NEO4J_URI_BOLT=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME} + - NEO4J_PASSWORD=${NEO4J_PASSWORD} + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + neo4j: + condition: service_healthy + networks: + - flowsint_network + + # Frontend + app: + build: + context: ./flowsint-app + dockerfile: Dockerfile.dev + container_name: flowsint-app-dev + ports: + - "5173:5173" + volumes: + - ./flowsint-app:/app + - /app/node_modules + environment: + - VITE_API_URL=http://localhost:5001 + networks: + - flowsint_network + stdin_open: true + tty: true + +networks: + flowsint_network: + name: flowsint_network_dev + driver: bridge + +volumes: + pg_data_dev: + neo4j_data_dev: + neo4j_logs_dev: + neo4j_import_dev: + neo4j_plugins_dev: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..6d497d6 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,148 @@ +name: flowsint-prod + +services: + # PostgreSQL database + postgres: + image: postgres:15 + container_name: flowsint-postgres-prod + restart: always + environment: + POSTGRES_USER: ${POSTGRES_USER:-flowsint} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flowsint} + POSTGRES_DB: ${POSTGRES_DB:-flowsint} + ports: + - "5433:5432" + volumes: + - pg_data_prod:/var/lib/postgresql/data + networks: + - flowsint_network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-flowsint}"] + interval: 10s + timeout: 5s + retries: 5 + + # Redis for Celery & cache + redis: + image: redis:alpine + container_name: flowsint-redis-prod + restart: always + ports: + - "6379:6379" + networks: + - flowsint_network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # Neo4j graph database + neo4j: + image: neo4j:5 + container_name: flowsint-neo4j-prod + restart: always + ports: + - "7474:7474" # Web UI + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_apoc_export_file_enabled=true + - NEO4J_apoc_import_file_enabled=true + - NEO4J_apoc_import_file_use__neo4j__config=true + volumes: + - neo4j_data_prod:/data + - neo4j_logs_prod:/logs + - neo4j_import_prod:/var/lib/neo4j/import + - neo4j_plugins_prod:/plugins + networks: + - flowsint_network + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + + # FastAPI Backend (Production) + api: + build: + context: . + dockerfile: flowsint-api/Dockerfile + target: prod + container_name: flowsint-api-prod + restart: always + ports: + - "5001:5001" + environment: + - DATABASE_URL=postgresql://${POSTGRES_USER:-flowsint}:${POSTGRES_PASSWORD:-flowsint}@postgres:5432/${POSTGRES_DB:-flowsint} + - NEO4J_URI_BOLT=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME} + - NEO4J_PASSWORD=${NEO4J_PASSWORD} + - AUTH_SECRET=${AUTH_SECRET} + - HIBP_API_KEY=${HIBP_API_KEY} + - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY} + - WHOXY_API_KEY=${WHOXY_API_KEY} + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + neo4j: + condition: service_healthy + networks: + - flowsint_network + + # Celery Worker (Production) + celery: + build: + context: . + dockerfile: flowsint-api/Dockerfile + target: prod + container_name: flowsint-celery-prod + restart: always + command: celery -A flowsint_core.core.celery worker --loglevel=info --pool=solo + environment: + - DATABASE_URL=postgresql://${POSTGRES_USER:-flowsint}:${POSTGRES_PASSWORD:-flowsint}@postgres:5432/${POSTGRES_DB:-flowsint} + - NEO4J_URI_BOLT=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME} + - NEO4J_PASSWORD=${NEO4J_PASSWORD} + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + neo4j: + condition: service_healthy + networks: + - flowsint_network + + # Frontend (Production) + app: + build: + context: ./flowsint-app + dockerfile: Dockerfile + args: + - VITE_API_URL=${VITE_API_URL:-http://localhost:5001} + container_name: flowsint-app-prod + restart: always + ports: + - "5173:5173" + networks: + - flowsint_network + depends_on: + - api + +networks: + flowsint_network: + name: flowsint_network_prod + driver: bridge + +volumes: + pg_data_prod: + neo4j_data_prod: + neo4j_logs_prod: + neo4j_import_prod: + neo4j_plugins_prod: diff --git a/flowsint-api/Dockerfile b/flowsint-api/Dockerfile index edfd437..50b7db1 100644 --- a/flowsint-api/Dockerfile +++ b/flowsint-api/Dockerfile @@ -1,16 +1,18 @@ -# --- Production stage --- -FROM python:3.12-slim AS prod +# --- Base stage with Python 3.12 --- +FROM python:3.12-slim AS base # Environment variables ENV PYTHONUNBUFFERED=1 ENV POETRY_VIRTUALENVS_CREATE=false -ENV APP_ENV=prod # Install system dependencies RUN apt-get update && apt-get install -y \ build-essential \ curl \ git \ + libpq-dev \ + pkg-config \ + libcairo2-dev \ && rm -rf /var/lib/apt/lists/* # Install Poetry @@ -19,17 +21,50 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \ WORKDIR /app -# Copy only dependency files to leverage Docker cache -COPY pyproject.toml poetry.lock ./ +# --- Development stage --- +FROM base AS dev -# Install production dependencies only (no dev) -RUN poetry install --no-dev --no-root +ENV APP_ENV=development -# Copy the source code -COPY . . +# Copy all dependency packages +COPY flowsint-core /app/flowsint-core +COPY flowsint-types /app/flowsint-types +COPY flowsint-transforms /app/flowsint-transforms + +# Copy API files +COPY flowsint-api /app/flowsint-api + +WORKDIR /app/flowsint-api + +# Install dependencies with dev packages +RUN poetry install --no-root # Expose FastAPI port EXPOSE 5001 -# Run FastAPI +ENTRYPOINT ["/app/flowsint-api/entrypoint.sh"] +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "5001", "--reload"] + +# --- Production stage --- +FROM base AS prod + +ENV APP_ENV=production + +# Copy all dependency packages +COPY flowsint-core /app/flowsint-core +COPY flowsint-types /app/flowsint-types +COPY flowsint-transforms /app/flowsint-transforms + +# Copy API files +COPY flowsint-api /app/flowsint-api + +WORKDIR /app/flowsint-api + +# Install production dependencies only (no dev) +RUN poetry install + +# Expose FastAPI port +EXPOSE 5001 + +ENTRYPOINT ["/app/flowsint-api/entrypoint.sh"] CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "5001"] diff --git a/flowsint-api/entrypoint.sh b/flowsint-api/entrypoint.sh index c36a0f6..d77c9ba 100755 --- a/flowsint-api/entrypoint.sh +++ b/flowsint-api/entrypoint.sh @@ -1,12 +1,8 @@ #!/bin/sh -set -e # Arrêter le script en cas d'erreur +set -e -if [ "$1" = "app" ]; then - echo "Démarrage de FastAPI..." - exec uvicorn app.main:app --host 0.0.0.0 --port 5001 -elif [ "$1" = "celery" ]; then - echo "Démarrage de Celery..." - exec celery -A app.core.celery worker --loglevel=info -else - exec "$@" -fi +echo "Running database migrations..." +alembic upgrade head + +echo "Starting application..." +exec "$@" diff --git a/flowsint-app/Dockerfile b/flowsint-app/Dockerfile index 956cd3f..68d56ea 100644 --- a/flowsint-app/Dockerfile +++ b/flowsint-app/Dockerfile @@ -1,27 +1,13 @@ -# Build stage -FROM node:20 AS builder +FROM node:24 WORKDIR /app -# Copy package files and install dependencies COPY package*.json ./ -RUN npm install --legacy-peer-deps +RUN yarn install COPY . . -# Type check and build -RUN npm run typecheck -RUN npm run build +RUN yarn build +EXPOSE 5173 -# Production stage -FROM nginx:alpine - -# Copy built files to nginx -COPY --from=builder /app/dist /usr/share/nginx/html - -# Copy nginx configuration -COPY nginx.conf /etc/nginx/conf.d/default.conf - -EXPOSE 80 - -CMD ["nginx", "-g", "daemon off;"] +CMD ["yarn", "preview", "--", "--port", "5173"] diff --git a/flowsint-app/vite.config.ts b/flowsint-app/vite.config.ts index 7b777ca..15d0b1d 100644 --- a/flowsint-app/vite.config.ts +++ b/flowsint-app/vite.config.ts @@ -32,6 +32,7 @@ export default defineConfig({ } }, server: { + open: true, proxy: { '/api': { target: 'http://localhost:5001', @@ -39,5 +40,10 @@ export default defineConfig({ secure: false } } + }, + preview: { + open: false, + host: '0.0.0.0', + port: 5173 } }) diff --git a/flowsint-core/src/flowsint_core/core/config.py b/flowsint-core/src/flowsint_core/core/config.py index bec1c20..cc46c81 100644 --- a/flowsint-core/src/flowsint_core/core/config.py +++ b/flowsint-core/src/flowsint_core/core/config.py @@ -5,8 +5,8 @@ load_dotenv() class Settings: - CELERY_BROKER_URL = os.getenv("REDIS_URI", "redis://localhost:6379/0") - CELERY_RESULT_BACKEND = os.getenv("REDIS_URI", "redis://localhost:6379/0") + CELERY_BROKER_URL = os.environ["REDIS_URL"] + CELERY_RESULT_BACKEND = os.environ["REDIS_URL"] settings = Settings() diff --git a/flowsint-core/src/flowsint_core/core/events.py b/flowsint-core/src/flowsint_core/core/events.py index a2d9a88..c2b633b 100644 --- a/flowsint-core/src/flowsint_core/core/events.py +++ b/flowsint-core/src/flowsint_core/core/events.py @@ -11,7 +11,7 @@ import os class EventEmitter: def __init__(self): self.id = uuid.uuid4() - self.redis = redis.from_url(os.getenv("REDIS_URI", "redis://localhost:6379/0")) + self.redis = redis.from_url(os.environ["REDIS_URL"]) self.pubsubs: Dict[str, redis.client.PubSub] = {} async def subscribe(self, channel: str): diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.py index 84a256c..7b28362 100644 --- a/flowsint-core/src/flowsint_core/core/graph_db.py +++ b/flowsint-core/src/flowsint_core/core/graph_db.py @@ -21,7 +21,6 @@ class Neo4jConnection: URI = os.getenv("NEO4J_URI_BOLT") -URI = "bolt://localhost:7687" USERNAME = os.getenv("NEO4J_USERNAME") PASSWORD = os.getenv("NEO4J_PASSWORD") diff --git a/flowsint-core/src/flowsint_core/tasks/event.py b/flowsint-core/src/flowsint_core/tasks/event.py index 2a2a6bb..9f421eb 100644 --- a/flowsint-core/src/flowsint_core/tasks/event.py +++ b/flowsint-core/src/flowsint_core/tasks/event.py @@ -17,9 +17,7 @@ def emit_event_task(log_id: str, sketch_id: str, log_type: EventLevel, content: event = Event( id=log_id, sketch_id=sketch_id, type=log_type, payload=content ).model_dump_json() - redis_client = redis.from_url( - os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0") - ) + redis_client = redis.from_url(os.environ["REDIS_URL"]) redis_client.publish(sketch_id, event) except Exception as e: raise diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.py index 0b718f2..e55683c 100644 --- a/flowsint-core/src/flowsint_core/tasks/flow.py +++ b/flowsint-core/src/flowsint_core/tasks/flow.py @@ -18,7 +18,6 @@ from flowsint_core.utils import to_json_serializable load_dotenv() URI = os.getenv("NEO4J_URI_BOLT") -URI = "bolt://localhost:7687" USERNAME = os.getenv("NEO4J_USERNAME") PASSWORD = os.getenv("NEO4J_PASSWORD") diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.py index 4a714bb..fe12b5a 100644 --- a/flowsint-core/src/flowsint_core/tasks/transform.py +++ b/flowsint-core/src/flowsint_core/tasks/transform.py @@ -18,7 +18,6 @@ from flowsint_core.utils import to_json_serializable load_dotenv() URI = os.getenv("NEO4J_URI_BOLT") -URI = "bolt://localhost:7687" USERNAME = os.getenv("NEO4J_USERNAME") PASSWORD = os.getenv("NEO4J_PASSWORD")