Refactor Docker Build, Celery Config, and Task Streaming for Cross-Platform Reliability

.gitignore

@@ -19,3 +19,5 @@ env/
 .vscode/
 
 /*.env
+
+/*regions.md

Dockerfile

@@ -1,150 +1,165 @@
-# Use an official Python runtime as a base image
-FROM python:3.10-slim AS build
+# Build stage with UV
+FROM ghcr.io/astral-sh/uv:0.8.22 AS uv
 
-# Set environment variables to production
+# Builder stage
+FROM python:3.12-slim AS builder
+
+# Set build-time environment variables
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONHASHSEED=random \
     PYTHONUNBUFFERED=1 \
-    PIP_NO_CACHE_DIR=1 \
     PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_DEFAULT_TIMEOUT=100 \
     DEBIAN_FRONTEND=noninteractive \
+    UV_COMPILE_BYTECODE=1 \
+    UV_NO_INSTALLER_METADATA=1 \
+    UV_LINK_MODE=copy
+
+ARG APP_HOME=/app
+WORKDIR ${APP_HOME}
+
+# Install build dependencies
+COPY requirements.txt .
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    gcc \
+    g++ \
+    python3-dev \
+    pkg-config \
+    libjpeg-dev \
+    cmake \
+    && rm -rf /var/lib/apt/lists/*
+
+# Use UV to build wheels
+RUN --mount=from=uv,source=/uv,target=/bin/uv \
+    --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system -r requirements.txt
+
+# Final stage
+FROM python:3.12-slim
+
+# Metadata
+LABEL maintainer="Prokopis Antoniadis" \
+      description="🔥🕷️ Crawl4AI: LLM Web Crawler & scraper" \
+      version="0.1.0"
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
     PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
-    PYTHON_ENV=production 
+    PYTHON_ENV=production \
+    UV_LINK_MODE=copy \
+    DISPLAY=:99
 
+# Set build arguments
 ARG APP_HOME=/app
-ARG PYTHON_VERSION=3.12
-ARG INSTALL_TYPE=default
-ARG ENABLE_GPU=false
-ARG TARGETARCH
-
-
-# Add Maintainer Info
-LABEL maintainer="Prokopis Antoniadis"
-LABEL description="🔥🕷️ Crawl4AI: LLM Web Crawler & scraper"
-LABEL version="1.0"
-
-# RUN apt-get update && apt-get install -y --no-install-recommends \
-#     build-essential \
-#     curl \
-#     wget \
-#     gnupg \
-#     cmake \
-#     pkg-config \
-#     python3-dev \
-#     libjpeg-dev \
-#     supervisor \
-#     && apt-get clean \ 
-#     && rm -rf /var/lib/apt/lists/*
-
-# Install minimal system dependencies for Playwright and Python
-RUN apt-get update -y && \
-    apt-get upgrade -y && \
-    apt-get install -y --no-install-recommends \
+
+# Install UV in final stage
+COPY --from=uv /uv /usr/local/bin/uv
+RUN chmod +x /usr/local/bin/uv
+
+WORKDIR ${APP_HOME}
+
+# Install system dependencies in a single layer
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    fonts-liberation \
+    ca-certificates \
+    lsof \
+    # Add build dependencies for madoka
+    build-essential \ 
+    curl \
+    gcc \ 
+    g++ \ 
+    python3-dev \
+    pkg-config \
+    libjpeg-dev \
+    cmake \
+    wget \
+    gnupg \
+    supervisor \
+    # Playwright system dependencies
+    libglib2.0-0 \
     libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxcb1 \
+    libxkbcommon0 \
+    libx11-6 \
     libxcomposite1 \
-    libxcursor1 \
     libxdamage1 \
+    libxext6 \
+    libxfixes3 \
     libxrandr2 \
-    libdrm2 \
     libgbm1 \
-    libxss1 \
+    libpango-1.0-0 \
+    libcairo2 \
     libasound2 \
-    libatk1.0-0 \
-    libatk-bridge2.0-0 \
+    libatspi2.0-0 \
+    libxcursor1 \
+    libxss1 \
     libgtk-3-0 \
     xvfb \
     x11vnc \
     git \
-    curl fonts-liberation ca-certificates lsof \
-    && git clone https://github.com/novnc/noVNC /opt/noVNC \
-    && git clone https://github.com/novnc/websockify /opt/noVNC/utils/websockify \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# RUN apt-get update && apt-get dist-upgrade -y \
-#     && rm -rf /var/lib/apt/lists/*
-
-# RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
-#     apt-get update && apt-get install -y --no-install-recommends \
-#     nvidia-cuda-toolkit \
-#     && apt-get clean \ 
-#     && rm -rf /var/lib/apt/lists/* ; \
-# else \
-#     echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
-# fi
-
-# RUN if [ "$TARGETARCH" = "arm64" ]; then \
-#     echo "🦾 Installing ARM-specific optimizations"; \
-#     apt-get update && apt-get install -y --no-install-recommends \
-#     libopenblas-dev \
-#     && apt-get clean \ 
-#     && rm -rf /var/lib/apt/lists/*; \
-# elif [ "$TARGETARCH" = "amd64" ]; then \
-#     echo "🖥️ Installing AMD64-specific optimizations"; \
-#     apt-get update && apt-get install -y --no-install-recommends \
-#     libomp-dev \
-#     && apt-get clean \ 
-#     && rm -rf /var/lib/apt/lists/*; \
-# else \
-#     echo "Skipping platform-specific optimizations (unsupported platform)"; \
-# fi
-
-# Create a non-root user and group
-# RUN groupadd -r appuser && useradd --no-log-init -r -g appuser appuser
-
-# Create and set permissions for appuser home directory
-# RUN mkdir -p /home/appuser && chown -R appuser:appuser /home/appuser
-
-# Set the working directory inside the container
-WORKDIR ${APP_HOME}
-
-# Copy the requirements file and install Python dependencies
-# Copy supervisor config first (might need root later, but okay for now)
-# COPY supervisord.conf .
-
-COPY requirements.txt .
-COPY config.yml .
-
-RUN pip install --no-cache-dir -r requirements.txt && \
-    pip install --no-cache-dir playwright
-
-RUN pip install --no-cache-dir --upgrade pip && \
-    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
+    # Add sudo for X11 management
+    && git clone --depth 1 https://github.com/novnc/noVNC /opt/noVNC \
+    && git clone --depth 1 https://github.com/novnc/websockify /opt/noVNC/utils/websockify \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /var/cache/apt/*
+
+# Create non-root user
+RUN groupadd -r appuser && \
+    useradd --no-log-init -r -g appuser appuser && \
+    mkdir -p /home/appuser/.cache /ms-playwright && \
+    chown -R appuser:appuser /home/appuser /ms-playwright ${APP_HOME}
+
+# Install Python dependencies using UV
+# COPY --from=builder /app/wheels /wheels
+
+# Copy dependencies and install
+COPY --from=builder ${APP_HOME}/requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system -r requirements.txt && \
+    uv pip install --system playwright && \
+    playwright install --with-deps chromium
+
+# Verify installations
+RUN python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
     python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
 
+# Copy application code
+COPY --chown=appuser:appuser . .
+COPY --chown=appuser:appuser config.yml .
 
-# RUN crawl4ai-setup
-
-# https://playwright.dev/docs/browsers
-# Install only the required Playwright browser (Chromium)
-RUN playwright install chromium
-
-# RUN playwright install --with-deps
+# Set display environment variable
+# ENV DISPLAY=:99
 
+# Run diagnostics
 RUN crawl4ai-doctor
 
-# RUN mkdir -p /home/appuser/.cache/ms-playwright \
-#     && cp -r /root/.cache/ms-playwright/chromium-* /home/appuser/.cache/ms-playwright/ \
-#     && chown -R appuser:appuser /home/appuser/.cache/ms-playwright
+# Expose ports
+EXPOSE 8000 9222 6080
 
-# Copy the application code into the container
-COPY . .
-# Change ownership of the application directory to the non-root user
-# RUN chown -R appuser:appuser ${APP_HOME}
+# Healthcheck dont need, fly io do this for us
+# HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+#     CMD curl -f http://localhost:8000/health || exit 1
 
-# Switch to the non-root user before starting the application
-# USER appuser
+# Copy and set permissions for the entrypoint script
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \
+    chown root:root /usr/local/bin/docker-entrypoint.sh && \
+    ls -la /usr/local/bin/docker-entrypoint.sh  # Verify permissions
 
-# # Install only the required Playwright browser (Chromium)
-# RUN playwright install chromium
+# Switch to non-root user
+USER appuser
 
-# Expose the port your FastAPI app will run on
-EXPOSE 8000 9222 6080
+# Set the entrypoint
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
 
-# Command to run the application
-# CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--ws", "websockets"]
-# CMD ["sh", "-c", "Xvfb :99 -screen 0 1280x720x24 & export DISPLAY=:99 && uvicorn server:app --host 0.0.0.0 --port 8000 --ws websockets"]
-CMD ["sh", "-c", "Xvfb :99 -screen 0 1280x720x24 & x11vnc -display :99 -nopw -forever -shared -rfbport 5900 -quiet & /opt/noVNC/utils/websockify/run 6080 localhost:5900 --web /opt/noVNC & uvicorn server:app --host 0.0.0.0 --port 8000 --ws websockets"]
-# Start the application using supervisord
-# CMD ["supervisord", "-c", "supervisord.conf"]
+# Start application
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--ws", "websockets"]