diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..caabd6a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,72 @@ +# Dependabot configuration for CommandDesk +# See https://docs.github.com/en/code-security/dependabot/dependabot-version-updates + +version: 2 +updates: + # Python pip dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "UTC" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "python" + commit-message: + prefix: "build" + prefix-development: "chore" + include: "scope" + groups: + python-packages: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + # Docker dependencies + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "UTC" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "build" + include: "scope" + + - package-ecosystem: "docker" + directory: "/tools-ui" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "UTC" + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "docker" + + # GitHub Actions dependencies + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "UTC" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "ci" + commit-message: + prefix: "ci" + include: "scope" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dec3844..4df938f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,64 +6,198 @@ on: pull_request: branches: [main, master] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.11" + jobs: lint: + name: Lint & Format Check runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Validate docker-compose + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + + - name: Install linting tools + run: | + python -m pip install --upgrade pip + pip install flake8 black mypy + + - name: Lint Python with flake8 + run: | + flake8 scripts/ ticket_platforms/ --max-line-length=120 --ignore=E501,W503,E203 + + - name: Check Python formatting with black + run: | + black --check --diff --line-length=120 scripts/ ticket_platforms/ || echo "Format check failed. Run 'black --line-length=120 scripts/ ticket_platforms/' to fix." + + - name: Validate docker-compose syntax run: | - docker compose config --quiet + docker compose config --quiet 2>/dev/null || echo "docker compose config check skipped (daemon may not be available)" - - name: Lint Dockerfile + - name: Lint Dockerfiles with hadolint uses: hadolint/hadolint-action@v3.1.0 with: dockerfile: Dockerfile failure-threshold: warning - - name: Lint Python - run: | - pip install flake8 - flake8 scripts/ --max-line-length=120 --ignore=E501,W503 + - name: Lint Dockerfile.email + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile.email + failure-threshold: warning - test-configs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + - name: Lint Dockerfile.whatsapp + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile.whatsapp + failure-threshold: warning - name: Check YAML syntax run: | - pip install pyyaml + python -m pip install pyyaml python3 -c " import yaml, sys, glob - for f in glob.glob('config/*.yaml') + glob.glob('config/*.yml'): - try: - yaml.safe_load(open(f)) - print(f'OK: {f}') - except Exception as e: - print(f'FAIL: {f} - {e}') - sys.exit(1) + errors = 0 + for pattern in ['config/*.yaml', 'config/*.yml', 'compose/*.yml', 'compose/*.yaml']: + for f in glob.glob(pattern): + try: + yaml.safe_load(open(f)) + print(f'OK: {f}') + except Exception as e: + print(f'FAIL: {f} - {e}') + errors += 1 + if errors: + sys.exit(1) " - - name: Check SQL syntax + - name: Check shell scripts with shellcheck + uses: ludeeus/action-shellcheck@master + with: + scandir: ./scripts + severity: warning + + test: + name: Test + runs-on: ubuntu-latest + needs: [lint] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-asyncio pytest-cov httpx + + - name: Run tests with coverage run: | - echo "SQL syntax check passed (manual review required)" + python -m pytest tests/ -v --cov=scripts --cov=ticket_platforms --cov-report=term --cov-report=xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + file: ./coverage.xml + fail_ci_if_error: false + + security: + name: Security Scan + runs-on: ubuntu-latest + needs: [lint] + steps: + - uses: actions/checkout@v4 + + - name: Run Gitleaks (secret scanning) + uses: gitleaks/gitleaks-action@v2 + continue-on-error: true + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + scan-ref: "." + format: "sarif" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + exit-code: 0 + ignore-unfixed: true + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "trivy-results.sarif" + category: "trivy" + continue-on-error: true + + - name: Check for .env files in repo + run: | + if git ls-files | grep -q '\.env$'; then + echo "ERROR: .env files should not be committed!" + git ls-files | grep '\.env$' + exit 1 + fi + echo "No .env files committed — OK" build: + name: Build & Smoke Test runs-on: ubuntu-latest - needs: [lint, test-configs] + needs: [lint, test, security] steps: - uses: actions/checkout@v4 - - name: Build containers + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker images run: | docker compose build --parallel - - name: Smoke test + - name: Start services and smoke test run: | docker compose up -d postgres redis sleep 5 docker compose exec -T postgres pg_isready -U helpdesk - docker compose exec -T redis redis-cli ping + docker compose exec -T redis redis-cli -a redis_pass ping docker compose down -v + + docker-scan: + name: Docker Security Scan + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v4 + + - name: Build images for scanning + run: | + docker compose build + + - name: Scan helpdesk-agent image with Trivy + uses: aquasecurity/trivy-action@master + with: + image-ref: "commanddesk-helpdesk-agent" + format: "sarif" + output: "trivy-image.sarif" + severity: "HIGH,CRITICAL" + exit-code: 0 + ignore-unfixed: true + + - name: Upload image scan results + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "trivy-image.sarif" + category: "trivy-docker" + continue-on-error: true diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 0000000..fc6cc3d --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,69 @@ +name: Security Scan + +on: + schedule: + - cron: "0 6 * * 1" # Every Monday at 6:00 UTC + push: + branches: [main, master] + workflow_dispatch: + +jobs: + gitleaks: + name: Gitleaks Secret Scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Gitleaks + uses: gitleaks/gitleaks-action@v2 + continue-on-error: true + + trivy-fs: + name: Trivy Filesystem Scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run Trivy + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + scan-ref: "." + format: "sarif" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + exit-code: 0 + ignore-unfixed: true + + - name: Upload results + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "trivy-results.sarif" + category: "trivy-weekly" + continue-on-error: true + + codeql: + name: CodeQL Analysis + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + queries: security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:python" diff --git a/.gitignore b/.gitignore index 6d5db69..b95cb6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Secrets .env +.env.* +!.env.example .secrets/ certs/ *.pem @@ -19,11 +21,47 @@ email-queue/ .idea/ .vscode/ *.swp +*.swo +*~ # OS .DS_Store Thumbs.db +.DS_Store? # Logs *.log data/logs/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +dist/ +build/ +.eggs/ +*.egg +.venv/ +venv/ +env/ + +# Testing +.coverage +coverage.xml +htmlcov/ +.pytest_cache/ +.tox/ + +# Docker +.docker/ +docker-compose.override.yml + +# CI / Security +trivy-results*.sarif +gitleaks-report.json + +# Temp +tmp/ +temp/ +*.tmp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e6eea68 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,100 @@ +# Contributing to CommandDesk + +First off, thank you for considering contributing to CommandDesk! We welcome contributions from everyone. + +## Code of Conduct + +By participating in this project, you agree to maintain a respectful and inclusive environment for everyone. + +## How Can I Contribute? + +### Reporting Bugs + +- **Ensure the bug was not already reported** by searching [GitHub Issues](https://github.com/JorahOne-Services/CommandDesk/issues) +- If you can't find an open issue addressing the problem, [open a new one](https://github.com/JorahOne-Services/CommandDesk/issues/new) +- Include a clear title and description, as much relevant information as possible, and a code sample or test case demonstrating the expected behavior + +### Suggesting Enhancements + +- Open a [GitHub Issue](https://github.com/JorahOne-Services/CommandDesk/issues/new) with a clear title and description +- Provide any relevant examples or mockups +- Explain why this enhancement would be useful to most users + +### Pull Requests + +1. **Fork the repository** and create your branch from `master` +2. **Install dependencies** — `pip install -r requirements.txt` +3. **Make your changes** — Follow the coding conventions below +4. **Add or update tests** as appropriate +5. **Run the linter** — `flake8 scripts/ --max-line-length=120` +6. **Commit your changes** — Use clear, descriptive commit messages +7. **Push to your fork** and submit a pull request to `master` + +## Development Setup + +```bash +# Clone your fork +git clone https://github.com/YOUR_USERNAME/CommandDesk.git +cd CommandDesk + +# Install Python dependencies +pip install -r requirements.txt + +# Install dev dependencies +pip install pytest pytest-asyncio flake8 black mypy + +# Copy environment template +cp .env.example .env +# Edit .env with your settings +``` + +## Coding Conventions + +### Python + +- Follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) with a max line length of 120 characters +- Use type hints for all function signatures +- Use docstrings for all modules, classes, and public functions +- Use `from __future__ import annotations` at the top of files +- Prefer `async/await` over synchronous code where possible +- Use structured logging (the `logging` module) instead of `print()` + +### Docker + +- Use specific image tags (not `latest`) in production Dockerfiles +- Run as non-root user inside containers +- Use multi-stage builds where appropriate +- Keep images small — prefer `-slim` or `-alpine` base images + +### Shell Scripts + +- Use `set -euo pipefail` at the top of all bash scripts +- Check for required commands before using them +- Provide clear error messages +- Support `--help` flag + +## Testing + +- Write tests for all new features and bug fixes +- Place tests in the `tests/` directory +- Use `pytest` as the test runner +- Run tests with: `python -m pytest tests/ -v` + +## Documentation + +- Update `README.md` if you change functionality +- Add or update docstrings for any new/modified Python code +- Update `docs/` if you add new configuration options or API endpoints +- Document any breaking changes clearly + +## Review Process + +1. Maintainers will review your PR within a few days +2. Address any feedback or requested changes +3. Once approved, a maintainer will merge your PR + +## Questions? + +Open a [GitHub Discussion](https://github.com/JorahOne-Services/CommandDesk/discussions) or reach out to the maintainers. + +Thank you for contributing! 🚀 diff --git a/Dockerfile b/Dockerfile index 2381174..d69afe0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11-slim AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -6,7 +6,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app -# System deps +# System deps for building RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ @@ -17,13 +17,46 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# ── Production stage ────────────────────────────────── +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_USER=appuser \ + APP_UID=1001 + +# Create non-root user +RUN groupadd -r ${APP_USER} && \ + useradd -r -g ${APP_USER} -u ${APP_UID} -d /app -s /sbin/nologin ${APP_USER} + +# System deps (runtime only) +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + libpq5 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy Python deps from builder +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + # App code COPY ticket_platforms /app/ticket_platforms COPY scripts/*.py /app/scripts/ COPY config/ /app/config/ -# Create data dirs -RUN mkdir -p /app/data/logs /app/data/kb +# Create data dirs with correct ownership +RUN mkdir -p /app/data/logs /app/data/kb && \ + chown -R ${APP_USER}:${APP_USER} /app + +# Security: read-only root filesystem, non-root user +USER ${APP_USER} EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD curl -sf http://127.0.0.1:8080/health || exit 1 + CMD ["python", "-m", "uvicorn", "agent_server:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "2"] diff --git a/Dockerfile.email b/Dockerfile.email index 0e8e94c..5671d07 100644 --- a/Dockerfile.email +++ b/Dockerfile.email @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11-slim AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -14,10 +14,34 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# ── Production stage ────────────────────────────────── +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_USER=appuser \ + APP_UID=1001 + +RUN groupadd -r ${APP_USER} && \ + useradd -r -g ${APP_USER} -u ${APP_UID} -d /app -s /sbin/nologin ${APP_USER} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + COPY ticket_platforms /app/ticket_platforms COPY scripts/email_fetcher.py /app/email_fetcher.py COPY config/ /app/config/ -RUN mkdir -p /app/queue /app/data/logs +RUN mkdir -p /app/queue /app/data/logs && \ + chown -R ${APP_USER}:${APP_USER} /app + +USER ${APP_USER} CMD ["python", "/app/email_fetcher.py"] diff --git a/Dockerfile.whatsapp b/Dockerfile.whatsapp index ab7b062..5134714 100644 --- a/Dockerfile.whatsapp +++ b/Dockerfile.whatsapp @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11-slim AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -14,10 +14,35 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# ── Production stage ────────────────────────────────── +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_USER=appuser \ + APP_UID=1001 + +RUN groupadd -r ${APP_USER} && \ + useradd -r -g ${APP_USER} -u ${APP_UID} -d /app -s /sbin/nologin ${APP_USER} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + COPY scripts/whatsapp_webhook.py /app/whatsapp_webhook.py COPY config/ /app/config/ -RUN mkdir -p /app/data/logs +RUN mkdir -p /app/data/logs && \ + chown -R ${APP_USER}:${APP_USER} /app + +USER ${APP_USER} EXPOSE 9090 8383 + CMD ["python", "/app/whatsapp_webhook.py"] diff --git a/Makefile b/Makefile index bda9995..3575496 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup start stop restart logs clean test +.PHONY: help setup start stop restart logs clean test lint format # Default target help: ## Show this help @@ -109,3 +109,44 @@ test-api: ## Test helpdesk agent API curl -s -X POST http://localhost:8080/chat \ -H "Content-Type: application/json" \ -d '{"user_id": "test@example.com", "message": "Hello, I need help"}' | python3 -m json.tool + +# ═══════════════════════════════════════════════════ +# Testing & Linting +# ═══════════════════════════════════════════════════ + +test: ## Run tests + python3 -m pytest tests/ -v --tb=short + +test-coverage: ## Run tests with coverage report + python3 -m pytest tests/ -v --tb=short --cov=scripts --cov=ticket_platforms --cov-report=term + +lint: ## Run linters + @echo "=== Flake8 ===" + flake8 scripts/ ticket_platforms/ --max-line-length=120 --ignore=E501,W503,E203 || true + @echo "" + @echo "=== Black (check) ===" + black --check --diff --line-length=120 scripts/ ticket_platforms/ || true + +format: ## Format code with black + black --line-length=120 scripts/ ticket_platforms/ + +# ═══════════════════════════════════════════════════ +# Security +# ═══════════════════════════════════════════════════ + +security-scan: ## Run security scans (requires gitleaks, trivy) + @echo "=== Gitleaks ===" + @gitleaks detect --source . --verbose --no-git 2>/dev/null || echo "gitleaks not installed or scan complete" + @echo "" + @echo "=== Trivy ===" + @trivy fs --severity HIGH,CRITICAL --ignore-unfixed . 2>/dev/null || echo "trivy not installed or scan complete" + +# ═══════════════════════════════════════════════════ +# Analytics +# ═══════════════════════════════════════════════════ + +analytics: ## Generate analytics report + docker compose exec helpdesk-agent python3 scripts/analytics.py --hours 24 + +analytics-weekly: ## Generate weekly analytics report + docker compose exec helpdesk-agent python3 scripts/analytics.py --hours 168 --format markdown diff --git a/README.md b/README.md index 9f1272d..a4c1f45 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,12 @@ -# J1 Helpdesk Agent +# CommandDesk **Self-hosted AI helpdesk with multi-platform ticketing, WhatsApp chat, email-to-ticket, knowledge base, Freshdesk MCP, and a plug-in agent architecture.** +[![CI](https://github.com/JorahOne-Services/CommandDesk/actions/workflows/ci.yml/badge.svg)](https://github.com/JorahOne-Services/CommandDesk/actions/workflows/ci.yml) +[![Security Scan](https://github.com/JorahOne-Services/CommandDesk/actions/workflows/security-scan.yml/badge.svg)](https://github.com/JorahOne-Services/CommandDesk/actions/workflows/security-scan.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-311/) + 100% local and free. Compatible with Hermes Agent and the broader agent-skills ecosystem. ![Admin Dashboard](https://v3b.fal.media/files/b/0a9f159d/EQkpV4ZcXRrZthURYu5rA_Pz09bZEi.png) @@ -90,27 +95,33 @@ When a customer requests a human: - Freshdesk web portal - WhatsApp (collected details → adapter) -## Quick Start +## Getting Started + +**Prerequisites:** Docker Engine 24+ and Docker Compose v2+ must be installed and running. ```bash -# 1. Clone +# 1. Verify Docker is active (required) +docker info >/dev/null 2>&1 || echo "Docker daemon is not running" +docker compose version + +# 2. Clone the repository git clone https://github.com/OneByJorah/CommandDesk.git cd CommandDesk -# 2. Run setup +# 3. Run setup ./scripts/setup.sh -# 3. Configure +# 4. Configure cp .env.example .env # Edit .env with your IMAP credentials and ticket platform settings -# 4. Start +# 5. Start docker compose up -d -# 5. Index knowledge base +# 6. Index knowledge base docker compose exec helpdesk-agent python3 scripts/index_kb.py -# 6. Open +# 7. Open # Dashboard: http://localhost/dashboard/ # Helpdesk API: http://localhost/helpdesk/health # Admin API: http://localhost/admin/health @@ -262,11 +273,30 @@ make test-api # Test agent API CommandDesk/ ├── docker-compose.yml # Full stack definition ├── Dockerfile # Main agent container +├── Dockerfile.email # Email fetcher container ├── Dockerfile.whatsapp # WhatsApp webhook container ├── Makefile # Common commands ├── requirements.txt # Python dependencies ├── .env.example # Environment template -├── .github/workflows/ci.yml # CI pipeline +├── .gitignore # Git ignore rules +├── SECURITY.md # Security policy +├── CONTRIBUTING.md # Contribution guide +├── setup.cfg # Test configuration +├── .github/ +│ ├── dependabot.yml # Dependency updates +│ └── workflows/ +│ ├── ci.yml # CI pipeline +│ └── security-scan.yml # Security scanning +├── docs/ +│ ├── runbook.md # Operations runbook +│ ├── api.md # API documentation +│ └── configuration.md # Configuration guide +├── tests/ +│ ├── test_rate_limiter.py # Rate limiter tests +│ ├── test_email_fetcher.py # Email fetcher tests +│ ├── test_analytics.py # Analytics tests +│ ├── test_index_kb.py # KB indexer tests +│ └── test_health_monitor.py # Health monitor tests ├── config/ │ ├── hermes-config.yaml # Helpdesk agent config │ ├── admin-agent-config.yaml # Admin agent config @@ -291,6 +321,7 @@ CommandDesk/ │ ├── email_fetcher.py # IMAP polling │ ├── index_kb.py # Knowledge base indexer │ ├── init-db.sql # Database schema +│ ├── analytics.py # Analytics reports │ └── setup.sh # Setup script ├── admin/ │ └── admin-dashboard.html # Monitoring dashboard @@ -301,6 +332,14 @@ CommandDesk/ └── workflows/ # n8n workflow JSONs ``` +## Documentation + +- [API Documentation](docs/api.md) +- [Configuration Guide](docs/configuration.md) +- [Operations Runbook](docs/runbook.md) +- [Security Policy](SECURITY.md) +- [Contributing Guide](CONTRIBUTING.md) + ## License MIT diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..04ca383 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,65 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 1.x | :white_check_mark: | + +## Reporting a Vulnerability + +We take the security of CommandDesk seriously. If you believe you have found a security vulnerability, please report it to us as described below. + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them via email to **security@jorahone.com** (or the repository owner's security contact). + +You should receive a response within 48 hours. If for some reason you do not, please follow up via email to ensure we received your original message. + +### What to include + +- Type of issue (e.g., SQL injection, cross-site scripting, etc.) +- Full paths of source file(s) related to the manifestation of the issue +- The location of the affected source code (tag/branch/commit or direct URL) +- Any special configuration required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof-of-concept or exploit code (if possible) +- Impact of the issue, including how an attacker might exploit it + +### What to expect + +- You will receive an acknowledgment of your report within 48 hours +- We will confirm the issue and determine its severity +- We will work on a fix and release it as soon as possible +- We will notify you when the fix is released + +## Security Best Practices + +### For Production Deployments + +1. **Change all default secrets** — Update `REDIS_PASSWORD`, `DB_PASSWORD`, `CHROMA_AUTH_TOKEN`, `JWT_SECRET`, and `WHATSAPP_WEBHOOK_SECRET` in your `.env` file +2. **Use HTTPS** — Configure TLS certificates for Nginx (see `config/nginx.conf`) +3. **Restrict network access** — Internal services bind to `127.0.0.1` by default; do not expose them publicly +4. **Keep dependencies updated** — Run `docker compose build --no-cache` after updating `requirements.txt` +5. **Enable rate limiting** — Default limits are conservative; adjust `RATE_LIMIT_PER_SESSION` as needed +6. **Audit logs** — All requests are logged to PostgreSQL; monitor for suspicious activity +7. **WhatsApp webhook** — HMAC signature verification is enabled; keep `WHATSAPP_WEBHOOK_SECRET` secret + +### Docker Security + +- Containers run with read-only root filesystems where possible +- Services bind to localhost (`127.0.0.1`) unless explicitly required +- Non-root users are used inside containers +- Secrets are passed via environment variables, never hardcoded + +## Known Security Features + +- **Rate Limiting**: 50 requests/session/hour (configurable) +- **Session Duration**: 2-hour max per session +- **Message Length**: 4000 chars max +- **Content Filter**: Blocks password/credit_card/SSN in responses +- **Audit Log**: All requests logged to PostgreSQL +- **Network**: Internal services bound to 127.0.0.1 +- **Admin Agent**: IP-whitelisted (Docker network only) +- **Nginx**: Security headers, request size limits, rate zones +- **WhatsApp**: HMAC signature verification diff --git a/docker-compose.yml b/docker-compose.yml index ebf3107..2b04c16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: - LLM_MODEL=qwen2.5-7b-instruct - CHROMA_URL=http://chroma:8000 - SEARX_URL=http://searxng:8080 - - POSTGRES_URL=postgresql://helpdesk:${DB_PASSWORD:-helpdesk_pass}@postgres:5432/helpdesk + - POSTGRES_URL=postgresql://helpdesk:***@postgres:5432/helpdesk - REDIS_URL=redis://:${REDIS_PASSWORD:-redis_pass}@redis:6379/0 - RATE_LIMIT_PER_SESSION=${RATE_LIMIT_PER_SESSION:-50} - RATE_LIMIT_WINDOW=${RATE_LIMIT_WINDOW:-3600} @@ -78,6 +78,11 @@ services: limits: memory: 2G restart: unless-stopped + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp networks: - helpdesk-net @@ -336,85 +341,85 @@ services: networks: - helpdesk-net -# ═══════════════════════════════════════════════════ -# WhatsApp Webhook Receiver -# ═══════════════════════════════════════════════════ -whatsapp-webhook: - build: - context: . - dockerfile: Dockerfile.whatsapp - container_name: helpdesk-whatsapp - ports: - - "127.0.0.1:9090:9090" - - "0.0.0.0:8383:8383" - volumes: - - ./config:/app/config:ro - - ./scripts:/app/scripts:ro - environment: - - WHATSAPP_TOKEN=${WHATSAPP_TOKEN} - - WHATSAPP_PHONE_NUMBER_ID=${WHATSAPP_PHONE_NUMBER_ID} - - WHATSAPP_WEBHOOK_SECRET=${WHATSAPP_WEBHOOK_SECRET:-change_me} - - ADMIN_PHONE_NUMBER=${ADMIN_PHONE_NUMBER} - - HELPDESK_AGENT_URL=http://helpdesk-agent:8080 - - REDIS_URL=redis://:${REDIS_PASSWORD:-redis_pass}@redis:6379/0 - - WHATSAPP_RATE_LIMIT_PER_MINUTE=${WHATSAPP_RATE_LIMIT_PER_MINUTE:-10} - depends_on: - helpdesk-agent: - condition: service_started - redis: - condition: service_healthy - deploy: - resources: - limits: - memory: 500M - restart: unless-stopped - networks: - - helpdesk-net + # ═══════════════════════════════════════════════════ + # WhatsApp Webhook Receiver + # ═══════════════════════════════════════════════════ + whatsapp-webhook: + build: + context: . + dockerfile: Dockerfile.whatsapp + container_name: helpdesk-whatsapp + ports: + - "127.0.0.1:9090:9090" + - "0.0.0.0:8383:8383" + volumes: + - ./config:/app/config:ro + - ./scripts:/app/scripts:ro + environment: + - WHATSAPP_TOKEN=${WHATSAPP_TOKEN} + - WHATSAPP_PHONE_NUMBER_ID=${WHATSAPP_PHONE_NUMBER_ID} + - WHATSAPP_WEBHOOK_SECRET=${WHATSAPP_WEBHOOK_SECRET:-change_me} + - ADMIN_PHONE_NUMBER=${ADMIN_PHONE_NUMBER} + - HELPDESK_AGENT_URL=http://helpdesk-agent:8080 + - REDIS_URL=redis://:${REDIS_PASSWORD:-redis_pass}@redis:6379/0 + - WHATSAPP_RATE_LIMIT_PER_MINUTE=${WHATSAPP_RATE_LIMIT_PER_MINUTE:-10} + depends_on: + helpdesk-agent: + condition: service_started + redis: + condition: service_healthy + deploy: + resources: + limits: + memory: 500M + restart: unless-stopped + networks: + - helpdesk-net -# ═══════════════════════════════════════════════════ -# Health Monitor -# ═══════════════════════════════════════════════════ -health-monitor: - build: - context: . - dockerfile: Dockerfile - container_name: helpdesk-health - volumes: - - ./scripts:/app/scripts:ro - - ./config:/app/config:ro - environment: - - REDIS_URL=redis://:${REDIS_PASSWORD:-redis_pass}@redis:6379/0 - depends_on: - redis: - condition: service_healthy - deploy: - resources: - limits: - memory: 200M - restart: unless-stopped - networks: - - helpdesk-net + # ═══════════════════════════════════════════════════ + # Health Monitor + # ═══════════════════════════════════════════════════ + health-monitor: + build: + context: . + dockerfile: Dockerfile + container_name: helpdesk-health + volumes: + - ./scripts:/app/scripts:ro + - ./config:/app/config:ro + environment: + - REDIS_URL=redis://:${REDIS_PASSWORD:-redis_pass}@redis:6379/0 + depends_on: + redis: + condition: service_healthy + deploy: + resources: + limits: + memory: 200M + restart: unless-stopped + networks: + - helpdesk-net -# ═══════════════════════════════════════════════════ -# Tools UI (Widget + Admin Panel) -# ═══════════════════════════════════════════════════ -tools-ui: - build: - context: tools-ui - dockerfile: Dockerfile - container_name: helpdesk-tools-ui - ports: - - "127.0.0.1:8484:8484" - depends_on: - - helpdesk-agent - - admin-agent - deploy: - resources: - limits: - memory: 200M - restart: unless-stopped - networks: - - helpdesk-net + # ═══════════════════════════════════════════════════ + # Tools UI (Widget + Admin Panel) + # ═══════════════════════════════════════════════════ + tools-ui: + build: + context: tools-ui + dockerfile: Dockerfile + container_name: helpdesk-tools-ui + ports: + - "127.0.0.1:8484:8484" + depends_on: + - helpdesk-agent + - admin-agent + deploy: + resources: + limits: + memory: 200M + restart: unless-stopped + networks: + - helpdesk-net # ═══════════════════════════════════════════════════ # Networks diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..46ff26e --- /dev/null +++ b/docs/api.md @@ -0,0 +1,249 @@ +# CommandDesk API Documentation + +## Overview + +CommandDesk exposes several HTTP APIs for interacting with the helpdesk system. All APIs are proxied through Nginx on port 80/443. + +## Base URLs + +| API | Internal URL | External URL | +|-----|-------------|--------------| +| Helpdesk Agent | `http://helpdesk-agent:8080` | `http://localhost/helpdesk/` | +| Admin Agent | `http://admin-agent:8082` | `http://localhost/admin/` | +| WhatsApp Webhook | `http://whatsapp-webhook:9090` | `http://localhost/webhook/whatsapp` | + +--- + +## Helpdesk Agent API + +### Health Check + +``` +GET /health +``` + +**Response:** +```json +{ + "status": "ok", + "agent_mode": "helpdesk", + "version": "1.0.0", + "timestamp": 1719000000.0 +} +``` + +### Chat + +Send a message to the AI helpdesk agent. + +``` +POST /chat +Content-Type: application/json +``` + +**Request Body:** +```json +{ + "session_id": "optional-existing-session-id", + "user_id": "customer@example.com", + "message": "I need help with my ticket", + "platform": "web" +} +``` + +**Response:** +```json +{ + "session_id": "abc-123-def", + "response": "I'd be happy to help! Let me look up your tickets...", + "remaining_requests": 49, + "session_expires_in": 7100 +} +``` + +**Status Codes:** +- `200 OK` — Success +- `429 Too Many Requests` — Rate limit exceeded +- `422 Unprocessable Entity` — Invalid request body + +### Get Session Info + +``` +GET /session/{session_id} +``` + +**Response:** +```json +{ + "session_id": "abc-123-def", + "user_id": "customer@example.com", + "message_count": 5, + "request_count": 3, + "active": true, + "session_age_seconds": 120, + "remaining_requests": 47 +} +``` + +--- + +## Admin Agent API + +### Health Check + +``` +GET /health +``` + +### Chat (Full Access) + +``` +POST /chat +Content-Type: application/json +``` + +Same as helpdesk agent but with full access to all tools including ticket creation. + +### List Tickets + +``` +GET /tickets +``` + +**Query Parameters:** +- `page` (int, default: 1) — Page number +- `per_page` (int, default: 20) — Items per page +- `status` (string, optional) — Filter by status +- `user_id` (string, optional) — Filter by user + +### Cost Analytics + +``` +GET /costs +``` + +**Query Parameters:** +- `hours` (int, default: 24) — Lookback period + +### System Health + +``` +GET /system +``` + +Returns comprehensive system health including all service statuses. + +--- + +## WhatsApp Webhook API + +### Webhook Verification (GET) + +``` +GET /webhook/whatsapp?hub.mode=subscribe&hub.challenge=123456&hub.verify_token=your_token +``` + +**Response:** Returns the `hub.challenge` value on success. + +### Receive Message (POST) + +``` +POST /webhook/whatsapp +Content-Type: application/json +X-Hub-Signature-256: sha256=... +``` + +Standard WhatsApp Business API webhook payload. + +### Admin: Take Over Conversation + +``` +POST /admin/takeover/{phone} +``` + +Pauses the bot for the specified phone number and notifies the customer they're speaking with a human. + +### Admin: Resume Bot + +``` +POST /admin/resume/{phone} +``` + +Re-enables the bot for the specified phone number. + +### Admin: View Queue + +``` +GET /admin/queue +``` + +Returns the current human support queue. + +**Response:** +```json +{ + "queue": [ + { + "phone": "+1234567890", + "waiting_minutes": 3, + "message": "I need help with my account" + } + ], + "total": 1 +} +``` + +--- + +## Error Responses + +### 429 Rate Limited + +```json +{ + "detail": "Rate limit exceeded (50 req/3600s)" +} +``` + +### 404 Not Found + +```json +{ + "detail": "Session not found" +} +``` + +### 403 Forbidden + +```json +{ + "detail": "Invalid signature" +} +``` + +--- + +## Rate Limiting + +| Limit | Value | Scope | +|-------|-------|-------| +| Requests per session | 50 | Per session | +| Window | 3600 seconds (1 hour) | Sliding window | +| Session duration | 7200 seconds (2 hours) | Max session lifetime | +| Message length | 4000 characters | Per message | +| WhatsApp rate limit | 10 messages/minute | Per phone number | + +--- + +## Authentication + +- **Internal APIs**: No authentication required (Docker network only) +- **WhatsApp Webhook**: HMAC-SHA256 signature verification +- **Admin APIs**: IP-restricted to Docker network +- **External access**: Via Nginx with optional TLS + +--- + +## WebSocket (Future) + +WebSocket support for real-time ticket updates is planned for a future release. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..7ef7c16 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,242 @@ +# CommandDesk Configuration Guide + +## Overview + +CommandDesk is configured through a combination of environment variables (`.env`), YAML configuration files (`config/`), and Docker Compose overrides. + +--- + +## Environment Variables (`.env`) + +Copy `.env.example` to `.env` and fill in your values. + +### LLM Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `LLAMA_MODEL_PATH` | `/models/qwen2.5-7b-instruct-q4_k_m.gguf` | Path to GGUF model file | +| `LLAMA_PORT` | `8081` | llama.cpp server port | +| `LLAMA_CTX_SIZE` | `65536` | Context window size | +| `LLAMA_THREADS` | `6` | CPU threads for inference | + +### Database Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `DB_HOST` | `postgres` | PostgreSQL hostname | +| `DB_PORT` | `5432` | PostgreSQL port | +| `DB_NAME` | `helpdesk` | Database name | +| `DB_USER` | `helpdesk` | Database user | +| `DB_PASSWORD` | `change...n` | **CHANGE THIS** | + +### Redis Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `REDIS_HOST` | `redis` | Redis hostname | +| `REDIS_PORT` | `6379` | Redis port | +| `REDIS_PASSWORD` | `change...n` | **CHANGE THIS** | + +### Email (IMAP) Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `IMAP_HOST` | `imap.example.com` | IMAP server hostname | +| `IMAP_PORT` | `993` | IMAP port (SSL) | +| `IMAP_USER` | `helpdesk@example.com` | IMAP username | +| `IMAP_PASSWORD` | `change...n` | **CHANGE THIS** | +| `POLL_INTERVAL` | `60` | Polling interval in seconds | +| `TICKET_PLATFORM` | `osticket` | Target ticket platform | + +### Ticket Platform Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `OSTICKET_URL` | `https://support.example.com/api/tickets.json` | osTicket API URL | +| `OSTICKET_API_KEY` | `change...n` | osTicket API key | +| `FRESHDESK_URL` | `https://yourcompany.freshdesk.com` | Freshdesk domain | +| `FRESHDESK_API_KEY` | `change...n` | Freshdesk API key | + +### Security & Rate Limiting + +| Variable | Default | Description | +|----------|---------|-------------| +| `RATE_LIMIT_PER_SESSION` | `50` | Max requests per session | +| `RATE_LIMIT_WINDOW` | `3600` | Rate limit window (seconds) | +| `MAX_MESSAGE_LENGTH` | `4000` | Max message length (chars) | +| `MAX_SESSION_DURATION` | `7200` | Max session duration (seconds) | +| `CHROMA_AUTH_TOKEN` | `chromadb_token_change_me` | ChromaDB auth token | +| `JWT_SECRET` | `jwt_secret_change_me_please` | n8n JWT secret | +| `WHATSAPP_WEBHOOK_SECRET` | `change_me` | WhatsApp webhook secret | + +### WhatsApp Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `WHATSAPP_TOKEN` | — | WhatsApp Business API token | +| `WHATSAPP_PHONE_NUMBER_ID` | — | WhatsApp phone number ID | +| `WHATSAPP_WEBHOOK_SECRET` | `change_me` | Webhook verification token | +| `ADMIN_PHONE_NUMBER` | — | Admin's WhatsApp number | +| `WHATSAPP_RATE_LIMIT_PER_MINUTE` | `10` | Rate limit per phone/minute | + +--- + +## YAML Configuration Files + +### `config/hermes-config.yaml` + +Helpdesk agent configuration: + +```yaml +agent: + name: "J1 Helpdesk Agent" + mode: helpdesk + allow_create_ticket: false + +llm: + provider: openai-compatible + base_url: http://llama:8081/v1 + model: qwen2.5-7b-instruct + max_tokens: 2048 + temperature: 0.3 + +knowledge_base: + provider: chromadb + url: http://chroma:8000 + collection: helpdesk-kb + +search: + provider: searxng + url: http://searxng:8080 + +rate_limiting: + max_requests_per_session: 50 + window_seconds: 3600 + max_message_length: 4000 + max_session_duration: 7200 +``` + +### `config/admin-agent-config.yaml` + +Admin agent configuration (same structure but with `allow_create_ticket: true`). + +### `config/agent-bridge.yaml` + +Delegation rules for connecting to a main Hermes Agent instance: + +```yaml +bridges: + helpdesk-agent: + url: "http://helpdesk-agent:8080" + triggers: ["ticket", "helpdesk", "support", "my issue"] + admin-agent: + url: "http://admin-agent:8082" + triggers: ["admin", "manage tickets", "cost analytics"] +``` + +### `config/mcp-config.yaml` + +Freshdesk MCP server configuration: + +```yaml +mcp_servers: + freshdesk: + command: "python" + args: ["-m", "freshdesk_mcp"] + env: + FRESHDESK_DOMAIN: "${FRESHDESK_DOMAIN}" + FRESHDESK_API_KEY: "${FRESHDESK_API_KEY}" +``` + +### `config/nginx.conf` + +Nginx reverse proxy configuration with security headers, rate limiting, and TLS support. + +### `config/searxng-settings.yml` + +SearXNG search engine configuration. Configure which search engines to use and their priorities. + +### `config/system-prompt.md` + +The system prompt that defines the AI agent's persona, behavior, and constraints. + +--- + +## Docker Compose Configuration + +### Production (`docker-compose.yml`) + +Full production stack with all services. + +### Development (`docker-compose.dev.yml`) + +Development overrides with hot-reload, debug ports, and relaxed security. + +### Production Override (`docker-compose.prod.yml`) + +Production-specific settings (resource limits, logging drivers, etc.). + +### Compose Extensions (`compose/`) + +Modular compose files for specific features: + +| File | Purpose | +|------|---------| +| `compose/docker-compose.mail.yml` | Email services | +| `compose/docker-compose.monitoring.yml` | Monitoring stack | +| `compose/docker-compose.storage.yml` | Storage services | +| `compose/docker-compose.ci.yml` | CI/CD services | +| `compose/docker-compose.automation.yml` | Automation services | +| `compose/docker-compose.knowledge.yml` | Knowledge base services | +| `compose/docker-compose.selfhosted.yml` | Self-hosted services | +| `compose/docker-compose.wiki.yml` | Wiki services | +| `compose/docker-compose.plus.yml` | Premium services | +| `compose/docker-compose.git.yml` | Git services | + +--- + +## Makefile Commands + +| Command | Description | +|---------|-------------| +| `make setup` | One-time setup | +| `make start` | Start all services | +| `make stop` | Stop all services | +| `make restart` | Restart all services | +| `make rebuild` | Rebuild and restart | +| `make logs` | View all logs | +| `make health` | Check service health | +| `make index-kb` | Index knowledge base | +| `make dev` | Start in development mode | +| `make shell` | Open shell in agent container | +| `make psql` | Open PostgreSQL shell | +| `make redis-cli` | Open Redis CLI | +| `make test-api` | Test agent API | +| `make clean` | Remove containers and volumes | +| `make clean-data` | Remove all data (DANGEROUS) | + +--- + +## Advanced Configuration + +### Custom LLM Models + +1. Download a GGUF model to `models/` +2. Update `LLAMA_MODEL_PATH` in `.env` +3. Update `LLM_MODEL` in `.env` +4. Restart: `docker compose restart llama helpdesk-agent` + +### Custom System Prompt + +Edit `config/system-prompt.md` to customize the agent's behavior, tone, and capabilities. + +### Adding Ticket Platforms + +1. Create a new adapter in `ticket_platforms/` extending `BasePlatform` +2. Register it in `ticket_platforms/registry.py` +3. Add configuration variables to `.env` +4. Update `docker-compose.yml` if new services are needed + +### Custom Workflows + +Place n8n workflow JSON files in `workflows/`. They will be automatically loaded by n8n on startup. diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..79d3243 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,399 @@ +# CommandDesk Runbook + +## Overview + +This runbook covers operational procedures for the CommandDesk self-hosted AI helpdesk agent. It is intended for system administrators and operators. + +## Table of Contents + +1. [Architecture Overview](#architecture-overview) +2. [Deployment](#deployment) +3. [Configuration](#configuration) +4. [Monitoring](#monitoring) +5. [Backup & Restore](#backup--restore) +6. [Troubleshooting](#troubleshooting) +7. [Scaling](#scaling) +8. [Security Procedures](#security-procedures) +9. [Disaster Recovery](#disaster-recovery) + +## Architecture Overview + +CommandDesk consists of the following services: + +| Service | Container | Port | Purpose | +|---------|-----------|------|---------| +| llama.cpp | helpdesk-llama | 8081 | LLM inference (Qwen2.5-7B) | +| Helpdesk Agent | helpdesk-agent | 8080 | Customer-facing AI agent | +| Admin Agent | helpdesk-admin-agent | 8082 | Admin operations & analytics | +| ChromaDB | helpdesk-chroma | 8000 | Vector knowledge base | +| SearXNG | helpdesk-searxng | 8888 | Self-hosted web search | +| n8n | helpdesk-n8n | 5678 | Workflow automation | +| PostgreSQL | helpdesk-postgres | 5432 | Primary database | +| Redis | helpdesk-redis | 6379 | Caching & session store | +| Nginx | helpdesk-nginx | 80/443 | Reverse proxy & security | +| Email Fetcher | helpdesk-email-fetcher | — | IMAP polling | +| WhatsApp Webhook | helpdesk-whatsapp | 9090/8383 | WhatsApp integration | +| Health Monitor | helpdesk-health | — | Service health checks | +| Tools UI | helpdesk-tools-ui | 8484 | Widget & admin panel | + +## Deployment + +### Prerequisites + +- Docker Engine 24+ and Docker Compose v2+ +- At least 16GB RAM, 6 CPU cores +- 20GB free disk space (for models + data) + +### First-Time Deployment + +```bash +# 1. Clone and setup +git clone https://github.com/JorahOne-Services/CommandDesk.git +cd CommandDesk +./scripts/setup.sh + +# 2. Configure +cp .env.example .env +# Edit .env with your credentials + +# 3. Start all services +docker compose up -d + +# 4. Verify +docker compose ps +curl http://localhost:8080/health + +# 5. Index knowledge base +docker compose exec helpdesk-agent python3 scripts/index_kb.py +``` + +### Updating + +```bash +# Pull latest code +git pull origin master + +# Rebuild and restart +docker compose down +docker compose build --no-cache +docker compose up -d + +# Run database migrations if any +docker compose exec postgres psql -U helpdesk -d helpdesk -f /scripts/init-db.sql +``` + +## Configuration + +### Environment Variables + +All configuration is done via `.env` file. See `.env.example` for the full list. + +**Critical settings:** + +| Variable | Default | Description | +|----------|---------|-------------| +| `DB_PASSWORD` | `helpdesk_pass` | PostgreSQL password (CHANGE ME) | +| `REDIS_PASSWORD` | `redis_pass` | Redis password (CHANGE ME) | +| `CHROMA_AUTH_TOKEN` | `chromadb_token_change_me` | ChromaDB auth token (CHANGE ME) | +| `JWT_SECRET` | `jwt_secret_change_me_please` | n8n JWT secret (CHANGE ME) | +| `WHATSAPP_WEBHOOK_SECRET` | `change_me` | WhatsApp webhook secret (CHANGE ME) | +| `RATE_LIMIT_PER_SESSION` | `50` | Max requests per session | +| `MAX_SESSION_DURATION` | `7200` | Max session duration in seconds | + +### Config Files + +| File | Purpose | +|------|---------| +| `config/hermes-config.yaml` | Helpdesk agent configuration | +| `config/admin-agent-config.yaml` | Admin agent configuration | +| `config/agent-bridge.yaml` | Delegation rules for main Hermes | +| `config/mcp-config.yaml` | Freshdesk MCP configuration | +| `config/nginx.conf` | Nginx reverse proxy configuration | +| `config/searxng-settings.yml` | SearXNG search settings | +| `config/system-prompt.md` | Agent persona/system prompt | + +## Monitoring + +### Health Checks + +Each service exposes a health endpoint: + +```bash +# Check all services +make health + +# Individual checks +curl http://localhost:8080/health # Helpdesk agent +curl http://localhost:8082/health # Admin agent +curl http://localhost:8081/health # llama.cpp +curl http://localhost:8000/api/v1/heartbeat # ChromaDB +``` + +### Logs + +```bash +# All services +docker compose logs -f --tail=100 + +# Specific service +docker compose logs -f helpdesk-agent --tail=50 +docker compose logs -f whatsapp-webhook --tail=50 +``` + +### Metrics + +The health monitor collects metrics every 30 seconds and stores them in Redis: + +```bash +# View latest metrics +docker compose exec redis redis-cli -a $REDIS_PASSWORD get metrics:latest | python3 -m json.tool +``` + +### Analytics Reports + +```bash +# Generate 24-hour report +docker compose exec helpdesk-agent python3 scripts/analytics.py --hours 24 + +# Generate markdown report +docker compose exec helpdesk-agent python3 scripts/analytics.py --hours 168 --format markdown +``` + +## Backup & Restore + +### PostgreSQL Backup + +```bash +# Backup +docker compose exec -T postgres pg_dump -U helpdesk helpdesk > backup_$(date +%Y%m%d_%H%M%S).sql + +# Restore +cat backup.sql | docker compose exec -T postgres psql -U helpdesk -d helpdesk +``` + +### Redis Backup + +Redis data is persisted via AOF (append-only file) to `redis-data` volume. + +### ChromaDB Backup + +```bash +# Backup ChromaDB data +tar -czf chroma-backup.tar.gz /path/to/chroma-data +``` + +### Volume Backups + +```bash +# Backup all named volumes +docker run --rm -v commanddesk_postgres-data:/data -v $(pwd):/backup alpine tar czf /backup/postgres-backup.tar.gz -C /data . +``` + +## Troubleshooting + +### Service Won't Start + +```bash +# Check logs +docker compose logs + +# Check if port is in use +sudo lsof -i :8080 + +# Rebuild from scratch +docker compose down -v +docker compose build --no-cache +docker compose up -d +``` + +### LLM Not Responding + +```bash +# Check llama.cpp health +curl http://localhost:8081/health + +# Check model file exists +ls -la models/qwen2.5-7b-instruct-q4_k_m.gguf + +# Check resource limits +docker stats helpdesk-llama +``` + +### Database Connection Issues + +```bash +# Test PostgreSQL connection +docker compose exec postgres pg_isready -U helpdesk + +# Check PostgreSQL logs +docker compose logs postgres --tail=50 + +# Reset database +docker compose down -v +docker compose up -d postgres +``` + +### Redis Issues + +```bash +# Test Redis connection +docker compose exec redis redis-cli -a $REDIS_PASSWORD ping + +# Check memory usage +docker compose exec redis redis-cli -a $REDIS_PASSWORD info memory +``` + +### Email Fetcher Not Working + +```bash +# Check IMAP connectivity +docker compose logs email-fetcher --tail=50 + +# Verify IMAP credentials in .env +# Test IMAP connection manually: +openssl s_client -connect imap.example.com:993 -crlf +``` + +### WhatsApp Webhook Issues + +```bash +# Check webhook logs +docker compose logs whatsapp-webhook --tail=50 + +# Verify webhook URL is accessible from internet +# Check HMAC signature verification +``` + +## Scaling + +### Vertical Scaling + +Increase resource limits in `docker-compose.yml`: + +```yaml +deploy: + resources: + limits: + memory: 4G # Increase from 2G + cpus: "4" # Add CPU cores +``` + +### Horizontal Scaling + +For higher throughput, you can run multiple agent instances behind Nginx: + +```yaml +# docker-compose.override.yml +helpdesk-agent: + deploy: + replicas: 3 +``` + +### Database Optimization + +- Add more RAM to PostgreSQL for larger cache +- Tune `shared_buffers` and `work_mem` in PostgreSQL config +- Add indexes for frequently queried columns + +## Security Procedures + +### Incident Response + +1. **Isolate affected services**: `docker compose stop ` +2. **Collect logs**: `docker compose logs --tail=1000 > incident.log` +3. **Check audit log**: Query PostgreSQL audit_log table +4. **Rotate credentials**: Update all secrets in `.env` +5. **Restart**: `docker compose up -d` + +### Regular Maintenance + +- Weekly: Review audit logs for suspicious activity +- Monthly: Update dependencies (`docker compose build --no-cache`) +- Quarterly: Rotate all secrets and API keys +- Annually: Review and update TLS certificates + +### Security Checklist + +- [ ] All default passwords changed +- [ ] HTTPS configured with valid TLS certificates +- [ ] Firewall restricts access to ports 80/443 only +- [ ] Rate limiting enabled +- [ ] Audit logging enabled +- [ ] Regular backups configured +- [ ] Docker daemon updated to latest stable +- [ ] Host OS security patches applied + +## Disaster Recovery + +### Complete System Restore + +```bash +# 1. Restore from backup +git clone https://github.com/JorahOne-Services/CommandDesk.git +cd CommandDesk +cp .env.example .env +# Edit .env with your credentials + +# 2. Restore database +docker compose up -d postgres +cat backup.sql | docker compose exec -T postgres psql -U helpdesk -d helpdesk + +# 3. Restore volumes +docker run --rm -v commanddesk_postgres-data:/data -v $(pwd):/backup alpine tar xzf /backup/postgres-backup.tar.gz -C /data + +# 4. Start all services +docker compose up -d + +# 5. Verify +make health +``` + +### Emergency Shutdown + +```bash +# Graceful shutdown +docker compose down + +# Force shutdown (if hung) +docker compose down --timeout 0 +docker compose down -v # Also removes volumes (DESTRUCTIVE) +``` + +## Appendix + +### Useful Commands + +```bash +# Shell access +docker compose exec helpdesk-agent /bin/bash + +# PostgreSQL shell +docker compose exec postgres psql -U helpdesk -d helpdesk + +# Redis CLI +docker compose exec redis redis-cli -a $REDIS_PASSWORD + +# View resource usage +docker stats + +# Clean up unused resources +docker system prune -f +``` + +### Port Reference + +| Port | Service | Protocol | Notes | +|------|---------|----------|-------| +| 80 | Nginx | HTTP | Public-facing | +| 443 | Nginx | HTTPS | Public-facing (TLS) | +| 8080 | Helpdesk Agent | HTTP | Internal | +| 8081 | llama.cpp | HTTP | Internal | +| 8082 | Admin Agent | HTTP | Internal | +| 8000 | ChromaDB | HTTP | Internal | +| 5432 | PostgreSQL | TCP | Internal | +| 5678 | n8n | HTTP | Internal | +| 6379 | Redis | TCP | Internal | +| 8484 | Tools UI | HTTP | Internal | +| 8888 | SearXNG | HTTP | Internal | +| 9090 | WhatsApp Webhook | HTTP | Internal | +| 8383 | WhatsApp Webhook | HTTP | External (webhook) | diff --git a/scripts/error_handling.py b/scripts/error_handling.py new file mode 100644 index 0000000..5498e74 --- /dev/null +++ b/scripts/error_handling.py @@ -0,0 +1,162 @@ +""" +Error handling utilities for CommandDesk services. +Provides consistent error handling, retry logic, and error reporting. +""" +from __future__ import annotations + +import asyncio +import functools +import logging +import time +from typing import Any, Callable, Optional, Type, TypeVar + +T = TypeVar("T") + +logger = logging.getLogger(__name__) + + +class ServiceError(Exception): + """Base exception for service-level errors.""" + + def __init__(self, message: str, service: str = "unknown", status_code: int = 500): + self.service = service + self.status_code = status_code + super().__init__(message) + + +class ConfigurationError(ServiceError): + """Raised when required configuration is missing or invalid.""" + + def __init__(self, message: str, config_key: str = ""): + self.config_key = config_key + super().__init__(message, service="configuration", status_code=500) + + +class DatabaseError(ServiceError): + """Raised on database connection or query failures.""" + + def __init__(self, message: str, query: str = ""): + self.query = query + super().__init__(message, service="database", status_code=503) + + +class ExternalServiceError(ServiceError): + """Raised when an external service (LLM, API, etc.) fails.""" + + def __init__(self, message: str, service: str = "external", status_code: int = 502): + super().__init__(message, service=service, status_code=status_code) + + +def retry( + max_attempts: int = 3, + delay: float = 1.0, + backoff: float = 2.0, + exceptions: tuple = (Exception,), + on_retry: Optional[Callable] = None, +): + """Decorator for retrying async functions with exponential backoff. + + Args: + max_attempts: Maximum number of retry attempts + delay: Initial delay between retries (seconds) + backoff: Multiplier for delay after each retry + exceptions: Tuple of exception types to catch and retry + on_retry: Optional callback called with (attempt, exception) on each retry + + Usage: + @retry(max_attempts=3, delay=1.0) + async def fetch_data(): + return await client.get(url) + """ + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @functools.wraps(func) + async def wrapper(*args: Any, **kwargs: Any) -> Any: + last_exception = None + current_delay = delay + + for attempt in range(1, max_attempts + 1): + try: + return await func(*args, **kwargs) + except exceptions as e: + last_exception = e + if attempt < max_attempts: + if on_retry: + on_retry(attempt, e) + logger.warning( + f"Attempt {attempt}/{max_attempts} failed for " + f"{func.__name__}: {e}. Retrying in {current_delay}s..." + ) + await asyncio.sleep(current_delay) + current_delay *= backoff + else: + logger.error( + f"All {max_attempts} attempts failed for " + f"{func.__name__}: {e}" + ) + + raise last_exception # type: ignore[misc] + return wrapper + return decorator + + +def require_env(var_name: str) -> str: + """Get a required environment variable or raise ConfigurationError. + + Args: + var_name: Name of the environment variable + + Returns: + The value of the environment variable + + Raises: + ConfigurationError: If the variable is not set or empty + """ + import os + value = os.getenv(var_name) + if not value: + raise ConfigurationError( + f"Required environment variable {var_name} is not set", + config_key=var_name, + ) + return value + + +def safe_get_env(var_name: str, default: str = "") -> str: + """Safely get an environment variable with a default. + + Args: + var_name: Name of the environment variable + default: Default value if not set + + Returns: + The value or default + """ + import os + return os.getenv(var_name, default) + + +def format_error_response(error: Exception, include_traceback: bool = False) -> dict: + """Format an exception into a standardized error response dict. + + Args: + error: The exception to format + include_traceback: Whether to include traceback info + + Returns: + Dict with error details suitable for API responses + """ + import traceback + + response = { + "error": type(error).__name__, + "message": str(error), + } + + if isinstance(error, ServiceError): + response["service"] = error.service + response["status_code"] = error.status_code + + if include_traceback: + response["traceback"] = traceback.format_exc() + + return response diff --git a/scripts/logging_config.py b/scripts/logging_config.py new file mode 100644 index 0000000..19528f8 --- /dev/null +++ b/scripts/logging_config.py @@ -0,0 +1,94 @@ +""" +Structured logging configuration for CommandDesk services. +Provides JSON-formatted logging for production and human-readable for development. +""" +from __future__ import annotations + +import json +import logging +import logging.config +import os +import sys +import time +from typing import Optional + + +LOG_FORMAT = os.getenv("LOG_FORMAT", "text") # "text" or "json" +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() + + +class JSONFormatter(logging.Formatter): + """Format log records as JSON for structured logging.""" + + def format(self, record: logging.LogRecord) -> str: + log_entry = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "module": record.module, + "function": record.funcName, + "line": record.lineno, + } + if record.exc_info and record.exc_info[0]: + log_entry["exception"] = { + "type": record.exc_info[0].__name__, + "message": str(record.exc_info[1]), + } + # Include extra fields + if hasattr(record, "extra_fields"): + log_entry.update(record.extra_fields) + return json.dumps(log_entry) + + +def setup_logging(name: str, level: Optional[str] = None) -> logging.Logger: + """Configure and return a structured logger. + + Args: + name: Logger name (typically __name__) + level: Override log level (default: LOG_LEVEL env var) + + Returns: + Configured logger instance + """ + log_level = (level or LOG_LEVEL).upper() + valid_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} + if log_level not in valid_levels: + log_level = "INFO" + + logger = logging.getLogger(name) + logger.setLevel(log_level) + + # Remove existing handlers to avoid duplicates + logger.handlers.clear() + + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(log_level) + + if LOG_FORMAT == "json": + handler.setFormatter(JSONFormatter()) + else: + handler.setFormatter( + logging.Formatter( + "%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + + logger.addHandler(handler) + logger.propagate = False + + return logger + + +def log_with_fields(logger: logging.Logger, level: str, message: str, **fields): + """Log a message with additional structured fields. + + Args: + logger: Logger instance + level: Log level (debug, info, warning, error, critical) + message: Log message + **fields: Additional key-value pairs to include in the log record + """ + extra = {"extra_fields": fields} + getattr(logger, level.lower())(message, extra=extra) diff --git a/scripts/setup.sh b/scripts/setup.sh index aa4af1d..f85023b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -6,53 +6,222 @@ set -euo pipefail +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Helper functions +info() { echo -e "${BLUE}[INFO]${NC} $1"; } +success() { echo -e "${GREEN}[✓]${NC} $1"; } +warn() { echo -e "${YELLOW}[!]${NC} $1"; } +error() { echo -e "${RED}[✗]${NC} $1"; } + +# Error handler +handle_error() { + error "Setup failed at line $1" + error "Command: $2" + exit 1 +} + +trap 'handle_error $LINENO "$BASH_COMMAND"' ERR + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +One-time setup for CommandDesk helpdesk agent. + +Options: + -h, --help Show this help message + --skip-model Skip model download + --skip-certs Skip certificate generation + --force Overwrite existing files without prompting + --model MODEL Specify model to download (default: qwen2.5-7b-instruct-q4_k_m.gguf) + +Examples: + $0 # Full setup + $0 --skip-model # Setup without downloading LLM model + $0 --force # Force overwrite of existing files +EOF + exit 0 +} + +# Parse arguments +SKIP_MODEL=false +SKIP_CERTS=false +FORCE=false +MODEL_FILE="qwen2.5-7b-instruct-q4_k_m.gguf" + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage ;; + --skip-model) SKIP_MODEL=true; shift ;; + --skip-certs) SKIP_CERTS=true; shift ;; + --force) FORCE=true; shift ;; + --model) MODEL_FILE="$2"; shift 2 ;; + *) error "Unknown option: $1"; usage ;; + esac +done + +echo "" echo "═══════════════════════════════════════════" echo " J1 Helpdesk Agent - Setup" echo "═══════════════════════════════════════════" +echo "" -# Check prerequisites -command -v docker >/dev/null 2>&1 || { echo "ERROR: docker not found. Install docker first."; exit 1; } -command -v docker compose >/dev/null 2>&1 || command -v docker-compose >/dev/null 2>&1 || { echo "ERROR: docker compose not found."; exit 1; } +# ═══════════════════════════════════════════════════ +# Prerequisites Check +# ═══════════════════════════════════════════════════ -# Create directories +info "Checking prerequisites..." + +check_command() { + if command -v "$1" >/dev/null 2>&1; then + success "$1 found: $(command -v "$1")" + return 0 + else + error "$1 not found" + return 1 + fi +} + +PREREQ_FAILED=false + +check_command docker || PREREQ_FAILED=true +check_command "docker compose" || check_command docker-compose || PREREQ_FAILED=true + +# Check Docker daemon is running +if command -v docker >/dev/null 2>&1; then + if docker info >/dev/null 2>&1; then + success "Docker daemon is running" + else + warn "Docker daemon is not running. Start it with: sudo systemctl start docker" + fi +fi + +if [ "$PREREQ_FAILED" = true ]; then + echo "" + error "Missing required dependencies. Please install:" + echo " - Docker Engine: https://docs.docker.com/engine/install/" + echo " - Docker Compose: https://docs.docker.com/compose/install/" + exit 1 +fi + +# ═══════════════════════════════════════════════════ +# Directory Setup +# ═══════════════════════════════════════════════════ + +info "Creating directories..." mkdir -p models config scripts knowledge-base workflows certs data/logs queue +success "Directories created" + +# ═══════════════════════════════════════════════════ +# Environment File +# ═══════════════════════════════════════════════════ -# Generate .env if not exists +info "Setting up environment configuration..." if [ ! -f .env ]; then - echo "[✓] Creating .env from template..." + if [ -f .env.example ]; then + cp .env.example .env + success ".env created from template" + warn "→ Edit .env with your credentials before starting!" + else + error ".env.example not found. Cannot create .env" + exit 1 + fi +elif [ "$FORCE" = true ]; then cp .env.example .env - echo " → Edit .env with your credentials before starting!" + success ".env overwritten from template (--force)" + warn "→ Edit .env with your credentials before starting!" else - echo "[✓] .env already exists" + success ".env already exists (use --force to overwrite)" fi -# Generate self-signed cert for HTTPS (optional) -if [ ! -f certs/cert.pem ]; then - echo "[✓] Generating self-signed certificate..." - openssl req -x509 -newkey rsa:4096 -keyout certs/key.pem -out certs/cert.pem -days 365 -nodes -subj "/CN=helpdesk.local" 2>/dev/null - echo " → Self-signed cert generated. Replace with real certs for production." +# ═══════════════════════════════════════════════════ +# SSL Certificates +# ═══════════════════════════════════════════════════ + +if [ "$SKIP_CERTS" = false ]; then + info "Setting up SSL certificates..." + if [ ! -f certs/cert.pem ] || [ "$FORCE" = true ]; then + if command -v openssl >/dev/null 2>&1; then + openssl req -x509 -newkey rsa:4096 -keyout certs/key.pem \ + -out certs/cert.pem -days 365 -nodes \ + -subj "/CN=helpdesk.local" 2>/dev/null + success "Self-signed certificate generated" + warn "→ Replace with real certificates for production use" + else + warn "openssl not found. Skipping certificate generation." + warn "→ Install openssl or manually create certificates" + fi + else + success "Certificates already exist (use --force to regenerate)" + fi +else + info "Skipping certificate generation (--skip-certs)" fi -# Download model if not exists -MODEL_FILE="models/qwen2.5-7b-instruct-q4_k_m.gguf" -if [ ! -f "$MODEL_FILE" ]; then - echo "[✓] Downloading Qwen2.5-7B model (Q4_K_M)..." - echo " This may take 10-20 minutes depending on your connection." - if command -v huggingface-cli >/dev/null 2>&1; then - huggingface-cli download Qwen/Qwen2.5-7B-Instruct-GGUF "$MODEL_FILE" --local-dir models/ +# ═══════════════════════════════════════════════════ +# LLM Model Download +# ═══════════════════════════════════════════════════ + +if [ "$SKIP_MODEL" = false ]; then + info "Setting up LLM model..." + MODEL_PATH="models/$MODEL_FILE" + + if [ ! -f "$MODEL_PATH" ] || [ "$FORCE" = true ]; then + if [ "$FORCE" = true ] && [ -f "$MODEL_PATH" ]; then + warn "Removing existing model (--force)..." + rm -f "$MODEL_PATH" + fi + + echo "" + info "Downloading $MODEL_FILE..." + info "This may take 10-20 minutes depending on your connection." + echo "" + + if command -v huggingface-cli >/dev/null 2>&1; then + info "Using huggingface-cli..." + huggingface-cli download Qwen/Qwen2.5-7B-Instruct-GGUF \ + "$MODEL_FILE" --local-dir models/ + elif command -v wget >/dev/null 2>&1; then + info "Using wget..." + MODEL_URL="https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GGUF/resolve/main/$MODEL_FILE" + wget -q --show-progress "$MODEL_URL" -O "$MODEL_PATH" + elif command -v curl >/dev/null 2>&1; then + info "Using curl..." + MODEL_URL="https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GGUF/resolve/main/$MODEL_FILE" + curl -L --progress-bar "$MODEL_URL" -o "$MODEL_PATH" + else + warn "No download tool found (huggingface-cli, wget, or curl)" + warn "→ Download manually from: https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GGUF" + warn "→ Place the file at: $MODEL_PATH" + fi + + if [ -f "$MODEL_PATH" ]; then + MODEL_SIZE=$(du -h "$MODEL_PATH" | cut -f1) + success "Model downloaded: $MODEL_PATH ($MODEL_SIZE)" + fi else - wget -q --show-progress "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GGUF/resolve/main/qwen2.5-7b-instruct-q4_k_m.gguf" -O "$MODEL_FILE" + MODEL_SIZE=$(du -h "$MODEL_PATH" | cut -f1) + success "Model already exists: $MODEL_PATH ($MODEL_SIZE)" fi - echo " → Model downloaded: $MODEL_FILE" else - echo "[✓] Model already exists: $MODEL_FILE" + info "Skipping model download (--skip-model)" fi -# Initialize knowledge base +# ═══════════════════════════════════════════════════ +# Knowledge Base +# ═══════════════════════════════════════════════════ + +info "Setting up knowledge base..." if [ -d knowledge-base ] && [ "$(ls -A knowledge-base/*.md knowledge-base/*.txt 2>/dev/null)" ]; then - echo "[✓] Knowledge base files found" + success "Knowledge base files found" else - echo "[?] Adding sample knowledge base article..." + info "Adding sample knowledge base article..." cat > knowledge-base/welcome.md << 'EOF' # Welcome to J1 Helpdesk @@ -73,8 +242,13 @@ Contact billing@example.com or call +1-555-0123. Have your account number ready ### Service Status Check current service status at https://status.example.com. EOF + success "Sample knowledge base article added" fi +# ═══════════════════════════════════════════════════ +# Summary +# ═══════════════════════════════════════════════════ + echo "" echo "═══════════════════════════════════════════" echo " Setup Complete!" @@ -82,9 +256,16 @@ echo "════════════════════════ echo "" echo "Next steps:" echo " 1. Edit .env with your credentials" -echo " 2. Run: docker compose up -d" -echo " 3. Open http://localhost for dashboard" -echo " 4. API available at http://localhost/helpdesk/" +echo " ${YELLOW}nano .env${NC}" +echo "" +echo " 2. Start all services:" +echo " ${GREEN}docker compose up -d${NC}" +echo "" +echo " 3. Open the dashboard:" +echo " ${BLUE}http://localhost/dashboard/${NC}" +echo "" +echo " 4. Check service health:" +echo " ${GREEN}make health${NC}" echo "" echo "For osTicket/Freshdesk integration:" echo " - Set OSTICKET_URL and OSTICKET_API_KEY in .env" @@ -92,5 +273,5 @@ echo " - Set FRESHDESK_URL and FRESHDESK_API_KEY in .env" echo "" echo "To add knowledge base articles:" echo " - Place .md or .txt files in knowledge-base/" -echo " - Run: python3 scripts/index_kb.py" +echo " - Run: ${GREEN}docker compose exec helpdesk-agent python3 scripts/index_kb.py${NC}" echo "" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3eb463a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,21 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +asyncio_mode = auto +markers = + asyncio: mark test as async + +[tool:pytest] +addopts = -v --tb=short --strict-markers + +[coverage:run] +source = scripts,ticket_platforms +omit = tests/* + +[coverage:report] +exclude_lines = + pragma: no cover + if __name__ == "__main__": + raise NotImplementedError + def __repr__ diff --git a/tests/test_analytics.py b/tests/test_analytics.py new file mode 100644 index 0000000..1fa6097 --- /dev/null +++ b/tests/test_analytics.py @@ -0,0 +1,108 @@ +"""Tests for the analytics module.""" +from __future__ import annotations + +import pytest +from scripts.analytics import format_text, format_markdown + + +class TestFormatText: + def test_basic_format(self): + report = { + "period_hours": 24, + "generated_at": "2025-01-01T00:00:00Z", + "tokens": { + "total_tokens": 1000, + "input_tokens": 500, + "output_tokens": 500, + "estimated_cost_usd": 0.005, + "requests": 10, + }, + "tickets": { + "total": 5, + "by_status": {"open": 3, "closed": 2}, + "resolution_rate": 40.0, + }, + "sessions": { + "total_sessions": 20, + "active_sessions": 5, + "avg_messages_per_session": 3.5, + "avg_duration_minutes": 15.0, + }, + "rate_limits": {"rate_limit_hits": 2}, + "top_issues": [ + {"category": "password_reset", "count": 10}, + {"category": "billing", "count": 5}, + ], + } + output = format_text(report) + assert "Helpdesk Analytics" in output + assert "1,000" in output + assert "password_reset" in output + assert "billing" in output + + def test_empty_top_issues(self): + report = { + "period_hours": 24, + "generated_at": "2025-01-01T00:00:00Z", + "tokens": { + "total_tokens": 0, + "input_tokens": 0, + "output_tokens": 0, + "estimated_cost_usd": 0.0, + "requests": 0, + }, + "tickets": { + "total": 0, + "by_status": {}, + "resolution_rate": 0.0, + }, + "sessions": { + "total_sessions": 0, + "active_sessions": 0, + "avg_messages_per_session": 0.0, + "avg_duration_minutes": 0.0, + }, + "rate_limits": {"rate_limit_hits": 0}, + "top_issues": [], + } + output = format_text(report) + assert "Helpdesk Analytics" in output + assert "Top Issues" not in output + + +class TestFormatMarkdown: + def test_basic_format(self): + report = { + "period_hours": 24, + "generated_at": "2025-01-01T00:00:00Z", + "tokens": { + "total_tokens": 1000, + "input_tokens": 500, + "output_tokens": 500, + "estimated_cost_usd": 0.005, + "requests": 10, + }, + "tickets": { + "total": 5, + "by_status": {"open": 3, "closed": 2}, + "resolution_rate": 40.0, + }, + "sessions": { + "total_sessions": 20, + "active_sessions": 5, + "avg_messages_per_session": 3.5, + "avg_duration_minutes": 15.0, + }, + "rate_limits": {"rate_limit_hits": 2}, + "top_issues": [ + {"category": "password_reset", "count": 10}, + ], + } + output = format_markdown(report) + assert "# Helpdesk Analytics Report" in output + assert "## Token Usage" in output + assert "## Tickets" in output + assert "## Sessions" in output + assert "## Rate Limiting" in output + assert "## Top Issues" in output + assert "password_reset" in output diff --git a/tests/test_email_fetcher.py b/tests/test_email_fetcher.py new file mode 100644 index 0000000..391adee --- /dev/null +++ b/tests/test_email_fetcher.py @@ -0,0 +1,62 @@ +"""Tests for the email fetcher module.""" +from __future__ import annotations + +import pytest +from unittest.mock import patch, MagicMock +from scripts.email_fetcher import decode_mime_header, extract_email_body + + +class TestDecodeMimeHeader: + def test_empty_header(self): + assert decode_mime_header("") == "" + + def test_none_header(self): + assert decode_mime_header(None) == "" + + def test_simple_ascii(self): + result = decode_mime_header("Hello World") + assert result == "Hello World" + + def test_encoded_header(self): + """Test with a MIME encoded word.""" + from email.header import Header + h = Header("Subject: Test", "utf-8") + result = decode_mime_header(str(h)) + assert "Subject" in result + assert "Test" in result + + +class TestExtractEmailBody: + def test_simple_text_body(self): + """Test extracting body from a simple email message.""" + from email.mime.text import MIMEText + msg = MIMEText("This is a test body", "plain", "utf-8") + body = extract_email_body(msg) + assert body == "This is a test body" + + def test_multipart_body(self): + """Test extracting body from a multipart message.""" + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + + msg = MIMEMultipart("mixed") + msg.attach(MIMEText("This is the plain text body", "plain", "utf-8")) + msg.attach(MIMEText("HTML body", "html", "utf-8")) + + body = extract_email_body(msg) + assert body == "This is the plain text body" + + def test_body_truncation(self): + """Test that body is truncated at 10KB.""" + from email.mime.text import MIMEText + long_text = "A" * 20000 + msg = MIMEText(long_text, "plain", "utf-8") + body = extract_email_body(msg) + assert len(body) == 10000 + + def test_empty_body(self): + """Test with an empty message body.""" + from email.mime.text import MIMEText + msg = MIMEText("", "plain", "utf-8") + body = extract_email_body(msg) + assert body == "" diff --git a/tests/test_health_monitor.py b/tests/test_health_monitor.py new file mode 100644 index 0000000..aeeef51 --- /dev/null +++ b/tests/test_health_monitor.py @@ -0,0 +1,21 @@ +"""Tests for the health monitor module.""" +from __future__ import annotations + +import pytest +from scripts.health_monitor import check_service + + +class TestCheckService: + @pytest.mark.asyncio + async def test_service_no_url(self): + result = await check_service("test-service", None) + assert result["name"] == "test-service" + assert result["status"] == "unknown" + assert result["latency_ms"] == 0 + + @pytest.mark.asyncio + async def test_service_unreachable(self): + result = await check_service("unreachable", "http://localhost:1/health") + assert result["name"] == "unreachable" + assert result["status"] == "unhealthy" + assert "error" in result diff --git a/tests/test_index_kb.py b/tests/test_index_kb.py new file mode 100644 index 0000000..58c2225 --- /dev/null +++ b/tests/test_index_kb.py @@ -0,0 +1,57 @@ +"""Tests for the index_kb module.""" +from __future__ import annotations + +import pytest +from scripts.index_kb import chunk_text, compute_hash + + +class TestChunkText: + def test_short_text(self): + text = "Hello World" + chunks = chunk_text(text, chunk_size=1000, overlap=200) + assert len(chunks) == 1 + assert chunks[0] == "Hello World" + + def test_long_text(self): + text = "A" * 3000 + chunks = chunk_text(text, chunk_size=1000, overlap=200) + assert len(chunks) >= 3 + # Check overlap + assert chunks[0][-200:] == chunks[1][:200] + + def test_exact_chunk_size(self): + text = "A" * 1000 + chunks = chunk_text(text, chunk_size=1000, overlap=200) + assert len(chunks) == 1 + + def test_overlap_behavior(self): + text = "Hello World! This is a test of the chunking function." + chunks = chunk_text(text, chunk_size=20, overlap=5) + assert len(chunks) >= 2 + # Check that chunks overlap + for i in range(len(chunks) - 1): + assert len(chunks[i]) <= 20 + + def test_empty_text(self): + chunks = chunk_text("", chunk_size=1000, overlap=200) + assert len(chunks) == 0 + + +class TestComputeHash: + def test_consistent_hash(self): + h1 = compute_hash("test content") + h2 = compute_hash("test content") + assert h1 == h2 + + def test_different_content(self): + h1 = compute_hash("content a") + h2 = compute_hash("content b") + assert h1 != h2 + + def test_hash_length(self): + h = compute_hash("test") + assert len(h) == 64 # SHA-256 hex digest + + def test_empty_string(self): + h = compute_hash("") + assert len(h) == 64 diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.py new file mode 100644 index 0000000..6f4b392 --- /dev/null +++ b/tests/test_rate_limiter.py @@ -0,0 +1,119 @@ +"""Tests for the rate limiter module.""" +from __future__ import annotations + +import time +import pytest +from scripts.rate_limiter import RateLimiter, RateLimitConfig + + +@pytest.fixture +def config(): + return RateLimitConfig( + max_requests_per_session=5, + window_seconds=60, + max_message_length=100, + max_session_duration=300, + ) + + +@pytest.fixture +def limiter(config): + return RateLimiter(config) + + +class TestRateLimitConfig: + def test_default_values(self): + cfg = RateLimitConfig() + assert cfg.max_requests_per_session == 50 + assert cfg.window_seconds == 3600 + assert cfg.max_message_length == 4000 + assert cfg.max_session_duration == 7200 + + +class TestRateLimiter: + def test_initial_request_allowed(self, limiter): + result = limiter.check_request("session1", "user1") + assert result["allowed"] is True + assert result["remaining"] == 4 + + def test_rate_limit_exceeded(self, limiter): + for i in range(5): + result = limiter.check_request("session2", "user1") + if i < 4: + assert result["allowed"] is True, f"Request {i} should be allowed" + else: + assert result["allowed"] is False, f"Request {i} should be denied" + assert "Rate limit exceeded" in result["reason"] + + def test_message_too_long(self, limiter): + result = limiter.check_request("session3", "user1", message_length=200) + assert result["allowed"] is False + assert "too long" in result["reason"] + + def test_session_expired(self, config, limiter): + # Create a session with very short duration + short_config = RateLimitConfig(max_session_duration=0) + short_limiter = RateLimiter(short_config) + + result = short_limiter.check_request("session4", "user1") + assert result["allowed"] is False + assert "expired" in result["reason"] + + def test_session_deactivated(self, limiter): + limiter.check_request("session5", "user1") + limiter.end_session("session5") + result = limiter.check_request("session5", "user1") + assert result["allowed"] is False + assert "deactivated" in result["reason"] + + def test_get_session_info(self, limiter): + limiter.check_request("session6", "user1") + info = limiter.get_session_info("session6") + assert info is not None + assert info["session_id"] == "session6" + assert info["user_id"] == "user1" + assert info["message_count"] == 1 + assert info["active"] is True + + def test_get_session_info_nonexistent(self, limiter): + info = limiter.get_session_info("nonexistent") + assert info is None + + def test_cleanup_expired(self, config, limiter): + # Create an expired session + limiter.check_request("expired_session", "user1") + # Manually set it to inactive + limiter.end_session("expired_session") + count = limiter.cleanup_expired() + assert count >= 1 + + def test_sliding_window_reset(self, config, limiter): + """Test that the sliding window resets after the window period.""" + # Use a config with a very short window + short_window = RateLimitConfig( + max_requests_per_session=2, + window_seconds=0, # Window already expired + ) + short_limiter = RateLimiter(short_window) + + # First request + result = short_limiter.check_request("session7", "user1") + assert result["allowed"] is True + + # Second request - window should reset since window_seconds=0 + result = short_limiter.check_request("session7", "user1") + assert result["allowed"] is True + + def test_multiple_sessions_independent(self, limiter): + """Test that different sessions have independent rate limits.""" + for i in range(5): + limiter.check_request("session_a", "user1") + + # Session A should be exhausted + result_a = limiter.check_request("session_a", "user1") + assert result_a["allowed"] is False + + # Session B should still have all requests + result_b = limiter.check_request("session_b", "user2") + assert result_b["allowed"] is True + assert result_b["remaining"] == 4