Admin console improvements + fixes

gateway/compose.production.yaml

@@ -19,6 +19,7 @@ volumes:
     sds-gateway-prod-temp-zips: {}
 
     # data that can be reconstructed:
+    sds-gateway-prod-app-logs: {}
     sds-gateway-prod-static: {}
     sds-gateway-prod-uv-cache: {}
     sds-gateway-prod-uv-venv-app: {}
@@ -76,10 +77,14 @@ services:
               target: /opt/uv-venv/
               type: volume
               read_only: false
+            - source: sds-gateway-prod-app-logs
+              target: /app/logs
+              type: volume
+              read_only: false
         post_start:
             - command:
                   chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/
-                  /opt/uv-venv/
+                  /opt/uv-venv/ /app/logs/
               user: root
         env_file:
             - ./.envs/production/django.env
@@ -363,10 +368,14 @@ services:
               target: /opt/uv-venv/
               type: volume
               read_only: false
+            - source: sds-gateway-prod-app-logs
+              target: /app/logs
+              type: volume
+              read_only: false
         post_start:
             - command:
                   chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/
-                  /opt/uv-venv/
+                  /opt/uv-venv/ /app/logs/
               user: root
         env_file:
             - ./.envs/production/django.env
@@ -420,10 +429,14 @@ services:
               target: /opt/uv-venv/
               type: volume
               read_only: false
+            - source: sds-gateway-prod-app-logs
+              target: /app/logs
+              type: volume
+              read_only: false
         post_start:
             - command:
                   chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/
-                  /opt/uv-venv/
+                  /opt/uv-venv/ /app/logs/
               user: root
         env_file:
             - ./.envs/production/django.env
@@ -472,10 +485,14 @@ services:
               target: /opt/uv-venv/
               type: volume
               read_only: false
+            - source: sds-gateway-prod-app-logs
+              target: /app/logs
+              type: volume
+              read_only: false
         post_start:
             - command:
                   chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/
-                  /opt/uv-venv/
+                  /opt/uv-venv/ /app/logs/
               user: root
         env_file:
             - ./.envs/production/django.env

gateway/config/settings/production.py

@@ -1,7 +1,8 @@
 """⚠️ Setting overrides for PRODUCTION ⚠️"""
-# ruff: noqa: F405, ERA001, E402
+# ruff: noqa: F405, ERA001, E402, E501
 
 import os
+from pathlib import Path
 
 import django_stubs_ext
 import sentry_sdk
@@ -138,6 +139,10 @@
 #   check ADMINS' email addresses in base.py if re-enabling this handler.
 DEFAULT_LOGGING["handlers"]["mail_admins"]["class"] = "logging.NullHandler"
 
+# Ensure log directory exists before RotatingFileHandler tries to write
+_LOG_DIR = Path("/app/logs/gateway.log").parent
+_LOG_DIR.mkdir(parents=True, exist_ok=True)
+
 # https://docs.djangoproject.com/en/4.2/ref/settings/#logging
 # See https://docs.djangoproject.com/en/4.2/topics/logging for
 # more details on how to customize your logging configuration.
@@ -149,6 +154,10 @@
         "colored": {
             "()": ColoredFormatter,
         },
+        "plain": {
+            "format": "%(asctime)s.%(msecs)03d | %(levelname)-8s | %(name)s:%(module)s:%(funcName)s:%(lineno)d - %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
     },
     "handlers": {
         "mail_admins": {
@@ -161,16 +170,24 @@
             "class": "logging.StreamHandler",
             "formatter": "colored",
         },
+        "file": {
+            "level": "INFO",
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "/app/logs/gateway.log",
+            "maxBytes": 50 * 1024 * 1024,
+            "backupCount": 5,
+            "formatter": "plain",
+        },
     },
-    "root": {"level": "DEBUG", "handlers": ["console"]},
+    "root": {"level": "DEBUG", "handlers": ["console", "file"]},
     "loggers": {
         "django.request": {
-            "handlers": ["console"],
+            "handlers": ["console", "file"],
             "level": "ERROR",
             "propagate": True,
         },
         "django.security.DisallowedHost": {
-            "handlers": ["console"],
+            "handlers": ["console", "file"],
             "level": "ERROR",
             "propagate": True,
         },

gateway/justfile

@@ -4,6 +4,7 @@ set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
 #   Add your machine hostname to ./scripts/prod-hostnames.env
 # constants
 
+COLUMNS := "119"
 env_selection_script := "./scripts/env-selection.sh"
 snapshot_log_dir := "./logs/snapshots"
 secrets_generator := "./scripts/generate-secrets.sh"

gateway/pyproject.toml

@@ -88,7 +88,7 @@
         # more verbose (live logs during every test):
         # addopts = "--maxfail=2 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout -o log_cli=true --showlocals --tb=long --capture=no"
         # less verbose (logs captured and shown only on failure; use --log-cli when debugging):
-        addopts              = "--maxfail=10 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout --tb=short"
+        addopts              = "--maxfail=10 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout --tb=short --durations=10"
         console_output_style = "progress"
         log_auto_indent      = "On"
         log_cli              = false

gateway/sds_gateway/admin.py

@@ -0,0 +1,167 @@
+"""Custom admin dashboard for the Django admin index page."""
+
+from __future__ import annotations
+
+import logging
+from copy import deepcopy
+from datetime import timedelta
+
+from django.contrib import admin
+from django.contrib.auth import get_user_model
+from django.db import DatabaseError
+from django.db import OperationalError
+from django.db import ProgrammingError
+from django.db.models import Count
+from django.db.models import Q
+from django.db.models import Sum
+from django.urls import reverse
+from django.utils import timezone
+
+from sds_gateway.api_methods.models import Capture
+from sds_gateway.api_methods.models import Dataset
+from sds_gateway.api_methods.models import File
+from sds_gateway.api_methods.utils.disk_utils import format_file_size
+from sds_gateway.monitoring.models import SystemHealthSnapshot
+
+logger = logging.getLogger(__name__)
+
+_DASHBOARD_FALLBACK: dict[str, object] = {
+    "active_file_count": 0,
+    "active_total_size": "0 B",
+    "cleanup_file_count": 0,
+    "cleanup_total_size": "0 B",
+    "top_users": [],
+    "capture_count": 0,
+    "dataset_count": 0,
+    "health_payload": None,
+    "recent_users": [],
+    "superusers": [],
+    "total_user_count": 0,
+    "file_admin_url": "#",
+    "capture_admin_url": "#",
+    "dataset_admin_url": "#",
+    "user_admin_url": "#",
+}
+
+
+def _file_stats() -> dict[str, object]:
+    """Return file-related stats for the dashboard."""
+    now = timezone.now()
+    thirty_days_ago = now - timedelta(days=30)
+
+    active_stats = File.objects.filter(is_deleted=False).aggregate(
+        count=Count("uuid"),
+        total_size=Sum("size"),
+    )
+    cleanup_stats = File.objects.filter(
+        is_deleted=True,
+        deleted_at__lt=thirty_days_ago,
+    ).aggregate(
+        count=Count("uuid"),
+        total_size=Sum("size"),
+    )
+    return {
+        "active_file_count": active_stats["count"] or 0,
+        "active_total_size": (
+            format_file_size(active_stats["total_size"])
+            if active_stats["total_size"]
+            else "0 B"
+        ),
+        "cleanup_file_count": cleanup_stats["count"] or 0,
+        "cleanup_total_size": (
+            format_file_size(cleanup_stats["total_size"])
+            if cleanup_stats["total_size"]
+            else "0 B"
+        ),
+    }
+
+
+def _capture_dataset_stats() -> dict[str, int]:
+    """Return capture and dataset counts for the dashboard."""
+    return {
+        "capture_count": Capture.objects.filter(is_deleted=False).count(),
+        "dataset_count": Dataset.objects.filter(is_deleted=False).count(),
+    }
+
+
+def _user_stats() -> dict[str, object]:
+    """Return user-related stats for the dashboard."""
+    user_model = get_user_model()
+    now = timezone.now()
+    fourteen_days_ago = now - timedelta(days=14)
+
+    top_users = (
+        user_model.objects.filter(files__is_deleted=False)
+        .annotate(
+            total_size=Sum("files__size", default=0),
+            file_count=Count("files__uuid"),
+        )
+        .order_by("-total_size")[:10]
+    )
+    recent_users = list(
+        user_model.objects.filter(date_joined__gte=fourteen_days_ago)
+        .order_by("-date_joined")
+        .values("email", "name", "is_active", "is_approved", "date_joined", "pk")
+    )
+    superusers = list(
+        user_model.objects.filter(Q(is_staff=True) | Q(is_superuser=True))
+        .order_by("-date_joined")
+        .values(
+            "email",
+            "name",
+            "is_active",
+            "is_approved",
+            "is_staff",
+            "is_superuser",
+            "pk",
+        )
+    )
+    return {
+        "top_users": top_users,
+        "recent_users": recent_users,
+        "superusers": superusers,
+        "total_user_count": user_model.objects.count(),
+    }
+
+
+def _system_health() -> dict[str, object]:
+    """Return system health snapshot for the dashboard."""
+    return {"health_payload": SystemHealthSnapshot.latest_snapshot_payload()}
+
+
+def _dashboard_context() -> dict[str, object]:
+    try:
+        ctx: dict[str, object] = {}
+        ctx.update(_file_stats())
+        ctx.update(_capture_dataset_stats())
+        ctx.update(_user_stats())
+        ctx.update(_system_health())
+        ctx["file_admin_url"] = reverse("admin:api_methods_file_changelist")
+        ctx["capture_admin_url"] = reverse("admin:api_methods_capture_changelist")
+        ctx["dataset_admin_url"] = reverse("admin:api_methods_dataset_changelist")
+        ctx["user_admin_url"] = reverse("admin:users_user_changelist")
+    except (OperationalError, ProgrammingError, DatabaseError):
+        logger.exception("Dashboard context query failed")
+        return deepcopy(_DASHBOARD_FALLBACK)
+    return ctx
+
+
+# TODO: Replace monkey-patch with AdminSite subclass.
+# Subclassing requires changing config/urls.py to use the custom site instance
+# instead of the default admin.site. For now, monkey-patch the index method
+# to inject dashboard context. Fragile if Django changes AdminSite internals.
+_original_admin_index = admin.site.index
+
+
+def _dashboard_index(request, extra_context=None):
+    extra_context = extra_context or {}
+    extra_context["dashboard"] = _dashboard_context()
+    return _original_admin_index(request, extra_context=extra_context)
+
+
+# Override the default admin site index
+admin.site.index = _dashboard_index
+admin.site.index_template = "admin/dashboard_index.html"
+admin.site.site_header = "SDS Gateway Admin"
+admin.site.site_title = "SDS Gateway Admin"
+admin.site.index_title = "Dashboard"