diff --git a/gateway/compose.production.yaml b/gateway/compose.production.yaml index 9cdefe3a0..b185ccd0c 100644 --- a/gateway/compose.production.yaml +++ b/gateway/compose.production.yaml @@ -19,6 +19,7 @@ volumes: sds-gateway-prod-temp-zips: {} # data that can be reconstructed: + sds-gateway-prod-app-logs: {} sds-gateway-prod-static: {} sds-gateway-prod-uv-cache: {} sds-gateway-prod-uv-venv-app: {} @@ -76,10 +77,14 @@ services: target: /opt/uv-venv/ type: volume read_only: false + - source: sds-gateway-prod-app-logs + target: /app/logs + type: volume + read_only: false post_start: - command: chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/ - /opt/uv-venv/ + /opt/uv-venv/ /app/logs/ user: root env_file: - ./.envs/production/django.env @@ -363,10 +368,14 @@ services: target: /opt/uv-venv/ type: volume read_only: false + - source: sds-gateway-prod-app-logs + target: /app/logs + type: volume + read_only: false post_start: - command: chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/ - /opt/uv-venv/ + /opt/uv-venv/ /app/logs/ user: root env_file: - ./.envs/production/django.env @@ -420,10 +429,14 @@ services: target: /opt/uv-venv/ type: volume read_only: false + - source: sds-gateway-prod-app-logs + target: /app/logs + type: volume + read_only: false post_start: - command: chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/ - /opt/uv-venv/ + /opt/uv-venv/ /app/logs/ user: root env_file: - ./.envs/production/django.env @@ -472,10 +485,14 @@ services: target: /opt/uv-venv/ type: volume read_only: false + - source: sds-gateway-prod-app-logs + target: /app/logs + type: volume + read_only: false post_start: - command: chown -R django:django /app/sds_gateway/media/ /opt/uv-cache/ - /opt/uv-venv/ + /opt/uv-venv/ /app/logs/ user: root env_file: - ./.envs/production/django.env diff --git a/gateway/config/settings/production.py b/gateway/config/settings/production.py index df30b9987..9733eec2e 100644 --- a/gateway/config/settings/production.py +++ b/gateway/config/settings/production.py @@ -1,7 +1,8 @@ """⚠️ Setting overrides for PRODUCTION ⚠️""" -# ruff: noqa: F405, ERA001, E402 +# ruff: noqa: F405, ERA001, E402, E501 import os +from pathlib import Path import django_stubs_ext import sentry_sdk @@ -138,6 +139,10 @@ # check ADMINS' email addresses in base.py if re-enabling this handler. DEFAULT_LOGGING["handlers"]["mail_admins"]["class"] = "logging.NullHandler" +# Ensure log directory exists before RotatingFileHandler tries to write +_LOG_DIR = Path("/app/logs/gateway.log").parent +_LOG_DIR.mkdir(parents=True, exist_ok=True) + # https://docs.djangoproject.com/en/4.2/ref/settings/#logging # See https://docs.djangoproject.com/en/4.2/topics/logging for # more details on how to customize your logging configuration. @@ -149,6 +154,10 @@ "colored": { "()": ColoredFormatter, }, + "plain": { + "format": "%(asctime)s.%(msecs)03d | %(levelname)-8s | %(name)s:%(module)s:%(funcName)s:%(lineno)d - %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + }, }, "handlers": { "mail_admins": { @@ -161,16 +170,24 @@ "class": "logging.StreamHandler", "formatter": "colored", }, + "file": { + "level": "INFO", + "class": "logging.handlers.RotatingFileHandler", + "filename": "/app/logs/gateway.log", + "maxBytes": 50 * 1024 * 1024, + "backupCount": 5, + "formatter": "plain", + }, }, - "root": {"level": "DEBUG", "handlers": ["console"]}, + "root": {"level": "DEBUG", "handlers": ["console", "file"]}, "loggers": { "django.request": { - "handlers": ["console"], + "handlers": ["console", "file"], "level": "ERROR", "propagate": True, }, "django.security.DisallowedHost": { - "handlers": ["console"], + "handlers": ["console", "file"], "level": "ERROR", "propagate": True, }, diff --git a/gateway/justfile b/gateway/justfile index 3267a269f..1839a5b05 100644 --- a/gateway/justfile +++ b/gateway/justfile @@ -4,6 +4,7 @@ set shell := ["bash", "-eu", "-o", "pipefail", "-c"] # Add your machine hostname to ./scripts/prod-hostnames.env # constants +COLUMNS := "119" env_selection_script := "./scripts/env-selection.sh" snapshot_log_dir := "./logs/snapshots" secrets_generator := "./scripts/generate-secrets.sh" diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index a88f1bdee..84e0ca511 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -88,7 +88,7 @@ # more verbose (live logs during every test): # addopts = "--maxfail=2 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout -o log_cli=true --showlocals --tb=long --capture=no" # less verbose (logs captured and shown only on failure; use --log-cli when debugging): - addopts = "--maxfail=10 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout --tb=short" + addopts = "--maxfail=10 --new-first -rf --strict-markers --cov=sds_gateway --cov-report=html --show-capture=stdout --tb=short --durations=10" console_output_style = "progress" log_auto_indent = "On" log_cli = false diff --git a/gateway/sds_gateway/admin.py b/gateway/sds_gateway/admin.py new file mode 100644 index 000000000..7a6a8bd41 --- /dev/null +++ b/gateway/sds_gateway/admin.py @@ -0,0 +1,167 @@ +"""Custom admin dashboard for the Django admin index page.""" + +from __future__ import annotations + +import logging +from copy import deepcopy +from datetime import timedelta + +from django.contrib import admin +from django.contrib.auth import get_user_model +from django.db import DatabaseError +from django.db import OperationalError +from django.db import ProgrammingError +from django.db.models import Count +from django.db.models import Q +from django.db.models import Sum +from django.urls import reverse +from django.utils import timezone + +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import File +from sds_gateway.api_methods.utils.disk_utils import format_file_size +from sds_gateway.monitoring.models import SystemHealthSnapshot + +logger = logging.getLogger(__name__) + +_DASHBOARD_FALLBACK: dict[str, object] = { + "active_file_count": 0, + "active_total_size": "0 B", + "cleanup_file_count": 0, + "cleanup_total_size": "0 B", + "top_users": [], + "capture_count": 0, + "dataset_count": 0, + "health_payload": None, + "recent_users": [], + "superusers": [], + "total_user_count": 0, + "file_admin_url": "#", + "capture_admin_url": "#", + "dataset_admin_url": "#", + "user_admin_url": "#", +} + + +def _file_stats() -> dict[str, object]: + """Return file-related stats for the dashboard.""" + now = timezone.now() + thirty_days_ago = now - timedelta(days=30) + + active_stats = File.objects.filter(is_deleted=False).aggregate( + count=Count("uuid"), + total_size=Sum("size"), + ) + cleanup_stats = File.objects.filter( + is_deleted=True, + deleted_at__lt=thirty_days_ago, + ).aggregate( + count=Count("uuid"), + total_size=Sum("size"), + ) + return { + "active_file_count": active_stats["count"] or 0, + "active_total_size": ( + format_file_size(active_stats["total_size"]) + if active_stats["total_size"] + else "0 B" + ), + "cleanup_file_count": cleanup_stats["count"] or 0, + "cleanup_total_size": ( + format_file_size(cleanup_stats["total_size"]) + if cleanup_stats["total_size"] + else "0 B" + ), + } + + +def _capture_dataset_stats() -> dict[str, int]: + """Return capture and dataset counts for the dashboard.""" + return { + "capture_count": Capture.objects.filter(is_deleted=False).count(), + "dataset_count": Dataset.objects.filter(is_deleted=False).count(), + } + + +def _user_stats() -> dict[str, object]: + """Return user-related stats for the dashboard.""" + user_model = get_user_model() + now = timezone.now() + fourteen_days_ago = now - timedelta(days=14) + + top_users = ( + user_model.objects.filter(files__is_deleted=False) + .annotate( + total_size=Sum("files__size", default=0), + file_count=Count("files__uuid"), + ) + .order_by("-total_size")[:10] + ) + recent_users = list( + user_model.objects.filter(date_joined__gte=fourteen_days_ago) + .order_by("-date_joined") + .values("email", "name", "is_active", "is_approved", "date_joined", "pk") + ) + superusers = list( + user_model.objects.filter(Q(is_staff=True) | Q(is_superuser=True)) + .order_by("-date_joined") + .values( + "email", + "name", + "is_active", + "is_approved", + "is_staff", + "is_superuser", + "pk", + ) + ) + return { + "top_users": top_users, + "recent_users": recent_users, + "superusers": superusers, + "total_user_count": user_model.objects.count(), + } + + +def _system_health() -> dict[str, object]: + """Return system health snapshot for the dashboard.""" + return {"health_payload": SystemHealthSnapshot.latest_snapshot_payload()} + + +def _dashboard_context() -> dict[str, object]: + try: + ctx: dict[str, object] = {} + ctx.update(_file_stats()) + ctx.update(_capture_dataset_stats()) + ctx.update(_user_stats()) + ctx.update(_system_health()) + ctx["file_admin_url"] = reverse("admin:api_methods_file_changelist") + ctx["capture_admin_url"] = reverse("admin:api_methods_capture_changelist") + ctx["dataset_admin_url"] = reverse("admin:api_methods_dataset_changelist") + ctx["user_admin_url"] = reverse("admin:users_user_changelist") + except (OperationalError, ProgrammingError, DatabaseError): + logger.exception("Dashboard context query failed") + return deepcopy(_DASHBOARD_FALLBACK) + return ctx + + +# TODO: Replace monkey-patch with AdminSite subclass. +# Subclassing requires changing config/urls.py to use the custom site instance +# instead of the default admin.site. For now, monkey-patch the index method +# to inject dashboard context. Fragile if Django changes AdminSite internals. +_original_admin_index = admin.site.index + + +def _dashboard_index(request, extra_context=None): + extra_context = extra_context or {} + extra_context["dashboard"] = _dashboard_context() + return _original_admin_index(request, extra_context=extra_context) + + +# Override the default admin site index +admin.site.index = _dashboard_index +admin.site.index_template = "admin/dashboard_index.html" +admin.site.site_header = "SDS Gateway Admin" +admin.site.site_title = "SDS Gateway Admin" +admin.site.index_title = "Dashboard" diff --git a/gateway/sds_gateway/api_methods/admin.py b/gateway/sds_gateway/api_methods/admin.py index 15cb4c757..8ed2e1ab0 100644 --- a/gateway/sds_gateway/api_methods/admin.py +++ b/gateway/sds_gateway/api_methods/admin.py @@ -1,33 +1,125 @@ import json from django.contrib import admin +from django.db.models import Count from sds_gateway.api_methods import models +from sds_gateway.api_methods.utils.disk_utils import format_file_size # Register your models here. @admin.register(models.File) class FileAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("name", "media_type", "size", "owner", "is_deleted") - search_fields = ("checksum", "name", "media_type", "owner") + list_display = ( + "name", + "capture_count", + "owner", + "media_type", + "formatted_size", + "is_public", + "is_deleted", + "expiration_date", + "created_at", + "updated_at", + ) + search_fields = ("sum_blake3", "name", "media_type", "owner__email") ordering = ("-updated_at",) + @admin.display(description="# Cap", ordering="_capture_count") + def capture_count(self, obj): + count = getattr(obj, "_capture_count", 0) + return count or "-" + + @admin.display(description="Size", ordering="size") + def formatted_size(self, obj): + return format_file_size(obj.size) if obj.size is not None else "-" + + def get_queryset(self, request): + return ( + super() + .get_queryset(request) + .select_related("owner") + .annotate(_capture_count=Count("captures")) + ) + @admin.register(models.Capture) class CaptureAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("name", "channel", "capture_type", "index_name") + list_display = ( + "name", + "dataset_count", + "owner", + "capture_type", + "file_count", + "origin", + "channel", + "index_name", + "is_deleted", + "created_at", + "updated_at", + ) search_fields = ("uuid", "name", "channel", "index_name") list_filter = ("channel", "capture_type", "index_name") ordering = ("-updated_at",) + @admin.display(description="# Ds", ordering="_dataset_count") + def dataset_count(self, obj): + count = getattr(obj, "_dataset_count", 0) + return count or "-" + + @admin.display(description="Files", ordering="_file_count") + def file_count(self, obj): + count = getattr(obj, "_file_count", 0) + return f"{count} files" if count else "-" + + def get_queryset(self, request): + return ( + super() + .get_queryset(request) + .select_related("owner") + .annotate(_dataset_count=Count("datasets"), _file_count=Count("files")) + ) + @admin.register(models.Dataset) class DatasetAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("name", "doi", "get_keywords", "status", "owner") + list_display = ( + "name", + "owner", + "status", + "capture_count", + "file_count", + "version", + "doi", + "get_keywords", + "license", + "release_date", + "is_deleted", + "created_at", + "updated_at", + ) search_fields = ("name", "doi", "keywords__name", "owner__email") list_filter = ("status", "keywords") ordering = ("-updated_at",) + @admin.display(description="# Cap", ordering="_capture_count") + def capture_count(self, obj): + count = getattr(obj, "_capture_count", 0) + return f"{count} captures" if count else "-" + + @admin.display(description="Artifact Files", ordering="_file_count") + def file_count(self, obj): + count = getattr(obj, "_file_count", 0) + return f"{count} artifact files" if count else "-" + + def get_queryset(self, request): + return ( + super() + .get_queryset(request) + .select_related("owner") + .annotate(_capture_count=Count("captures"), _file_count=Count("files")) + ) + @admin.display(description="Keywords") def get_keywords(self, obj): """Display comma-separated list of keywords.""" @@ -75,18 +167,65 @@ def save_model(self, request, obj, form, change): @admin.register(models.TemporaryZipFile) class TemporaryZipFileAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("uuid", "owner", "created_at", "expires_at") - search_fields = ("uuid", "owner") + list_display = ( + "filename", + "owner", + "formatted_file_size", + "creation_status", + "is_downloaded", + "created_at", + "expires_at", + ) + search_fields = ("uuid", "owner__email") ordering = ("-created_at",) + @admin.display(description="File Size", ordering="file_size") + def formatted_file_size(self, obj): + return format_file_size(obj.file_size) if obj.file_size is not None else "-" + + def get_queryset(self, request): + return super().get_queryset(request).select_related("owner") + @admin.register(models.UserSharePermission) class UserSharePermissionAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("item_uuid", "item_type", "shared_with", "owner", "is_enabled") - search_fields = ("item_uuid", "item_type", "shared_with", "owner") + list_display = ( + "owner", + "shared_with", + "item_type", + "item_name", + "permission_level", + "notified", + "is_enabled", + "created_at", + ) + search_fields = ("item_uuid", "item_type", "shared_with__email", "owner__email") list_filter = ("item_type", "is_enabled") ordering = ("-updated_at",) + @admin.display(description="Item Name") + def item_name(self, obj): + """Resolve item_uuid to the actual dataset/capture name. + + Note: UserSharePermission uses item_type + item_uuid (UUID polymorphic), + not FK relations, so Prefetch/select_related can't batch-load items. + Consider denormalizing item_name onto the model if N+1 becomes a problem. + """ + from sds_gateway.api_methods.models import Capture # noqa: PLC0415 + from sds_gateway.api_methods.models import Dataset # noqa: PLC0415 + from sds_gateway.api_methods.models import ItemType # noqa: PLC0415 + + if obj.item_type == ItemType.DATASET: + item = Dataset.objects.filter(uuid=obj.item_uuid).first() + return item.name if item else str(obj.item_uuid) + if obj.item_type == ItemType.CAPTURE: + item = Capture.objects.filter(uuid=obj.item_uuid).first() + return item.name if item else str(obj.item_uuid) + return str(obj.item_uuid) + + def get_queryset(self, request): + return super().get_queryset(request).select_related("owner", "shared_with") + @admin.register(models.DEPRECATEDPostProcessedData) class PostProcessedDataAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] @@ -129,10 +268,22 @@ class PostProcessedDataAdmin(admin.ModelAdmin): # pyright: ignore[reportMissing @admin.register(models.ShareGroup) class ShareGroupAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] - list_display = ("name", "owner") - search_fields = ("name", "owner") + list_display = ("name", "owner", "member_count", "created_at", "updated_at") + search_fields = ("name", "owner__email") ordering = ("-updated_at",) + @admin.display(description="Members", ordering="_member_count") + def member_count(self, obj): + return getattr(obj, "_member_count", "-") + + def get_queryset(self, request): + return ( + super() + .get_queryset(request) + .select_related("owner") + .annotate(_member_count=Count("group_share_permissions")) + ) + @admin.register(models.Keyword) class KeywordAdmin(admin.ModelAdmin): # pyright: ignore[reportMissingTypeArgument] diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index f1c150367..fe369f215 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -215,6 +215,7 @@ def test_capture_on_draft_only_skips_without_fed_doc( def test_deleted_capture_reindexes_with_is_deleted( self, mock_indexer_cls: MagicMock, + _mock_publish: MagicMock, ) -> None: mock_indexer = mock_indexer_cls.return_value capture = CaptureFactory(is_deleted=True) @@ -243,6 +244,7 @@ class TestDatasetDisconnectReindex(TestCase): def test_disconnect_captures_reindexes_orphans( self, mock_indexer_cls: MagicMock, + _mock_publish: MagicMock, ) -> None: mock_indexer = mock_indexer_cls.return_value dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py index 47359baed..0d05d8b05 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py @@ -70,5 +70,9 @@ def test_mint_via_http(self) -> None: HTTP_AUTHORIZATION=f"Token {token.key}", ) assert response.status_code == status.HTTP_200_OK - assert "api_key" in response.json() - assert response.json()["email"] == target.email + payload = response.json() + assert "api_key" in payload + assert payload["email"] == sync_user.email + key = UserAPIKey.objects.get_from_key(payload["api_key"]) + assert key.user_id == sync_user.pk + assert UserAPIKey.objects.filter(user=target).exists() is False diff --git a/gateway/sds_gateway/context_processors.py b/gateway/sds_gateway/context_processors.py index d0d4bae47..b3aea7297 100644 --- a/gateway/sds_gateway/context_processors.py +++ b/gateway/sds_gateway/context_processors.py @@ -49,11 +49,16 @@ def _load_version() -> dict[str, str]: # File missing or unparseable — try env var fallback. if not version_path.is_file(): - logger.warning("version file not found at %s", version_path) + logger.debug("version file not found at %s (expected in dev)", version_path) commit = os.environ.get("SDS_COMMIT_HASH", "unknown") return {"commit": commit, "version": commit} +# Cache result so version.json is parsed and logged once at startup, +# not on every request via static_cache_busting(). +_VERSION_CACHE = _load_version() + + def _latest_admin_monitoring_status() -> dict[str, Any] | None: try: from sds_gateway.monitoring.models import SystemHealthSnapshot # noqa: PLC0415 @@ -123,5 +128,4 @@ def static_cache_busting(_request: HttpRequest) -> dict[str, Any]: which includes both the nearest release tag and the commit count/hash. Falls back to ``SDS_COMMIT_HASH`` or ``"unknown"``. """ - version = _load_version() - return {"STATIC_CACHE_BUSTING_VERSION": version["version"]} + return {"STATIC_CACHE_BUSTING_VERSION": _VERSION_CACHE["version"]} diff --git a/gateway/sds_gateway/static/admin/css/dashboard.css b/gateway/sds_gateway/static/admin/css/dashboard.css new file mode 100644 index 000000000..6a983fc1d --- /dev/null +++ b/gateway/sds_gateway/static/admin/css/dashboard.css @@ -0,0 +1,245 @@ +/* Dashboard styles — uses Django admin's native CSS variables */ + +/* Layout */ +.dashboard-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(200px, 100%), 1fr)); + gap: 16px; + margin-bottom: 24px; +} + +/* Stat cards */ +.dashboard-card { + background: var(--darkened-bg); + border-radius: 8px; + padding: 20px; + text-align: center; + border: 1px solid var(--hairline-color); +} + +/* Stat card numbers — var(--primary) ≈ #79aec8 is borderline on dark bg. + Brighter blue ensures 7:1+ contrast for large bold text. */ +.dashboard-card__number { + font-size: 2.2em; + font-weight: 700; + color: #60a5fa; + line-height: 1.2; +} + +@media (prefers-color-scheme: light) { + .dashboard-card__number { + color: #0055aa; + } +} + +.dashboard-card__number a { + color: inherit; + text-decoration: none; +} + +.dashboard-card__number a:hover { + color: var(--secondary); +} + +/* Card labels — var(--body-quiet-color) can be too low contrast. + Explicit lighter gray ensures 6:1+ contrast on dark bg. */ +.dashboard-card__label { + color: #b0b0b0; + margin-top: 4px; + font-size: 0.9em; +} + +/* Card sub-text */ +.dashboard-card__sub { + color: #a0a0a0; + font-size: 0.85em; + margin-top: 2px; +} + +@media (prefers-color-scheme: light) { + .dashboard-card__label { + color: #555; + } + .dashboard-card__sub { + color: #666; + } +} + +/* Cleanup banner */ +.dashboard-cleanup { + background: var(--message-error-bg); + border-left: 4px solid var(--error-fg); + border-radius: 6px; + padding: 16px 20px; + margin-bottom: 24px; +} + +.dashboard-cleanup__title { + margin: 0 0 8px; + color: var(--error-fg); +} + +.dashboard-cleanup__body { + margin: 0; + color: var(--body-fg); +} + +/* Section modules */ +.dashboard-section { + margin-bottom: 24px; + padding: 16px 20px; +} + +/* Health status pill */ +.dashboard-pill { + display: inline-block; + padding: 4px 12px; + border-radius: 12px; + font-size: 0.9em; + font-weight: 700; + color: #fff; + letter-spacing: 0.03em; +} + +.dashboard-pill--healthy { + background: #28a745; +} +.dashboard-pill--degraded { + background: var(--accent); + color: var(--body-fg); +} +.dashboard-pill--down { + background: var(--error-fg); +} + +/* Health meta */ +.dashboard-meta { + color: #a0a0a0; + font-size: 0.85em; + margin-top: 6px; +} + +@media (prefers-color-scheme: light) { + .dashboard-meta { + color: #666; + } +} + +/* Tables */ +.dashboard-table { + width: 100%; + border-collapse: collapse; + margin-top: 12px; +} + +.dashboard-table th { + text-align: left; + padding: 10px 12px; + border-bottom: 2px solid var(--border-color); + font-size: 0.85em; + text-transform: uppercase; + letter-spacing: 0.04em; + color: #b0b0b0; +} + +@media (prefers-color-scheme: light) { + .dashboard-table th { + color: #555; + } +} + +.dashboard-table td { + padding: 10px 12px; + border-bottom: 1px solid var(--hairline-color); + color: var(--body-fg); +} + +.dashboard-table tbody tr:nth-child(even) { + background: var(--darkened-bg); +} + +.dashboard-table tbody tr:hover { + background: var(--selected-bg); +} + +/* Clickable rows */ +.dashboard-clickable { + cursor: pointer; +} + +.dashboard-clickable:hover { + background: var(--selected-bg); +} + +/* Status indicator dot */ +.dashboard-dot { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 50%; + margin-right: 6px; + vertical-align: middle; +} + +.dashboard-dot--healthy { + background: #28a745; +} +.dashboard-dot--degraded { + background: var(--accent); +} +.dashboard-dot--down { + background: var(--error-fg); +} + +/* Check/cross marks — #28a745 on success bg is ~2.3:1, fails WCAG AA */ +.dashboard-check { + color: #1a7a2e; + background: var(--message-success-bg); + padding: 2px 6px; + border-radius: 3px; + font-weight: bold; +} + +@media (prefers-color-scheme: light) { + .dashboard-check { + color: #145a22; + } +} + +.dashboard-cross { + color: var(--error-fg); +} + +/* Muted empty state */ +.dashboard-empty { + color: #a0a0a0; + margin: 8px 0 0; + font-style: italic; +} + +@media (prefers-color-scheme: light) { + .dashboard-empty { + color: #666; + } +} + +/* Utility classes */ +.text-center { + text-align: center; +} + +/* Dashboard heading — WCAG AA: var(--secondary) ≈ #417690 fails on dark bg (~2.6:1). + Use explicit high-contrast colors with theme adaptation. */ +.dashboard-heading { + color: #e0e0e0; + margin-bottom: 20px; + padding-bottom: 8px; + border-bottom: 2px solid #5cb8ff; +} + +@media (prefers-color-scheme: light) { + .dashboard-heading { + color: #1a1a2e; + border-bottom-color: #417690; + } +} diff --git a/gateway/sds_gateway/templates/admin/dashboard_index.html b/gateway/sds_gateway/templates/admin/dashboard_index.html new file mode 100644 index 000000000..4808b1972 --- /dev/null +++ b/gateway/sds_gateway/templates/admin/dashboard_index.html @@ -0,0 +1,177 @@ +{% extends "admin/base_site.html" %} + +{% load i18n static %} + +{% block title %}Dashboard | {{ title }}{% endblock title %} +{% block extrahead %} + {{ block.super }} + +{% endblock extrahead %} +{% block content %} +
+

Gateway Dashboard

+ +
+
+
+ {{ dashboard.active_file_count }} +
+
Files
+
{{ dashboard.active_total_size }}
+
+
+
+ {{ dashboard.capture_count }} +
+
Captures
+
+
+
+ {{ dashboard.dataset_count }} +
+
Datasets
+
+
+
+ {{ dashboard.total_user_count }} +
+
Users
+
+
+ + {% if dashboard.cleanup_file_count > 0 %} +
+

Cleanup Candidates

+

+ {{ dashboard.cleanup_file_count }} soft-deleted files older than 30 days + — {{ dashboard.cleanup_total_size }} recoverable +

+
+ {% endif %} + + {% if dashboard.health_payload %} +
+

System Health

+ {% with status=dashboard.health_payload.overall_status %} + {{ status|upper }} + {% endwith %} +
Checked at: {{ dashboard.health_payload.checked_at }}
+ + + + + + + + + {% for service in dashboard.health_payload.child_statuses %} + + + + + {% endfor %} + +
ServiceStatus
{{ service.service_name }} + + {{ service.status }} +
+
+ {% endif %} + +
+

Recent Users (14 days)

+ {% if dashboard.recent_users %} + + + + + + + + + + + + {% for user in dashboard.recent_users %} + + + + + + + + {% endfor %} + +
NameEmailJoinedActiveApproved
{{ user.name }}{{ user.email }}{{ user.date_joined|date:"M d, Y" }} + {% if user.is_active %}✓{% else %}✗{% endif %} + + {% if user.is_approved %}✓{% else %}✗{% endif %} +
+ {% else %} +

No users joined in the last 14 days.

+ {% endif %} +
+ +
+

Superusers & Admins

+ {% if dashboard.superusers %} + + + + + + + + + + + + {% for user in dashboard.superusers %} + + + + + + + + {% endfor %} + +
NameEmailStaffSuperuserActive
{{ user.name }}{{ user.email }} + {% if user.is_staff %}✓{% else %}✗{% endif %} + + {% if user.is_superuser %}✓{% else %}✗{% endif %} + + {% if user.is_active %}✓{% else %}✗{% endif %} +
+ {% else %} +

No staff or superuser accounts found.

+ {% endif %} +
+ + {% if dashboard.top_users %} +
+

Top Users by Storage

+ + + + + + + + + + {% for user in dashboard.top_users %} + + + + + + {% endfor %} + +
UserFilesTotal Size
{{ user.email }}{{ user.file_count }}{{ user.total_size|filesizeformat }}
+
+ {% endif %} +
+{% endblock content %} diff --git a/gateway/sds_gateway/tests/__init__.py b/gateway/sds_gateway/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gateway/sds_gateway/tests/test_admin_dashboard.py b/gateway/sds_gateway/tests/test_admin_dashboard.py new file mode 100644 index 000000000..e3f02995e --- /dev/null +++ b/gateway/sds_gateway/tests/test_admin_dashboard.py @@ -0,0 +1,253 @@ +"""Tests for the custom admin dashboard.""" + +from datetime import timedelta +from unittest.mock import patch + +import pytest +from django.db import OperationalError +from django.db import connection +from django.test.utils import CaptureQueriesContext +from django.urls import reverse +from django.utils import timezone +from rest_framework.status import HTTP_200_OK + +from sds_gateway.admin import _dashboard_context +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import File +from sds_gateway.users.tests.factories import UserFactory + +pytestmark = pytest.mark.django_db + + +def test_dashboard_context_empty_database() -> None: + ctx = _dashboard_context() + + assert ctx["active_file_count"] == 0 + assert ctx["active_total_size"] == "0 B" + assert ctx["cleanup_file_count"] == 0 + assert ctx["cleanup_total_size"] == "0 B" + assert list(ctx["top_users"]) == [] + assert ctx["capture_count"] == 0 + assert ctx["dataset_count"] == 0 + + +def test_dashboard_context_with_fixture_data() -> None: + user = UserFactory() + now = timezone.now() + + File.objects.create( + owner=user, + name="active1.h5", + size=1024 * 1024, + directory="files/", + file="files/active1.h5", + ) + File.objects.create( + owner=user, + name="active2.h5", + size=2 * 1024 * 1024, + directory="files/", + file="files/active2.h5", + ) + + # Deleted file older than 30 days (cleanup candidate) + File.objects.create( + owner=user, + name="old_deleted.h5", + size=5 * 1024 * 1024, + directory="files/", + file="files/old_deleted.h5", + is_deleted=True, + deleted_at=now - timedelta(days=45), + ) + + # Deleted file less than 30 days ago (not a cleanup candidate) + File.objects.create( + owner=user, + name="recent_deleted.h5", + size=3 * 1024 * 1024, + directory="files/", + file="files/recent_deleted.h5", + is_deleted=True, + deleted_at=now - timedelta(days=5), + ) + + Capture.objects.create( + name="cap1", + owner=user, + capture_type="drf", + top_level_dir="/data/cap1", + ) + Capture.objects.create( + name="cap_deleted", + owner=user, + capture_type="drf", + top_level_dir="/data/cap_deleted", + is_deleted=True, + ) + + Dataset.objects.create(name="ds1", owner=user) + Dataset.objects.create(name="ds_deleted", owner=user, is_deleted=True) + + ctx = _dashboard_context() + + active_file_count = 2 + capture_count = 1 + dataset_count = 1 + cleanup_file_count = 1 + + assert ctx["active_file_count"] == active_file_count + assert ctx["capture_count"] == capture_count + assert ctx["dataset_count"] == dataset_count + assert ctx["cleanup_file_count"] == cleanup_file_count + assert ctx["cleanup_total_size"] != "0 B" + + +def test_cleanup_candidates_exclude_recent_deletes() -> None: + """Only files deleted >30 days ago appear as cleanup candidates.""" + user = UserFactory() + now = timezone.now() + + File.objects.create( + owner=user, + name="old.h5", + size=100, + directory="files/", + file="files/old.h5", + is_deleted=True, + deleted_at=now - timedelta(days=31), + ) + File.objects.create( + owner=user, + name="new.h5", + size=200, + directory="files/", + file="files/new.h5", + is_deleted=True, + deleted_at=now - timedelta(days=29), + ) + + ctx = _dashboard_context() + + assert ctx["cleanup_file_count"] == 1 + + +def test_top_users_ordered_by_total_size() -> None: + user_a = UserFactory() + user_b = UserFactory() + user_c = UserFactory() + + # user_b: 3MB, user_a: 1MB, user_c: 0 + File.objects.create( + owner=user_a, + name="a.h5", + size=1024 * 1024, + directory="files/", + file="files/a.h5", + ) + for i in range(3): + File.objects.create( + owner=user_b, + name=f"b{i}.h5", + size=1024 * 1024, + directory="files/", + file=f"files/b{i}.h5", + ) + + ctx = _dashboard_context() + top_users = list(ctx["top_users"]) + + assert top_users[0].email == user_b.email + assert top_users[1].email == user_a.email + # user_c has no files, not in the list + assert all(u.email != user_c.email for u in top_users) + + +def test_dashboard_context_query_count() -> None: + """Verify no N+1 queries — dashboard context should use a bounded number.""" + with CaptureQueriesContext(connection) as queries: + _dashboard_context() + + # Expected queries: active files aggregate, cleanup files aggregate, + # top users, capture count, dataset count, health snapshot = 6 + max_queries = 8 + assert len(queries) <= max_queries + + +def test_dashboard_index_view_returns_200(client) -> None: + admin_user = UserFactory(is_staff=True, is_superuser=True) + client.force_login(admin_user) + + response = client.get(reverse("admin:index")) + + assert response.status_code == HTTP_200_OK + assert b"Gateway Dashboard" in response.content + assert b"Files" in response.content + assert b"Captures" in response.content + assert b"Datasets" in response.content + + +def test_dashboard_context_recent_users() -> None: + """Only users who joined within the last 14 days appear in recent_users.""" + recent = UserFactory(date_joined=timezone.now() - timedelta(days=10)) + old = UserFactory(date_joined=timezone.now() - timedelta(days=20)) + + ctx = _dashboard_context() + recent_emails = [u["email"] for u in ctx["recent_users"]] + + assert recent.email in recent_emails + assert old.email not in recent_emails + + +def test_dashboard_context_superusers() -> None: + """Staff and superuser users appear in the superusers list.""" + staff = UserFactory(is_staff=True, is_superuser=False) + superuser = UserFactory(is_staff=False, is_superuser=True) + regular = UserFactory(is_staff=False, is_superuser=False) + + ctx = _dashboard_context() + su_emails = [u["email"] for u in ctx["superusers"]] + + assert staff.email in su_emails + assert superuser.email in su_emails + assert regular.email not in su_emails + + +def test_dashboard_context_admin_urls() -> None: + """Admin URL fields are non-empty strings.""" + ctx = _dashboard_context() + + for key in ( + "file_admin_url", + "capture_admin_url", + "dataset_admin_url", + "user_admin_url", + ): + assert isinstance(ctx[key], str) + assert len(ctx[key]) > 0 + + +def test_dashboard_context_db_error_returns_fallback() -> None: + """When DB queries fail, _dashboard_context returns safe defaults.""" + with patch( + "sds_gateway.admin.File.objects.filter", + side_effect=OperationalError("DB connection lost"), + ): + ctx = _dashboard_context() + + assert ctx["active_file_count"] == 0 + assert ctx["active_total_size"] == "0 B" + assert ctx["cleanup_file_count"] == 0 + assert ctx["cleanup_total_size"] == "0 B" + assert ctx["top_users"] == [] + assert ctx["capture_count"] == 0 + assert ctx["dataset_count"] == 0 + assert ctx["health_payload"] is None + assert ctx["recent_users"] == [] + assert ctx["superusers"] == [] + assert ctx["total_user_count"] == 0 + assert ctx["file_admin_url"] == "#" + assert ctx["capture_admin_url"] == "#" + assert ctx["dataset_admin_url"] == "#" + assert ctx["user_admin_url"] == "#" diff --git a/gateway/sds_gateway/users/admin.py b/gateway/sds_gateway/users/admin.py index 231844cfa..20f1324a3 100644 --- a/gateway/sds_gateway/users/admin.py +++ b/gateway/sds_gateway/users/admin.py @@ -43,7 +43,16 @@ class UserAdmin(auth_admin.UserAdmin): # pyright: ignore[reportMissingTypeArgum ), (_("Important dates"), {"fields": ("last_login", "date_joined")}), ) - list_display = ["email", "name", "is_superuser"] + list_display = [ + "email", + "name", + "is_active", + "is_approved", + "is_staff", + "is_superuser", + "last_login", + "date_joined", + ] search_fields = ["name"] ordering = ["id"] add_fieldsets = ( diff --git a/gateway/sds_gateway/visualizations/admin.py b/gateway/sds_gateway/visualizations/admin.py index d1cfd430b..1efacfa6e 100644 --- a/gateway/sds_gateway/visualizations/admin.py +++ b/gateway/sds_gateway/visualizations/admin.py @@ -12,9 +12,11 @@ class PostProcessedDataAdmin(admin.ModelAdmin): list_display = ( "processing_type", "capture", + "get_owner", "processing_status", - "processed_at", "pipeline_id", + "processed_at", + "has_error", "created_at", ) list_filter = ( @@ -30,7 +32,6 @@ class PostProcessedDataAdmin(admin.ModelAdmin): "capture__name", "capture__uuid", "pipeline_id", - "processing_error", ) readonly_fields = ( "uuid", @@ -85,6 +86,18 @@ class PostProcessedDataAdmin(admin.ModelAdmin): ), ) + @admin.display(description="Owner") + def get_owner(self, obj): + """Get owner through the capture FK.""" + if obj.capture and obj.capture.owner: + return obj.capture.owner.email + return "-" + + @admin.display(boolean=True, description="Error") + def has_error(self, obj): + """Show if processing has an error.""" + return obj.processing_error is not None or obj.cog_error is not None + def get_queryset(self, request): """Optimize queryset with related fields.""" - return super().get_queryset(request).select_related("capture") + return super().get_queryset(request).select_related("capture__owner") diff --git a/sdk/justfile b/sdk/justfile index c351e811b..5aa738d26 100644 --- a/sdk/justfile +++ b/sdk/justfile @@ -4,6 +4,7 @@ python_version := `cat .python-version` supported_python_versions := "3.11 3.12 3.13 3.14" test_marker_default := "not integration" test_marker_integration := "integration" +COLUMNS := "119" sdk_root := justfile_directory() git_root := sdk_root + "/.." @@ -116,7 +117,7 @@ pyrefly *args: test python=python_version *pytest_args: #!/usr/bin/env bash set -euo pipefail - export COLUMNS=88 + export COLUMNS={{ COLUMNS }} echo -e "\n\t\033[34mRunning tests against Python {{ python }}\033[0m\n" # no coverage if -k is used (likely a focused test run) @@ -146,7 +147,7 @@ test python=python_version *pytest_args: test-all *args: #!/usr/bin/env bash set -euo pipefail - COLUMNS=80 + COLUMNS={{ COLUMNS }} echo -e "\n\t\033[34mRunning all local (non-integration) tests\033[0m\n" # run higher bound tests against default python version @@ -171,7 +172,7 @@ test-integration python=python_version workers="auto" *pytest_args: @echo -e "\n\t\033[33mRunning integration tests in parallel\033[0m" @echo -e "\t\033[33m Workers: {{ workers }}\033[0m" @echo -e "\t\033[33m Pass -n 1 or use 'just test-integration-sequential' for sequential execution\033[0m\n" - COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution highest -p '{{ python }}' pytest \ -m '{{ test_marker_integration }}' \ -n {{ workers }} \ --dist loadgroup \ @@ -183,7 +184,7 @@ test-integration python=python_version workers="auto" *pytest_args: test-integration-sequential python=python_version *pytest_args: @echo -e "\n\t\033[33mRunning integration tests sequentially\033[0m" @echo -e "\t\033[33m Use 'just test-integration' for parallel execution\033[0m\n" - COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution highest -p '{{ python }}' pytest \ -m '{{ test_marker_integration }}' \ {{ pytest_args }} \ tests @@ -192,7 +193,7 @@ test-integration-sequential python=python_version *pytest_args: [group('qa')] test-lowest python=python_version *pytest_args: @echo "Running lowest dep resolution tests for Python '{{ python }}'" - COLUMNS=88 uv run --resolution lowest-direct -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution lowest-direct -p '{{ python }}' pytest \ -m '{{ test_marker_default }}' \ {{ pytest_args }} \ tests @@ -200,7 +201,7 @@ test-lowest python=python_version *pytest_args: # runs local tests in verbose mode with stdout capture; useful for debugging test failures [group('qa')] test-verbose python=python_version *pytest_args: - COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution highest -p '{{ python }}' pytest \ -vvv \ -m '{{ test_marker_default }}' \ --show-capture=stdout \ @@ -215,7 +216,7 @@ test-verbose python=python_version *pytest_args: test-integration-verbose python=python_version *pytest_args: @echo -e "\n\t\033[33mRunning integration tests: make sure you set up the web application\033[0m" @echo -e "\t\033[33mand configure 'tests/integration/integration.env'\033[0m\n" - COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution highest -p '{{ python }}' pytest \ -m '{{ test_marker_integration }}' \ -vvv \ --capture=no \ @@ -228,7 +229,7 @@ test-integration-quick python=python_version workers="auto" *pytest_args: @echo -e "\n\t\033[33mRunning quick integration tests in parallel (skipping heavy tests)\033[0m" @echo -e "\t\033[33m Workers: {{ workers }}\033[0m" @echo -e "\t\033[33m Marker: {{ test_marker_integration }} and not heavy\033[0m\n" - COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \ + COLUMNS={{ COLUMNS }} uv run --resolution highest -p '{{ python }}' pytest \ -m '{{ test_marker_integration }} and not heavy' \ -n {{ workers }} \ --dist loadgroup \ diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index be7f45d35..01e753a72 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -257,6 +257,7 @@ # "-o", # "log_cli=true", "--tb=short", + "--durations=10", # "--tb=long", # more verbose # "--capture=no", # more verbose ]