Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -1941,8 +1941,8 @@ analytics-monthly-update: ## Stage verified archives into all monthly dashboard
python backend/scripts/update_monthly_analytics.py $(ARCHIVES) --supplemental "$(SUPPLEMENTAL)" $(if $(THROUGH),--through $(THROUGH),)

analytics-monthly-check: ## Validate published analytics periods, rankings, and existing dashboard layouts
cd backend && python -m pytest --confcutdir=tests/scripts tests/scripts/test_update_monthly_analytics.py tests/scripts/test_export_analytics_snapshots.py tests/scripts/test_analytics_evidence.py -q
cd frontend && npx vitest run src/__tests__/components/AnalyticsPeriods.test.ts src/__tests__/components/FutureAnalyticsMonth.test.tsx src/__tests__/components/TopSearchesData.test.ts src/__tests__/components/MonthlyComparison.test.tsx src/__tests__/pages/SeptemberAnalyticsPage.test.tsx src/__tests__/components/DiscoveryOutcomes.test.tsx
cd backend && python -m pytest --confcutdir=tests/scripts tests/scripts/test_update_monthly_analytics.py tests/scripts/test_export_analytics_snapshots.py tests/scripts/test_analytics_evidence.py tests/scripts/test_public_analytics_export.py -q
cd frontend && npx vitest run src/__tests__/components/AnalyticsPeriods.test.ts src/__tests__/components/FutureAnalyticsMonth.test.tsx src/__tests__/components/TopSearchesData.test.ts src/__tests__/components/MonthlyComparison.test.tsx src/__tests__/pages/SeptemberAnalyticsPage.test.tsx src/__tests__/components/DiscoveryOutcomes.test.tsx src/__tests__/components/AnalyticsRoutes.test.ts src/__tests__/components/AugustAnalyticsData.test.ts src/__tests__/pages/AnalyticsPage.test.tsx

.PHONY: analytics-evidence analytics-evidence-check
analytics-evidence: ## Generate screened evidence for existing analytics snapshots (optional ARCHIVES)
Expand Down
31 changes: 31 additions & 0 deletions backend/scripts/export_analytics_design_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@
from app.services.analytics_reporting.storage import checksum # noqa: E402

COLORS = ["#003C5B", "#1E40AF", "#2563EB", "#3B82F6", "#60A5FA", "#93C5FD", "#BFDBFE"]
ENDPOINT_METHOD = (
"July and August retain their saved route coverage and unlisted remainder. "
"Later months use preserved request aggregates. Each period shows up to 100 leading "
"request paths; remaining requests and server errors are combined under "
"Other / unlisted routes. All time is ranked before this limit is applied, "
"not by adding the displayed monthly rankings. Browse 15 rows per page."
)


def counts(row):
Expand Down Expand Up @@ -123,6 +130,26 @@ def endpoint_rows(rows):
]


def compact_endpoint_rows(rows, limit=100):
"""Bound dashboard payload size without discarding request or error totals."""
other_label = "Other / unlisted routes"
ranked = sorted(
(row for row in rows if row["label"] != other_label),
key=lambda row: (-row["requests"], row["label"]),
)
remainder = ranked[limit:] + [row for row in rows if row["label"] == other_label]
result = ranked[:limit]
if remainder:
result.append(
dict(
label=other_label,
requests=sum(row["requests"] for row in remainder),
errors=sum(row["errors"] for row in remainder),
)
)
return result


def outcomes(report, details):
def ranking(key, metric):
return [
Expand Down Expand Up @@ -741,6 +768,10 @@ def query_rows(report, key):
endpoints["periods"]["all"] = [
dict(label=k, **v) for k, v in sorted(totals.items(), key=lambda x: -x[1]["requests"])
]
endpoints["periods"] = {
key: compact_endpoint_rows(rows) for key, rows in endpoints["periods"].items()
}
endpoints["method"] = ENDPOINT_METHOD
historical_outcomes = json.loads((output / "outcomes2026.json").read_text())["periods"]
for historical_period in ("2026-07", "2026-08"):
for ranking in ("outlinks", "downloads"):
Expand Down
23 changes: 23 additions & 0 deletions backend/tests/scripts/test_export_analytics_snapshots.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,32 @@
import csv
import io

from scripts.export_analytics_design_data import compact_endpoint_rows
from scripts.export_analytics_snapshots import RANKINGS, preview, table_csv


def test_endpoint_display_limit_preserves_tail_errors_and_existing_remainder():
rows = [
{"label": f"/resource/{i:03}", "requests": 10, "errors": 1} for i in reversed(range(105))
]
rows.append({"label": "Other / unlisted routes", "requests": 7, "errors": 2})
compact = compact_endpoint_rows(rows)
assert len(compact) == 101
assert compact[0]["label"] == "/resource/000"
assert compact[99]["label"] == "/resource/099"
assert compact[-1] == {"label": "Other / unlisted routes", "requests": 57, "errors": 7}
assert sum(row["requests"] for row in compact) == 1057
assert sum(row["errors"] for row in compact) == 107
assert compact_endpoint_rows(compact) == compact
assert len(rows) == 106


def test_small_endpoint_list_has_no_invented_remainder():
rows = [{"label": "/api/docs", "requests": 12, "errors": 0}]
assert compact_endpoint_rows(rows) == rows
assert compact_endpoint_rows([]) == []


def test_preview_limits_rankings_but_keeps_distinct_member_leaders():
report = {key: [{"id": str(n)} for n in range(150)] for key in RANKINGS}
report.update(
Expand Down
67 changes: 54 additions & 13 deletions docs/frontend/monthly-analytics.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,68 @@ Report links preserve selection in the URL and support reloads and browser histo

| URL | Report | Period |
| ----------------------------------------- | ---------------------------------------------------- | ---------------- |
| `/analytics` | Overview, daily activity, and traffic peaks | September 2026 |
| `/analytics/comparison` | Monthly totals, chart, CSV/JSON downloads | July–September 2026 |
| `/analytics/content` | Popular resources, collections, downloads | September 2026 |
| `/analytics` | Overview, daily activity, and traffic peaks | October 2026 |
| `/analytics/comparison` | Monthly totals, chart, CSV/JSON downloads | July–October 2026 |
| `/analytics/content` | Popular resources, collections, downloads | October 2026 |
| `/analytics/content?month=2026-07` | Preserved popular-content report | July 2026 |
| `/analytics/members` | Alliance and campus detail | September 2026 |
| `/analytics/activity` | Legacy alias for Overview | September 2026 |
| `/analytics/discovery` | Search terms, filters, zero results | September 2026 |
| `/analytics/clients` | Clients, channels, key attribution, MCP/QGIS signals | September 2026 |
| `/analytics/platform` | API traffic and reliability | September 2026 |
| `/analytics/members` | Alliance and campus detail | October 2026 |
| `/analytics/activity` | Legacy alias for Overview | October 2026 |
| `/analytics/discovery` | Search terms, filters, zero results | October 2026 |
| `/analytics/clients` | Clients, channels, key attribution, MCP/QGIS signals | October 2026 |
| `/analytics/platform` | API traffic and reliability | October 2026 |

Unknown report paths return 404. Legacy `?report=...` links redirect to report paths,
preserving the month and other query parameters. `/analytics` is Overview;
`/analytics/overview` and `/analytics/activity` redirect there. Navigation marks the current report and
wraps on small screens; a skip link leads to the report. Single-month reports
default to September. Use `month=2026-07`, `month=2026-08`, or `month=2026-09`
to select a month, or `month=all` for July–September. Navigation preserves the
default to October. Use `month=2026-07`, `month=2026-08`, `month=2026-09`, or `month=2026-10`
to select a month, or `month=all` for July–October. Navigation preserves the
selected period. Unpublished periods show an unavailable message. The comparison
report shows all three months.
report shows all four months.

The page uses checked-in aggregates, not live database queries. September and
The page uses checked-in aggregates, not live database queries. October and
refreshed cumulative reports include full CSV/JSON table downloads in ZIP files.
See the [September update](#september-2026-update) for their source coverage and
See the [October update](#october-2026-update) for their source coverage and
local regeneration workflow. The sections below describe the original July and
August snapshot definitions and presentations, which remain available.

## October 2026 update

The November 3 export covers October 1 inclusive through November 1 exclusive
(UTC), with
records on all 31 days in each of the four analytics source tables. Source counts
are 3,208,142 API requests, 7,973 searches, 130,889 search appearances, and 15,191
interaction events, including 10,577 resource views, 1,049 source-site clicks,
and 551 download clicks. The supplemental aggregate records 6,390 distinct non-null
event visit tokens and 2,477 searches with non-empty query text. Catalog metadata
was captured November 3, not at October month-end. Visits describe
recorded tokens, not unique people.

The month comparison table marks October’s API request count with an asterisk.
Its footnote explains that Geoportal server-side requests began being recorded on
October 14, so most of the increase reflects expanded logging coverage rather
than equivalent growth in usage. Earlier monthly request totals are not directly
comparable.

October and July–October cumulative reports use the existing monthly refresh
workflow with `THROUGH=2026-11-01`. Preserve the July–September archive revisions;
rebuild cumulative rankings from complete inputs, rather than adding monthly
rankings. Historical July coverage gaps remain disclosed in All time. The Summary
page defaults to October because the four-month cumulative report no longer
matches the latest three calendar months.

API endpoint tables retain up to 100 leading request paths per period. Additional
paths are combined under “Other / unlisted routes,” preserving every request and
server error in the totals. Cumulative leaders are selected from the full combined
counts before limiting the table, not from the displayed monthly top lists. This
bounds dashboard payload size when automated traffic requests many distinct paths.
Complete inputs remain in the private archives.

Private source archives and supplemental counts stay outside tracked files.
Source extraction and preservation procedures belong in restricted operations
documentation. Counts reconciled to source tables establish reproducibility;
they do not prove that every real-world interaction was recorded.

## Summary widget styling

Use `analytics-metric-card` for summary widgets on every dashboard report.
Expand Down Expand Up @@ -601,6 +638,10 @@ make analytics-monthly-check
cd frontend && npm run build
```

The monthly check includes download privacy and stale-revision checks, route
metadata, and dashboard navigation across published months and All time. Remove
superseded report directories so only manifest-listed revisions ship.

The default boundary is the first day of the current UTC month, so only completed
months are included. For a reproducible rerun, add `THROUGH=2026-10-01` to regenerate
through September. Supply exactly one reconciled, checksummed archive for every
Expand Down
Loading
Loading