diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9575aba..3d59612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,28 @@ on: branches: [main] workflow_dispatch: +# One run at a time per PR (or per branch/ref for push/workflow_dispatch). +# Added 2026-08-12: rapid pushes to an open PR were each auto-triggering a +# full run, and because this workflow shares one dev backend across every +# run (not just within a run), overlapping runs contended with each other - +# api-tests took 42m26s against a 6m31s baseline while two runs overlapped +# (see docs/ci_workflow_notes.md). cancel-in-progress means a new push +# supersedes the old run's results immediately rather than letting a stale +# commit's run keep consuming the shared backend. +# +# Verified live 2026-08-12 that `if: always()` on a job ignores workflow +# cancellation and lets it start anyway even after the run is marked +# cancelled: api-tests correctly cancelled, but accessibility-tests (then +# `if: always()`) started right after and kept running against the shared +# backend regardless. Removed always() from the actual test jobs +# (accessibility-tests, visual-tests) so they respect cancellation via +# default success()-gating. test-summary still needs always() (it must +# tolerate e2e-tests' deliberate skip while paused) - paired with +# !cancelled() there instead of removing it outright. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: contents: read @@ -33,7 +55,14 @@ env: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} TEST_USER_INDEX: "1" - SANDBOX_ORG_SLUG: ${{ secrets.SANDBOX_ORG_SLUG }} + # Repo Variable, not a Secret (2026-08-14): its value ("1", CivicDataLab's + # org id) isn't sensitive, and GitHub was masking every digit "1" in the job + # summary numbers because it exactly matched this secret's value — moving + # it here stops that without touching the actual secret, which is left in + # place for run-smoke.yml/scheduled.yml (unrelated blast radius: run-smoke + # is the reusable workflow ParakhAI-frontend's CD pipeline calls, not worth + # touching for a cosmetic fix scoped to this file's own job summary). + SANDBOX_ORG_SLUG: ${{ vars.SANDBOX_ORG_SLUG }} jobs: # ────────────────────────────────────────────── Lint @@ -71,6 +100,14 @@ jobs: # so e2e-tests (the heaviest, most concurrency-sensitive suite) gets the # backend to itself. This trades total pipeline wall-clock time for # reliability. + # + # TRIED 2026-08-12: made api-tests + accessibility-tests concurrent (both + # `needs: lint`) on the theory that #13/#14 being independently re-verified + # fixed that same day, plus the backend's runserver -> Docker migration, + # meant the underlying capacity problem was gone. It wasn't (or isn't + # fully) — the concurrent run started failing in CI within ~13 minutes and + # was reverted same-day. Re-attempt only with a real repro of what failed + # (job logs from a completed run, not a cancelled one) in hand first. api-tests: name: API Tests runs-on: ubuntu-latest @@ -98,6 +135,7 @@ jobs: --tb=short \ --html=reports/api_report.html \ --self-contained-html \ + --json-report-file=reports/api.json \ -m api - name: Upload API test report @@ -105,7 +143,9 @@ jobs: uses: actions/upload-artifact@v7 with: name: api-test-report - path: reports/api_report.html + path: | + reports/api_report.html + reports/api.json retention-days: 30 # ────────────────────────────────────────────── E2E tests (3-shard matrix) @@ -117,12 +157,15 @@ jobs: name: E2E Tests (shard ${{ matrix.shard }}/${{ strategy.job-total }}) runs-on: ubuntu-latest needs: visual-tests - # PAUSED 2026-08-11 — remove this line (restoring `if: always()`) to - # resume. test-summary's E2E-report steps are guarded on - # `needs.e2e-tests.result != 'skipped'` and degrade cleanly while this - # is off. + # PAUSED 2026-08-11 — remove this line to resume. test-summary's + # E2E-report steps are guarded on `needs.e2e-tests.result != 'skipped'` + # and degrade cleanly while this is off. Do NOT restore `if: always()` + # (that was the original value) — removed the same pattern from + # accessibility-tests/visual-tests 2026-08-12 after verifying live that + # it ignores workflow cancellation; default success()-gating is correct + # here for the same reason. if: false - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -175,8 +218,15 @@ jobs: name: Accessibility Tests (axe) runs-on: ubuntu-latest needs: api-tests - if: always() - timeout-minutes: 30 + # No if: always() here on purpose (removed 2026-08-12) - it was letting + # this job start even after the run was cancelled (verified live: a + # superseded run's api-tests correctly cancelled, but this job ignored + # that and kept running against the shared backend for up to its + # timeout). Every test step already uses continue-on-error, so + # api-tests' job conclusion is "success" even when its tests fail - + # default gating only actually skips this job on a genuine infra + # failure or cancellation, both cases where skipping is correct. + timeout-minutes: 60 steps: - uses: actions/checkout@v6 @@ -200,6 +250,7 @@ jobs: --tb=short \ --html=reports/a11y_report.html \ --self-contained-html \ + --json-report-file=reports/accessibility.json \ -m accessibility - name: Upload accessibility reports @@ -209,6 +260,7 @@ jobs: name: accessibility-report path: | reports/a11y_report.html + reports/accessibility.json reports/accessibility_report.json reports/accessibility_login_report.json retention-days: 30 @@ -218,8 +270,9 @@ jobs: name: Visual Regression Tests runs-on: ubuntu-latest needs: accessibility-tests - if: always() - timeout-minutes: 30 + # No if: always() here either - see the comment on accessibility-tests' + # if: line above; same reasoning applies. + timeout-minutes: 60 steps: - uses: actions/checkout@v6 @@ -235,14 +288,22 @@ jobs: - name: Install Playwright browsers run: playwright install --with-deps chromium - # Restore cached baselines so we compare against a known good state + # Restore cached baselines so we compare against a known good state. + # + # The `v2-` generation prefix deliberately orphans every cache written + # before 2026-08-12. Those caches hold the poisoned baselines documented in + # docs/visual_diffs.md (a 404, stuck "Loading …" curtains, and four copies + # of the logged-out homepage). Without the bump, editing tests/visual/** + # changes the primary key but `restore-keys` still falls back to the newest + # old cache — so CI would restore the poisoned generation and diff correct + # captures against it. Bump this again if baselines ever need a clean slate. - name: Restore visual baselines cache uses: actions/cache@v5 with: path: snapshots/ - key: visual-baselines-${{ runner.os }}-${{ hashFiles('tests/visual/**') }} + key: visual-baselines-v2-${{ runner.os }}-${{ hashFiles('tests/visual/**') }} restore-keys: | - visual-baselines-${{ runner.os }}- + visual-baselines-v2-${{ runner.os }}- - name: Run visual regression tests continue-on-error: true @@ -252,6 +313,7 @@ jobs: --tb=short \ --html=reports/visual_report.html \ --self-contained-html \ + --json-report-file=reports/visual.json \ -m visual # Save updated baselines back to cache @@ -260,7 +322,8 @@ jobs: uses: actions/cache@v5 with: path: snapshots/ - key: visual-baselines-${{ runner.os }}-${{ hashFiles('tests/visual/**') }} + # Must stay in lockstep with the restore step's generation prefix above. + key: visual-baselines-v2-${{ runner.os }}-${{ hashFiles('tests/visual/**') }} - name: Upload visual regression report and diffs if: always() @@ -269,6 +332,7 @@ jobs: name: visual-regression-report path: | reports/visual_report.html + reports/visual.json screenshots/DIFF_* snapshots/ retention-days: 30 @@ -278,7 +342,14 @@ jobs: name: Test Summary runs-on: ubuntu-latest needs: [api-tests, e2e-tests, accessibility-tests, visual-tests] - if: always() + # Kept always() here (unlike the test jobs above) because this is the + # reporting job, not a test job - it must still run while e2e-tests is + # deliberately paused (if: false -> always "skipped", and default + # success() gating treats a skipped dependency as not-satisfied, which + # would stop this job from running at all). Paired with !cancelled() so + # it still respects a real concurrency-group cancellation rather than + # repeating the bug just fixed on the test jobs. + if: always() && !cancelled() timeout-minutes: 10 steps: - uses: actions/checkout@v6 @@ -292,6 +363,16 @@ jobs: - name: Install report dependencies run: pip install pytest-json-report + # All suite reports, so the summary table can show real test counts + # instead of job status. Each suite's pytest step is `continue-on-error`, + # so its job result is ~always "success" no matter how many tests failed. + - name: Download all suite reports + uses: actions/download-artifact@v4 + with: + path: suite-artifacts/ + merge-multiple: true + continue-on-error: true + - name: Download E2E shard reports if: needs.e2e-tests.result != 'skipped' uses: actions/download-artifact@v4 @@ -314,17 +395,37 @@ jobs: if: needs.e2e-tests.result != 'skipped' run: cat reports/e2e_summary.md >> $GITHUB_STEP_SUMMARY - - name: Print suite status table + # Real per-suite test counts, read from the JSON reports rather than from + # `needs..result`. The old table printed job status, which is + # structurally incapable of showing a test failure here: every suite runs + # its pytest step with `continue-on-error: true`, so the job succeeds even + # when tests fail, and the most-read signal in the run always said + # "success". See scripts/suite_summary.py. + - name: Print suite results table run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Suite | Status |" >> $GITHUB_STEP_SUMMARY - echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| API Tests | ${{ needs.api-tests.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| E2E Tests | ${{ needs.e2e-tests.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Accessibility | ${{ needs.accessibility-tests.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Visual Regression | ${{ needs.visual-tests.result }} |" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Platform:** ${{ env.BASE_URL }}" >> $GITHUB_STEP_SUMMARY + { + echo "" + echo "## Test results by suite" + echo "" + python scripts/suite_summary.py \ + "API=suite-artifacts/api.json" \ + "E2E=shard-artifacts/*/reports/e2e_shard_*.json" \ + "Accessibility=suite-artifacts/accessibility.json" \ + "Visual Regression=suite-artifacts/reports/visual.json" + echo "" + echo "**Platform:** ${{ env.BASE_URL }}" + echo "" + echo "
Job status (plumbing, not test results)" + echo "" + echo "| Job | Result |" + echo "|-----|--------|" + echo "| API Tests | ${{ needs.api-tests.result }} |" + echo "| E2E Tests | ${{ needs.e2e-tests.result }} |" + echo "| Accessibility | ${{ needs.accessibility-tests.result }} |" + echo "| Visual Regression | ${{ needs.visual-tests.result }} |" + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" - name: Upload combined E2E report if: needs.e2e-tests.result != 'skipped' diff --git a/docs/app_bugs.md b/docs/app_bugs.md index 9a28d53..82009f0 100644 --- a/docs/app_bugs.md +++ b/docs/app_bugs.md @@ -19,9 +19,9 @@ Format: append-only. When a bug is fixed in the app, mark `status: fixed` and th | 10 | wizard / cancel | The **Cancel Evaluation** button (`"button:has-text('Cancel Evaluation')"` and its fallback selectors) is not found in the wizard after clicking **Start** from the modal. The wizard Configuration tab renders successfully (`WIZARD_TAB_CONFIGURATION` is visible) but the Cancel Evaluation button never appears within 5 seconds. Previously hidden by a `pytest.skip` guard — now surfaces as a failure after the guard was converted to `assert`. Could be a timing issue (the button renders after a further delay) or the button text / element has changed. | open | Log in → AI Maker → CivicDataLab → New Evaluation → Start → wait for wizard. Inspect DOM for `button` elements containing "Cancel" — the button may be absent or have a different label. | `tests/e2e/test_evaluations.py::TestNewEvaluationWizard::test_cancel_evaluation_returns_to_list` | 2026-05-23 | | 12 | auth / keycloak-dev | **Keycloak dev instance rate-limits ≥5 concurrent logins from the same account.** Under a 5-parallel-login load test, 3/5 workers fail with "login form not rendered" even after a 15 s wait — the form simply never appears. The 3-concurrent test (same account) passes consistently. Root causes (any combination): Keycloak brute-force-protection window, single-node session-table lock contention, or the dev nginx upstream thread pool exhaustion. **Not a bug in the Parakh frontend.** Upgrade path: HA Keycloak with session affinity disabled and per-user rate-limit tuning. | open (infra) | Run `pytest tests/load/test_load.py::TestConcurrentAuthentication -v -s` and observe 3/5 workers timing out on the login form. The 3-concurrent variant in the same class passes. | `tests/load/test_load.py::TestConcurrentAuthentication::test_5_concurrent_logins_success_rate` | 2026-05-23 | | 11 | wizard / manual-workspace | The manual-evaluation workspace Test Cases tab fails to render its expected UI elements when reached via the Domain type flow: (a) module cards appear (counter confirmed) but clicking the first card does not open an input/output panel — neither `MANUAL_INPUT_TEXTAREA` nor `MANUAL_CHANGE_MODULE_LINK` becomes visible; (b) the minimum test-cases hint ("Evaluate at least 3 test cases per module…") is not rendered on the Test Cases tab. These were previously skipped with `pytest.skip` guards; they now surface as failures. | open | Log in → AI Maker → CivicDataLab → New Evaluation → Start → select Domain type → fill Objective → click Test Cases tab. Click any module card and observe whether an input panel appears. Also check whether the min-cases hint text is rendered below the module list. | `tests/e2e/test_evaluation_workspace_manual.py::TestManualWorkspaceTestEntry::test_module_click_opens_input_panel`; `tests/e2e/test_evaluation_workspace_manual.py::TestManualWorkspaceTestEntry::test_min_test_cases_note_visible` | 2026-05-23 | -| 13 | evaluations list / backend perf | The frontend `GetAudits` query that powers the AI-Maker Evaluations list (`/dashboard/ai-maker/{org}/evaluations`) **hangs and never returns** on dev — the page sits on "Loading evaluations…" indefinitely (still up at 90 s). Root-caused 2026-06-12 via live Playwright + direct GraphQL probing on org 1 (981 total audits): requesting the **light** field set (`id name status modelId`, `limit:100`) returns in ~10.6 s, and `metrics`-only or `modules`-only at `limit:100` each return in ~16–18 s, but the **full field set the frontend actually requests** (`name modelName status modules metrics evaluationMode auditType totalTests passedTests failedTests createdAt startedAt completedAt`) **times out past 30 s even at `limit:5`** — so it is the combined per-row resolver cost (likely N+1 on `metrics`/`modules`/test-count fields), not row volume. Backend should batch/precompute these fields or paginate server-side. Also observed intermittent `ERR_NETWORK_CHANGED` / "Backend server is not available" GraphQL errors on dev during the same session (degraded backend). **Not a test defect** — no client-side timeout bump can help since the query never completes. | open | Log in → AI Maker → CivicDataLab → Evaluations. Page stays on "Loading evaluations…". In DevTools, the `GetAudits` POST to `dev.api.parakh.civicdataspace.in/graphql/` has no response after 90 s. Confirmed via Playwright MCP 2026-06-12: full-field `audits` query aborts at 30 s even with `limit:5`, while light-field returns in ~10 s. | `tests/e2e/test_evaluations.py::TestStatusFilterTabs::*` (all 6 — tabs/column/rows render only after the list query resolves); transitively any test that asserts on rendered evaluations-list content for org 1; `tests/e2e/test_evaluations.py::TestEvaluationsListPage::test_table_column_headers_are_present` (added 2026-07-31 — re-run isolated, no concurrent CI load: page still stuck on "Loading evaluations..." past 40s, confirming the hang reproduces without concurrent-shard traffic) | 2026-06-12 | -| 14 | a11y / dashboard heading | The AI Maker Dashboard (`/dashboard/ai-maker/1`) never renders any `

`/`

`/`

` heading — `document.querySelectorAll('h1,h2,h3').length` is `0` even after a 30s wait. Root cause: the "Overview" widget (which appears to be where the page heading lives) is permanently stuck on "Loading overview..." and never resolves — the same failure shape as bug #13 (backend `GetAudits`/overview resolver hang), just surfacing on the dashboard-home overview query instead of the evaluations-list query. Because the widget never mounts, the page has zero heading elements, violating WCAG 1.3.1 (Info and Relationships) / a broken document outline for screen-reader users. | open | Log in as `TEST_EMAIL_1` → navigate directly to `https://dev.parakh.civicdataspace.in/dashboard/ai-maker/1`. Page loads nav/sidebar/breadcrumbs but the main content area shows only "Loading overview..." indefinitely (confirmed stuck past 30s). Run `document.querySelectorAll('h1,h2,h3').length` → `0`. Confirmed via Playwright 2026-07-10. | `tests/accessibility/test_accessibility_auth.py::TestDashboardAccessibility::test_dashboard_has_exactly_one_h1`; `tests/e2e/test_ai_maker_dashboard.py::TestAIMakerDashboardLoads::test_civicdatalab_name_visible_in_sidebar`, `TestModelCardsonHome::test_add_new_model_button_is_visible`, `TestModelCardsonHome::test_model_cards_are_rendered`, `TestModelCardsonHome::test_text_generation_badge_is_present` (added 2026-07-31 — re-run isolated (no concurrent CI load), direct repro script confirmed the whole home content area, not just the heading, is gated behind the same stuck "Loading overview..." widget: org identity block (`CivicdataLab` text), model cards, `Add A New Model` button, and model-type badges never render because they all read from the same overview payload) | 2026-07-10 | -| 15 | routing / auth deep-link | Direct navigation to authenticated AI-Maker / Auditor sub-routes (`/dashboard/ai-maker/1/ai-models`, `/dashboard/ai-maker/1/evaluations/new`, `/dashboard/ai-maker/1/auditors`, `/dashboard/ai-maker/1/prompt-libraries`, `/dashboard/auditor`) **intermittently renders the public marketing homepage** (hero "Build AI that's trustworthy from day one.", feature tabs, CTA cards, footer) instead of the requested dashboard page, even though the user is authenticated (avatar/session valid) and the URL bar shows the correct path. Root-caused 2026-07-10 via a scripted repro: a fresh browser context seeded with a valid `storage_state` (cookies + localStorage from a real Keycloak login) navigating straight to `/dashboard/ai-maker/1/ai-models` with `wait_until="load"` + 2.5s settle reproduced the homepage fallback in **1 of 6 trials (~17%)**; the same repro against `/dashboard/ai-maker/1/prompt-libraries` reproduced it in **1 of 4 trials (~25%)**, and a further capture landed in a visibly half-transitioned state (both "Prompt Libraries" page chrome and homepage CTA/footer content stacked in the same viewport — see `screenshots/DIFF_auth_prompt_libraries_desktop_1440x900.png`). When it does NOT reproduce, the correct page renders fully and correctly (light theme, correct heading, working sidebar, "Loading AI models…" spinner resolving normally). This is a materially different and more severe symptom than the "non-deterministic async content" (live stats, model dropdown ordering) that `_NON_DETERMINISTIC_VISUAL` in `tests/visual/test_visual_regression.py` currently attributes these routes' diffs to — the wizard/models/auditors/auditor-dashboard visual tests see ~43% pixel diffs because they are sporadically capturing the wrong page entirely, not just varying live data. Likely a client-side routing/auth-guard race on hard navigation (the route guard falls through to the default/public route before the session check resolves), since it only reproduces on deep-link/full-page loads, not in-app client-side navigation. | open | Using a Playwright context seeded with a valid Keycloak `storage_state`, repeatedly `page.goto()` directly to `/dashboard/ai-maker/1/ai-models` (or any route in the list above) with `wait_until="load"` and a 2.5s+ settle delay; repeat ~5-6 times. Roughly 1 in 5-6 loads renders the homepage hero/CTA/footer instead of the dashboard page content. Confirmed via ad hoc Playwright script 2026-07-10 (not yet encoded as a standing test — see `docs/visual_diffs.md` Run 2026-07-10 for the diff images that first surfaced it). | `tests/visual/test_visual_regression.py::TestAuthenticatedPageVisuals::test_authenticated_page_desktop[chromium-models_list]`, `[chromium-new_evaluation_wizard]`, `[chromium-auditors_management]`, `[chromium-auditor_dashboard]`, `[chromium-prompt_libraries]` (all xfail under VISUAL-002 — the xfail reason text should be revisited given this finding); `tests/e2e/test_user_flows.py::TestFlow04_EvaluationDetailAndReport::test_evaluation_detail_shows_full_results` (added 2026-07-13 — deep-link to `/evaluations/{id}` rendered the public homepage; avatar/session chrome still present, confirming auth was intact) | 2026-07-10 | +| 13 | evaluations list / backend perf | The frontend `GetAudits` query that powers the AI-Maker Evaluations list (`/dashboard/ai-maker/{org}/evaluations`) **hangs and never returns** on dev — the page sits on "Loading evaluations…" indefinitely (still up at 90 s). Root-caused 2026-06-12 via live Playwright + direct GraphQL probing on org 1 (981 total audits): requesting the **light** field set (`id name status modelId`, `limit:100`) returns in ~10.6 s, and `metrics`-only or `modules`-only at `limit:100` each return in ~16–18 s, but the **full field set the frontend actually requests** (`name modelName status modules metrics evaluationMode auditType totalTests passedTests failedTests createdAt startedAt completedAt`) **times out past 30 s even at `limit:5`** — so it is the combined per-row resolver cost (likely N+1 on `metrics`/`modules`/test-count fields), not row volume. Backend should batch/precompute these fields or paginate server-side. Also observed intermittent `ERR_NETWORK_CHANGED` / "Backend server is not available" GraphQL errors on dev during the same session (degraded backend). **Not a test defect** — no client-side timeout bump can help since the query never completes. | fixed 2026-08-12 (see verification note) | Log in → AI Maker → CivicDataLab → Evaluations. Page stays on "Loading evaluations…". In DevTools, the `GetAudits` POST to `dev.api.parakh.civicdataspace.in/graphql/` has no response after 90 s. Confirmed via Playwright MCP 2026-06-12: full-field `audits` query aborts at 30 s even with `limit:5`, while light-field returns in ~10 s. | `tests/e2e/test_evaluations.py::TestStatusFilterTabs::*` (all 6 — tabs/column/rows render only after the list query resolves); transitively any test that asserts on rendered evaluations-list content for org 1; `tests/e2e/test_evaluations.py::TestEvaluationsListPage::test_table_column_headers_are_present` (added 2026-07-31 — re-run isolated, no concurrent CI load: page still stuck on "Loading evaluations..." past 40s, confirming the hang reproduces without concurrent-shard traffic) | 2026-06-12 | +| 14 | a11y / dashboard heading | The AI Maker Dashboard (`/dashboard/ai-maker/1`) never renders any `

`/`

`/`

` heading — `document.querySelectorAll('h1,h2,h3').length` is `0` even after a 30s wait. Root cause: the "Overview" widget (which appears to be where the page heading lives) is permanently stuck on "Loading overview..." and never resolves — the same failure shape as bug #13 (backend `GetAudits`/overview resolver hang), just surfacing on the dashboard-home overview query instead of the evaluations-list query. Because the widget never mounts, the page has zero heading elements, violating WCAG 1.3.1 (Info and Relationships) / a broken document outline for screen-reader users. **Update 2026-08-13:** Phase 12 (2026-08-12) marked this `fixed` after a clean re-verification, but noted the xfails referencing it hadn't been removed pending a re-run. That re-run surfaced different evidence: in a full e2e run (`--splits 12 --group 2`), `test_add_new_model_button_is_visible` and `test_model_cards_are_rendered` genuinely XFAILED (the bug fired), while `test_civicdatalab_name_visible_in_sidebar` and `test_text_generation_badge_is_present` XPASSED in the same run. Re-running the two XFAILing tests in true isolation (`-n 1 --reruns 0`, no other suite active) immediately afterward, both XPASSED cleanly. This is NOT the bug #3 concurrency-noise shape (isolation didn't consistently clear it either way) — it's a genuinely intermittent defect, not a fixed-vs-broken binary. **Do not remove the `strict=False` xfails on any of the four related tests** — they correctly tolerate a defect that still recurs some fraction of the time. | intermittent (re-opened 2026-08-13 — was marked fixed 2026-08-12, see update above; keep the xfails) | Log in as `TEST_EMAIL_1` → navigate directly to `https://dev.parakh.civicdataspace.in/dashboard/ai-maker/1`. Page loads nav/sidebar/breadcrumbs but the main content area shows only "Loading overview..." indefinitely (confirmed stuck past 30s). Run `document.querySelectorAll('h1,h2,h3').length` → `0`. Confirmed via Playwright 2026-07-10. | `tests/accessibility/test_accessibility_auth.py::TestDashboardAccessibility::test_dashboard_has_exactly_one_h1`; `tests/e2e/test_ai_maker_dashboard.py::TestAIMakerDashboardLoads::test_civicdatalab_name_visible_in_sidebar`, `TestModelCardsonHome::test_add_new_model_button_is_visible`, `TestModelCardsonHome::test_model_cards_are_rendered`, `TestModelCardsonHome::test_text_generation_badge_is_present` (added 2026-07-31 — re-run isolated (no concurrent CI load), direct repro script confirmed the whole home content area, not just the heading, is gated behind the same stuck "Loading overview..." widget: org identity block (`CivicdataLab` text), model cards, `Add A New Model` button, and model-type badges never render because they all read from the same overview payload) | 2026-07-10 | +| 15 | routing / auth deep-link | Direct navigation to authenticated AI-Maker / Auditor sub-routes (`/dashboard/ai-maker/1/ai-models`, `/dashboard/ai-maker/1/evaluations/new`, `/dashboard/ai-maker/1/auditors`, `/dashboard/ai-maker/1/prompt-libraries`, `/dashboard/auditor`) **intermittently renders the public marketing homepage** (hero "Build AI that's trustworthy from day one.", feature tabs, CTA cards, footer) instead of the requested dashboard page, even though the user is authenticated (avatar/session valid) and the URL bar shows the correct path. Root-caused 2026-07-10 via a scripted repro: a fresh browser context seeded with a valid `storage_state` (cookies + localStorage from a real Keycloak login) navigating straight to `/dashboard/ai-maker/1/ai-models` with `wait_until="load"` + 2.5s settle reproduced the homepage fallback in **1 of 6 trials (~17%)**; the same repro against `/dashboard/ai-maker/1/prompt-libraries` reproduced it in **1 of 4 trials (~25%)**, and a further capture landed in a visibly half-transitioned state (both "Prompt Libraries" page chrome and homepage CTA/footer content stacked in the same viewport — see `screenshots/DIFF_auth_prompt_libraries_desktop_1440x900.png`). When it does NOT reproduce, the correct page renders fully and correctly (light theme, correct heading, working sidebar, "Loading AI models…" spinner resolving normally). This is a materially different and more severe symptom than the "non-deterministic async content" (live stats, model dropdown ordering) that `_NON_DETERMINISTIC_VISUAL` in `tests/visual/test_visual_regression.py` currently attributes these routes' diffs to — the wizard/models/auditors/auditor-dashboard visual tests see ~43% pixel diffs because they are sporadically capturing the wrong page entirely, not just varying live data. Likely a client-side routing/auth-guard race on hard navigation (the route guard falls through to the default/public route before the session check resolves), since it only reproduces on deep-link/full-page loads, not in-app client-side navigation. | open | Using a Playwright context seeded with a valid Keycloak `storage_state`, repeatedly `page.goto()` directly to `/dashboard/ai-maker/1/ai-models` (or any route in the list above) with `wait_until="load"` and a 2.5s+ settle delay; repeat ~5-6 times. Roughly 1 in 5-6 loads renders the homepage hero/CTA/footer instead of the dashboard page content. Confirmed via ad hoc Playwright script 2026-07-10 (not yet encoded as a standing test — see `docs/visual_diffs.md` Run 2026-07-10 for the diff images that first surfaced it). | `tests/visual/test_visual_regression.py::TestAuthenticatedPageVisuals::test_authenticated_page_desktop[chromium-models_list]`, `[chromium-new_evaluation_wizard]`, `[chromium-auditors_management]`, `[chromium-auditor_dashboard]`, `[chromium-prompt_libraries]` (all xfail under VISUAL-002 — the xfail reason text should be revisited given this finding); `tests/e2e/test_user_flows.py::TestFlow04_EvaluationDetailAndReport::test_evaluation_detail_shows_full_results` (added 2026-07-13 — deep-link to `/evaluations/{id}` rendered the public homepage; avatar/session chrome still present, confirming auth was intact); `tests/visual/test_visual_regression.py::TestAuthenticatedPageVisuals::*` (2026-08-12 — the new capture-integrity gate in `utils/visual_guards.py` now detects this automatically and skips instead of baselining it) **— SEVERITY UPDATE 2026-08-12: observed at ~50%, far above the 17-25% recorded above.** Two consecutive full visual runs hit the logged-out fallback on **12 of 24 authenticated captures** (5/12 then 7/12). Affected: `/dashboard/ai-maker/1`, `/ai-models`, `/evaluations`, `/evaluations/new`, `/auditors`, `/prompt-libraries`, `/dashboard/auditor`. NOT affected in either run: `/dashboard` (role selector), `/dashboard/ai-maker` (org selector), `/dashboard/auditor/evaluations` — so it is not uniform across depth. New detail vs the 2026-07-10 characterisation: the fallback now renders **logged-out chrome (LOGIN / SIGN UP)**, i.e. the session is not merely failing to apply to the route — the shell itself renders unauthenticated. Caveat on the comparison: both figures are small samples (the original was 1/6 and 1/4 trials) and load conditions differed, so treat this as "materially worse under current conditions" rather than a precise regression measurement. Independent confirmation that it also fired historically: four baselines were byte-identical copies of the logged-out homepage. `test_evaluation_detail_shows_full_results`'s xfail was missing `strict=False` (this repo's `pytest.ini` sets `xfail_strict = true` globally) — added 2026-08-13 so a lucky run's XPASS is tolerated instead of failing the suite, consistent with bug #14's established convention for intermittent (not always-reproducing) bugs. | 2026-07-10 | | 16 | SEO / homepage content | Homepage `` contains a spelling error: `"Paricipatory AI Evaluation"` (missing the first `t` in "Participatory"). Root-caused in frontend source: `app/layout.tsx` line 180, `export const metadata = { ..., description: "Paricipatory AI Evaluation" }`. One-line content fix. | open | Load `https://dev.parakh.civicdataspace.in/`. Run `document.querySelector('meta[name="description"]').content` → `"Paricipatory AI Evaluation"`. Confirmed against `ParakhAI-frontend/app/layout.tsx:180` 2026-07-13. | `tests/e2e/test_platform_issues.py::TestHomepageSEO::test_meta_description_has_no_typo` | 2026-07-13 | | 17 | SEO / homepage content | Homepage has no Open Graph `` tag (or any other `og:*` tags) — the root `metadata` export in `app/layout.tsx` only sets `title` and `description`, no `openGraph` block. Links to the homepage shared on social media / chat apps will render with no title card. | open | Load `https://dev.parakh.civicdataspace.in/`. Run `document.querySelector('meta[property="og:title"]')` → `null`. Confirmed against `ParakhAI-frontend/app/layout.tsx` (metadata export has no `openGraph` key) 2026-07-13. | `tests/e2e/test_platform_issues.py::TestHomepageSEO::test_og_title_is_present` | 2026-07-13 | | 18 | routing / 404 | Visiting any non-existent URL as an anonymous user does not show a 404/not-found message — it silently redirects to the generic NextAuth "sign in with Keycloak" page instead, with no indication the URL was invalid. Root cause confirmed in `middleware.ts`: `publicPages = ['/']` is the *only* unauthenticated route; the matcher (`'/((?!_next|api|.*\\..*).*)'`) sends every other path — including typos and genuinely nonexistent routes — through `withAuth`, which redirects unauthenticated requests to `/api/auth/signin` before Next.js ever gets a chance to render a not-found page. This is a distinct, more severe variant of the two existing sibling xfails in this file (`test_404_page_has_home_navigation_link` / `test_404_page_has_logo_or_branding`), which assumed a bare-but-present Next.js 404 page — there isn't one for anonymous users at all. | open | As an anonymous user, navigate to `https://dev.parakh.civicdataspace.in/this-page-does-not-exist-xyz-123`. Observe the generic Keycloak/NextAuth sign-in screen, not a 404 page. Confirmed against `ParakhAI-frontend/middleware.ts` (`publicPages = ['/']`) 2026-07-13. | `tests/e2e/test_platform_issues.py::TestNotFoundPage::test_404_page_shows_not_found_message` | 2026-07-13 | @@ -29,9 +29,18 @@ Format: append-only. When a bug is fixed in the app, mark `status: fixed` and th | 21 | ai-maker / breadcrumb | The deepest breadcrumb crumb on every `/dashboard/ai-maker/{orgId}/*` sub-page (AI Models, Evaluations, Auditors, etc.) shows the organization name or a static **"Dashboard"** fallback — never the actual section name ("Models", "Evaluations", etc.). Root cause confirmed in `app/[locale]/dashboard/ai-maker/[orgId]/layout.tsx:30`: the shared layout's `BreadCrumbs` config hardcodes the last crumb as `label: organization?.name \|\| "Dashboard"`, computed once at the org-layout level — sub-route pages (`ai-models/page.tsx`, etc.) never override or extend this with their own section label. Live repro on `/dashboard/ai-maker/1/ai-models`: last breadcrumb `
  • ` renders literally as `DashboardDashboard` (duplicated because the layout renders both a desktop and a mobile-truncated span for the same crumb). This was already known — flagged in a 2026-06-22 MCP exploration per this test file's own docstring — but never given a ledger row, so the `assert` was a bare failing assertion rather than a proper `xfail`. | open | Log in as `TEST_EMAIL_1` → navigate to `https://dev.parakh.civicdataspace.in/dashboard/ai-maker/1/ai-models`. Run `document.querySelector("[aria-label='breadcrumb'] li:last-child").textContent` → `"Dashboard"` (or the org name), never `"Models"`. Confirmed against `ParakhAI-frontend/app/[locale]/dashboard/ai-maker/[orgId]/layout.tsx:30` 2026-07-31. Two sibling assertions in the same test file (`test_evaluations_breadcrumb_is_not_dashboard`, `test_evaluation_detail_breadcrumb_has_org_name`) pass reliably in isolation — only the AI Models sub-page repros consistently. | `tests/e2e/test_breadcrumb.py::TestBreadcrumbLabels::test_models_breadcrumb_is_not_dashboard` | 2026-06-22 | | 20 | wizard / start-evaluation gating | The **Start Evaluation** button in the "Start an Evaluation" modal (step 2) intermittently never enables despite a fully valid form (evaluator type selected, non-empty objective typed via real keystrokes). First observed live 2026-07-02: identical keystroke input enabled the button in one session and not the next — suspected server-side validation state race, not a client-rendering issue (retyping/re-selecting fields doesn't reliably recover it within a session). This was previously only documented as an inline `xfail` comment in `test_add_evaluation_modal.py` without a numbered ledger row, which violates this file's own "every xfail must reference a row by id" convention — formalized here retroactively since the same defect surfaced independently in `test_evaluations.py` on 2026-07-28. | open | Open "Start an Evaluation" modal → step 2 → select an evaluator type → type an objective via real keystrokes (not `.fill()`) → wait up to 5s. The Start Evaluation button should enable but intermittently stays disabled. No client-side workaround found; re-selecting the evaluator type or retyping the objective does not reliably unstick it. | `tests/e2e/test_add_evaluation_modal.py::TestStartEvaluationValidation::test_start_enables_with_objective_and_disables_when_cleared`; `tests/e2e/test_evaluations.py::TestNewEvaluationModalStep2::test_start_requires_objective_filled` | 2026-07-02 | +| 25 | evaluations list / responsive | Evaluations table overflows horizontally at tablet width (768px) — the Model column is cut off at the right edge and the "Rows: 10" pagination control is clipped at the left edge. Found by new responsive visual coverage capturing `/dashboard/ai-maker/1/evaluations` at 768×1024. | open | Log in → AI Maker → Evaluations, at 768px viewport width. Table extends past the visible content area on both sides; no horizontal scroll affordance visible. | `tests/visual/test_visual_responsive.py::TestResponsiveAuthenticatedVisuals::test_authenticated_page_responsive[chromium-evaluations_list-tablet]` (baseline `auth_evaluations_list_tablet_768x1024.png` shows the clipped state) | 2026-08-12 | +| 24 | auditor / assignments hang | `/dashboard/auditor/assignments` hangs indefinitely on "Loading your assignments…" — still curtained past a 45s budget. Found 2026-08-12 by the visual suite's new capture-integrity gate (`utils/visual_guards.py`), which refused to save a baseline of the stuck state. Same family as #13/#14 (a route-level data query that never resolves) but on a route those rows never covered, and notably this one did **not** clear within 45s in the same session where the AI-Maker overview and evaluations list both loaded fully — so it is not merely slow. | open | Log in as `TEST_EMAIL_1` → switch to Evaluator role → navigate to `https://dev.parakh.civicdataspace.in/dashboard/auditor/assignments`. Page renders sidebar/chrome but the content area stays on "Loading your assignments…" past 45s. | `tests/visual/test_visual_regression.py::TestAuthenticatedPageVisuals::test_authenticated_page_desktop[chromium-auditor_assignments]` (skips via the integrity gate rather than baselining the stuck state) | 2026-08-12 | | 23 | auth / backend | ParakhAPI's `DataSpaceAuthMiddleware` (`graphql_api/middleware/dataspace_user.py::process_request`) wraps its entire Bearer-token validation path (DataSpace `/api/auth/user/info/` lookup) in a blanket `except Exception`, and on **any** failure — expired token, invalid token, DataSpace being briefly unreachable — silently sets `request.user = AnonymousUser()` and continues. The failure is only `logger.warning`'d server-side; the client gets a normal `200` response shaped exactly like a legitimate anonymous request (e.g. `audits` returns `{data: [], totalItemsCount: 0}`, no `errors` array, nothing containing "token"/"expired"/401/403). A client has no way to distinguish "you are genuinely unauthenticated" from "your session just died" — it can only infer this indirectly from previously-authenticated-looking context. Root-caused 2026-08-11 while investigating 3 seemingly-flaky `ParakhAI_test` security tests that passed in isolation but failed only when reached ~4-5 minutes into a long suite run; confirmed by reading the middleware source directly. This is very likely the true root cause behind this file's own previously-vague "session expired mid-suite" attributions in the Phase 10 log entries (bug #3/#12 family) — those blamed Keycloak/dev-env instability in general, but the precise mechanism is this middleware's silent-swallow, which a well-behaved API should instead surface as a 401. | open | With a valid Bearer token, wait past the token's TTL (~5 min on dev), then issue any authenticated GraphQL query (e.g. `{ audits { data { id } totalItemsCount } }`) with the now-stale token. Response is `200` with the same shape as an anonymous request, no error. Confirmed against `ParakhAPI` `graphql_api/middleware/dataspace_user.py:99-104` (dev branch) 2026-08-11. | Indirectly, any test using `tests/conftest.py::authenticated_graphql_client` on a long-running suite — see the fixture's own docstring (fixed 2026-08-11 to proactively refresh before this can bite) for the test-side mitigation; this row documents the underlying product-side gap that mitigation works around. | 2026-08-11 | | 22 | audits query / backend perf | Sorting the `audits` GraphQL query by `passed_tests` is measurably slower than the default `created_at` ordering under concurrent load. 5 concurrent authenticated `audits(sortOptions: [{field: "passed_tests", direction: "desc"}])` requests (light field selection: `id status passedTests` only, `limit: 20`) took 9-15s each on the cleanest run and up to 35s under noisier conditions, vs. a 6s budget for the equivalent unsorted 5-concurrent read (`TestConcurrentGraphQLReads`). 100% success, zero errors — not a correctness bug, but the delta suggests the `(organization, -created_at)` index added for this query (commit `4f51ab2`) doesn't help an `ORDER BY passed_tests`, which has no index of its own. Worth a backend follow-up (composite/partial index, or precomputing `passed_tests` ordering) if sort-by-pass-rate becomes a real UI feature rather than just an API capability. | open | Run `pytest tests/load/test_load_graphql.py -m load -v -k TestAuditsSortPerformance` — compare reported latencies in `reports/load_metrics_graphql.json` (`concurrent_audits_sorted_n5`) against the unsorted baseline in the same file. | `tests/load/test_load_graphql.py::TestAuditsSortPerformance::test_5_concurrent_audits_sorted_by_passed_tests` | 2026-08-11 | +| 29 | a11y / new-evaluation dialog | The New Evaluation dialog's `aria-describedby` attribute intermittently references an element id that doesn't exist in the DOM, breaking the screen-reader description link (WCAG 1.3.1). Previously only an inline `xfail` in `test_accessibility_auth.py` with no ledger row, violating this file's own "every xfail must reference a row by id" convention — formalized here. Confirmed intermittent, not fixed: XPASSed in the 2026-08-14 CI run (`31779734378`) but the underlying check (`aria-describedby` value must resolve to a real DOM id, or be absent entirely) is a static markup issue, not an obviously timing-driven one, so the XPASS may reflect a genuine but inconsistent fix rather than pure luck — needs a few more observations before either closing or root-causing in the frontend. `strict=False` applied so it doesn't flap the suite red either direction. | intermittent | Log in → AI Maker → CivicDataLab → Evaluations → click "New Evaluation". Read the opened dialog's `aria-describedby` attribute; check whether an element with that id exists in the DOM. | `tests/accessibility/test_accessibility_auth.py::TestDialogAccessibility::test_ux010_new_evaluation_dialog_aria_describedby_resolves` | 2026-08-14 | +| 26 | wizard / evaluator-type label | The wizard's Evaluation Overview card renders the "Evaluator" field as the literal string `Evaluator` instead of the selected evaluator type ("Technical"/"Domain"/"Cultural"), for all three evaluator-type selections. Root-caused in frontend source (`ParakhAI-frontend`, `dev` branch @ `a1c5b84` — note the local checkout used for earlier bug rows was several commits behind live `origin/dev` at investigation time; always diff against `origin/dev`, not a stale local clone, when re-verifying this row): `getEvaluatorLabel()` in `app/[locale]/dashboard/ai-maker/[orgId]/evaluations/components/NewEvaluationContent.tsx:1958-1969` switches on `auditType` (a raw backend enum, e.g. `AUDIT_TYPE.TECHNICAL_AUDIT` = `'TECHNICAL_AUDIT'`) but its `case` labels compare against `AUDIT_TYPE_LABELS.TECHNICAL_AUDIT` (the *display* label, `'Technical'`) — an enum-vs-label comparison that can never match, so every selection falls through to `default: return 'Evaluator'`. Introduced in commit `34f221c` (2026-08-03, "refactor: update audit type handling in evaluation components"), which correctly migrated the `auditType` state to hold the raw enum but missed updating this one downstream consumer to compare against `AUDIT_TYPE.*` instead of `AUDIT_TYPE_LABELS.*`. Same-shaped drift also broke this repo's Playwright locator for the step-2 radio inputs (`input[name='evaluatorType'][value='Technical']` → DOM now emits `[value='TECHNICAL_AUDIT']`) — fixed test-side in this session (`locators/evaluations_locators.py`, `pages/new_evaluation_page.py::get_checked_evaluator_type`), since that half was a pure test-locator drift, not a product defect. | open | Log in as `TEST_EMAIL_1` → AI Maker → CivicDataLab → New Evaluation → select any model → Bulk method → Next → leave evaluator type on the default "a technical evaluator" (or pick Domain/Cultural) → fill Objective → Start Evaluation. On the wizard's Evaluation Overview card, the "Evaluator" row reads `Evaluator : Evaluator` regardless of which type was selected (Scope/Mode/Objective all render correctly in the same card). Confirmed via a live scripted repro 2026-08-13 (draft audit id 1911, since cancelled via `scripts/cleanup_drafts.py`) — full page-body dump showed `Evaluator : Evaluator`. | `tests/e2e/test_add_evaluation_bulk.py::TestBulkDraftCreation::test_overview_reflects_modal_inputs`; `tests/e2e/test_add_evaluation_bulk.py::TestEvaluatorTypeVariants::test_evaluator_type_reflected_in_overview[domain-Domain]`, `[cultural-Cultural]`; `tests/e2e/test_new_evaluation_regression.py::TestEvaluationTypeRadio::test_changing_eval_type_updates_name_label` (added 2026-08-13 — same defect, reached via the `authenticated_page`/modal flow instead of `_create_bulk_draft`) | 2026-08-13 | + +| 27 | evaluations list / filtering | The evaluations list's "Filter Status" column-filter popover (part of the ~2026-08 evaluation-table-listing redesign, which replaced the old StatusFilterTabs bar) **intermittently** fails to actually filter the table: the `Status: