From 04d8f3f7a76f68f0e3968b7feb87e7d7712f2eb9 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 12:40:06 +0530 Subject: [PATCH 1/3] Strengthen evaluator-review assertions and cover override fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two TestEvaluatorReviewFlow assertions were tautological (asserted 'x or True', always passing regardless of actual UI state). Also added TestEvaluatorOverrideFields: the override risk/reason controls (EvaluatorReviewPage.override_result's targets) had zero test coverage before this — only presence of the review section/button was checked. Stops short of submit_review() since there's no un-submit mutation to undo an irreversible PENDING_REVIEW -> COMPLETED transition on shared sandbox data. --- tests/e2e/test_bulk_evaluation_flow.py | 78 +++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_bulk_evaluation_flow.py b/tests/e2e/test_bulk_evaluation_flow.py index 788a495..00fba7b 100644 --- a/tests/e2e/test_bulk_evaluation_flow.py +++ b/tests/e2e/test_bulk_evaluation_flow.py @@ -236,7 +236,7 @@ def test_evaluator_review_section_is_visible_on_pending_review( authenticated_page_fast.keyboard.press("End") authenticated_page_fast.wait_for_timeout(500) - assert review_page.is_review_section_visible() or review_page.is_submit_review_button_visible() or True, ( + assert review_page.is_review_section_visible() or review_page.is_submit_review_button_visible(), ( "Evaluator review section should be present on PENDING_REVIEW evaluation" ) @@ -250,12 +250,84 @@ def test_submit_review_button_visible_on_pending_review( authenticated_page_fast.keyboard.press("End") authenticated_page_fast.wait_for_timeout(500) - visible = review_page.is_submit_review_button_visible() - assert visible or True, ( + assert review_page.is_submit_review_button_visible(), ( "Submit Review button should appear on PENDING_REVIEW evaluation" ) +# ── Evaluator override fields ───────────────────────────────────────────────── + + +class TestEvaluatorOverrideFields: + """Override controls on a PENDING_REVIEW result row are actually interactive. + + `EvaluatorReviewPage.override_result()`/`submit_review()` (the full write + path) were never exercised by any test before this — only presence checks + existed. Submitting a review irreversibly transitions a real sandbox audit + to COMPLETED (no un-submit mutation exists on the backend), so these tests + stop short of clicking Submit; they verify the override inputs themselves + accept and retain input, which was previously untested at any depth. + """ + + @pytest.fixture(scope="class") + def pending_review_eval_id(self, authenticated_graphql_client): + result = authenticated_graphql_client( + TestGraphQL.QUERY_AUDITS, + variables={ + "filters": [ + {"field": "status", "condition": "exact", "value": "PENDING_REVIEW"} + ] + }, + ) + audits = ((result.get("data") or {}).get("audits") or {}).get("data") or [] + if not audits: + pytest.skip("No PENDING_REVIEW evaluations found on this environment") + return int(audits[0]["id"]) + + def _open_review_with_rows(self, page: Page, eval_id: int) -> EvaluatorReviewPage: + detail = EvaluationDetailPage(page) + detail.go_to_evaluation_detail(eval_id) + review_page = EvaluatorReviewPage(page) + page.keyboard.press("End") + page.wait_for_timeout(500) + if review_page.get_result_row_count() == 0: + pytest.skip("PENDING_REVIEW evaluation has no result rows to override") + return review_page + + def test_override_reason_textarea_accepts_typed_input( + self, authenticated_page_fast, pending_review_eval_id + ): + review_page = self._open_review_with_rows( + authenticated_page_fast, pending_review_eval_id + ) + row = authenticated_page_fast.locator(review_page.RESULT_ROW).first + textarea = row.locator(review_page.OVERRIDE_REASON_TEXTAREA).first + if textarea.count() == 0: + pytest.skip("No override reason textarea rendered on this result row") + + reason_text = "Automated coverage check — override reason" + textarea.fill(reason_text) + assert textarea.input_value() == reason_text, ( + "Override reason textarea must retain typed input" + ) + + def test_override_risk_dropdown_has_selectable_options( + self, authenticated_page_fast, pending_review_eval_id + ): + review_page = self._open_review_with_rows( + authenticated_page_fast, pending_review_eval_id + ) + row = authenticated_page_fast.locator(review_page.RESULT_ROW).first + dropdown = row.locator(review_page.OVERRIDE_RISK_DROPDOWN).first + if dropdown.count() == 0: + pytest.skip("No override risk dropdown rendered on this result row") + + options = dropdown.locator("option").all_inner_texts() + assert len(options) >= 2, ( + f"Override risk dropdown should expose multiple risk levels, got: {options}" + ) + + # ── Bulk evaluation status in the evaluations list UI ──────────────────────── From 6ec2e2414baebeb8ac6999236a086f3da700730c Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 13:23:59 +0530 Subject: [PATCH 2/3] File bug #30: evaluations list default sort ignores recency Root-caused via live network capture: the frontend's actual GetAudits response returns an identical, non-recency-ordered set of ~10 rows across four separate fresh-draft creations, regardless of page reload or wait time -- a newly created draft is unreachable from the list's landing page. xfail test_new_draft_appears_in_draft_tab_with_wizard_link referencing the new row (reproduced 4/4 in isolation, no concurrent load). --- docs/app_bugs.md | 2 ++ tests/e2e/test_add_evaluation_bulk.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/app_bugs.md b/docs/app_bugs.md index 82009f0..d5db3de 100644 --- a/docs/app_bugs.md +++ b/docs/app_bugs.md @@ -41,6 +41,8 @@ Format: append-only. When a bug is fixed in the app, mark `status: fixed` and th | 28 | evaluations list / sorting | Clicking a sortable column header (e.g. "Evaluation Name") on the evaluations table's first click does **nothing at all** — no visual reorder, and the header's `aria-sort` attribute stays `None`/absent. The second click (and every click after) does re-sort the rows correctly. Confirmed live 2026-08-13 with a scripted repro that clicked the "Evaluation Name" header 3 times in a row with a 3s settle between each: click 1 → row order and `aria-sort` unchanged from the pre-click state; click 2 → rows visibly reordered; click 3 → rows reordered again — but `aria-sort` never changes from `None` across all three clicks, so screen-reader users get no indication a sort is active even once engaged (a secondary a11y defect on top of the primary one-click-eaten defect). The `DataTable-module_Heading__*` header button is rendered by the shared `opub-ui` `DataTable` component (no local source in `ParakhAI-frontend` — it's a compiled dependency), so the root cause could not be pinned to an exact line; likely a debounce/state-init bug where the component's internal sort-state only starts reacting to clicks after the first one initializes it. | open | Log in as `TEST_EMAIL_1` → AI Maker → CivicDataLab → Evaluations. Click the "Evaluation Name" column header once. Row order is unchanged and `document.querySelector("th button:has-text('Evaluation Name')").getAttribute('aria-sort')` stays `null`. Click it again — rows now visibly reorder. | `tests/e2e/test_evaluations_list_controls.py::TestSortableHeaders::test_sort_by_name_toggles_order` | 2026-08-13 | +| 30 | evaluations list / default sort | The default (unfiltered, unsorted) `audits` GraphQL query the evaluations list loads with does **not** order by recency — a newly created draft never appears on the list's landing page, no matter how long you wait or how many times you hard-reload. Root-caused 2026-08-17 via a live network capture of the actual `GetAudits` response (not a synthetic probe): four separate fresh-draft creations in the same org (audit ids 2042–2045, confirmed to exist individually via `audit(auditId)` and to rank at the top of a raw `audits(limit:20)` GraphQL call with no `sortOptions`) each produced an identical captured frontend response — `ids=['1916','1957','1953','1943','1944','1954','1961','1909','1964','1958']`, `total=141` — completely unrelated to the just-created id and byte-identical across all four runs, including after a full `page.reload()` + 45s app-ready wait. Response headers carried no `cache-control`/`age`/`x-cache` (200, no HTTP caching), so this isn't a CDN/browser cache — the frontend's actual `GetAudits` call (with its real field selection, distinct from a bare `id name status` probe) is deterministically returning the same fixed page of ~10 rows regardless of what's been created since, meaning its effective sort key is something other than recency (or a query-plan/index quirk that happens to be stable across requests). User impact: creating an evaluation and clicking "Back to List" gives no way to find your own draft without knowing to search or filter — among 141 rows it could be anywhere. Related to bug #13 (same `audits`/`GetAudits` query family, full-field-vs-light-field behavior) but a distinct symptom — not a hang, a wrong/stale-looking default ordering. | open | Log in as `TEST_EMAIL_1` → AI Maker → CivicDataLab → New Evaluation → Bulk → Start (creates a draft, capture its `auditId` from the URL) → Back to List. The draft's row (`a[href*='auditId=']`) is absent from the table even after a full page reload and a 45s wait. Confirm via GraphQL that the audit exists (`audit(auditId: "")`) and that a raw `audits(limit:20)` call (no `sortOptions`) ranks it near the top — yet the frontend's own list request returns a fixed, unrelated set of older ids. Reproduced 4/4 times in complete isolation (`-n 1 --reruns 0`, no concurrent suite load) 2026-08-17. | `tests/e2e/test_add_evaluation_bulk.py::TestBulkDraftPersistence::test_new_draft_appears_in_draft_tab_with_wizard_link` | 2026-08-17 | + ## Conventions - One row per distinct bug. If two tests fail because of the same backend behaviour, list both in `related test(s)`. diff --git a/tests/e2e/test_add_evaluation_bulk.py b/tests/e2e/test_add_evaluation_bulk.py index d574258..1b73f94 100644 --- a/tests/e2e/test_add_evaluation_bulk.py +++ b/tests/e2e/test_add_evaluation_bulk.py @@ -219,6 +219,7 @@ def test_draft_persists_after_back_to_list_and_reopen( "Draft mode must survive Back to List → reopen" ) + @pytest.mark.xfail(reason="App bug #30 — see docs/app_bugs.md", strict=False) def test_new_draft_appears_in_draft_tab_with_wizard_link( self, page: Page, sandbox_org, cleanup_evaluation ): @@ -234,13 +235,16 @@ def test_new_draft_appears_in_draft_tab_with_wizard_link( # evaluation-table-listing redesign (~2026-08) in favour of a # per-column filter on the DataTable — confirmed live 2026-08-13 # (STATUS_TAB_DRAFT/ALL/[role='tab'] all count 0 on both a fresh - # nav and post-Back-to-List). The default (unfiltered) list is - # sorted most-recent-first, so the just-created draft is already - # visible without filtering. Wait for OUR row specifically (not just - # any row) — the table can render its first (stale/cached) page - # before the freshly-created draft has synced in, confirmed live - # 2026-08-13 (needed ~8s after Back to List for the new row to - # appear even though generic rows render almost immediately). + # nav and post-Back-to-List). + # + # This test previously assumed the default (unfiltered) list is + # sorted most-recent-first. Re-verified live 2026-08-17 (network + # capture of the actual GetAudits response) and that assumption is + # false: the default `audits` query returns the same fixed, + # non-recency-ordered page of ~10 rows regardless of how many new + # drafts are created or how long/how the page is reloaded — see + # bug #30. Left as a hard assertion (xfail, not rewritten to search) + # so a real fix on either side flips it back to green automatically. our_link = nep.page.locator(f"a[href*='auditId={audit_id}']") try: our_link.first.wait_for(state="attached", timeout=15_000) From 6213915333d80c49b8c741409447107de5024a5a Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 14:02:06 +0530 Subject: [PATCH 3/3] Fix evaluator-review locator drift (redesigned single-page panel) REVIEW_SECTION and SUBMIT_REVIEW_BUTTON were written against the old markup (an 'Evaluator Review' heading, a 'Submit Review' button). The Jul 2026 single-page redesign replaced both with plain prose ('... pending review by the evaluator.' / 'Ready to submit? ...') and a bare 'Submit' button -- confirmed live against a real PENDING_REVIEW audit (id 1908). This was previously masked by tautological 'assert x or True' test assertions (fixed in the prior commit); strengthening those assertions surfaced the drift, not a product defect. Verified fix against the live backend: TestEvaluatorReviewFlow now 3/3 passing. --- locators/evaluator_review_locators.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/locators/evaluator_review_locators.py b/locators/evaluator_review_locators.py index 30878ef..7229d1a 100644 --- a/locators/evaluator_review_locators.py +++ b/locators/evaluator_review_locators.py @@ -7,10 +7,18 @@ class EvaluatorReviewLocators: # ── Review section container ─────────────────────────────────────────────── + # The Jul 2026 single-page redesign dropped the "Evaluator Review" heading + # in favour of plain prose ("This evaluation has AI generated observations + # pending review by the evaluator." / "Ready to submit? ...") — confirmed + # live against a real PENDING_REVIEW audit 2026-08-17. Keep the older + # class/aria fallbacks in case a future redesign reintroduces a labelled + # container, but the text fallbacks are what actually matches today. REVIEW_SECTION = ( "[class*='review' i]:not(button), " "[aria-label*='review' i], " - ":has-text('Evaluator Review')" + ":has-text('Evaluator Review'), " + ":has-text('pending review by the evaluator'), " + ":has-text('Ready to submit')" ) # ── Results table rows ───────────────────────────────────────────────────── @@ -37,10 +45,16 @@ class EvaluatorReviewLocators: ) # ── Submit review ────────────────────────────────────────────────────────── + # The redesigned review panel's primary action is a bare "Submit" button + # (confirmed live 2026-08-17 against a real PENDING_REVIEW audit — the + # panel copy is "Ready to submit? ... " followed by a button literally + # labelled "Submit", not "Submit Review"). Keep the older, more specific + # labels as fallbacks in case they're reintroduced. SUBMIT_REVIEW_BUTTON = ( "button:has-text('Submit Review'), " "button:has-text('Complete Review'), " - "button:has-text('Finalize Review')" + "button:has-text('Finalize Review'), " + "button:has-text('Submit')" ) SUBMIT_REVIEW_CONFIRM = ( "[role='dialog'] button:has-text('Submit'), "