From d71416c5716d8b47271aa5eea7a6afc791daaae0 Mon Sep 17 00:00:00 2001 From: emmaoke-w Date: Wed, 12 Aug 2026 20:20:17 +0200 Subject: [PATCH 1/2] ci: sanitize Android UI workflow logs and remove column flavors (WPB-27951) --- .../qa-android-critical-flow-tests.yml | 10 ++- .../qa-android-ui-test-manual-deflake.yml | 13 ++- scripts/qa/android-ui/resolve_flavor.py | 12 ++- scripts/qa/android-ui/run_ui_tests.sh | 37 ++++++++- .../sanitize_instrumentation_log.py | 82 +++++++++++++++++++ 5 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 scripts/qa/android-ui/sanitize_instrumentation_log.py diff --git a/.github/workflows/qa-android-critical-flow-tests.yml b/.github/workflows/qa-android-critical-flow-tests.yml index 674889f073b..e6509180efd 100644 --- a/.github/workflows/qa-android-critical-flow-tests.yml +++ b/.github/workflows/qa-android-critical-flow-tests.yml @@ -243,8 +243,6 @@ jobs: contents: write env: - AWS_REGION: eu-west-1 - OP_VAULT: "Test Automation" FLAVORS_CONFIG_PATH: "/etc/android-qa/flavors.json" DEVICE_GROUPS_JSON: ${{ vars.ANDROID_DEVICE_GROUPS_JSON }} @@ -253,6 +251,11 @@ jobs: shell: bash steps: + - name: Register log masks + run: | + echo "::add-mask::eu-west-1" + echo "::add-mask::mation" + - name: Checkout (with submodules) uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -276,7 +279,7 @@ jobs: run: make qa-ui-setup cmd=ensure-required-tools # Flavor resolution is runner-driven (from /etc/android-qa/flavors.json), not hardcoded in repo. - # This bash subcommand exports S3_FOLDER, APP_ID, and PACKAGES_TO_UNINSTALL for downstream steps. + # Keep app install data job-wide, but pass the S3 folder only to the download step. - name: Resolve flavor (runner config) id: resolve_flavor env: @@ -298,6 +301,7 @@ jobs: IS_UPGRADE: ${{ needs.validate-and-resolve-inputs.outputs.resolvedIsUpgrade }} OLD_BUILD_NUMBER: ${{ needs.validate-and-resolve-inputs.outputs.resolvedOldBuildNumber }} S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + S3_FOLDER: ${{ steps.resolve_flavor.outputs.s3Folder }} run: make qa-ui-setup cmd=download-apks # Select device(s): use input device when provided, otherwise auto-pick. diff --git a/.github/workflows/qa-android-ui-test-manual-deflake.yml b/.github/workflows/qa-android-ui-test-manual-deflake.yml index 08ebdbcba36..62032a7080b 100644 --- a/.github/workflows/qa-android-ui-test-manual-deflake.yml +++ b/.github/workflows/qa-android-ui-test-manual-deflake.yml @@ -85,8 +85,6 @@ jobs: actions: read env: - AWS_REGION: eu-west-1 - OP_VAULT: "Test Automation" FLAVORS_CONFIG_PATH: "/etc/android-qa/flavors.json" DEVICE_GROUPS_JSON: ${{ vars.ANDROID_DEVICE_GROUPS_JSON }} @@ -95,6 +93,11 @@ jobs: shell: bash steps: + - name: Register log masks + run: | + echo "::add-mask::eu-west-1" + echo "::add-mask::mation" + # Mirror qa-android-critical-flow-tests.yml runner setup so manual # deflake executes in the same QA office runner environment. - name: Checkout (with submodules) @@ -181,9 +184,10 @@ jobs: RERUN_FAILED_COUNT: ${{ steps.validate_deflake_input.outputs.rerunFailedCount }} run: make qa-ui-validate cmd=validate-rerun-inputs - # Reuse the same flavor resolver and APK download path so the deflake run - # stays aligned with qa-android-critical-flow-tests.yml app setup. + # Reuse the same flavor resolver and pass its S3 folder output into the APK + # download step so deflake stays aligned with qa-android-critical-flow-tests.yml. - name: Resolve flavor (runner config) + id: resolve_flavor env: FLAVOR_INPUT: ${{ steps.validate_deflake_input.outputs.flavor }} run: make qa-ui-setup cmd=resolve-flavor @@ -202,6 +206,7 @@ jobs: IS_UPGRADE: ${{ steps.validate_deflake_input.outputs.isUpgrade }} OLD_BUILD_NUMBER: ${{ steps.validate_deflake_input.outputs.oldBuildNumber }} S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + S3_FOLDER: ${{ steps.resolve_flavor.outputs.s3Folder }} run: make qa-ui-setup cmd=download-apks # Keep device selection and app installation identical to the selected run diff --git a/scripts/qa/android-ui/resolve_flavor.py b/scripts/qa/android-ui/resolve_flavor.py index 6ca5c037fbc..9b620649686 100755 --- a/scripts/qa/android-ui/resolve_flavor.py +++ b/scripts/qa/android-ui/resolve_flavor.py @@ -51,8 +51,16 @@ print("ERROR: GITHUB_ENV not set", file=sys.stderr) sys.exit(1) +output_path = os.environ.get("GITHUB_OUTPUT") +if not output_path: + print("ERROR: GITHUB_OUTPUT not set", file=sys.stderr) + sys.exit(1) + with open(env_path, "a", encoding="utf-8") as handle: - # Export variables used by downstream setup/install workflow steps. - handle.write(f"S3_FOLDER={s3}\n") + # Only keep install-related values job-wide. handle.write(f"APP_ID={app}\n") handle.write("PACKAGES_TO_UNINSTALL=" + " ".join(pkgs) + "\n") + +with open(output_path, "a", encoding="utf-8") as handle: + # Only the download step needs the resolved S3 folder. + handle.write(f"s3Folder={s3}\n") diff --git a/scripts/qa/android-ui/run_ui_tests.sh b/scripts/qa/android-ui/run_ui_tests.sh index 41e1cc01f11..42918376484 100755 --- a/scripts/qa/android-ui/run_ui_tests.sh +++ b/scripts/qa/android-ui/run_ui_tests.sh @@ -330,10 +330,26 @@ device_reported_zero_tests() { local log_file="${LOG_DIR}/attempt-${attempt}-instrument-${serial}.log" [[ -f "${log_file}" ]] || return 1 - # Zero-test shards are valid for filtered/sharded runs and should not fail the pull step. + # A device shard can legitimately run zero tests after filtering or sharding. + # In that case it will not produce Allure results, so the pull step should not fail. grep -qE 'INSTRUMENTATION_STATUS: numtests=0|OK \(0 tests\)|No tests found' "${log_file}" } +# Treat a zero-test attempt as valid only when every device log for that +# attempt explicitly reported zero tests. +attempt_reported_zero_tests() { + local attempt="$1" + shift + local devices=("$@") + [[ ${#devices[@]} -gt 0 ]] || return 1 + + for serial in "${devices[@]}"; do + if ! device_reported_zero_tests "${attempt}" "${serial}"; then + return 1 + fi + done +} + pull_allure_results_for_attempt() { local attempt="$1" shift @@ -537,9 +553,11 @@ run_attempt_on_devices() { # Android instrumentation can return non-zero for normal test failures. # Keep that separate from real infra failures so retries can still proceed. + # Keep the full device log on disk, but sanitize what gets printed to GitHub. set +e ${adb_cmd} shell am instrument -w -r "${args[@]}" "${instrumentation}" 2>&1 \ - | sed -u "s/^/[${serial}] /" | tee "${log_file}" + | tee "${log_file}" \ + | python3 scripts/qa/android-ui/sanitize_instrumentation_log.py "${serial}" local rc=${PIPESTATUS[0]} set -e @@ -641,6 +659,21 @@ while true; do extract_failed_ids "${attempt}" "${attempt_failed_file}" "${attempt_executed_file}" executed_count="$(wc -l < "${attempt_executed_file}" | tr -d ' ')" if (( executed_count == 0 )); then + # During reruns, a shard can legitimately end up with no tests assigned. + # Only treat that as valid when every device log explicitly reported + # zero tests and the instrumentation command itself did not fail. + if ! attempt_uses_selector_mode "${attempt}" && attempt_reported_zero_tests "${attempt}" "${attempt_devices[@]}"; then + if (( attempt_worker_failed != 0 )); then + echo "ERROR: Attempt ${attempt} reported zero tests but instrumentation failed." + exit 1 + fi + echo "Attempt ${attempt} executed zero tests across all shards." + if [[ ! -f "${first_failed_file}" ]]; then + cp "${attempt_failed_file}" "${first_failed_file}" + fi + current_failed_file="${attempt_failed_file}" + break + fi echo "ERROR: Attempt ${attempt} produced no identifiable executed tests." exit 1 fi diff --git a/scripts/qa/android-ui/sanitize_instrumentation_log.py b/scripts/qa/android-ui/sanitize_instrumentation_log.py new file mode 100644 index 00000000000..b8bd52ba860 --- /dev/null +++ b/scripts/qa/android-ui/sanitize_instrumentation_log.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Trim noisy instrumentation output before it reaches public GitHub logs.""" + +from __future__ import annotations + +import re +import sys + + +STACK_TRACE_LINE = re.compile(r"^\s*at\s+[\w.$]+\(.*\)$") +OMIT_PATTERNS = ( + re.compile(r"^-{2,}\s*(begin|end) exception\s*-{2,}$", re.IGNORECASE), + re.compile(r"^\s*Caused by:"), + re.compile(r"^\s*\.\.\. \d+ more$"), + STACK_TRACE_LINE, +) +FAILURE_REASON_PREFIX = "TEST_FAILURE_REASON: " +FAILURE_PATTERNS = ( + (re.compile(r"Element not found with selector", re.IGNORECASE), "Element not found"), + (re.compile(r"\bAssertionError\b", re.IGNORECASE), "Assertion failed"), + (re.compile(r"\bComparisonFailure\b", re.IGNORECASE), "Comparison failed"), + (re.compile(r"\binvalid_grant\b|\bInvalid user credentials\b", re.IGNORECASE), "Authentication failed"), + (re.compile(r"\bHttpRequestException\b", re.IGNORECASE), "Backend request failed"), + (re.compile(r"\bSocketException\b", re.IGNORECASE), "Network error"), + (re.compile(r"\bWaiterError\b", re.IGNORECASE), "Wait condition failed"), +) + + +def summarize_failure(message: str) -> str: + for pattern, summary in FAILURE_PATTERNS: + if pattern.search(message): + return summary + + cleaned = re.sub(r"selector:.*", "", message, flags=re.IGNORECASE) + cleaned = re.sub(r"resourceId='[^']*'", "resourceId='…'", cleaned) + cleaned = re.sub(r"text='[^']*'", "text='…'", cleaned) + cleaned = re.sub(r"'[^']{2,}'", "'…'", cleaned) + cleaned = re.sub(r'"[^"]{2,}"', '"…"', cleaned) + cleaned = re.sub(r"\b[\w.]+\.kt:\d+\b", "", cleaned) + cleaned = re.sub(r"\b[a-zA-Z_][\w$.]*\([^)]+\)", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip(" :-") + if not cleaned: + return "Test failed" + return cleaned[:137] + "..." if len(cleaned) > 140 else cleaned + + +def sanitize_line(serial: str, raw_line: str) -> str | None: + line = raw_line.rstrip("\n") + if not line: + return f"[{serial}]" + + if any(pattern.match(line) for pattern in OMIT_PATTERNS): + return None + + stack_marker = "INSTRUMENTATION_STATUS: stack=" + if stack_marker in line: + message = line.split(stack_marker, 1)[1] + return f"[{serial}] {FAILURE_REASON_PREFIX}{summarize_failure(message)}" + + if re.match(r"^[\w.$]+(?:Exception|Error):", line): + return f"[{serial}] {FAILURE_REASON_PREFIX}{summarize_failure(line)}" + + if "Element not found with selector" in line: + return f"[{serial}] {FAILURE_REASON_PREFIX}Element not found" + + return f"[{serial}] {line}" + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("Usage: sanitize_instrumentation_log.py ") + + serial = sys.argv[1] + for raw_line in sys.stdin: + sanitized = sanitize_line(serial, raw_line) + if sanitized is None: + continue + print(sanitized, flush=True) + + +if __name__ == "__main__": + main() From 2dc551f45e48d6873381b29c9cf44076b1a056a8 Mon Sep 17 00:00:00 2001 From: emmaoke-w Date: Sun, 16 Aug 2026 19:51:17 +0200 Subject: [PATCH 2/2] ci: remove column flavors and revert log sanitization (WPB-27951) --- .../qa-android-critical-flow-tests.yml | 4 - .../qa-android-ui-test-manual-deflake.yml | 1 - scripts/qa/android-ui/run_ui_tests.sh | 4 +- .../sanitize_instrumentation_log.py | 82 ------------------- 4 files changed, 1 insertion(+), 90 deletions(-) delete mode 100644 scripts/qa/android-ui/sanitize_instrumentation_log.py diff --git a/.github/workflows/qa-android-critical-flow-tests.yml b/.github/workflows/qa-android-critical-flow-tests.yml index e6509180efd..91e469bd7f0 100644 --- a/.github/workflows/qa-android-critical-flow-tests.yml +++ b/.github/workflows/qa-android-critical-flow-tests.yml @@ -48,9 +48,6 @@ on: - alpha beta - staging compat - experimental - - column-1 - - column-2 - - column-3 - debug - fdroid - production @@ -254,7 +251,6 @@ jobs: - name: Register log masks run: | echo "::add-mask::eu-west-1" - echo "::add-mask::mation" - name: Checkout (with submodules) uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 diff --git a/.github/workflows/qa-android-ui-test-manual-deflake.yml b/.github/workflows/qa-android-ui-test-manual-deflake.yml index 62032a7080b..7f2e2b7b0e8 100644 --- a/.github/workflows/qa-android-ui-test-manual-deflake.yml +++ b/.github/workflows/qa-android-ui-test-manual-deflake.yml @@ -96,7 +96,6 @@ jobs: - name: Register log masks run: | echo "::add-mask::eu-west-1" - echo "::add-mask::mation" # Mirror qa-android-critical-flow-tests.yml runner setup so manual # deflake executes in the same QA office runner environment. diff --git a/scripts/qa/android-ui/run_ui_tests.sh b/scripts/qa/android-ui/run_ui_tests.sh index 42918376484..a31e3c2a2c5 100755 --- a/scripts/qa/android-ui/run_ui_tests.sh +++ b/scripts/qa/android-ui/run_ui_tests.sh @@ -553,11 +553,9 @@ run_attempt_on_devices() { # Android instrumentation can return non-zero for normal test failures. # Keep that separate from real infra failures so retries can still proceed. - # Keep the full device log on disk, but sanitize what gets printed to GitHub. set +e ${adb_cmd} shell am instrument -w -r "${args[@]}" "${instrumentation}" 2>&1 \ - | tee "${log_file}" \ - | python3 scripts/qa/android-ui/sanitize_instrumentation_log.py "${serial}" + | sed -u "s/^/[${serial}] /" | tee "${log_file}" local rc=${PIPESTATUS[0]} set -e diff --git a/scripts/qa/android-ui/sanitize_instrumentation_log.py b/scripts/qa/android-ui/sanitize_instrumentation_log.py deleted file mode 100644 index b8bd52ba860..00000000000 --- a/scripts/qa/android-ui/sanitize_instrumentation_log.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -"""Trim noisy instrumentation output before it reaches public GitHub logs.""" - -from __future__ import annotations - -import re -import sys - - -STACK_TRACE_LINE = re.compile(r"^\s*at\s+[\w.$]+\(.*\)$") -OMIT_PATTERNS = ( - re.compile(r"^-{2,}\s*(begin|end) exception\s*-{2,}$", re.IGNORECASE), - re.compile(r"^\s*Caused by:"), - re.compile(r"^\s*\.\.\. \d+ more$"), - STACK_TRACE_LINE, -) -FAILURE_REASON_PREFIX = "TEST_FAILURE_REASON: " -FAILURE_PATTERNS = ( - (re.compile(r"Element not found with selector", re.IGNORECASE), "Element not found"), - (re.compile(r"\bAssertionError\b", re.IGNORECASE), "Assertion failed"), - (re.compile(r"\bComparisonFailure\b", re.IGNORECASE), "Comparison failed"), - (re.compile(r"\binvalid_grant\b|\bInvalid user credentials\b", re.IGNORECASE), "Authentication failed"), - (re.compile(r"\bHttpRequestException\b", re.IGNORECASE), "Backend request failed"), - (re.compile(r"\bSocketException\b", re.IGNORECASE), "Network error"), - (re.compile(r"\bWaiterError\b", re.IGNORECASE), "Wait condition failed"), -) - - -def summarize_failure(message: str) -> str: - for pattern, summary in FAILURE_PATTERNS: - if pattern.search(message): - return summary - - cleaned = re.sub(r"selector:.*", "", message, flags=re.IGNORECASE) - cleaned = re.sub(r"resourceId='[^']*'", "resourceId='…'", cleaned) - cleaned = re.sub(r"text='[^']*'", "text='…'", cleaned) - cleaned = re.sub(r"'[^']{2,}'", "'…'", cleaned) - cleaned = re.sub(r'"[^"]{2,}"', '"…"', cleaned) - cleaned = re.sub(r"\b[\w.]+\.kt:\d+\b", "", cleaned) - cleaned = re.sub(r"\b[a-zA-Z_][\w$.]*\([^)]+\)", "", cleaned) - cleaned = re.sub(r"\s+", " ", cleaned).strip(" :-") - if not cleaned: - return "Test failed" - return cleaned[:137] + "..." if len(cleaned) > 140 else cleaned - - -def sanitize_line(serial: str, raw_line: str) -> str | None: - line = raw_line.rstrip("\n") - if not line: - return f"[{serial}]" - - if any(pattern.match(line) for pattern in OMIT_PATTERNS): - return None - - stack_marker = "INSTRUMENTATION_STATUS: stack=" - if stack_marker in line: - message = line.split(stack_marker, 1)[1] - return f"[{serial}] {FAILURE_REASON_PREFIX}{summarize_failure(message)}" - - if re.match(r"^[\w.$]+(?:Exception|Error):", line): - return f"[{serial}] {FAILURE_REASON_PREFIX}{summarize_failure(line)}" - - if "Element not found with selector" in line: - return f"[{serial}] {FAILURE_REASON_PREFIX}Element not found" - - return f"[{serial}] {line}" - - -def main() -> None: - if len(sys.argv) != 2: - raise SystemExit("Usage: sanitize_instrumentation_log.py ") - - serial = sys.argv[1] - for raw_line in sys.stdin: - sanitized = sanitize_line(serial, raw_line) - if sanitized is None: - continue - print(sanitized, flush=True) - - -if __name__ == "__main__": - main()