From 504dc47ae62c2d5eb47b4aa91188f46164033970 Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Tue, 22 Sep 2026 21:32:40 +0300 Subject: [PATCH 1/6] `Fix Semaphore test report grouping for MCP` | #3290 --- .semaphore/semaphore.yml | 2 +- script/ci-publish-test-results.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4e07a519b..8bc5c6e94 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -162,4 +162,4 @@ after_pipeline: jobs: - name: Publish Results commands: - - test-results gen-pipeline-report + - test-results gen-pipeline-report --generate-mcp-summary diff --git a/script/ci-publish-test-results.sh b/script/ci-publish-test-results.sh index d5c21df8c..e6ad97b40 100755 --- a/script/ci-publish-test-results.sh +++ b/script/ci-publish-test-results.sh @@ -10,7 +10,7 @@ set -euo pipefail if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/outputs/androidTest-results/connected/" if [[ -d "$results_dir" ]]; then - test-results publish "$results_dir" --name "$SEMAPHORE_JOB_NAME" + test-results publish "$results_dir" --name "Instrumentation tests" --generate-mcp-summary else echo "Instrumentation test results directory does not exist: $results_dir" fi @@ -19,7 +19,7 @@ fi if [[ "$SEMAPHORE_JOB_NAME" =~ ^JUnit.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/test-results/" if [[ -d "$results_dir" ]]; then - test-results publish "$results_dir" --name "$SEMAPHORE_JOB_NAME" + test-results publish "$results_dir" --name "JUnit tests" --generate-mcp-summary else echo "JUnit test results directory does not exist: $results_dir" fi From 1ccc36ab1ed8fed0ac8d7f49ca9e6806d15cb386 Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Tue, 22 Sep 2026 21:45:14 +0300 Subject: [PATCH 2/6] Add Markdown test summary to Semaphore reports | #3290 --- .semaphore/semaphore.yml | 4 + script/ci-generate-test-report.sh | 173 ++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100755 script/ci-generate-test-report.sh diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 8bc5c6e94..ba0413cbb 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -162,4 +162,8 @@ after_pipeline: jobs: - name: Publish Results commands: + - sudo apt-get install -y jq - test-results gen-pipeline-report --generate-mcp-summary + - artifact pull workflow "test-results/${SEMAPHORE_PIPELINE_ID}.json" -d pipeline-test-results.json + - ./script/ci-generate-test-report.sh pipeline-test-results.json test-report.md + - artifact push workflow -f -d .semaphore/REPORT.md test-report.md diff --git a/script/ci-generate-test-report.sh b/script/ci-generate-test-report.sh new file mode 100755 index 000000000..221198849 --- /dev/null +++ b/script/ci-generate-test-report.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +# +# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com +# Contributors: denbond7 +# + +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +input_file="$1" +output_file="$2" + +if [[ ! -f "$input_file" ]]; then + echo "Pipeline test results file does not exist: $input_file" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to generate the test report" + exit 1 +fi + +work_dir="$(mktemp -d)" +json_file="$input_file" + +cleanup() { + rm -rf "$work_dir" +} + +trap cleanup EXIT + +# test-results stores JSON reports compressed with gzip while retaining the .json extension. +if gzip -t "$input_file" 2>/dev/null; then + json_file="$work_dir/pipeline-test-results.json" + gzip -cd "$input_file" > "$json_file" +fi + +branch="${SEMAPHORE_GIT_BRANCH:-local}" +commit_sha="${SEMAPHORE_GIT_SHA:-$(git rev-parse HEAD 2>/dev/null || printf 'unknown')}" +short_commit_sha="${commit_sha:0:7}" + +jq -r \ + --arg branch "$branch" \ + --arg commit "$short_commit_sha" \ + ' + def markdown: + tostring + | gsub("\\|"; "|") + | gsub("`"; "`") + | gsub("\\r"; "") + | gsub("\\n"; "
"); + + def duration: + (. // 0) as $nanoseconds + | if $nanoseconds >= 60000000000 then + (((($nanoseconds / 6000000000) | floor) / 10) | tostring) + "m" + else + (((($nanoseconds / 1000000) | floor) / 1000) | tostring) + "s" + end; + + [.testResults[]?] as $groups + | [ + $groups[] as $group + | $group.suites[]? as $suite + | $suite.tests[]? + | { + group: $group.name, + suite: $suite.name, + name: .name, + state: .state, + duration: .duration, + message: (.failure.message // .error.message // "") + } + ] as $tests + | [$tests[] | select(.state == "failed" or .state == "error")] as $failed_tests + | ($tests | sort_by(.duration // 0) | reverse | .[0:10]) as $slowest_tests + | ([$groups[].summary.total] | add // 0) as $total + | ([$groups[].summary.passed] | add // 0) as $passed + | ([$groups[].summary.failed] | add // 0) as $failed + | ([$groups[].summary.error] | add // 0) as $errors + | ([$groups[].summary.skipped] | add // 0) as $skipped + | ([$groups[].summary.disabled] | add // 0) as $disabled + | ([$groups[].summary.duration] | add // 0) as $total_duration + | ( + [ + "# Test report", + "", + (if $total == 0 + then "**Status:** ⚠️ No test results were published" + elif ($failed + $errors) > 0 + then "**Status:** ❌ " + (($failed + $errors) | tostring) + " test(s) failed" + else "**Status:** ✅ All tests passed" + end), + "", + "_Branch `" + ($branch | markdown) + "`, commit `" + ($commit | markdown) + "`._", + "", + "## Summary", + "", + "| Total | Passed | Failed | Errors | Skipped | Disabled | Test duration |", + "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + "| " + ($total | tostring) + + " | " + ($passed | tostring) + + " | " + ($failed | tostring) + + " | " + ($errors | tostring) + + " | " + ($skipped | tostring) + + " | " + ($disabled | tostring) + + " | " + ($total_duration | duration) + " |", + "", + "## Test groups", + "", + "| Group | Tests | Passed | Failed | Errors | Skipped | Suites | Duration |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |" + ] + + ($groups | map( + "| " + (.name | markdown) + + " | " + (.summary.total | tostring) + + " | " + (.summary.passed | tostring) + + " | " + (.summary.failed | tostring) + + " | " + (.summary.error | tostring) + + " | " + (.summary.skipped | tostring) + + " | " + ((.suites | length) | tostring) + + " | " + (.summary.duration | duration) + " |" + )) + + [ + "", + "## Failed tests", + "" + ] + + (if ($failed_tests | length) == 0 then + ["✅ No failed tests."] + else + [ + "| Group | Suite | Test | Result | Duration | Message |", + "| --- | --- | --- | --- | ---: | --- |" + ] + + ($failed_tests | map( + "| " + (.group | markdown) + + " | " + (.suite | markdown) + + " | `" + (.name | markdown) + "`" + + " | " + (.state | markdown) + + " | " + (.duration | duration) + + " | " + (.message | markdown) + " |" + )) + end) + + [ + "", + "## Slowest tests", + "" + ] + + (if ($slowest_tests | length) == 0 then + ["No test results were published."] + else + [ + "| Suite | Test | Result | Duration |", + "| --- | --- | --- | ---: |" + ] + + ($slowest_tests | map( + "| " + (.suite | markdown) + + " | `" + (.name | markdown) + "`" + + " | " + (.state | markdown) + + " | " + (.duration | duration) + " |" + )) + end) + ) + | join("\n") + ' "$json_file" > "$output_file" + +echo "Generated test report: $output_file" From 945dee794f5ff6690fdf0d414fac08a357ab2397 Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Tue, 22 Sep 2026 22:34:21 +0300 Subject: [PATCH 3/6] Align Semaphore reports with passed instrumentation jobs | #3290 --- ...ormalize-passed-instrumentation-results.py | 58 +++++++++++++++++++ script/ci-publish-test-results.sh | 6 ++ 2 files changed, 64 insertions(+) create mode 100755 script/ci-normalize-passed-instrumentation-results.py diff --git a/script/ci-normalize-passed-instrumentation-results.py b/script/ci-normalize-passed-instrumentation-results.py new file mode 100755 index 000000000..56932a8c0 --- /dev/null +++ b/script/ci-normalize-passed-instrumentation-results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# +# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com +# Contributors: denbond7 +# + +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def normalize(xml_file: Path) -> bool: + tree = ET.parse(xml_file) + root = tree.getroot() + changed = False + + for test_case in root.iter(): + if local_name(test_case.tag) != "testcase": + continue + + for child in list(test_case): + if local_name(child.tag) in {"failure", "error"}: + test_case.remove(child) + changed = True + + for test_suite in root.iter(): + if local_name(test_suite.tag) not in {"testsuite", "testsuites"}: + continue + + for attribute in ("failures", "errors"): + if test_suite.get(attribute) not in {None, "0"}: + test_suite.set(attribute, "0") + changed = True + + if changed: + tree.write(xml_file, encoding="UTF-8", xml_declaration=True) + + return changed + + +def main() -> int: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + return 1 + + results_dir = Path(sys.argv[1]) + normalized_files = sum(normalize(xml_file) for xml_file in results_dir.rglob("*.xml")) + print(f"Normalized non-final failures in {normalized_files} passed instrumentation report(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/script/ci-publish-test-results.sh b/script/ci-publish-test-results.sh index e6ad97b40..c0286d608 100755 --- a/script/ci-publish-test-results.sh +++ b/script/ci-publish-test-results.sh @@ -10,6 +10,12 @@ set -euo pipefail if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/outputs/androidTest-results/connected/" if [[ -d "$results_dir" ]]; then + # Android reports can retain failed entries even when Gradle's final result is successful. + # For a passed job, publish its authoritative final outcome instead of non-final failures. + job_result="${SEMAPHORE_JOB_RESULT:-}" + if [[ "${job_result,,}" == "passed" ]]; then + python3 ./script/ci-normalize-passed-instrumentation-results.py "$results_dir" + fi test-results publish "$results_dir" --name "Instrumentation tests" --generate-mcp-summary else echo "Instrumentation test results directory does not exist: $results_dir" From 03d08117f413a96ccdc22ac897a242eae10d7f62 Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Wed, 23 Sep 2026 08:36:52 +0300 Subject: [PATCH 4/6] Fix Semaphore report generation and ADB diagnostics timeout | #3290 --- .semaphore/semaphore.yml | 1 + script/ci-after-fail-debug.sh | 6 ++++++ .../ci-get-and-publish-debug-info-as-artifact.sh | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ba0413cbb..d596d36ca 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -162,6 +162,7 @@ after_pipeline: jobs: - name: Publish Results commands: + - checkout - sudo apt-get install -y jq - test-results gen-pipeline-report --generate-mcp-summary - artifact pull workflow "test-results/${SEMAPHORE_PIPELINE_ID}.json" -d pipeline-test-results.json diff --git a/script/ci-after-fail-debug.sh b/script/ci-after-fail-debug.sh index 93497a533..b5515a6d5 100755 --- a/script/ci-after-fail-debug.sh +++ b/script/ci-after-fail-debug.sh @@ -7,8 +7,14 @@ set -euo pipefail +ADB_BIN="$(command -v adb)" +ADB_COMMAND_TIMEOUT="${ADB_COMMAND_TIMEOUT:-15s}" SUMMARY_FAILURES=0 +adb() { + timeout "$ADB_COMMAND_TIMEOUT" "$ADB_BIN" "$@" +} + print_section() { local title="$1" diff --git a/script/ci-get-and-publish-debug-info-as-artifact.sh b/script/ci-get-and-publish-debug-info-as-artifact.sh index 1088399f1..69cc07bf5 100755 --- a/script/ci-get-and-publish-debug-info-as-artifact.sh +++ b/script/ci-get-and-publish-debug-info-as-artifact.sh @@ -7,6 +7,13 @@ set -euo pipefail +ADB_BIN="$(command -v adb)" +ADB_COMMAND_TIMEOUT="${ADB_COMMAND_TIMEOUT:-15s}" + +adb() { + timeout "$ADB_COMMAND_TIMEOUT" "$ADB_BIN" "$@" +} + if [[ "$SEMAPHORE_JOB_NAME" =~ ^Lint.* ]]; then # Do nothing for 'Lint(structural quality)' job. exit 0 @@ -23,8 +30,11 @@ fi if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then # store full logcat log echo "Collect logcat logs as logcat.txt.gz for $SEMAPHORE_JOB_NAME" - adb logcat -d | gzip > "$HOME/logcat.txt.gz" - artifact push job "$HOME/logcat.txt.gz" + if adb logcat -d | gzip > "$HOME/logcat.txt.gz"; then + artifact push job "$HOME/logcat.txt.gz" + else + echo "Could not collect logcat within $ADB_COMMAND_TIMEOUT, skipping" + fi echo "Store the device's screenshot for $SEMAPHORE_JOB_NAME" if adb shell screencap -p /sdcard/screencap.png; then From 5bcda63ee730fce3d8f4692dcab199b44b4d91c1 Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Wed, 23 Sep 2026 08:44:49 +0300 Subject: [PATCH 5/6] Report final instrumentation test outcomes in Semaphore | #3290 --- ...ormalize-passed-instrumentation-results.py | 55 ++++++++++++--- script/ci-publish-test-results.sh | 10 +-- ...ci-stream-instrumentation-test-progress.sh | 69 +++++++++++-------- 3 files changed, 92 insertions(+), 42 deletions(-) diff --git a/script/ci-normalize-passed-instrumentation-results.py b/script/ci-normalize-passed-instrumentation-results.py index 56932a8c0..063e65fa8 100755 --- a/script/ci-normalize-passed-instrumentation-results.py +++ b/script/ci-normalize-passed-instrumentation-results.py @@ -14,43 +14,80 @@ def local_name(tag: str) -> str: return tag.rsplit("}", 1)[-1] -def normalize(xml_file: Path) -> bool: +def normalize(xml_file: Path, non_passed_tests: set[str]) -> tuple[bool, int]: tree = ET.parse(xml_file) root = tree.getroot() changed = False + removed_failures = 0 for test_case in root.iter(): if local_name(test_case.tag) != "testcase": continue + test_identity = f"{test_case.get('name', '')}({test_case.get('classname', '')})" + if test_identity in non_passed_tests: + continue + for child in list(test_case): if local_name(child.tag) in {"failure", "error"}: test_case.remove(child) changed = True + removed_failures += 1 for test_suite in root.iter(): if local_name(test_suite.tag) not in {"testsuite", "testsuites"}: continue - for attribute in ("failures", "errors"): - if test_suite.get(attribute) not in {None, "0"}: - test_suite.set(attribute, "0") + test_cases = [node for node in test_suite.iter() if local_name(node.tag) == "testcase"] + counts = { + "failures": sum( + any(local_name(child.tag) == "failure" for child in test_case) + for test_case in test_cases + ), + "errors": sum( + any(local_name(child.tag) == "error" for child in test_case) + for test_case in test_cases + ), + } + + for attribute, count in counts.items(): + value = str(count) + if test_suite.get(attribute) != value: + test_suite.set(attribute, value) changed = True if changed: tree.write(xml_file, encoding="UTF-8", xml_declaration=True) - return changed + return changed, removed_failures def main() -> int: - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + if len(sys.argv) not in {2, 3}: + print( + f"Usage: {sys.argv[0]} [non-passed-tests-file]", + file=sys.stderr, + ) return 1 results_dir = Path(sys.argv[1]) - normalized_files = sum(normalize(xml_file) for xml_file in results_dir.rglob("*.xml")) - print(f"Normalized non-final failures in {normalized_files} passed instrumentation report(s)") + non_passed_tests_file = Path(sys.argv[2]) if len(sys.argv) == 3 else None + non_passed_tests = ( + set(non_passed_tests_file.read_text().splitlines()) + if non_passed_tests_file and non_passed_tests_file.is_file() + else set() + ) + + results = [ + normalize(xml_file, non_passed_tests) + for xml_file in results_dir.rglob("*.xml") + ] + normalized_files = sum(changed for changed, _ in results) + removed_failures = sum(removed for _, removed in results) + print( + f"Normalized {removed_failures} non-final failure(s) " + f"in {normalized_files} instrumentation report(s)" + ) return 0 diff --git a/script/ci-publish-test-results.sh b/script/ci-publish-test-results.sh index c0286d608..9eaa9de46 100755 --- a/script/ci-publish-test-results.sh +++ b/script/ci-publish-test-results.sh @@ -10,11 +10,13 @@ set -euo pipefail if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/outputs/androidTest-results/connected/" if [[ -d "$results_dir" ]]; then - # Android reports can retain failed entries even when Gradle's final result is successful. - # For a passed job, publish its authoritative final outcome instead of non-final failures. + # Android XML can retain non-final failures after retries. Align it with the final runner events. job_result="${SEMAPHORE_JOB_RESULT:-}" - if [[ "${job_result,,}" == "passed" ]]; then - python3 ./script/ci-normalize-passed-instrumentation-results.py "$results_dir" + non_passed_tests_file="${INSTRUMENTATION_NON_PASSED_TESTS_FILE:-$HOME/instrumentation-non-passed-tests.txt}" + if [[ "${job_result,,}" == "passed" || -f "$non_passed_tests_file" ]]; then + python3 ./script/ci-normalize-passed-instrumentation-results.py \ + "$results_dir" \ + "$non_passed_tests_file" fi test-results publish "$results_dir" --name "Instrumentation tests" --generate-mcp-summary else diff --git a/script/ci-stream-instrumentation-test-progress.sh b/script/ci-stream-instrumentation-test-progress.sh index e99030d1c..823d3f0b5 100755 --- a/script/ci-stream-instrumentation-test-progress.sh +++ b/script/ci-stream-instrumentation-test-progress.sh @@ -8,9 +8,9 @@ set -euo pipefail format_test_progress() { - local failed_tests_file="${1:-}" + local non_passed_tests_file="${1:-}" - awk -v failed_tests_file="$failed_tests_file" ' + awk -v non_passed_tests_file="$non_passed_tests_file" ' function readable_test_name(raw_name, opening_parenthesis, method_name, qualified_class_name, class_name) { opening_parenthesis = index(raw_name, "(") @@ -43,20 +43,15 @@ format_test_progress() { run_started_at = started_at[test_name] } outcome[test_name] = "" + final_outcome[test_name] = "INCOMPLETE" print_progress(test_name, "STARTED", -1) next } /^failed: / { test_name = substr($0, length("failed: ") + 1) - if (outcome[test_name] != "FAILED") { - failed_count++ - outcome[test_name] = "FAILED" - if (failed_tests_file != "") { - print test_name >> failed_tests_file - close(failed_tests_file) - } - } + outcome[test_name] = "FAILED" + final_outcome[test_name] = "FAILED" elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 print_progress(test_name, "FAILED", elapsed_seconds) next @@ -64,10 +59,8 @@ format_test_progress() { /^assumption failed: / { test_name = substr($0, length("assumption failed: ") + 1) - if (outcome[test_name] != "SKIPPED") { - skipped_count++ - outcome[test_name] = "SKIPPED" - } + outcome[test_name] = "SKIPPED" + final_outcome[test_name] = "SKIPPED" elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 print_progress(test_name, "SKIPPED", elapsed_seconds) next @@ -75,10 +68,8 @@ format_test_progress() { /^ignored: / { test_name = substr($0, length("ignored: ") + 1) - if (outcome[test_name] != "SKIPPED") { - skipped_count++ - outcome[test_name] = "SKIPPED" - } + outcome[test_name] = "SKIPPED" + final_outcome[test_name] = "SKIPPED" print_progress(test_name, "SKIPPED", -1) next } @@ -88,7 +79,7 @@ format_test_progress() { if (outcome[test_name] == "") { elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 - passed_count++ + final_outcome[test_name] = "PASSED" print_progress(test_name, "PASSED", elapsed_seconds) } @@ -97,11 +88,31 @@ format_test_progress() { } END { - incomplete_count = 0 for (test_name in started_at) { if (outcome[test_name] == "") { + final_outcome[test_name] = "INCOMPLETE" + } + } + + for (test_name in final_outcome) { + if (final_outcome[test_name] == "PASSED") { + passed_count++ + } else if (final_outcome[test_name] == "FAILED") { + failed_count++ + } else if (final_outcome[test_name] == "SKIPPED") { + skipped_count++ + } else if (final_outcome[test_name] == "INCOMPLETE") { incomplete_count++ } + + if (non_passed_tests_file != "" && \ + (final_outcome[test_name] == "FAILED" || final_outcome[test_name] == "INCOMPLETE")) { + print test_name >> non_passed_tests_file + } + } + + if (non_passed_tests_file != "") { + close(non_passed_tests_file) } total_count = passed_count + failed_count + skipped_count + incomplete_count @@ -152,9 +163,9 @@ github_repository_slug() { esac } -print_failed_tests() { - local failed_tests_file="$1" - [[ -s "$failed_tests_file" ]] || return 0 +print_non_passed_tests() { + local non_passed_tests_file="$1" + [[ -s "$non_passed_tests_file" ]] || return 0 local repository_slug local commit_sha @@ -162,7 +173,7 @@ print_failed_tests() { commit_sha="${SEMAPHORE_GIT_SHA:-$(git rev-parse HEAD)}" echo "" - echo "[TEST] FAILED TESTS" + echo "[TEST] FAILED OR INCOMPLETE TESTS" echo "[TEST] ------------------------------------------------------------" while IFS= read -r raw_test_name; do @@ -205,7 +216,7 @@ print_failed_tests() { else echo "[TEST] - ${display_name}: ${source_path}" fi - done < "$failed_tests_file" + done < "$non_passed_tests_file" echo "[TEST] ------------------------------------------------------------" } @@ -223,7 +234,7 @@ fi logcat_log_file="${LOGCAT_LOG_FILE:-$HOME/logcat_log.txt}" stream_dir="$(mktemp -d)" logcat_fifo="$stream_dir/logcat" -failed_tests_file="$stream_dir/failed-tests.txt" +non_passed_tests_file="${INSTRUMENTATION_NON_PASSED_TESTS_FILE:-$HOME/instrumentation-non-passed-tests.txt}" logcat_pid="" formatter_pid="" @@ -243,7 +254,7 @@ cleanup() { wait "$formatter_pid" 2>/dev/null || true fi - print_failed_tests "$failed_tests_file" + print_non_passed_tests "$non_passed_tests_file" rm -rf "$stream_dir" return "$test_command_result" @@ -255,9 +266,9 @@ trap cleanup EXIT # Keep the original events as an artifact and print concise progress to the CI log. adb logcat -c mkfifo "$logcat_fifo" -touch "$failed_tests_file" +: > "$non_passed_tests_file" -tee "$logcat_log_file" < "$logcat_fifo" | format_test_progress "$failed_tests_file" & +tee "$logcat_log_file" < "$logcat_fifo" | format_test_progress "$non_passed_tests_file" & formatter_pid=$! adb logcat -v raw TestRunner:I '*:S' > "$logcat_fifo" & From 0ac16ea0bedb6a2c7ffd22d877e6a221cdc3f26b Mon Sep 17 00:00:00 2001 From: Denys Bondarenko Date: Wed, 23 Sep 2026 09:01:44 +0300 Subject: [PATCH 6/6] Show instrumentation stacktraces in Semaphore reports | #3290 --- ...ormalize-passed-instrumentation-results.py | 148 ++++++++++++++++-- script/ci-publish-test-results.sh | 4 +- ...ci-stream-instrumentation-test-progress.sh | 9 +- 3 files changed, 145 insertions(+), 16 deletions(-) diff --git a/script/ci-normalize-passed-instrumentation-results.py b/script/ci-normalize-passed-instrumentation-results.py index 063e65fa8..102975fea 100755 --- a/script/ci-normalize-passed-instrumentation-results.py +++ b/script/ci-normalize-passed-instrumentation-results.py @@ -8,17 +8,131 @@ import sys import xml.etree.ElementTree as ET from pathlib import Path +from typing import NamedTuple + + +class FailureDetails(NamedTuple): + type: str + message: str + stacktrace: str def local_name(tag: str) -> str: return tag.rsplit("}", 1)[-1] -def normalize(xml_file: Path, non_passed_tests: set[str]) -> tuple[bool, int]: +def read_non_passed_tests(non_passed_tests_file: Path | None) -> dict[str, str]: + if not non_passed_tests_file or not non_passed_tests_file.is_file(): + return {} + + non_passed_tests: dict[str, str] = {} + for line in non_passed_tests_file.read_text().splitlines(): + status, separator, test_identity = line.partition("\t") + if separator: + non_passed_tests[test_identity] = status + elif line: + # Support state files created by an older version of the progress script. + non_passed_tests[line] = "FAILED" + return non_passed_tests + + +def failure_details(stacktrace: str) -> FailureDetails: + first_line = stacktrace.splitlines()[0] if stacktrace else "Instrumentation test failed" + exception_type, separator, message = first_line.partition(":") + return FailureDetails( + type=exception_type.strip() if separator else "TestFailure", + message=message.strip() if separator else first_line, + stacktrace=stacktrace, + ) + + +def read_test_runner_failures(logcat_log_file: Path | None) -> dict[str, FailureDetails]: + if not logcat_log_file or not logcat_log_file.is_file(): + return {} + + failures: dict[str, FailureDetails] = {} + failed_test = "" + stacktrace_lines: list[str] | None = None + + for raw_line in logcat_log_file.read_text(errors="replace").splitlines(): + line = raw_line.rstrip("\r") + if line.startswith("failed: "): + failed_test = line.removeprefix("failed: ") + stacktrace_lines = None + elif line == "----- begin exception -----" and failed_test: + stacktrace_lines = [] + elif line == "----- end exception -----" and stacktrace_lines is not None: + stacktrace = "\n".join(stacktrace_lines).strip() + if stacktrace: + failures[failed_test] = failure_details(stacktrace) + stacktrace_lines = None + elif stacktrace_lines is not None: + stacktrace_lines.append(line) + + return failures + + +def set_failure_details( + test_case: ET.Element, + status: str, + details: FailureDetails | None, +) -> bool: + result = next( + ( + child + for child in test_case + if local_name(child.tag) in {"failure", "error"} + ), + None, + ) + changed = False + + if result is None: + result = ET.SubElement(test_case, "failure") + changed = True + + if status == "INCOMPLETE": + details = FailureDetails( + type="IncompleteTest", + message="Instrumentation test did not finish", + stacktrace=( + "The Android test runner stopped before reporting a final result. " + "Check the Semaphore job log and logcat artifact for the underlying " + "device or test process failure." + ), + ) + elif details is None and not (result.text or "").strip(): + details = FailureDetails( + type="TestFailure", + message="Instrumentation test failed", + stacktrace=( + "Android TestRunner reported a failure without an exception stacktrace. " + "Check the Semaphore job log and logcat artifact for more details." + ), + ) + + if details is not None: + for attribute, value in {"type": details.type, "message": details.message}.items(): + if result.get(attribute) != value: + result.set(attribute, value) + changed = True + if result.text != details.stacktrace: + result.text = details.stacktrace + changed = True + + return changed + + +def normalize( + xml_file: Path, + non_passed_tests: dict[str, str], + test_runner_failures: dict[str, FailureDetails], +) -> tuple[bool, int, int]: tree = ET.parse(xml_file) root = tree.getroot() changed = False removed_failures = 0 + enriched_failures = 0 for test_case in root.iter(): if local_name(test_case.tag) != "testcase": @@ -26,6 +140,13 @@ def normalize(xml_file: Path, non_passed_tests: set[str]) -> tuple[bool, int]: test_identity = f"{test_case.get('name', '')}({test_case.get('classname', '')})" if test_identity in non_passed_tests: + if set_failure_details( + test_case, + non_passed_tests[test_identity], + test_runner_failures.get(test_identity), + ): + changed = True + enriched_failures += 1 continue for child in list(test_case): @@ -59,33 +180,34 @@ def normalize(xml_file: Path, non_passed_tests: set[str]) -> tuple[bool, int]: if changed: tree.write(xml_file, encoding="UTF-8", xml_declaration=True) - return changed, removed_failures + return changed, removed_failures, enriched_failures def main() -> int: - if len(sys.argv) not in {2, 3}: + if len(sys.argv) not in {2, 3, 4}: print( - f"Usage: {sys.argv[0]} [non-passed-tests-file]", + f"Usage: {sys.argv[0]} " + "[non-passed-tests-file] [test-runner-logcat-file]", file=sys.stderr, ) return 1 results_dir = Path(sys.argv[1]) - non_passed_tests_file = Path(sys.argv[2]) if len(sys.argv) == 3 else None - non_passed_tests = ( - set(non_passed_tests_file.read_text().splitlines()) - if non_passed_tests_file and non_passed_tests_file.is_file() - else set() - ) + non_passed_tests_file = Path(sys.argv[2]) if len(sys.argv) >= 3 else None + logcat_log_file = Path(sys.argv[3]) if len(sys.argv) >= 4 else None + non_passed_tests = read_non_passed_tests(non_passed_tests_file) + test_runner_failures = read_test_runner_failures(logcat_log_file) results = [ - normalize(xml_file, non_passed_tests) + normalize(xml_file, non_passed_tests, test_runner_failures) for xml_file in results_dir.rglob("*.xml") ] - normalized_files = sum(changed for changed, _ in results) - removed_failures = sum(removed for _, removed in results) + normalized_files = sum(changed for changed, _, _ in results) + removed_failures = sum(removed for _, removed, _ in results) + enriched_failures = sum(enriched for _, _, enriched in results) print( f"Normalized {removed_failures} non-final failure(s) " + f"and enriched {enriched_failures} final failure(s) " f"in {normalized_files} instrumentation report(s)" ) return 0 diff --git a/script/ci-publish-test-results.sh b/script/ci-publish-test-results.sh index 9eaa9de46..10cf92736 100755 --- a/script/ci-publish-test-results.sh +++ b/script/ci-publish-test-results.sh @@ -13,10 +13,12 @@ if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then # Android XML can retain non-final failures after retries. Align it with the final runner events. job_result="${SEMAPHORE_JOB_RESULT:-}" non_passed_tests_file="${INSTRUMENTATION_NON_PASSED_TESTS_FILE:-$HOME/instrumentation-non-passed-tests.txt}" + logcat_log_file="${LOGCAT_LOG_FILE:-$HOME/logcat_log.txt}" if [[ "${job_result,,}" == "passed" || -f "$non_passed_tests_file" ]]; then python3 ./script/ci-normalize-passed-instrumentation-results.py \ "$results_dir" \ - "$non_passed_tests_file" + "$non_passed_tests_file" \ + "$logcat_log_file" fi test-results publish "$results_dir" --name "Instrumentation tests" --generate-mcp-summary else diff --git a/script/ci-stream-instrumentation-test-progress.sh b/script/ci-stream-instrumentation-test-progress.sh index 823d3f0b5..e5e185b39 100755 --- a/script/ci-stream-instrumentation-test-progress.sh +++ b/script/ci-stream-instrumentation-test-progress.sh @@ -107,7 +107,7 @@ format_test_progress() { if (non_passed_tests_file != "" && \ (final_outcome[test_name] == "FAILED" || final_outcome[test_name] == "INCOMPLETE")) { - print test_name >> non_passed_tests_file + print final_outcome[test_name] "\t" test_name >> non_passed_tests_file } } @@ -176,7 +176,12 @@ print_non_passed_tests() { echo "[TEST] FAILED OR INCOMPLETE TESTS" echo "[TEST] ------------------------------------------------------------" - while IFS= read -r raw_test_name; do + while IFS= read -r non_passed_test; do + local raw_test_name="${non_passed_test#*$'\t'}" + if [[ "$raw_test_name" == "$non_passed_test" ]]; then + raw_test_name="$non_passed_test" + fi + local test_method="${raw_test_name%%(*}" local qualified_class_name="${raw_test_name#*(}" qualified_class_name="${qualified_class_name%)}"