diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4e07a519b..d596d36ca 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -162,4 +162,9 @@ after_pipeline: jobs: - name: Publish Results commands: - - test-results gen-pipeline-report + - checkout + - sudo apt-get install -y jq + - test-results gen-pipeline-report --generate-mcp-summary + - artifact pull workflow "test-results/${SEMAPHORE_PIPELINE_ID}.json" -d pipeline-test-results.json + - ./script/ci-generate-test-report.sh pipeline-test-results.json test-report.md + - artifact push workflow -f -d .semaphore/REPORT.md test-report.md diff --git a/script/ci-after-fail-debug.sh b/script/ci-after-fail-debug.sh index 93497a533..b5515a6d5 100755 --- a/script/ci-after-fail-debug.sh +++ b/script/ci-after-fail-debug.sh @@ -7,8 +7,14 @@ set -euo pipefail +ADB_BIN="$(command -v adb)" +ADB_COMMAND_TIMEOUT="${ADB_COMMAND_TIMEOUT:-15s}" SUMMARY_FAILURES=0 +adb() { + timeout "$ADB_COMMAND_TIMEOUT" "$ADB_BIN" "$@" +} + print_section() { local title="$1" diff --git a/script/ci-generate-test-report.sh b/script/ci-generate-test-report.sh new file mode 100755 index 000000000..221198849 --- /dev/null +++ b/script/ci-generate-test-report.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +# +# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com +# Contributors: denbond7 +# + +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +input_file="$1" +output_file="$2" + +if [[ ! -f "$input_file" ]]; then + echo "Pipeline test results file does not exist: $input_file" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to generate the test report" + exit 1 +fi + +work_dir="$(mktemp -d)" +json_file="$input_file" + +cleanup() { + rm -rf "$work_dir" +} + +trap cleanup EXIT + +# test-results stores JSON reports compressed with gzip while retaining the .json extension. +if gzip -t "$input_file" 2>/dev/null; then + json_file="$work_dir/pipeline-test-results.json" + gzip -cd "$input_file" > "$json_file" +fi + +branch="${SEMAPHORE_GIT_BRANCH:-local}" +commit_sha="${SEMAPHORE_GIT_SHA:-$(git rev-parse HEAD 2>/dev/null || printf 'unknown')}" +short_commit_sha="${commit_sha:0:7}" + +jq -r \ + --arg branch "$branch" \ + --arg commit "$short_commit_sha" \ + ' + def markdown: + tostring + | gsub("\\|"; "|") + | gsub("`"; "`") + | gsub("\\r"; "") + | gsub("\\n"; "
"); + + def duration: + (. // 0) as $nanoseconds + | if $nanoseconds >= 60000000000 then + (((($nanoseconds / 6000000000) | floor) / 10) | tostring) + "m" + else + (((($nanoseconds / 1000000) | floor) / 1000) | tostring) + "s" + end; + + [.testResults[]?] as $groups + | [ + $groups[] as $group + | $group.suites[]? as $suite + | $suite.tests[]? + | { + group: $group.name, + suite: $suite.name, + name: .name, + state: .state, + duration: .duration, + message: (.failure.message // .error.message // "") + } + ] as $tests + | [$tests[] | select(.state == "failed" or .state == "error")] as $failed_tests + | ($tests | sort_by(.duration // 0) | reverse | .[0:10]) as $slowest_tests + | ([$groups[].summary.total] | add // 0) as $total + | ([$groups[].summary.passed] | add // 0) as $passed + | ([$groups[].summary.failed] | add // 0) as $failed + | ([$groups[].summary.error] | add // 0) as $errors + | ([$groups[].summary.skipped] | add // 0) as $skipped + | ([$groups[].summary.disabled] | add // 0) as $disabled + | ([$groups[].summary.duration] | add // 0) as $total_duration + | ( + [ + "# Test report", + "", + (if $total == 0 + then "**Status:** ⚠️ No test results were published" + elif ($failed + $errors) > 0 + then "**Status:** ❌ " + (($failed + $errors) | tostring) + " test(s) failed" + else "**Status:** ✅ All tests passed" + end), + "", + "_Branch `" + ($branch | markdown) + "`, commit `" + ($commit | markdown) + "`._", + "", + "## Summary", + "", + "| Total | Passed | Failed | Errors | Skipped | Disabled | Test duration |", + "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + "| " + ($total | tostring) + + " | " + ($passed | tostring) + + " | " + ($failed | tostring) + + " | " + ($errors | tostring) + + " | " + ($skipped | tostring) + + " | " + ($disabled | tostring) + + " | " + ($total_duration | duration) + " |", + "", + "## Test groups", + "", + "| Group | Tests | Passed | Failed | Errors | Skipped | Suites | Duration |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |" + ] + + ($groups | map( + "| " + (.name | markdown) + + " | " + (.summary.total | tostring) + + " | " + (.summary.passed | tostring) + + " | " + (.summary.failed | tostring) + + " | " + (.summary.error | tostring) + + " | " + (.summary.skipped | tostring) + + " | " + ((.suites | length) | tostring) + + " | " + (.summary.duration | duration) + " |" + )) + + [ + "", + "## Failed tests", + "" + ] + + (if ($failed_tests | length) == 0 then + ["✅ No failed tests."] + else + [ + "| Group | Suite | Test | Result | Duration | Message |", + "| --- | --- | --- | --- | ---: | --- |" + ] + + ($failed_tests | map( + "| " + (.group | markdown) + + " | " + (.suite | markdown) + + " | `" + (.name | markdown) + "`" + + " | " + (.state | markdown) + + " | " + (.duration | duration) + + " | " + (.message | markdown) + " |" + )) + end) + + [ + "", + "## Slowest tests", + "" + ] + + (if ($slowest_tests | length) == 0 then + ["No test results were published."] + else + [ + "| Suite | Test | Result | Duration |", + "| --- | --- | --- | ---: |" + ] + + ($slowest_tests | map( + "| " + (.suite | markdown) + + " | `" + (.name | markdown) + "`" + + " | " + (.state | markdown) + + " | " + (.duration | duration) + " |" + )) + end) + ) + | join("\n") + ' "$json_file" > "$output_file" + +echo "Generated test report: $output_file" diff --git a/script/ci-get-and-publish-debug-info-as-artifact.sh b/script/ci-get-and-publish-debug-info-as-artifact.sh index 1088399f1..69cc07bf5 100755 --- a/script/ci-get-and-publish-debug-info-as-artifact.sh +++ b/script/ci-get-and-publish-debug-info-as-artifact.sh @@ -7,6 +7,13 @@ set -euo pipefail +ADB_BIN="$(command -v adb)" +ADB_COMMAND_TIMEOUT="${ADB_COMMAND_TIMEOUT:-15s}" + +adb() { + timeout "$ADB_COMMAND_TIMEOUT" "$ADB_BIN" "$@" +} + if [[ "$SEMAPHORE_JOB_NAME" =~ ^Lint.* ]]; then # Do nothing for 'Lint(structural quality)' job. exit 0 @@ -23,8 +30,11 @@ fi if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then # store full logcat log echo "Collect logcat logs as logcat.txt.gz for $SEMAPHORE_JOB_NAME" - adb logcat -d | gzip > "$HOME/logcat.txt.gz" - artifact push job "$HOME/logcat.txt.gz" + if adb logcat -d | gzip > "$HOME/logcat.txt.gz"; then + artifact push job "$HOME/logcat.txt.gz" + else + echo "Could not collect logcat within $ADB_COMMAND_TIMEOUT, skipping" + fi echo "Store the device's screenshot for $SEMAPHORE_JOB_NAME" if adb shell screencap -p /sdcard/screencap.png; then diff --git a/script/ci-normalize-passed-instrumentation-results.py b/script/ci-normalize-passed-instrumentation-results.py new file mode 100755 index 000000000..102975fea --- /dev/null +++ b/script/ci-normalize-passed-instrumentation-results.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 + +# +# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com +# Contributors: denbond7 +# + +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import NamedTuple + + +class FailureDetails(NamedTuple): + type: str + message: str + stacktrace: str + + +def local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def read_non_passed_tests(non_passed_tests_file: Path | None) -> dict[str, str]: + if not non_passed_tests_file or not non_passed_tests_file.is_file(): + return {} + + non_passed_tests: dict[str, str] = {} + for line in non_passed_tests_file.read_text().splitlines(): + status, separator, test_identity = line.partition("\t") + if separator: + non_passed_tests[test_identity] = status + elif line: + # Support state files created by an older version of the progress script. + non_passed_tests[line] = "FAILED" + return non_passed_tests + + +def failure_details(stacktrace: str) -> FailureDetails: + first_line = stacktrace.splitlines()[0] if stacktrace else "Instrumentation test failed" + exception_type, separator, message = first_line.partition(":") + return FailureDetails( + type=exception_type.strip() if separator else "TestFailure", + message=message.strip() if separator else first_line, + stacktrace=stacktrace, + ) + + +def read_test_runner_failures(logcat_log_file: Path | None) -> dict[str, FailureDetails]: + if not logcat_log_file or not logcat_log_file.is_file(): + return {} + + failures: dict[str, FailureDetails] = {} + failed_test = "" + stacktrace_lines: list[str] | None = None + + for raw_line in logcat_log_file.read_text(errors="replace").splitlines(): + line = raw_line.rstrip("\r") + if line.startswith("failed: "): + failed_test = line.removeprefix("failed: ") + stacktrace_lines = None + elif line == "----- begin exception -----" and failed_test: + stacktrace_lines = [] + elif line == "----- end exception -----" and stacktrace_lines is not None: + stacktrace = "\n".join(stacktrace_lines).strip() + if stacktrace: + failures[failed_test] = failure_details(stacktrace) + stacktrace_lines = None + elif stacktrace_lines is not None: + stacktrace_lines.append(line) + + return failures + + +def set_failure_details( + test_case: ET.Element, + status: str, + details: FailureDetails | None, +) -> bool: + result = next( + ( + child + for child in test_case + if local_name(child.tag) in {"failure", "error"} + ), + None, + ) + changed = False + + if result is None: + result = ET.SubElement(test_case, "failure") + changed = True + + if status == "INCOMPLETE": + details = FailureDetails( + type="IncompleteTest", + message="Instrumentation test did not finish", + stacktrace=( + "The Android test runner stopped before reporting a final result. " + "Check the Semaphore job log and logcat artifact for the underlying " + "device or test process failure." + ), + ) + elif details is None and not (result.text or "").strip(): + details = FailureDetails( + type="TestFailure", + message="Instrumentation test failed", + stacktrace=( + "Android TestRunner reported a failure without an exception stacktrace. " + "Check the Semaphore job log and logcat artifact for more details." + ), + ) + + if details is not None: + for attribute, value in {"type": details.type, "message": details.message}.items(): + if result.get(attribute) != value: + result.set(attribute, value) + changed = True + if result.text != details.stacktrace: + result.text = details.stacktrace + changed = True + + return changed + + +def normalize( + xml_file: Path, + non_passed_tests: dict[str, str], + test_runner_failures: dict[str, FailureDetails], +) -> tuple[bool, int, int]: + tree = ET.parse(xml_file) + root = tree.getroot() + changed = False + removed_failures = 0 + enriched_failures = 0 + + for test_case in root.iter(): + if local_name(test_case.tag) != "testcase": + continue + + test_identity = f"{test_case.get('name', '')}({test_case.get('classname', '')})" + if test_identity in non_passed_tests: + if set_failure_details( + test_case, + non_passed_tests[test_identity], + test_runner_failures.get(test_identity), + ): + changed = True + enriched_failures += 1 + continue + + for child in list(test_case): + if local_name(child.tag) in {"failure", "error"}: + test_case.remove(child) + changed = True + removed_failures += 1 + + for test_suite in root.iter(): + if local_name(test_suite.tag) not in {"testsuite", "testsuites"}: + continue + + test_cases = [node for node in test_suite.iter() if local_name(node.tag) == "testcase"] + counts = { + "failures": sum( + any(local_name(child.tag) == "failure" for child in test_case) + for test_case in test_cases + ), + "errors": sum( + any(local_name(child.tag) == "error" for child in test_case) + for test_case in test_cases + ), + } + + for attribute, count in counts.items(): + value = str(count) + if test_suite.get(attribute) != value: + test_suite.set(attribute, value) + changed = True + + if changed: + tree.write(xml_file, encoding="UTF-8", xml_declaration=True) + + return changed, removed_failures, enriched_failures + + +def main() -> int: + if len(sys.argv) not in {2, 3, 4}: + print( + f"Usage: {sys.argv[0]} " + "[non-passed-tests-file] [test-runner-logcat-file]", + file=sys.stderr, + ) + return 1 + + results_dir = Path(sys.argv[1]) + non_passed_tests_file = Path(sys.argv[2]) if len(sys.argv) >= 3 else None + logcat_log_file = Path(sys.argv[3]) if len(sys.argv) >= 4 else None + non_passed_tests = read_non_passed_tests(non_passed_tests_file) + test_runner_failures = read_test_runner_failures(logcat_log_file) + + results = [ + normalize(xml_file, non_passed_tests, test_runner_failures) + for xml_file in results_dir.rglob("*.xml") + ] + normalized_files = sum(changed for changed, _, _ in results) + removed_failures = sum(removed for _, removed, _ in results) + enriched_failures = sum(enriched for _, _, enriched in results) + print( + f"Normalized {removed_failures} non-final failure(s) " + f"and enriched {enriched_failures} final failure(s) " + f"in {normalized_files} instrumentation report(s)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/script/ci-publish-test-results.sh b/script/ci-publish-test-results.sh index d5c21df8c..10cf92736 100755 --- a/script/ci-publish-test-results.sh +++ b/script/ci-publish-test-results.sh @@ -10,7 +10,17 @@ set -euo pipefail if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/outputs/androidTest-results/connected/" if [[ -d "$results_dir" ]]; then - test-results publish "$results_dir" --name "$SEMAPHORE_JOB_NAME" + # Android XML can retain non-final failures after retries. Align it with the final runner events. + job_result="${SEMAPHORE_JOB_RESULT:-}" + non_passed_tests_file="${INSTRUMENTATION_NON_PASSED_TESTS_FILE:-$HOME/instrumentation-non-passed-tests.txt}" + logcat_log_file="${LOGCAT_LOG_FILE:-$HOME/logcat_log.txt}" + if [[ "${job_result,,}" == "passed" || -f "$non_passed_tests_file" ]]; then + python3 ./script/ci-normalize-passed-instrumentation-results.py \ + "$results_dir" \ + "$non_passed_tests_file" \ + "$logcat_log_file" + fi + test-results publish "$results_dir" --name "Instrumentation tests" --generate-mcp-summary else echo "Instrumentation test results directory does not exist: $results_dir" fi @@ -19,7 +29,7 @@ fi if [[ "$SEMAPHORE_JOB_NAME" =~ ^JUnit.* ]]; then results_dir="$HOME/git/flowcrypt-android/FlowCrypt/build/test-results/" if [[ -d "$results_dir" ]]; then - test-results publish "$results_dir" --name "$SEMAPHORE_JOB_NAME" + test-results publish "$results_dir" --name "JUnit tests" --generate-mcp-summary else echo "JUnit test results directory does not exist: $results_dir" fi diff --git a/script/ci-stream-instrumentation-test-progress.sh b/script/ci-stream-instrumentation-test-progress.sh index e99030d1c..e5e185b39 100755 --- a/script/ci-stream-instrumentation-test-progress.sh +++ b/script/ci-stream-instrumentation-test-progress.sh @@ -8,9 +8,9 @@ set -euo pipefail format_test_progress() { - local failed_tests_file="${1:-}" + local non_passed_tests_file="${1:-}" - awk -v failed_tests_file="$failed_tests_file" ' + awk -v non_passed_tests_file="$non_passed_tests_file" ' function readable_test_name(raw_name, opening_parenthesis, method_name, qualified_class_name, class_name) { opening_parenthesis = index(raw_name, "(") @@ -43,20 +43,15 @@ format_test_progress() { run_started_at = started_at[test_name] } outcome[test_name] = "" + final_outcome[test_name] = "INCOMPLETE" print_progress(test_name, "STARTED", -1) next } /^failed: / { test_name = substr($0, length("failed: ") + 1) - if (outcome[test_name] != "FAILED") { - failed_count++ - outcome[test_name] = "FAILED" - if (failed_tests_file != "") { - print test_name >> failed_tests_file - close(failed_tests_file) - } - } + outcome[test_name] = "FAILED" + final_outcome[test_name] = "FAILED" elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 print_progress(test_name, "FAILED", elapsed_seconds) next @@ -64,10 +59,8 @@ format_test_progress() { /^assumption failed: / { test_name = substr($0, length("assumption failed: ") + 1) - if (outcome[test_name] != "SKIPPED") { - skipped_count++ - outcome[test_name] = "SKIPPED" - } + outcome[test_name] = "SKIPPED" + final_outcome[test_name] = "SKIPPED" elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 print_progress(test_name, "SKIPPED", elapsed_seconds) next @@ -75,10 +68,8 @@ format_test_progress() { /^ignored: / { test_name = substr($0, length("ignored: ") + 1) - if (outcome[test_name] != "SKIPPED") { - skipped_count++ - outcome[test_name] = "SKIPPED" - } + outcome[test_name] = "SKIPPED" + final_outcome[test_name] = "SKIPPED" print_progress(test_name, "SKIPPED", -1) next } @@ -88,7 +79,7 @@ format_test_progress() { if (outcome[test_name] == "") { elapsed_seconds = (test_name in started_at) ? systime() - started_at[test_name] : -1 - passed_count++ + final_outcome[test_name] = "PASSED" print_progress(test_name, "PASSED", elapsed_seconds) } @@ -97,11 +88,31 @@ format_test_progress() { } END { - incomplete_count = 0 for (test_name in started_at) { if (outcome[test_name] == "") { + final_outcome[test_name] = "INCOMPLETE" + } + } + + for (test_name in final_outcome) { + if (final_outcome[test_name] == "PASSED") { + passed_count++ + } else if (final_outcome[test_name] == "FAILED") { + failed_count++ + } else if (final_outcome[test_name] == "SKIPPED") { + skipped_count++ + } else if (final_outcome[test_name] == "INCOMPLETE") { incomplete_count++ } + + if (non_passed_tests_file != "" && \ + (final_outcome[test_name] == "FAILED" || final_outcome[test_name] == "INCOMPLETE")) { + print final_outcome[test_name] "\t" test_name >> non_passed_tests_file + } + } + + if (non_passed_tests_file != "") { + close(non_passed_tests_file) } total_count = passed_count + failed_count + skipped_count + incomplete_count @@ -152,9 +163,9 @@ github_repository_slug() { esac } -print_failed_tests() { - local failed_tests_file="$1" - [[ -s "$failed_tests_file" ]] || return 0 +print_non_passed_tests() { + local non_passed_tests_file="$1" + [[ -s "$non_passed_tests_file" ]] || return 0 local repository_slug local commit_sha @@ -162,10 +173,15 @@ print_failed_tests() { commit_sha="${SEMAPHORE_GIT_SHA:-$(git rev-parse HEAD)}" echo "" - echo "[TEST] FAILED TESTS" + echo "[TEST] FAILED OR INCOMPLETE TESTS" echo "[TEST] ------------------------------------------------------------" - while IFS= read -r raw_test_name; do + while IFS= read -r non_passed_test; do + local raw_test_name="${non_passed_test#*$'\t'}" + if [[ "$raw_test_name" == "$non_passed_test" ]]; then + raw_test_name="$non_passed_test" + fi + local test_method="${raw_test_name%%(*}" local qualified_class_name="${raw_test_name#*(}" qualified_class_name="${qualified_class_name%)}" @@ -205,7 +221,7 @@ print_failed_tests() { else echo "[TEST] - ${display_name}: ${source_path}" fi - done < "$failed_tests_file" + done < "$non_passed_tests_file" echo "[TEST] ------------------------------------------------------------" } @@ -223,7 +239,7 @@ fi logcat_log_file="${LOGCAT_LOG_FILE:-$HOME/logcat_log.txt}" stream_dir="$(mktemp -d)" logcat_fifo="$stream_dir/logcat" -failed_tests_file="$stream_dir/failed-tests.txt" +non_passed_tests_file="${INSTRUMENTATION_NON_PASSED_TESTS_FILE:-$HOME/instrumentation-non-passed-tests.txt}" logcat_pid="" formatter_pid="" @@ -243,7 +259,7 @@ cleanup() { wait "$formatter_pid" 2>/dev/null || true fi - print_failed_tests "$failed_tests_file" + print_non_passed_tests "$non_passed_tests_file" rm -rf "$stream_dir" return "$test_command_result" @@ -255,9 +271,9 @@ trap cleanup EXIT # Keep the original events as an artifact and print concise progress to the CI log. adb logcat -c mkfifo "$logcat_fifo" -touch "$failed_tests_file" +: > "$non_passed_tests_file" -tee "$logcat_log_file" < "$logcat_fifo" | format_test_progress "$failed_tests_file" & +tee "$logcat_log_file" < "$logcat_fifo" | format_test_progress "$non_passed_tests_file" & formatter_pid=$! adb logcat -v raw TestRunner:I '*:S' > "$logcat_fifo" &