diff --git a/app/Models/Build.php b/app/Models/Build.php index a0d8fe0185..c45518ef3f 100644 --- a/app/Models/Build.php +++ b/app/Models/Build.php @@ -33,6 +33,7 @@ * @property int $buildwarnings * @property int $buildduration * @property int $testnotrun + * @property int $testnotrunwarning * @property int $testfailed * @property int $testpassed * @property int $testtimestatusfailed @@ -84,6 +85,7 @@ class Build extends Model 'buildwarnings', 'buildduration', 'testnotrun', + 'testnotrunwarning', 'testfailed', 'testpassed', 'testtimestatusfailed', @@ -118,6 +120,7 @@ class Build extends Model 'buildwarnings' => 'integer', 'buildduration' => 'integer', 'testnotrun' => 'integer', + 'testnotrunwarning' => 'integer', 'testfailed' => 'integer', 'testpassed' => 'integer', 'testduration' => 'integer', diff --git a/app/Models/Test.php b/app/Models/Test.php index a89eb00a3c..578230e89c 100644 --- a/app/Models/Test.php +++ b/app/Models/Test.php @@ -32,6 +32,8 @@ * @property ?Carbon $starttime * @property TestTimeStatusCategory $timestatuscategory * + * @method static Builder notRunWarning() + * * @mixin Builder */ class Test extends Model @@ -84,6 +86,22 @@ class Test extends Model 'starttime' => 'datetime', ]; + /** + * Tests which did not run for a reason other than being explicitly disabled. + * + * CTest reports a completion status of "Disabled" for tests marked DISABLED. + * + * @param Builder $query + */ + public function scopeNotRunWarning(Builder $query): void + { + $query->where('status', self::NOTRUN) + ->where(static function (Builder $query): void { + $query->whereNull('details') + ->orWhere('details', '!=', self::DISABLED); + }); + } + /** * @return BelongsTo */ diff --git a/app/cdash/app/Controller/Api/Index.php b/app/cdash/app/Controller/Api/Index.php index b70f94fd90..37662aa2c3 100644 --- a/app/cdash/app/Controller/Api/Index.php +++ b/app/cdash/app/Controller/Api/Index.php @@ -17,6 +17,7 @@ namespace CDash\Controller\Api; +use App\Models\Test; use App\Utils\TestingDay; use CDash\Database; use CDash\Model\BuildGroup; @@ -345,6 +346,7 @@ public function getIndexSelect(): string b.buildwarnings AS countbuildwarnings, b.buildduration, b.testnotrun AS counttestsnotrun, + b.testnotrunwarning AS counttestsnotrunwarning, b.testfailed AS counttestsfailed, b.testpassed AS counttestspassed, b.testtimestatusfailed AS countteststimestatusfailed, @@ -656,6 +658,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false $selected_build_warnings = 0; $selected_build_duration = 0; $selected_tests_not_run = 0; + $selected_tests_not_run_warning = 0; $selected_tests_failed = 0; $selected_tests_passed = 0; $selected_proc_time = 0; @@ -685,6 +688,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false b.starttime, b.endtime, testnotrun, + testnotrunwarning, testfailed, testpassed, testduration, @@ -705,6 +709,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false $selected_build_warnings += max(0, $select_array->buildwarnings); $selected_build_duration += max(0, $select_array->buildduration); $selected_tests_not_run += max(0, $select_array->testnotrun); + $selected_tests_not_run_warning += max(0, $select_array->testnotrunwarning); $selected_tests_failed += max(0, $select_array->testfailed); $selected_tests_passed += max(0, $select_array->testpassed); $selected_proc_time += max(0, $select_array->testtime); @@ -1013,6 +1018,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false $test_response = []; $nnotrun = $build_array['counttestsnotrun']; + $nnotrunwarning = $build_array['counttestsnotrunwarning']; $nfail = $build_array['counttestsfailed']; $npass = $build_array['counttestspassed']; $proc_time = $build_array['testtime']; @@ -1022,11 +1028,13 @@ public function generateBuildResponseFromRow(array $build_array): array|false if (!$this->childView) { if ($this->includeSubProjects) { $nnotrun = $selected_tests_not_run; + $nnotrunwarning = $selected_tests_not_run_warning; $nfail = $selected_tests_failed; $npass = $selected_tests_passed; $proc_time = $selected_proc_time; } else { $nnotrun -= $selected_tests_not_run; + $nnotrunwarning -= $selected_tests_not_run_warning; $nfail -= $selected_tests_failed; $npass -= $selected_tests_passed; $proc_time -= $selected_proc_time; @@ -1055,7 +1063,8 @@ public function generateBuildResponseFromRow(array $build_array): array|false $labels_result = DB::select(" SELECT b2t.status, - b2t.newstatus + b2t.newstatus, + b2t.details FROM build2test AS b2t INNER JOIN label2test AS l2t ON l2t.testid = b2t.id WHERE @@ -1065,6 +1074,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false ", array_merge([$buildid], $this->labelIds)); $nnotrun = 0; + $nnotrunwarning = 0; $nfail = 0; $npass = 0; $test_response['nfaildiffp'] = 0; @@ -1089,6 +1099,9 @@ public function generateBuildResponseFromRow(array $build_array): array|false break; case 'notrun': $nnotrun++; + if ($label_row->details !== Test::DISABLED) { + $nnotrunwarning++; + } if ((int) $label_row->newstatus === 1) { $test_response['nnotrundiffp']++; } @@ -1098,6 +1111,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false } $test_response['notrun'] = $nnotrun; + $test_response['notrunwarning'] = max(0, $nnotrunwarning); $test_response['fail'] = $nfail; $test_response['pass'] = $npass; diff --git a/app/cdash/app/Controller/Api/QueryTests.php b/app/cdash/app/Controller/Api/QueryTests.php index 7f00f75295..19a58a9963 100644 --- a/app/cdash/app/Controller/Api/QueryTests.php +++ b/app/cdash/app/Controller/Api/QueryTests.php @@ -19,6 +19,7 @@ use App\Models\PinnedTestMeasurement; use App\Models\Project as EloquentProject; +use App\Models\Test; use App\Models\TestMeasurement; use CDash\Database; use CDash\Model\Build; @@ -423,7 +424,7 @@ public function getResponse(): array case 'notrun': $test['status'] = 'Not Run'; - $test['statusclass'] = 'warning'; + $test['statusclass'] = $row->details === Test::DISABLED ? 'normal' : 'warning'; break; } diff --git a/app/cdash/app/Model/Build.php b/app/cdash/app/Model/Build.php index cd07084685..206d1006b1 100644 --- a/app/cdash/app/Model/Build.php +++ b/app/cdash/app/Model/Build.php @@ -773,7 +773,7 @@ public function Save() /** Helper function for test number accessors. */ private function GetNumberOfTestsByField(string $field): int { - if (!in_array($field, ['testpassed', 'testfailed', 'testnotrun'], true)) { + if (!in_array($field, ['testpassed', 'testfailed', 'testnotrun', 'testnotrunwarning'], true)) { throw new InvalidArgumentException('Invalid field specified.'); } @@ -799,20 +799,36 @@ public function GetNumberOfNotRunTests(): int|false return $this->GetNumberOfTestsByField('testnotrun'); } + /** Get number of not run tests which were not explicitly disabled */ + public function GetNumberOfNotRunWarningTests(): int|false + { + return $this->GetNumberOfTestsByField('testnotrunwarning'); + } + /** Update the test numbers */ public function UpdateTestNumbers(int $numberTestsPassed, int $numberTestsFailed, int $numberTestsNotRun): void { $this->TestFailedCount = $numberTestsFailed; + // Disabled tests aren't worth warning about, so they are tallied separately from + // the other not-run tests. The tests for this build have all been recorded by + // now, so they can be counted directly. + $numberTestsNotRunWarning = EloquentBuild::find((int) $this->Id) + ?->tests() + ->notRunWarning() + ->count() ?? 0; + // If this is a subproject build, we also have to update its parents test numbers. $newFailed = $numberTestsFailed - $this->GetNumberOfFailedTests(); $newNotRun = $numberTestsNotRun - $this->GetNumberOfNotRunTests(); $newPassed = $numberTestsPassed - $this->GetNumberOfPassedTests(); + $newNotRunWarning = $numberTestsNotRunWarning - $this->GetNumberOfNotRunWarningTests(); $this->SetParentId($this->LookupParentBuildId()); - $this->UpdateParentTestNumbers($newFailed, $newNotRun, $newPassed); + $this->UpdateParentTestNumbers($newFailed, $newNotRun, $newPassed, $newNotRunWarning); EloquentBuild::whereKey($this->Id)->update([ 'testnotrun' => $numberTestsNotRun, + 'testnotrunwarning' => $numberTestsNotRunWarning, 'testfailed' => $numberTestsFailed, 'testpassed' => $numberTestsPassed, ]); @@ -1612,22 +1628,24 @@ public function UpdateBuild($buildid, $newErrors, $newWarnings): void } /** Update the testing numbers for our parent build. */ - private function UpdateParentTestNumbers(int $newFailed, int $newNotRun, int $newPassed): void + private function UpdateParentTestNumbers(int $newFailed, int $newNotRun, int $newPassed, int $newNotRunWarning): void { if ($this->ParentId < 1) { return; } - DB::transaction(function () use ($newFailed, $newNotRun, $newPassed): void { + DB::transaction(function () use ($newFailed, $newNotRun, $newPassed, $newNotRunWarning): void { $parent = EloquentBuild::findOrFail($this->ParentId); // Don't let the -1 default value screw up our math. $parent_testfailed = self::ConvertMissingToZero($parent->testfailed); $parent_testnotrun = self::ConvertMissingToZero($parent->testnotrun); + $parent_testnotrunwarning = self::ConvertMissingToZero($parent->testnotrunwarning); $parent_testpassed = self::ConvertMissingToZero($parent->testpassed); $parent->update([ 'testnotrun' => $newNotRun + $parent_testnotrun, + 'testnotrunwarning' => $newNotRunWarning + $parent_testnotrunwarning, 'testfailed' => $newFailed + $parent_testfailed, 'testpassed' => $newPassed + $parent_testpassed, ]); diff --git a/app/cdash/tests/test_testhistory.php b/app/cdash/tests/test_testhistory.php index 7f06fb55fc..e9bd2f3a24 100644 --- a/app/cdash/tests/test_testhistory.php +++ b/app/cdash/tests/test_testhistory.php @@ -134,6 +134,9 @@ private function generateXML($mm, $timestamp, $include_sporadic, $flaky_passed) ./notrun + + Skipped + diff --git a/database/migrations/2026_08_25_184512_build_testnotrunwarning.php b/database/migrations/2026_08_25_184512_build_testnotrunwarning.php new file mode 100644 index 0000000000..92c9d67f96 --- /dev/null +++ b/database/migrations/2026_08_25_184512_build_testnotrunwarning.php @@ -0,0 +1,53 @@ += 0'); + + DB::update(" + UPDATE build + SET testnotrunwarning = counted.total + FROM ( + SELECT buildid, COUNT(*) AS total + FROM build2test + WHERE status = 'notrun' + AND (details IS NULL OR details != 'Disabled') + GROUP BY buildid + ) AS counted + WHERE build.id = counted.buildid + "); + + // Parent builds aggregate the results of their children. + DB::update(" + UPDATE build + SET testnotrunwarning = GREATEST(build.testnotrunwarning, 0) + counted.total + FROM ( + SELECT child.parentid, COUNT(*) AS total + FROM build2test + INNER JOIN build AS child ON child.id = build2test.buildid + WHERE child.parentid > 0 + AND build2test.status = 'notrun' + AND (build2test.details IS NULL OR build2test.details != 'Disabled') + GROUP BY child.parentid + ) AS counted + WHERE build.id = counted.parentid + "); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + } +}; diff --git a/graphql/schema.graphql b/graphql/schema.graphql index 1eab0ea949..89955af855 100644 --- a/graphql/schema.graphql +++ b/graphql/schema.graphql @@ -856,6 +856,11 @@ type Build { "The number of tests not run for this build." notRunTestsCount: Int @rename(attribute: "testnotrun") + """ + The number of not-run tests counted as warnings (excludes tests which were explicitly disabled). + """ + notRunTestsWarningCount: Int @rename(attribute: "testnotrunwarning") + "The duration of the test step in seconds." testDuration: Int! @rename(attribute: "testduration") diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 8e1ac63409..383b3d8860 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -7848,7 +7848,7 @@ parameters: - rawMessage: 'Only numeric types are allowed in -, int|false given on the right side.' identifier: minus.rightNonNumeric - count: 3 + count: 4 path: app/cdash/app/Model/Build.php - diff --git a/resources/js/angular/views/partials/build.html b/resources/js/angular/views/partials/build.html index 4d7e2de8a4..f3520efcc6 100644 --- a/resources/js/angular/views/partials/build.html +++ b/resources/js/angular/views/partials/build.html @@ -325,7 +325,7 @@ - +
diff --git a/resources/js/vue/components/BuildSummaryPage.vue b/resources/js/vue/components/BuildSummaryPage.vue index 1c93385fdc..082ebb7ad2 100644 --- a/resources/js/vue/components/BuildSummaryPage.vue +++ b/resources/js/vue/components/BuildSummaryPage.vue @@ -516,6 +516,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount passedTestsCount site { id @@ -540,6 +541,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } nextBuild: build(id: $nextId) @include(if: $hasNext) { id @@ -549,6 +551,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } } `, @@ -577,7 +580,7 @@ export default { nerrors: Math.max(0, prev.buildErrorsCount), nwarnings: Math.max(0, prev.buildWarningsCount), ntestfailed: Math.max(0, prev.failedTestsCount), - ntestnotrun: Math.max(0, prev.notRunTestsCount), + ntestnotrun: Math.max(0, prev.notRunTestsWarningCount), }; } else { this.cdash.previousbuild = null; @@ -592,7 +595,7 @@ export default { nerrors: Math.max(0, next.buildErrorsCount), nwarnings: Math.max(0, next.buildWarningsCount), ntestfailed: Math.max(0, next.failedTestsCount), - ntestnotrun: Math.max(0, next.notRunTestsCount), + ntestnotrun: Math.max(0, next.notRunTestsWarningCount), }; } else { this.cdash.nextbuild = null; @@ -617,7 +620,7 @@ export default { this.cdash.test = { nfailed: Math.max(0, build.failedTestsCount), - nnotrun: Math.max(0, build.notRunTestsCount), + nnotrun: Math.max(0, build.notRunTestsWarningCount), npassed: Math.max(0, build.passedTestsCount), }; diff --git a/resources/js/vue/components/BuildSummaryPage/BuildTimelineCard.vue b/resources/js/vue/components/BuildSummaryPage/BuildTimelineCard.vue index f09dbc5516..1ada74a51c 100644 --- a/resources/js/vue/components/BuildSummaryPage/BuildTimelineCard.vue +++ b/resources/js/vue/components/BuildSummaryPage/BuildTimelineCard.vue @@ -201,6 +201,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount passedTestsCount } } @@ -278,7 +279,7 @@ export default { if (this.build.failedTestsCount > 0) { return 'error'; } - if (this.build.notRunTestsCount > 0) { + if (this.build.notRunTestsWarningCount > 0) { return 'warning'; } return 'success'; diff --git a/resources/js/vue/components/BuildTestsPage.vue b/resources/js/vue/components/BuildTestsPage.vue index 788c83b846..1242aae130 100644 --- a/resources/js/vue/components/BuildTestsPage.vue +++ b/resources/js/vue/components/BuildTestsPage.vue @@ -73,6 +73,7 @@ import LoadingIndicator from './shared/LoadingIndicator.vue'; import BuildSummaryCard from './shared/BuildSummaryCard.vue'; import BuildSidebar from './shared/BuildSidebar.vue'; import { DateTime } from 'luxon'; +import { testStatusToColorClass } from './shared/TestDisplay'; const TEST_QUERY = gql` query( @@ -325,14 +326,15 @@ export default { value: edge.node.status, text: this.humanReadableTestStatus(edge.node.status), href: `${this.$baseURL}/tests/${edge.node.id}`, - classes: [this.testStatusToColorClass(edge.node.status)], + classes: [testStatusToColorClass(edge.node.status, edge.node.details)], + testId: `test-status-${edge.node.id}`, }, subProject: edge.subProject ?? '', timeStatus: { value: edge.node.timeStatusCategory, text: this.humanReadableTestStatus(edge.node.timeStatusCategory), href: `${this.$baseURL}/tests/${edge.node.id}?graph=time`, - classes: [this.testStatusToColorClass(edge.node.timeStatusCategory)], + classes: [testStatusToColorClass(edge.node.timeStatusCategory, edge.node.details)], }, history: { value: '', @@ -349,19 +351,6 @@ export default { }, methods: { - testStatusToColorClass(status) { - switch (status) { - case 'PASSED': - return 'normal'; - case 'FAILED': - return 'error'; - case 'NOT_RUN': - return 'warning'; - default: - return ''; - } - }, - humanReadableTestStatus(status) { switch (status) { case 'PASSED': diff --git a/resources/js/vue/components/TestIdPage.vue b/resources/js/vue/components/TestIdPage.vue index 12cb8f31e1..9b6a010873 100644 --- a/resources/js/vue/components/TestIdPage.vue +++ b/resources/js/vue/components/TestIdPage.vue @@ -334,6 +334,7 @@ import { faChartLine, faLink, } from '@fortawesome/free-solid-svg-icons'; +import { isAcceptableNotRun } from './shared/TestDisplay'; export default { name: 'TestIdPage', @@ -517,6 +518,14 @@ export default { }, testStatusPillClass() { + if (!this.test) { + return 'tw-bg-neutral tw-text-neutral-content'; + } + + if (this.test.status === 'NOT_RUN' && isAcceptableNotRun(this.test.details)) { + return 'tw-bg-success tw-text-success-content'; + } + switch (this.test.status) { case 'PASSED': return 'tw-bg-success tw-text-success-content'; diff --git a/resources/js/vue/components/shared/DataTable.vue b/resources/js/vue/components/shared/DataTable.vue index a51fa1f358..ecc7e322b0 100644 --- a/resources/js/vue/components/shared/DataTable.vue +++ b/resources/js/vue/components/shared/DataTable.vue @@ -54,7 +54,7 @@ :class="(row[column.name]?.classes ?? []).concat(column.expand ? [] : ['shrink'])" class="table-cell" data-cy="data-table-cell" - :data-test="testId + '-cell'" + :data-test="row[column.name]?.testId ?? (testId + '-cell')" >