Handle skipped tests in "Not run" column

app/Models/Build.php

@@ -33,6 +33,7 @@
  * @property int $buildwarnings
  * @property int $buildduration
  * @property int $testnotrun
+ * @property int $testnotrunwarning
  * @property int $testfailed
  * @property int $testpassed
  * @property int $testtimestatusfailed
@@ -84,6 +85,7 @@ class Build extends Model
         'buildwarnings',
         'buildduration',
         'testnotrun',
+        'testnotrunwarning',
         'testfailed',
         'testpassed',
         'testtimestatusfailed',
@@ -118,6 +120,7 @@ class Build extends Model
         'buildwarnings' => 'integer',
         'buildduration' => 'integer',
         'testnotrun' => 'integer',
+        'testnotrunwarning' => 'integer',
         'testfailed' => 'integer',
         'testpassed' => 'integer',
         'testduration' => 'integer',

app/Models/Test.php

@@ -32,6 +32,8 @@
  * @property ?Carbon $starttime
  * @property TestTimeStatusCategory $timestatuscategory
  *
+ * @method static Builder<Test> notRunWarning()
+ *
  * @mixin Builder<Test>
  */
 class Test extends Model
@@ -84,6 +86,22 @@ class Test extends Model
         'starttime' => 'datetime',
     ];
 
+    /**
+     * Tests which did not run for a reason other than being explicitly disabled.
+     *
+     * CTest reports a completion status of "Disabled" for tests marked DISABLED.
+     *
+     * @param Builder<self> $query
+     */
+    public function scopeNotRunWarning(Builder $query): void
+    {
+        $query->where('status', self::NOTRUN)
+            ->where(static function (Builder $query): void {
+                $query->whereNull('details')
+                    ->orWhere('details', '!=', self::DISABLED);
+            });
+    }
+
     /**
      * @return BelongsTo<Build, $this>
      */

app/cdash/app/Controller/Api/Index.php

@@ -17,6 +17,7 @@
 
 namespace CDash\Controller\Api;
 
+use App\Models\Test;
 use App\Utils\TestingDay;
 use CDash\Database;
 use CDash\Model\BuildGroup;
@@ -345,6 +346,7 @@ public function getIndexSelect(): string
                 b.buildwarnings AS countbuildwarnings,
                 b.buildduration,
                 b.testnotrun AS counttestsnotrun,
+                b.testnotrunwarning AS counttestsnotrunwarning,
                 b.testfailed AS counttestsfailed,
                 b.testpassed AS counttestspassed,
                 b.testtimestatusfailed AS countteststimestatusfailed,
@@ -656,6 +658,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
         $selected_build_warnings = 0;
         $selected_build_duration = 0;
         $selected_tests_not_run = 0;
+        $selected_tests_not_run_warning = 0;
         $selected_tests_failed = 0;
         $selected_tests_passed = 0;
         $selected_proc_time = 0;
@@ -685,6 +688,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
                                           b.starttime,
                                           b.endtime,
                                           testnotrun,
+                                          testnotrunwarning,
                                           testfailed,
                                           testpassed,
                                           testduration,
@@ -705,6 +709,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
                     $selected_build_warnings += max(0, $select_array->buildwarnings);
                     $selected_build_duration += max(0, $select_array->buildduration);
                     $selected_tests_not_run += max(0, $select_array->testnotrun);
+                    $selected_tests_not_run_warning += max(0, $select_array->testnotrunwarning);
                     $selected_tests_failed += max(0, $select_array->testfailed);
                     $selected_tests_passed += max(0, $select_array->testpassed);
                     $selected_proc_time += max(0, $select_array->testtime);
@@ -1013,6 +1018,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
             $test_response = [];
 
             $nnotrun = $build_array['counttestsnotrun'];
+            $nnotrunwarning = $build_array['counttestsnotrunwarning'];
             $nfail = $build_array['counttestsfailed'];
             $npass = $build_array['counttestspassed'];
             $proc_time = $build_array['testtime'];
@@ -1022,11 +1028,13 @@ public function generateBuildResponseFromRow(array $build_array): array|false
             if (!$this->childView) {
                 if ($this->includeSubProjects) {
                     $nnotrun = $selected_tests_not_run;
+                    $nnotrunwarning = $selected_tests_not_run_warning;
                     $nfail = $selected_tests_failed;
                     $npass = $selected_tests_passed;
                     $proc_time = $selected_proc_time;
                 } else {
                     $nnotrun -= $selected_tests_not_run;
+                    $nnotrunwarning -= $selected_tests_not_run_warning;
                     $nfail -= $selected_tests_failed;
                     $npass -= $selected_tests_passed;
                     $proc_time -= $selected_proc_time;
@@ -1055,7 +1063,8 @@ public function generateBuildResponseFromRow(array $build_array): array|false
                 $labels_result = DB::select("
                                      SELECT
                                          b2t.status,
-                                         b2t.newstatus
+                                         b2t.newstatus,
+                                         b2t.details
                                      FROM build2test AS b2t
                                      INNER JOIN label2test AS l2t ON l2t.testid = b2t.id
                                      WHERE
@@ -1065,6 +1074,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
                                  ", array_merge([$buildid], $this->labelIds));
 
                 $nnotrun = 0;
+                $nnotrunwarning = 0;
                 $nfail = 0;
                 $npass = 0;
                 $test_response['nfaildiffp'] = 0;
@@ -1089,6 +1099,9 @@ public function generateBuildResponseFromRow(array $build_array): array|false
                             break;
                         case 'notrun':
                             $nnotrun++;
+                            if ($label_row->details !== Test::DISABLED) {
+                                $nnotrunwarning++;
+                            }
                             if ((int) $label_row->newstatus === 1) {
                                 $test_response['nnotrundiffp']++;
                             }
@@ -1098,6 +1111,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false
             }
 
             $test_response['notrun'] = $nnotrun;
+            $test_response['notrunwarning'] = max(0, $nnotrunwarning);
             $test_response['fail'] = $nfail;
             $test_response['pass'] = $npass;
 

app/cdash/app/Controller/Api/QueryTests.php

@@ -19,6 +19,7 @@
 
 use App\Models\PinnedTestMeasurement;
 use App\Models\Project as EloquentProject;
+use App\Models\Test;
 use App\Models\TestMeasurement;
 use CDash\Database;
 use CDash\Model\Build;
@@ -423,7 +424,7 @@ public function getResponse(): array
 
                 case 'notrun':
                     $test['status'] = 'Not Run';
-                    $test['statusclass'] = 'warning';
+                    $test['statusclass'] = $row->details === Test::DISABLED ? 'normal' : 'warning';
                     break;
             }
 

app/cdash/app/Model/Build.php

@@ -773,7 +773,7 @@ public function Save()
     /** Helper function for test number accessors. */
     private function GetNumberOfTestsByField(string $field): int
     {
-        if (!in_array($field, ['testpassed', 'testfailed', 'testnotrun'], true)) {
+        if (!in_array($field, ['testpassed', 'testfailed', 'testnotrun', 'testnotrunwarning'], true)) {
             throw new InvalidArgumentException('Invalid field specified.');
         }
 
@@ -799,20 +799,36 @@ public function GetNumberOfNotRunTests(): int|false
         return $this->GetNumberOfTestsByField('testnotrun');
     }
 
+    /** Get number of not run tests which were not explicitly disabled */
+    public function GetNumberOfNotRunWarningTests(): int|false
+    {
+        return $this->GetNumberOfTestsByField('testnotrunwarning');
+    }
+
     /** Update the test numbers */
     public function UpdateTestNumbers(int $numberTestsPassed, int $numberTestsFailed, int $numberTestsNotRun): void
     {
         $this->TestFailedCount = $numberTestsFailed;
 
+        // Disabled tests aren't worth warning about, so they are tallied separately from
+        // the other not-run tests.  The tests for this build have all been recorded by
+        // now, so they can be counted directly.
+        $numberTestsNotRunWarning = EloquentBuild::find((int) $this->Id)
+            ?->tests()
+            ->notRunWarning()
+            ->count() ?? 0;
+
         // If this is a subproject build, we also have to update its parents test numbers.
         $newFailed = $numberTestsFailed - $this->GetNumberOfFailedTests();
         $newNotRun = $numberTestsNotRun - $this->GetNumberOfNotRunTests();
         $newPassed = $numberTestsPassed - $this->GetNumberOfPassedTests();
+        $newNotRunWarning = $numberTestsNotRunWarning - $this->GetNumberOfNotRunWarningTests();
         $this->SetParentId($this->LookupParentBuildId());
-        $this->UpdateParentTestNumbers($newFailed, $newNotRun, $newPassed);
+        $this->UpdateParentTestNumbers($newFailed, $newNotRun, $newPassed, $newNotRunWarning);
 
         EloquentBuild::whereKey($this->Id)->update([
             'testnotrun' => $numberTestsNotRun,
+            'testnotrunwarning' => $numberTestsNotRunWarning,
             'testfailed' => $numberTestsFailed,
             'testpassed' => $numberTestsPassed,
         ]);
@@ -1612,22 +1628,24 @@ public function UpdateBuild($buildid, $newErrors, $newWarnings): void
     }
 
     /** Update the testing numbers for our parent build. */
-    private function UpdateParentTestNumbers(int $newFailed, int $newNotRun, int $newPassed): void
+    private function UpdateParentTestNumbers(int $newFailed, int $newNotRun, int $newPassed, int $newNotRunWarning): void
     {
         if ($this->ParentId < 1) {
             return;
         }
 
-        DB::transaction(function () use ($newFailed, $newNotRun, $newPassed): void {
+        DB::transaction(function () use ($newFailed, $newNotRun, $newPassed, $newNotRunWarning): void {
             $parent = EloquentBuild::findOrFail($this->ParentId);
 
             // Don't let the -1 default value screw up our math.
             $parent_testfailed = self::ConvertMissingToZero($parent->testfailed);
             $parent_testnotrun = self::ConvertMissingToZero($parent->testnotrun);
+            $parent_testnotrunwarning = self::ConvertMissingToZero($parent->testnotrunwarning);
             $parent_testpassed = self::ConvertMissingToZero($parent->testpassed);
 
             $parent->update([
                 'testnotrun' => $newNotRun + $parent_testnotrun,
+                'testnotrunwarning' => $newNotRunWarning + $parent_testnotrunwarning,
                 'testfailed' => $newFailed + $parent_testfailed,
                 'testpassed' => $newPassed + $parent_testpassed,
             ]);

app/cdash/tests/test_testhistory.php

@@ -134,6 +134,9 @@ private function generateXML($mm, $timestamp, $include_sporadic, $flaky_passed)
                   <FullName>./notrun</FullName>
                   <FullCommandLine></FullCommandLine>
                   <Results>
+                    <NamedMeasurement type="text/string" name="Exit Code">
+                      <Value>Skipped</Value>
+                    </NamedMeasurement>
                     <NamedMeasurement type="text/string" name="Command Line">
                       <Value></Value>
                     </NamedMeasurement>

database/migrations/2026_08_25_184512_build_testnotrunwarning.php

@@ -0,0 +1,53 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+return new class extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        DB::statement("ALTER TABLE build ADD COLUMN testnotrunwarning smallint DEFAULT '-1'::smallint");
+
+        // Builds which reported test results have no warnings until proven otherwise.
+        DB::update('UPDATE build SET testnotrunwarning = 0 WHERE testnotrun >= 0');
+
+        DB::update("
+            UPDATE build
+            SET testnotrunwarning = counted.total
+            FROM (
+                SELECT buildid, COUNT(*) AS total
+                FROM build2test
+                WHERE status = 'notrun'
+                    AND (details IS NULL OR details != 'Disabled')
+                GROUP BY buildid
+            ) AS counted
+            WHERE build.id = counted.buildid
+        ");
+
+        // Parent builds aggregate the results of their children.
+        DB::update("
+            UPDATE build
+            SET testnotrunwarning = GREATEST(build.testnotrunwarning, 0) + counted.total
+            FROM (
+                SELECT child.parentid, COUNT(*) AS total
+                FROM build2test
+                INNER JOIN build AS child ON child.id = build2test.buildid
+                WHERE child.parentid > 0
+                    AND build2test.status = 'notrun'
+                    AND (build2test.details IS NULL OR build2test.details != 'Disabled')
+                GROUP BY child.parentid
+            ) AS counted
+            WHERE build.id = counted.parentid
+        ");
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+    }
+};

graphql/schema.graphql

@@ -856,6 +856,11 @@ type Build {
   "The number of tests not run for this build."
   notRunTestsCount: Int @rename(attribute: "testnotrun")
 
+  """
+  The number of not-run tests counted as warnings (excludes tests which were explicitly disabled).
+  """
+  notRunTestsWarningCount: Int @rename(attribute: "testnotrunwarning")
+
   "The duration of the test step in seconds."
   testDuration: Int! @rename(attribute: "testduration")
 

phpstan-baseline.neon

@@ -7848,7 +7848,7 @@ parameters:
 		-
 			rawMessage: 'Only numeric types are allowed in -, int|false given on the right side.'
 			identifier: minus.rightNonNumeric
-			count: 3
+			count: 4
 			path: app/cdash/app/Model/Build.php
 
 		-

resources/js/angular/views/partials/build.html

@@ -325,7 +325,7 @@
   </div>
 </td>
 
-<td ng-if="::buildgroup.hastestdata" align="center" ng-class="::{'warning': build.test.notrun > 0, 'normal': build.test.notrun == 0}">
+<td ng-if="::buildgroup.hastestdata" align="center" ng-class="::{'warning': build.test.notrunwarning > 0, 'normal': build.test.notrunwarning == 0}">
   <div ng-if="::build.hastest"
        ng-class="::{'valuewithsub': build.test.nnotrundiffp > 0 || build.test.nnotrundiffn > 0}">
     <a class="cdash-link" ng-href="{{ 'builds/' + build.id + '/tests?filters=%7B%22all%22%3A%5B%7B%22eq%22%3A%7B%22status%22%3A%22NOT_RUN%22%7D%7D%5D%7D' }}">

resources/js/vue/components/BuildSummaryPage.vue

@@ -516,6 +516,7 @@ export default {
             buildWarningsCount
             failedTestsCount
             notRunTestsCount
+            notRunTestsWarningCount
             passedTestsCount
             site {
               id
@@ -540,6 +541,7 @@ export default {
             buildWarningsCount
             failedTestsCount
             notRunTestsCount
+            notRunTestsWarningCount
           }
           nextBuild: build(id: $nextId) @include(if: $hasNext) {
             id
@@ -549,6 +551,7 @@ export default {
             buildWarningsCount
             failedTestsCount
             notRunTestsCount
+            notRunTestsWarningCount
           }
         }
       `,
@@ -577,7 +580,7 @@ export default {
             nerrors: Math.max(0, prev.buildErrorsCount),
             nwarnings: Math.max(0, prev.buildWarningsCount),
             ntestfailed: Math.max(0, prev.failedTestsCount),
-            ntestnotrun: Math.max(0, prev.notRunTestsCount),
+            ntestnotrun: Math.max(0, prev.notRunTestsWarningCount),
           };
         } else {
           this.cdash.previousbuild = null;
@@ -592,7 +595,7 @@ export default {
             nerrors: Math.max(0, next.buildErrorsCount),
             nwarnings: Math.max(0, next.buildWarningsCount),
             ntestfailed: Math.max(0, next.failedTestsCount),
-            ntestnotrun: Math.max(0, next.notRunTestsCount),
+            ntestnotrun: Math.max(0, next.notRunTestsWarningCount),
           };
         } else {
           this.cdash.nextbuild = null;
@@ -617,7 +620,7 @@ export default {
 
         this.cdash.test = {
           nfailed: Math.max(0, build.failedTestsCount),
-          nnotrun: Math.max(0, build.notRunTestsCount),
+          nnotrun: Math.max(0, build.notRunTestsWarningCount),
           npassed: Math.max(0, build.passedTestsCount),
         };