From 67d08f57052889b8d13813fa93776bc4c4db2ba3 Mon Sep 17 00:00:00 2001 From: Julien Jomier Date: Tue, 26 May 2026 20:10:02 -0400 Subject: [PATCH 01/11] Handle skipped tests in "Not run" column This commit introduces a new `notRunSkippedDetailsRegex` property to the Project model, allowing users to define regex patterns for not-run test details. A corresponding validation rule is added to ensure the regex is valid. The Build model now includes a method to count not-run tests that do not match the specified patterns, and the GraphQL schema is updated to expose this functionality. Additionally, UI components are modified to display the warning count and adjust test status colors based on the new regex patterns. Tests are also added to verify the new functionality. --- .../UpdateProjectInputValidator.php | 6 + app/Models/Build.php | 19 +++ app/Models/Project.php | 4 + app/Rules/NotRunSkippedDetailsRegexRule.php | 26 ++++ app/Utils/TestDisplay.php | 142 ++++++++++++++++++ app/cdash/app/Controller/Api/Index.php | 14 ++ app/cdash/app/Controller/Api/QueryTests.php | 8 +- app/cdash/tests/test_testhistory.php | 3 + ...otrun_skipped_details_regex_to_project.php | 15 ++ graphql/schema.graphql | 19 +++ .../js/angular/views/partials/build.html | 2 +- .../js/vue/components/BuildSummaryPage.vue | 9 +- .../js/vue/components/BuildTestsPage.vue | 33 ++-- .../components/ProjectSettings/GeneralTab.vue | 13 +- .../js/vue/components/ProjectSettingsPage.vue | 17 +++ resources/js/vue/components/TestIdPage.vue | 16 ++ .../js/vue/components/shared/FormSection.vue | 11 +- .../js/vue/components/shared/TestDisplay.js | 99 ++++++++++++ tests/Browser/Pages/BuildTestsPageTest.php | 46 ++++++ tests/Feature/GraphQL/BuildTypeTest.php | 54 +++++++ .../GraphQL/Mutations/UpdateProjectTest.php | 26 ++++ tests/Feature/GraphQL/ProjectTypeTest.php | 2 + tests/Unit/Utils/TestDisplayTest.php | 82 ++++++++++ 23 files changed, 644 insertions(+), 22 deletions(-) create mode 100644 app/Rules/NotRunSkippedDetailsRegexRule.php create mode 100644 app/Utils/TestDisplay.php create mode 100644 database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php create mode 100644 resources/js/vue/components/shared/TestDisplay.js create mode 100644 tests/Unit/Utils/TestDisplayTest.php diff --git a/app/GraphQL/Validators/UpdateProjectInputValidator.php b/app/GraphQL/Validators/UpdateProjectInputValidator.php index 0dc314ed3a..9d176a6436 100644 --- a/app/GraphQL/Validators/UpdateProjectInputValidator.php +++ b/app/GraphQL/Validators/UpdateProjectInputValidator.php @@ -3,6 +3,7 @@ namespace App\GraphQL\Validators; use App\Models\Project; +use App\Rules\NotRunSkippedDetailsRegexRule; use App\Rules\ProjectAuthenticateSubmissionsRule; use App\Rules\ProjectNameRule; use App\Rules\ProjectVisibilityRule; @@ -25,6 +26,11 @@ public function rules(): array 'authenticateSubmissions' => [ new ProjectAuthenticateSubmissionsRule(), ], + 'notRunSkippedDetailsRegex' => [ + 'nullable', + 'string', + new NotRunSkippedDetailsRegexRule(), + ], ]; } diff --git a/app/Models/Build.php b/app/Models/Build.php index a0d8fe0185..98402a0684 100644 --- a/app/Models/Build.php +++ b/app/Models/Build.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Utils\TestDisplay; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -400,4 +401,22 @@ public function percentCoverageForPath(string $path): ?float return ($loctested / $total_lines) * 100; } + + /** + * The number of not-run tests whose details do not match the project's skipped pattern. + */ + public function notRunTestsWarningCount(): int + { + $patterns = $this->project?->notrun_skipped_details_regex + ?? TestDisplay::DEFAULT_NOTRUN_SKIPPED_DETAILS_REGEX; + + $count = 0; + foreach ($this->tests()->where('status', Test::NOTRUN)->pluck('details') as $details) { + if (!TestDisplay::isAcceptableNotRun($details, $patterns)) { + $count++; + } + } + + return $count; + } } diff --git a/app/Models/Project.php b/app/Models/Project.php index 11b7987ed4..c102b9200e 100644 --- a/app/Models/Project.php +++ b/app/Models/Project.php @@ -3,6 +3,7 @@ namespace App\Models; use App\Enums\ProjectRole; +use App\Utils\TestDisplay; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; @@ -49,6 +50,7 @@ * @property ?string $banner * @property ?string $logoUrl * @property ?string $cmakeprojectroot + * @property string $notrun_skipped_details_regex * * @method Builder forUser() * @method Builder administeredByUser() @@ -95,6 +97,7 @@ class Project extends Model 'ldapfilter', 'banner', 'cmakeprojectroot', + 'notrun_skipped_details_regex', ]; protected $casts = [ @@ -137,6 +140,7 @@ class Project extends Model 'uploadquota' => 10, 'showcoveragecode' => true, 'authenticatesubmissions' => false, + 'notrun_skipped_details_regex' => TestDisplay::DEFAULT_NOTRUN_SKIPPED_DETAILS_REGEX, ]; public const ACCESS_PRIVATE = 0; diff --git a/app/Rules/NotRunSkippedDetailsRegexRule.php b/app/Rules/NotRunSkippedDetailsRegexRule.php new file mode 100644 index 0000000000..d6d9c402dd --- /dev/null +++ b/app/Rules/NotRunSkippedDetailsRegexRule.php @@ -0,0 +1,26 @@ + + */ + public static function parsePatterns(?string $patternsText): array + { + if ($patternsText === null || trim($patternsText) === '') { + return []; + } + + $patterns = []; + foreach (preg_split("/\r\n|\n|\r/", $patternsText) as $line) { + $line = trim($line); + if ($line !== '') { + $patterns[] = $line; + } + } + + return $patterns; + } + + public static function patternToPcre(string $pattern): string + { + $pattern = trim($pattern); + if ($pattern === '') { + return ''; + } + + if (str_starts_with($pattern, '/')) { + return $pattern; + } + + $parts = preg_split('/(\*|\?)/', $pattern, -1, PREG_SPLIT_DELIM_CAPTURE); + if ($parts === false) { + return ''; + } + + $regex = ''; + foreach ($parts as $part) { + if ($part === '*') { + $regex .= '.*'; + } elseif ($part === '?') { + $regex .= '.'; + } else { + $regex .= preg_quote($part, '/'); + } + } + + return '/(?i)' . $regex . '/'; + } + + public static function detailsMatchesSkippedPattern(?string $details, ?string $patternsText): bool + { + if ($details === null || $details === '') { + return false; + } + + foreach (self::parsePatterns($patternsText) as $pattern) { + $pcre = self::patternToPcre($pattern); + if ($pcre === '') { + continue; + } + + $result = @preg_match($pcre, $details); + if ($result === 1) { + return true; + } + } + + return false; + } + + public static function isAcceptableNotRun(?string $details, ?string $patternsText): bool + { + return self::detailsMatchesSkippedPattern($details, $patternsText); + } + + public static function statusColorClass(string $status, ?string $details, ?string $patternsText): string + { + if ($status === Test::NOTRUN && self::isAcceptableNotRun($details, $patternsText)) { + return 'normal'; + } + + return match ($status) { + Test::PASSED => 'normal', + Test::FAILED => 'error', + Test::NOTRUN => 'warning', + default => '', + }; + } + + public static function statusTextColorClass(string $status, ?string $details, ?string $patternsText): string + { + return match (self::statusColorClass($status, $details, $patternsText)) { + 'normal' => 'normal-text', + 'warning' => 'warning-text', + 'error' => 'error-text', + default => '', + }; + } + + public static function graphqlStatusColorClass(string $status, ?string $details, ?string $patternsText): string + { + $dbStatus = match ($status) { + 'NOT_RUN' => Test::NOTRUN, + 'PASSED' => Test::PASSED, + 'FAILED' => Test::FAILED, + default => strtolower($status), + }; + + return self::statusColorClass($dbStatus, $details, $patternsText); + } + + public static function isValidPatternsText(?string $patternsText): bool + { + foreach (self::parsePatterns($patternsText) as $pattern) { + $pcre = self::patternToPcre($pattern); + if ($pcre === '') { + return false; + } + + set_error_handler(static fn (): bool => true); + $result = preg_match($pcre, ''); + restore_error_handler(); + + if ($result === false) { + return false; + } + } + + return true; + } +} diff --git a/app/cdash/app/Controller/Api/Index.php b/app/cdash/app/Controller/Api/Index.php index b70f94fd90..7542f771bf 100644 --- a/app/cdash/app/Controller/Api/Index.php +++ b/app/cdash/app/Controller/Api/Index.php @@ -17,6 +17,7 @@ namespace CDash\Controller\Api; +use App\Models\Build as EloquentBuild; use App\Utils\TestingDay; use CDash\Database; use CDash\Model\BuildGroup; @@ -1098,6 +1099,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false } $test_response['notrun'] = $nnotrun; + $test_response['notrunwarning'] = $this->computeNotRunTestsWarningCount((int) $buildid, $nnotrun); $test_response['fail'] = $nfail; $test_response['pass'] = $npass; @@ -1555,4 +1557,16 @@ public function recordGenerationTime(array &$response): void { $this->pageTimer->end($response); } + + /** + * Count not-run tests that should display as warnings on the index page. + */ + private function computeNotRunTestsWarningCount(int $buildid, int $notRunCount): int + { + if ($notRunCount <= 0) { + return 0; + } + + return EloquentBuild::find($buildid)?->notRunTestsWarningCount() ?? 0; + } } diff --git a/app/cdash/app/Controller/Api/QueryTests.php b/app/cdash/app/Controller/Api/QueryTests.php index 7f00f75295..0899abe869 100644 --- a/app/cdash/app/Controller/Api/QueryTests.php +++ b/app/cdash/app/Controller/Api/QueryTests.php @@ -20,6 +20,7 @@ use App\Models\PinnedTestMeasurement; use App\Models\Project as EloquentProject; use App\Models\TestMeasurement; +use App\Utils\TestDisplay; use CDash\Database; use CDash\Model\Build; use CDash\Model\Project; @@ -375,6 +376,7 @@ public function getResponse(): array ", $query_params); // Rows of test data to be displayed to the user. + $notRunSkippedDetailsRegex = EloquentProject::findOrFail($this->project->Id)->notrun_skipped_details_regex; $tests = []; foreach ($rows as $row) { $test = []; @@ -423,7 +425,11 @@ public function getResponse(): array case 'notrun': $test['status'] = 'Not Run'; - $test['statusclass'] = 'warning'; + $test['statusclass'] = TestDisplay::statusColorClass( + 'notrun', + $row->details, + $notRunSkippedDetailsRegex, + ); break; } diff --git a/app/cdash/tests/test_testhistory.php b/app/cdash/tests/test_testhistory.php index 7f06fb55fc..e9bd2f3a24 100644 --- a/app/cdash/tests/test_testhistory.php +++ b/app/cdash/tests/test_testhistory.php @@ -134,6 +134,9 @@ private function generateXML($mm, $timestamp, $include_sporadic, $flaky_passed) ./notrun + + Skipped + diff --git a/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php b/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php new file mode 100644 index 0000000000..e2a61e4547 --- /dev/null +++ b/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php @@ -0,0 +1,15 @@ + - +
diff --git a/resources/js/vue/components/BuildSummaryPage.vue b/resources/js/vue/components/BuildSummaryPage.vue index 1c93385fdc..082ebb7ad2 100644 --- a/resources/js/vue/components/BuildSummaryPage.vue +++ b/resources/js/vue/components/BuildSummaryPage.vue @@ -516,6 +516,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount passedTestsCount site { id @@ -540,6 +541,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } nextBuild: build(id: $nextId) @include(if: $hasNext) { id @@ -549,6 +551,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } } `, @@ -577,7 +580,7 @@ export default { nerrors: Math.max(0, prev.buildErrorsCount), nwarnings: Math.max(0, prev.buildWarningsCount), ntestfailed: Math.max(0, prev.failedTestsCount), - ntestnotrun: Math.max(0, prev.notRunTestsCount), + ntestnotrun: Math.max(0, prev.notRunTestsWarningCount), }; } else { this.cdash.previousbuild = null; @@ -592,7 +595,7 @@ export default { nerrors: Math.max(0, next.buildErrorsCount), nwarnings: Math.max(0, next.buildWarningsCount), ntestfailed: Math.max(0, next.failedTestsCount), - ntestnotrun: Math.max(0, next.notRunTestsCount), + ntestnotrun: Math.max(0, next.notRunTestsWarningCount), }; } else { this.cdash.nextbuild = null; @@ -617,7 +620,7 @@ export default { this.cdash.test = { nfailed: Math.max(0, build.failedTestsCount), - nnotrun: Math.max(0, build.notRunTestsCount), + nnotrun: Math.max(0, build.notRunTestsWarningCount), npassed: Math.max(0, build.passedTestsCount), }; diff --git a/resources/js/vue/components/BuildTestsPage.vue b/resources/js/vue/components/BuildTestsPage.vue index 788c83b846..7422904b7b 100644 --- a/resources/js/vue/components/BuildTestsPage.vue +++ b/resources/js/vue/components/BuildTestsPage.vue @@ -73,6 +73,7 @@ import LoadingIndicator from './shared/LoadingIndicator.vue'; import BuildSummaryCard from './shared/BuildSummaryCard.vue'; import BuildSidebar from './shared/BuildSidebar.vue'; import { DateTime } from 'luxon'; +import { testStatusToColorClass } from './shared/TestDisplay'; const TEST_QUERY = gql` query( @@ -82,6 +83,9 @@ const TEST_QUERY = gql` ) { build(id: $buildid) { id + project { + notRunSkippedDetailsRegex + } tests(filters: $filters, first: 1000000) { edges { node { @@ -205,6 +209,11 @@ export default { }); return tests; }, + result({ data }) { + if (data?.build?.project?.notRunSkippedDetailsRegex !== undefined) { + this.notRunSkippedDetailsRegex = data.build.project.notRunSkippedDetailsRegex; + } + }, variables() { return { buildid: this.buildId, @@ -237,6 +246,11 @@ export default { }); return tests; }, + result({ data }) { + if (data?.build?.project?.notRunSkippedDetailsRegex !== undefined) { + this.notRunSkippedDetailsRegex = data.build.project.notRunSkippedDetailsRegex; + } + }, variables() { return { buildid: this.previousBuildId, @@ -259,6 +273,7 @@ export default { data() { return { changedFilters: JSON.parse(JSON.stringify(this.initialFilters)), + notRunSkippedDetailsRegex: '*skip*', }; }, @@ -308,6 +323,7 @@ export default { }, formattedTestRows() { + const notRunSkippedDetailsRegex = this.notRunSkippedDetailsRegex; return this.filteredTests?.map((edge) => { return { name: { @@ -325,14 +341,14 @@ export default { value: edge.node.status, text: this.humanReadableTestStatus(edge.node.status), href: `${this.$baseURL}/tests/${edge.node.id}`, - classes: [this.testStatusToColorClass(edge.node.status)], + classes: [testStatusToColorClass(edge.node.status, edge.node.details, notRunSkippedDetailsRegex)], }, subProject: edge.subProject ?? '', timeStatus: { value: edge.node.timeStatusCategory, text: this.humanReadableTestStatus(edge.node.timeStatusCategory), href: `${this.$baseURL}/tests/${edge.node.id}?graph=time`, - classes: [this.testStatusToColorClass(edge.node.timeStatusCategory)], + classes: [testStatusToColorClass(edge.node.timeStatusCategory, edge.node.details, notRunSkippedDetailsRegex)], }, history: { value: '', @@ -349,19 +365,6 @@ export default { }, methods: { - testStatusToColorClass(status) { - switch (status) { - case 'PASSED': - return 'normal'; - case 'FAILED': - return 'error'; - case 'NOT_RUN': - return 'warning'; - default: - return ''; - } - }, - humanReadableTestStatus(status) { switch (status) { case 'PASSED': diff --git a/resources/js/vue/components/ProjectSettings/GeneralTab.vue b/resources/js/vue/components/ProjectSettings/GeneralTab.vue index 2e68a274c6..96d9468323 100644 --- a/resources/js/vue/components/ProjectSettings/GeneralTab.vue +++ b/resources/js/vue/components/ProjectSettings/GeneralTab.vue @@ -362,7 +362,8 @@ + + @@ -541,6 +550,7 @@ export default { fileUploadLimit: 50, showCoverageCode: true, banner: '', + notRunSkippedDetailsRegex: '*skip*', }, validationErrors: {}, fatalError: null, @@ -584,6 +594,7 @@ export default { fileUploadLimit showCoverageCode banner + notRunSkippedDetailsRegex } } `, diff --git a/resources/js/vue/components/ProjectSettingsPage.vue b/resources/js/vue/components/ProjectSettingsPage.vue index 66a6d89bc7..eeb4012426 100644 --- a/resources/js/vue/components/ProjectSettingsPage.vue +++ b/resources/js/vue/components/ProjectSettingsPage.vue @@ -97,5 +97,22 @@ export default { currentSection: 'general', }; }, + + mounted() { + this.navigateToHashSection(); + }, + + methods: { + navigateToHashSection() { + if (window.location.hash !== '#Testing') { + return; + } + + this.currentSection = 'general'; + this.$nextTick(() => { + document.getElementById('Testing')?.scrollIntoView(); + }); + }, + }, }; diff --git a/resources/js/vue/components/TestIdPage.vue b/resources/js/vue/components/TestIdPage.vue index 12cb8f31e1..395e2beffb 100644 --- a/resources/js/vue/components/TestIdPage.vue +++ b/resources/js/vue/components/TestIdPage.vue @@ -334,6 +334,7 @@ import { faChartLine, faLink, } from '@fortawesome/free-solid-svg-icons'; +import { detailsMatchesSkippedPattern } from './shared/TestDisplay'; export default { name: 'TestIdPage', @@ -382,6 +383,7 @@ export default { enableTestTiming testTimeStdMultiplier testTimeStdThreshold + notRunSkippedDetailsRegex } } } @@ -517,6 +519,20 @@ export default { }, testStatusPillClass() { + if (!this.test) { + return 'tw-bg-neutral tw-text-neutral-content'; + } + + if ( + this.test.status === 'NOT_RUN' + && detailsMatchesSkippedPattern( + this.test.details, + this.build?.project?.notRunSkippedDetailsRegex ?? '*skip*', + ) + ) { + return 'tw-bg-success tw-text-success-content'; + } + switch (this.test.status) { case 'PASSED': return 'tw-bg-success tw-text-success-content'; diff --git a/resources/js/vue/components/shared/FormSection.vue b/resources/js/vue/components/shared/FormSection.vue index c95a864cb6..3061ef9487 100644 --- a/resources/js/vue/components/shared/FormSection.vue +++ b/resources/js/vue/components/shared/FormSection.vue @@ -1,5 +1,8 @@