Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DOCS.md
Original file line number Diff line number Diff line change
Expand Up @@ -5210,7 +5210,7 @@ The same findings and gates always produce the same bytes. Case order is fixed,
| `findings` | Always | One case per drift finding. `classname` is `vg.findings`. `name` is `{ruleId} {location}` (` #2`, ` #3`, … on ties). No findings → one passing case, `no findings`. |
| `architecture` | `--fail-on architecture-finding` or `architecture-warning` | One failure per boundary row (`classname` `vg.architecture`), a passing `architecture` case when the gate is clean, or one failure when the gate could not run. |
| `security` | `--fail-on iac-finding` or `security-finding` | One case per security finding when the packs ran (`classname` `vg.security`). Findings below the severity are `<skipped>`. A passing `security` case when there are none. One failure when the gate could not run. |
| `gates` | A drift budget or an EPSS threshold was judged | `drift-budget` / `drift-worsening` for the DriftScore flags, or `drift-budget maxScore` and `drift-budget maxWorseningPercent` for [`driftBudget`](#drift-budget) in project config. `fail-on-epss` for [`--fail-on-epss`](#epss-gate) or `failOnEpss`. |
| `gates` | A drift budget or an EPSS threshold was judged | `drift-budget` / `drift-worsening` for the DriftScore flags. For [`driftBudget`](#drift-budget) in project config: `drift-budget maxScore`, `drift-budget maxWorseningPercent`, `drift-budget maxRiskScore`, and `drift-budget maxRiskWorseningPercent`, in that order when configured. The two RiskScore cases are `<skipped>` on a local scan because `vg scan` does not compute RiskScore. `agents.maxWorseningPercent` never appears here; the GitHub App check evaluates it. `fail-on-epss` for [`--fail-on-epss`](#epss-gate) or `failOnEpss`. |

A finding is a `<failure>` only when its level fails the drift gate you set. Anything else is `<skipped>` (present, not gating):

Expand Down
80 changes: 80 additions & 0 deletions src/reporting/junit-report.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { readFileSync } from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
import { XMLParser } from 'fast-xml-parser';
import { describe, expect, it } from 'vitest';
import { evaluateConfigDriftBudget } from './drift-budget-gate.js';
import { evaluateEpssGate } from './epss-gate.js';
Expand Down Expand Up @@ -217,6 +218,85 @@ describe('scan JUnit gates', () => {
expect(cases[0]?.status === 'skipped' && cases[0].message).toContain('Run with --baseline to compare.');
});

it.each(['warn', 'enforce', 'shadow'] as const)('keeps local RiskScore cases skipped after DriftScore cases in %s mode', (mode) => {
const gate = evaluateConfigDriftBudget({
raw: {
mode,
maxRiskWorseningPercent: 0,
maxRiskScore: 0,
agents: { maxWorseningPercent: 0 },
maxWorseningPercent: 5,
maxScore: 40,
},
configFile: '.vibgrate/config.yml',
headScore: 40,
baseScore: 40,
});
const cases = configDriftBudgetCases(gate);
expect(cases.map((c) => [c.name, c.status])).toEqual([
['drift-budget maxScore', 'pass'],
['drift-budget maxWorseningPercent', 'pass'],
['drift-budget maxRiskScore', 'skipped'],
['drift-budget maxRiskWorseningPercent', 'skipped'],
]);

const xml = renderScanJUnit({ findings: [], extraSuites: [{ name: 'gates', cases }] });
const parsed = new XMLParser({ ignoreAttributes: false }).parse(xml);
const suite = parsed.testsuites.testsuite[1];
expect(suite['@_name']).toBe('gates');
expect(suite.testcase.map((c: { '@_name': string }) => c['@_name'])).toEqual(cases.map((c) => c.name));
expect(suite.testcase.slice(2).map((c: { skipped: { '@_message': string } }) => c.skipped['@_message'])).toEqual([
'drift budget: Not evaluated on a local scan. The GitHub App computes RiskScore on a Team plan or above.',
'drift budget: Not evaluated on a local scan. The GitHub App computes RiskScore on a Team plan or above.',
]);
expect(suite['@_failures']).toBe('0');
expect(suite['@_skipped']).toBe('2');
expect(xml).not.toContain('agents.maxWorseningPercent');
expect(gate.exitCode).toBe(0);
});

it('does not fail an enforced RiskScore-only budget on a local scan', () => {
const gate = evaluateConfigDriftBudget({
raw: { mode: 'enforce', maxRiskScore: 0, maxRiskWorseningPercent: 0 },
configFile: '.vibgrate/config.yml',
headScore: 99,
baseScore: null,
});
const cases = configDriftBudgetCases(gate);
expect(cases.map((c) => [c.name, c.status])).toEqual([
['drift-budget maxRiskScore', 'skipped'],
['drift-budget maxRiskWorseningPercent', 'skipped'],
]);
const xml = renderScanJUnit({ findings: [], extraSuites: [{ name: 'gates', cases }] });
const parsed = new XMLParser({ ignoreAttributes: false }).parse(xml);
expect(parsed.testsuites['@_failures']).toBe('0');
expect(parsed.testsuites.testsuite[1]).toMatchObject({
'@_name': 'gates', '@_tests': '2', '@_failures': '0', '@_skipped': '2',
});
expect(xml).not.toContain('<failure');
expect(gate.exitCode).toBe(0);
});

it('preserves an enforced DriftScore failure alongside skipped RiskScore cases', () => {
const gate = evaluateConfigDriftBudget({
raw: { mode: 'enforce', maxScore: 40, maxRiskScore: 0, maxRiskWorseningPercent: 0 },
configFile: '.vibgrate/config.yml',
headScore: 44,
baseScore: null,
});
const cases = configDriftBudgetCases(gate);
const xml = renderScanJUnit({ findings: [], extraSuites: [{ name: 'gates', cases }] });
const parsed = new XMLParser({ ignoreAttributes: false }).parse(xml);
expect(parsed.testsuites.testsuite[1]).toMatchObject({
'@_name': 'gates', '@_tests': '3', '@_failures': '1', '@_skipped': '2',
});
expect(parsed.testsuites.testsuite[1].testcase[0]).toMatchObject({
'@_name': 'drift-budget maxScore',
failure: { '@_type': 'gate' },
});
expect(gate.exitCode).toBe(2);
});

it('sorts architecture rows and keeps a blocked gate as one failure', () => {
const failed = architectureGateSuite({
status: 'failed',
Expand Down
Loading