Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { ONLINE_VALIDATION_PLUGIN_NAME } from './validation-report';
import type { ValidateResult } from '../../actions/validate';
import type { IMessageSpan } from '../../api/io/private/span';

/**
* Add counters describing the outcome of a validate run to the given span
*
* Offline violations (policy plugin reports and construct annotations read
* from the cloud assembly) are counted per severity. An offline report with a
* 'failure' conclusion is the exact condition that makes deploy-like actions
* throw (see `throwIfValidationFailures`), so `offlineWouldFailDeploy` records
* that offline validation caught an error before a deployment attempt.
*/
export function countValidationResults(span: IMessageSpan<any>, result: ValidateResult) {
const offline = result.pluginReports.filter((r) => r.pluginName !== ONLINE_VALIDATION_PLUGIN_NAME);
const online = result.pluginReports.filter((r) => r.pluginName === ONLINE_VALIDATION_PLUGIN_NAME);
Comment on lines +15 to +16

for (const report of offline) {
for (const violation of report.violations) {
span.incCounter(`offlineViolations:${violation.severity}`);
}
}

span.incCounter('onlineViolations', sum(online.map((r) => r.violations.length)));
span.incCounter('offlineWouldFailDeploy', offline.some((r) => r.conclusion === 'failure') ? 1 : 0);
}

function sum(xs: number[]) {
return xs.reduce((a, b) => a + b, 0);
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ import type { MinimumSeverity } from '../types';

const VALIDATION_REPORT_FILE = 'validation-report.json';

/**
* The plugin name under which online (CloudFormation change set) validation results are reported.
*
* All other plugin names in a unified validation report are offline sources:
* policy validation plugins and construct annotations, both read from the
* cloud assembly.
*/
export const ONLINE_VALIDATION_PLUGIN_NAME = 'CloudFormation';

/**
* The name of the plugin that emits construct annotations into the validation report.
*
Expand Down
4 changes: 2 additions & 2 deletions packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ import { formatErrorMessage, formatExpressStabilizationWarning, formatTime, obsc
import { pLimit } from '../util/concurrency';
import { createIgnoreMatcher } from '../util/glob-matcher';
import { promiseWithResolvers } from '../util/promises';
import { combineConclusions, obtainUnifiedValidationReport, throwIfValidationFailures } from './private/validation-report';
import { combineConclusions, obtainUnifiedValidationReport, ONLINE_VALIDATION_PLUGIN_NAME, throwIfValidationFailures } from './private/validation-report';

export interface ToolkitOptions {
/**
Expand Down Expand Up @@ -765,7 +765,7 @@ export class Toolkit extends CloudAssemblySourceBuilder {
}

return {
pluginName: 'CloudFormation',
pluginName: ONLINE_VALIDATION_PLUGIN_NAME,
conclusion: 'failure',
violations,
};
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import type { PluginReportJson } from '@aws-cdk/cloud-assembly-schema';
import type { ValidateResult } from '../../lib/actions/validate';
import type { IMessageSpan } from '../../lib/api/io/private/span';
import { countValidationResults } from '../../lib/toolkit/private/count-validation-results';

let span: IMessageSpan<any>;
let counters: Record<string, number>;

beforeEach(() => {
counters = {};
span = {
incCounter: (name: string, delta: number = 1) => {
counters[name] = (counters[name] ?? 0) + delta;
},
} as IMessageSpan<any>;
});

function report(pluginName: string, conclusion: 'success' | 'failure', severities: string[]): PluginReportJson {
return {
pluginName,
conclusion,
violations: severities.map((severity) => ({
ruleName: 'some-rule',
description: 'some description',
severity: severity as any,
violatingConstructs: [],
})),
};
}

function result(...pluginReports: PluginReportJson[]): ValidateResult {
return {
conclusion: pluginReports.some((r) => r.conclusion === 'failure') ? 'failure' : 'success',
pluginReports,
};
}

test('counts offline violations per severity', () => {
countValidationResults(span, result(
report('SomePlugin', 'failure', ['error', 'error', 'warning']),
report('Construct Annotations', 'success', ['warning', 'info']),
));

expect(counters).toEqual({
'offlineViolations:error': 2,
'offlineViolations:warning': 2,
'offlineViolations:info': 1,
'onlineViolations': 0,
'offlineWouldFailDeploy': 1,
});
});

test('online violations are counted separately from offline severities', () => {
countValidationResults(span, result(
report('CloudFormation', 'failure', ['fatal', 'fatal']),
));

expect(counters).toEqual({
onlineViolations: 2,
offlineWouldFailDeploy: 0,
});
});

test('offlineWouldFailDeploy is 0 when offline reports succeed', () => {
countValidationResults(span, result(
report('SomePlugin', 'success', ['warning']),
));

expect(counters).toEqual({
'offlineViolations:warning': 1,
'onlineViolations': 0,
'offlineWouldFailDeploy': 0,
});
});

test('no reports produce zero counters', () => {
countValidationResults(span, result());

expect(counters).toEqual({
onlineViolations: 0,
offlineWouldFailDeploy: 0,
});
});
1 change: 1 addition & 0 deletions packages/aws-cdk/lib/api-private.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ export * from '../../@aws-cdk/toolkit-lib/lib/api/tags/private';
export * from '../../@aws-cdk/toolkit-lib/lib/private/activity-printer';
export * from '../../@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/borrowed-assembly';
export * from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/count-assembly-results';
export * from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results';
export { throwIfValidationFailures } from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report';
27 changes: 24 additions & 3 deletions packages/aws-cdk/lib/cli/cdk-toolkit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { CliIoHost } from './io-host';
import type { Configuration } from './user-configuration';
import { PROJECT_CONFIG } from './user-configuration';
import type { ActionLessRequest, IMessageSpan, IoHelper } from '../../lib/api-private';
import { asIoHelper, cfnApi, createIgnoreMatcher, formatExpressStabilizationWarning, IO, tagsForStack, throwIfValidationFailures } from '../../lib/api-private';
import { asIoHelper, cfnApi, countValidationResults, createIgnoreMatcher, formatExpressStabilizationWarning, IO, tagsForStack, throwIfValidationFailures } from '../../lib/api-private';
import type { AssetBuildNode, AssetPublishNode, Concurrency, MarkerNode, StackNode, WorkGraph, WorkGraphActions } from '../api';
import {
CloudWatchLogEventMonitor,
Expand Down Expand Up @@ -644,8 +644,29 @@ export class CdkToolkit {
return this.validateWatch(validateOptions);
}

const result = await this.toolkit.validate(this.props.cloudExecutable, validateOptions);
return result.conclusion === 'failure' ? 1 : 0;
// Synthesize before starting the VALIDATE span, so the span measures only
// the validation phase (offline report collection and, unless disabled,
// online CloudFormation validation). Synthesis is reported as its own
// SYNTH event; the assembly is cached, so the synthesis inside
// `toolkit.validate()` below is a cache hit.
await this.props.cloudExecutable.synthesize();

// The span is ended even if the engine crashes, so telemetry always
// records that a validation was started.
const validateSpan = await this.ioHost.asIoHelper().span(CLI_PRIVATE_SPAN.VALIDATE).begin({});
let error: ErrorDetails | undefined;
try {
const result = await this.toolkit.validate(this.props.cloudExecutable, validateOptions);
countValidationResults(validateSpan, result);
return result.conclusion === 'failure' ? 1 : 0;
} catch (e: any) {
error = {
name: cdkCliErrorName(e),
};
throw e;
} finally {
await validateSpan.end({ error });
}
}

/**
Expand Down
3 changes: 3 additions & 0 deletions packages/aws-cdk/lib/cli/io-host/cli-io-host.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1165,6 +1165,9 @@ function eventFromMessage(msg: IoMessage<unknown>): TelemetryEvent | undefined {
if (CLI_PRIVATE_IO.CDK_CLI_I3003.is(msg)) {
return eventResult('ASSET', msg);
}
if (CLI_PRIVATE_IO.CDK_CLI_I4001.is(msg)) {
return eventResult('VALIDATE', msg);
}
// Hotswap lives in the cdk-toolkit so it cannot be a CDK_CLI error code.
// Instead we reuse the existing Hotswap span.
if (IO.CDK_TOOLKIT_I5410.is(msg)) {
Expand Down
15 changes: 15 additions & 0 deletions packages/aws-cdk/lib/cli/telemetry/messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,16 @@ export const CLI_PRIVATE_IO = {
description: 'Finished asset building and publishing',
interface: 'EventResult',
}),
CDK_CLI_I4000: make.trace<EventStart>({
code: 'CDK_CLI_I4000',
description: 'Validation has started',
interface: 'EventStart',
}),
CDK_CLI_I4001: make.trace<EventResult>({
code: 'CDK_CLI_I4001',
description: 'Validation has finished',
interface: 'EventResult',
}),
};

/**
Expand All @@ -85,4 +95,9 @@ export const CLI_PRIVATE_SPAN = {
start: CLI_PRIVATE_IO.CDK_CLI_I3002,
end: CLI_PRIVATE_IO.CDK_CLI_I3003,
},
VALIDATE: {
name: 'Validation',
start: CLI_PRIVATE_IO.CDK_CLI_I4000,
end: CLI_PRIVATE_IO.CDK_CLI_I4001,
},
} satisfies Record<string, SpanDefinition<any, any>>;
2 changes: 1 addition & 1 deletion packages/aws-cdk/lib/cli/telemetry/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ interface SessionEvent {
readonly command: Command;
}

export type EventType = 'SYNTH' | 'INVOKE' | 'DEPLOY' | 'HOTSWAP' | 'ASSET';
export type EventType = 'SYNTH' | 'INVOKE' | 'DEPLOY' | 'HOTSWAP' | 'ASSET' | 'VALIDATE';
export type State = 'ABORTED' | 'FAILED' | 'SUCCEEDED';
interface Event extends SessionEvent {
readonly state: State;
Expand Down
33 changes: 33 additions & 0 deletions packages/aws-cdk/test/cli/io-host/cli-io-host.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -781,6 +781,39 @@ describe('CliIoHost', () => {
}));
});

test('emit telemetry on VALIDATE event', async () => {
// Create a message that should trigger telemetry using the actual message code
const message: IoMessage<unknown> = {
time: new Date(),
level: 'trace',
action: 'validate',
code: 'CDK_CLI_I4001',
message: 'telemetry message',
data: {
duration: 123,
counters: {
'offlineViolations:error': 2,
'offlineWouldFailDeploy': 1,
'onlineViolations': 0,
},
},
};

// Send the notification
await telemetryIoHost.notify(message);

// Verify that the emit method was called with the correct parameters
expect(telemetryEmitSpy).toHaveBeenCalledWith(expect.objectContaining({
eventType: 'VALIDATE',
duration: 123,
counters: {
'offlineViolations:error': 2,
'offlineWouldFailDeploy': 1,
'onlineViolations': 0,
},
}));
});

test('do not emit telemetry on non telemetry codes', async () => {
// Create a message that should trigger telemetry using the actual message code
const message: IoMessage<unknown> = {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."}
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."}
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":4,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."}
{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: <DURATION>\n"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":4,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (SecurityPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'SecurityPlugin::no-public-buckets'\n\nWARNING All resources must have cost allocation tags (CostPlugin)\n Test-Stack-B/MyTopic/Resource (MyTopic)\n Acknowledge with 'CostPlugin::require-cost-tags'"}
{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."}
{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."}
{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: <DURATION>"}
{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (SecurityPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'SecurityPlugin::no-public-buckets'\n\nWARNING All resources must have cost allocation tags (CostPlugin)\n Test-Stack-B/MyTopic/Resource (MyTopic)\n Acknowledge with 'CostPlugin::require-cost-tags'"}
{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: <DURATION>\n"}
Loading
Loading