From 05059e6fcf0481ec343ec8b564746944c77dbdd9 Mon Sep 17 00:00:00 2001 From: Ian Hou <45278651+iankhou@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:25:43 -0400 Subject: [PATCH] feat(cli): emit telemetry for the validate action Adds a VALIDATE telemetry event covering the validation phase of cdk validate (offline report collection and online CloudFormation validation, excluding synthesis), with counters for offline violations per severity, offlineWouldFailDeploy (offline validation found a report that would have failed cdk deploy), and onlineViolations. --- .../private/count-validation-results.ts | 30 +++++++ .../lib/toolkit/private/validation-report.ts | 9 ++ .../toolkit-lib/lib/toolkit/toolkit.ts | 4 +- .../toolkit/count-validation-results.test.ts | 83 +++++++++++++++++ packages/aws-cdk/lib/api-private.ts | 1 + packages/aws-cdk/lib/cli/cdk-toolkit.ts | 27 +++++- .../aws-cdk/lib/cli/io-host/cli-io-host.ts | 3 + .../aws-cdk/lib/cli/telemetry/messages.ts | 15 ++++ packages/aws-cdk/lib/cli/telemetry/schema.ts | 2 +- .../test/cli/io-host/cli-io-host.test.ts | 33 +++++++ ...ss_when_no_validation_report_exists.ndjson | 12 +-- ...n_validates_a_single_selected_stack.ndjson | 12 +-- ...e_even_when_no_violations_are_found.ndjson | 7 ++ ...age_with_offline_violation_counters.ndjson | 7 ++ ..._error_name_when_the_engine_crashes.ndjson | 6 ++ ...en_validation_report_has_violations.ndjson | 8 +- ...le_violations_from_multiple_plugins.ndjson | 8 +- .../aws-cdk/test/commands/validate.test.ts | 88 +++++++++++++++++++ 18 files changed, 333 insertions(+), 22 deletions(-) create mode 100644 packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts create mode 100644 packages/@aws-cdk/toolkit-lib/test/toolkit/count-validation-results.test.ts create mode 100644 packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_even_when_no_violations_are_found.ndjson create mode 100644 packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_with_offline_violation_counters.ndjson create mode 100644 packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_ends_the_VALIDATE_span_with_the_error_name_when_the_engine_crashes.ndjson diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts new file mode 100644 index 000000000..92902f0de --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts @@ -0,0 +1,30 @@ +import { ONLINE_VALIDATION_PLUGIN_NAME } from './validation-report'; +import type { ValidateResult } from '../../actions/validate'; +import type { IMessageSpan } from '../../api/io/private/span'; + +/** + * Add counters describing the outcome of a validate run to the given span + * + * Offline violations (policy plugin reports and construct annotations read + * from the cloud assembly) are counted per severity. An offline report with a + * 'failure' conclusion is the exact condition that makes deploy-like actions + * throw (see `throwIfValidationFailures`), so `offlineWouldFailDeploy` records + * that offline validation caught an error before a deployment attempt. + */ +export function countValidationResults(span: IMessageSpan, result: ValidateResult) { + const offline = result.pluginReports.filter((r) => r.pluginName !== ONLINE_VALIDATION_PLUGIN_NAME); + const online = result.pluginReports.filter((r) => r.pluginName === ONLINE_VALIDATION_PLUGIN_NAME); + + for (const report of offline) { + for (const violation of report.violations) { + span.incCounter(`offlineViolations:${violation.severity}`); + } + } + + span.incCounter('onlineViolations', sum(online.map((r) => r.violations.length))); + span.incCounter('offlineWouldFailDeploy', offline.some((r) => r.conclusion === 'failure') ? 1 : 0); +} + +function sum(xs: number[]) { + return xs.reduce((a, b) => a + b, 0); +} diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts index a1a9eeb47..f68ba4f20 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts @@ -12,6 +12,15 @@ import type { MinimumSeverity } from '../types'; const VALIDATION_REPORT_FILE = 'validation-report.json'; +/** + * The plugin name under which online (CloudFormation change set) validation results are reported. + * + * All other plugin names in a unified validation report are offline sources: + * policy validation plugins and construct annotations, both read from the + * cloud assembly. + */ +export const ONLINE_VALIDATION_PLUGIN_NAME = 'CloudFormation'; + /** * The name of the plugin that emits construct annotations into the validation report. * diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts index 0cb2e0355..664e98a77 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts @@ -116,7 +116,7 @@ import { formatErrorMessage, formatExpressStabilizationWarning, formatTime, obsc import { pLimit } from '../util/concurrency'; import { createIgnoreMatcher } from '../util/glob-matcher'; import { promiseWithResolvers } from '../util/promises'; -import { combineConclusions, obtainUnifiedValidationReport, throwIfValidationFailures } from './private/validation-report'; +import { combineConclusions, obtainUnifiedValidationReport, ONLINE_VALIDATION_PLUGIN_NAME, throwIfValidationFailures } from './private/validation-report'; export interface ToolkitOptions { /** @@ -765,7 +765,7 @@ export class Toolkit extends CloudAssemblySourceBuilder { } return { - pluginName: 'CloudFormation', + pluginName: ONLINE_VALIDATION_PLUGIN_NAME, conclusion: 'failure', violations, }; diff --git a/packages/@aws-cdk/toolkit-lib/test/toolkit/count-validation-results.test.ts b/packages/@aws-cdk/toolkit-lib/test/toolkit/count-validation-results.test.ts new file mode 100644 index 000000000..86b17ff2e --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/test/toolkit/count-validation-results.test.ts @@ -0,0 +1,83 @@ +import type { PluginReportJson } from '@aws-cdk/cloud-assembly-schema'; +import type { ValidateResult } from '../../lib/actions/validate'; +import type { IMessageSpan } from '../../lib/api/io/private/span'; +import { countValidationResults } from '../../lib/toolkit/private/count-validation-results'; + +let span: IMessageSpan; +let counters: Record; + +beforeEach(() => { + counters = {}; + span = { + incCounter: (name: string, delta: number = 1) => { + counters[name] = (counters[name] ?? 0) + delta; + }, + } as IMessageSpan; +}); + +function report(pluginName: string, conclusion: 'success' | 'failure', severities: string[]): PluginReportJson { + return { + pluginName, + conclusion, + violations: severities.map((severity) => ({ + ruleName: 'some-rule', + description: 'some description', + severity: severity as any, + violatingConstructs: [], + })), + }; +} + +function result(...pluginReports: PluginReportJson[]): ValidateResult { + return { + conclusion: pluginReports.some((r) => r.conclusion === 'failure') ? 'failure' : 'success', + pluginReports, + }; +} + +test('counts offline violations per severity', () => { + countValidationResults(span, result( + report('SomePlugin', 'failure', ['error', 'error', 'warning']), + report('Construct Annotations', 'success', ['warning', 'info']), + )); + + expect(counters).toEqual({ + 'offlineViolations:error': 2, + 'offlineViolations:warning': 2, + 'offlineViolations:info': 1, + 'onlineViolations': 0, + 'offlineWouldFailDeploy': 1, + }); +}); + +test('online violations are counted separately from offline severities', () => { + countValidationResults(span, result( + report('CloudFormation', 'failure', ['fatal', 'fatal']), + )); + + expect(counters).toEqual({ + onlineViolations: 2, + offlineWouldFailDeploy: 0, + }); +}); + +test('offlineWouldFailDeploy is 0 when offline reports succeed', () => { + countValidationResults(span, result( + report('SomePlugin', 'success', ['warning']), + )); + + expect(counters).toEqual({ + 'offlineViolations:warning': 1, + 'onlineViolations': 0, + 'offlineWouldFailDeploy': 0, + }); +}); + +test('no reports produce zero counters', () => { + countValidationResults(span, result()); + + expect(counters).toEqual({ + onlineViolations: 0, + offlineWouldFailDeploy: 0, + }); +}); diff --git a/packages/aws-cdk/lib/api-private.ts b/packages/aws-cdk/lib/api-private.ts index 2a2d8106d..942e56444 100644 --- a/packages/aws-cdk/lib/api-private.ts +++ b/packages/aws-cdk/lib/api-private.ts @@ -12,4 +12,5 @@ export * from '../../@aws-cdk/toolkit-lib/lib/api/tags/private'; export * from '../../@aws-cdk/toolkit-lib/lib/private/activity-printer'; export * from '../../@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/borrowed-assembly'; export * from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/count-assembly-results'; +export * from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results'; export { throwIfValidationFailures } from '../../@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report'; diff --git a/packages/aws-cdk/lib/cli/cdk-toolkit.ts b/packages/aws-cdk/lib/cli/cdk-toolkit.ts index 6030293d5..fb68c8070 100644 --- a/packages/aws-cdk/lib/cli/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cli/cdk-toolkit.ts @@ -14,7 +14,7 @@ import { CliIoHost } from './io-host'; import type { Configuration } from './user-configuration'; import { PROJECT_CONFIG } from './user-configuration'; import type { ActionLessRequest, IMessageSpan, IoHelper } from '../../lib/api-private'; -import { asIoHelper, cfnApi, createIgnoreMatcher, formatExpressStabilizationWarning, IO, tagsForStack, throwIfValidationFailures } from '../../lib/api-private'; +import { asIoHelper, cfnApi, countValidationResults, createIgnoreMatcher, formatExpressStabilizationWarning, IO, tagsForStack, throwIfValidationFailures } from '../../lib/api-private'; import type { AssetBuildNode, AssetPublishNode, Concurrency, MarkerNode, StackNode, WorkGraph, WorkGraphActions } from '../api'; import { CloudWatchLogEventMonitor, @@ -644,8 +644,29 @@ export class CdkToolkit { return this.validateWatch(validateOptions); } - const result = await this.toolkit.validate(this.props.cloudExecutable, validateOptions); - return result.conclusion === 'failure' ? 1 : 0; + // Synthesize before starting the VALIDATE span, so the span measures only + // the validation phase (offline report collection and, unless disabled, + // online CloudFormation validation). Synthesis is reported as its own + // SYNTH event; the assembly is cached, so the synthesis inside + // `toolkit.validate()` below is a cache hit. + await this.props.cloudExecutable.synthesize(); + + // The span is ended even if the engine crashes, so telemetry always + // records that a validation was started. + const validateSpan = await this.ioHost.asIoHelper().span(CLI_PRIVATE_SPAN.VALIDATE).begin({}); + let error: ErrorDetails | undefined; + try { + const result = await this.toolkit.validate(this.props.cloudExecutable, validateOptions); + countValidationResults(validateSpan, result); + return result.conclusion === 'failure' ? 1 : 0; + } catch (e: any) { + error = { + name: cdkCliErrorName(e), + }; + throw e; + } finally { + await validateSpan.end({ error }); + } } /** diff --git a/packages/aws-cdk/lib/cli/io-host/cli-io-host.ts b/packages/aws-cdk/lib/cli/io-host/cli-io-host.ts index 6f1f7b547..733dd4a26 100644 --- a/packages/aws-cdk/lib/cli/io-host/cli-io-host.ts +++ b/packages/aws-cdk/lib/cli/io-host/cli-io-host.ts @@ -1165,6 +1165,9 @@ function eventFromMessage(msg: IoMessage): TelemetryEvent | undefined { if (CLI_PRIVATE_IO.CDK_CLI_I3003.is(msg)) { return eventResult('ASSET', msg); } + if (CLI_PRIVATE_IO.CDK_CLI_I4001.is(msg)) { + return eventResult('VALIDATE', msg); + } // Hotswap lives in the cdk-toolkit so it cannot be a CDK_CLI error code. // Instead we reuse the existing Hotswap span. if (IO.CDK_TOOLKIT_I5410.is(msg)) { diff --git a/packages/aws-cdk/lib/cli/telemetry/messages.ts b/packages/aws-cdk/lib/cli/telemetry/messages.ts index 00f33b65a..625781df6 100644 --- a/packages/aws-cdk/lib/cli/telemetry/messages.ts +++ b/packages/aws-cdk/lib/cli/telemetry/messages.ts @@ -59,6 +59,16 @@ export const CLI_PRIVATE_IO = { description: 'Finished asset building and publishing', interface: 'EventResult', }), + CDK_CLI_I4000: make.trace({ + code: 'CDK_CLI_I4000', + description: 'Validation has started', + interface: 'EventStart', + }), + CDK_CLI_I4001: make.trace({ + code: 'CDK_CLI_I4001', + description: 'Validation has finished', + interface: 'EventResult', + }), }; /** @@ -85,4 +95,9 @@ export const CLI_PRIVATE_SPAN = { start: CLI_PRIVATE_IO.CDK_CLI_I3002, end: CLI_PRIVATE_IO.CDK_CLI_I3003, }, + VALIDATE: { + name: 'Validation', + start: CLI_PRIVATE_IO.CDK_CLI_I4000, + end: CLI_PRIVATE_IO.CDK_CLI_I4001, + }, } satisfies Record>; diff --git a/packages/aws-cdk/lib/cli/telemetry/schema.ts b/packages/aws-cdk/lib/cli/telemetry/schema.ts index c2affc98d..e60edac09 100644 --- a/packages/aws-cdk/lib/cli/telemetry/schema.ts +++ b/packages/aws-cdk/lib/cli/telemetry/schema.ts @@ -25,7 +25,7 @@ interface SessionEvent { readonly command: Command; } -export type EventType = 'SYNTH' | 'INVOKE' | 'DEPLOY' | 'HOTSWAP' | 'ASSET'; +export type EventType = 'SYNTH' | 'INVOKE' | 'DEPLOY' | 'HOTSWAP' | 'ASSET' | 'VALIDATE'; export type State = 'ABORTED' | 'FAILED' | 'SUCCEEDED'; interface Event extends SessionEvent { readonly state: State; diff --git a/packages/aws-cdk/test/cli/io-host/cli-io-host.test.ts b/packages/aws-cdk/test/cli/io-host/cli-io-host.test.ts index 1e6b95b86..8d82c3e90 100644 --- a/packages/aws-cdk/test/cli/io-host/cli-io-host.test.ts +++ b/packages/aws-cdk/test/cli/io-host/cli-io-host.test.ts @@ -781,6 +781,39 @@ describe('CliIoHost', () => { })); }); + test('emit telemetry on VALIDATE event', async () => { + // Create a message that should trigger telemetry using the actual message code + const message: IoMessage = { + time: new Date(), + level: 'trace', + action: 'validate', + code: 'CDK_CLI_I4001', + message: 'telemetry message', + data: { + duration: 123, + counters: { + 'offlineViolations:error': 2, + 'offlineWouldFailDeploy': 1, + 'onlineViolations': 0, + }, + }, + }; + + // Send the notification + await telemetryIoHost.notify(message); + + // Verify that the emit method was called with the correct parameters + expect(telemetryEmitSpy).toHaveBeenCalledWith(expect.objectContaining({ + eventType: 'VALIDATE', + duration: 123, + counters: { + 'offlineViolations:error': 2, + 'offlineWouldFailDeploy': 1, + 'onlineViolations': 0, + }, + })); + }); + test('do not emit telemetry on non telemetry codes', async () => { // Create a message that should trigger telemetry using the actual message code const message: IoMessage = { diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/no_violations_reports_success_when_no_validation_report_exists.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/no_violations_reports_success_when_no_validation_report_exists.ndjson index 9388eb1c0..89358b246 100644 --- a/packages/aws-cdk/test/commands/__io_snapshots__/validate/no_violations_reports_success_when_no_validation_report_exists.ndjson +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/no_violations_reports_success_when_no_validation_report_exists.ndjson @@ -1,5 +1,7 @@ -{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} -{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} -{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} -{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} -{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."} +{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} +{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/stack_selection_validates_a_single_selected_stack.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/stack_selection_validates_a_single_selected_stack.ndjson index 9388eb1c0..89358b246 100644 --- a/packages/aws-cdk/test/commands/__io_snapshots__/validate/stack_selection_validates_a_single_selected_stack.ndjson +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/stack_selection_validates_a_single_selected_stack.ndjson @@ -1,5 +1,7 @@ -{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} -{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} -{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} -{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} -{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."} +{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} +{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_even_when_no_violations_are_found.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_even_when_no_violations_are_found.ndjson new file mode 100644 index 000000000..89358b246 --- /dev/null +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_even_when_no_violations_are_found.ndjson @@ -0,0 +1,7 @@ +{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} +{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I9600","message":"Validation did not find any problems."} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_with_offline_violation_counters.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_with_offline_violation_counters.ndjson new file mode 100644 index 000000000..609d3ca6e --- /dev/null +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_emits_a_VALIDATE_span_end_message_with_offline_violation_counters.ndjson @@ -0,0 +1,7 @@ +{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} +{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_ends_the_VALIDATE_span_with_the_error_name_when_the_engine_crashes.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_ends_the_VALIDATE_span_with_the_error_name_when_the_engine_crashes.ndjson new file mode 100644 index 000000000..d989c066c --- /dev/null +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/telemetry_ends_the_VALIDATE_span_with_the_error_name_when_the_engine_crashes.ndjson @@ -0,0 +1,6 @@ +{"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} +{"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_failure_when_validation_report_has_violations.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_failure_when_validation_report_has_violations.ndjson index 17b16301d..609d3ca6e 100644 --- a/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_failure_when_validation_report_has_violations.ndjson +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_failure_when_validation_report_has_violations.ndjson @@ -1,5 +1,7 @@ {"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} {"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} -{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} -{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} -{"seq":4,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (TestPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'TestPlugin::no-public-buckets'"} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_multiple_violations_from_multiple_plugins.ndjson b/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_multiple_violations_from_multiple_plugins.ndjson index 0ad412ccb..b46112b2e 100644 --- a/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_multiple_violations_from_multiple_plugins.ndjson +++ b/packages/aws-cdk/test/commands/__io_snapshots__/validate/with_violations_reports_multiple_violations_from_multiple_plugins.ndjson @@ -1,5 +1,7 @@ {"seq":0,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1000","message":"Starting Synthesis ..."} {"seq":1,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I1001","message":"\n✨ Synthesis time: \n"} -{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} -{"seq":3,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} -{"seq":4,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (SecurityPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'SecurityPlugin::no-public-buckets'\n\nWARNING All resources must have cost allocation tags (CostPlugin)\n Test-Stack-B/MyTopic/Resource (MyTopic)\n Acknowledge with 'CostPlugin::require-cost-tags'"} +{"seq":2,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4000","message":"Starting Validation ..."} +{"seq":3,"type":"notify","action":"validate","level":"trace","code":"CDK_TOOLKIT_I1001","message":"Starting Synthesis ..."} +{"seq":4,"type":"notify","action":"validate","level":"info","code":"CDK_TOOLKIT_I1000","message":"✨ Synthesis time: "} +{"seq":5,"type":"notify","action":"validate","level":"error","code":"CDK_TOOLKIT_E9600","message":"ERROR S3 Buckets must not be publicly accessible (SecurityPlugin)\n Test-Stack-A-Display-Name/MyBucket/Resource (MyBucket)\n Acknowledge with 'SecurityPlugin::no-public-buckets'\n\nWARNING All resources must have cost allocation tags (CostPlugin)\n Test-Stack-B/MyTopic/Resource (MyTopic)\n Acknowledge with 'CostPlugin::require-cost-tags'"} +{"seq":6,"type":"notify","action":"validate","level":"trace","code":"CDK_CLI_I4001","message":"\n✨ Validation time: \n"} diff --git a/packages/aws-cdk/test/commands/validate.test.ts b/packages/aws-cdk/test/commands/validate.test.ts index 372a8f6d5..1a6f268f8 100644 --- a/packages/aws-cdk/test/commands/validate.test.ts +++ b/packages/aws-cdk/test/commands/validate.test.ts @@ -152,6 +152,94 @@ describe('with violations', () => { }); }); +describe('telemetry', () => { + // Remove the spies installed by these tests; the file-level `resetAllMocks` + // would otherwise strip the passthrough implementation from `ioHost.notify` + // and break tests that run later (test order is randomized). + afterEach(() => { + jest.restoreAllMocks(); + }); + + test('emits a VALIDATE span end message with offline violation counters', async () => { + const assembly = await cloudExecutable.synthesize(); + await fs.writeJSON(path.join(assembly.directory, 'validation-report.json'), { + version: '1.0.0', + pluginReports: [{ + pluginName: 'TestPlugin', + conclusion: 'failure', + violations: [{ + ruleName: 'no-public-buckets', + description: 'S3 Buckets must not be publicly accessible', + severity: 'error', + violatingConstructs: [{ + constructPath: 'Test-Stack-A-Display-Name/MyBucket/Resource', + cloudFormationResource: { + templatePath: 'Test-Stack-A.template.json', + logicalId: 'MyBucket', + }, + }], + }], + }], + }); + + const notifySpy = jest.spyOn(ioHost, 'notify'); + await toolkit.validate({ + stacks: { patterns: [], strategy: StackSelectionStrategy.ALL_STACKS }, + online: false, + }); + + expect(notifySpy).toHaveBeenCalledWith(expect.objectContaining({ + code: 'CDK_CLI_I4001', + data: expect.objectContaining({ + duration: expect.any(Number), + counters: { + 'offlineViolations:error': 1, + 'offlineWouldFailDeploy': 1, + 'onlineViolations': 0, + }, + }), + })); + }); + + test('emits a VALIDATE span end message even when no violations are found', async () => { + const notifySpy = jest.spyOn(ioHost, 'notify'); + await toolkit.validate({ + stacks: { patterns: [], strategy: StackSelectionStrategy.ALL_STACKS }, + online: false, + }); + + expect(notifySpy).toHaveBeenCalledWith(expect.objectContaining({ + code: 'CDK_CLI_I4001', + data: expect.objectContaining({ + duration: expect.any(Number), + counters: { + onlineViolations: 0, + offlineWouldFailDeploy: 0, + }, + }), + })); + }); + + test('ends the VALIDATE span with the error name when the engine crashes', async () => { + // The CLI synthesizes (and caches) the assembly before the VALIDATE span + // begins, so failing `produce()` crashes the engine inside the span. + jest.spyOn(cloudExecutable, 'produce').mockRejectedValue(new Error('engine exploded')); + + const notifySpy = jest.spyOn(ioHost, 'notify'); + await expect(toolkit.validate({ + stacks: { patterns: [], strategy: StackSelectionStrategy.ALL_STACKS }, + online: false, + })).rejects.toThrow('engine exploded'); + + expect(notifySpy).toHaveBeenCalledWith(expect.objectContaining({ + code: 'CDK_CLI_I4001', + data: expect.objectContaining({ + error: { name: 'UnknownError' }, + }), + })); + }); +}); + describe('stack selection', () => { test('validates a single selected stack', async () => { const exitCode = await toolkit.validate({