From fad094f19afcd797b2f4280497833efbfbfc3014 Mon Sep 17 00:00:00 2001 From: Yashwanth Reddy Verupaka <59129034+yashwanth195@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:35:30 -0800 Subject: [PATCH 1/3] Fix environment snapshot hashing --- .../rush/environment-snapshot_2026-09-16.json | 10 + .../src/logic/incremental/InputsSnapshot.ts | 57 +++--- .../incremental/test/InputsSnapshot.test.ts | 186 ++++++++++++++++++ .../__snapshots__/InputsSnapshot.test.ts.snap | 4 +- 4 files changed, 229 insertions(+), 28 deletions(-) create mode 100644 common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json diff --git a/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json new file mode 100644 index 00000000000..670090e0c5c --- /dev/null +++ b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Fix declared environment input hashing to preserve Windows case-insensitive lookup, distinguish absent and empty values, and snapshot supplied environments before querying operation hashes. Operations with declared environment inputs will receive new cache keys.", + "type": "patch" + } + ], + "packageName": "@microsoft/rush" +} diff --git a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index ad96a3e9007..c46b597db55 100644 --- a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -7,7 +7,7 @@ import { createHash, type Hash } from 'node:crypto'; import ignore, { type Ignore } from 'ignore'; import { type IReadonlyLookupByPath, LookupByPath } from '@rushstack/lookup-by-path'; -import { InternalError, Path, Sort } from '@rushstack/node-core-library'; +import { EnvironmentMap, InternalError, Path, Sort } from '@rushstack/node-core-library'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { @@ -90,7 +90,7 @@ export interface IInputsSnapshotParameters { */ additionalHashes?: ReadonlyMap; /** - * The environment to use for `dependsOnEnvVars`. By default performs a snapshot of process.env upon construction. + * The environment to snapshot for `dependsOnEnvVars`. Defaults to process.env upon construction. * @defaultValue \{ ...process.env \} */ environment?: Record; @@ -251,7 +251,7 @@ export class InputsSnapshot implements IInputsSnapshot { /** * The environment to use for `dependsOnEnvVars`. */ - readonly #environment: Record; + readonly #environment: EnvironmentMap; /** * Pre-computed Node.js version strings at each granularity level for `dependsOnNodeVersion`. */ @@ -272,7 +272,7 @@ export class InputsSnapshot implements IInputsSnapshot { public constructor(params: IInputsSnapshotParameters) { const { additionalHashes, - environment = { ...process.env }, + environment = process.env, globalAdditionalFiles, hashes, hasUncommittedChanges, @@ -306,11 +306,13 @@ export class InputsSnapshot implements IInputsSnapshot { const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap); + // Snapshot the environment before comparing or querying it. + this.#environment = new EnvironmentMap(environment); this.#projectMetadataMap = (previousSnapshot && previousSnapshot.#tryDeriveProjectMetadataMap( params, - environment, + this.#environment, nodeVersion, globalAdditionalHashes, operationAdditionalFileCount @@ -318,8 +320,6 @@ export class InputsSnapshot implements IInputsSnapshot { createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir); this.#additionalHashes = additionalHashes; this.#globalAdditionalHashes = globalAdditionalHashes; - // Snapshot the environment so that queries are not impacted by when they happen - this.#environment = environment; // Parse Node.js version once so it doesn't need to be re-parsed per operation this.#nodeVersionByGranularity = _parseNodeVersion(nodeVersion); this.#lookupByPath = lookupByPath; @@ -427,15 +427,22 @@ export class InputsSnapshot implements IInputsSnapshot { const operationSettings: Readonly | undefined = operationName ? record.projectConfig?.operationSettingsByOperationName.get(operationName) : undefined; - const snapshotEnvironment: Readonly> = this.#environment; - if ( - environment && - operationSettings?.dependsOnEnvVars?.some( - (envVar: string) => (environment[envVar] || '') !== (snapshotEnvironment[envVar] || '') - ) - ) { - // The operation's environment differs from the snapshot's in a variable that it hashes, so don't memoize. - return this.#computeOperationOwnStateHash(project, operationName, operationSettings, environment); + const snapshotEnvironment: EnvironmentMap = this.#environment; + if (environment && operationSettings?.dependsOnEnvVars?.length) { + const operationEnvironment: EnvironmentMap = new EnvironmentMap(environment); + if ( + operationSettings.dependsOnEnvVars.some( + (envVar: string) => operationEnvironment.get(envVar) !== snapshotEnvironment.get(envVar) + ) + ) { + // The operation's environment differs from the snapshot's in a variable that it hashes, so don't memoize. + return this.#computeOperationOwnStateHash( + project, + operationName, + operationSettings, + operationEnvironment + ); + } } const { hashByOperationName } = record; @@ -457,7 +464,7 @@ export class InputsSnapshot implements IInputsSnapshot { project: IRushConfigurationProjectForSnapshot, operationName: string | undefined, operationSettings: Readonly | undefined, - environment: Readonly> + environment: EnvironmentMap ): string { const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation(project, operationName); @@ -469,7 +476,8 @@ export class InputsSnapshot implements IInputsSnapshot { // As long as we enumerate environment variables in a consistent order, we will get a stable hash. // Changing the order in rush-project.json will change the hash anyway since the file contents are part of the hash. for (const envVar of dependsOnEnvVars) { - hasher.update(`${hashDelimiter}$${envVar}=${environment[envVar] || ''}`); + // Encode absence separately from an empty value, and escape delimiters within values. + hasher.update(`${hashDelimiter}$${JSON.stringify([envVar, environment.get(envVar)])}`); } } @@ -512,7 +520,7 @@ export class InputsSnapshot implements IInputsSnapshot { */ #tryDeriveProjectMetadataMap( params: IInputsSnapshotParameters, - environment: Readonly>, + environment: EnvironmentMap, nodeVersion: string, globalAdditionalHashes: ReadonlyMap | undefined, operationAdditionalFileCount: number @@ -748,17 +756,14 @@ function areMapsEqual(a: ReadonlyMap, b: ReadonlyMap>, - b: Readonly> -): boolean { - const keys: string[] = Object.keys(a); - if (keys.length !== Object.keys(b).length) { +function areEnvironmentsEqual(a: EnvironmentMap, b: EnvironmentMap): boolean { + const keys: string[] = Array.from(a.names()); + if (keys.length !== Array.from(b.names()).length) { return false; } for (const key of keys) { - if (a[key] !== b[key] || !Object.prototype.hasOwnProperty.call(b, key)) { + if (a.get(key) !== b.get(key)) { return false; } } diff --git a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts index fc29aaa8487..4ed187c5b05 100644 --- a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts +++ b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts @@ -4,6 +4,7 @@ import { LookupByPath } from '@rushstack/lookup-by-path'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import { RushConstants } from '../../RushConstants'; import { InputsSnapshot, type IInputsSnapshotParameters, @@ -795,6 +796,122 @@ describe(InputsSnapshot.name, () => { }); }); + describe('environment identity', () => { + function createSnapshot( + environment: Record, + names: string[] = ['ENV_VAR'] + ): ReturnType { + const { project, options } = getTestConfig(); + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + ['_phase:build', { operationName: '_phase:build', dependsOnEnvVars: names }] + ]) + }; + return { + project, + input: new InputsSnapshot({ + ...options, + environment, + projectMap: new Map([[project, { projectConfig: projectConfig as RushProjectConfiguration }]]) + }) + }; + } + + describe.each([false, true])('with an explicit operation environment: %s', (useOperationEnvironment) => { + function getHash(environment: Record, names?: string[]): string { + const { project, input } = createSnapshot( + useOperationEnvironment ? { ENV_VAR: 'snapshot' } : environment, + names + ); + return input.getOperationOwnStateHash( + project, + '_phase:build', + useOperationEnvironment ? environment : undefined + ); + } + + it('distinguishes absence, empty values and literal sentinel strings', () => { + const values: Array = [undefined, '', 'undefined', 'null', 'value']; + expect(new Set(values.map((value) => getHash({ ENV_VAR: value }))).size).toBe(values.length); + expect(getHash({})).toBe(getHash({ ENV_VAR: undefined })); + }); + + it.each(['win32', 'linux'])('uses %s environment-name semantics', (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const upper: string = getHash({ ENV_VAR: 'first' }); + const lower: string = getHash({ env_var: 'first' }); + const changed: string = getHash({ env_var: 'second' }); + if (platform === 'win32') { + expect(lower).toBe(upper); + expect(changed).not.toBe(lower); + } else { + expect(lower).toBe(getHash({})); + expect(changed).toBe(lower); + expect(lower).not.toBe(upper); + } + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + + it('does not include undeclared environment inputs', () => { + expect(getHash({ ENV_VAR: 'same', OTHER: 'first' })).toBe( + getHash({ ENV_VAR: 'same', OTHER: 'second' }) + ); + }); + + it('does not confuse embedded separators with another declared variable', () => { + const delimiter: string = RushConstants.hashDelimiter; + expect(getHash({ A: `a${delimiter}$B=b`, B: 'c' }, ['A', 'B'])).not.toBe( + getHash({ A: 'a', B: `b${delimiter}$B=c` }, ['A', 'B']) + ); + }); + }); + + it('captures supplied values before the first hash query', () => { + const environment: Record = { ENV_VAR: 'captured' }; + const { project, input } = createSnapshot(environment); + const { project: baselineProject, input: baseline } = createSnapshot({ ...environment }); + environment.ENV_VAR = 'changed'; + expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe( + baseline.getOperationOwnStateHash(baselineProject, '_phase:build') + ); + expect(input.getOperationOwnStateHash(project, '_phase:build', environment)).not.toBe( + input.getOperationOwnStateHash(project, '_phase:build') + ); + }); + + it.each([undefined, ''])( + 'does not reuse or overwrite the snapshot hash for a different operation value (snapshot: %s)', + (snapshotValue) => { + const { project, input } = createSnapshot({ ENV_VAR: snapshotValue }); + const operationEnvironment: Record = { + ENV_VAR: snapshotValue === undefined ? '' : undefined + }; + const { project: operationProject, input: operationSnapshot } = createSnapshot(operationEnvironment); + const operationHash: string = operationSnapshot.getOperationOwnStateHash( + operationProject, + '_phase:build' + ); + expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( + operationHash + ); + const snapshotHash: string = input.getOperationOwnStateHash(project, '_phase:build'); + expect(snapshotHash).not.toBe(operationHash); + expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( + operationHash + ); + operationEnvironment.ENV_VAR = snapshotValue; + expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( + snapshotHash + ); + expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(snapshotHash); + } + ); + }); + describe('previousSnapshot', () => { const operationNames: (string | undefined)[] = [undefined, '_phase:build', '_phase:test']; @@ -1064,6 +1181,75 @@ describe(InputsSnapshot.name, () => { } }); + it.each([false, true])( + 'does not reuse stale environment state after caller mutation (previous hash computed: %s)', + (computePreviousHash) => { + const { projects, a, options } = getDerivationTestConfig(); + const environment: Record = { FOO: 'captured' }; + options.environment = environment; + const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); + const baselineHash: string = new InputsSnapshot(options).getOperationOwnStateHash(a, '_phase:build'); + if (computePreviousHash) { + expect(previousSnapshot.getOperationOwnStateHash(a, '_phase:build')).toBe(baselineHash); + } + + environment.FOO = 'changed'; + const derivedOptions: IInputsSnapshotParameters = { ...options, previousSnapshot }; + const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); + expect(derived.getOperationOwnStateHash(a, '_phase:build')).not.toBe(baselineHash); + expect(previousSnapshot.getOperationOwnStateHash(a, '_phase:build')).toBe(baselineHash); + expectSameAsNewSnapshot(derived, derivedOptions, projects); + } + ); + + it.each([undefined, ''])('distinguishes absent and empty environments during reuse: %s', (value) => { + const { projects, a, options } = getDerivationTestConfig(); + options.environment = { FOO: value }; + const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); + const baselineHash: string = previousSnapshot.getOperationOwnStateHash(a, '_phase:build'); + const derivedOptions: IInputsSnapshotParameters = { + ...options, + environment: { FOO: value === undefined ? '' : undefined }, + previousSnapshot + }; + const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); + expect(derived.getOperationOwnStateHash(a, '_phase:build')).not.toBe(baselineHash); + expectSameAsNewSnapshot(derived, derivedOptions, projects); + }); + + it.each(['win32', 'linux'])('uses %s environment-name semantics during reuse', (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const { projects, a, options } = getDerivationTestConfig(); + const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); + const previousFiles: ReadonlyMap = previousSnapshot.getTrackedFileHashesForOperation( + a, + '_phase:build' + ); + queryAll(previousSnapshot, projects); + + const derivedOptions: IInputsSnapshotParameters = { + ...options, + environment: { foo: '1' }, + previousSnapshot + }; + const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); + expect(derived.getTrackedFileHashesForOperation(a, '_phase:build') === previousFiles).toBe( + platform === 'win32' + ); + expectSameAsNewSnapshot(derived, derivedOptions, projects); + + const changedOptions: IInputsSnapshotParameters = { + ...derivedOptions, + environment: { foo: '2' } + }; + expectSameAsNewSnapshot(new InputsSnapshot(changedOptions), changedOptions, projects); + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + it('Computes the same state as a new snapshot after any sequence of changes', () => { const { projects, bAdditionalFiles, options } = getDerivationTestConfig(); const files: string[] = [ diff --git a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap index f8dcd553d3f..fb766150fdb 100644 --- a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap +++ b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap @@ -6,9 +6,9 @@ exports[`InputsSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"a exports[`InputsSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"47a31b2ea9deae3582aa2b3e87d0313a240220fc"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"3ded0c253e5e9d12f32fd502285cc4707e88af76"`; exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnNodeVersion 1`] = `"cc182bde6aa81c0410ede7db22f3af2f0a24d3e3"`; From 27c921f153b29a944fd2cb3889b00fc9fe88107e Mon Sep 17 00:00:00 2001 From: yashwanth195 <59129034+yashwanth195@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:52:02 -0800 Subject: [PATCH 2/3] Preserve legacy input behavior --- .../rush/environment-snapshot_2026-09-16.json | 4 +- common/reviews/api/rush-lib.api.md | 11 +- libraries/rush-lib/src/index.ts | 1 + .../rush-lib/src/logic/ProjectWatcher.ts | 33 +- .../src/logic/incremental/InputsSnapshot.ts | 232 ++++++++++---- .../incremental/test/InputsSnapshot.test.ts | 284 +++++++++++++++--- .../__snapshots__/InputsSnapshot.test.ts.snap | 4 +- .../src/logic/operations/BuildPlanPlugin.ts | 5 +- .../operations/CacheableOperationPlugin.ts | 19 +- .../operations/IOperationExecutionResult.ts | 6 + .../src/logic/operations/IOperationGraph.ts | 14 +- .../IncrementalExecutionGuardPlugin.ts | 40 ++- .../src/logic/operations/LegacySkipPlugin.ts | 34 ++- .../operations/OperationExecutionRecord.ts | 18 +- .../src/logic/operations/OperationGraph.ts | 50 ++- .../operations/OperationInputFilesCheck.ts | 10 +- .../IncrementalExecutionGuardPlugin.test.ts | 16 + .../operations/test/LegacySkipPlugin.test.ts | 54 +++- .../test/OperationGraphStateHash.test.ts | 217 ++++++++++++- .../src/logic/test/ProjectWatcher.test.ts | 102 ++++++- 20 files changed, 1000 insertions(+), 154 deletions(-) diff --git a/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json index 670090e0c5c..322b63c78f0 100644 --- a/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json +++ b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json @@ -2,8 +2,8 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "Fix declared environment input hashing to preserve Windows case-insensitive lookup, distinguish absent and empty values, and snapshot supplied environments before querying operation hashes. Operations with declared environment inputs will receive new cache keys.", - "type": "patch" + "comment": "Add optional per-operation strict input snapshots with captured environment values, collision-safe identity, and additional-input/output validation. Legacy input hashing and validation remain the default. Strict operations use versioned cache keys and bypass legacy skip and incremental-command equivalence.", + "type": "minor" } ], "packageName": "@microsoft/rush" diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 914ae2604f6..d91316b3016 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -406,6 +406,7 @@ export function getWorkspaceRequestOperationEnvironment(hostEnvironment: Readonl export interface IBaseOperationExecutionResult { getStateHash(): string; getStateHashComponents(): IOperationStateHashComponents; + readonly inputSnapshotPolicy?: InputSnapshotPolicy; readonly metadataFolderPath: string; readonly operation: Operation; } @@ -652,10 +653,11 @@ export interface IIndividualVersionJson extends IVersionPolicyJson { // @beta export interface IInputsSnapshot { - getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string, environment?: Readonly>): string; - getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string): ReadonlyMap; + getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string, environment?: Readonly>, policy?: InputSnapshotPolicy): string; + getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string, policy?: InputSnapshotPolicy): ReadonlyMap; readonly hashes: ReadonlyMap; readonly hasUncommittedChanges: boolean; + readonly inputSnapshotPolicyVersion?: 1; readonly rootDirectory: string; readonly workingTreeReadStartTimeMs?: number; } @@ -714,6 +716,9 @@ export class IndividualVersionPolicy extends VersionPolicy { export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { } +// @beta +export type InputSnapshotPolicy = 'legacy' | 'strict-v1'; + // @internal export interface _IOperationActivityOptions { readonly operationId?: string; @@ -775,6 +780,7 @@ export interface IOperationGraph { deleteResults?(operations: Iterable): void; discardScheduledIteration(): boolean; executeScheduledIterationAsync(): Promise; + getInputSnapshotPolicy?(operation: Operation): InputSnapshotPolicy; readonly hasScheduledIteration: boolean; readonly hooks: OperationGraphHooks; invalidateOperations(operations?: Iterable, reason?: string): void; @@ -788,6 +794,7 @@ export interface IOperationGraph { retainHeldOperations?(): (() => void) | undefined; scheduleIterationAsync(options: IOperationGraphIterationOptions): Promise; setEnabledStates(operations: Iterable, targetState: Operation['enabled'], mode: 'safe' | 'unsafe'): boolean; + setInputSnapshotPolicy?(operations: Iterable, policy: 'strict-v1'): void; readonly status: OperationStatus; readonly terminalDestinations: ReadonlySet; tryExtendCurrentIteration?(options: IOperationGraphExtensionOptions): IOperationGraphExtensionResult; diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index e181e2bce72..2cc3fc2c4f2 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -155,6 +155,7 @@ export { export { ProjectChangeAnalyzer, type IGetChangedProjectsOptions } from './logic/ProjectChangeAnalyzer'; export type { IInputsSnapshot, + InputSnapshotPolicy, GetInputsSnapshotAsyncFn as GetInputsSnapshotAsyncFn, IRushConfigurationProjectForSnapshot } from './logic/incremental/InputsSnapshot'; diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index aa43c903957..76b03d781e7 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -11,7 +11,12 @@ import { Path } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; import { Git } from './Git'; -import type { GetInputsSnapshotAsyncFn, IInputsSnapshot } from './incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type GetInputsSnapshotAsyncFn, + type IInputsSnapshot, + type InputSnapshotPolicy +} from './incremental/InputsSnapshot'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; import type { IOperationGraph, IOperationGraphIterationOptions } from './operations/IOperationGraph'; @@ -544,13 +549,23 @@ export class ProjectWatcher { let changedOperation: Operation | undefined; try { const currentSnapshot: IInputsSnapshot | undefined = await getInputsSnapshotAsync(); - if (this.#inputsCheck !== check || !currentSnapshot) { + if (this.#inputsCheck !== check) { + return; + } + if (!currentSnapshot) { + for (const operation of this.#graph.operations) { + assertInputSnapshotPolicySupported( + currentSnapshot, + this.#graph.getInputSnapshotPolicy?.(operation) ?? 'legacy' + ); + } return; } changedOperation = _findOperationWithChangedInputs( this.#graph.operations, iterationSnapshot, - currentSnapshot + currentSnapshot, + this.#graph.getInputSnapshotPolicy ); } catch (e) { if (this.#inputsCheck === check) { @@ -719,16 +734,20 @@ export class ProjectWatcher { * Returns an operation whose own state hash differs between the two snapshots, if there is one. The own state * hash covers the operation's tracked files, additional files and environment variables. */ -function _findOperationWithChangedInputs( +export function _findOperationWithChangedInputs( operations: Iterable, previousSnapshot: IInputsSnapshot, - currentSnapshot: IInputsSnapshot + currentSnapshot: IInputsSnapshot, + getInputSnapshotPolicy?: (operation: Operation) => InputSnapshotPolicy ): Operation | undefined { for (const operation of operations) { const { associatedProject, associatedPhase } = operation; + const policy: InputSnapshotPolicy = getInputSnapshotPolicy?.(operation) ?? 'legacy'; + assertInputSnapshotPolicySupported(previousSnapshot, policy); + assertInputSnapshotPolicySupported(currentSnapshot, policy); if ( - currentSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name) !== - previousSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name) + currentSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name, undefined, policy) !== + previousSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name, undefined, policy) ) { return operation; } diff --git a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index c46b597db55..c3e369b2fd3 100644 --- a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -25,6 +25,25 @@ export type IRushConfigurationProjectForSnapshot = Pick< 'projectFolder' | 'projectRelativeFolder' >; +/** + * Selects the compatibility contract for operation inputs. + * @beta + */ +export type InputSnapshotPolicy = 'legacy' | 'strict-v1'; + +/** + * Rejects unsupported strict queries, including snapshots supplied by older hosts. + * @internal + */ +export function assertInputSnapshotPolicySupported( + snapshot: IInputsSnapshot | undefined, + policy: InputSnapshotPolicy +): void { + if (policy !== 'legacy' && (policy !== 'strict-v1' || snapshot?.inputSnapshotPolicyVersion !== 1)) { + throw new Error('Strict operation inputs require a snapshot supporting input snapshot policy version 1.'); + } +} + /** * @internal */ @@ -90,7 +109,8 @@ export interface IInputsSnapshotParameters { */ additionalHashes?: ReadonlyMap; /** - * The environment to snapshot for `dependsOnEnvVars`. Defaults to process.env upon construction. + * The environment for `dependsOnEnvVars`. Strict queries capture its values upon construction. + * Legacy queries retain an explicitly supplied object, or copy process.env by default. * @defaultValue \{ ...process.env \} */ environment?: Record; @@ -152,6 +172,12 @@ const EMPTY_MAP: ReadonlyMap = new Map(); * @beta */ export interface IInputsSnapshot { + /** + * When 1, supports strict-v1 queries, including construction-time environment capture and output validation. + * Absent on legacy-only implementations. + */ + readonly inputSnapshotPolicyVersion?: 1; + /** * The raw hashes of all tracked files in the repository. */ @@ -183,11 +209,13 @@ export interface IInputsSnapshot { * * @param project - The Rush project to get hashes for * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a default set for the project, as used for bulk commands. + * @param policy - Defaults to legacy validation. Strict queries also reject additional inputs inside output folders. * @returns A map of file name to Git hash. For local files paths will be relative. Configured additional files may be absolute paths. */ getTrackedFileHashesForOperation( project: IRushConfigurationProjectForSnapshot, - operationName?: string + operationName?: string, + policy?: InputSnapshotPolicy ): ReadonlyMap; /** @@ -197,12 +225,15 @@ export interface IInputsSnapshot { * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a generic hash for the whole project, as used for bulk commands. * @param environment - The environment that the operation runs with, if it differs from the environment of the * snapshot. The operation's `dependsOnEnvVars` are hashed from it. + * @param policy - Defaults to legacy identity. Strict queries use captured platform-aware environment values, + * unambiguous encoding, output validation, and a versioned hash namespace. * @returns The local state hash for the project. This is a hash of the environment, the project's tracked files, and any additional files. */ getOperationOwnStateHash( project: IRushConfigurationProjectForSnapshot, operationName?: string, - environment?: Readonly> + environment?: Readonly>, + policy?: InputSnapshotPolicy ): string; } @@ -216,6 +247,8 @@ export interface IInputsSnapshot { * @internal */ export class InputsSnapshot implements IInputsSnapshot { + public readonly inputSnapshotPolicyVersion: 1 = 1; + /** * {@inheritdoc IInputsSnapshot.hashes} */ @@ -240,6 +273,9 @@ export class InputsSnapshot implements IInputsSnapshot { IRushConfigurationProjectForSnapshot, IInternalInputsSnapshotProjectMetadata >; + #strictProjectMetadataMap: + | Map + | undefined; /** * Hashes of files to be included in all result sets. */ @@ -251,7 +287,8 @@ export class InputsSnapshot implements IInputsSnapshot { /** * The environment to use for `dependsOnEnvVars`. */ - readonly #environment: EnvironmentMap; + readonly #environment: Record; + readonly #strictEnvironment: EnvironmentMap; /** * Pre-computed Node.js version strings at each granularity level for `dependsOnNodeVersion`. */ @@ -272,7 +309,7 @@ export class InputsSnapshot implements IInputsSnapshot { public constructor(params: IInputsSnapshotParameters) { const { additionalHashes, - environment = process.env, + environment = { ...process.env }, globalAdditionalFiles, hashes, hasUncommittedChanges, @@ -306,18 +343,31 @@ export class InputsSnapshot implements IInputsSnapshot { const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap); - // Snapshot the environment before comparing or querying it. - this.#environment = new EnvironmentMap(environment); + this.#environment = environment; + this.#strictEnvironment = new EnvironmentMap(environment); this.#projectMetadataMap = - (previousSnapshot && - previousSnapshot.#tryDeriveProjectMetadataMap( - params, - this.#environment, - nodeVersion, - globalAdditionalHashes, - operationAdditionalFileCount - )) ?? - createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir); + (previousSnapshot && areEnvironmentsEqual(environment, previousSnapshot.#environment) + ? previousSnapshot.#tryDeriveProjectMetadataMap( + params, + nodeVersion, + globalAdditionalHashes, + operationAdditionalFileCount, + previousSnapshot.#projectMetadataMap + ) + : undefined) ?? createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir); + if ( + previousSnapshot && + previousSnapshot.#strictProjectMetadataMap && + areEnvironmentMapsEqual(this.#strictEnvironment, previousSnapshot.#strictEnvironment) + ) { + this.#strictProjectMetadataMap = previousSnapshot.#tryDeriveProjectMetadataMap( + params, + nodeVersion, + globalAdditionalHashes, + operationAdditionalFileCount, + previousSnapshot.#strictProjectMetadataMap + ); + } this.#additionalHashes = additionalHashes; this.#globalAdditionalHashes = globalAdditionalHashes; // Parse Node.js version once so it doesn't need to be re-parsed per operation @@ -337,9 +387,11 @@ export class InputsSnapshot implements IInputsSnapshot { */ public getTrackedFileHashesForOperation( project: IRushConfigurationProjectForSnapshot, - operationName?: string + operationName?: string, + policy: InputSnapshotPolicy = 'legacy' ): ReadonlyMap { - const record: IInternalInputsSnapshotProjectMetadata | undefined = this.#projectMetadataMap.get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = + this.#getProjectMetadataMap(policy).get(project); if (!record) { throw new InternalError(`No information available for project at ${project.projectFolder}`); } @@ -352,6 +404,22 @@ export class InputsSnapshot implements IInputsSnapshot { const filter: (filePath: string) => boolean = getOrCreateProjectFilter(record); let outputValidator: LookupByPath | undefined; + const validateInputPath: (filePath: string, inputKind: 'tracked' | 'additional') => void = ( + filePath, + inputKind + ) => { + // This should eventually use a global file origin map to detect outputs of *any* operation. + const outputMatch: string | undefined = outputValidator?.findChildPath( + policy === 'strict-v1' ? getOutputLookupPath(this.rootDirectory, filePath) : filePath + ); + if (outputMatch) { + throw new Error( + `Configured output folder "${outputMatch}" for operation "${operationName}" in project "${project.projectRelativeFolder}" contains ${inputKind} input file "${filePath}".` + + ` If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from "outputFolderNames".` + + ` This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder.` + ); + } + }; if (operationName) { const operationSettings: Readonly | undefined = @@ -359,10 +427,14 @@ export class InputsSnapshot implements IInputsSnapshot { const outputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; if (outputFolderNames) { - const { relativePrefix } = record; outputValidator = new LookupByPath(); for (const folderName of outputFolderNames) { - outputValidator.setItem(`${relativePrefix}/${folderName}`, folderName); + outputValidator.setItem( + policy === 'strict-v1' + ? getOutputLookupPath(this.rootDirectory, path.resolve(project.projectFolder, folderName)) + : `${record.relativePrefix}/${folderName}`, + folderName + ); } } @@ -373,6 +445,9 @@ export class InputsSnapshot implements IInputsSnapshot { // Sort the additional files to ensure deterministic hash computation const sortedAdditionalFiles: string[] = Array.from(additionalFilesForOperation).sort(); for (const [filePath, hash] of this.#resolveHashes(sortedAdditionalFiles)) { + if (policy === 'strict-v1') { + validateInputPath(filePath, 'additional'); + } hashes.set(filePath, hash); } } @@ -391,17 +466,7 @@ export class InputsSnapshot implements IInputsSnapshot { hashes.set(filePath, hash); } - // Ensure that the configured output folders for this operation do not contain any input files - // This should be reworked to operate on a global file origin map to ensure a hashed input - // is not a declared output of *any* operation. - const outputMatch: string | undefined = outputValidator?.findChildPath(filePath); - if (outputMatch) { - throw new Error( - `Configured output folder "${outputMatch}" for operation "${operationName}" in project "${project.projectRelativeFolder}" contains tracked input file "${filePath}".` + - ` If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from "outputFolderNames".` + - ` This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder.` - ); - } + validateInputPath(filePath, 'tracked'); } // Only a complete result is kept, so that a query that fails fails again @@ -417,9 +482,11 @@ export class InputsSnapshot implements IInputsSnapshot { public getOperationOwnStateHash( project: IRushConfigurationProjectForSnapshot, operationName?: string, - environment?: Readonly> + environment?: Readonly>, + policy: InputSnapshotPolicy = 'legacy' ): string { - const record: IInternalInputsSnapshotProjectMetadata | undefined = this.#projectMetadataMap.get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = + this.#getProjectMetadataMap(policy).get(project); if (!record) { throw new Error(`No information available for project at ${project.projectFolder}`); } @@ -427,20 +494,25 @@ export class InputsSnapshot implements IInputsSnapshot { const operationSettings: Readonly | undefined = operationName ? record.projectConfig?.operationSettingsByOperationName.get(operationName) : undefined; - const snapshotEnvironment: EnvironmentMap = this.#environment; + const snapshotEnvironment: EnvironmentMap | Readonly> = + policy === 'strict-v1' ? this.#strictEnvironment : this.#environment; if (environment && operationSettings?.dependsOnEnvVars?.length) { - const operationEnvironment: EnvironmentMap = new EnvironmentMap(environment); + const operationEnvironment: EnvironmentMap | Readonly> = + policy === 'strict-v1' ? new EnvironmentMap(environment) : environment; if ( - operationSettings.dependsOnEnvVars.some( - (envVar: string) => operationEnvironment.get(envVar) !== snapshotEnvironment.get(envVar) - ) + operationSettings.dependsOnEnvVars.some((envVar: string) => { + const value: string | undefined = getEnvironmentValue(operationEnvironment, envVar); + const snapshotValue: string | undefined = getEnvironmentValue(snapshotEnvironment, envVar); + return policy === 'strict-v1' ? value !== snapshotValue : (value || '') !== (snapshotValue || ''); + }) ) { // The operation's environment differs from the snapshot's in a variable that it hashes, so don't memoize. return this.#computeOperationOwnStateHash( project, operationName, operationSettings, - operationEnvironment + operationEnvironment, + policy ); } } @@ -452,7 +524,8 @@ export class InputsSnapshot implements IInputsSnapshot { project, operationName, operationSettings, - snapshotEnvironment + snapshotEnvironment, + policy ); hashByOperationName.set(operationName, hash); } @@ -464,11 +537,19 @@ export class InputsSnapshot implements IInputsSnapshot { project: IRushConfigurationProjectForSnapshot, operationName: string | undefined, operationSettings: Readonly | undefined, - environment: EnvironmentMap + environment: EnvironmentMap | Readonly>, + policy: InputSnapshotPolicy ): string { - const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation(project, operationName); + const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation( + project, + operationName, + policy + ); const hasher: Hash = createHash('sha1'); + if (policy === 'strict-v1') { + hasher.update('rush-input-snapshot:strict-v1\0'); + } // If this is for a specific operation, apply operation-specific options if (operationSettings) { const { dependsOnEnvVars, dependsOnNodeVersion, outputFolderNames } = operationSettings; @@ -476,8 +557,12 @@ export class InputsSnapshot implements IInputsSnapshot { // As long as we enumerate environment variables in a consistent order, we will get a stable hash. // Changing the order in rush-project.json will change the hash anyway since the file contents are part of the hash. for (const envVar of dependsOnEnvVars) { - // Encode absence separately from an empty value, and escape delimiters within values. - hasher.update(`${hashDelimiter}$${JSON.stringify([envVar, environment.get(envVar)])}`); + const value: string | undefined = getEnvironmentValue(environment, envVar); + hasher.update( + policy === 'strict-v1' + ? `${hashDelimiter}$${JSON.stringify([envVar, value])}` + : `${hashDelimiter}$${envVar}=${value || ''}` + ); } } @@ -500,6 +585,21 @@ export class InputsSnapshot implements IInputsSnapshot { return hasher.digest('hex'); } + #getProjectMetadataMap( + policy: InputSnapshotPolicy + ): Map { + assertInputSnapshotPolicySupported(this, policy); + if (policy === 'legacy') { + return this.#projectMetadataMap; + } + return (this.#strictProjectMetadataMap ??= createProjectMetadataMap( + this.hashes, + this.#lookupByPath, + this.#projectMap, + this.rootDirectory + )); + } + *#resolveHashes(filePaths: Iterable): Generator<[string, string]> { const { hashes } = this; const additionalHashes: ReadonlyMap | undefined = this.#additionalHashes; @@ -520,10 +620,13 @@ export class InputsSnapshot implements IInputsSnapshot { */ #tryDeriveProjectMetadataMap( params: IInputsSnapshotParameters, - environment: EnvironmentMap, nodeVersion: string, globalAdditionalHashes: ReadonlyMap | undefined, - operationAdditionalFileCount: number + operationAdditionalFileCount: number, + previousMetadataMap: ReadonlyMap< + IRushConfigurationProjectForSnapshot, + IInternalInputsSnapshotProjectMetadata + > ): Map | undefined { const { additionalHashes, hashes, lookupByPath, projectMap, rootDir } = params; if ( @@ -532,8 +635,7 @@ export class InputsSnapshot implements IInputsSnapshot { rootDir !== this.rootDirectory || nodeVersion !== this.#nodeVersion || operationAdditionalFileCount !== this.#operationAdditionalFileCount || - !areMapsEqual(globalAdditionalHashes ?? EMPTY_MAP, this.#globalAdditionalHashes ?? EMPTY_MAP) || - !areEnvironmentsEqual(environment, this.#environment) + !areMapsEqual(globalAdditionalHashes ?? EMPTY_MAP, this.#globalAdditionalHashes ?? EMPTY_MAP) ) { return undefined; } @@ -542,10 +644,6 @@ export class InputsSnapshot implements IInputsSnapshot { addChangedFiles(this.hashes, hashes, changedFiles); addChangedFiles(this.#additionalHashes ?? EMPTY_MAP, additionalHashes ?? EMPTY_MAP, changedFiles); - const previousMetadataMap: ReadonlyMap< - IRushConfigurationProjectForSnapshot, - IInternalInputsSnapshotProjectMetadata - > = this.#projectMetadataMap; const projectMetadataMap: Map< IRushConfigurationProjectForSnapshot, IInternalInputsSnapshotProjectMetadata @@ -756,7 +854,30 @@ function areMapsEqual(a: ReadonlyMap, b: ReadonlyMap>, + name: string +): string | undefined { + return environment instanceof EnvironmentMap ? environment.get(name) : environment[name]; +} + +function areEnvironmentsEqual( + a: Readonly>, + b: Readonly> +): boolean { + const keys: string[] = Object.keys(a); + if (keys.length !== Object.keys(b).length) { + return false; + } + for (const key of keys) { + if (a[key] !== b[key] || !Object.prototype.hasOwnProperty.call(b, key)) { + return false; + } + } + return true; +} + +function areEnvironmentMapsEqual(a: EnvironmentMap, b: EnvironmentMap): boolean { const keys: string[] = Array.from(a.names()); if (keys.length !== Array.from(b.names()).length) { return false; @@ -827,6 +948,11 @@ function createInternalRecord( }; } +function getOutputLookupPath(rootDir: string, filePath: string): string { + const relativePath: string = Path.convertToSlashes(path.relative(rootDir, path.resolve(rootDir, filePath))); + return process.platform === 'win32' ? relativePath.toLowerCase() : relativePath; +} + function getRelativePrefix(project: IRushConfigurationProjectForSnapshot, rootDir: string): string { return Path.convertToSlashes(path.relative(rootDir, project.projectFolder)); } diff --git a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts index 4ed187c5b05..21270ca3d5f 100644 --- a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts +++ b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as path from 'node:path'; + import { LookupByPath } from '@rushstack/lookup-by-path'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; @@ -9,6 +11,7 @@ import { InputsSnapshot, type IInputsSnapshotParameters, type IInputsSnapshotProjectMetadata, + type InputSnapshotPolicy, type IRushConfigurationProjectForSnapshot } from '../InputsSnapshot'; @@ -51,6 +54,14 @@ describe(InputsSnapshot.name, () => { return { project, input }; } + function getStrictHash( + input: InputsSnapshot, + project: IRushConfigurationProjectForSnapshot, + environment?: Record + ): string { + return input.getOperationOwnStateHash(project, '_phase:build', environment, 'strict-v1'); + } + it('Exposes the time at which it began reading the working tree', () => { const { options } = getTestConfig(); expect(new InputsSnapshot(options).workingTreeReadStartTimeMs).toBeUndefined(); @@ -178,6 +189,116 @@ describe(InputsSnapshot.name, () => { expect(result.get('/ext/config.json')).toEqual('hash4'); }); + function getOutputSnapshot( + inputKind: 'tracked' | 'additional', + filePath: string + ): ReturnType { + const { project, options } = getTestConfig(); + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + ['_phase:build', { operationName: '_phase:build', outputFolderNames: ['lib'] }] + ]) + }; + return { + project, + input: new InputsSnapshot({ + ...options, + hashes: new Map([ + ['a/source.js', 'source-hash'], + ...(inputKind === 'tracked' ? [[filePath, 'uncovered-hash'] as const] : []) + ]), + additionalHashes: new Map([[filePath, 'uncovered-hash']]), + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration, + additionalFilesByOperationName: + inputKind === 'additional' ? new Map([['_phase:build', new Set([filePath])]]) : undefined + } + ] + ]) + }) + }; + } + + it.each([ + ['tracked', 'a/lib/uncovered.js'], + ['additional', 'a/lib/uncovered.js'], + ['additional', 'a/lib/../lib/uncovered.js'], + ['additional', path.resolve('/root/a/lib/uncovered.js')] + ] as const)('Rejects %s uncovered output inputs before operation hashing: %s', (inputKind, filePath) => { + const { project, input } = getOutputSnapshot(inputKind, filePath); + // Repeated failures must not expose a partially validated input map or cached operation hash. + for (let attempt: number = 0; attempt < 2; attempt++) { + expect(() => getStrictHash(input, project)).toThrow(`contains ${inputKind} input file "${filePath}"`); + expect(() => input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1')).toThrow( + 'output to a disjoint folder' + ); + } + }); + + it.each(['win32', 'linux'])('Uses %s path casing for output input validation', (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + for (const [inputKind, filePath] of [ + ['tracked', 'a/LIB/UNCOVERED.js'], + ['additional', 'A/LIB/UNCOVERED.js'], + ['additional', path.resolve('/root/A/LIB/UNCOVERED.js')] + ] as const) { + const { project, input } = getOutputSnapshot(inputKind, filePath); + if (platform === 'win32') { + expect(() => getStrictHash(input, project)).toThrow( + `contains ${inputKind} input file "${filePath}"` + ); + } else { + expect( + input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1').get(filePath) + ).toBe('uncovered-hash'); + } + } + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + + it.each([ + 'a/lib-sources/uncovered.js', + path.resolve('/root/a/lib-sources/uncovered.js'), + path.resolve('/ext/config.json') + ])('Preserves valid additional input keys and hashes outside output folders: %s', (filePath) => { + const { project, input } = getOutputSnapshot('additional', filePath); + const hashes: ReadonlyMap = input.getTrackedFileHashesForOperation( + project, + '_phase:build', + 'strict-v1' + ); + expect([...hashes]).toEqual([ + [filePath, 'uncovered-hash'], + ['a/source.js', 'source-hash'] + ]); + expect(input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1')).toBe(hashes); + expect(() => getStrictHash(input, project)).not.toThrow(); + }); + + it.each([false, true])( + 'keeps strict validation separate from legacy queries (strict first: %s)', + (strictFirst) => { + const { project, input } = getOutputSnapshot('additional', 'a/lib/uncovered.js'); + const strictQuery = (): string => getStrictHash(input, project); + if (strictFirst) { + expect(strictQuery).toThrow('contains additional input file'); + } + const legacyHash: string = input.getOperationOwnStateHash(project, '_phase:build'); + expect( + input.getTrackedFileHashesForOperation(project, '_phase:build').get('a/lib/uncovered.js') + ).toBe('uncovered-hash'); + expect(strictQuery).toThrow('contains additional input file'); + expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(legacyHash); + } + ); + it('Respects globalAdditionalFiles', () => { const { project, options } = getTestConfig(); @@ -823,11 +944,7 @@ describe(InputsSnapshot.name, () => { useOperationEnvironment ? { ENV_VAR: 'snapshot' } : environment, names ); - return input.getOperationOwnStateHash( - project, - '_phase:build', - useOperationEnvironment ? environment : undefined - ); + return getStrictHash(input, project, useOperationEnvironment ? environment : undefined); } it('distinguishes absence, empty values and literal sentinel strings', () => { @@ -875,14 +992,48 @@ describe(InputsSnapshot.name, () => { const { project, input } = createSnapshot(environment); const { project: baselineProject, input: baseline } = createSnapshot({ ...environment }); environment.ENV_VAR = 'changed'; + expect(getStrictHash(input, project)).toBe(getStrictHash(baseline, baselineProject)); + expect(getStrictHash(input, project, environment)).not.toBe(getStrictHash(input, project)); expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe( - baseline.getOperationOwnStateHash(baselineProject, '_phase:build') + input.getOperationOwnStateHash(project, '_phase:build', environment) ); - expect(input.getOperationOwnStateHash(project, '_phase:build', environment)).not.toBe( - input.getOperationOwnStateHash(project, '_phase:build') + expect(input.getOperationOwnStateHash(project, '_phase:build')).not.toBe( + baseline.getOperationOwnStateHash(baselineProject, '_phase:build') ); }); + it.each(['win32', 'linux'])('preserves raw legacy environment identity on %s', (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const getLegacyHash = (environment: Record): string => { + const { input, project } = createSnapshot(environment); + const hash: string = input.getOperationOwnStateHash(project, '_phase:build'); + expect(input.getOperationOwnStateHash(project, '_phase:build', { ENV_VAR: undefined })).toBe(hash); + return hash; + }; + expect(getLegacyHash({})).toBe(getLegacyHash({ ENV_VAR: '' })); + expect(getLegacyHash({ env_var: 'nonempty' })).toBe(getLegacyHash({})); + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + + it.each([false, true])( + 'partitions identities with no declared environment (strict first: %s)', + (strictFirst) => { + const { project, input } = createSnapshot({}, []); + if (strictFirst) { + getStrictHash(input, project); + } + const legacy: string = input.getOperationOwnStateHash(project, '_phase:build'); + const strict: string = getStrictHash(input, project); + expect(strict).not.toBe(legacy); + expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(legacy); + expect(getStrictHash(input, project)).toBe(strict); + } + ); + it.each([undefined, ''])( 'does not reuse or overwrite the snapshot hash for a different operation value (snapshot: %s)', (snapshotValue) => { @@ -891,23 +1042,14 @@ describe(InputsSnapshot.name, () => { ENV_VAR: snapshotValue === undefined ? '' : undefined }; const { project: operationProject, input: operationSnapshot } = createSnapshot(operationEnvironment); - const operationHash: string = operationSnapshot.getOperationOwnStateHash( - operationProject, - '_phase:build' - ); - expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( - operationHash - ); - const snapshotHash: string = input.getOperationOwnStateHash(project, '_phase:build'); + const operationHash: string = getStrictHash(operationSnapshot, operationProject); + expect(getStrictHash(input, project, operationEnvironment)).toBe(operationHash); + const snapshotHash: string = getStrictHash(input, project); expect(snapshotHash).not.toBe(operationHash); - expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( - operationHash - ); + expect(getStrictHash(input, project, operationEnvironment)).toBe(operationHash); operationEnvironment.ENV_VAR = snapshotValue; - expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( - snapshotHash - ); - expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(snapshotHash); + expect(getStrictHash(input, project, operationEnvironment)).toBe(snapshotHash); + expect(getStrictHash(input, project)).toBe(snapshotHash); } ); }); @@ -1003,18 +1145,27 @@ describe(InputsSnapshot.name, () => { }; } - function queryAll(snapshot: InputsSnapshot, projects: IRushConfigurationProjectForSnapshot[]): string[] { + function queryAll( + snapshot: InputsSnapshot, + projects: IRushConfigurationProjectForSnapshot[], + policy: InputSnapshotPolicy = 'legacy' + ): string[] { const results: string[] = []; for (const project of projects) { for (const operationName of operationNames) { try { const tracked: [string, string][] = Array.from( - snapshot.getTrackedFileHashesForOperation(project, operationName) + snapshot.getTrackedFileHashesForOperation(project, operationName, policy) + ); + const hash: string = snapshot.getOperationOwnStateHash(project, operationName, undefined, policy); + const hashInOtherEnvironment: string = snapshot.getOperationOwnStateHash( + project, + operationName, + { + FOO: 'other' + }, + policy ); - const hash: string = snapshot.getOperationOwnStateHash(project, operationName); - const hashInOtherEnvironment: string = snapshot.getOperationOwnStateHash(project, operationName, { - FOO: 'other' - }); results.push(JSON.stringify({ tracked, hash, hashInOtherEnvironment })); } catch (error) { results.push(`error: ${error.message}`); @@ -1028,15 +1179,55 @@ describe(InputsSnapshot.name, () => { function expectSameAsNewSnapshot( derived: InputsSnapshot, options: IInputsSnapshotParameters, - projects: IRushConfigurationProjectForSnapshot[] + projects: IRushConfigurationProjectForSnapshot[], + policy: InputSnapshotPolicy = 'legacy' ): void { const created: InputsSnapshot = new InputsSnapshot({ ...options, previousSnapshot: undefined }); - expect(queryAll(derived, projects)).toEqual(queryAll(created, projects)); + expect(queryAll(derived, projects, policy)).toEqual(queryAll(created, projects, policy)); expect(derived.hashes).toBe(options.hashes); expect(derived.hasUncommittedChanges).toBe(options.hasUncommittedChanges); expect(derived.workingTreeReadStartTimeMs).toBe(options.workingTreeReadStartTimeMs); } + it.each([false, true])( + 'reuses raw legacy and captured strict state independently (strict first: %s)', + (strictFirst) => { + const { options, a } = getDerivationTestConfig(); + const environment: Record = { FOO: 'old' }; + const previousSnapshot: InputsSnapshot = new InputsSnapshot({ ...options, environment }); + if (strictFirst) { + getStrictHash(previousSnapshot, a); + } + const legacy: string = previousSnapshot.getOperationOwnStateHash(a, '_phase:build'); + const strict: string = getStrictHash(previousSnapshot, a); + environment.FOO = 'new'; + const next: InputsSnapshot = new InputsSnapshot({ ...options, environment, previousSnapshot }); + const fresh: InputsSnapshot = new InputsSnapshot({ ...options, environment }); + expect(next.getOperationOwnStateHash(a, '_phase:build')).toBe(legacy); + expect(fresh.getOperationOwnStateHash(a, '_phase:build')).not.toBe(legacy); + expect(getStrictHash(next, a)).not.toBe(strict); + expect(getStrictHash(next, a)).toBe(getStrictHash(fresh, a)); + expect(getStrictHash(previousSnapshot, a)).toBe(strict); + } + ); + + it.each(['legacy', 'strict-v1'] as const)('reuses only validated %s maps across snapshots', (policy) => { + const { options, a, b, projects } = getDerivationTestConfig(); + const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); + queryAll(previousSnapshot, projects, policy); + const hashes: Map = new Map(options.hashes); + hashes.set('a/src/x.ts', 'changed'); + const nextOptions: IInputsSnapshotParameters = { ...options, hashes, previousSnapshot }; + const next: InputsSnapshot = new InputsSnapshot(nextOptions); + expect(next.getTrackedFileHashesForOperation(b, '_phase:build', policy)).toBe( + previousSnapshot.getTrackedFileHashesForOperation(b, '_phase:build', policy) + ); + expect(next.getTrackedFileHashesForOperation(a, '_phase:build', policy)).not.toBe( + previousSnapshot.getTrackedFileHashesForOperation(a, '_phase:build', policy) + ); + expectSameAsNewSnapshot(next, nextOptions, projects, policy); + }); + it('Reuses the state of each project whose inputs did not change', () => { const { projects, a, b, options } = getDerivationTestConfig(); const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); @@ -1188,17 +1379,17 @@ describe(InputsSnapshot.name, () => { const environment: Record = { FOO: 'captured' }; options.environment = environment; const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - const baselineHash: string = new InputsSnapshot(options).getOperationOwnStateHash(a, '_phase:build'); + const baselineHash: string = getStrictHash(new InputsSnapshot(options), a); if (computePreviousHash) { - expect(previousSnapshot.getOperationOwnStateHash(a, '_phase:build')).toBe(baselineHash); + expect(getStrictHash(previousSnapshot, a)).toBe(baselineHash); } environment.FOO = 'changed'; const derivedOptions: IInputsSnapshotParameters = { ...options, previousSnapshot }; const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect(derived.getOperationOwnStateHash(a, '_phase:build')).not.toBe(baselineHash); - expect(previousSnapshot.getOperationOwnStateHash(a, '_phase:build')).toBe(baselineHash); - expectSameAsNewSnapshot(derived, derivedOptions, projects); + expect(getStrictHash(derived, a)).not.toBe(baselineHash); + expect(getStrictHash(previousSnapshot, a)).toBe(baselineHash); + expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); } ); @@ -1206,15 +1397,15 @@ describe(InputsSnapshot.name, () => { const { projects, a, options } = getDerivationTestConfig(); options.environment = { FOO: value }; const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - const baselineHash: string = previousSnapshot.getOperationOwnStateHash(a, '_phase:build'); + const baselineHash: string = getStrictHash(previousSnapshot, a); const derivedOptions: IInputsSnapshotParameters = { ...options, environment: { FOO: value === undefined ? '' : undefined }, previousSnapshot }; const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect(derived.getOperationOwnStateHash(a, '_phase:build')).not.toBe(baselineHash); - expectSameAsNewSnapshot(derived, derivedOptions, projects); + expect(getStrictHash(derived, a)).not.toBe(baselineHash); + expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); }); it.each(['win32', 'linux'])('uses %s environment-name semantics during reuse', (platform) => { @@ -1225,9 +1416,10 @@ describe(InputsSnapshot.name, () => { const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); const previousFiles: ReadonlyMap = previousSnapshot.getTrackedFileHashesForOperation( a, - '_phase:build' + '_phase:build', + 'strict-v1' ); - queryAll(previousSnapshot, projects); + queryAll(previousSnapshot, projects, 'strict-v1'); const derivedOptions: IInputsSnapshotParameters = { ...options, @@ -1235,16 +1427,16 @@ describe(InputsSnapshot.name, () => { previousSnapshot }; const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect(derived.getTrackedFileHashesForOperation(a, '_phase:build') === previousFiles).toBe( - platform === 'win32' - ); - expectSameAsNewSnapshot(derived, derivedOptions, projects); + expect( + derived.getTrackedFileHashesForOperation(a, '_phase:build', 'strict-v1') === previousFiles + ).toBe(platform === 'win32'); + expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); const changedOptions: IInputsSnapshotParameters = { ...derivedOptions, environment: { foo: '2' } }; - expectSameAsNewSnapshot(new InputsSnapshot(changedOptions), changedOptions, projects); + expectSameAsNewSnapshot(new InputsSnapshot(changedOptions), changedOptions, projects, 'strict-v1'); } finally { Object.defineProperty(process, 'platform', descriptor); } diff --git a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap index fb766150fdb..f8dcd553d3f 100644 --- a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap +++ b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap @@ -6,9 +6,9 @@ exports[`InputsSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"a exports[`InputsSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"47a31b2ea9deae3582aa2b3e87d0313a240220fc"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"3ded0c253e5e9d12f32fd502285cc4707e88af76"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnNodeVersion 1`] = `"cc182bde6aa81c0410ede7db22f3af2f0a24d3e3"`; diff --git a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts index 12299578730..4041336870f 100644 --- a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts @@ -14,6 +14,7 @@ import { clusterOperations, type IOperationBuildCacheContext } from './Cacheable import { DisjointSet } from '../cobuild/DisjointSet'; import type { IConfigurableOperation } from './IOperationExecutionResult'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { assertInputSnapshotPolicySupported, type InputSnapshotPolicy } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'BuildPlanPlugin' = 'BuildPlanPlugin'; @@ -68,11 +69,13 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { for (const operation of operations) { const { associatedProject, associatedPhase } = operation; + const policy: InputSnapshotPolicy = recordByOperation.get(operation)?.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(associatedProject); const fileHashes: ReadonlyMap | undefined = - inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); + inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name, policy); if (!fileHashes) { continue; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index c1c7020f8a9..8cd9e292cc8 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -70,7 +70,11 @@ import { setTrustedStateHash } from './RetainedResultVerification'; import { getSnapshotStartTimeMs } from './OperationInputFilesCheck'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import { isBuildCacheReadSkipped, wasExecutedIncrementally } from './IncrementalExecutionState'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; @@ -326,7 +330,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { operation, inputsSnapshot, snapshotStartTimeMs, - projectConfigurations.get(associatedProject) + projectConfigurations.get(associatedProject), + graph.getInputSnapshotPolicy?.(operation) ) }; // Upstream runners may mutate the property of build cache context for downstream runners @@ -390,7 +395,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { operation, inputsSnapshot, snapshotStartTimeMs, - context.projectConfigurations.get(operation.associatedProject) + context.projectConfigurations.get(operation.associatedProject), + graph.getInputSnapshotPolicy?.(operation) ) ); } @@ -1209,18 +1215,21 @@ function readCacheInputs( operation: Operation, inputsSnapshot: IInputsSnapshot, snapshotStartTimeMs: number, - projectConfiguration: RushProjectConfiguration | undefined + projectConfiguration: RushProjectConfiguration | undefined, + policy: InputSnapshotPolicy = 'legacy' ): Pick< IOperationBuildCacheContext, 'getCacheDisabledReason' | 'inputsSnapshot' | 'snapshotStartTimeMs' | 'inputFileHashes' > { const { name: phaseName } = operation.associatedPhase; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); // This value can *currently* be cached per-project, but in the future the list of files will vary // depending on the selected phase. const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( operation.associatedProject, - phaseName + phaseName, + policy ); // Computing the reason checks each tracked file of the project, and an iteration of a long-lived graph diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index 8400f371d59..8ae8ddbe6c0 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -8,6 +8,7 @@ import type { IOperationLastState } from './IOperationRunner'; import type { Operation } from './Operation'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { ILogFilePaths } from './ProjectLogWritable'; +import type { InputSnapshotPolicy } from '../incremental/InputsSnapshot'; /** * Structured components of the state hash for an operation. @@ -33,6 +34,11 @@ export interface IOperationStateHashComponents { * @alpha */ export interface IBaseOperationExecutionResult { + /** + * This operation's frozen graph-owned input policy. Absent on legacy-only hosts. + */ + readonly inputSnapshotPolicy?: InputSnapshotPolicy; + /** * The operation itself */ diff --git a/libraries/rush-lib/src/logic/operations/IOperationGraph.ts b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts index 85d73919c4a..387c1f98aad 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationGraph.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts @@ -7,7 +7,7 @@ import type { Operation } from './Operation'; import type { IExecutionResult, IOperationExecutionResult } from './IOperationExecutionResult'; import type { Parallelism } from './ParseParallelism'; import type { OperationStatus } from './OperationStatus'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot, InputSnapshotPolicy } from '../incremental/InputsSnapshot'; import type { OperationGraphHooks } from '../../pluginFramework/OperationGraphHooks'; /** @@ -189,6 +189,18 @@ export interface IOperationGraphRequestResult extends IExecutionResult { * @alpha */ export interface IOperationGraph { + /** + * Atomically opts operations into strict input identity and validation. Must be called before the first + * iteration is requested. Unsupported on older hosts; plugins requiring strict inputs must fail if absent. + * Selection is frozen before iteration snapshot acquisition and cannot change for the life of this graph. + */ + setInputSnapshotPolicy?(operations: Iterable, policy: 'strict-v1'): void; + + /** + * Reads this graph's input policy for an operation. Unselected operations use legacy semantics. + */ + getInputSnapshotPolicy?(operation: Operation): InputSnapshotPolicy; + /** * Hooks into the execution process for operations */ diff --git a/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts b/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts index 78939acc3b9..41bb07ea659 100644 --- a/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts @@ -8,7 +8,11 @@ import * as path from 'node:path'; import { FileSystem, InternalError, Path } from '@rushstack/node-core-library'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import type { IOperationGraphContext, IPhasedCommandPlugin, @@ -72,6 +76,7 @@ const TOOLING_PACKAGE_NAME_REGEXP: RegExp = * The inputs of an operation that decide whether it may run its incremental command, as of one inputs snapshot. */ export interface IIncrementalInputState { + readonly inputSnapshotPolicy?: InputSnapshotPolicy; /** * The path of the operation's project folder relative to the root of the inputs snapshot, with a trailing `/`, * or an empty string for a project at the root. @@ -121,10 +126,13 @@ export function captureIncrementalInputState( const { operation } = record; const { associatedProject: project, associatedPhase: phase, settings } = operation; const { config } = record.getStateHashComponents(); + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); const inputFiles: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( project, - phase.name + phase.name, + policy ); const projectPrefix: string = getProjectPrefix(inputsSnapshot, project); const inputPathsHash: Hash = createHash('sha1'); @@ -146,6 +154,7 @@ export function captureIncrementalInputState( } return { + ...(policy === 'strict-v1' ? { inputSnapshotPolicy: policy } : {}), projectPrefix, configHash: config, environment: (settings?.dependsOnEnvVars ?? []) @@ -187,6 +196,9 @@ export function getIncrementalInputChangeReason( lastState: IIncrementalInputState, currentState: IIncrementalInputState ): string | undefined { + if (lastState.inputSnapshotPolicy === 'strict-v1' || currentState.inputSnapshotPolicy === 'strict-v1') { + return 'strict input policy does not use legacy incremental-command equivalence'; + } if (lastState.configHash !== currentState.configHash) { return 'its command line changed'; } @@ -378,7 +390,7 @@ function applyToGraph(graph: IOperationGraph): void { // Read before the command starts, so that the next check notices a folder that is recreated while it runs, // e.g. during the first run in a new warm worker, which no check precedes. The check reads them first if its // base has them. - recordState.inputFolders ??= readInputFolderIdentities(record.operation, recordState.inputsSnapshot); + recordState.inputFolders ??= readInputFolderIdentities(record, recordState.inputsSnapshot); }); } @@ -393,9 +405,11 @@ function applyToGraph(graph: IOperationGraph): void { const { runner, associatedProject: project, associatedPhase: phase } = record.operation; if (recordState && record.enabled && runner && !runner.isNoOp && !areInputFilesChecked(record)) { const { inputsSnapshot, snapshotStartTimeMs } = recordState; + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); recordState.inputFilesState = captureInputFilesState( inputsSnapshot.rootDirectory, - inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name).keys(), + inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name, policy).keys(), snapshotStartTimeMs ); } @@ -408,6 +422,9 @@ function applyToGraph(graph: IOperationGraph): void { { outputsMayBeBundles = false }: IIncrementalExecutionGuardOptions = {} ): Promise { const { operation } = record; + if (record.inputSnapshotPolicy === 'strict-v1') { + return 'strict input policy does not use legacy incremental-command equivalence'; + } const cleanOnlyReason: string | undefined = cleanOnlyReasonByOperation.get(operation); if (cleanOnlyReason) { return cleanOnlyReason; @@ -422,7 +439,7 @@ function applyToGraph(graph: IOperationGraph): void { if (base.inputFolders) { // Read before the command starts, so that the next check notices a folder that is recreated while it runs. const inputFolders: ReadonlyMap = readInputFolderIdentities( - operation, + record, recordState.inputsSnapshot ); recordState.inputFolders = inputFolders; @@ -543,6 +560,10 @@ function applyToGraph(graph: IOperationGraph): void { const recordState: IRecordState | undefined = stateByRecord.get(record); stateByRecord.delete(record); clearIncrementalExecutionGuard(record); + if (record.inputSnapshotPolicy === 'strict-v1') { + forgetBase(operation); + return; + } const execution: ICommandExecution | undefined = getCommandExecution(record); if (!execution) { @@ -689,13 +710,18 @@ function formatExamplePaths(paths: string[]): string { * the folder relative to the root of the inputs snapshot. */ function readInputFolderIdentities( - operation: Operation, + record: IOperationExecutionResult, inputsSnapshot: IInputsSnapshot ): ReadonlyMap { + const { operation } = record; + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); const { associatedProject: project, associatedPhase: phase } = operation; const projectPrefix: string = getProjectPrefix(inputsSnapshot, project); const folderPaths: Set = new Set(); - for (const filePath of inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name).keys()) { + for (const filePath of inputsSnapshot + .getTrackedFileHashesForOperation(project, phase.name, policy) + .keys()) { if (filePath.startsWith(projectPrefix) && !path.isAbsolute(filePath)) { folderPaths.add(filePath.slice(0, Math.max(filePath.lastIndexOf('/'), 0))); } diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts index 10d51eb14aa..579726e139c 100644 --- a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -9,7 +9,11 @@ import { PrintUtilities, Colorize, type ITerminal } from '@rushstack/terminal'; import type { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import type { IOperationGraphIterationOptions } from './IOperationGraph'; import type { IOperationRunnerContext } from './IOperationRunner'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; @@ -138,6 +142,8 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { for (const record of operations.values()) { const { operation } = record; const { runner } = operation; + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); if (!runner) { continue; } @@ -162,7 +168,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { stateMap.set(operation, { packageDepsPath, - allowSkip, + allowSkip: policy === 'legacy' && allowSkip, dependencyChanged: false, inputsUnchanged: false, ...readInputs(record, inputsSnapshot, snapshotStartTimeMs) @@ -215,6 +221,22 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { snapshotStartTimeMs: number ): Pick { const { associatedProject, associatedPhase, runner } = record.operation; + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); + if (policy === 'strict-v1') { + const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( + associatedProject, + associatedPhase.name, + policy + ); + return { + packageDeps: undefined, + inputFilesCheck: + record.enabled && !runner!.isNoOp + ? { inputsSnapshot, snapshotStartTimeMs, fileHashes } + : undefined + }; + } try { const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( associatedProject, @@ -280,6 +302,12 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { return; } + if (record.inputSnapshotPolicy === 'strict-v1') { + // A strict run must not consume or leave a receipt written using legacy equivalence. + await FileSystem.deleteFileAsync(packageDepsPath); + return; + } + let lastProjectDeps: IProjectDeps | undefined = undefined; try { @@ -398,7 +426,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { } } - if (!record.operation.runner!.cacheable) { + if (!record.operation.runner!.cacheable || record.inputSnapshotPolicy === 'strict-v1') { // This operation doesn't support skip detection. return; } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 3cf1622541d..0ccbb9a422a 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -30,7 +30,11 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; import type { IOperationExecutionResult, IOperationStateHashComponents } from './IOperationExecutionResult'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import { OperationError } from './OperationError'; import { RushConstants } from '../RushConstants'; import type { IEnvironment } from '../../utilities/Utilities'; @@ -59,6 +63,7 @@ export interface IOperationExecutionRecordContext { * the environment of the inputs snapshot. The operation's `dependsOnEnvVars` are hashed from it. */ getOperationEnvironment?: (operation: Operation) => Readonly>; + getInputSnapshotPolicy?: (operation: Operation) => InputSnapshotPolicy; invalidate?: (operations: Iterable, reason: string) => void; inputsSnapshot: IInputsSnapshot | undefined; /** @@ -203,6 +208,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera public logFilePaths: ILogFilePaths | undefined; #context: IOperationExecutionRecordContext; + readonly #inputSnapshotPolicy: InputSnapshotPolicy; #collatedWriter: CollatedWriter | undefined = undefined; #status: OperationStatus; @@ -232,6 +238,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera }); this.#context = context; + this.#inputSnapshotPolicy = context.getInputSnapshotPolicy?.(operation) ?? 'legacy'; this.#status = operation.dependencies.size > 0 ? OperationStatus.Waiting : OperationStatus.Ready; this.#stateHashEntry = undefined; } @@ -240,6 +247,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return this.runner.name; } + public get inputSnapshotPolicy(): InputSnapshotPolicy { + return this.#inputSnapshotPolicy; + } + public get iterationId(): number { return this.#context.iterationId; } @@ -466,6 +477,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera #calculateStateHash(): IOperationStateHashCacheEntry { const { inputsSnapshot, stateHashCache } = this.#context; + const policy: InputSnapshotPolicy = this.inputSnapshotPolicy; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); if (!inputsSnapshot) { throw new Error(`Cannot calculate state hash without git.`); @@ -493,7 +506,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera const local: string = inputsSnapshot.getOperationOwnStateHash( associatedProject, associatedPhase.name, - this.#context.getOperationEnvironment?.(this.operation) + this.#context.getOperationEnvironment?.(this.operation), + policy ); // Examples of data in the config hash: diff --git a/libraries/rush-lib/src/logic/operations/OperationGraph.ts b/libraries/rush-lib/src/logic/operations/OperationGraph.ts index 4d624a5aa31..6b36630e0aa 100644 --- a/libraries/rush-lib/src/logic/operations/OperationGraph.ts +++ b/libraries/rush-lib/src/logic/operations/OperationGraph.ts @@ -28,7 +28,11 @@ import type { IExecutionResult, IOperationStateHashComponents } from './IOperationExecutionResult'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import type { IEnvironment } from '../../utilities/Utilities'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { @@ -219,12 +223,18 @@ class ConfigurableOperation implements IConfigurableOperation { public enabled: boolean; public shouldRunnerPersist: boolean = true; readonly #stateHashEntry: IOperationStateHashCacheEntry; + readonly #record: OperationExecutionRecord; public constructor(record: OperationExecutionRecord, stateHashEntry: IOperationStateHashCacheEntry) { this.operation = record.operation; this.metadataFolderPath = record.metadataFolderPath; this.enabled = !!record.operation.enabled; this.#stateHashEntry = stateHashEntry; + this.#record = record; + } + + public get inputSnapshotPolicy(): InputSnapshotPolicy { + return this.#record.inputSnapshotPolicy; } public getStateHash(): string { @@ -256,6 +266,8 @@ export class OperationGraph implements IOperationGraph { public readonly operations: Set; public readonly abortController: AbortController; readonly #sortedOperations: readonly Operation[]; + readonly #strictInputOperations: Set = new Set(); + #inputSnapshotPolicyFrozen: boolean = false; public resultByOperation: Map; @@ -357,6 +369,34 @@ export class OperationGraph implements IOperationGraph { ); } + public setInputSnapshotPolicy(operations: Iterable, policy: 'strict-v1'): void { + if (policy !== 'strict-v1') { + throw new Error(`Unsupported input snapshot policy: ${policy}`); + } + const requested: Set = new Set(operations); + if (this.#inputSnapshotPolicyFrozen) { + throw new Error('Input snapshot policy cannot change after iteration planning has started.'); + } + for (const operation of requested) { + if (!this.operations.has(operation)) { + throw new Error('Cannot set input snapshot policy for an operation outside this graph.'); + } + } + for (const operation of requested) { + this.#strictInputOperations.add(operation); + } + } + + public readonly getInputSnapshotPolicy = (operation: Operation): InputSnapshotPolicy => { + return this.#strictInputOperations.has(operation) ? 'strict-v1' : 'legacy'; + }; + + #assertSnapshotSupported(snapshot: IInputsSnapshot | undefined): void { + if (this.#strictInputOperations.size > 0) { + assertInputSnapshotPolicySupported(snapshot, 'strict-v1'); + } + } + /** * {@inheritDoc IOperationGraph.setEnabledStates} */ @@ -630,6 +670,7 @@ export class OperationGraph implements IOperationGraph { const { records } = iteration; const { inputsSnapshot } = options; + this.#assertSnapshotSupported(inputsSnapshot); const joiningRecords: Set = collectNeededRecords( records, Array.from(options.neededOperations, (operation: Operation) => getRecord(records, operation)) @@ -646,7 +687,8 @@ export class OperationGraph implements IOperationGraph { inputsSnapshot.getOperationOwnStateHash( record.associatedProject, record.associatedPhase.name, - iteration.getOperationEnvironment?.(record.operation) + iteration.getOperationEnvironment?.(record.operation), + this.getInputSnapshotPolicy(record.operation) ); // An operation that was dispatched before the joining request's inputs changed has no result for it @@ -800,6 +842,7 @@ export class OperationGraph implements IOperationGraph { * @returns A promise which is resolved when all operations have been processed to a final state. */ public async executeAsync(iterationOptions: IOperationGraphIterationOptions): Promise { + this.#inputSnapshotPolicyFrozen = true; await this.abortCurrentIterationAsync(); const scheduled: IExecutionIterationContext | undefined = await this.#scheduleIterationAsync(iterationOptions); @@ -955,6 +998,7 @@ export class OperationGraph implements IOperationGraph { async #scheduleIterationAsync( iterationOptions: IOperationGraphIterationOptions ): Promise { + this.#inputSnapshotPolicyFrozen = true; const getInputsSnapshotAsync: (() => Promise) | undefined = this.#getInputsSnapshotAsync; @@ -966,6 +1010,7 @@ export class OperationGraph implements IOperationGraph { getOperationRequestId, holdUnneededOperations } = iterationOptions; + this.#assertSnapshotSupported(inputsSnapshot); const iterationOptionsForCallbacks: IOperationGraphIterationOptions = { startTime, inputsSnapshot, @@ -1023,6 +1068,7 @@ export class OperationGraph implements IOperationGraph { maxParallelism: this.#maxParallelism, onOperationStateChanged: undefined, createEnvironment: createEnvironmentForOperation, + getInputSnapshotPolicy: this.getInputSnapshotPolicy, getOperationEnvironment, getOperationRequestId, holdUnneededOperations, diff --git a/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts b/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts index 820c93738a4..1a54b581d7e 100644 --- a/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts +++ b/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts @@ -6,7 +6,11 @@ import * as path from 'node:path'; import { Executable, Path } from '@rushstack/node-core-library'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { + assertInputSnapshotPolicySupported, + type IInputsSnapshot, + type InputSnapshotPolicy +} from '../incremental/InputsSnapshot'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import { haveInputFilesChanged, @@ -124,9 +128,11 @@ export async function haveOperationInputFilesChangedAsync( ); }; const { associatedProject: project, associatedPhase: phase } = record.operation; + const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; + assertInputSnapshotPolicySupported(inputsSnapshot, policy); return await haveInputFilesChangedSinceSnapshotAsync({ inputFilesState, - snapshotHashes: inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name), + snapshotHashes: inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name, policy), getGitPath, isNewInput }); diff --git a/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts index e6ea0d51c65..051acfdf273 100644 --- a/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts @@ -226,6 +226,22 @@ interface ITestWorkspace { const workspaceFolders: string[] = []; +it.each([undefined, 'reported'] as const)( + 'strict policy executes env-only changes without legacy skip or incremental equivalence (%s runner)', + async (pluginRunner) => { + const workspace: ITestWorkspace = await createWorkspaceAsync( + [{ name: 'a', dependsOnEnvVars: ['MODE'], pluginRunner }], + { hasLegacySkipDetection: true } + ); + workspace.graph.setInputSnapshotPolicy(workspace.operations.values(), 'strict-v1'); + expect((await workspace.executeAsync({})).commands).toEqual(['a:initial']); + expect((await workspace.executeAsync({ MODE: '' })).commands).toEqual(['a:initial']); + expect((await workspace.executeAsync({ MODE: 'next' })).commands).toEqual(['a:initial']); + workspace.writeFile('a/src/one.ts', 'one edited'); + expect((await workspace.executeAsync({ MODE: 'next' })).commands).toEqual(['a:initial']); + } +); + afterEach(() => { jest.restoreAllMocks(); for (const folder of workspaceFolders.splice(0)) { diff --git a/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts index 0f41034d107..905d0735766 100644 --- a/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts @@ -55,7 +55,7 @@ import type { RushConfigurationProject } from '../../../api/RushConfigurationPro import type { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; import { PhasedCommandHooks, type IOperationGraphContext } from '../../../pluginFramework/PhasedCommandHooks'; -import type { IInputsSnapshot } from '../../incremental/InputsSnapshot'; +import type { IInputsSnapshot, InputSnapshotPolicy } from '../../incremental/InputsSnapshot'; import { OperationBuildCache } from '../../buildCache/OperationBuildCache'; import { CacheableOperationPlugin } from '../CacheableOperationPlugin'; import { LegacySkipInvalidationPlugin, LegacySkipPlugin } from '../LegacySkipPlugin'; @@ -159,6 +159,8 @@ class MockRunner implements IOperationRunner { * Options of a command */ interface ICommandOptions { + readonly strictNames?: ReadonlySet; + readonly strictInputError?: Error; /** * The names of the projects whose operations the command doesn't execute, e.g. because of `--only` or because * the graph of a daemon's engine contains every operation in the repo @@ -279,6 +281,12 @@ async function runCommandAsync( destinations: [new MockWritable()], abortController: new AbortController() }); + if (options.strictNames) { + graph.setInputSnapshotPolicy( + Array.from(options.strictNames, (name) => operations.get(name)!), + 'strict-v1' + ); + } await hooks.onGraphCreatedAsync.promise(graph, { isIncrementalBuildAllowed, projectConfigurations @@ -307,14 +315,27 @@ async function runCommandAsync( ); } const inputsSnapshot: IInputsSnapshot = { + inputSnapshotPolicyVersion: 1, hashes, rootDirectory: folder, hasUncommittedChanges: false, - getTrackedFileHashesForOperation: (project: RushConfigurationProject) => { + getTrackedFileHashesForOperation: ( + project: RushConfigurationProject, + operationName?: string, + policy?: InputSnapshotPolicy + ) => { + if (policy === 'strict-v1' && options.strictInputError) { + throw options.strictInputError; + } const inputFilePath: string = `${project.packageName}/src/index.ts`; return new Map([[inputFilePath, hashes.get(inputFilePath)!]]); }, - getOperationOwnStateHash: (project: RushConfigurationProject) => inputs.get(project.packageName)! + getOperationOwnStateHash: ( + project: RushConfigurationProject, + operationName?: string, + environment?: Readonly>, + policy?: InputSnapshotPolicy + ) => `${policy === 'strict-v1' ? 'strict:' : ''}${inputs.get(project.packageName)!}` }; executions.length = 0; @@ -344,6 +365,33 @@ describe(LegacySkipPlugin.name, () => { }; }); + it('neither consumes nor writes legacy receipts for strict operations', async () => { + expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Success }); + expect(await runCommandAsync(checkout, 'legacy-skip', { strictNames: new Set(['a']) })).toEqual({ + a: OperationStatus.Success + }); + expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Success }); + expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Skipped }); + }); + + it('does not swallow a strict input query failure in the legacy readInputs catch', async () => { + await expect( + runCommandAsync(checkout, 'legacy-skip', { + strictNames: new Set(['a']), + strictInputError: new Error('strict input/output conflict') + }) + ).rejects.toThrow('strict input/output conflict'); + expect(checkout.executions).toEqual([]); + }); + + it('retains strict archive caching without reusing legacy archive identities', async () => { + expect(await runCommandAsync(checkout, 'build-cache')).toEqual({ a: OperationStatus.Success }); + const options: ICommandOptions = { strictNames: new Set(['a']) }; + expect(await runCommandAsync(checkout, 'build-cache', options)).toEqual({ a: OperationStatus.Success }); + expect(await runCommandAsync(checkout, 'build-cache', options)).toEqual({ a: OperationStatus.FromCache }); + expect(checkout.cacheEntries.size).toBe(2); + }); + afterEach(() => { fs.rmSync(checkout.folder, { recursive: true, force: true }); }); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts index 92c1bf617bf..a85b3f8820f 100644 --- a/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts @@ -21,14 +21,22 @@ jest.mock('../ProjectLogWritable', () => { }); import { MockWritable } from '@rushstack/terminal'; +import { LookupByPath } from '@rushstack/lookup-by-path'; +import * as path from 'node:path'; import type { IPhase } from '../../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; -import type { IInputsSnapshot, IRushConfigurationProjectForSnapshot } from '../../incremental/InputsSnapshot'; +import { + InputsSnapshot, + type IInputsSnapshot, + type IRushConfigurationProjectForSnapshot, + type InputSnapshotPolicy +} from '../../incremental/InputsSnapshot'; +import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; import type { IOperationStateHashComponents } from '../IOperationExecutionResult'; import { Operation } from '../Operation'; import { calculateOperationStateHashEntry } from '../OperationExecutionRecord'; -import { OperationGraph } from '../OperationGraph'; +import { OperationGraph, type IOperationGraphOptions } from '../OperationGraph'; import { OperationStatus } from '../OperationStatus'; import { MockOperationRunner } from './MockOperationRunner'; @@ -57,9 +65,9 @@ interface IStateHashes { class StateHashGraph { public readonly iterations: IStateHashes[] = []; public readonly graph: OperationGraph; - readonly #inputsSnapshot: IInputsSnapshot; + public readonly inputsSnapshot: IInputsSnapshot; - public constructor(localHashes: ReadonlyMap) { + public constructor(localHashes: ReadonlyMap, options?: Partial) { const operations: Map = new Map(); for (const name of ['a', 'b', 'c']) { operations.set( @@ -68,7 +76,11 @@ class StateHashGraph { runner: new MockOperationRunner(name), logFilenameIdentifier: name, phase: mockPhase, - project: { packageName: name } as unknown as RushConfigurationProject + project: { + packageName: name, + projectFolder: path.resolve('/repo', name), + projectRelativeFolder: name + } as unknown as RushConfigurationProject }) ); } @@ -80,7 +92,8 @@ class StateHashGraph { parallelism: 1, allowOversubscription: true, destinations: [new MockWritable()], - abortController: new AbortController() + abortController: new AbortController(), + ...options }); this.graph.hooks.configureIteration.tap('test', (records) => { const components: Map = new Map(); @@ -92,18 +105,26 @@ class StateHashGraph { this.iterations.push({ components, hashes }); }); - this.#inputsSnapshot = { + this.inputsSnapshot = { + inputSnapshotPolicyVersion: 1, hashes: new Map(), rootDirectory: '/repo', hasUncommittedChanges: false, getTrackedFileHashesForOperation: () => new Map(), - getOperationOwnStateHash: (project: IRushConfigurationProjectForSnapshot) => - localHashes.get((project as RushConfigurationProject).packageName)! + getOperationOwnStateHash: ( + project: IRushConfigurationProjectForSnapshot, + operationName?: string, + environment?: Readonly>, + policy?: InputSnapshotPolicy + ) => + `${policy === 'strict-v1' ? 'strict:' : ''}${localHashes.get( + (project as RushConfigurationProject).packageName + )!}` }; } public async executeAsync(): Promise { - expect((await this.graph.executeAsync({ inputsSnapshot: this.#inputsSnapshot })).status).toBe( + expect((await this.graph.executeAsync({ inputsSnapshot: this.inputsSnapshot })).status).toBe( OperationStatus.Success ); return this.iterations[this.iterations.length - 1]; @@ -117,27 +138,195 @@ function getReusedNames(previous: IStateHashes, current: IStateHashes): string[] } describe('OperationGraph state hashes', () => { + const localHashes: ReadonlyMap = new Map([ + ['a', 'local-a'], + ['b', 'local-b'], + ['c', 'local-c'] + ]); + + it('registers policy atomically and leaves unselected own hashes in the legacy namespace', async () => { + const legacy: StateHashGraph = new StateHashGraph(localHashes); + const strict: StateHashGraph = new StateHashGraph(localHashes); + const [a, b] = Array.from(strict.graph.operations); + expect(() => + strict.graph.setInputSnapshotPolicy([a, Array.from(legacy.graph.operations)[0]], 'strict-v1') + ).toThrow('outside this graph'); + expect(strict.graph.getInputSnapshotPolicy(a)).toBe('legacy'); + expect(() => + Reflect.apply(strict.graph.setInputSnapshotPolicy, strict.graph, [[a], 'strict-v2']) + ).toThrow('Unsupported'); + expect(strict.graph.getInputSnapshotPolicy(a)).toBe('legacy'); + strict.graph.setInputSnapshotPolicy([a], 'strict-v1'); + strict.graph.setInputSnapshotPolicy([a], 'strict-v1'); + const before: IStateHashes = await legacy.executeAsync(); + const after: IStateHashes = await strict.executeAsync(); + expect(after.components.get('a')!.local).not.toBe(before.components.get('a')!.local); + expect(after.components.get('b')!.local).toBe(before.components.get('b')!.local); + expect(after.hashes.get('b')).not.toBe(before.hashes.get('b')); + expect(strict.graph.resultByOperation.get(a)!.inputSnapshotPolicy).toBe('strict-v1'); + expect(strict.graph.resultByOperation.get(b)!.inputSnapshotPolicy).toBe('legacy'); + }); + + it('freezes registration before asynchronous snapshot acquisition', async () => { + let provideSnapshot!: (snapshot: IInputsSnapshot) => void; + const pending: Promise = new Promise((resolve) => { + provideSnapshot = resolve; + }); + const fixture: StateHashGraph = new StateHashGraph(localHashes, { + getInputsSnapshotAsync: () => pending + }); + fixture.graph.setInputSnapshotPolicy(fixture.graph.operations, 'strict-v1'); + const scheduled: Promise = fixture.graph.scheduleIterationAsync({}); + expect(() => fixture.graph.setInputSnapshotPolicy([], 'strict-v1')).toThrow('cannot change'); + expect(fixture.iterations).toHaveLength(0); + provideSnapshot(fixture.inputsSnapshot); + expect(await scheduled).toBe(true); + expect(fixture.iterations).toHaveLength(1); + fixture.graph.discardScheduledIteration(); + }); + + it.each([false, true])( + 'fails closed before configuration for unsupported snapshots (missing: %s)', + async (missing) => { + const fixture: StateHashGraph = new StateHashGraph(localHashes); + const snapshot: IInputsSnapshot = { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }; + fixture.graph.setInputSnapshotPolicy(fixture.graph.operations, 'strict-v1'); + await expect( + fixture.graph.executeAsync({ inputsSnapshot: missing ? undefined : snapshot }) + ).rejects.toThrow('snapshot supporting'); + expect(fixture.iterations).toHaveLength(0); + expect(() => fixture.graph.setInputSnapshotPolicy([], 'strict-v1')).toThrow('cannot change'); + } + ); + + it('accepts old snapshots for legacy graphs but checks capability again on every strict iteration', async () => { + const fixture: StateHashGraph = new StateHashGraph(localHashes); + const snapshot: IInputsSnapshot = { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }; + await fixture.graph.executeAsync({ inputsSnapshot: snapshot }); + expect(fixture.iterations).toHaveLength(1); + const strict: StateHashGraph = new StateHashGraph(localHashes); + strict.graph.setInputSnapshotPolicy(strict.graph.operations, 'strict-v1'); + await strict.executeAsync(); + await expect(strict.graph.executeAsync({ inputsSnapshot: snapshot })).rejects.toThrow( + 'snapshot supporting' + ); + expect(strict.iterations).toHaveLength(1); + }); + + it('rejects strict additional/output conflicts before configuration or execution preparation', async () => { + const fixture: StateHashGraph = new StateHashGraph(localHashes); + const operations: Operation[] = Array.from(fixture.graph.operations); + const a: Operation = operations[0]; + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + ['phase', { operationName: 'phase', outputFolderNames: ['lib'] }] + ]) + }; + const inputsSnapshot: InputsSnapshot = new InputsSnapshot({ + rootDir: path.resolve('/repo'), + hashes: new Map(), + additionalHashes: new Map([['a/lib/sorter.js', 'input']]), + hasUncommittedChanges: false, + lookupByPath: new LookupByPath( + operations.map(({ associatedProject: project }) => [project.projectRelativeFolder, project]) + ), + projectMap: new Map( + operations.map(({ associatedProject: project }) => [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration, + additionalFilesByOperationName: + project === a.associatedProject ? new Map([['phase', new Set(['a/lib/sorter.js'])]]) : undefined + } + ]) + ) + }); + inputsSnapshot.getOperationOwnStateHash(a.associatedProject, 'phase'); + fixture.graph.setInputSnapshotPolicy([a], 'strict-v1'); + const prepare: jest.Mock = jest.fn(); + fixture.graph.hooks.beforeExecuteIterationAsync.tap('test', prepare); + await expect(fixture.graph.executeAsync({ inputsSnapshot })).rejects.toThrow( + 'contains additional input file' + ); + expect(fixture.iterations).toHaveLength(0); + expect(prepare).not.toHaveBeenCalled(); + }); + + it('uses the frozen resolver and capability guard for joining request records', async () => { + const hashes: Map = new Map(localHashes); + const fixture: StateHashGraph = new StateHashGraph(hashes); + const [a, b, c] = Array.from(fixture.graph.operations); + fixture.graph.setInputSnapshotPolicy([b], 'strict-v1'); + let started!: () => void; + const whenStarted: Promise = new Promise((resolve) => { + started = resolve; + }); + let finish!: () => void; + const whenFinished: Promise = new Promise((resolve) => { + finish = resolve; + }); + a.runner = new MockOperationRunner('a', async () => { + started(); + await whenFinished; + return OperationStatus.Success; + }); + let joining: boolean = false; + fixture.graph.hooks.configureIteration.tap('test-policy', (records) => { + expect(records.get(b)!.inputSnapshotPolicy).toBe('strict-v1'); + expect(records.get(c)!.inputSnapshotPolicy).toBe('legacy'); + records.get(b)!.enabled = joining; + records.get(c)!.enabled = false; + }); + const run = fixture.graph.executeAsync({ + inputsSnapshot: fixture.inputsSnapshot, + holdUnneededOperations: true + }); + await whenStarted; + try { + joining = true; + const count: number = fixture.iterations.length; + expect(() => + fixture.graph.tryExtendCurrentIteration({ + inputsSnapshot: { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }, + neededOperations: [b] + }) + ).toThrow('snapshot supporting'); + expect(fixture.iterations).toHaveLength(count); + hashes.set('b', 'changed'); + expect( + fixture.graph.tryExtendCurrentIteration({ + inputsSnapshot: fixture.inputsSnapshot, + neededOperations: [b] + }).extended + ).toBe(true); + expect(fixture.iterations.at(-1)!.components.get('b')!.local).toBe('strict:changed'); + } finally { + finish(); + await run; + } + }); + it('reuses the state hash of each operation whose inputs did not change since the previous iteration', async () => { - const localHashes: Map = new Map([ + const changedHashes: Map = new Map([ ['a', 'local-a'], ['b', 'local-b'], ['c', 'local-c'] ]); - const graph: StateHashGraph = new StateHashGraph(localHashes); + const graph: StateHashGraph = new StateHashGraph(changedHashes); const first: IStateHashes = await graph.executeAsync(); const second: IStateHashes = await graph.executeAsync(); expect(getReusedNames(first, second)).toEqual(['a', 'b', 'c']); expect(second.hashes).toEqual(first.hashes); - localHashes.set('a', 'local-a-edited'); + changedHashes.set('a', 'local-a-edited'); const third: IStateHashes = await graph.executeAsync(); expect(getReusedNames(second, third)).toEqual(['c']); expect(third.hashes.get('a')).not.toBe(second.hashes.get('a')); expect(third.hashes.get('b')).not.toBe(second.hashes.get('b')); // A new graph has nothing to reuse, and calculates the same state hashes - const uncached: IStateHashes = await new StateHashGraph(localHashes).executeAsync(); + const uncached: IStateHashes = await new StateHashGraph(changedHashes).executeAsync(); expect(third.hashes).toEqual(uncached.hashes); expect(third.components).toEqual(uncached.components); }); diff --git a/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts b/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts index 0943e7a0e2d..0645e0e7623 100644 --- a/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts @@ -64,23 +64,70 @@ jest.mock('../operations/ProjectLogWritable', () => { }); import { setTimeout as sleepAsync } from 'node:timers/promises'; +import * as path from 'node:path'; +import { LookupByPath } from '@rushstack/lookup-by-path'; import { AnsiEscape, MockWritable, StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { InputsSnapshot, type IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import type { IOperationRunner, IOperationRunnerContext } from '../operations/IOperationRunner'; import type { IOperationExecutionResult } from '../operations/IOperationExecutionResult'; import { Operation } from '../operations/Operation'; import { OperationGraph } from '../operations/OperationGraph'; import { OperationStatus } from '../operations/OperationStatus'; -import { ProjectWatcher } from '../ProjectWatcher'; +import { ProjectWatcher, _findOperationWithChangedInputs } from '../ProjectWatcher'; const DEBOUNCE_MS: number = 10; const REQUESTOR: string = 'test-requestor'; +it('compares watched inputs with the graph policy and fails closed on legacy-only snapshots', () => { + const { operation } = createOperation('a'); + const project: RushConfigurationProject = operation.associatedProject; + const projectConfig: Pick = { + operationSettingsByOperationName: new Map([ + ['phase', { operationName: 'phase', dependsOnEnvVars: ['MODE'] }] + ]) + }; + const snapshot = (environment: Record): InputsSnapshot => + new InputsSnapshot({ + rootDir: path.resolve('/repo'), + hashes: new Map(), + lookupByPath: new LookupByPath([['a', project]]), + hasUncommittedChanges: false, + projectMap: new Map([ + [ + project, + { + projectConfig: projectConfig as RushProjectConfiguration + } + ] + ]), + environment + }); + const before: InputsSnapshot = snapshot({}); + const after: InputsSnapshot = snapshot({ MODE: '' }); + expect(_findOperationWithChangedInputs([operation], before, after)).toBeUndefined(); + const graph: OperationGraph = createWatchSession([operation]).graph; + graph.setInputSnapshotPolicy([operation], 'strict-v1'); + expect(_findOperationWithChangedInputs([operation], before, after, graph.getInputSnapshotPolicy)).toBe( + operation + ); + const unsupported: IInputsSnapshot = { + hashes: new Map(), + rootDirectory: '/repo', + hasUncommittedChanges: false, + getOperationOwnStateHash: () => 'ignored-policy', + getTrackedFileHashesForOperation: () => new Map() + }; + expect(() => + _findOperationWithChangedInputs([operation], before, unsupported, graph.getInputSnapshotPolicy) + ).toThrow('snapshot supporting'); +}); + // The statuses of a result that PhasedOperationPlugin keeps if the operation's inputs did not change. const RETAINED_STATUSES: ReadonlySet = new Set([ OperationStatus.Success, @@ -673,6 +720,57 @@ describe(ProjectWatcher.name, () => { }); describe('inputs that change during an iteration', () => { + it.each([ + ['legacy', 'current'], + ['strict-v1', 'current'], + ['strict-v1', 'superseded'], + ['strict-v1', 'cancelled'] + ] as const)( + 'handles a missing idle-check snapshot for %s operations when the check is %s', + async (policy, checkState) => { + const a: ITestOperation = createOperation('a'); + const session: IWatchSession = createWatchSession([a.operation], { hasInputsSnapshots: true }); + if (policy === 'strict-v1') { + session.graph.setInputSnapshotPolicy([a.operation], policy); + } + session.snapshots.getGraphSnapshotAsync = async () => ({ + ...createInputsSnapshot(session.snapshots.inputVersions), + inputSnapshotPolicyVersion: 1 + }); + let completeCheck!: (snapshot: undefined) => void; + session.snapshots.getWatcherSnapshotAsync = () => + new Promise((resolve) => { + completeCheck = resolve; + }); + await startWatchSessionAsync(session); + expect(session.iterations).toEqual([['a']]); + expect(session.snapshots.watcherSnapshotCount).toBe(1); + + if (checkState === 'superseded') { + session.snapshots.getWatcherSnapshotAsync = session.snapshots.getGraphSnapshotAsync; + editInputs(session, a.operation); + simulateFileChange(); + await settleAsync(session); + expect(session.iterations).toEqual([['a'], ['a']]); + expect(session.snapshots.watcherSnapshotCount).toBe(2); + } else if (checkState === 'cancelled') { + session.abortController.abort(); + } + + completeCheck(undefined); + await settleAsync(session); + if (policy === 'strict-v1' && checkState === 'current') { + expect(session.terminalProvider.getErrorOutput()).toContain( + 'Failed to check for file changes made during the iteration: ' + + 'Strict operation inputs require a snapshot supporting input snapshot policy version 1.' + ); + } else { + expect(session.terminalProvider.getErrorOutput()).toBe(''); + } + expect(session.iterations).toHaveLength(checkState === 'superseded' ? 2 : 1); + } + ); + it('queues an iteration for inputs that changed while an iteration ran', async () => { const a: ITestOperation = createOperation('a'); const b: ITestOperation = createOperation('b'); From f257099202af0ead90c52b7a52cbe76a3eb722b6 Mon Sep 17 00:00:00 2001 From: yashwanth195 <59129034+yashwanth195@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:24:06 -0800 Subject: [PATCH 3/3] Fix declared environment hashing --- .../rush/environment-snapshot_2026-09-16.json | 4 +- .../environment-map-equals_2026-11-16.json | 10 + common/reviews/api/node-core-library.api.md | 1 + common/reviews/api/rush-lib.api.md | 11 +- .../node-core-library/src/EnvironmentMap.ts | 17 + .../src/test/EnvironmentMap.test.ts | 46 ++ libraries/rush-lib/src/index.ts | 1 - .../src/logic/ProjectChangeAnalyzer.ts | 14 +- .../rush-lib/src/logic/ProjectWatcher.ts | 33 +- .../src/logic/incremental/InputsSnapshot.ts | 274 +++------- .../incremental/test/InputsSnapshot.test.ts | 491 +++++------------- .../__snapshots__/InputsSnapshot.test.ts.snap | 4 +- .../src/logic/operations/BuildPlanPlugin.ts | 5 +- .../operations/CacheableOperationPlugin.ts | 19 +- .../operations/IOperationExecutionResult.ts | 6 - .../src/logic/operations/IOperationGraph.ts | 14 +- .../IncrementalExecutionGuardPlugin.ts | 40 +- .../src/logic/operations/LegacySkipPlugin.ts | 34 +- .../operations/OperationExecutionRecord.ts | 18 +- .../src/logic/operations/OperationGraph.ts | 50 +- .../operations/OperationInputFilesCheck.ts | 10 +- .../IncrementalExecutionGuardPlugin.test.ts | 20 +- .../operations/test/LegacySkipPlugin.test.ts | 54 +- .../test/OperationGraphStateHash.test.ts | 217 +------- .../logic/test/ProjectChangeAnalyzer.test.ts | 9 +- .../src/logic/test/ProjectWatcher.test.ts | 102 +--- 26 files changed, 351 insertions(+), 1153 deletions(-) create mode 100644 common/changes/@rushstack/node-core-library/environment-map-equals_2026-11-16.json diff --git a/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json index 322b63c78f0..780e33b7472 100644 --- a/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json +++ b/common/changes/@microsoft/rush/environment-snapshot_2026-09-16.json @@ -2,8 +2,8 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "Add optional per-operation strict input snapshots with captured environment values, collision-safe identity, and additional-input/output validation. Legacy input hashing and validation remain the default. Strict operations use versioned cache keys and bypass legacy skip and incremental-command equivalence.", - "type": "minor" + "comment": "Fix declared-environment cache hashes to distinguish unset and empty values, encode values unambiguously, and use Windows environment-variable name semantics. Operations with declared environment inputs receive new cache keys.", + "type": "patch" } ], "packageName": "@microsoft/rush" diff --git a/common/changes/@rushstack/node-core-library/environment-map-equals_2026-11-16.json b/common/changes/@rushstack/node-core-library/environment-map-equals_2026-11-16.json new file mode 100644 index 00000000000..d363821cc4d --- /dev/null +++ b/common/changes/@rushstack/node-core-library/environment-map-equals_2026-11-16.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Add EnvironmentMap.equals() to compare variable names and values without allocating temporary arrays.", + "type": "minor" + } + ], + "packageName": "@rushstack/node-core-library" +} diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index 679736f30f6..1d6ca3717ed 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -98,6 +98,7 @@ export class EnvironmentMap { readonly caseSensitive: boolean; clear(): void; entries(): IterableIterator; + equals(environmentMap: EnvironmentMap): boolean; get(name: string): string | undefined; mergeFrom(environmentMap: EnvironmentMap): void; mergeFromObject(environmentObject?: Record): void; diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index d91316b3016..914ae2604f6 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -406,7 +406,6 @@ export function getWorkspaceRequestOperationEnvironment(hostEnvironment: Readonl export interface IBaseOperationExecutionResult { getStateHash(): string; getStateHashComponents(): IOperationStateHashComponents; - readonly inputSnapshotPolicy?: InputSnapshotPolicy; readonly metadataFolderPath: string; readonly operation: Operation; } @@ -653,11 +652,10 @@ export interface IIndividualVersionJson extends IVersionPolicyJson { // @beta export interface IInputsSnapshot { - getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string, environment?: Readonly>, policy?: InputSnapshotPolicy): string; - getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string, policy?: InputSnapshotPolicy): ReadonlyMap; + getOperationOwnStateHash(project: IRushConfigurationProjectForSnapshot, operationName?: string, environment?: Readonly>): string; + getTrackedFileHashesForOperation(project: IRushConfigurationProjectForSnapshot, operationName?: string): ReadonlyMap; readonly hashes: ReadonlyMap; readonly hasUncommittedChanges: boolean; - readonly inputSnapshotPolicyVersion?: 1; readonly rootDirectory: string; readonly workingTreeReadStartTimeMs?: number; } @@ -716,9 +714,6 @@ export class IndividualVersionPolicy extends VersionPolicy { export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { } -// @beta -export type InputSnapshotPolicy = 'legacy' | 'strict-v1'; - // @internal export interface _IOperationActivityOptions { readonly operationId?: string; @@ -780,7 +775,6 @@ export interface IOperationGraph { deleteResults?(operations: Iterable): void; discardScheduledIteration(): boolean; executeScheduledIterationAsync(): Promise; - getInputSnapshotPolicy?(operation: Operation): InputSnapshotPolicy; readonly hasScheduledIteration: boolean; readonly hooks: OperationGraphHooks; invalidateOperations(operations?: Iterable, reason?: string): void; @@ -794,7 +788,6 @@ export interface IOperationGraph { retainHeldOperations?(): (() => void) | undefined; scheduleIterationAsync(options: IOperationGraphIterationOptions): Promise; setEnabledStates(operations: Iterable, targetState: Operation['enabled'], mode: 'safe' | 'unsafe'): boolean; - setInputSnapshotPolicy?(operations: Iterable, policy: 'strict-v1'): void; readonly status: OperationStatus; readonly terminalDestinations: ReadonlySet; tryExtendCurrentIteration?(options: IOperationGraphExtensionOptions): IOperationGraphExtensionResult; diff --git a/libraries/node-core-library/src/EnvironmentMap.ts b/libraries/node-core-library/src/EnvironmentMap.ts index c672f1be36b..b73e03fdb29 100644 --- a/libraries/node-core-library/src/EnvironmentMap.ts +++ b/libraries/node-core-library/src/EnvironmentMap.ts @@ -106,6 +106,23 @@ export class EnvironmentMap { return this._map.values(); } + /** + * Returns whether the maps have the same case sensitivity, variable names, and values. + * Entry order and, for case-insensitive maps, variable name casing are ignored. + */ + public equals(environmentMap: EnvironmentMap): boolean { + if (this.caseSensitive !== environmentMap.caseSensitive || this._map.size !== environmentMap._map.size) { + return false; + } + + for (const [name, entry] of this._map) { + if (environmentMap._map.get(name)?.value !== entry.value) { + return false; + } + } + return true; + } + /** * Adds each entry from `environmentMap` to this map. */ diff --git a/libraries/node-core-library/src/test/EnvironmentMap.test.ts b/libraries/node-core-library/src/test/EnvironmentMap.test.ts index be51e8163b4..bdce9a2a5d1 100644 --- a/libraries/node-core-library/src/test/EnvironmentMap.test.ts +++ b/libraries/node-core-library/src/test/EnvironmentMap.test.ts @@ -6,6 +6,52 @@ import process from 'node:process'; import { EnvironmentMap } from '../EnvironmentMap'; describe(EnvironmentMap.name, () => { + describe(EnvironmentMap.prototype.equals.name, () => { + it('compares values and names independently of insertion order', () => { + const map = new EnvironmentMap({ A: '1', B: '' }); + expect(map.equals(new EnvironmentMap({ B: '', A: '1' }))).toBe(true); + expect(map.equals(new EnvironmentMap({ A: '2', B: '' }))).toBe(false); + expect(map.equals(new EnvironmentMap({ A: '1', C: '' }))).toBe(false); + expect(map.equals(new EnvironmentMap({ A: '1' }))).toBe(false); + expect(map.equals(new EnvironmentMap({ A: '1', B: '', C: '3' }))).toBe(false); + }); + + it('distinguishes empty values from missing variables', () => { + const missing = new EnvironmentMap(); + expect(missing.equals(new EnvironmentMap({ A: undefined }))).toBe(true); + expect(missing.equals(new EnvironmentMap({ A: '' }))).toBe(false); + expect(new EnvironmentMap({ A: '' }).equals(missing)).toBe(false); + }); + + it.each(['win32', 'linux'])('respects %s variable-name semantics', (platform) => { + const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const upper = new EnvironmentMap({ A: '1' }); + const lower = new EnvironmentMap({ a: '1' }); + expect(upper.equals(lower)).toBe(platform === 'win32'); + expect(lower.equals(upper)).toBe(platform === 'win32'); + expect(upper.equals(new EnvironmentMap({ a: '2' }))).toBe(false); + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + + it('does not equate maps with different lookup semantics', () => { + const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + try { + Object.defineProperty(process, 'platform', { ...descriptor, value: 'win32' }); + const insensitive = new EnvironmentMap({ A: '1' }); + Object.defineProperty(process, 'platform', { ...descriptor, value: 'linux' }); + const sensitive = new EnvironmentMap({ A: '1' }); + expect(insensitive.equals(sensitive)).toBe(false); + expect(sensitive.equals(insensitive)).toBe(false); + } finally { + Object.defineProperty(process, 'platform', descriptor); + } + }); + }); + test('_sanityCheck() throws', () => { const map = new EnvironmentMap(); const environmentObject = { A: '123' }; diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 2cc3fc2c4f2..e181e2bce72 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -155,7 +155,6 @@ export { export { ProjectChangeAnalyzer, type IGetChangedProjectsOptions } from './logic/ProjectChangeAnalyzer'; export type { IInputsSnapshot, - InputSnapshotPolicy, GetInputsSnapshotAsyncFn as GetInputsSnapshotAsyncFn, IRushConfigurationProjectForSnapshot } from './logic/incremental/InputsSnapshot'; diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index bb43f6b1311..7e75eab84f1 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -14,6 +14,7 @@ import { Sort, JsonFile, Objects, + EnvironmentMap, type IPackageJson } from '@rushstack/node-core-library'; import { @@ -496,7 +497,7 @@ export class ProjectChangeAnalyzer { const additionalHashes: Map = new Map( additionalFiles.size > 0 ? await hashFilesAsync(rootDirectory, additionalFiles, gitPath) : [] ); - const environment: Record = { ...process.env }; + const environment: EnvironmentMap = new EnvironmentMap(process.env); // The files that each operation depends on accumulate across snapshots const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap); @@ -505,7 +506,7 @@ export class ProjectChangeAnalyzer { previousSnapshot.repoState === repoState && previousSnapshot.operationAdditionalFileCount === operationAdditionalFileCount && areMapsEqual(previousSnapshot.additionalHashes, additionalHashes) && - areEnvironmentsEqual(previousSnapshot.environment, environment) + previousSnapshot.environment.equals(environment) ) { // Every hash that a new snapshot would compute is the same. The previous snapshot began reading the // working tree earlier, which only widens the window in which a saved file may be newer than its hash. @@ -841,7 +842,7 @@ interface IAdditionalGlob { */ interface IReusableInputsSnapshot { readonly additionalHashes: ReadonlyMap; - readonly environment: Readonly>; + readonly environment: EnvironmentMap; readonly operationAdditionalFileCount: number; readonly repoState: IDetailedRepoState; readonly snapshot: InputsSnapshot; @@ -896,13 +897,6 @@ function areMapsEqual(a: ReadonlyMap, b: ReadonlyMap>, - b: Readonly> -): boolean { - return areMapsEqual(new Map(Object.entries(a)), new Map(Object.entries(b))); -} - async function getAdditionalFilesFromRushProjectConfigurationAsync( additionalGlobs: IAdditionalGlob[], rootRelativeLookupByPath: IReadonlyLookupByPath, diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index 76b03d781e7..aa43c903957 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -11,12 +11,7 @@ import { Path } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; import { Git } from './Git'; -import { - assertInputSnapshotPolicySupported, - type GetInputsSnapshotAsyncFn, - type IInputsSnapshot, - type InputSnapshotPolicy -} from './incremental/InputsSnapshot'; +import type { GetInputsSnapshotAsyncFn, IInputsSnapshot } from './incremental/InputsSnapshot'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; import type { IOperationGraph, IOperationGraphIterationOptions } from './operations/IOperationGraph'; @@ -549,23 +544,13 @@ export class ProjectWatcher { let changedOperation: Operation | undefined; try { const currentSnapshot: IInputsSnapshot | undefined = await getInputsSnapshotAsync(); - if (this.#inputsCheck !== check) { - return; - } - if (!currentSnapshot) { - for (const operation of this.#graph.operations) { - assertInputSnapshotPolicySupported( - currentSnapshot, - this.#graph.getInputSnapshotPolicy?.(operation) ?? 'legacy' - ); - } + if (this.#inputsCheck !== check || !currentSnapshot) { return; } changedOperation = _findOperationWithChangedInputs( this.#graph.operations, iterationSnapshot, - currentSnapshot, - this.#graph.getInputSnapshotPolicy + currentSnapshot ); } catch (e) { if (this.#inputsCheck === check) { @@ -734,20 +719,16 @@ export class ProjectWatcher { * Returns an operation whose own state hash differs between the two snapshots, if there is one. The own state * hash covers the operation's tracked files, additional files and environment variables. */ -export function _findOperationWithChangedInputs( +function _findOperationWithChangedInputs( operations: Iterable, previousSnapshot: IInputsSnapshot, - currentSnapshot: IInputsSnapshot, - getInputSnapshotPolicy?: (operation: Operation) => InputSnapshotPolicy + currentSnapshot: IInputsSnapshot ): Operation | undefined { for (const operation of operations) { const { associatedProject, associatedPhase } = operation; - const policy: InputSnapshotPolicy = getInputSnapshotPolicy?.(operation) ?? 'legacy'; - assertInputSnapshotPolicySupported(previousSnapshot, policy); - assertInputSnapshotPolicySupported(currentSnapshot, policy); if ( - currentSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name, undefined, policy) !== - previousSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name, undefined, policy) + currentSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name) !== + previousSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name) ) { return operation; } diff --git a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts index c3e369b2fd3..587d17c83f1 100644 --- a/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts +++ b/libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts @@ -25,25 +25,6 @@ export type IRushConfigurationProjectForSnapshot = Pick< 'projectFolder' | 'projectRelativeFolder' >; -/** - * Selects the compatibility contract for operation inputs. - * @beta - */ -export type InputSnapshotPolicy = 'legacy' | 'strict-v1'; - -/** - * Rejects unsupported strict queries, including snapshots supplied by older hosts. - * @internal - */ -export function assertInputSnapshotPolicySupported( - snapshot: IInputsSnapshot | undefined, - policy: InputSnapshotPolicy -): void { - if (policy !== 'legacy' && (policy !== 'strict-v1' || snapshot?.inputSnapshotPolicyVersion !== 1)) { - throw new Error('Strict operation inputs require a snapshot supporting input snapshot policy version 1.'); - } -} - /** * @internal */ @@ -109,11 +90,10 @@ export interface IInputsSnapshotParameters { */ additionalHashes?: ReadonlyMap; /** - * The environment for `dependsOnEnvVars`. Strict queries capture its values upon construction. - * Legacy queries retain an explicitly supplied object, or copy process.env by default. - * @defaultValue \{ ...process.env \} + * The environment for `dependsOnEnvVars`. The caller must not modify it after construction. + * @defaultValue new EnvironmentMap(process.env) */ - environment?: Record; + environment?: EnvironmentMap; /** * The Node.js version string to use for `dependsOnNodeVersion`. Defaults to `process.version`. * @defaultValue process.version @@ -172,12 +152,6 @@ const EMPTY_MAP: ReadonlyMap = new Map(); * @beta */ export interface IInputsSnapshot { - /** - * When 1, supports strict-v1 queries, including construction-time environment capture and output validation. - * Absent on legacy-only implementations. - */ - readonly inputSnapshotPolicyVersion?: 1; - /** * The raw hashes of all tracked files in the repository. */ @@ -209,13 +183,11 @@ export interface IInputsSnapshot { * * @param project - The Rush project to get hashes for * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a default set for the project, as used for bulk commands. - * @param policy - Defaults to legacy validation. Strict queries also reject additional inputs inside output folders. * @returns A map of file name to Git hash. For local files paths will be relative. Configured additional files may be absolute paths. */ getTrackedFileHashesForOperation( project: IRushConfigurationProjectForSnapshot, - operationName?: string, - policy?: InputSnapshotPolicy + operationName?: string ): ReadonlyMap; /** @@ -225,15 +197,12 @@ export interface IInputsSnapshot { * @param operationName - The name of the operation (phase) to get hashes for. If omitted, returns a generic hash for the whole project, as used for bulk commands. * @param environment - The environment that the operation runs with, if it differs from the environment of the * snapshot. The operation's `dependsOnEnvVars` are hashed from it. - * @param policy - Defaults to legacy identity. Strict queries use captured platform-aware environment values, - * unambiguous encoding, output validation, and a versioned hash namespace. * @returns The local state hash for the project. This is a hash of the environment, the project's tracked files, and any additional files. */ getOperationOwnStateHash( project: IRushConfigurationProjectForSnapshot, operationName?: string, - environment?: Readonly>, - policy?: InputSnapshotPolicy + environment?: Readonly> ): string; } @@ -247,8 +216,6 @@ export interface IInputsSnapshot { * @internal */ export class InputsSnapshot implements IInputsSnapshot { - public readonly inputSnapshotPolicyVersion: 1 = 1; - /** * {@inheritdoc IInputsSnapshot.hashes} */ @@ -273,9 +240,6 @@ export class InputsSnapshot implements IInputsSnapshot { IRushConfigurationProjectForSnapshot, IInternalInputsSnapshotProjectMetadata >; - #strictProjectMetadataMap: - | Map - | undefined; /** * Hashes of files to be included in all result sets. */ @@ -287,8 +251,7 @@ export class InputsSnapshot implements IInputsSnapshot { /** * The environment to use for `dependsOnEnvVars`. */ - readonly #environment: Record; - readonly #strictEnvironment: EnvironmentMap; + readonly #environment: EnvironmentMap; /** * Pre-computed Node.js version strings at each granularity level for `dependsOnNodeVersion`. */ @@ -309,7 +272,7 @@ export class InputsSnapshot implements IInputsSnapshot { public constructor(params: IInputsSnapshotParameters) { const { additionalHashes, - environment = { ...process.env }, + environment = new EnvironmentMap(process.env), globalAdditionalFiles, hashes, hasUncommittedChanges, @@ -343,33 +306,18 @@ export class InputsSnapshot implements IInputsSnapshot { const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap); - this.#environment = environment; - this.#strictEnvironment = new EnvironmentMap(environment); this.#projectMetadataMap = - (previousSnapshot && areEnvironmentsEqual(environment, previousSnapshot.#environment) - ? previousSnapshot.#tryDeriveProjectMetadataMap( - params, - nodeVersion, - globalAdditionalHashes, - operationAdditionalFileCount, - previousSnapshot.#projectMetadataMap - ) - : undefined) ?? createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir); - if ( - previousSnapshot && - previousSnapshot.#strictProjectMetadataMap && - areEnvironmentMapsEqual(this.#strictEnvironment, previousSnapshot.#strictEnvironment) - ) { - this.#strictProjectMetadataMap = previousSnapshot.#tryDeriveProjectMetadataMap( + this.#tryDeriveProjectMetadataMap( + previousSnapshot, params, + environment, nodeVersion, globalAdditionalHashes, - operationAdditionalFileCount, - previousSnapshot.#strictProjectMetadataMap - ); - } + operationAdditionalFileCount + ) ?? createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir); this.#additionalHashes = additionalHashes; this.#globalAdditionalHashes = globalAdditionalHashes; + this.#environment = environment; // Parse Node.js version once so it doesn't need to be re-parsed per operation this.#nodeVersionByGranularity = _parseNodeVersion(nodeVersion); this.#lookupByPath = lookupByPath; @@ -387,11 +335,9 @@ export class InputsSnapshot implements IInputsSnapshot { */ public getTrackedFileHashesForOperation( project: IRushConfigurationProjectForSnapshot, - operationName?: string, - policy: InputSnapshotPolicy = 'legacy' + operationName?: string ): ReadonlyMap { - const record: IInternalInputsSnapshotProjectMetadata | undefined = - this.#getProjectMetadataMap(policy).get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = this.#projectMetadataMap.get(project); if (!record) { throw new InternalError(`No information available for project at ${project.projectFolder}`); } @@ -404,22 +350,6 @@ export class InputsSnapshot implements IInputsSnapshot { const filter: (filePath: string) => boolean = getOrCreateProjectFilter(record); let outputValidator: LookupByPath | undefined; - const validateInputPath: (filePath: string, inputKind: 'tracked' | 'additional') => void = ( - filePath, - inputKind - ) => { - // This should eventually use a global file origin map to detect outputs of *any* operation. - const outputMatch: string | undefined = outputValidator?.findChildPath( - policy === 'strict-v1' ? getOutputLookupPath(this.rootDirectory, filePath) : filePath - ); - if (outputMatch) { - throw new Error( - `Configured output folder "${outputMatch}" for operation "${operationName}" in project "${project.projectRelativeFolder}" contains ${inputKind} input file "${filePath}".` + - ` If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from "outputFolderNames".` + - ` This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder.` - ); - } - }; if (operationName) { const operationSettings: Readonly | undefined = @@ -427,14 +357,10 @@ export class InputsSnapshot implements IInputsSnapshot { const outputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; if (outputFolderNames) { + const { relativePrefix } = record; outputValidator = new LookupByPath(); for (const folderName of outputFolderNames) { - outputValidator.setItem( - policy === 'strict-v1' - ? getOutputLookupPath(this.rootDirectory, path.resolve(project.projectFolder, folderName)) - : `${record.relativePrefix}/${folderName}`, - folderName - ); + outputValidator.setItem(`${relativePrefix}/${folderName}`, folderName); } } @@ -445,9 +371,6 @@ export class InputsSnapshot implements IInputsSnapshot { // Sort the additional files to ensure deterministic hash computation const sortedAdditionalFiles: string[] = Array.from(additionalFilesForOperation).sort(); for (const [filePath, hash] of this.#resolveHashes(sortedAdditionalFiles)) { - if (policy === 'strict-v1') { - validateInputPath(filePath, 'additional'); - } hashes.set(filePath, hash); } } @@ -466,7 +389,17 @@ export class InputsSnapshot implements IInputsSnapshot { hashes.set(filePath, hash); } - validateInputPath(filePath, 'tracked'); + // Ensure that the configured output folders for this operation do not contain any input files + // This should be reworked to operate on a global file origin map to ensure a hashed input + // is not a declared output of *any* operation. + const outputMatch: string | undefined = outputValidator?.findChildPath(filePath); + if (outputMatch) { + throw new Error( + `Configured output folder "${outputMatch}" for operation "${operationName}" in project "${project.projectRelativeFolder}" contains tracked input file "${filePath}".` + + ` If it is intended that this operation modifies its own input files, modify the build process to emit a warning if the output version differs from the input, and remove the directory from "outputFolderNames".` + + ` This will ensure cache correctness. Otherwise, change the build process to output to a disjoint folder.` + ); + } } // Only a complete result is kept, so that a query that fails fails again @@ -482,11 +415,9 @@ export class InputsSnapshot implements IInputsSnapshot { public getOperationOwnStateHash( project: IRushConfigurationProjectForSnapshot, operationName?: string, - environment?: Readonly>, - policy: InputSnapshotPolicy = 'legacy' + environment?: Readonly> ): string { - const record: IInternalInputsSnapshotProjectMetadata | undefined = - this.#getProjectMetadataMap(policy).get(project); + const record: IInternalInputsSnapshotProjectMetadata | undefined = this.#projectMetadataMap.get(project); if (!record) { throw new Error(`No information available for project at ${project.projectFolder}`); } @@ -494,25 +425,20 @@ export class InputsSnapshot implements IInputsSnapshot { const operationSettings: Readonly | undefined = operationName ? record.projectConfig?.operationSettingsByOperationName.get(operationName) : undefined; - const snapshotEnvironment: EnvironmentMap | Readonly> = - policy === 'strict-v1' ? this.#strictEnvironment : this.#environment; + const snapshotEnvironment: EnvironmentMap = this.#environment; if (environment && operationSettings?.dependsOnEnvVars?.length) { - const operationEnvironment: EnvironmentMap | Readonly> = - policy === 'strict-v1' ? new EnvironmentMap(environment) : environment; + const operationEnvironment: EnvironmentMap = new EnvironmentMap(environment); if ( - operationSettings.dependsOnEnvVars.some((envVar: string) => { - const value: string | undefined = getEnvironmentValue(operationEnvironment, envVar); - const snapshotValue: string | undefined = getEnvironmentValue(snapshotEnvironment, envVar); - return policy === 'strict-v1' ? value !== snapshotValue : (value || '') !== (snapshotValue || ''); - }) + operationSettings.dependsOnEnvVars.some( + (envVar: string) => operationEnvironment.get(envVar) !== snapshotEnvironment.get(envVar) + ) ) { - // The operation's environment differs from the snapshot's in a variable that it hashes, so don't memoize. + // The operation's environment differs in a variable that it hashes, so don't memoize. return this.#computeOperationOwnStateHash( project, operationName, operationSettings, - operationEnvironment, - policy + operationEnvironment ); } } @@ -524,8 +450,7 @@ export class InputsSnapshot implements IInputsSnapshot { project, operationName, operationSettings, - snapshotEnvironment, - policy + snapshotEnvironment ); hashByOperationName.set(operationName, hash); } @@ -537,32 +462,21 @@ export class InputsSnapshot implements IInputsSnapshot { project: IRushConfigurationProjectForSnapshot, operationName: string | undefined, operationSettings: Readonly | undefined, - environment: EnvironmentMap | Readonly>, - policy: InputSnapshotPolicy + environment: EnvironmentMap ): string { - const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation( - project, - operationName, - policy - ); + const hashes: ReadonlyMap = this.getTrackedFileHashesForOperation(project, operationName); const hasher: Hash = createHash('sha1'); - if (policy === 'strict-v1') { - hasher.update('rush-input-snapshot:strict-v1\0'); - } // If this is for a specific operation, apply operation-specific options if (operationSettings) { const { dependsOnEnvVars, dependsOnNodeVersion, outputFolderNames } = operationSettings; - if (dependsOnEnvVars) { + if (dependsOnEnvVars?.length) { + // Earlier hashes start with hashDelimiter (or are empty). Keep this encoding in a distinct namespace. + hasher.update('env-v2\0'); // As long as we enumerate environment variables in a consistent order, we will get a stable hash. // Changing the order in rush-project.json will change the hash anyway since the file contents are part of the hash. for (const envVar of dependsOnEnvVars) { - const value: string | undefined = getEnvironmentValue(environment, envVar); - hasher.update( - policy === 'strict-v1' - ? `${hashDelimiter}$${JSON.stringify([envVar, value])}` - : `${hashDelimiter}$${envVar}=${value || ''}` - ); + hasher.update(`${hashDelimiter}$${JSON.stringify([envVar, environment.get(envVar)])}`); } } @@ -585,21 +499,6 @@ export class InputsSnapshot implements IInputsSnapshot { return hasher.digest('hex'); } - #getProjectMetadataMap( - policy: InputSnapshotPolicy - ): Map { - assertInputSnapshotPolicySupported(this, policy); - if (policy === 'legacy') { - return this.#projectMetadataMap; - } - return (this.#strictProjectMetadataMap ??= createProjectMetadataMap( - this.hashes, - this.#lookupByPath, - this.#projectMap, - this.rootDirectory - )); - } - *#resolveHashes(filePaths: Iterable): Generator<[string, string]> { const { hashes } = this; const additionalHashes: ReadonlyMap | undefined = this.#additionalHashes; @@ -614,36 +513,46 @@ export class InputsSnapshot implements IInputsSnapshot { } /** - * Returns the state of each project for a later snapshot with the given parameters. It is the state of each - * project in this snapshot whose inputs have the same hashes, and new state for each other project. Returns - * undefined if the later snapshot differs from this one in more than the hashes of files. + * Reuses the state of projects whose input hashes are unchanged. Returns undefined if there is no previous + * snapshot or it differs in more than file hashes. */ #tryDeriveProjectMetadataMap( + previousSnapshot: InputsSnapshot | undefined, params: IInputsSnapshotParameters, + environment: EnvironmentMap, nodeVersion: string, globalAdditionalHashes: ReadonlyMap | undefined, - operationAdditionalFileCount: number, - previousMetadataMap: ReadonlyMap< - IRushConfigurationProjectForSnapshot, - IInternalInputsSnapshotProjectMetadata - > + operationAdditionalFileCount: number ): Map | undefined { const { additionalHashes, hashes, lookupByPath, projectMap, rootDir } = params; if ( - lookupByPath !== this.#lookupByPath || - projectMap !== this.#projectMap || - rootDir !== this.rootDirectory || - nodeVersion !== this.#nodeVersion || - operationAdditionalFileCount !== this.#operationAdditionalFileCount || - !areMapsEqual(globalAdditionalHashes ?? EMPTY_MAP, this.#globalAdditionalHashes ?? EMPTY_MAP) + !previousSnapshot || + lookupByPath !== previousSnapshot.#lookupByPath || + projectMap !== previousSnapshot.#projectMap || + rootDir !== previousSnapshot.rootDirectory || + nodeVersion !== previousSnapshot.#nodeVersion || + operationAdditionalFileCount !== previousSnapshot.#operationAdditionalFileCount || + !areMapsEqual( + globalAdditionalHashes ?? EMPTY_MAP, + previousSnapshot.#globalAdditionalHashes ?? EMPTY_MAP + ) || + !environment.equals(previousSnapshot.#environment) ) { return undefined; } const changedFiles: Set = new Set(); - addChangedFiles(this.hashes, hashes, changedFiles); - addChangedFiles(this.#additionalHashes ?? EMPTY_MAP, additionalHashes ?? EMPTY_MAP, changedFiles); + addChangedFiles(previousSnapshot.hashes, hashes, changedFiles); + addChangedFiles( + previousSnapshot.#additionalHashes ?? EMPTY_MAP, + additionalHashes ?? EMPTY_MAP, + changedFiles + ); + const previousMetadataMap: ReadonlyMap< + IRushConfigurationProjectForSnapshot, + IInternalInputsSnapshotProjectMetadata + > = previousSnapshot.#projectMetadataMap; const projectMetadataMap: Map< IRushConfigurationProjectForSnapshot, IInternalInputsSnapshotProjectMetadata @@ -854,44 +763,6 @@ function areMapsEqual(a: ReadonlyMap, b: ReadonlyMap>, - name: string -): string | undefined { - return environment instanceof EnvironmentMap ? environment.get(name) : environment[name]; -} - -function areEnvironmentsEqual( - a: Readonly>, - b: Readonly> -): boolean { - const keys: string[] = Object.keys(a); - if (keys.length !== Object.keys(b).length) { - return false; - } - for (const key of keys) { - if (a[key] !== b[key] || !Object.prototype.hasOwnProperty.call(b, key)) { - return false; - } - } - return true; -} - -function areEnvironmentMapsEqual(a: EnvironmentMap, b: EnvironmentMap): boolean { - const keys: string[] = Array.from(a.names()); - if (keys.length !== Array.from(b.names()).length) { - return false; - } - - for (const key of keys) { - if (a.get(key) !== b.get(key)) { - return false; - } - } - - return true; -} - /** * Parses a Node.js version string once and returns pre-computed strings for each granularity level. * @@ -948,11 +819,6 @@ function createInternalRecord( }; } -function getOutputLookupPath(rootDir: string, filePath: string): string { - const relativePath: string = Path.convertToSlashes(path.relative(rootDir, path.resolve(rootDir, filePath))); - return process.platform === 'win32' ? relativePath.toLowerCase() : relativePath; -} - function getRelativePrefix(project: IRushConfigurationProjectForSnapshot, rootDir: string): string { return Path.convertToSlashes(path.relative(rootDir, project.projectFolder)); } diff --git a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts index 21270ca3d5f..8689d6f8639 100644 --- a/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts +++ b/libraries/rush-lib/src/logic/incremental/test/InputsSnapshot.test.ts @@ -1,19 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as path from 'node:path'; - import { LookupByPath } from '@rushstack/lookup-by-path'; +import { EnvironmentMap } from '@rushstack/node-core-library'; +import { createHash } from 'node:crypto'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; -import { RushConstants } from '../../RushConstants'; import { InputsSnapshot, type IInputsSnapshotParameters, type IInputsSnapshotProjectMetadata, - type InputSnapshotPolicy, type IRushConfigurationProjectForSnapshot } from '../InputsSnapshot'; +import { RushConstants } from '../../RushConstants'; describe(InputsSnapshot.name, () => { function getTestConfig(): { @@ -54,14 +53,6 @@ describe(InputsSnapshot.name, () => { return { project, input }; } - function getStrictHash( - input: InputsSnapshot, - project: IRushConfigurationProjectForSnapshot, - environment?: Record - ): string { - return input.getOperationOwnStateHash(project, '_phase:build', environment, 'strict-v1'); - } - it('Exposes the time at which it began reading the working tree', () => { const { options } = getTestConfig(); expect(new InputsSnapshot(options).workingTreeReadStartTimeMs).toBeUndefined(); @@ -189,116 +180,6 @@ describe(InputsSnapshot.name, () => { expect(result.get('/ext/config.json')).toEqual('hash4'); }); - function getOutputSnapshot( - inputKind: 'tracked' | 'additional', - filePath: string - ): ReturnType { - const { project, options } = getTestConfig(); - const projectConfig: Pick = { - operationSettingsByOperationName: new Map([ - ['_phase:build', { operationName: '_phase:build', outputFolderNames: ['lib'] }] - ]) - }; - return { - project, - input: new InputsSnapshot({ - ...options, - hashes: new Map([ - ['a/source.js', 'source-hash'], - ...(inputKind === 'tracked' ? [[filePath, 'uncovered-hash'] as const] : []) - ]), - additionalHashes: new Map([[filePath, 'uncovered-hash']]), - projectMap: new Map([ - [ - project, - { - projectConfig: projectConfig as RushProjectConfiguration, - additionalFilesByOperationName: - inputKind === 'additional' ? new Map([['_phase:build', new Set([filePath])]]) : undefined - } - ] - ]) - }) - }; - } - - it.each([ - ['tracked', 'a/lib/uncovered.js'], - ['additional', 'a/lib/uncovered.js'], - ['additional', 'a/lib/../lib/uncovered.js'], - ['additional', path.resolve('/root/a/lib/uncovered.js')] - ] as const)('Rejects %s uncovered output inputs before operation hashing: %s', (inputKind, filePath) => { - const { project, input } = getOutputSnapshot(inputKind, filePath); - // Repeated failures must not expose a partially validated input map or cached operation hash. - for (let attempt: number = 0; attempt < 2; attempt++) { - expect(() => getStrictHash(input, project)).toThrow(`contains ${inputKind} input file "${filePath}"`); - expect(() => input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1')).toThrow( - 'output to a disjoint folder' - ); - } - }); - - it.each(['win32', 'linux'])('Uses %s path casing for output input validation', (platform) => { - const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; - Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); - try { - for (const [inputKind, filePath] of [ - ['tracked', 'a/LIB/UNCOVERED.js'], - ['additional', 'A/LIB/UNCOVERED.js'], - ['additional', path.resolve('/root/A/LIB/UNCOVERED.js')] - ] as const) { - const { project, input } = getOutputSnapshot(inputKind, filePath); - if (platform === 'win32') { - expect(() => getStrictHash(input, project)).toThrow( - `contains ${inputKind} input file "${filePath}"` - ); - } else { - expect( - input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1').get(filePath) - ).toBe('uncovered-hash'); - } - } - } finally { - Object.defineProperty(process, 'platform', descriptor); - } - }); - - it.each([ - 'a/lib-sources/uncovered.js', - path.resolve('/root/a/lib-sources/uncovered.js'), - path.resolve('/ext/config.json') - ])('Preserves valid additional input keys and hashes outside output folders: %s', (filePath) => { - const { project, input } = getOutputSnapshot('additional', filePath); - const hashes: ReadonlyMap = input.getTrackedFileHashesForOperation( - project, - '_phase:build', - 'strict-v1' - ); - expect([...hashes]).toEqual([ - [filePath, 'uncovered-hash'], - ['a/source.js', 'source-hash'] - ]); - expect(input.getTrackedFileHashesForOperation(project, '_phase:build', 'strict-v1')).toBe(hashes); - expect(() => getStrictHash(input, project)).not.toThrow(); - }); - - it.each([false, true])( - 'keeps strict validation separate from legacy queries (strict first: %s)', - (strictFirst) => { - const { project, input } = getOutputSnapshot('additional', 'a/lib/uncovered.js'); - const strictQuery = (): string => getStrictHash(input, project); - if (strictFirst) { - expect(strictQuery).toThrow('contains additional input file'); - } - const legacyHash: string = input.getOperationOwnStateHash(project, '_phase:build'); - expect( - input.getTrackedFileHashesForOperation(project, '_phase:build').get('a/lib/uncovered.js') - ).toBe('uncovered-hash'); - expect(strictQuery).toThrow('contains additional input file'); - expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(legacyHash); - } - ); - it('Respects globalAdditionalFiles', () => { const { project, options } = getTestConfig(); @@ -852,7 +733,7 @@ describe(InputsSnapshot.name, () => { } ] ]), - environment: {} + environment: new EnvironmentMap() }); const result1: string = input1.getOperationOwnStateHash(project, '_phase:build'); @@ -870,7 +751,7 @@ describe(InputsSnapshot.name, () => { } ] ]), - environment: { ENV_VAR: 'some_value' } + environment: new EnvironmentMap({ ENV_VAR: 'some_value' }) }); const result2: string = input2.getOperationOwnStateHash(project, '_phase:build'); @@ -891,7 +772,7 @@ describe(InputsSnapshot.name, () => { new InputsSnapshot({ ...options, projectMap: new Map([[project, { projectConfig: projectConfig as RushProjectConfiguration }]]), - environment + environment: new EnvironmentMap(environment) }); const snapshotA: InputsSnapshot = createSnapshot({ ENV_VAR: 'a', OTHER: 'a' }); const snapshotB: InputsSnapshot = createSnapshot({ ENV_VAR: 'b' }); @@ -917,141 +798,126 @@ describe(InputsSnapshot.name, () => { }); }); - describe('environment identity', () => { + describe('declared environment cache identity', () => { function createSnapshot( environment: Record, - names: string[] = ['ENV_VAR'] + dependsOnEnvVars: string[] = ['ENV_VAR'] ): ReturnType { const { project, options } = getTestConfig(); const projectConfig: Pick = { operationSettingsByOperationName: new Map([ - ['_phase:build', { operationName: '_phase:build', dependsOnEnvVars: names }] + ['_phase:build', { operationName: '_phase:build', dependsOnEnvVars }] ]) }; return { project, input: new InputsSnapshot({ ...options, - environment, + environment: new EnvironmentMap(environment), projectMap: new Map([[project, { projectConfig: projectConfig as RushProjectConfiguration }]]) }) }; } - describe.each([false, true])('with an explicit operation environment: %s', (useOperationEnvironment) => { + describe.each([false, true])('with an explicit operation environment: %s', (explicit) => { function getHash(environment: Record, names?: string[]): string { - const { project, input } = createSnapshot( - useOperationEnvironment ? { ENV_VAR: 'snapshot' } : environment, - names - ); - return getStrictHash(input, project, useOperationEnvironment ? environment : undefined); + const { project, input } = createSnapshot(explicit ? { ENV_VAR: 'snapshot' } : environment, names); + return input.getOperationOwnStateHash(project, '_phase:build', explicit ? environment : undefined); } - it('distinguishes absence, empty values and literal sentinel strings', () => { + it('does not reuse cached output when an empty value overrides a default', () => { + const render = (environment: Record): string => + JSON.stringify({ publicPath: environment.ENV_VAR ?? '/default/' }); + const absent: Record = {}; + const empty: Record = { ENV_VAR: '' }; + const cache: Map = new Map([[getHash(absent), render(absent)]]); + expect(render(absent)).not.toBe(render(empty)); + expect(cache.has(getHash(empty))).toBe(false); + cache.set(getHash(empty), render(empty)); + expect(cache.get(getHash(absent))).toBe('{"publicPath":"/default/"}'); + expect(cache.get(getHash(empty))).toBe('{"publicPath":""}'); + }); + + it('distinguishes unset, empty and literal sentinel values', () => { const values: Array = [undefined, '', 'undefined', 'null', 'value']; expect(new Set(values.map((value) => getHash({ ENV_VAR: value }))).size).toBe(values.length); expect(getHash({})).toBe(getHash({ ENV_VAR: undefined })); }); - it.each(['win32', 'linux'])('uses %s environment-name semantics', (platform) => { - const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; - Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); - try { - const upper: string = getHash({ ENV_VAR: 'first' }); - const lower: string = getHash({ env_var: 'first' }); - const changed: string = getHash({ env_var: 'second' }); - if (platform === 'win32') { - expect(lower).toBe(upper); - expect(changed).not.toBe(lower); - } else { - expect(lower).toBe(getHash({})); - expect(changed).toBe(lower); - expect(lower).not.toBe(upper); + it.each(['win32', 'linux'])( + 'uses %s environment-name lookup without changing declarations', + (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const upper: string = getHash({ ENV_VAR: 'first' }); + const lower: string = getHash({ env_var: 'first' }); + expect(lower === upper).toBe(platform === 'win32'); + expect(lower === getHash({ env_var: 'second' })).toBe(platform !== 'win32'); + if (platform !== 'win32') { + expect(lower).toBe(getHash({})); + } + } finally { + Object.defineProperty(process, 'platform', descriptor); } - } finally { - Object.defineProperty(process, 'platform', descriptor); } - }); + ); - it('does not include undeclared environment inputs', () => { + it('excludes undeclared values and disambiguates embedded separators', () => { expect(getHash({ ENV_VAR: 'same', OTHER: 'first' })).toBe( getHash({ ENV_VAR: 'same', OTHER: 'second' }) ); - }); - - it('does not confuse embedded separators with another declared variable', () => { const delimiter: string = RushConstants.hashDelimiter; - expect(getHash({ A: `a${delimiter}$B=b`, B: 'c' }, ['A', 'B'])).not.toBe( - getHash({ A: 'a', B: `b${delimiter}$B=c` }, ['A', 'B']) - ); + const names: string[] = ['A', 'B']; + const first = { A: `a${delimiter}$B=b`, B: 'c' }; + const second = { A: 'a', B: `b${delimiter}$B=c` }; + const legacyEncoding = (environment: Record): string => + names.map((name) => `${delimiter}$${name}=${environment[name] || ''}`).join(''); + expect(legacyEncoding(first)).toBe(legacyEncoding(second)); + expect(getHash(first, names)).not.toBe(getHash(second, names)); }); }); - it('captures supplied values before the first hash query', () => { - const environment: Record = { ENV_VAR: 'captured' }; - const { project, input } = createSnapshot(environment); - const { project: baselineProject, input: baseline } = createSnapshot({ ...environment }); - environment.ENV_VAR = 'changed'; - expect(getStrictHash(input, project)).toBe(getStrictHash(baseline, baselineProject)); - expect(getStrictHash(input, project, environment)).not.toBe(getStrictHash(input, project)); - expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe( - input.getOperationOwnStateHash(project, '_phase:build', environment) - ); - expect(input.getOperationOwnStateHash(project, '_phase:build')).not.toBe( - baseline.getOperationOwnStateHash(baselineProject, '_phase:build') - ); + it('does not reuse the previous encoding for any tested environment value', () => { + const values: Array = [undefined, '', 'undefined', 'null', 'a|$ENV_VAR=b', '"\\']; + const oldHashes: Set = new Set(); + const newHashes: Set = new Set(); + for (const value of values) { + const { input, project } = createSnapshot({ ENV_VAR: value }); + const hasher = createHash('sha1'); + hasher.update(`${RushConstants.hashDelimiter}$ENV_VAR=${value || ''}`); + for (const [file, hash] of input.getTrackedFileHashesForOperation(project, '_phase:build')) { + hasher.update(`${RushConstants.hashDelimiter}${file}${RushConstants.hashDelimiter}${hash}`); + } + oldHashes.add(hasher.digest('hex')); + newHashes.add(input.getOperationOwnStateHash(project, '_phase:build')); + } + expect([...newHashes].some((hash) => oldHashes.has(hash))).toBe(false); }); - it.each(['win32', 'linux'])('preserves raw legacy environment identity on %s', (platform) => { - const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; - Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); - try { - const getLegacyHash = (environment: Record): string => { - const { input, project } = createSnapshot(environment); - const hash: string = input.getOperationOwnStateHash(project, '_phase:build'); - expect(input.getOperationOwnStateHash(project, '_phase:build', { ENV_VAR: undefined })).toBe(hash); - return hash; - }; - expect(getLegacyHash({})).toBe(getLegacyHash({ ENV_VAR: '' })); - expect(getLegacyHash({ env_var: 'nonempty' })).toBe(getLegacyHash({})); - } finally { - Object.defineProperty(process, 'platform', descriptor); - } + it('preserves hashes when no environment variables are declared', () => { + const { project, input } = getTrivialSnapshot(); + const { project: emptyProject, input: empty } = createSnapshot({ ENV_VAR: 'ignored' }, []); + expect(empty.getOperationOwnStateHash(emptyProject, '_phase:build', { ENV_VAR: 'different' })).toBe( + input.getOperationOwnStateHash(project, '_phase:build') + ); }); - it.each([false, true])( - 'partitions identities with no declared environment (strict first: %s)', - (strictFirst) => { - const { project, input } = createSnapshot({}, []); - if (strictFirst) { - getStrictHash(input, project); - } - const legacy: string = input.getOperationOwnStateHash(project, '_phase:build'); - const strict: string = getStrictHash(input, project); - expect(strict).not.toBe(legacy); - expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(legacy); - expect(getStrictHash(input, project)).toBe(strict); - } - ); - - it.each([undefined, ''])( - 'does not reuse or overwrite the snapshot hash for a different operation value (snapshot: %s)', - (snapshotValue) => { - const { project, input } = createSnapshot({ ENV_VAR: snapshotValue }); - const operationEnvironment: Record = { - ENV_VAR: snapshotValue === undefined ? '' : undefined - }; - const { project: operationProject, input: operationSnapshot } = createSnapshot(operationEnvironment); - const operationHash: string = getStrictHash(operationSnapshot, operationProject); - expect(getStrictHash(input, project, operationEnvironment)).toBe(operationHash); - const snapshotHash: string = getStrictHash(input, project); - expect(snapshotHash).not.toBe(operationHash); - expect(getStrictHash(input, project, operationEnvironment)).toBe(operationHash); - operationEnvironment.ENV_VAR = snapshotValue; - expect(getStrictHash(input, project, operationEnvironment)).toBe(snapshotHash); - expect(getStrictHash(input, project)).toBe(snapshotHash); - } - ); + it.each([undefined, ''])('isolates an explicit unset/empty override from memoization (%s)', (value) => { + const { project, input } = createSnapshot({ ENV_VAR: value }); + const operationEnvironment = { ENV_VAR: value === undefined ? '' : undefined }; + const { project: otherProject, input: other } = createSnapshot(operationEnvironment); + const operationHash: string = other.getOperationOwnStateHash(otherProject, '_phase:build'); + expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( + operationHash + ); + const snapshotHash: string = input.getOperationOwnStateHash(project, '_phase:build'); + expect(snapshotHash).not.toBe(operationHash); + expect(input.getOperationOwnStateHash(project, '_phase:build', operationEnvironment)).toBe( + operationHash + ); + expect(input.getOperationOwnStateHash(project, '_phase:build')).toBe(snapshotHash); + }); }); describe('previousSnapshot', () => { @@ -1113,7 +979,7 @@ describe(InputsSnapshot.name, () => { options: { rootDir: '/root', additionalHashes: new Map([['/ext/tool.json', 'ext1']]), - environment: { FOO: '1' }, + environment: new EnvironmentMap({ FOO: '1' }), globalAdditionalFiles: ['common/config/global.json'], hashes: new Map([ ['a/README.md', 'a0'], @@ -1145,27 +1011,18 @@ describe(InputsSnapshot.name, () => { }; } - function queryAll( - snapshot: InputsSnapshot, - projects: IRushConfigurationProjectForSnapshot[], - policy: InputSnapshotPolicy = 'legacy' - ): string[] { + function queryAll(snapshot: InputsSnapshot, projects: IRushConfigurationProjectForSnapshot[]): string[] { const results: string[] = []; for (const project of projects) { for (const operationName of operationNames) { try { const tracked: [string, string][] = Array.from( - snapshot.getTrackedFileHashesForOperation(project, operationName, policy) - ); - const hash: string = snapshot.getOperationOwnStateHash(project, operationName, undefined, policy); - const hashInOtherEnvironment: string = snapshot.getOperationOwnStateHash( - project, - operationName, - { - FOO: 'other' - }, - policy + snapshot.getTrackedFileHashesForOperation(project, operationName) ); + const hash: string = snapshot.getOperationOwnStateHash(project, operationName); + const hashInOtherEnvironment: string = snapshot.getOperationOwnStateHash(project, operationName, { + FOO: 'other' + }); results.push(JSON.stringify({ tracked, hash, hashInOtherEnvironment })); } catch (error) { results.push(`error: ${error.message}`); @@ -1179,53 +1036,58 @@ describe(InputsSnapshot.name, () => { function expectSameAsNewSnapshot( derived: InputsSnapshot, options: IInputsSnapshotParameters, - projects: IRushConfigurationProjectForSnapshot[], - policy: InputSnapshotPolicy = 'legacy' + projects: IRushConfigurationProjectForSnapshot[] ): void { const created: InputsSnapshot = new InputsSnapshot({ ...options, previousSnapshot: undefined }); - expect(queryAll(derived, projects, policy)).toEqual(queryAll(created, projects, policy)); + expect(queryAll(derived, projects)).toEqual(queryAll(created, projects)); expect(derived.hashes).toBe(options.hashes); expect(derived.hasUncommittedChanges).toBe(options.hasUncommittedChanges); expect(derived.workingTreeReadStartTimeMs).toBe(options.workingTreeReadStartTimeMs); } - it.each([false, true])( - 'reuses raw legacy and captured strict state independently (strict first: %s)', - (strictFirst) => { - const { options, a } = getDerivationTestConfig(); - const environment: Record = { FOO: 'old' }; - const previousSnapshot: InputsSnapshot = new InputsSnapshot({ ...options, environment }); - if (strictFirst) { - getStrictHash(previousSnapshot, a); - } - const legacy: string = previousSnapshot.getOperationOwnStateHash(a, '_phase:build'); - const strict: string = getStrictHash(previousSnapshot, a); - environment.FOO = 'new'; - const next: InputsSnapshot = new InputsSnapshot({ ...options, environment, previousSnapshot }); - const fresh: InputsSnapshot = new InputsSnapshot({ ...options, environment }); - expect(next.getOperationOwnStateHash(a, '_phase:build')).toBe(legacy); - expect(fresh.getOperationOwnStateHash(a, '_phase:build')).not.toBe(legacy); - expect(getStrictHash(next, a)).not.toBe(strict); - expect(getStrictHash(next, a)).toBe(getStrictHash(fresh, a)); - expect(getStrictHash(previousSnapshot, a)).toBe(strict); - } - ); - - it.each(['legacy', 'strict-v1'] as const)('reuses only validated %s maps across snapshots', (policy) => { - const { options, a, b, projects } = getDerivationTestConfig(); - const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - queryAll(previousSnapshot, projects, policy); - const hashes: Map = new Map(options.hashes); - hashes.set('a/src/x.ts', 'changed'); - const nextOptions: IInputsSnapshotParameters = { ...options, hashes, previousSnapshot }; + it.each([undefined, ''])('invalidates reuse for unset/empty changes (%s)', (value) => { + const { options, a, projects } = getDerivationTestConfig(); + const previousSnapshot: InputsSnapshot = new InputsSnapshot({ + ...options, + environment: new EnvironmentMap({ FOO: value }) + }); + const before: string = previousSnapshot.getOperationOwnStateHash(a, '_phase:build'); + const nextOptions: IInputsSnapshotParameters = { + ...options, + environment: new EnvironmentMap({ FOO: value === undefined ? '' : undefined }), + previousSnapshot + }; const next: InputsSnapshot = new InputsSnapshot(nextOptions); - expect(next.getTrackedFileHashesForOperation(b, '_phase:build', policy)).toBe( - previousSnapshot.getTrackedFileHashesForOperation(b, '_phase:build', policy) - ); - expect(next.getTrackedFileHashesForOperation(a, '_phase:build', policy)).not.toBe( - previousSnapshot.getTrackedFileHashesForOperation(a, '_phase:build', policy) - ); - expectSameAsNewSnapshot(next, nextOptions, projects, policy); + expect(next.getOperationOwnStateHash(a, '_phase:build')).not.toBe(before); + expectSameAsNewSnapshot(next, nextOptions, projects); + }); + + it.each(['win32', 'linux'])('uses %s environment comparison when deriving a snapshot', (platform) => { + const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); + try { + const { options, a, projects } = getDerivationTestConfig(); + const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); + const previousFiles = previousSnapshot.getTrackedFileHashesForOperation(a, '_phase:build'); + queryAll(previousSnapshot, projects); + const nextOptions: IInputsSnapshotParameters = { + ...options, + environment: new EnvironmentMap({ foo: '1' }), + previousSnapshot + }; + const next: InputsSnapshot = new InputsSnapshot(nextOptions); + expect(next.getTrackedFileHashesForOperation(a, '_phase:build') === previousFiles).toBe( + platform === 'win32' + ); + expectSameAsNewSnapshot(next, nextOptions, projects); + const changedOptions: IInputsSnapshotParameters = { + ...nextOptions, + environment: new EnvironmentMap({ foo: '2' }) + }; + expectSameAsNewSnapshot(new InputsSnapshot(changedOptions), changedOptions, projects); + } finally { + Object.defineProperty(process, 'platform', descriptor); + } }); it('Reuses the state of each project whose inputs did not change', () => { @@ -1332,8 +1194,11 @@ describe(InputsSnapshot.name, () => { const hashes: Map = new Map(options.hashes); hashes.set('a/src/x.ts', 'a1-changed'); const changes: [string, () => Partial][] = [ - ['environment', () => ({ environment: { FOO: '2' } })], - ['an environment variable that no operation hashes', () => ({ environment: { FOO: '1', BAR: '' } })], + ['environment', () => ({ environment: new EnvironmentMap({ FOO: '2' }) })], + [ + 'an environment variable that no operation hashes', + () => ({ environment: new EnvironmentMap({ FOO: '1', BAR: '' }) }) + ], ['node version', () => ({ nodeVersion: 'v23.0.0' })], [ 'global additional file', @@ -1372,76 +1237,6 @@ describe(InputsSnapshot.name, () => { } }); - it.each([false, true])( - 'does not reuse stale environment state after caller mutation (previous hash computed: %s)', - (computePreviousHash) => { - const { projects, a, options } = getDerivationTestConfig(); - const environment: Record = { FOO: 'captured' }; - options.environment = environment; - const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - const baselineHash: string = getStrictHash(new InputsSnapshot(options), a); - if (computePreviousHash) { - expect(getStrictHash(previousSnapshot, a)).toBe(baselineHash); - } - - environment.FOO = 'changed'; - const derivedOptions: IInputsSnapshotParameters = { ...options, previousSnapshot }; - const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect(getStrictHash(derived, a)).not.toBe(baselineHash); - expect(getStrictHash(previousSnapshot, a)).toBe(baselineHash); - expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); - } - ); - - it.each([undefined, ''])('distinguishes absent and empty environments during reuse: %s', (value) => { - const { projects, a, options } = getDerivationTestConfig(); - options.environment = { FOO: value }; - const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - const baselineHash: string = getStrictHash(previousSnapshot, a); - const derivedOptions: IInputsSnapshotParameters = { - ...options, - environment: { FOO: value === undefined ? '' : undefined }, - previousSnapshot - }; - const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect(getStrictHash(derived, a)).not.toBe(baselineHash); - expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); - }); - - it.each(['win32', 'linux'])('uses %s environment-name semantics during reuse', (platform) => { - const descriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; - Object.defineProperty(process, 'platform', { ...descriptor, value: platform }); - try { - const { projects, a, options } = getDerivationTestConfig(); - const previousSnapshot: InputsSnapshot = new InputsSnapshot(options); - const previousFiles: ReadonlyMap = previousSnapshot.getTrackedFileHashesForOperation( - a, - '_phase:build', - 'strict-v1' - ); - queryAll(previousSnapshot, projects, 'strict-v1'); - - const derivedOptions: IInputsSnapshotParameters = { - ...options, - environment: { foo: '1' }, - previousSnapshot - }; - const derived: InputsSnapshot = new InputsSnapshot(derivedOptions); - expect( - derived.getTrackedFileHashesForOperation(a, '_phase:build', 'strict-v1') === previousFiles - ).toBe(platform === 'win32'); - expectSameAsNewSnapshot(derived, derivedOptions, projects, 'strict-v1'); - - const changedOptions: IInputsSnapshotParameters = { - ...derivedOptions, - environment: { foo: '2' } - }; - expectSameAsNewSnapshot(new InputsSnapshot(changedOptions), changedOptions, projects, 'strict-v1'); - } finally { - Object.defineProperty(process, 'platform', descriptor); - } - }); - it('Computes the same state as a new snapshot after any sequence of changes', () => { const { projects, bAdditionalFiles, options } = getDerivationTestConfig(); const files: string[] = [ diff --git a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap index f8dcd553d3f..91b3e542ffb 100644 --- a/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap +++ b/libraries/rush-lib/src/logic/incremental/test/__snapshots__/InputsSnapshot.test.ts.snap @@ -6,9 +6,9 @@ exports[`InputsSnapshot getOperationOwnStateHash Handles trivial input 1`] = `"a exports[`InputsSnapshot getOperationOwnStateHash Respects additionalOutputFilesByOperationName 1`] = `"07f4147294d21a07865de84875d60bfbcc690652"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"ad1f915d0ac6331c09febbbe1496c970a5401b73"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 1`] = `"980bd656afb55515b53be36f9d9ea7aaebfaa76e"`; -exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"2c68d56fc9278b6495496070a6a992b929c37a83"`; +exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnEnvVars 2`] = `"00211b6f51668d85598e5ca390fd2e6a630a103b"`; exports[`InputsSnapshot getOperationOwnStateHash Respects dependsOnNodeVersion 1`] = `"cc182bde6aa81c0410ede7db22f3af2f0a24d3e3"`; diff --git a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts index 4041336870f..12299578730 100644 --- a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts @@ -14,7 +14,6 @@ import { clusterOperations, type IOperationBuildCacheContext } from './Cacheable import { DisjointSet } from '../cobuild/DisjointSet'; import type { IConfigurableOperation } from './IOperationExecutionResult'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { assertInputSnapshotPolicySupported, type InputSnapshotPolicy } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'BuildPlanPlugin' = 'BuildPlanPlugin'; @@ -69,13 +68,11 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { for (const operation of operations) { const { associatedProject, associatedPhase } = operation; - const policy: InputSnapshotPolicy = recordByOperation.get(operation)?.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(associatedProject); const fileHashes: ReadonlyMap | undefined = - inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name, policy); + inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); if (!fileHashes) { continue; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 8cd9e292cc8..c1c7020f8a9 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -70,11 +70,7 @@ import { setTrustedStateHash } from './RetainedResultVerification'; import { getSnapshotStartTimeMs } from './OperationInputFilesCheck'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import { isBuildCacheReadSkipped, wasExecutedIncrementally } from './IncrementalExecutionState'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; @@ -330,8 +326,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { operation, inputsSnapshot, snapshotStartTimeMs, - projectConfigurations.get(associatedProject), - graph.getInputSnapshotPolicy?.(operation) + projectConfigurations.get(associatedProject) ) }; // Upstream runners may mutate the property of build cache context for downstream runners @@ -395,8 +390,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { operation, inputsSnapshot, snapshotStartTimeMs, - context.projectConfigurations.get(operation.associatedProject), - graph.getInputSnapshotPolicy?.(operation) + context.projectConfigurations.get(operation.associatedProject) ) ); } @@ -1215,21 +1209,18 @@ function readCacheInputs( operation: Operation, inputsSnapshot: IInputsSnapshot, snapshotStartTimeMs: number, - projectConfiguration: RushProjectConfiguration | undefined, - policy: InputSnapshotPolicy = 'legacy' + projectConfiguration: RushProjectConfiguration | undefined ): Pick< IOperationBuildCacheContext, 'getCacheDisabledReason' | 'inputsSnapshot' | 'snapshotStartTimeMs' | 'inputFileHashes' > { const { name: phaseName } = operation.associatedPhase; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); // This value can *currently* be cached per-project, but in the future the list of files will vary // depending on the selected phase. const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( operation.associatedProject, - phaseName, - policy + phaseName ); // Computing the reason checks each tracked file of the project, and an iteration of a long-lived graph diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index 8ae8ddbe6c0..8400f371d59 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -8,7 +8,6 @@ import type { IOperationLastState } from './IOperationRunner'; import type { Operation } from './Operation'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { ILogFilePaths } from './ProjectLogWritable'; -import type { InputSnapshotPolicy } from '../incremental/InputsSnapshot'; /** * Structured components of the state hash for an operation. @@ -34,11 +33,6 @@ export interface IOperationStateHashComponents { * @alpha */ export interface IBaseOperationExecutionResult { - /** - * This operation's frozen graph-owned input policy. Absent on legacy-only hosts. - */ - readonly inputSnapshotPolicy?: InputSnapshotPolicy; - /** * The operation itself */ diff --git a/libraries/rush-lib/src/logic/operations/IOperationGraph.ts b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts index 387c1f98aad..85d73919c4a 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationGraph.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts @@ -7,7 +7,7 @@ import type { Operation } from './Operation'; import type { IExecutionResult, IOperationExecutionResult } from './IOperationExecutionResult'; import type { Parallelism } from './ParseParallelism'; import type { OperationStatus } from './OperationStatus'; -import type { IInputsSnapshot, InputSnapshotPolicy } from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { OperationGraphHooks } from '../../pluginFramework/OperationGraphHooks'; /** @@ -189,18 +189,6 @@ export interface IOperationGraphRequestResult extends IExecutionResult { * @alpha */ export interface IOperationGraph { - /** - * Atomically opts operations into strict input identity and validation. Must be called before the first - * iteration is requested. Unsupported on older hosts; plugins requiring strict inputs must fail if absent. - * Selection is frozen before iteration snapshot acquisition and cannot change for the life of this graph. - */ - setInputSnapshotPolicy?(operations: Iterable, policy: 'strict-v1'): void; - - /** - * Reads this graph's input policy for an operation. Unselected operations use legacy semantics. - */ - getInputSnapshotPolicy?(operation: Operation): InputSnapshotPolicy; - /** * Hooks into the execution process for operations */ diff --git a/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts b/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts index 41bb07ea659..78939acc3b9 100644 --- a/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/IncrementalExecutionGuardPlugin.ts @@ -8,11 +8,7 @@ import * as path from 'node:path'; import { FileSystem, InternalError, Path } from '@rushstack/node-core-library'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { IOperationGraphContext, IPhasedCommandPlugin, @@ -76,7 +72,6 @@ const TOOLING_PACKAGE_NAME_REGEXP: RegExp = * The inputs of an operation that decide whether it may run its incremental command, as of one inputs snapshot. */ export interface IIncrementalInputState { - readonly inputSnapshotPolicy?: InputSnapshotPolicy; /** * The path of the operation's project folder relative to the root of the inputs snapshot, with a trailing `/`, * or an empty string for a project at the root. @@ -126,13 +121,10 @@ export function captureIncrementalInputState( const { operation } = record; const { associatedProject: project, associatedPhase: phase, settings } = operation; const { config } = record.getStateHashComponents(); - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); const inputFiles: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( project, - phase.name, - policy + phase.name ); const projectPrefix: string = getProjectPrefix(inputsSnapshot, project); const inputPathsHash: Hash = createHash('sha1'); @@ -154,7 +146,6 @@ export function captureIncrementalInputState( } return { - ...(policy === 'strict-v1' ? { inputSnapshotPolicy: policy } : {}), projectPrefix, configHash: config, environment: (settings?.dependsOnEnvVars ?? []) @@ -196,9 +187,6 @@ export function getIncrementalInputChangeReason( lastState: IIncrementalInputState, currentState: IIncrementalInputState ): string | undefined { - if (lastState.inputSnapshotPolicy === 'strict-v1' || currentState.inputSnapshotPolicy === 'strict-v1') { - return 'strict input policy does not use legacy incremental-command equivalence'; - } if (lastState.configHash !== currentState.configHash) { return 'its command line changed'; } @@ -390,7 +378,7 @@ function applyToGraph(graph: IOperationGraph): void { // Read before the command starts, so that the next check notices a folder that is recreated while it runs, // e.g. during the first run in a new warm worker, which no check precedes. The check reads them first if its // base has them. - recordState.inputFolders ??= readInputFolderIdentities(record, recordState.inputsSnapshot); + recordState.inputFolders ??= readInputFolderIdentities(record.operation, recordState.inputsSnapshot); }); } @@ -405,11 +393,9 @@ function applyToGraph(graph: IOperationGraph): void { const { runner, associatedProject: project, associatedPhase: phase } = record.operation; if (recordState && record.enabled && runner && !runner.isNoOp && !areInputFilesChecked(record)) { const { inputsSnapshot, snapshotStartTimeMs } = recordState; - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); recordState.inputFilesState = captureInputFilesState( inputsSnapshot.rootDirectory, - inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name, policy).keys(), + inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name).keys(), snapshotStartTimeMs ); } @@ -422,9 +408,6 @@ function applyToGraph(graph: IOperationGraph): void { { outputsMayBeBundles = false }: IIncrementalExecutionGuardOptions = {} ): Promise { const { operation } = record; - if (record.inputSnapshotPolicy === 'strict-v1') { - return 'strict input policy does not use legacy incremental-command equivalence'; - } const cleanOnlyReason: string | undefined = cleanOnlyReasonByOperation.get(operation); if (cleanOnlyReason) { return cleanOnlyReason; @@ -439,7 +422,7 @@ function applyToGraph(graph: IOperationGraph): void { if (base.inputFolders) { // Read before the command starts, so that the next check notices a folder that is recreated while it runs. const inputFolders: ReadonlyMap = readInputFolderIdentities( - record, + operation, recordState.inputsSnapshot ); recordState.inputFolders = inputFolders; @@ -560,10 +543,6 @@ function applyToGraph(graph: IOperationGraph): void { const recordState: IRecordState | undefined = stateByRecord.get(record); stateByRecord.delete(record); clearIncrementalExecutionGuard(record); - if (record.inputSnapshotPolicy === 'strict-v1') { - forgetBase(operation); - return; - } const execution: ICommandExecution | undefined = getCommandExecution(record); if (!execution) { @@ -710,18 +689,13 @@ function formatExamplePaths(paths: string[]): string { * the folder relative to the root of the inputs snapshot. */ function readInputFolderIdentities( - record: IOperationExecutionResult, + operation: Operation, inputsSnapshot: IInputsSnapshot ): ReadonlyMap { - const { operation } = record; - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); const { associatedProject: project, associatedPhase: phase } = operation; const projectPrefix: string = getProjectPrefix(inputsSnapshot, project); const folderPaths: Set = new Set(); - for (const filePath of inputsSnapshot - .getTrackedFileHashesForOperation(project, phase.name, policy) - .keys()) { + for (const filePath of inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name).keys()) { if (filePath.startsWith(projectPrefix) && !path.isAbsolute(filePath)) { folderPaths.add(filePath.slice(0, Math.max(filePath.lastIndexOf('/'), 0))); } diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts index 579726e139c..10d51eb14aa 100644 --- a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -9,11 +9,7 @@ import { PrintUtilities, Colorize, type ITerminal } from '@rushstack/terminal'; import type { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { IOperationGraphIterationOptions } from './IOperationGraph'; import type { IOperationRunnerContext } from './IOperationRunner'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; @@ -142,8 +138,6 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { for (const record of operations.values()) { const { operation } = record; const { runner } = operation; - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); if (!runner) { continue; } @@ -168,7 +162,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { stateMap.set(operation, { packageDepsPath, - allowSkip: policy === 'legacy' && allowSkip, + allowSkip, dependencyChanged: false, inputsUnchanged: false, ...readInputs(record, inputsSnapshot, snapshotStartTimeMs) @@ -221,22 +215,6 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { snapshotStartTimeMs: number ): Pick { const { associatedProject, associatedPhase, runner } = record.operation; - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); - if (policy === 'strict-v1') { - const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( - associatedProject, - associatedPhase.name, - policy - ); - return { - packageDeps: undefined, - inputFilesCheck: - record.enabled && !runner!.isNoOp - ? { inputsSnapshot, snapshotStartTimeMs, fileHashes } - : undefined - }; - } try { const fileHashes: ReadonlyMap = inputsSnapshot.getTrackedFileHashesForOperation( associatedProject, @@ -302,12 +280,6 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { return; } - if (record.inputSnapshotPolicy === 'strict-v1') { - // A strict run must not consume or leave a receipt written using legacy equivalence. - await FileSystem.deleteFileAsync(packageDepsPath); - return; - } - let lastProjectDeps: IProjectDeps | undefined = undefined; try { @@ -426,7 +398,7 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { } } - if (!record.operation.runner!.cacheable || record.inputSnapshotPolicy === 'strict-v1') { + if (!record.operation.runner!.cacheable) { // This operation doesn't support skip detection. return; } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 0ccbb9a422a..3cf1622541d 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -30,11 +30,7 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; import type { IOperationExecutionResult, IOperationStateHashComponents } from './IOperationExecutionResult'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import { OperationError } from './OperationError'; import { RushConstants } from '../RushConstants'; import type { IEnvironment } from '../../utilities/Utilities'; @@ -63,7 +59,6 @@ export interface IOperationExecutionRecordContext { * the environment of the inputs snapshot. The operation's `dependsOnEnvVars` are hashed from it. */ getOperationEnvironment?: (operation: Operation) => Readonly>; - getInputSnapshotPolicy?: (operation: Operation) => InputSnapshotPolicy; invalidate?: (operations: Iterable, reason: string) => void; inputsSnapshot: IInputsSnapshot | undefined; /** @@ -208,7 +203,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera public logFilePaths: ILogFilePaths | undefined; #context: IOperationExecutionRecordContext; - readonly #inputSnapshotPolicy: InputSnapshotPolicy; #collatedWriter: CollatedWriter | undefined = undefined; #status: OperationStatus; @@ -238,7 +232,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera }); this.#context = context; - this.#inputSnapshotPolicy = context.getInputSnapshotPolicy?.(operation) ?? 'legacy'; this.#status = operation.dependencies.size > 0 ? OperationStatus.Waiting : OperationStatus.Ready; this.#stateHashEntry = undefined; } @@ -247,10 +240,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return this.runner.name; } - public get inputSnapshotPolicy(): InputSnapshotPolicy { - return this.#inputSnapshotPolicy; - } - public get iterationId(): number { return this.#context.iterationId; } @@ -477,8 +466,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera #calculateStateHash(): IOperationStateHashCacheEntry { const { inputsSnapshot, stateHashCache } = this.#context; - const policy: InputSnapshotPolicy = this.inputSnapshotPolicy; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); if (!inputsSnapshot) { throw new Error(`Cannot calculate state hash without git.`); @@ -506,8 +493,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera const local: string = inputsSnapshot.getOperationOwnStateHash( associatedProject, associatedPhase.name, - this.#context.getOperationEnvironment?.(this.operation), - policy + this.#context.getOperationEnvironment?.(this.operation) ); // Examples of data in the config hash: diff --git a/libraries/rush-lib/src/logic/operations/OperationGraph.ts b/libraries/rush-lib/src/logic/operations/OperationGraph.ts index 6b36630e0aa..4d624a5aa31 100644 --- a/libraries/rush-lib/src/logic/operations/OperationGraph.ts +++ b/libraries/rush-lib/src/logic/operations/OperationGraph.ts @@ -28,11 +28,7 @@ import type { IExecutionResult, IOperationStateHashComponents } from './IOperationExecutionResult'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { IEnvironment } from '../../utilities/Utilities'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { @@ -223,18 +219,12 @@ class ConfigurableOperation implements IConfigurableOperation { public enabled: boolean; public shouldRunnerPersist: boolean = true; readonly #stateHashEntry: IOperationStateHashCacheEntry; - readonly #record: OperationExecutionRecord; public constructor(record: OperationExecutionRecord, stateHashEntry: IOperationStateHashCacheEntry) { this.operation = record.operation; this.metadataFolderPath = record.metadataFolderPath; this.enabled = !!record.operation.enabled; this.#stateHashEntry = stateHashEntry; - this.#record = record; - } - - public get inputSnapshotPolicy(): InputSnapshotPolicy { - return this.#record.inputSnapshotPolicy; } public getStateHash(): string { @@ -266,8 +256,6 @@ export class OperationGraph implements IOperationGraph { public readonly operations: Set; public readonly abortController: AbortController; readonly #sortedOperations: readonly Operation[]; - readonly #strictInputOperations: Set = new Set(); - #inputSnapshotPolicyFrozen: boolean = false; public resultByOperation: Map; @@ -369,34 +357,6 @@ export class OperationGraph implements IOperationGraph { ); } - public setInputSnapshotPolicy(operations: Iterable, policy: 'strict-v1'): void { - if (policy !== 'strict-v1') { - throw new Error(`Unsupported input snapshot policy: ${policy}`); - } - const requested: Set = new Set(operations); - if (this.#inputSnapshotPolicyFrozen) { - throw new Error('Input snapshot policy cannot change after iteration planning has started.'); - } - for (const operation of requested) { - if (!this.operations.has(operation)) { - throw new Error('Cannot set input snapshot policy for an operation outside this graph.'); - } - } - for (const operation of requested) { - this.#strictInputOperations.add(operation); - } - } - - public readonly getInputSnapshotPolicy = (operation: Operation): InputSnapshotPolicy => { - return this.#strictInputOperations.has(operation) ? 'strict-v1' : 'legacy'; - }; - - #assertSnapshotSupported(snapshot: IInputsSnapshot | undefined): void { - if (this.#strictInputOperations.size > 0) { - assertInputSnapshotPolicySupported(snapshot, 'strict-v1'); - } - } - /** * {@inheritDoc IOperationGraph.setEnabledStates} */ @@ -670,7 +630,6 @@ export class OperationGraph implements IOperationGraph { const { records } = iteration; const { inputsSnapshot } = options; - this.#assertSnapshotSupported(inputsSnapshot); const joiningRecords: Set = collectNeededRecords( records, Array.from(options.neededOperations, (operation: Operation) => getRecord(records, operation)) @@ -687,8 +646,7 @@ export class OperationGraph implements IOperationGraph { inputsSnapshot.getOperationOwnStateHash( record.associatedProject, record.associatedPhase.name, - iteration.getOperationEnvironment?.(record.operation), - this.getInputSnapshotPolicy(record.operation) + iteration.getOperationEnvironment?.(record.operation) ); // An operation that was dispatched before the joining request's inputs changed has no result for it @@ -842,7 +800,6 @@ export class OperationGraph implements IOperationGraph { * @returns A promise which is resolved when all operations have been processed to a final state. */ public async executeAsync(iterationOptions: IOperationGraphIterationOptions): Promise { - this.#inputSnapshotPolicyFrozen = true; await this.abortCurrentIterationAsync(); const scheduled: IExecutionIterationContext | undefined = await this.#scheduleIterationAsync(iterationOptions); @@ -998,7 +955,6 @@ export class OperationGraph implements IOperationGraph { async #scheduleIterationAsync( iterationOptions: IOperationGraphIterationOptions ): Promise { - this.#inputSnapshotPolicyFrozen = true; const getInputsSnapshotAsync: (() => Promise) | undefined = this.#getInputsSnapshotAsync; @@ -1010,7 +966,6 @@ export class OperationGraph implements IOperationGraph { getOperationRequestId, holdUnneededOperations } = iterationOptions; - this.#assertSnapshotSupported(inputsSnapshot); const iterationOptionsForCallbacks: IOperationGraphIterationOptions = { startTime, inputsSnapshot, @@ -1068,7 +1023,6 @@ export class OperationGraph implements IOperationGraph { maxParallelism: this.#maxParallelism, onOperationStateChanged: undefined, createEnvironment: createEnvironmentForOperation, - getInputSnapshotPolicy: this.getInputSnapshotPolicy, getOperationEnvironment, getOperationRequestId, holdUnneededOperations, diff --git a/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts b/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts index 1a54b581d7e..820c93738a4 100644 --- a/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts +++ b/libraries/rush-lib/src/logic/operations/OperationInputFilesCheck.ts @@ -6,11 +6,7 @@ import * as path from 'node:path'; import { Executable, Path } from '@rushstack/node-core-library'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import { - assertInputSnapshotPolicySupported, - type IInputsSnapshot, - type InputSnapshotPolicy -} from '../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import { haveInputFilesChanged, @@ -128,11 +124,9 @@ export async function haveOperationInputFilesChangedAsync( ); }; const { associatedProject: project, associatedPhase: phase } = record.operation; - const policy: InputSnapshotPolicy = record.inputSnapshotPolicy ?? 'legacy'; - assertInputSnapshotPolicySupported(inputsSnapshot, policy); return await haveInputFilesChangedSinceSnapshotAsync({ inputFilesState, - snapshotHashes: inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name, policy), + snapshotHashes: inputsSnapshot.getTrackedFileHashesForOperation(project, phase.name), getGitPath, isNewInput }); diff --git a/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts index 051acfdf273..c4b62f6f1b4 100644 --- a/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/IncrementalExecutionGuardPlugin.test.ts @@ -55,7 +55,7 @@ import * as path from 'node:path'; import { PassThrough } from 'node:stream'; import { LookupByPath } from '@rushstack/lookup-by-path'; -import { Executable, SubprocessTerminator } from '@rushstack/node-core-library'; +import { EnvironmentMap, Executable, SubprocessTerminator } from '@rushstack/node-core-library'; import { type ITerminal, MockWritable, StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import type { IPhase } from '../../../api/CommandLineConfiguration'; @@ -226,22 +226,6 @@ interface ITestWorkspace { const workspaceFolders: string[] = []; -it.each([undefined, 'reported'] as const)( - 'strict policy executes env-only changes without legacy skip or incremental equivalence (%s runner)', - async (pluginRunner) => { - const workspace: ITestWorkspace = await createWorkspaceAsync( - [{ name: 'a', dependsOnEnvVars: ['MODE'], pluginRunner }], - { hasLegacySkipDetection: true } - ); - workspace.graph.setInputSnapshotPolicy(workspace.operations.values(), 'strict-v1'); - expect((await workspace.executeAsync({})).commands).toEqual(['a:initial']); - expect((await workspace.executeAsync({ MODE: '' })).commands).toEqual(['a:initial']); - expect((await workspace.executeAsync({ MODE: 'next' })).commands).toEqual(['a:initial']); - workspace.writeFile('a/src/one.ts', 'one edited'); - expect((await workspace.executeAsync({ MODE: 'next' })).commands).toEqual(['a:initial']); - } -); - afterEach(() => { jest.restoreAllMocks(); for (const folder of workspaceFolders.splice(0)) { @@ -673,7 +657,7 @@ async function createWorkspaceAsync( hasUncommittedChanges: false, lookupByPath, projectMap, - environment: { ...environment }, + environment: new EnvironmentMap(environment), workingTreeReadStartTimeMs: startTimeMs }); }; diff --git a/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts index 905d0735766..0f41034d107 100644 --- a/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/LegacySkipPlugin.test.ts @@ -55,7 +55,7 @@ import type { RushConfigurationProject } from '../../../api/RushConfigurationPro import type { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; import { PhasedCommandHooks, type IOperationGraphContext } from '../../../pluginFramework/PhasedCommandHooks'; -import type { IInputsSnapshot, InputSnapshotPolicy } from '../../incremental/InputsSnapshot'; +import type { IInputsSnapshot } from '../../incremental/InputsSnapshot'; import { OperationBuildCache } from '../../buildCache/OperationBuildCache'; import { CacheableOperationPlugin } from '../CacheableOperationPlugin'; import { LegacySkipInvalidationPlugin, LegacySkipPlugin } from '../LegacySkipPlugin'; @@ -159,8 +159,6 @@ class MockRunner implements IOperationRunner { * Options of a command */ interface ICommandOptions { - readonly strictNames?: ReadonlySet; - readonly strictInputError?: Error; /** * The names of the projects whose operations the command doesn't execute, e.g. because of `--only` or because * the graph of a daemon's engine contains every operation in the repo @@ -281,12 +279,6 @@ async function runCommandAsync( destinations: [new MockWritable()], abortController: new AbortController() }); - if (options.strictNames) { - graph.setInputSnapshotPolicy( - Array.from(options.strictNames, (name) => operations.get(name)!), - 'strict-v1' - ); - } await hooks.onGraphCreatedAsync.promise(graph, { isIncrementalBuildAllowed, projectConfigurations @@ -315,27 +307,14 @@ async function runCommandAsync( ); } const inputsSnapshot: IInputsSnapshot = { - inputSnapshotPolicyVersion: 1, hashes, rootDirectory: folder, hasUncommittedChanges: false, - getTrackedFileHashesForOperation: ( - project: RushConfigurationProject, - operationName?: string, - policy?: InputSnapshotPolicy - ) => { - if (policy === 'strict-v1' && options.strictInputError) { - throw options.strictInputError; - } + getTrackedFileHashesForOperation: (project: RushConfigurationProject) => { const inputFilePath: string = `${project.packageName}/src/index.ts`; return new Map([[inputFilePath, hashes.get(inputFilePath)!]]); }, - getOperationOwnStateHash: ( - project: RushConfigurationProject, - operationName?: string, - environment?: Readonly>, - policy?: InputSnapshotPolicy - ) => `${policy === 'strict-v1' ? 'strict:' : ''}${inputs.get(project.packageName)!}` + getOperationOwnStateHash: (project: RushConfigurationProject) => inputs.get(project.packageName)! }; executions.length = 0; @@ -365,33 +344,6 @@ describe(LegacySkipPlugin.name, () => { }; }); - it('neither consumes nor writes legacy receipts for strict operations', async () => { - expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Success }); - expect(await runCommandAsync(checkout, 'legacy-skip', { strictNames: new Set(['a']) })).toEqual({ - a: OperationStatus.Success - }); - expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Success }); - expect(await runCommandAsync(checkout, 'legacy-skip')).toEqual({ a: OperationStatus.Skipped }); - }); - - it('does not swallow a strict input query failure in the legacy readInputs catch', async () => { - await expect( - runCommandAsync(checkout, 'legacy-skip', { - strictNames: new Set(['a']), - strictInputError: new Error('strict input/output conflict') - }) - ).rejects.toThrow('strict input/output conflict'); - expect(checkout.executions).toEqual([]); - }); - - it('retains strict archive caching without reusing legacy archive identities', async () => { - expect(await runCommandAsync(checkout, 'build-cache')).toEqual({ a: OperationStatus.Success }); - const options: ICommandOptions = { strictNames: new Set(['a']) }; - expect(await runCommandAsync(checkout, 'build-cache', options)).toEqual({ a: OperationStatus.Success }); - expect(await runCommandAsync(checkout, 'build-cache', options)).toEqual({ a: OperationStatus.FromCache }); - expect(checkout.cacheEntries.size).toBe(2); - }); - afterEach(() => { fs.rmSync(checkout.folder, { recursive: true, force: true }); }); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts index a85b3f8820f..92c1bf617bf 100644 --- a/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/OperationGraphStateHash.test.ts @@ -21,22 +21,14 @@ jest.mock('../ProjectLogWritable', () => { }); import { MockWritable } from '@rushstack/terminal'; -import { LookupByPath } from '@rushstack/lookup-by-path'; -import * as path from 'node:path'; import type { IPhase } from '../../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; -import { - InputsSnapshot, - type IInputsSnapshot, - type IRushConfigurationProjectForSnapshot, - type InputSnapshotPolicy -} from '../../incremental/InputsSnapshot'; -import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import type { IInputsSnapshot, IRushConfigurationProjectForSnapshot } from '../../incremental/InputsSnapshot'; import type { IOperationStateHashComponents } from '../IOperationExecutionResult'; import { Operation } from '../Operation'; import { calculateOperationStateHashEntry } from '../OperationExecutionRecord'; -import { OperationGraph, type IOperationGraphOptions } from '../OperationGraph'; +import { OperationGraph } from '../OperationGraph'; import { OperationStatus } from '../OperationStatus'; import { MockOperationRunner } from './MockOperationRunner'; @@ -65,9 +57,9 @@ interface IStateHashes { class StateHashGraph { public readonly iterations: IStateHashes[] = []; public readonly graph: OperationGraph; - public readonly inputsSnapshot: IInputsSnapshot; + readonly #inputsSnapshot: IInputsSnapshot; - public constructor(localHashes: ReadonlyMap, options?: Partial) { + public constructor(localHashes: ReadonlyMap) { const operations: Map = new Map(); for (const name of ['a', 'b', 'c']) { operations.set( @@ -76,11 +68,7 @@ class StateHashGraph { runner: new MockOperationRunner(name), logFilenameIdentifier: name, phase: mockPhase, - project: { - packageName: name, - projectFolder: path.resolve('/repo', name), - projectRelativeFolder: name - } as unknown as RushConfigurationProject + project: { packageName: name } as unknown as RushConfigurationProject }) ); } @@ -92,8 +80,7 @@ class StateHashGraph { parallelism: 1, allowOversubscription: true, destinations: [new MockWritable()], - abortController: new AbortController(), - ...options + abortController: new AbortController() }); this.graph.hooks.configureIteration.tap('test', (records) => { const components: Map = new Map(); @@ -105,26 +92,18 @@ class StateHashGraph { this.iterations.push({ components, hashes }); }); - this.inputsSnapshot = { - inputSnapshotPolicyVersion: 1, + this.#inputsSnapshot = { hashes: new Map(), rootDirectory: '/repo', hasUncommittedChanges: false, getTrackedFileHashesForOperation: () => new Map(), - getOperationOwnStateHash: ( - project: IRushConfigurationProjectForSnapshot, - operationName?: string, - environment?: Readonly>, - policy?: InputSnapshotPolicy - ) => - `${policy === 'strict-v1' ? 'strict:' : ''}${localHashes.get( - (project as RushConfigurationProject).packageName - )!}` + getOperationOwnStateHash: (project: IRushConfigurationProjectForSnapshot) => + localHashes.get((project as RushConfigurationProject).packageName)! }; } public async executeAsync(): Promise { - expect((await this.graph.executeAsync({ inputsSnapshot: this.inputsSnapshot })).status).toBe( + expect((await this.graph.executeAsync({ inputsSnapshot: this.#inputsSnapshot })).status).toBe( OperationStatus.Success ); return this.iterations[this.iterations.length - 1]; @@ -138,195 +117,27 @@ function getReusedNames(previous: IStateHashes, current: IStateHashes): string[] } describe('OperationGraph state hashes', () => { - const localHashes: ReadonlyMap = new Map([ - ['a', 'local-a'], - ['b', 'local-b'], - ['c', 'local-c'] - ]); - - it('registers policy atomically and leaves unselected own hashes in the legacy namespace', async () => { - const legacy: StateHashGraph = new StateHashGraph(localHashes); - const strict: StateHashGraph = new StateHashGraph(localHashes); - const [a, b] = Array.from(strict.graph.operations); - expect(() => - strict.graph.setInputSnapshotPolicy([a, Array.from(legacy.graph.operations)[0]], 'strict-v1') - ).toThrow('outside this graph'); - expect(strict.graph.getInputSnapshotPolicy(a)).toBe('legacy'); - expect(() => - Reflect.apply(strict.graph.setInputSnapshotPolicy, strict.graph, [[a], 'strict-v2']) - ).toThrow('Unsupported'); - expect(strict.graph.getInputSnapshotPolicy(a)).toBe('legacy'); - strict.graph.setInputSnapshotPolicy([a], 'strict-v1'); - strict.graph.setInputSnapshotPolicy([a], 'strict-v1'); - const before: IStateHashes = await legacy.executeAsync(); - const after: IStateHashes = await strict.executeAsync(); - expect(after.components.get('a')!.local).not.toBe(before.components.get('a')!.local); - expect(after.components.get('b')!.local).toBe(before.components.get('b')!.local); - expect(after.hashes.get('b')).not.toBe(before.hashes.get('b')); - expect(strict.graph.resultByOperation.get(a)!.inputSnapshotPolicy).toBe('strict-v1'); - expect(strict.graph.resultByOperation.get(b)!.inputSnapshotPolicy).toBe('legacy'); - }); - - it('freezes registration before asynchronous snapshot acquisition', async () => { - let provideSnapshot!: (snapshot: IInputsSnapshot) => void; - const pending: Promise = new Promise((resolve) => { - provideSnapshot = resolve; - }); - const fixture: StateHashGraph = new StateHashGraph(localHashes, { - getInputsSnapshotAsync: () => pending - }); - fixture.graph.setInputSnapshotPolicy(fixture.graph.operations, 'strict-v1'); - const scheduled: Promise = fixture.graph.scheduleIterationAsync({}); - expect(() => fixture.graph.setInputSnapshotPolicy([], 'strict-v1')).toThrow('cannot change'); - expect(fixture.iterations).toHaveLength(0); - provideSnapshot(fixture.inputsSnapshot); - expect(await scheduled).toBe(true); - expect(fixture.iterations).toHaveLength(1); - fixture.graph.discardScheduledIteration(); - }); - - it.each([false, true])( - 'fails closed before configuration for unsupported snapshots (missing: %s)', - async (missing) => { - const fixture: StateHashGraph = new StateHashGraph(localHashes); - const snapshot: IInputsSnapshot = { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }; - fixture.graph.setInputSnapshotPolicy(fixture.graph.operations, 'strict-v1'); - await expect( - fixture.graph.executeAsync({ inputsSnapshot: missing ? undefined : snapshot }) - ).rejects.toThrow('snapshot supporting'); - expect(fixture.iterations).toHaveLength(0); - expect(() => fixture.graph.setInputSnapshotPolicy([], 'strict-v1')).toThrow('cannot change'); - } - ); - - it('accepts old snapshots for legacy graphs but checks capability again on every strict iteration', async () => { - const fixture: StateHashGraph = new StateHashGraph(localHashes); - const snapshot: IInputsSnapshot = { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }; - await fixture.graph.executeAsync({ inputsSnapshot: snapshot }); - expect(fixture.iterations).toHaveLength(1); - const strict: StateHashGraph = new StateHashGraph(localHashes); - strict.graph.setInputSnapshotPolicy(strict.graph.operations, 'strict-v1'); - await strict.executeAsync(); - await expect(strict.graph.executeAsync({ inputsSnapshot: snapshot })).rejects.toThrow( - 'snapshot supporting' - ); - expect(strict.iterations).toHaveLength(1); - }); - - it('rejects strict additional/output conflicts before configuration or execution preparation', async () => { - const fixture: StateHashGraph = new StateHashGraph(localHashes); - const operations: Operation[] = Array.from(fixture.graph.operations); - const a: Operation = operations[0]; - const projectConfig: Pick = { - operationSettingsByOperationName: new Map([ - ['phase', { operationName: 'phase', outputFolderNames: ['lib'] }] - ]) - }; - const inputsSnapshot: InputsSnapshot = new InputsSnapshot({ - rootDir: path.resolve('/repo'), - hashes: new Map(), - additionalHashes: new Map([['a/lib/sorter.js', 'input']]), - hasUncommittedChanges: false, - lookupByPath: new LookupByPath( - operations.map(({ associatedProject: project }) => [project.projectRelativeFolder, project]) - ), - projectMap: new Map( - operations.map(({ associatedProject: project }) => [ - project, - { - projectConfig: projectConfig as RushProjectConfiguration, - additionalFilesByOperationName: - project === a.associatedProject ? new Map([['phase', new Set(['a/lib/sorter.js'])]]) : undefined - } - ]) - ) - }); - inputsSnapshot.getOperationOwnStateHash(a.associatedProject, 'phase'); - fixture.graph.setInputSnapshotPolicy([a], 'strict-v1'); - const prepare: jest.Mock = jest.fn(); - fixture.graph.hooks.beforeExecuteIterationAsync.tap('test', prepare); - await expect(fixture.graph.executeAsync({ inputsSnapshot })).rejects.toThrow( - 'contains additional input file' - ); - expect(fixture.iterations).toHaveLength(0); - expect(prepare).not.toHaveBeenCalled(); - }); - - it('uses the frozen resolver and capability guard for joining request records', async () => { - const hashes: Map = new Map(localHashes); - const fixture: StateHashGraph = new StateHashGraph(hashes); - const [a, b, c] = Array.from(fixture.graph.operations); - fixture.graph.setInputSnapshotPolicy([b], 'strict-v1'); - let started!: () => void; - const whenStarted: Promise = new Promise((resolve) => { - started = resolve; - }); - let finish!: () => void; - const whenFinished: Promise = new Promise((resolve) => { - finish = resolve; - }); - a.runner = new MockOperationRunner('a', async () => { - started(); - await whenFinished; - return OperationStatus.Success; - }); - let joining: boolean = false; - fixture.graph.hooks.configureIteration.tap('test-policy', (records) => { - expect(records.get(b)!.inputSnapshotPolicy).toBe('strict-v1'); - expect(records.get(c)!.inputSnapshotPolicy).toBe('legacy'); - records.get(b)!.enabled = joining; - records.get(c)!.enabled = false; - }); - const run = fixture.graph.executeAsync({ - inputsSnapshot: fixture.inputsSnapshot, - holdUnneededOperations: true - }); - await whenStarted; - try { - joining = true; - const count: number = fixture.iterations.length; - expect(() => - fixture.graph.tryExtendCurrentIteration({ - inputsSnapshot: { ...fixture.inputsSnapshot, inputSnapshotPolicyVersion: undefined }, - neededOperations: [b] - }) - ).toThrow('snapshot supporting'); - expect(fixture.iterations).toHaveLength(count); - hashes.set('b', 'changed'); - expect( - fixture.graph.tryExtendCurrentIteration({ - inputsSnapshot: fixture.inputsSnapshot, - neededOperations: [b] - }).extended - ).toBe(true); - expect(fixture.iterations.at(-1)!.components.get('b')!.local).toBe('strict:changed'); - } finally { - finish(); - await run; - } - }); - it('reuses the state hash of each operation whose inputs did not change since the previous iteration', async () => { - const changedHashes: Map = new Map([ + const localHashes: Map = new Map([ ['a', 'local-a'], ['b', 'local-b'], ['c', 'local-c'] ]); - const graph: StateHashGraph = new StateHashGraph(changedHashes); + const graph: StateHashGraph = new StateHashGraph(localHashes); const first: IStateHashes = await graph.executeAsync(); const second: IStateHashes = await graph.executeAsync(); expect(getReusedNames(first, second)).toEqual(['a', 'b', 'c']); expect(second.hashes).toEqual(first.hashes); - changedHashes.set('a', 'local-a-edited'); + localHashes.set('a', 'local-a-edited'); const third: IStateHashes = await graph.executeAsync(); expect(getReusedNames(second, third)).toEqual(['c']); expect(third.hashes.get('a')).not.toBe(second.hashes.get('a')); expect(third.hashes.get('b')).not.toBe(second.hashes.get('b')); // A new graph has nothing to reuse, and calculates the same state hashes - const uncached: IStateHashes = await new StateHashGraph(changedHashes).executeAsync(); + const uncached: IStateHashes = await new StateHashGraph(localHashes).executeAsync(); expect(third.hashes).toEqual(uncached.hashes); expect(third.components).toEqual(uncached.components); }); diff --git a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts index 28f922e1754..2408765621b 100644 --- a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts @@ -378,12 +378,12 @@ describe(ProjectChangeAnalyzer.name, () => { reuseOptions ); const snapshot: IInputsSnapshot | undefined = await provider!(); - expect(mockSnapshot.mock.calls[0][0].environment).toEqual(process.env); + expect(mockSnapshot.mock.calls[0][0].environment.toObject()).toEqual(process.env); process.env.PROJECT_CHANGE_ANALYZER_TEST = '1'; const nextSnapshot: IInputsSnapshot | undefined = await provider!(); expect(nextSnapshot).not.toBe(snapshot); - expect(mockSnapshot.mock.calls[1][0].environment).toEqual(process.env); + expect(mockSnapshot.mock.calls[1][0].environment.toObject()).toEqual(process.env); await expect(provider!()).resolves.toBe(nextSnapshot); process.env.PROJECT_CHANGE_ANALYZER_TEST = '2'; @@ -715,10 +715,7 @@ describe(ProjectChangeAnalyzer.name, () => { mockGetRepoChanges.mockReturnValue( new Map([ - [ - 'b/package.json', - { mode: 'modified', newhash: 'newhash-b', oldhash: 'oldhash-b', status: 'M' } - ] + ['b/package.json', { mode: 'modified', newhash: 'newhash-b', oldhash: 'oldhash-b', status: 'M' }] ]) ); const packageJsonByHash: Record = { diff --git a/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts b/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts index 0645e0e7623..0943e7a0e2d 100644 --- a/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectWatcher.test.ts @@ -64,70 +64,23 @@ jest.mock('../operations/ProjectLogWritable', () => { }); import { setTimeout as sleepAsync } from 'node:timers/promises'; -import * as path from 'node:path'; -import { LookupByPath } from '@rushstack/lookup-by-path'; import { AnsiEscape, MockWritable, StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { InputsSnapshot, type IInputsSnapshot } from '../incremental/InputsSnapshot'; -import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; import type { IOperationRunner, IOperationRunnerContext } from '../operations/IOperationRunner'; import type { IOperationExecutionResult } from '../operations/IOperationExecutionResult'; import { Operation } from '../operations/Operation'; import { OperationGraph } from '../operations/OperationGraph'; import { OperationStatus } from '../operations/OperationStatus'; -import { ProjectWatcher, _findOperationWithChangedInputs } from '../ProjectWatcher'; +import { ProjectWatcher } from '../ProjectWatcher'; const DEBOUNCE_MS: number = 10; const REQUESTOR: string = 'test-requestor'; -it('compares watched inputs with the graph policy and fails closed on legacy-only snapshots', () => { - const { operation } = createOperation('a'); - const project: RushConfigurationProject = operation.associatedProject; - const projectConfig: Pick = { - operationSettingsByOperationName: new Map([ - ['phase', { operationName: 'phase', dependsOnEnvVars: ['MODE'] }] - ]) - }; - const snapshot = (environment: Record): InputsSnapshot => - new InputsSnapshot({ - rootDir: path.resolve('/repo'), - hashes: new Map(), - lookupByPath: new LookupByPath([['a', project]]), - hasUncommittedChanges: false, - projectMap: new Map([ - [ - project, - { - projectConfig: projectConfig as RushProjectConfiguration - } - ] - ]), - environment - }); - const before: InputsSnapshot = snapshot({}); - const after: InputsSnapshot = snapshot({ MODE: '' }); - expect(_findOperationWithChangedInputs([operation], before, after)).toBeUndefined(); - const graph: OperationGraph = createWatchSession([operation]).graph; - graph.setInputSnapshotPolicy([operation], 'strict-v1'); - expect(_findOperationWithChangedInputs([operation], before, after, graph.getInputSnapshotPolicy)).toBe( - operation - ); - const unsupported: IInputsSnapshot = { - hashes: new Map(), - rootDirectory: '/repo', - hasUncommittedChanges: false, - getOperationOwnStateHash: () => 'ignored-policy', - getTrackedFileHashesForOperation: () => new Map() - }; - expect(() => - _findOperationWithChangedInputs([operation], before, unsupported, graph.getInputSnapshotPolicy) - ).toThrow('snapshot supporting'); -}); - // The statuses of a result that PhasedOperationPlugin keeps if the operation's inputs did not change. const RETAINED_STATUSES: ReadonlySet = new Set([ OperationStatus.Success, @@ -720,57 +673,6 @@ describe(ProjectWatcher.name, () => { }); describe('inputs that change during an iteration', () => { - it.each([ - ['legacy', 'current'], - ['strict-v1', 'current'], - ['strict-v1', 'superseded'], - ['strict-v1', 'cancelled'] - ] as const)( - 'handles a missing idle-check snapshot for %s operations when the check is %s', - async (policy, checkState) => { - const a: ITestOperation = createOperation('a'); - const session: IWatchSession = createWatchSession([a.operation], { hasInputsSnapshots: true }); - if (policy === 'strict-v1') { - session.graph.setInputSnapshotPolicy([a.operation], policy); - } - session.snapshots.getGraphSnapshotAsync = async () => ({ - ...createInputsSnapshot(session.snapshots.inputVersions), - inputSnapshotPolicyVersion: 1 - }); - let completeCheck!: (snapshot: undefined) => void; - session.snapshots.getWatcherSnapshotAsync = () => - new Promise((resolve) => { - completeCheck = resolve; - }); - await startWatchSessionAsync(session); - expect(session.iterations).toEqual([['a']]); - expect(session.snapshots.watcherSnapshotCount).toBe(1); - - if (checkState === 'superseded') { - session.snapshots.getWatcherSnapshotAsync = session.snapshots.getGraphSnapshotAsync; - editInputs(session, a.operation); - simulateFileChange(); - await settleAsync(session); - expect(session.iterations).toEqual([['a'], ['a']]); - expect(session.snapshots.watcherSnapshotCount).toBe(2); - } else if (checkState === 'cancelled') { - session.abortController.abort(); - } - - completeCheck(undefined); - await settleAsync(session); - if (policy === 'strict-v1' && checkState === 'current') { - expect(session.terminalProvider.getErrorOutput()).toContain( - 'Failed to check for file changes made during the iteration: ' + - 'Strict operation inputs require a snapshot supporting input snapshot policy version 1.' - ); - } else { - expect(session.terminalProvider.getErrorOutput()).toBe(''); - } - expect(session.iterations).toHaveLength(checkState === 'superseded' ? 2 : 1); - } - ); - it('queues an iteration for inputs that changed while an iteration ran', async () => { const a: ITestOperation = createOperation('a'); const b: ITestOperation = createOperation('b');