Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Fix declared-environment cache hashes to distinguish unset and empty values, encode values unambiguously, and use Windows environment-variable name semantics. Operations with declared environment inputs receive new cache keys.",
"type": "patch"
}
],
"packageName": "@microsoft/rush"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@rushstack/node-core-library",
"comment": "Add EnvironmentMap.equals() to compare variable names and values without allocating temporary arrays.",
"type": "minor"
}
],
"packageName": "@rushstack/node-core-library"
}
1 change: 1 addition & 0 deletions common/reviews/api/node-core-library.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ export class EnvironmentMap {
readonly caseSensitive: boolean;
clear(): void;
entries(): IterableIterator<IEnvironmentEntry>;
equals(environmentMap: EnvironmentMap): boolean;
get(name: string): string | undefined;
mergeFrom(environmentMap: EnvironmentMap): void;
mergeFromObject(environmentObject?: Record<string, string | undefined>): void;
Expand Down
17 changes: 17 additions & 0 deletions libraries/node-core-library/src/EnvironmentMap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,23 @@ export class EnvironmentMap {
return this._map.values();
}

/**
* Returns whether the maps have the same case sensitivity, variable names, and values.
* Entry order and, for case-insensitive maps, variable name casing are ignored.
*/
public equals(environmentMap: EnvironmentMap): boolean {
if (this.caseSensitive !== environmentMap.caseSensitive || this._map.size !== environmentMap._map.size) {
return false;
}

for (const [name, entry] of this._map) {
if (environmentMap._map.get(name)?.value !== entry.value) {
return false;
}
}
return true;
}

/**
* Adds each entry from `environmentMap` to this map.
*/
Expand Down
46 changes: 46 additions & 0 deletions libraries/node-core-library/src/test/EnvironmentMap.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,52 @@ import process from 'node:process';
import { EnvironmentMap } from '../EnvironmentMap';

describe(EnvironmentMap.name, () => {
describe(EnvironmentMap.prototype.equals.name, () => {
it('compares values and names independently of insertion order', () => {
const map = new EnvironmentMap({ A: '1', B: '' });
expect(map.equals(new EnvironmentMap({ B: '', A: '1' }))).toBe(true);
expect(map.equals(new EnvironmentMap({ A: '2', B: '' }))).toBe(false);
expect(map.equals(new EnvironmentMap({ A: '1', C: '' }))).toBe(false);
expect(map.equals(new EnvironmentMap({ A: '1' }))).toBe(false);
expect(map.equals(new EnvironmentMap({ A: '1', B: '', C: '3' }))).toBe(false);
});

it('distinguishes empty values from missing variables', () => {
const missing = new EnvironmentMap();
expect(missing.equals(new EnvironmentMap({ A: undefined }))).toBe(true);
expect(missing.equals(new EnvironmentMap({ A: '' }))).toBe(false);
expect(new EnvironmentMap({ A: '' }).equals(missing)).toBe(false);
});

it.each(['win32', 'linux'])('respects %s variable-name semantics', (platform) => {
const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!;
Object.defineProperty(process, 'platform', { ...descriptor, value: platform });
try {
const upper = new EnvironmentMap({ A: '1' });
const lower = new EnvironmentMap({ a: '1' });
expect(upper.equals(lower)).toBe(platform === 'win32');
expect(lower.equals(upper)).toBe(platform === 'win32');
expect(upper.equals(new EnvironmentMap({ a: '2' }))).toBe(false);
} finally {
Object.defineProperty(process, 'platform', descriptor);
}
});

it('does not equate maps with different lookup semantics', () => {
const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!;
try {
Object.defineProperty(process, 'platform', { ...descriptor, value: 'win32' });
const insensitive = new EnvironmentMap({ A: '1' });
Object.defineProperty(process, 'platform', { ...descriptor, value: 'linux' });
const sensitive = new EnvironmentMap({ A: '1' });
expect(insensitive.equals(sensitive)).toBe(false);
expect(sensitive.equals(insensitive)).toBe(false);
} finally {
Object.defineProperty(process, 'platform', descriptor);
}
});
});

test('_sanityCheck() throws', () => {
const map = new EnvironmentMap();
const environmentObject = { A: '123' };
Expand Down
14 changes: 4 additions & 10 deletions libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
Sort,
JsonFile,
Objects,
EnvironmentMap,
type IPackageJson
} from '@rushstack/node-core-library';
import {
Expand Down Expand Up @@ -496,7 +497,7 @@ export class ProjectChangeAnalyzer {
const additionalHashes: Map<string, string> = new Map(
additionalFiles.size > 0 ? await hashFilesAsync(rootDirectory, additionalFiles, gitPath) : []
);
const environment: Record<string, string | undefined> = { ...process.env };
const environment: EnvironmentMap = new EnvironmentMap(process.env);
// The files that each operation depends on accumulate across snapshots
const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap);

Expand All @@ -505,7 +506,7 @@ export class ProjectChangeAnalyzer {
previousSnapshot.repoState === repoState &&
previousSnapshot.operationAdditionalFileCount === operationAdditionalFileCount &&
areMapsEqual(previousSnapshot.additionalHashes, additionalHashes) &&
areEnvironmentsEqual(previousSnapshot.environment, environment)
previousSnapshot.environment.equals(environment)
) {
// Every hash that a new snapshot would compute is the same. The previous snapshot began reading the
// working tree earlier, which only widens the window in which a saved file may be newer than its hash.
Expand Down Expand Up @@ -841,7 +842,7 @@ interface IAdditionalGlob {
*/
interface IReusableInputsSnapshot {
readonly additionalHashes: ReadonlyMap<string, string>;
readonly environment: Readonly<Record<string, string | undefined>>;
readonly environment: EnvironmentMap;
readonly operationAdditionalFileCount: number;
readonly repoState: IDetailedRepoState;
readonly snapshot: InputsSnapshot;
Expand Down Expand Up @@ -896,13 +897,6 @@ function areMapsEqual<TValue>(a: ReadonlyMap<string, TValue>, b: ReadonlyMap<str
return true;
}

function areEnvironmentsEqual(
a: Readonly<Record<string, string | undefined>>,
b: Readonly<Record<string, string | undefined>>
): boolean {
return areMapsEqual(new Map(Object.entries(a)), new Map(Object.entries(b)));
}

async function getAdditionalFilesFromRushProjectConfigurationAsync(
additionalGlobs: IAdditionalGlob[],
rootRelativeLookupByPath: IReadonlyLookupByPath<RushConfigurationProject>,
Expand Down
117 changes: 57 additions & 60 deletions libraries/rush-lib/src/logic/incremental/InputsSnapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { createHash, type Hash } from 'node:crypto';
import ignore, { type Ignore } from 'ignore';

import { type IReadonlyLookupByPath, LookupByPath } from '@rushstack/lookup-by-path';
import { InternalError, Path, Sort } from '@rushstack/node-core-library';
import { EnvironmentMap, InternalError, Path, Sort } from '@rushstack/node-core-library';

import type { RushConfigurationProject } from '../../api/RushConfigurationProject';
import type {
Expand Down Expand Up @@ -90,10 +90,10 @@ export interface IInputsSnapshotParameters {
*/
additionalHashes?: ReadonlyMap<string, string>;
/**
* The environment to use for `dependsOnEnvVars`. By default performs a snapshot of process.env upon construction.
* @defaultValue \{ ...process.env \}
* The environment for `dependsOnEnvVars`. The caller must not modify it after construction.
* @defaultValue new EnvironmentMap(process.env)
*/
environment?: Record<string, string | undefined>;
environment?: EnvironmentMap;
/**
* The Node.js version string to use for `dependsOnNodeVersion`. Defaults to `process.version`.
* @defaultValue process.version
Expand Down Expand Up @@ -251,7 +251,7 @@ export class InputsSnapshot implements IInputsSnapshot {
/**
* The environment to use for `dependsOnEnvVars`.
*/
readonly #environment: Record<string, string | undefined>;
readonly #environment: EnvironmentMap;
/**
* Pre-computed Node.js version strings at each granularity level for `dependsOnNodeVersion`.
*/
Expand All @@ -272,7 +272,7 @@ export class InputsSnapshot implements IInputsSnapshot {
public constructor(params: IInputsSnapshotParameters) {
const {
additionalHashes,
environment = { ...process.env },
environment = new EnvironmentMap(process.env),
globalAdditionalFiles,
hashes,
hasUncommittedChanges,
Expand Down Expand Up @@ -307,18 +307,16 @@ export class InputsSnapshot implements IInputsSnapshot {
const operationAdditionalFileCount: number = countOperationAdditionalFiles(projectMap);

this.#projectMetadataMap =
(previousSnapshot &&
previousSnapshot.#tryDeriveProjectMetadataMap(
params,
environment,
nodeVersion,
globalAdditionalHashes,
operationAdditionalFileCount
)) ??
createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir);
this.#tryDeriveProjectMetadataMap(
previousSnapshot,
params,
environment,
nodeVersion,
globalAdditionalHashes,
operationAdditionalFileCount
) ?? createProjectMetadataMap(hashes, lookupByPath, projectMap, rootDir);
this.#additionalHashes = additionalHashes;
this.#globalAdditionalHashes = globalAdditionalHashes;
// Snapshot the environment so that queries are not impacted by when they happen
this.#environment = environment;
// Parse Node.js version once so it doesn't need to be re-parsed per operation
this.#nodeVersionByGranularity = _parseNodeVersion(nodeVersion);
Expand Down Expand Up @@ -427,15 +425,22 @@ export class InputsSnapshot implements IInputsSnapshot {
const operationSettings: Readonly<IOperationSettings> | undefined = operationName
? record.projectConfig?.operationSettingsByOperationName.get(operationName)
: undefined;
const snapshotEnvironment: Readonly<Record<string, string | undefined>> = this.#environment;
if (
environment &&
operationSettings?.dependsOnEnvVars?.some(
(envVar: string) => (environment[envVar] || '') !== (snapshotEnvironment[envVar] || '')
)
) {
// The operation's environment differs from the snapshot's in a variable that it hashes, so don't memoize.
return this.#computeOperationOwnStateHash(project, operationName, operationSettings, environment);
const snapshotEnvironment: EnvironmentMap = this.#environment;
if (environment && operationSettings?.dependsOnEnvVars?.length) {
const operationEnvironment: EnvironmentMap = new EnvironmentMap(environment);
if (
operationSettings.dependsOnEnvVars.some(
(envVar: string) => operationEnvironment.get(envVar) !== snapshotEnvironment.get(envVar)
)
) {
// The operation's environment differs in a variable that it hashes, so don't memoize.
return this.#computeOperationOwnStateHash(
project,
operationName,
operationSettings,
operationEnvironment
);
}
}

const { hashByOperationName } = record;
Expand All @@ -457,19 +462,21 @@ export class InputsSnapshot implements IInputsSnapshot {
project: IRushConfigurationProjectForSnapshot,
operationName: string | undefined,
operationSettings: Readonly<IOperationSettings> | undefined,
environment: Readonly<Record<string, string | undefined>>
environment: EnvironmentMap
): string {
const hashes: ReadonlyMap<string, string> = this.getTrackedFileHashesForOperation(project, operationName);

const hasher: Hash = createHash('sha1');
// If this is for a specific operation, apply operation-specific options
if (operationSettings) {
const { dependsOnEnvVars, dependsOnNodeVersion, outputFolderNames } = operationSettings;
if (dependsOnEnvVars) {
if (dependsOnEnvVars?.length) {
// Earlier hashes start with hashDelimiter (or are empty). Keep this encoding in a distinct namespace.
hasher.update('env-v2\0');
// As long as we enumerate environment variables in a consistent order, we will get a stable hash.
// Changing the order in rush-project.json will change the hash anyway since the file contents are part of the hash.
for (const envVar of dependsOnEnvVars) {
hasher.update(`${hashDelimiter}$${envVar}=${environment[envVar] || ''}`);
hasher.update(`${hashDelimiter}$${JSON.stringify([envVar, environment.get(envVar)])}`);
}
}

Expand Down Expand Up @@ -506,38 +513,46 @@ export class InputsSnapshot implements IInputsSnapshot {
}

/**
* Returns the state of each project for a later snapshot with the given parameters. It is the state of each
* project in this snapshot whose inputs have the same hashes, and new state for each other project. Returns
* undefined if the later snapshot differs from this one in more than the hashes of files.
* Reuses the state of projects whose input hashes are unchanged. Returns undefined if there is no previous
* snapshot or it differs in more than file hashes.
*/
#tryDeriveProjectMetadataMap(
previousSnapshot: InputsSnapshot | undefined,
params: IInputsSnapshotParameters,
environment: Readonly<Record<string, string | undefined>>,
environment: EnvironmentMap,
nodeVersion: string,
globalAdditionalHashes: ReadonlyMap<string, string> | undefined,
operationAdditionalFileCount: number
): Map<IRushConfigurationProjectForSnapshot, IInternalInputsSnapshotProjectMetadata> | undefined {
const { additionalHashes, hashes, lookupByPath, projectMap, rootDir } = params;
if (
lookupByPath !== this.#lookupByPath ||
projectMap !== this.#projectMap ||
rootDir !== this.rootDirectory ||
nodeVersion !== this.#nodeVersion ||
operationAdditionalFileCount !== this.#operationAdditionalFileCount ||
!areMapsEqual(globalAdditionalHashes ?? EMPTY_MAP, this.#globalAdditionalHashes ?? EMPTY_MAP) ||
!areEnvironmentsEqual(environment, this.#environment)
!previousSnapshot ||
lookupByPath !== previousSnapshot.#lookupByPath ||
projectMap !== previousSnapshot.#projectMap ||
rootDir !== previousSnapshot.rootDirectory ||
nodeVersion !== previousSnapshot.#nodeVersion ||
operationAdditionalFileCount !== previousSnapshot.#operationAdditionalFileCount ||
!areMapsEqual(
globalAdditionalHashes ?? EMPTY_MAP,
previousSnapshot.#globalAdditionalHashes ?? EMPTY_MAP
) ||
!environment.equals(previousSnapshot.#environment)
) {
return undefined;
}

const changedFiles: Set<string> = new Set();
addChangedFiles(this.hashes, hashes, changedFiles);
addChangedFiles(this.#additionalHashes ?? EMPTY_MAP, additionalHashes ?? EMPTY_MAP, changedFiles);
addChangedFiles(previousSnapshot.hashes, hashes, changedFiles);
addChangedFiles(
previousSnapshot.#additionalHashes ?? EMPTY_MAP,
additionalHashes ?? EMPTY_MAP,
changedFiles
);

const previousMetadataMap: ReadonlyMap<
IRushConfigurationProjectForSnapshot,
IInternalInputsSnapshotProjectMetadata
> = this.#projectMetadataMap;
> = previousSnapshot.#projectMetadataMap;
const projectMetadataMap: Map<
IRushConfigurationProjectForSnapshot,
IInternalInputsSnapshotProjectMetadata
Expand Down Expand Up @@ -748,24 +763,6 @@ function areMapsEqual(a: ReadonlyMap<string, string>, b: ReadonlyMap<string, str
return true;
}

function areEnvironmentsEqual(
a: Readonly<Record<string, string | undefined>>,
b: Readonly<Record<string, string | undefined>>
): boolean {
const keys: string[] = Object.keys(a);
if (keys.length !== Object.keys(b).length) {
return false;
}

for (const key of keys) {
if (a[key] !== b[key] || !Object.prototype.hasOwnProperty.call(b, key)) {
return false;
}
}

return true;
}

/**
* Parses a Node.js version string once and returns pre-computed strings for each granularity level.
*
Expand Down
Loading
Loading