diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index c1aa9f00b..eb1ded15c 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -102,6 +102,22 @@ class NodejsNpmInstallAction(NodejsNpmInstallOrUpdateBaseAction): NAME = "NpmInstall" DESCRIPTION = "Installing dependencies from NPM" + def __init__(self, install_dir: str, subprocess_npm: SubprocessNpm, install_links: Optional[bool] = False): + """ + Parameters + ---------- + install_dir : str + Dependencies will be installed in this directory. + subprocess_npm : SubprocessNpm + An instance of the NPM process wrapper + install_links : Optional[bool] + Uses the --install-links npm option if True, by default False. Required when installing into the + source directory, so that local file dependencies are installed as regular dependencies. + """ + + super().__init__(install_dir=install_dir, subprocess_npm=subprocess_npm) + self.install_links = install_links + def execute(self): """ Runs the action. @@ -112,6 +128,8 @@ def execute(self): LOG.debug("NODEJS installing production dependencies in: %s", self.install_dir) command = ["install", "-q", "--no-audit", "--no-save", "--omit=dev"] + if self.install_links: + command.append("--install-links") self.subprocess_npm.run(command, cwd=self.install_dir) except NpmExecutionError as ex: @@ -120,7 +138,11 @@ def execute(self): class NodejsNpmUpdateAction(NodejsNpmInstallOrUpdateBaseAction): """ - A Lambda Builder Action that installs NPM project dependencies + A Lambda Builder Action that installs NPM project dependencies, ignoring any lockfile. + + Only used when building in source for a project that has no lockfile: `--no-package-lock` means + dependency versions are resolved afresh on every build, so a project that does have a lockfile + is installed with NodejsNpmInstallAction instead, to keep builds reproducible. """ NAME = "NpmUpdate" diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 83aab8165..b07642b83 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -25,7 +25,7 @@ NodejsNpmTestAction, NodejsNpmUpdateAction, ) -from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm +from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError, SubprocessNpm from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils LOG = logging.getLogger(__name__) @@ -174,6 +174,7 @@ def _actions_for_cleanup(self): @property def _actions_for_linking_source_dependencies_to_artifacts(self): + # Known gap in a workspaces monorepo - no node_modules beside the function: aws/aws-lambda-builders#933 source_dependencies_path = os.path.join(self.source_dir, "node_modules") artifact_dependencies_path = os.path.join(self.artifacts_dir, "node_modules") return [LinkSinglePathAction(source=source_dependencies_path, dest=artifact_dependencies_path)] @@ -256,16 +257,67 @@ def get_install_action( "Dev dependencies are omitted from the Lambda artifacts package" ) + # `npm ci` stays gated on a lockfile in the source directory itself: run from a workspace package it + # reifies from the lockfile and empties the workspace root's dev dependencies, including build tools + # the build needs. `--omit=dev` prunes the install directory's own dev dependencies on every version. if (osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path)) and npm_ci_option: return NodejsNpmCIAction( install_dir=install_dir, subprocess_npm=subprocess_npm, install_links=is_building_in_source ) if is_building_in_source: + # With a lockfile npm will use, install the locked versions. Without one there is nothing to be + # reproducible about, so keep updating, which also prunes dependencies that were removed from the + # manifest since the last build. + if NodejsNpmWorkflow.get_lockfile_path(install_dir, subprocess_npm, osutils): + return NodejsNpmInstallAction( + install_dir=install_dir, subprocess_npm=subprocess_npm, install_links=True + ) + return NodejsNpmUpdateAction(install_dir=install_dir, subprocess_npm=subprocess_npm) return NodejsNpmInstallAction(install_dir=install_dir, subprocess_npm=subprocess_npm) + @staticmethod + def get_lockfile_path(install_dir: str, subprocess_npm: SubprocessNpm, osutils: OSUtils) -> Optional[str]: + """ + Find the lockfile npm would use when it runs in the given directory. + + The lockfile does not have to sit in that directory: in an npm workspaces monorepo a single + package-lock.json lives at the repository root and covers every workspace package. Which directory that + is, is npm's own decision - `npm prefix` reports it, resolving to the workspace root for a workspace + package and to the directory itself for any other nested package, whose ancestors' lockfiles npm + ignores. Asking npm keeps this answer identical to the one the install will act on, and bounded to the + project. + + Parameters + ---------- + install_dir : str + the directory npm will run in, where dependencies will be installed + subprocess_npm : SubprocessNpm + An instance of the NPM process wrapper + osutils : OSUtils + An instance of OS Utilities for file manipulation + + Returns + ------- + Optional[str] + Path of the lockfile in npm's project root, or None if that project does not have one + """ + try: + project_root = subprocess_npm.run(["prefix"], cwd=install_dir).strip() + except NpmExecutionError as ex: + # without npm's answer there is no evidence a lockfile applies, so install as if there were none + LOG.debug("NODEJS could not resolve the npm project root of %s: %s", install_dir, ex) + return None + + for lockfile_name in ("package-lock.json", "npm-shrinkwrap.json"): + lockfile_path = osutils.joinpath(project_root, lockfile_name) + if osutils.file_exists(lockfile_path): + return lockfile_path + + return None + @staticmethod def can_use_install_links(npm_process: SubprocessNpm) -> bool: """ diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index cc613b60b..c8bbf1990 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -1,4 +1,5 @@ import itertools +import json import logging import os import shutil @@ -11,6 +12,8 @@ from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.supported_runtimes import NODEJS_RUNTIMES +from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils from tests.testing_utils import read_link_without_junction_prefix logger = logging.getLogger("aws_lambda_builders.workflows.nodejs_npm.workflow") @@ -306,6 +309,149 @@ def test_build_in_source_with_download_dependencies(self, runtime): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + @parameterized.expand(SUPPORTED_RUNTIMES) + def test_build_in_source_with_removed_dependencies_and_a_lockfile(self, runtime): + # a project with a lockfile installs the locked versions, and still drops a dependency that was + # removed from the manifest even though the lockfile it reads still lists it + source_dir = os.path.join(self.temp_testdata_dir, "npm-deps-with-lockfile") + lockfile_path = os.path.join(source_dir, "package-lock.json") + with open(lockfile_path, "rb") as lockfile: + original_lockfile = lockfile.read() + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + build_in_source=True, + ) + + source_node_modules = os.path.join(source_dir, "node_modules") + self.assertIn("minimal-request-promise", set(os.listdir(source_node_modules))) + installed_manifest = os.path.join(source_node_modules, "minimal-request-promise", "package.json") + with open(installed_manifest) as manifest: + # the lockfile pins 1.3.0 while the manifest allows ^1.3.0 + self.assertEqual(json.load(manifest)["version"], "1.3.0") + + # the install runs in the developer's own directory, so it must leave their lockfile untouched - + # including the `ms` devDependency entry that `--omit=dev` keeps out of node_modules + with open(lockfile_path, "rb") as lockfile: + self.assertEqual(lockfile.read(), original_lockfile) + + shutil.copy2( + os.path.join(self.temp_testdata_dir, "no-deps", "package.json"), + os.path.join(source_dir, "package.json"), + ) + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + build_in_source=True, + ) + + self.assertNotIn("minimal-request-promise", set(os.listdir(source_node_modules))) + # still untouched with the lockfile now out of date: the manifest no longer lists the dependency + with open(lockfile_path, "rb") as lockfile: + self.assertEqual(lockfile.read(), original_lockfile) + + @parameterized.expand(SUPPORTED_RUNTIMES) + def test_build_in_source_with_a_version_1_lockfile(self, runtime): + # npm 6 wrote lockfileVersion 1 and npm 7+ has to migrate it in memory before it can reify, which + # is a different write path from reifying a version 2 or 3 lockfile directly. The locked versions + # still have to win, and the developer's file still has to come back untouched - in its original + # format, not migrated in place. + source_dir = os.path.join(self.temp_testdata_dir, "npm-deps-with-v1-lockfile") + lockfile_path = os.path.join(source_dir, "package-lock.json") + with open(lockfile_path, "rb") as lockfile: + original_lockfile = lockfile.read() + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + build_in_source=True, + ) + + source_node_modules = os.path.join(source_dir, "node_modules") + installed_manifest = os.path.join(source_node_modules, "minimal-request-promise", "package.json") + with open(installed_manifest) as manifest: + # the lockfile pins 1.3.0 while the manifest allows ^1.3.0, so this version can only come + # from npm having read the version 1 lockfile + self.assertEqual(json.load(manifest)["version"], "1.3.0") + + with open(lockfile_path, "rb") as lockfile: + self.assertEqual(lockfile.read(), original_lockfile) + + @parameterized.expand(SUPPORTED_RUNTIMES) + def test_build_in_source_drops_already_installed_dev_dependencies(self, runtime): + # building in source installs into the developer's own directory, which normally already holds the + # dev dependencies their own `npm install` put there. Those must not reach the artifacts, which for + # this workflow are a symlink to the same node_modules. + source_dir = os.path.join(self.temp_testdata_dir, "npm-deps-with-lockfile") + source_node_modules = os.path.join(source_dir, "node_modules") + + # the developer's own install: the `ms` devDependency is present before the build. Go through + # SubprocessNpm rather than a bare `npm`, since the executable is `npm.cmd` on Windows. + SubprocessNpm(OSUtils()).run(["install", "--silent", "--no-audit", "--no-fund"], cwd=source_dir) + self.assertIn("ms", set(os.listdir(source_node_modules))) + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + build_in_source=True, + ) + + installed = set(os.listdir(source_node_modules)) + self.assertNotIn("ms", installed) + self.assertIn("minimal-request-promise", installed) + self.assertEqual(set(os.listdir(os.path.join(self.artifacts_dir, "node_modules"))), installed) + + @parameterized.expand(SUPPORTED_RUNTIMES) + def test_build_in_source_with_a_local_dependency_and_a_lockfile(self, runtime): + # a lockfile written by a plain `npm install` records a file: dependency as a link entry + # ("resolved": "../npm-deps", "link": true), which is the tree --install-links exists to override. + # Reading a lockfile and passing --install-links used to be mutually exclusive here, because every + # build-in-source install ran with --no-package-lock, so this combination needs pinning: the local + # dependency has to land as a real directory, or the artifacts ship a symlink pointing outside them. + source_dir = os.path.join(self.temp_testdata_dir, "with-local-dependency-and-lockfile") + lockfile_path = os.path.join(source_dir, "package-lock.json") + with open(lockfile_path, "rb") as lockfile: + original_lockfile = lockfile.read() + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + build_in_source=True, + ) + + source_node_modules = os.path.join(source_dir, "node_modules") + local_dependency = os.path.join(source_node_modules, "local-dependency") + # --install-links wins over the lockfile's link entry: a real directory holding the package's files + self.assertFalse(os.path.islink(local_dependency)) + self.assertTrue(os.path.isdir(local_dependency)) + self.assertTrue(os.path.isfile(os.path.join(local_dependency, "included.js"))) + + installed_manifest = os.path.join(source_node_modules, "minimal-request-promise", "package.json") + with open(installed_manifest) as manifest: + # the lockfile pins 1.3.0 while the manifest allows ^1.3.0, so the lockfile was read even + # though --install-links was also in effect + self.assertEqual(json.load(manifest)["version"], "1.3.0") + + with open(lockfile_path, "rb") as lockfile: + self.assertEqual(lockfile.read(), original_lockfile) + @parameterized.expand(SUPPORTED_RUNTIMES) def test_build_in_source_with_removed_dependencies(self, runtime): # run a build with default requirements and confirm dependencies are downloaded diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/included.js b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/included.js new file mode 100644 index 000000000..e8f963aee --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/included.js @@ -0,0 +1,2 @@ +//included +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package-lock.json new file mode 100644 index 000000000..f3ae3b18c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package-lock.json @@ -0,0 +1,32 @@ +{ + "name": "npmdepswithlockfile", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "npmdepswithlockfile", + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "minimal-request-promise": "^1.3.0" + }, + "devDependencies": { + "ms": "^2.1.3" + } + }, + "node_modules/minimal-request-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.3.0.tgz", + "integrity": "sha512-eaD7GFjLCG9glxI1UOXqsKjAUAau9JIMUAG+39sT/MSqJgGP3AJuYjAyEvhgYjSBiK+ROU5cPoaiwx/C6OV2uw==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package.json b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package.json new file mode 100644 index 000000000..7446006cd --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-lockfile/package.json @@ -0,0 +1,18 @@ +{ + "name": "npmdepswithlockfile", + "version": "1.0.0", + "description": "", + "files": [ + "included.js" + ], + "keywords": [], + "author": "", + "license": "APACHE2.0", + "main": "included.js", + "dependencies": { + "minimal-request-promise": "^1.3.0" + }, + "devDependencies": { + "ms": "^2.1.3" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/included.js b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/included.js new file mode 100644 index 000000000..e8f963aee --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/included.js @@ -0,0 +1,2 @@ +//included +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package-lock.json new file mode 100644 index 000000000..63e4cc425 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package-lock.json @@ -0,0 +1,19 @@ +{ + "name": "npmdepswithv1lockfile", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "minimal-request-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.3.0.tgz", + "integrity": "sha512-eaD7GFjLCG9glxI1UOXqsKjAUAau9JIMUAG+39sT/MSqJgGP3AJuYjAyEvhgYjSBiK+ROU5cPoaiwx/C6OV2uw==" + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package.json b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package.json new file mode 100644 index 000000000..08ea1035a --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/npm-deps-with-v1-lockfile/package.json @@ -0,0 +1,18 @@ +{ + "name": "npmdepswithv1lockfile", + "version": "1.0.0", + "description": "", + "files": [ + "included.js" + ], + "keywords": [], + "author": "", + "license": "APACHE2.0", + "main": "included.js", + "dependencies": { + "minimal-request-promise": "^1.3.0" + }, + "devDependencies": { + "ms": "^2.1.3" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/included.js b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/included.js new file mode 100644 index 000000000..70292404f --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/included.js @@ -0,0 +1,5 @@ +//included +const localdep = require('local-dependency'); +exports.handler = async (event, context) => { + return localdep; +}; \ No newline at end of file diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package-lock.json new file mode 100644 index 000000000..bf3b1f06a --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package-lock.json @@ -0,0 +1,35 @@ +{ + "name": "with-local-dependency-and-lockfile", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "with-local-dependency-and-lockfile", + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "local-dependency": "file:../npm-deps", + "minimal-request-promise": "^1.3.0" + } + }, + "../npm-deps": { + "name": "npmdeps", + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "minimal-request-promise": "*" + } + }, + "node_modules/local-dependency": { + "resolved": "../npm-deps", + "link": true + }, + "node_modules/minimal-request-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.3.0.tgz", + "integrity": "sha512-eaD7GFjLCG9glxI1UOXqsKjAUAau9JIMUAG+39sT/MSqJgGP3AJuYjAyEvhgYjSBiK+ROU5cPoaiwx/C6OV2uw==", + "license": "MIT" + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package.json b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package.json new file mode 100644 index 000000000..79d28aa48 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-local-dependency-and-lockfile/package.json @@ -0,0 +1,16 @@ +{ + "name": "with-local-dependency-and-lockfile", + "version": "1.0.0", + "description": "", + "files": [ + "included.js" + ], + "keywords": [], + "author": "", + "license": "APACHE2.0", + "main": "included.js", + "dependencies": { + "local-dependency": "file:../npm-deps", + "minimal-request-promise": "^1.3.0" + } +} diff --git a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py index badff3675..d9d476763 100644 --- a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py +++ b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py @@ -1,6 +1,7 @@ import json import os import shutil +import subprocess import tempfile from pathlib import Path from unittest import TestCase @@ -500,6 +501,62 @@ def test_esbuild_can_build_in_source_with_local_dependency(self, runtime): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + @parameterized.expand(SUPPORTED_RUNTIMES) + def test_esbuild_can_build_in_source_in_workspaces_monorepo_with_locked_versions(self, runtime): + # npm workspaces keep one lockfile at the monorepo root rather than next to each function. This one pins + # minimal-request-promise to 1.3.0 while the function's manifest allows ^1.3.0, so a build that ignored + # the lockfile would silently upgrade the dependency in the developer's own source tree. + monorepo_dir = os.path.join(self.temp_testdata_dir, "workspaces-monorepo") + source_dir = os.path.join(monorepo_dir, "packages", "fn") + lockfile_path = os.path.join(monorepo_dir, "package-lock.json") + with open(lockfile_path, "rb") as lockfile: + original_lockfile = lockfile.read() + + options = {"entry_points": ["included.js"]} + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + options=options, + executable_search_paths=[self.binpath], + build_in_source=True, + ) + + # the locked version is what got installed, and npm hoists it to the monorepo root + installed_manifest = os.path.join(monorepo_dir, "node_modules", "minimal-request-promise", "package.json") + self.assertTrue(os.path.isfile(installed_manifest)) + with open(installed_manifest) as manifest: + self.assertEqual(json.load(manifest)["version"], "1.3.0") + + # the root lockfile records the workspace package as a link entry, which is the tree --install-links + # overrides for a file: dependency. npm exempts workspaces from that, so the link still resolves to + # the developer's own packages/fn rather than a packed snapshot of it. Compare resolved paths rather + # than calling os.path.islink: npm links a workspace with a junction on Windows, which is a directory + # to Python, not a link. + workspace_link = os.path.join(monorepo_dir, "node_modules", "@workspaces-monorepo", "fn") + self.assertEqual(os.path.realpath(workspace_link), os.path.realpath(source_dir)) + + # the install runs inside the workspace package but reifies the root, so the root lockfile is the + # developer file most at risk - it has to come back untouched. This fixture is lockfileVersion 2, + # which npm migrates in memory before reifying; npm-deps-with-lockfile covers version 3. + with open(lockfile_path, "rb") as lockfile: + self.assertEqual(lockfile.read(), original_lockfile) + + # bundle is in artifacts, and it resolved the hoisted dependency: requiring it would raise + # MODULE_NOT_FOUND if esbuild had left the import unbundled + expected_files = {"included.js"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + bundle = os.path.join(self.artifacts_dir, "included.js") + require_bundle = subprocess.run( + ["node", "-e", "require(process.argv[1])", bundle], capture_output=True, text=True + ) + self.assertEqual(require_bundle.returncode, 0, require_bundle.stderr) + @parameterized.expand(SUPPORTED_RUNTIMES) def test_builds_javascript_project_ignoring_relevant_flags(self, runtime): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package-lock.json b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package-lock.json new file mode 100644 index 000000000..1f6fb750a --- /dev/null +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package-lock.json @@ -0,0 +1,47 @@ +{ + "name": "workspaces-monorepo", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "workspaces-monorepo", + "version": "1.0.0", + "license": "APACHE2.0", + "workspaces": [ + "packages/*" + ] + }, + "node_modules/@workspaces-monorepo/fn": { + "resolved": "packages/fn", + "link": true + }, + "node_modules/minimal-request-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.3.0.tgz", + "integrity": "sha512-eaD7GFjLCG9glxI1UOXqsKjAUAau9JIMUAG+39sT/MSqJgGP3AJuYjAyEvhgYjSBiK+ROU5cPoaiwx/C6OV2uw==", + "license": "MIT" + }, + "packages/fn": { + "name": "@workspaces-monorepo/fn", + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "minimal-request-promise": "^1.3.0" + } + } + }, + "dependencies": { + "@workspaces-monorepo/fn": { + "version": "file:packages/fn", + "requires": { + "minimal-request-promise": "^1.3.0" + } + }, + "minimal-request-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.3.0.tgz", + "integrity": "sha512-eaD7GFjLCG9glxI1UOXqsKjAUAau9JIMUAG+39sT/MSqJgGP3AJuYjAyEvhgYjSBiK+ROU5cPoaiwx/C6OV2uw==" + } + } +} diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package.json b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package.json new file mode 100644 index 000000000..d525dc4d0 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/package.json @@ -0,0 +1,7 @@ +{ + "name": "workspaces-monorepo", + "version": "1.0.0", + "private": true, + "license": "APACHE2.0", + "workspaces": ["packages/*"] +} diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/included.js b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/included.js new file mode 100644 index 000000000..545a88dd7 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/included.js @@ -0,0 +1,3 @@ +const request = require('minimal-request-promise'); + +module.exports = () => typeof request.get; diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/package.json b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/package.json new file mode 100644 index 000000000..1333d5fbc --- /dev/null +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/workspaces-monorepo/packages/fn/package.json @@ -0,0 +1,12 @@ +{ + "name": "@workspaces-monorepo/fn", + "version": "1.0.0", + "license": "APACHE2.0", + "files": [ + "included.js" + ], + "main": "included.js", + "dependencies": { + "minimal-request-promise": "^1.3.0" + } +} diff --git a/tests/unit/workflows/nodejs_npm/test_actions.py b/tests/unit/workflows/nodejs_npm/test_actions.py index a63337330..e648f020b 100644 --- a/tests/unit/workflows/nodejs_npm/test_actions.py +++ b/tests/unit/workflows/nodejs_npm/test_actions.py @@ -71,6 +71,19 @@ def test_installs_npm_production_dependencies_for_npm_project(self, SubprocessNp subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") + def test_installs_with_install_links_when_requested(self, SubprocessNpmMock): + subprocess_npm = SubprocessNpmMock.return_value + + action = NodejsNpmInstallAction("source", subprocess_npm=subprocess_npm, install_links=True) + + action.execute() + + # deliberately no --no-package-lock: a project that has a lockfile gets the locked versions installed + expected_args = ["install", "-q", "--no-audit", "--no-save", "--omit=dev", "--install-links"] + + subprocess_npm.run.assert_called_with(expected_args, cwd="source") + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): subprocess_npm = SubprocessNpmMock.return_value diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index 85133dc76..e45846d17 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -1,4 +1,6 @@ import os +import shutil +import tempfile from unittest import TestCase from unittest.mock import ANY, patch, call, Mock @@ -12,6 +14,8 @@ MoveDependenciesAction, ) from aws_lambda_builders.architecture import ARM64 +from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow from aws_lambda_builders.workflows.nodejs_npm.actions import ( NodejsNpmPackAction, @@ -89,11 +93,13 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_without_depende self.assertIsInstance(workflow.actions[6], NodejsNpmrcCleanUpAction) self.assertIsInstance(workflow.actions[7], NodejsNpmLockFileCleanUpAction) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir_external_manifest_and_build_in_source( - self, can_use_links_mock + self, can_use_links_mock, get_lockfile_path_mock ): can_use_links_mock.return_value = True + get_lockfile_path_mock.return_value = os.path.join("not_source", "package-lock.json") self.osutils.dirname.return_value = "not_source" self.osutils.file_exists.return_value = True @@ -111,7 +117,8 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_without_depende self.assertIsInstance(workflow.actions[3], CopySourceAction) self.assertEqual(workflow.actions[3].source_dir, "source") self.assertEqual(workflow.actions[3].dest_dir, "artifacts") - self.assertIsInstance(workflow.actions[4], NodejsNpmUpdateAction) + self.assertIsInstance(workflow.actions[4], NodejsNpmInstallAction) + self.assertTrue(workflow.actions[4].install_links) self.assertEqual(workflow.actions[4].install_dir, "not_source") self.assertIsInstance(workflow.actions[5], NodejsNpmTestAction) self.assertEqual(workflow.actions[5].install_dir, "not_source") @@ -323,9 +330,11 @@ def test_build_in_source_without_download_dependencies_and_without_dependencies_ self.assertIsInstance(workflow.actions[2], CopySourceAction) self.assertIsInstance(workflow.actions[3], NodejsNpmrcCleanUpAction) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") - def test_build_in_source_with_download_dependencies(self, can_use_links_mock): + def test_build_in_source_with_download_dependencies(self, can_use_links_mock, get_lockfile_path_mock): can_use_links_mock.return_value = True + get_lockfile_path_mock.return_value = os.path.join("source", "package-lock.json") source_dir = "source" artifacts_dir = "artifacts" @@ -342,7 +351,8 @@ def test_build_in_source_with_download_dependencies(self, can_use_links_mock): self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcAndLockfileCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) - self.assertIsInstance(workflow.actions[3], NodejsNpmUpdateAction) + self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) + self.assertTrue(workflow.actions[3].install_links) self.assertEqual(workflow.actions[3].install_dir, source_dir) self.assertIsInstance(workflow.actions[4], NodejsNpmTestAction) self.assertEqual(workflow.actions[4].install_dir, source_dir) @@ -351,9 +361,36 @@ def test_build_in_source_with_download_dependencies(self, can_use_links_mock): self.assertEqual(workflow.actions[5]._dest, os.path.join(artifacts_dir, "node_modules")) self.assertIsInstance(workflow.actions[6], NodejsNpmrcCleanUpAction) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") + def test_build_in_source_without_lockfile_keeps_updating_dependencies( + self, can_use_links_mock, get_lockfile_path_mock + ): + # with no lockfile anywhere there are no locked versions to install, and `npm update` additionally + # prunes dependencies that were removed from the manifest since the previous build + can_use_links_mock.return_value = True + get_lockfile_path_mock.return_value = None + + source_dir = "source" + workflow = NodejsNpmWorkflow( + source_dir=source_dir, + artifacts_dir="artifacts", + scratch_dir="scratch_dir", + manifest_path="source/manifest", + osutils=self.osutils, + build_in_source=True, + ) + + self.assertIsInstance(workflow.actions[3], NodejsNpmUpdateAction) + self.assertEqual(workflow.actions[3].install_dir, source_dir) + + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") - def test_build_in_source_with_download_dependencies_and_dependencies_dir(self, can_use_links_mock): + def test_build_in_source_with_download_dependencies_and_dependencies_dir( + self, can_use_links_mock, get_lockfile_path_mock + ): can_use_links_mock.return_value = True + get_lockfile_path_mock.return_value = os.path.join("source", "package-lock.json") source_dir = "source" artifacts_dir = "artifacts" @@ -371,7 +408,8 @@ def test_build_in_source_with_download_dependencies_and_dependencies_dir(self, c self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcAndLockfileCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) - self.assertIsInstance(workflow.actions[3], NodejsNpmUpdateAction) + self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) + self.assertTrue(workflow.actions[3].install_links) self.assertEqual(workflow.actions[3].install_dir, source_dir) self.assertIsInstance(workflow.actions[4], NodejsNpmTestAction) self.assertEqual(workflow.actions[4].install_dir, source_dir) @@ -459,3 +497,121 @@ def test_workflow_revert_build_in_source(self, install_action_mock, install_link build_options=ANY, is_building_in_source=False, ) + + +class TestNodejsNpmWorkflowGetLockfilePath(TestCase): + """ + the lockfile is looked for in the directory `npm prefix` names, so these tests use a real + temporary directory tree with a stubbed npm + """ + + def setUp(self): + self.osutils = OSUtils() + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, True) + self.subprocess_npm = Mock() + + def _touch(self, *path_parts): + file_path = os.path.join(self.tmp_dir, *path_parts) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w") as f: + f.write("{}") + return file_path + + def _npm_prefix_is(self, *path_parts): + # npm prints the project root with a trailing newline + self.subprocess_npm.run.return_value = os.path.join(self.tmp_dir, *path_parts) + os.linesep + + def _lockfile_path(self, install_dir_parts): + return NodejsNpmWorkflow.get_lockfile_path( + os.path.join(self.tmp_dir, *install_dir_parts), self.subprocess_npm, self.osutils + ) + + def _install_action(self, source_dir, install_dir): + return NodejsNpmWorkflow.get_install_action( + source_dir=os.path.join(self.tmp_dir, source_dir), + install_dir=os.path.join(self.tmp_dir, install_dir), + subprocess_npm=self.subprocess_npm, + osutils=self.osutils, + build_options=None, + is_building_in_source=True, + ) + + def test_asks_npm_for_the_project_root(self): + self._touch("fn", "package.json") + self._npm_prefix_is("fn") + + self._lockfile_path(["fn"]) + + self.subprocess_npm.run.assert_called_with(["prefix"], cwd=os.path.join(self.tmp_dir, "fn")) + + def test_finds_lockfile_in_the_project_root(self): + self._touch("fn", "package.json") + lockfile = self._touch("fn", "package-lock.json") + self._npm_prefix_is("fn") + + self.assertEqual(self._lockfile_path(["fn"]), lockfile) + + def test_finds_lockfile_at_the_workspaces_root(self): + # for a workspace package npm reports the monorepo root, where the single lockfile lives + self._touch("package.json") + lockfile = self._touch("package-lock.json") + self._touch("endpoints", "a", "package.json") + self._npm_prefix_is() + + self.assertEqual(self._lockfile_path(["endpoints", "a"]), lockfile) + + def test_finds_shrinkwrap_at_the_workspaces_root(self): + self._touch("package.json") + shrinkwrap = self._touch("npm-shrinkwrap.json") + self._touch("endpoints", "a", "package.json") + self._npm_prefix_is() + + self.assertEqual(self._lockfile_path(["endpoints", "a"]), shrinkwrap) + + def test_ignores_a_lockfile_outside_the_project_root(self): + # a nested package that is not a declared workspace is its own project root, and npm does not read + # an ancestor's lockfile for it + self._touch("package.json") + self._touch("package-lock.json") + self._touch("src", "fn", "package.json") + self._npm_prefix_is("src", "fn") + + self.assertIsNone(self._lockfile_path(["src", "fn"])) + + def test_returns_none_when_the_project_root_has_no_lockfile(self): + self._touch("fn", "package.json") + self._npm_prefix_is("fn") + + self.assertIsNone(self._lockfile_path(["fn"])) + + def test_returns_none_when_npm_cannot_report_the_project_root(self): + self._touch("fn", "package.json") + self._touch("fn", "package-lock.json") + self.subprocess_npm.run.side_effect = NpmExecutionError(message="boom!") + + self.assertIsNone(self._lockfile_path(["fn"])) + + def test_install_action_reads_the_lockfile_of_an_external_manifest_directory(self): + # building in source with an external manifest installs in the manifest directory, which is a sibling of + # the source directory, so that is the directory npm resolves its project root from + self._touch("src", "included.js") + self._touch("manifest", "package.json") + self._touch("manifest", "package-lock.json") + self._npm_prefix_is("manifest") + + action = self._install_action(source_dir="src", install_dir="manifest") + + self.assertIsInstance(action, NodejsNpmInstallAction) + self.assertTrue(action.install_links) + + def test_install_action_ignores_a_lockfile_outside_the_install_dir(self): + # a lockfile beside the source code says nothing about the tree npm installs into + self._touch("src", "package.json") + self._touch("src", "package-lock.json") + self._touch("manifest", "package.json") + self._npm_prefix_is("manifest") + + action = self._install_action(source_dir="src", install_dir="manifest") + + self.assertIsInstance(action, NodejsNpmUpdateAction) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py index 4e41ca30c..4896f93cf 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py @@ -347,9 +347,11 @@ def test_no_download_dependencies_and_no_dependencies_dir_fails(self): download_dependencies=False, ) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") - def test_build_in_source(self, install_links_mock): + def test_build_in_source(self, install_links_mock, get_lockfile_path_mock): install_links_mock.return_value = True + get_lockfile_path_mock.return_value = os.path.join("source", "package-lock.json") source_dir = "source" workflow = NodejsNpmEsbuildWorkflow( @@ -363,11 +365,31 @@ def test_build_in_source(self, install_links_mock): self.assertEqual(len(workflow.actions), 2) - self.assertIsInstance(workflow.actions[0], NodejsNpmUpdateAction) + self.assertIsInstance(workflow.actions[0], NodejsNpmInstallAction) + self.assertTrue(workflow.actions[0].install_links) self.assertEqual(workflow.actions[0].install_dir, source_dir) self.assertIsInstance(workflow.actions[1], EsbuildBundleAction) self.assertEqual(workflow.actions[1]._working_directory, source_dir) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") + def test_build_in_source_without_lockfile(self, install_links_mock, get_lockfile_path_mock): + install_links_mock.return_value = True + get_lockfile_path_mock.return_value = None + + source_dir = "source" + workflow = NodejsNpmEsbuildWorkflow( + source_dir=source_dir, + artifacts_dir="artifacts", + scratch_dir="scratch_dir", + manifest_path="source/manifest", + osutils=self.osutils, + build_in_source=True, + ) + + self.assertIsInstance(workflow.actions[0], NodejsNpmUpdateAction) + self.assertEqual(workflow.actions[0].install_dir, source_dir) + def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir_external_manifest(self): self.osutils.dirname.return_value = "not_source" @@ -389,11 +411,13 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_without_depende self.assertEqual(workflow.actions[2].install_dir, "scratch_dir") self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) + @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.get_lockfile_path") @patch("aws_lambda_builders.workflows.nodejs_npm.workflow.NodejsNpmWorkflow.can_use_install_links") def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir_external_manifest_and_build_in_source( - self, install_links_mock + self, install_links_mock, get_lockfile_path_mock ): install_links_mock.return_value = True + get_lockfile_path_mock.return_value = os.path.join("not_source", "package-lock.json") self.osutils.dirname.return_value = "not_source" @@ -408,7 +432,8 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_without_depende self.assertEqual(len(workflow.actions), 3) - self.assertIsInstance(workflow.actions[0], NodejsNpmUpdateAction) + self.assertIsInstance(workflow.actions[0], NodejsNpmInstallAction) + self.assertTrue(workflow.actions[0].install_links) self.assertEqual(workflow.actions[0].install_dir, "not_source") self.assertIsInstance(workflow.actions[1], LinkSinglePathAction) self.assertEqual(workflow.actions[1]._source, os.path.join("not_source", "node_modules"))