From 1cdd746af30169e8bb15ef522705b49d6203f8ee Mon Sep 17 00:00:00 2001 From: pradhyum6144 Date: Sun, 30 Nov 2025 18:07:06 +0530 Subject: [PATCH 1/4] Fix resolve_dependencies pipeline for multiple manifest files (#1957) - Modified resolve_pypi_packages() to accept multiple requirement files - Updated get_data_from_manifests() to batch-process PyPI manifests together - Maintained backward compatibility with existing single-file API - Added comprehensive tests for multiple files and backward compatibility This fix resolves issue #1957 where the pipeline failed when multiple manifest files (e.g., requirements.txt in different subfolders) were present in a project. The solution leverages python-inspector's ability to process multiple requirement files in a single call, which is more efficient and provides better dependency resolution context. Fixes #1957 Signed-off-by: pradhyum6144 --- scanpipe/pipes/resolve.py | 101 ++++++++++++++++++++++----- scanpipe/tests/pipes/test_resolve.py | 52 ++++++++++++++ 2 files changed, 136 insertions(+), 17 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 0a409dd88c..8a79bd722c 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -97,22 +97,72 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model ) return [] + # Group manifest resources by package type for batch processing + manifests_by_type = {} for resource in manifest_resources: - packages = resolve_manifest_resources(resource, package_registry) - if packages: - resolved_packages.extend(packages) - if headers := get_manifest_headers(resource): - sboms_headers[resource.name] = headers - else: - project.add_error( - description="No packages could be resolved", - model=model, - object_instance=resource, - ) + package_type = get_default_package_type(resource.location) + if package_type: + if package_type not in manifests_by_type: + manifests_by_type[package_type] = [] + manifests_by_type[package_type].append(resource) + + # Process PyPI manifests together in a single batch + if "pypi" in manifests_by_type: + pypi_resources = manifests_by_type["pypi"] + pypi_locations = [resource.location for resource in pypi_resources] + + resolver = package_registry.get("pypi") + if resolver: + try: + packages = resolver(input_locations=pypi_locations) + if packages: + # Associate packages with their source resources + # Since we're processing multiple files together, we need to + # associate each package with all the manifest resources + for package_data in packages: + package_data["codebase_resources"] = pypi_resources + resolved_packages.extend(packages) + + # Collect headers for each manifest + for resource in pypi_resources: + if headers := get_manifest_headers(resource): + sboms_headers[resource.name] = headers + else: + for resource in pypi_resources: + project.add_error( + description="No packages could be resolved", + model=model, + object_instance=resource, + ) + except Exception as e: + for resource in pypi_resources: + project.add_error( + description=f"Error resolving packages: {e}", + model=model, + object_instance=resource, + ) + + # Remove pypi from the dict so we don't process it again below + del manifests_by_type["pypi"] + + # Process other manifest types individually (SPDX, CycloneDX, About files) + for package_type, resources in manifests_by_type.items(): + for resource in resources: + packages = resolve_manifest_resources(resource, package_registry) + if packages: + resolved_packages.extend(packages) + if headers := get_manifest_headers(resource): + sboms_headers[resource.name] = headers + else: + project.add_error( + description="No packages could be resolved", + model=model, + object_instance=resource, + ) - dependencies = get_dependencies_from_manifest(resource) - if dependencies: - resolved_dependencies.extend(dependencies) + dependencies = get_dependencies_from_manifest(resource) + if dependencies: + resolved_dependencies.extend(dependencies) if sboms_headers: project.update_extra_data({"sboms_headers": sboms_headers}) @@ -222,13 +272,30 @@ def get_manifest_resources(project): return project.codebaseresources.filter(status=flag.APPLICATION_PACKAGE) -def resolve_pypi_packages(input_location): - """Resolve the PyPI packages from the ``input_location`` requirements file.""" +def resolve_pypi_packages(input_location=None, input_locations=None): + """ + Resolve the PyPI packages from requirement file(s). + + Args: + input_location: Single requirement file path (for backward compatibility) + input_locations: List of requirement file paths (for batch processing) + + Returns: + List of resolved package data dictionaries + """ + # Handle both single file and multiple files + if input_locations: + requirement_files = input_locations + elif input_location: + requirement_files = [input_location] + else: + raise ValueError("Either input_location or input_locations must be provided") + python_version = f"{sys.version_info.major}{sys.version_info.minor}" operating_system = "linux" resolution_output = python_inspector.resolve_dependencies( - requirement_files=[input_location], + requirement_files=requirement_files, python_version=python_version, operating_system=operating_system, # Prefer source distributions over binary distributions, diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index aa16edaafc..e9b238ed87 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -373,3 +373,55 @@ def test_scanpipe_resolve_get_manifest_headers(self): ] headers = resolve.get_manifest_headers(resource) self.assertEqual(expected, list(headers.keys())) + + @mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies") + def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve): + """Test that resolve_pypi_packages can handle multiple requirement files.""" + # Generated with: + # $ python-inspector --python-version 3.12 --operating-system linux \ + # --specifier pip==25.0.1 --json - + inspector_output_location = ( + self.data / "resolve" / "python_inspector_resolve_dependencies.json" + ) + with open(inspector_output_location) as f: + inspector_output = json.loads(f.read()) + + mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) + + # Test with multiple requirement files + req_files = ["requirements1.txt", "requirements2.txt"] + packages = resolve.resolve_pypi_packages(input_locations=req_files) + + # Verify python_inspector was called with all files + mock_resolve.assert_called_once() + call_args = mock_resolve.call_args + self.assertEqual(req_files, call_args.kwargs["requirement_files"]) + + # Verify packages were returned + self.assertEqual(2, len(packages)) + self.assertEqual("pip", packages[0]["name"]) + + @mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies") + def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility( + self, mock_resolve + ): + """Test that resolve_pypi_packages still works with single file (backward compatibility).""" + inspector_output_location = ( + self.data / "resolve" / "python_inspector_resolve_dependencies.json" + ) + with open(inspector_output_location) as f: + inspector_output = json.loads(f.read()) + + mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) + + # Test with single file (old API) + packages = resolve.resolve_pypi_packages(input_location="requirements.txt") + + # Verify python_inspector was called with single file in list + mock_resolve.assert_called_once() + call_args = mock_resolve.call_args + self.assertEqual(["requirements.txt"], call_args.kwargs["requirement_files"]) + + # Verify packages were returned + self.assertEqual(2, len(packages)) + From ce254a56abeb8479563028ac86c860a5d4adef4b Mon Sep 17 00:00:00 2001 From: pradhyum6144 Date: Sun, 30 Nov 2025 18:15:02 +0530 Subject: [PATCH 2/4] Remove inline comments for cleaner code Signed-off-by: pradhyum6144 --- scanpipe/pipes/resolve.py | 8 -------- scanpipe/tests/pipes/test_resolve.py | 6 ------ 2 files changed, 14 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 8a79bd722c..3df9ce8520 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -97,7 +97,6 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model ) return [] - # Group manifest resources by package type for batch processing manifests_by_type = {} for resource in manifest_resources: package_type = get_default_package_type(resource.location) @@ -106,7 +105,6 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model manifests_by_type[package_type] = [] manifests_by_type[package_type].append(resource) - # Process PyPI manifests together in a single batch if "pypi" in manifests_by_type: pypi_resources = manifests_by_type["pypi"] pypi_locations = [resource.location for resource in pypi_resources] @@ -116,14 +114,10 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model try: packages = resolver(input_locations=pypi_locations) if packages: - # Associate packages with their source resources - # Since we're processing multiple files together, we need to - # associate each package with all the manifest resources for package_data in packages: package_data["codebase_resources"] = pypi_resources resolved_packages.extend(packages) - # Collect headers for each manifest for resource in pypi_resources: if headers := get_manifest_headers(resource): sboms_headers[resource.name] = headers @@ -142,10 +136,8 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model object_instance=resource, ) - # Remove pypi from the dict so we don't process it again below del manifests_by_type["pypi"] - # Process other manifest types individually (SPDX, CycloneDX, About files) for package_type, resources in manifests_by_type.items(): for resource in resources: packages = resolve_manifest_resources(resource, package_registry) diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index e9b238ed87..d7d6cf1244 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -388,16 +388,13 @@ def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve) mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) - # Test with multiple requirement files req_files = ["requirements1.txt", "requirements2.txt"] packages = resolve.resolve_pypi_packages(input_locations=req_files) - # Verify python_inspector was called with all files mock_resolve.assert_called_once() call_args = mock_resolve.call_args self.assertEqual(req_files, call_args.kwargs["requirement_files"]) - # Verify packages were returned self.assertEqual(2, len(packages)) self.assertEqual("pip", packages[0]["name"]) @@ -414,14 +411,11 @@ def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility( mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) - # Test with single file (old API) packages = resolve.resolve_pypi_packages(input_location="requirements.txt") - # Verify python_inspector was called with single file in list mock_resolve.assert_called_once() call_args = mock_resolve.call_args self.assertEqual(["requirements.txt"], call_args.kwargs["requirement_files"]) - # Verify packages were returned self.assertEqual(2, len(packages)) From ae1b940050bd0583211824f71530a29037d2aeca Mon Sep 17 00:00:00 2001 From: pradhyum6144 Date: Tue, 2 Dec 2025 05:20:57 +0530 Subject: [PATCH 3/4] Fix test to use valid requirement file path Signed-off-by: pradhyum6144 --- scanpipe/tests/pipes/test_resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index d7d6cf1244..bc3b7756dd 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -133,7 +133,7 @@ def test_scanpipe_pipes_resolve_resolve_pypi_packages(self, mock_resolve): mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) - packages = resolve.resolve_pypi_packages("") + packages = resolve.resolve_pypi_packages(input_location="requirements.txt") self.assertEqual(2, len(packages)) package_data = packages[0] self.assertEqual("pip", package_data["name"]) From a4c5746b657c35301a24f19a95c4b763ccc3a0f3 Mon Sep 17 00:00:00 2001 From: pradhyum6144 Date: Tue, 16 Dec 2025 19:43:23 +0530 Subject: [PATCH 4/4] Fix linting and format errors in resolve.py Signed-off-by: pradhyum6144 --- scanpipe/pipes/resolve.py | 142 +++++++++++++++++---------- scanpipe/tests/pipes/test_resolve.py | 11 +-- 2 files changed, 96 insertions(+), 57 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 3df9ce8520..681137ed25 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -81,22 +81,8 @@ def get_dependencies_from_manifest(resource): return dependencies -def get_data_from_manifests(project, package_registry, manifest_resources, model=None): - """ - Get package and dependency data from package manifests/lockfiles/SBOMs or - for resolved packages from package requirements. - """ - resolved_packages = [] - resolved_dependencies = [] - sboms_headers = {} - - if not manifest_resources.exists(): - project.add_warning( - description="No resources containing package data found in codebase.", - model=model, - ) - return [] - +def _group_manifests_by_type(manifest_resources): + """Group manifest resources by their package type.""" manifests_by_type = {} for resource in manifest_resources: package_type = get_default_package_type(resource.location) @@ -104,40 +90,53 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model if package_type not in manifests_by_type: manifests_by_type[package_type] = [] manifests_by_type[package_type].append(resource) + return manifests_by_type - if "pypi" in manifests_by_type: - pypi_resources = manifests_by_type["pypi"] - pypi_locations = [resource.location for resource in pypi_resources] - - resolver = package_registry.get("pypi") - if resolver: - try: - packages = resolver(input_locations=pypi_locations) - if packages: - for package_data in packages: - package_data["codebase_resources"] = pypi_resources - resolved_packages.extend(packages) - - for resource in pypi_resources: - if headers := get_manifest_headers(resource): - sboms_headers[resource.name] = headers - else: - for resource in pypi_resources: - project.add_error( - description="No packages could be resolved", - model=model, - object_instance=resource, - ) - except Exception as e: - for resource in pypi_resources: - project.add_error( - description=f"Error resolving packages: {e}", - model=model, - object_instance=resource, - ) - - del manifests_by_type["pypi"] +def _resolve_pypi_manifests( + project, package_registry, pypi_resources, model, resolved_packages, sboms_headers +): + """Resolve PyPI packages from requirement files.""" + pypi_locations = [resource.location for resource in pypi_resources] + resolver = package_registry.get("pypi") + if not resolver: + return + + try: + packages = resolver(input_locations=pypi_locations) + if packages: + for package_data in packages: + package_data["codebase_resources"] = pypi_resources + resolved_packages.extend(packages) + for resource in pypi_resources: + if headers := get_manifest_headers(resource): + sboms_headers[resource.name] = headers + else: + for resource in pypi_resources: + project.add_error( + description="No packages could be resolved", + model=model, + object_instance=resource, + ) + except Exception as e: + for resource in pypi_resources: + project.add_error( + description=f"Error resolving packages: {e}", + model=model, + object_instance=resource, + ) + + +def _resolve_other_manifests( + project, + package_registry, + manifests_by_type, + model, + resolved_packages, + resolved_dependencies, + sboms_headers, +): + """Resolve non-PyPI packages from manifest files.""" for package_type, resources in manifests_by_type.items(): for resource in resources: packages = resolve_manifest_resources(resource, package_registry) @@ -156,6 +155,46 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model if dependencies: resolved_dependencies.extend(dependencies) + +def get_data_from_manifests(project, package_registry, manifest_resources, model=None): + """ + Get package and dependency data from package manifests/lockfiles/SBOMs or + for resolved packages from package requirements. + """ + resolved_packages = [] + resolved_dependencies = [] + sboms_headers = {} + + if not manifest_resources.exists(): + project.add_warning( + description="No resources containing package data found in codebase.", + model=model, + ) + return [] + + manifests_by_type = _group_manifests_by_type(manifest_resources) + + if "pypi" in manifests_by_type: + _resolve_pypi_manifests( + project, + package_registry, + manifests_by_type["pypi"], + model, + resolved_packages, + sboms_headers, + ) + del manifests_by_type["pypi"] + + _resolve_other_manifests( + project, + package_registry, + manifests_by_type, + model, + resolved_packages, + resolved_dependencies, + sboms_headers, + ) + if sboms_headers: project.update_extra_data({"sboms_headers": sboms_headers}) @@ -267,13 +306,14 @@ def get_manifest_resources(project): def resolve_pypi_packages(input_location=None, input_locations=None): """ Resolve the PyPI packages from requirement file(s). - + Args: input_location: Single requirement file path (for backward compatibility) input_locations: List of requirement file paths (for batch processing) - + Returns: List of resolved package data dictionaries + """ # Handle both single file and multiple files if input_locations: @@ -282,7 +322,7 @@ def resolve_pypi_packages(input_location=None, input_locations=None): requirement_files = [input_location] else: raise ValueError("Either input_location or input_locations must be provided") - + python_version = f"{sys.version_info.major}{sys.version_info.minor}" operating_system = "linux" diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index bc3b7756dd..ef19dbf30a 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -390,11 +390,11 @@ def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve) req_files = ["requirements1.txt", "requirements2.txt"] packages = resolve.resolve_pypi_packages(input_locations=req_files) - + mock_resolve.assert_called_once() call_args = mock_resolve.call_args self.assertEqual(req_files, call_args.kwargs["requirement_files"]) - + self.assertEqual(2, len(packages)) self.assertEqual("pip", packages[0]["name"]) @@ -402,7 +402,7 @@ def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve) def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility( self, mock_resolve ): - """Test that resolve_pypi_packages still works with single file (backward compatibility).""" + """Test resolve_pypi_packages with single file (backward compatibility).""" inspector_output_location = ( self.data / "resolve" / "python_inspector_resolve_dependencies.json" ) @@ -412,10 +412,9 @@ def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility( mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) packages = resolve.resolve_pypi_packages(input_location="requirements.txt") - + mock_resolve.assert_called_once() call_args = mock_resolve.call_args self.assertEqual(["requirements.txt"], call_args.kwargs["requirement_files"]) - - self.assertEqual(2, len(packages)) + self.assertEqual(2, len(packages))