diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 0a409dd88c..681137ed25 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -81,6 +81,81 @@ def get_dependencies_from_manifest(resource): return dependencies +def _group_manifests_by_type(manifest_resources): + """Group manifest resources by their package type.""" + manifests_by_type = {} + for resource in manifest_resources: + package_type = get_default_package_type(resource.location) + if package_type: + if package_type not in manifests_by_type: + manifests_by_type[package_type] = [] + manifests_by_type[package_type].append(resource) + return manifests_by_type + + +def _resolve_pypi_manifests( + project, package_registry, pypi_resources, model, resolved_packages, sboms_headers +): + """Resolve PyPI packages from requirement files.""" + pypi_locations = [resource.location for resource in pypi_resources] + resolver = package_registry.get("pypi") + if not resolver: + return + + try: + packages = resolver(input_locations=pypi_locations) + if packages: + for package_data in packages: + package_data["codebase_resources"] = pypi_resources + resolved_packages.extend(packages) + for resource in pypi_resources: + if headers := get_manifest_headers(resource): + sboms_headers[resource.name] = headers + else: + for resource in pypi_resources: + project.add_error( + description="No packages could be resolved", + model=model, + object_instance=resource, + ) + except Exception as e: + for resource in pypi_resources: + project.add_error( + description=f"Error resolving packages: {e}", + model=model, + object_instance=resource, + ) + + +def _resolve_other_manifests( + project, + package_registry, + manifests_by_type, + model, + resolved_packages, + resolved_dependencies, + sboms_headers, +): + """Resolve non-PyPI packages from manifest files.""" + for package_type, resources in manifests_by_type.items(): + for resource in resources: + packages = resolve_manifest_resources(resource, package_registry) + if packages: + resolved_packages.extend(packages) + if headers := get_manifest_headers(resource): + sboms_headers[resource.name] = headers + else: + project.add_error( + description="No packages could be resolved", + model=model, + object_instance=resource, + ) + + dependencies = get_dependencies_from_manifest(resource) + if dependencies: + resolved_dependencies.extend(dependencies) + + def get_data_from_manifests(project, package_registry, manifest_resources, model=None): """ Get package and dependency data from package manifests/lockfiles/SBOMs or @@ -97,22 +172,28 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model ) return [] - for resource in manifest_resources: - packages = resolve_manifest_resources(resource, package_registry) - if packages: - resolved_packages.extend(packages) - if headers := get_manifest_headers(resource): - sboms_headers[resource.name] = headers - else: - project.add_error( - description="No packages could be resolved", - model=model, - object_instance=resource, - ) + manifests_by_type = _group_manifests_by_type(manifest_resources) - dependencies = get_dependencies_from_manifest(resource) - if dependencies: - resolved_dependencies.extend(dependencies) + if "pypi" in manifests_by_type: + _resolve_pypi_manifests( + project, + package_registry, + manifests_by_type["pypi"], + model, + resolved_packages, + sboms_headers, + ) + del manifests_by_type["pypi"] + + _resolve_other_manifests( + project, + package_registry, + manifests_by_type, + model, + resolved_packages, + resolved_dependencies, + sboms_headers, + ) if sboms_headers: project.update_extra_data({"sboms_headers": sboms_headers}) @@ -222,13 +303,31 @@ def get_manifest_resources(project): return project.codebaseresources.filter(status=flag.APPLICATION_PACKAGE) -def resolve_pypi_packages(input_location): - """Resolve the PyPI packages from the ``input_location`` requirements file.""" +def resolve_pypi_packages(input_location=None, input_locations=None): + """ + Resolve the PyPI packages from requirement file(s). + + Args: + input_location: Single requirement file path (for backward compatibility) + input_locations: List of requirement file paths (for batch processing) + + Returns: + List of resolved package data dictionaries + + """ + # Handle both single file and multiple files + if input_locations: + requirement_files = input_locations + elif input_location: + requirement_files = [input_location] + else: + raise ValueError("Either input_location or input_locations must be provided") + python_version = f"{sys.version_info.major}{sys.version_info.minor}" operating_system = "linux" resolution_output = python_inspector.resolve_dependencies( - requirement_files=[input_location], + requirement_files=requirement_files, python_version=python_version, operating_system=operating_system, # Prefer source distributions over binary distributions, diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index aa16edaafc..ef19dbf30a 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -133,7 +133,7 @@ def test_scanpipe_pipes_resolve_resolve_pypi_packages(self, mock_resolve): mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) - packages = resolve.resolve_pypi_packages("") + packages = resolve.resolve_pypi_packages(input_location="requirements.txt") self.assertEqual(2, len(packages)) package_data = packages[0] self.assertEqual("pip", package_data["name"]) @@ -373,3 +373,48 @@ def test_scanpipe_resolve_get_manifest_headers(self): ] headers = resolve.get_manifest_headers(resource) self.assertEqual(expected, list(headers.keys())) + + @mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies") + def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve): + """Test that resolve_pypi_packages can handle multiple requirement files.""" + # Generated with: + # $ python-inspector --python-version 3.12 --operating-system linux \ + # --specifier pip==25.0.1 --json - + inspector_output_location = ( + self.data / "resolve" / "python_inspector_resolve_dependencies.json" + ) + with open(inspector_output_location) as f: + inspector_output = json.loads(f.read()) + + mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) + + req_files = ["requirements1.txt", "requirements2.txt"] + packages = resolve.resolve_pypi_packages(input_locations=req_files) + + mock_resolve.assert_called_once() + call_args = mock_resolve.call_args + self.assertEqual(req_files, call_args.kwargs["requirement_files"]) + + self.assertEqual(2, len(packages)) + self.assertEqual("pip", packages[0]["name"]) + + @mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies") + def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility( + self, mock_resolve + ): + """Test resolve_pypi_packages with single file (backward compatibility).""" + inspector_output_location = ( + self.data / "resolve" / "python_inspector_resolve_dependencies.json" + ) + with open(inspector_output_location) as f: + inspector_output = json.loads(f.read()) + + mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"]) + + packages = resolve.resolve_pypi_packages(input_location="requirements.txt") + + mock_resolve.assert_called_once() + call_args = mock_resolve.call_args + self.assertEqual(["requirements.txt"], call_args.kwargs["requirement_files"]) + + self.assertEqual(2, len(packages))