diff --git a/.trivyignore b/.trivyignore index ced74e1b..557073a5 100644 --- a/.trivyignore +++ b/.trivyignore @@ -52,3 +52,11 @@ CVE-2026-84243 # Reason: Fixed in Go 1.26.11 / 1.27.4. # Status: Temporary ignore - remove once go.mod / CI toolchain is bumped to Go >= 1.26.11 CVE-2026-97054 + +# CVE-2026-128085: OpenTelemetry-Go denial of service via attribute truncation bypass +# Severity: MEDIUM +# Component: go.opentelemetry.io/otel/trace v1.45.0 (indirect dependency) +# Current Version: v1.45.0 (fixed in 1.47.0) +# Reason: Indirect dependency; not yet bumped in go.mod. +# Status: Temporary ignore - remove once go.opentelemetry.io/otel modules are bumped to >= 1.47.0 +CVE-2026-128085 diff --git a/internal/commands/chat-sast.go b/internal/commands/chat-sast.go index cbad0774..8cfbbf7c 100644 --- a/internal/commands/chat-sast.go +++ b/internal/commands/chat-sast.go @@ -178,7 +178,7 @@ func GetTenantConfigurationResponses(tenantWrapper wrappers.TenantConfigurationW func GetTenantConfiguration(tenantConfigurationResponses *[]*wrappers.TenantConfigurationResponse, configKey string) (string, error) { if tenantConfigurationResponses != nil { for _, resp := range *tenantConfigurationResponses { - if resp.Key == configKey { + if resp != nil && resp.Key == configKey { return resp.Value, nil } } diff --git a/internal/commands/file_size_limit_test.go b/internal/commands/file_size_limit_test.go new file mode 100644 index 00000000..5edc82fb --- /dev/null +++ b/internal/commands/file_size_limit_test.go @@ -0,0 +1,175 @@ +//go:build !integration + +package commands + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/checkmarx/ast-cli/internal/filtering" + commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/services/realtimeengine/ossrealtime" + "gotest.tools/assert" +) + +const testSizeLimitBytes = 64 + +// withTestSizeLimit lowers the default filter size limit to testSizeLimitBytes for the duration of the test. +func withTestSizeLimit(t *testing.T) { + t.Helper() + previous := MaxDefaultFilterFileSizeBytes + MaxDefaultFilterFileSizeBytes = testSizeLimitBytes + t.Cleanup(func() { MaxDefaultFilterFileSizeBytes = previous }) +} + +// writeSizedFiles creates each file with the given number of bytes and returns the root with a trailing separator. +func writeSizedFiles(t *testing.T, files map[string]int) string { + t.Helper() + root := t.TempDir() + for rel, size := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + assert.NilError(t, os.WriteFile(p, []byte(strings.Repeat("a", size)), 0o600)) + } + return root + string(os.PathSeparator) +} + +func compressWith(t *testing.T, root, userInclude string, skipDefaultFilter bool) []string { + t.Helper() + zipPath, err := compressFolder(root, "", userInclude, "", filtering.NopMatcher{}, skipDefaultFilter, false, false, + commonParams.DefaultIncludeFilters(true)) + assert.NilError(t, err) + t.Cleanup(func() { _ = os.Remove(zipPath) }) + data, err := os.ReadFile(zipPath) + assert.NilError(t, err) + return zipEntryNames(t, data) +} + +func TestIsScaFile(t *testing.T) { + scaFiles := []string{ + "pom.xml", "package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", + "go.mod", "go.sum", "build.gradle", "build.gradle.kts", "libs.versions.toml", "build.sbt", + "requirements.txt", "requirements-dev.txt", "dev-requirements.txt", "constraints.txt", "packages.txt", + "setup.py", "setup.cfg", "pyproject.toml", "poetry.lock", "App.csproj", "APP.CSPROJ", "packages.config", + "packages.lock.json", "Directory.Packages.props", "Directory.Build.props", "Gemfile", "Gemfile.lock", + "composer.json", "composer.lock", "Podfile", "Podfile.lock", "Lib.podspec", "Lib.podspec.json", "Cartfile", "Cartfile.private", + "Package.swift", "Package@swift-5.9.swift", "pubspec.yaml", "bower.json", "lib.jar", ".cxsca-results.json", "scan.cxsca.sig", + } + for _, name := range scaFiles { + assert.Assert(t, isScaFile(name), "expected an SCA file: %s", name) + } + + otherFiles := []string{"Main.java", "app.js", "data.json", "config.yaml", "notes.txt", "Main.swift", "lib.dll", "deploy.sh", "app.war", "Pipfile"} + for _, name := range otherFiles { + assert.Assert(t, !isScaFile(name), "expected a non-SCA file: %s", name) + } +} + +// Every manifest the OSS realtime scanner accepts must be treated as an SCA file. +func TestScaFilePatterns_CoverOssRealtimeManifests(t *testing.T) { + manifests := []string{ + "pom.xml", "package.json", "bower.json", "Directory.Packages.props", "packages.config", "App.csproj", "go.mod", + "build.gradle", "build.gradle.kts", "libs.versions.toml", "build.sbt", "setup.cfg", "setup.py", "pyproject.toml", + "requirements.txt", "packages.txt", "constraints.txt", "Podfile", "Lib.podspec", "Lib.podspec.json", "Cartfile", + "Cartfile.private", "Gemfile", "composer.json", "pubspec.yaml", "Package.swift", "Package@swift-5.9.swift", + } + for _, name := range manifests { + assert.Assert(t, ossrealtime.IsSupportedManifestFile(name), "not an OSS realtime manifest: %s", name) + assert.Assert(t, isScaFile(name), "OSS realtime manifest not treated as an SCA file: %s", name) + } +} + +// SCA patterns must come from existing code: the OSS realtime manifest list or the default include filter. +func TestScaFilePatterns_ComeFromExistingCode(t *testing.T) { + for _, pattern := range commonParams.ScaFilePatterns { + name := strings.ReplaceAll(pattern, "*", "x") + assert.Assert(t, ossrealtime.IsSupportedManifestFile(name) || filterMatched(commonParams.BaseIncludeFilters, name), + "SCA pattern is neither an OSS realtime manifest nor in the default include filter: %s", pattern) + } +} + +func TestExceedsDefaultFilterSizeLimit(t *testing.T) { + withTestSizeLimit(t) + big := int64(testSizeLimitBytes + 1) + + assert.Assert(t, exceedsDefaultFilterSizeLimit(true, "Main.java", big)) + assert.Assert(t, !exceedsDefaultFilterSizeLimit(true, "Main.java", testSizeLimitBytes), "a file at the limit is kept") + assert.Assert(t, !exceedsDefaultFilterSizeLimit(true, "pom.xml", big), "SCA files are exempt") + assert.Assert(t, !exceedsDefaultFilterSizeLimit(false, "Main.java", big), "no limit with --skip-default-filter") +} + +func TestCompressFolder_SkipsOversizedFilesExceptSca(t *testing.T) { + withTestSizeLimit(t) + big := testSizeLimitBytes + 1 + root := writeSizedFiles(t, map[string]int{ + "src/Main.java": big, + "src/App.java": testSizeLimitBytes, + "deploy.sh": big, + "pom.xml": big, + "package-lock.json": big, + "requirements-dev.txt": big, + "libs/dependency.jar": big, + "web/yarn.lock": big, + "custom.xyz": big, + "go.sum": big, + "ios/Package.resolved": big, + "dotnet/App.csproj": big, + "dotnet/data.json": big, + "dotnet/appsettings.cs": 1, + }) + + assert.DeepEqual(t, compressWith(t, root, "", false), []string{ + "dotnet/App.csproj", "dotnet/appsettings.cs", "go.sum", "libs/dependency.jar", + "package-lock.json", "pom.xml", "requirements-dev.txt", "src/App.java", "web/yarn.lock", + }) + + // Files matched by --file-include are limited too, SCA files stay exempt. + entries := compressWith(t, root, "*.xyz,*.lock", false) + assert.Assert(t, !contains(entries, "custom.xyz")) + assert.Assert(t, !contains(entries, "src/Main.java")) + assert.Assert(t, contains(entries, "web/yarn.lock")) + + // A small file matched by --file-include is kept. + small := writeSizedFiles(t, map[string]int{"custom.xyz": 1}) + assert.Assert(t, contains(compressWith(t, small, "*.xyz", false), "custom.xyz")) + + // --skip-default-filter turns the limit off. + entries = compressWith(t, root, "", true) + assert.Assert(t, contains(entries, "src/Main.java")) + assert.Assert(t, contains(entries, "dotnet/data.json")) + assert.Assert(t, contains(compressWith(t, root, "*.xyz", true), "custom.xyz")) +} + +func TestCompressFolder_AllFilesOversizedAddsPlaceholder(t *testing.T) { + withTestSizeLimit(t) + root := writeSizedFiles(t, map[string]int{"Main.java": testSizeLimitBytes + 1}) + + logs := captureStdLog(t) + + assert.DeepEqual(t, compressWith(t, root, "", false), []string{".container-scan"}) + assert.Equal(t, strings.Count(logs.String(), "Excluded (larger than 100MB)"), 1) + assert.Assert(t, strings.Contains(logs.String(), "Main.java")) +} + +func TestCompressFolder_OversizedFileLoggedOnceWhenOtherFilesIncluded(t *testing.T) { + withTestSizeLimit(t) + root := writeSizedFiles(t, map[string]int{"Big.java": testSizeLimitBytes + 1, "Small.java": 1}) + logs := captureStdLog(t) + + assert.DeepEqual(t, compressWith(t, root, "", false), []string{"Small.java"}) + assert.Equal(t, strings.Count(logs.String(), "Excluded (larger than 100MB)"), 1) + assert.Assert(t, strings.Contains(logs.String(), "Big.java")) +} + +// captureStdLog redirects the standard logger, which logger.Print writes to, for the duration of the test. +func captureStdLog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + return &buf +} diff --git a/internal/commands/linguist_filters_test.go b/internal/commands/linguist_filters_test.go new file mode 100644 index 00000000..2bc37825 --- /dev/null +++ b/internal/commands/linguist_filters_test.go @@ -0,0 +1,97 @@ +//go:build !integration + +package commands + +import ( + "archive/zip" + "bytes" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/checkmarx/ast-cli/internal/filtering" + commonParams "github.com/checkmarx/ast-cli/internal/params" + "gotest.tools/assert" +) + +func TestFilterMatchedMatchesEveryBaseIncludeFilter(t *testing.T) { + for _, f := range commonParams.BaseIncludeFilters { + name := strings.NewReplacer("*", "x", "?", "y").Replace(f) + assert.Assert(t, filterMatched(commonParams.BaseIncludeFilters, name), "file %q should match filter %q", name, f) + assert.Assert(t, filterMatched(commonParams.BaseIncludeFilters, strings.ToUpper(name)), + "file %q should match filter %q case-insensitively", strings.ToUpper(name), f) + } +} + +func TestFilterMatchedLinguistFiles(t *testing.T) { + included := []string{"deploy.sh", "build.PS1", "run.bat", "notebook.ipynb", "tmpl.j2", "Jenkinsfile", "Makefile", "GNUmakefile", + "Containerfile", "Rakefile", ".bash_history", "BUILD.bazel", "Deploy.SH", "JENKINSFILE"} + for _, name := range included { + assert.Assert(t, filterMatched(commonParams.BaseIncludeFilters, name), "%q should be included", name) + } + excluded := []string{"logo.png", "archive.zip", "makefile.bak", "jenkinsfile.txt", "data.bin", "sh"} + for _, name := range excluded { + assert.Assert(t, !filterMatched(commonParams.BaseIncludeFilters, name), "%q should not be included", name) + } +} + +func TestFilterMatchedExcludeWinsOverLinguistInclude(t *testing.T) { + filters := append([]string{"!*.min.sh"}, commonParams.BaseIncludeFilters...) + assert.Assert(t, filterMatched(filters, "deploy.sh")) + assert.Assert(t, !filterMatched(filters, "deploy.min.sh"), "exclude filter must win over Linguist include") + assert.Assert(t, !filterMatched(commonParams.BaseExcludeFilters, ".vscode")) +} + +// writeLinguistTree builds a source tree mixing Linguist files, SAST files and files that must be skipped. +func writeLinguistTree(t *testing.T, files []string) string { + t.Helper() + root := t.TempDir() + for _, rel := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + assert.NilError(t, os.WriteFile(p, []byte("content of "+rel), 0o600)) + } + return root +} + +func zipEntryNames(t *testing.T, data []byte) []string { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + assert.NilError(t, err) + names := make([]string, 0, len(zr.File)) + for _, f := range zr.File { + names = append(names, f.Name) + } + sort.Strings(names) + return names +} + +func TestAddDirFilesIncludesLinguistFiles(t *testing.T) { + root := writeLinguistTree(t, []string{ + "deploy.sh", "Jenkinsfile", "Makefile", "build.ps1", "Main.java", "scripts/run.bat", + "logo.png", "data.bin", "node_modules/dep.sh", ".idea/config.sh", + }) + var out bytes.Buffer + zw := zip.NewWriter(&out) + + err := addDirFiles(zw, "", root+string(os.PathSeparator), commonParams.BaseExcludeFilters, commonParams.BaseIncludeFilters, true, + filtering.NopMatcher{}, false) + assert.NilError(t, err) + assert.NilError(t, zw.Close()) + + assert.DeepEqual(t, []string{"Jenkinsfile", "Main.java", "Makefile", "build.ps1", "deploy.sh", "scripts/run.bat"}, zipEntryNames(t, out.Bytes())) +} + +func TestIsDirEmptyWithLinguistFiles(t *testing.T) { + onlyLinguist := writeLinguistTree(t, []string{"logo.png", "ci/deploy.sh"}) + empty, err := isDirEmpty(onlyLinguist, commonParams.BaseExcludeFilters, commonParams.BaseIncludeFilters, true, filtering.NopMatcher{}) + assert.NilError(t, err) + assert.Assert(t, !empty, "a Linguist-only file must count as scannable content") + + onlyBinary := writeLinguistTree(t, []string{"logo.png", "data.bin", "node_modules/dep.sh"}) + empty, err = isDirEmpty(onlyBinary, commonParams.BaseExcludeFilters, commonParams.BaseIncludeFilters, true, filtering.NopMatcher{}) + assert.NilError(t, err) + assert.Assert(t, empty, "files outside the filters and excluded directories must not count") +} diff --git a/internal/commands/llm_settings.go b/internal/commands/llm_settings.go new file mode 100644 index 00000000..ce872be4 --- /dev/null +++ b/internal/commands/llm_settings.go @@ -0,0 +1,96 @@ +package commands + +import ( + "strings" + + "github.com/checkmarx/ast-cli/internal/logger" + commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/wrappers" +) + +const ( + // allowAiUsageKey is the tenant setting that must be on before the LLM-based scanning settings are checked. + allowAiUsageKey = "scan.config.ai.allowAiUsage" + // sastExtendedAnalysisKey is the LLM-based scanning setting, available at tenant and project level. + sastExtendedAnalysisKey = "scan.config.sast.extendedAnalysis" +) + +// resolveDefaultIncludeFilters returns the default include filter for the source zip. The Linguist patterns are +// added only when Allow AI Usage is on and LLM-based scanning is enabled, with the project setting overriding the +// tenant setting; any failure to read the tenant settings falls back to the SAST patterns so the scan runs as before. +func resolveDefaultIncludeFilters( + tenantWrapper wrappers.TenantConfigurationWrapper, + projectsWrapper wrappers.ProjectsWrapper, + projectID string, +) []string { + enabled := isLlmScanningEnabled(tenantWrapper, projectsWrapper, projectID) + if enabled { + logger.PrintIfVerbose("LLM-based scanning is enabled: adding Linguist patterns to the default include filter.") + } else { + logger.PrintIfVerbose("LLM-based scanning is disabled: using the default SAST include filter only.") + } + return commonParams.DefaultIncludeFilters(enabled) +} + +// isLlmScanningEnabled reports whether LLM-based scanning is enabled for the scan. +func isLlmScanningEnabled(tenantWrapper wrappers.TenantConfigurationWrapper, projectsWrapper wrappers.ProjectsWrapper, projectID string) bool { + tenantConfig, err := GetTenantConfigurationResponses(tenantWrapper) + if err != nil { + logger.PrintfIfVerbose("Failed to read the tenant configuration, LLM-based scanning treated as disabled: %v", err) + return false + } + if !isTenantSettingTrue(tenantConfig, allowAiUsageKey) { + logger.PrintIfVerbose("Allow AI Usage is off at tenant level.") + return false + } + if projectID != "" { + if enabled, ok := projectLlmScanningSetting(projectsWrapper, projectID); ok { + logger.PrintfIfVerbose("LLM-based scanning is %s at project level.", enabledText(enabled)) + return enabled + } + } + enabled := isTenantSettingTrue(tenantConfig, sastExtendedAnalysisKey) + logger.PrintfIfVerbose("LLM-based scanning is %s at tenant level.", enabledText(enabled)) + return enabled +} + +// projectLlmScanningSetting returns the project's LLM-based scanning setting; ok is false when the project does not +// set it or its settings cannot be read. +func projectLlmScanningSetting(projectsWrapper wrappers.ProjectsWrapper, projectID string) (enabled, ok bool) { + projectConfig, errorModel, err := projectsWrapper.GetConfiguration(projectID) + if err != nil || errorModel != nil { + logger.PrintfIfVerbose("Failed to read the project configuration, using the tenant setting: %v %v", err, errorModel) + return false, false + } + value := projectSettingsMap(projectConfig)[sastExtendedAnalysisKey] + if strings.TrimSpace(value) == "" { + return false, false + } + return isTrueValue(value), true +} + +func enabledText(enabled bool) string { + if enabled { + return "enabled" + } + return "disabled" +} + +// isTenantSettingTrue reports whether a tenant setting is on; a missing setting is off. +func isTenantSettingTrue(tenantConfig *[]*wrappers.TenantConfigurationResponse, key string) bool { + value, _ := GetTenantConfiguration(tenantConfig, key) + return isTrueValue(value) +} + +// isTrueValue reports whether a setting value is "true", ignoring case and surrounding spaces. +func isTrueValue(value string) bool { + return strings.EqualFold(strings.TrimSpace(value), "true") +} + +func projectSettingsMap(config []wrappers.ProjectConfiguration) map[string]string { + settings := make(map[string]string, len(config)) + for _, c := range config { + settings[c.Key] = c.Value + } + return settings +} diff --git a/internal/commands/llm_settings_test.go b/internal/commands/llm_settings_test.go new file mode 100644 index 00000000..4da5a74c --- /dev/null +++ b/internal/commands/llm_settings_test.go @@ -0,0 +1,200 @@ +//go:build !integration + +package commands + +import ( + "errors" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/checkmarx/ast-cli/internal/filtering" + commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/checkmarx/ast-cli/internal/wrappers/mock" + "github.com/spf13/cobra" + "gotest.tools/assert" +) + +func tenantSettings(kv ...string) mock.TenantConfigurationMockWrapper { + var config []*wrappers.TenantConfigurationResponse + for i := 0; i+1 < len(kv); i += 2 { + config = append(config, &wrappers.TenantConfigurationResponse{Key: kv[i], Value: kv[i+1]}) + } + return mock.TenantConfigurationMockWrapper{ + CustomGetTenantConfiguration: func() (*[]*wrappers.TenantConfigurationResponse, *wrappers.WebError, error) { + return &config, nil, nil + }, + } +} + +// projectSettingsWrapper serves a fixed project configuration and records whether it was asked for it. +type projectSettingsWrapper struct { + *mock.ProjectsMockWrapper + config []wrappers.ProjectConfiguration + err error + called *bool +} + +func (p projectSettingsWrapper) GetConfiguration(string) ([]wrappers.ProjectConfiguration, *wrappers.ErrorModel, error) { + *p.called = true + return p.config, nil, p.err +} + +func projectSettings(called *bool, kv ...string) projectSettingsWrapper { + var config []wrappers.ProjectConfiguration + for i := 0; i+1 < len(kv); i += 2 { + config = append(config, wrappers.ProjectConfiguration{Key: kv[i], Value: kv[i+1]}) + } + return projectSettingsWrapper{ProjectsMockWrapper: &mock.ProjectsMockWrapper{}, config: config, called: called} +} + +func TestIsLlmScanningEnabled(t *testing.T) { + aiOn := []string{allowAiUsageKey, "true"} + llmOn := []string{sastExtendedAnalysisKey, "true"} + llmOff := []string{sastExtendedAnalysisKey, "false"} + tests := []struct { + name string + tenant []string + project []string + projectID string + expected bool + projectCalled bool + }{ + {name: "AI usage off: tenant and project LLM on ignored", tenant: slices.Concat([]string{allowAiUsageKey, "false"}, llmOn), + project: llmOn, projectID: "p1", expected: false}, + {name: "AI usage missing is off", tenant: llmOn, project: llmOn, projectID: "p1", expected: false}, + {name: "tenant on, project on", tenant: slices.Concat(aiOn, llmOn), project: llmOn, projectID: "p1", expected: true, projectCalled: true}, + {name: "tenant on, project off", tenant: slices.Concat(aiOn, llmOn), project: llmOff, projectID: "p1", expected: false, projectCalled: true}, + {name: "tenant off, project on", tenant: slices.Concat(aiOn, llmOff), project: llmOn, projectID: "p1", expected: true, projectCalled: true}, + {name: "tenant off, project off", tenant: slices.Concat(aiOn, llmOff), project: llmOff, projectID: "p1", expected: false, projectCalled: true}, + {name: "tenant on, project not set", tenant: slices.Concat(aiOn, llmOn), projectID: "p1", expected: true, projectCalled: true}, + {name: "tenant off, project not set", tenant: aiOn, projectID: "p1", expected: false, projectCalled: true}, + {name: "tenant on, no project", tenant: slices.Concat(aiOn, llmOn), project: llmOff, projectID: "", expected: true}, + {name: "values are case-insensitive", tenant: []string{allowAiUsageKey, "TRUE"}, project: []string{sastExtendedAnalysisKey, " True "}, + projectID: "p1", expected: true, projectCalled: true}, + {name: "findings analysis is not checked", tenant: aiOn, + project: []string{"scan.config.sast.findingsAnalysis", "false", sastExtendedAnalysisKey, "true"}, projectID: "p1", expected: true, projectCalled: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + called := false + got := isLlmScanningEnabled(tenantSettings(tt.tenant...), projectSettings(&called, tt.project...), tt.projectID) + assert.Equal(t, tt.expected, got) + assert.Equal(t, tt.projectCalled, called, "project configuration call") + }) + } +} + +func TestIsLlmScanningEnabled_SettingsErrors(t *testing.T) { + failingTenant := mock.TenantConfigurationMockWrapper{ + CustomGetTenantConfiguration: func() (*[]*wrappers.TenantConfigurationResponse, *wrappers.WebError, error) { + return nil, nil, errors.New("tenant unavailable") + }, + } + tenantWebError := mock.TenantConfigurationMockWrapper{ + CustomGetTenantConfiguration: func() (*[]*wrappers.TenantConfigurationResponse, *wrappers.WebError, error) { + return nil, &wrappers.WebError{Message: "bad request"}, nil + }, + } + // Allow AI Usage cannot be confirmed, so the project is not read and the filter stays as it is. + called := false + assert.Assert(t, !isLlmScanningEnabled(failingTenant, projectSettings(&called, sastExtendedAnalysisKey, "true"), "p1")) + assert.Assert(t, !isLlmScanningEnabled(tenantWebError, projectSettings(&called, sastExtendedAnalysisKey, "true"), "p1")) + assert.Assert(t, !called, "project settings must not be read when the tenant call fails") + + // A failed project call uses the tenant setting. + failingProject := projectSettings(&called, sastExtendedAnalysisKey, "false") + failingProject.err = errors.New("project unavailable") + assert.Assert(t, isLlmScanningEnabled(tenantSettings(allowAiUsageKey, "true", sastExtendedAnalysisKey, "true"), failingProject, "p1")) + assert.Assert(t, !isLlmScanningEnabled(tenantSettings(allowAiUsageKey, "true", sastExtendedAnalysisKey, "false"), failingProject, "p1")) +} + +func TestResolveDefaultIncludeFilters(t *testing.T) { + called := false + on := tenantSettings(allowAiUsageKey, "true", sastExtendedAnalysisKey, "true") + assert.DeepEqual(t, resolveDefaultIncludeFilters(on, projectSettings(&called), "p1"), commonParams.BaseIncludeFilters) + + off := tenantSettings(allowAiUsageKey, "false", sastExtendedAnalysisKey, "true") + filters := resolveDefaultIncludeFilters(off, projectSettings(&called), "p1") + assert.DeepEqual(t, filters, commonParams.SastIncludeFilters) + assert.Assert(t, !slices.Contains(filters, "*.sh"), "Linguist patterns must be left out when LLM-based scanning is off") + assert.Assert(t, slices.Contains(filters, "*.java"), "SAST patterns must stay when LLM-based scanning is off") +} + +func writeSourceFiles(t *testing.T, files ...string) string { + t.Helper() + root := t.TempDir() + for _, rel := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + assert.NilError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + assert.NilError(t, os.WriteFile(p, []byte("content of "+rel), 0o600)) + } + return root + string(os.PathSeparator) +} + +func compressedEntries(t *testing.T, root string, defaultIncludeFilters []string) []string { + t.Helper() + zipPath, err := compressFolder(root, "", "", "", filtering.NopMatcher{}, false, false, false, defaultIncludeFilters) + assert.NilError(t, err) + t.Cleanup(func() { _ = os.Remove(zipPath) }) + data, err := os.ReadFile(zipPath) + assert.NilError(t, err) + return zipEntryNames(t, data) +} + +func TestCompressFolder_LinguistFilesFollowLlmSetting(t *testing.T) { + root := writeSourceFiles(t, "Main.java", "deploy.sh", "Jenkinsfile") + + withLinguist := compressedEntries(t, root, commonParams.DefaultIncludeFilters(true)) + assert.DeepEqual(t, []string{"Jenkinsfile", "Main.java", "deploy.sh"}, withLinguist) + + withoutLinguist := compressedEntries(t, root, commonParams.DefaultIncludeFilters(false)) + assert.DeepEqual(t, []string{"Main.java"}, withoutLinguist) +} + +// filterFlagsCommand builds a command with the flags getScanDefaultIncludeFilters reads. +func filterFlagsCommand(t *testing.T, args ...string) *cobra.Command { + t.Helper() + cmd := &cobra.Command{} + cmd.Flags().String(commonParams.SourcesFlag, "", "") + cmd.Flags().Bool(commonParams.SkipDefaultFilterFlag, false, "") + assert.NilError(t, cmd.Flags().Parse(args)) + return cmd +} + +func TestGetScanDefaultIncludeFilters(t *testing.T) { + tests := []struct { + name string + args []string + expected []string + tenantCalled bool + }{ + {name: "LLM settings decide for a local source", args: []string{"--" + commonParams.SourcesFlag, "."}, + expected: commonParams.BaseIncludeFilters, tenantCalled: true}, + {name: "--skip-default-filter uses no default filter", args: []string{"--" + commonParams.SourcesFlag, ".", "--" + commonParams.SkipDefaultFilterFlag}, + expected: nil, tenantCalled: false}, + {name: "git source builds no zip", args: []string{"--" + commonParams.SourcesFlag, "https://github.com/org/repo.git"}, + expected: nil, tenantCalled: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tenantCalled, projectCalled := false, false + llmOn := mock.TenantConfigurationMockWrapper{ + CustomGetTenantConfiguration: func() (*[]*wrappers.TenantConfigurationResponse, *wrappers.WebError, error) { + tenantCalled = true + config := []*wrappers.TenantConfigurationResponse{ + {Key: allowAiUsageKey, Value: "true"}, + {Key: sastExtendedAnalysisKey, Value: "true"}, + } + return &config, nil, nil + }, + } + got := getScanDefaultIncludeFilters(filterFlagsCommand(t, tt.args...), llmOn, projectSettings(&projectCalled), "p1") + assert.DeepEqual(t, tt.expected, got) + assert.Equal(t, tt.tenantCalled, tenantCalled, "tenant configuration call") + assert.Equal(t, tt.tenantCalled, projectCalled, "the project is read first; it does not set the value here, so the tenant decides") + }) + } +} diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 816e3dc7..437eb862 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -47,6 +47,9 @@ import ( "github.com/spf13/viper" ) +// maxDefaultFilterFileSizeMB is the size limit for files picked up by the default include filter. +const maxDefaultFilterFileSizeMB = 100 + const ( failedCreating = "Failed creating a scan" failedGetting = "Failed showing a scan" @@ -179,6 +182,8 @@ var ( kicsErrorCodes = []string{"60", "50", "40", "30", "20"} containerResolver wrappers.ContainerResolverWrapper MaxSizeBytes int64 = 5 * 1024 * 1024 * 1024 // 5 GB in bytes + // MaxDefaultFilterFileSizeBytes caps files picked up by the default include filter; larger ones are left out of the zip. + MaxDefaultFilterFileSizeBytes int64 = maxDefaultFilterFileSizeMB * mbBytes ) func NewScanCommand( @@ -1666,7 +1671,15 @@ func scanTypeEnabled(scanType string) bool { return false } -func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, antMatcher filtering.Matcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder bool) (string, error) { +// compressFolder zips sourceDir. defaultIncludeFilters is the default include filter in effect: the SAST patterns, +// plus the Linguist patterns when LLM-based scanning is enabled. Files matched by the default filter or by +// --file-include are subject to the size limit, except SCA files. +func compressFolder( + sourceDir, filter, userIncludeFilter, scaResolver string, + antMatcher filtering.Matcher, + skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder bool, + defaultIncludeFilters []string, +) (string, error) { scaToolPath := scaResolver outputFile, err := os.CreateTemp(os.TempDir(), "cx-*.zip") if err != nil { @@ -1676,10 +1689,13 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an zipWriter := zip.NewWriter(outputFile) excludeFilters := getExcludeFilters(filter, skipDefaultFilter) - includeFilters := getIncludeFilters(userIncludeFilter, skipDefaultFilter) + includeFilters := getIncludeFilters(userIncludeFilter, skipDefaultFilter, defaultIncludeFilters) + + // The size limit applies to files matched by the default filter or --file-include, unless --skip-default-filter is set. + applySizeLimit := !skipDefaultFilter // First check if the directory is empty or all files are filtered out - isEmpty, err := isDirEmpty(sourceDir, excludeFilters, includeFilters, antMatcher) + isEmpty, err := isDirEmpty(sourceDir, excludeFilters, includeFilters, applySizeLimit, antMatcher) if err != nil { return "", err } @@ -1697,7 +1713,7 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an } } else { // Add directory files normally - err = addDirFiles(zipWriter, "", sourceDir, excludeFilters, includeFilters, antMatcher, excludeGitFolder) + err = addDirFiles(zipWriter, "", sourceDir, excludeFilters, includeFilters, applySizeLimit, antMatcher, excludeGitFolder) if err != nil { return "", err } @@ -1735,9 +1751,15 @@ func isSingleContainerScanTriggered() bool { return len(scanTypeList) == 1 && scanTypeList[0] == commonParams.ContainersType } -// isDirEmpty checks if a directory is empty or if all files are filtered out -func isDirEmpty(dir string, excludeFilters, includeFilters []string, antMatcher filtering.Matcher) (bool, error) { +// isDirEmpty checks if a directory is empty or if all files are filtered out. When the only matching files are over +// the size limit, they are logged here, because addDirFiles (which logs them otherwise) does not run for an empty folder. +func isDirEmpty(dir string, excludeFilters, includeFilters []string, applySizeLimit bool, antMatcher filtering.Matcher) (bool, error) { empty := true + type oversizedFile struct { + path string + size int64 + } + var oversized []oversizedFile err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -1756,25 +1778,18 @@ func isDirEmpty(dir string, excludeFilters, includeFilters []string, antMatcher } relPathSlash := filepath.ToSlash(relPath) - // Prune excluded directories rather than descending into them. if info.IsDir() { - legacyFiltered, fErr := isDirFiltered(info.Name(), excludeFilters) - if fErr != nil { - return fErr - } - if legacyFiltered { - return filepath.SkipDir - } - if antMatcher.Excluded(relPathSlash, true) && !antMatcher.MustDescend(relPathSlash) { - return filepath.SkipDir - } - return nil + return pruneExcludedDir(info.Name(), relPathSlash, excludeFilters, antMatcher) } // Check if file passes filters filename := filepath.Base(relPath) if filterMatched(includeFilters, filename) && filterMatched(excludeFilters, filename) && !antMatcher.Excluded(relPathSlash, false) { + if size := targetFileSize(path, info); exceedsDefaultFilterSizeLimit(applySizeLimit, filename, size) { + oversized = append(oversized, oversizedFile{path: path, size: size}) + return nil + } empty = false return filepath.SkipAll // We found at least one file that will be included } @@ -1782,15 +1797,67 @@ func isDirEmpty(dir string, excludeFilters, includeFilters []string, antMatcher return nil }) + if err == nil && empty { + for _, f := range oversized { + logSizeLimitExclusion(f.path, f.size) + } + } return empty, err } -func getIncludeFilters(userIncludeFilter string, skipDefaultFilter bool) []string { +// pruneExcludedDir returns filepath.SkipDir for an excluded directory so the walk does not descend into it. +func pruneExcludedDir(name, relPathSlash string, excludeFilters []string, antMatcher filtering.Matcher) error { + legacyFiltered, err := isDirFiltered(name, excludeFilters) + if err != nil { + return err + } + if legacyFiltered { + return filepath.SkipDir + } + if antMatcher.Excluded(relPathSlash, true) && !antMatcher.MustDescend(relPathSlash) { + return filepath.SkipDir + } + return nil +} + +// logSizeLimitExclusion reports a file left out of the zip because it is over the default filter size limit. +func logSizeLimitExclusion(fileName string, size int64) { + logger.Printf("Excluded (larger than %dMB): %s (%.2fMB)", maxDefaultFilterFileSizeMB, fileName, float64(size)/mbBytes) +} + +func getIncludeFilters(userIncludeFilter string, skipDefaultFilter bool, defaultIncludeFilters []string) []string { if skipDefaultFilter { logger.PrintIfVerbose("--skip-default-filter set: skipping default base include file filter.") return buildFilters([]string{}, userIncludeFilter) } - return buildFilters(commonParams.BaseIncludeFilters, userIncludeFilter) + return buildFilters(defaultIncludeFilters, userIncludeFilter) +} + +// exceedsDefaultFilterSizeLimit reports whether a file is too large for the source zip. Callers only use it for files +// already matched by the include filter (default filter plus --file-include); applySizeLimit is false with +// --skip-default-filter. SCA files are exempt. The size is compared first so the SCA check only runs for the rare +// oversized file. +func exceedsDefaultFilterSizeLimit(applySizeLimit bool, fileName string, size int64) bool { + return applySizeLimit && size > MaxDefaultFilterFileSizeBytes && !isScaFile(fileName) +} + +// isScaFile reports whether the file belongs to the SCA (OSS) engine: a manifest, lock file, SCA Resolver output +// or packaged binary. +func isScaFile(fileName string) bool { + return filterMatched(commonParams.ScaFilePatterns, fileName) +} + +// targetFileSize returns the size of the file, following a symlink to its target. A size that cannot be read is +// reported as 0 so the file is handled as it was before the size limit (e.g. a dangling symlink). +func targetFileSize(filePath string, info fs.FileInfo) int64 { + if info.Mode()&fs.ModeSymlink == 0 { + return info.Size() + } + target, err := os.Stat(filePath) + if err != nil { + return 0 + } + return target.Size() } func getExcludeFilters(userExcludeFilter string, skipDefaultFilter bool) []string { @@ -1835,7 +1902,14 @@ func addDirFilesIgnoreFilter(zipWriter *zip.Writer, baseDir, parentDir string) e return nil } -func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, includeFilters []string, antMatcher filtering.Matcher, excludeGitFolder bool) error { +func addDirFiles( + zipWriter *zip.Writer, + baseDir, parentDir string, + filters, includeFilters []string, + applySizeLimit bool, + antMatcher filtering.Matcher, + excludeGitFolder bool, +) error { fileEntries, err := os.ReadDir(parentDir) if err != nil { return err @@ -1848,9 +1922,9 @@ func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, incl } if util.IsDirOrSymLinkToDir(parentDir, fileInfo) { - err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher, excludeGitFolder) + err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, applySizeLimit, fileInfo, antMatcher, excludeGitFolder) } else { - err = handleFile(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher) + err = handleFile(zipWriter, baseDir, parentDir, filters, includeFilters, applySizeLimit, fileInfo, antMatcher) } if err != nil { @@ -1866,6 +1940,7 @@ func handleFile( parentDir string, filters, includeFilters []string, + applySizeLimit bool, file fs.FileInfo, antMatcher filtering.Matcher, ) error { @@ -1885,6 +1960,10 @@ func handleFile( return nil } if filterMatched(includeFilters, file.Name()) && filterMatched(filters, file.Name()) && !antMatcher.Excluded(relPath, false) { + if size := targetFileSize(fileName, file); exceedsDefaultFilterSizeLimit(applySizeLimit, file.Name(), size) { + logSizeLimitExclusion(fileName, size) + return nil + } logger.PrintIfVerbose("Included: " + fileName) dat, err := ioutil.ReadFile(parentDir + file.Name()) if err != nil { @@ -1914,6 +1993,7 @@ func handleDir( parentDir string, filters, includeFilters []string, + applySizeLimit bool, file fs.FileInfo, antMatcher filtering.Matcher, excludeGitFolder bool, @@ -1955,7 +2035,7 @@ func handleDir( } newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) - return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, antMatcher, excludeGitFolder) + return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, applySizeLimit, antMatcher, excludeGitFolder) } func isDirFiltered(filename string, filters []string) (bool, error) { @@ -2166,8 +2246,17 @@ func addScaResults(zipWriter *zip.Writer) error { return nil } -func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsWrapper, featureFlagsWrapper wrappers.FeatureFlagsWrapper) ( - url, zipFilePath string, err error) { +// getUploadURLFromSource prepares the scan source (SBOM file, zip or directory), applies filters, +// runs the SCA/container resolvers when needed, compresses the result and uploads it. +// Its length and cyclomatic complexity predate the default include filter changes. +// +//nolint:gocyclo,funlen +func getUploadURLFromSource( + cmd *cobra.Command, + uploadsWrapper wrappers.UploadsWrapper, + featureFlagsWrapper wrappers.FeatureFlagsWrapper, + defaultIncludeFilters []string, +) (url, zipFilePath string, err error) { var preSignedURL string sourceDirFilter, _ := cmd.Flags().GetString(commonParams.SourceDirFilterFlag) @@ -2359,7 +2448,8 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW includeGeneratedCsvJson = true } } - zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder) + zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, + skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder, defaultIncludeFilters) // Clean up generated contributors files after successful zip creation if dirPathErr == nil && includeGeneratedCsvJson { @@ -2842,7 +2932,9 @@ func runCreateScanCommand( // those apply when the scan is not submitted (and it avoids an unnecessary policy // permission API call for the no-scan case). if noScan { - _, zipFilePath, handlerErr := setupScanHandler(cmd, uploadsWrapper, featureFlagsWrapper) + // No project is created for --no-scan, so only the tenant-level settings apply. + defaultIncludeFilters := getScanDefaultIncludeFilters(cmd, tenantWrapper, projectsWrapper, "") + _, zipFilePath, handlerErr := setupScanHandler(cmd, uploadsWrapper, featureFlagsWrapper, defaultIncludeFilters) defer cleanUpTempZip(zipFilePath) if handlerErr != nil { return errors.Errorf("%s", handlerErr) @@ -3014,7 +3106,8 @@ func createScanModel( } // Set up the scan handler (either git or upload) - scanHandler, zipFilePath, err := setupScanHandler(cmd, uploadsWrapper, featureFlagsWrapper) + defaultIncludeFilters := getScanDefaultIncludeFilters(cmd, tenantWrapper, projectsWrapper, scanModel.Project.ID) + scanHandler, zipFilePath, err := setupScanHandler(cmd, uploadsWrapper, featureFlagsWrapper, defaultIncludeFilters) if err != nil { return nil, zipFilePath, err } @@ -3041,7 +3134,27 @@ func getUploadType(cmd *cobra.Command) string { return "upload" } -func setupScanHandler(cmd *cobra.Command, uploadsWrapper wrappers.UploadsWrapper, featureFlagsWrapper wrappers.FeatureFlagsWrapper) ( +// getScanDefaultIncludeFilters resolves the default include filter for a local source; git sources and +// --skip-default-filter never build a filtered zip, so the settings are not fetched for them. +func getScanDefaultIncludeFilters( + cmd *cobra.Command, + tenantWrapper wrappers.TenantConfigurationWrapper, + projectsWrapper wrappers.ProjectsWrapper, + projectID string, +) []string { + skipDefaultFilter, _ := cmd.Flags().GetBool(commonParams.SkipDefaultFilterFlag) + if skipDefaultFilter || getUploadType(cmd) == git { + return nil + } + return resolveDefaultIncludeFilters(tenantWrapper, projectsWrapper, projectID) +} + +func setupScanHandler( + cmd *cobra.Command, + uploadsWrapper wrappers.UploadsWrapper, + featureFlagsWrapper wrappers.FeatureFlagsWrapper, + defaultIncludeFilters []string, +) ( wrappers.ScanHandler, string, error, @@ -3058,7 +3171,7 @@ func setupScanHandler(cmd *cobra.Command, uploadsWrapper wrappers.UploadsWrapper } else { var err error var uploadURL string - uploadURL, zipFilePath, err = getUploadURLFromSource(cmd, uploadsWrapper, featureFlagsWrapper) + uploadURL, zipFilePath, err = getUploadURLFromSource(cmd, uploadsWrapper, featureFlagsWrapper, defaultIncludeFilters) if err != nil { return scanHandler, zipFilePath, err } diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index fadd433a..507565d0 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -5441,7 +5441,7 @@ func TestSbomFileExcludedFromZip_WithCustomOutputName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5472,7 +5472,7 @@ func TestDefaultSbomFileAlwaysExcludedFromZip(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5505,7 +5505,7 @@ func TestSbomFileExcludedFromZip_InSubdirectory(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5544,7 +5544,7 @@ func TestSbomFileExcludedFromZip_AbsoluteSubdirWithCustomName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5604,10 +5604,10 @@ func cleanupMockAccessToken() { // --skip-default-filter tests func TestGetFilters_SkipDefaultFilter(t *testing.T) { - assert.DeepEqual(t, getIncludeFilters("*.foo", true), []string{"*.foo"}) + assert.DeepEqual(t, getIncludeFilters("*.foo", true, commonParams.BaseIncludeFilters), []string{"*.foo"}) assert.DeepEqual(t, getExcludeFilters("!bar", true), []string{"!bar"}) - includeDefault := getIncludeFilters("*.foo", false) + includeDefault := getIncludeFilters("*.foo", false, commonParams.BaseIncludeFilters) assert.Assert(t, slices.Contains(includeDefault, "*.go")) assert.Assert(t, slices.Contains(includeDefault, "*.foo")) @@ -5629,7 +5629,7 @@ func TestCompressFolder_DefaultBehaviorUnchanged(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5650,7 +5650,7 @@ func TestCompressFolder_SkipDefaultFilter(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5680,7 +5680,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterExclude(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!excluded_by_ant/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5701,7 +5701,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterIncludeOnly(t *testing.T) antMatcher, matcherErr := filtering.NewAntMatcher([]string{"**/*.customext"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5727,7 +5727,7 @@ func TestCompressFolder_DefaultFilters_WithAntFilter(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!keep_dir/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6150,7 +6150,7 @@ func TestCompressFolder_GitExcluded_WhenContributorsCsvEnabled(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, true, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6171,7 +6171,7 @@ func TestCompressFolder_GitIncluded_WhenContributorsCsvDisabled(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6195,7 +6195,7 @@ func TestCompressFolder_ContributorsFilesForceIncluded_WhenEnabled(t *testing.T) noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6219,7 +6219,7 @@ func TestCompressFolder_ContributorsFilesNotIncluded_WhenDisabled(t *testing.T) noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6247,7 +6247,7 @@ func TestCompressFolder_StaleFilesNotIncluded_WhenGenerationFailedThisRun(t *tes noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, true, commonParams.BaseIncludeFilters) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() diff --git a/internal/params/filters.go b/internal/params/filters.go index 65b1aee4..7885b42e 100644 --- a/internal/params/filters.go +++ b/internal/params/filters.go @@ -1,6 +1,77 @@ package params -var BaseIncludeFilters = []string{ +import "slices" + +// BaseIncludeFilters is the default include filter: the SAST patterns followed by the Linguist extension and filename patterns. +var BaseIncludeFilters = slices.Clip(slices.Concat(sastIncludeFilters, linguistExtensionFilters, linguistFilenameFilters)) + +// SastIncludeFilters is the default include filter without the Linguist patterns, used when LLM-based scanning is off. +var SastIncludeFilters = slices.Clip(sastIncludeFilters) + +// ScaFilePatterns are the files that belong to the SCA (OSS) engine: package manifests, lock files, SCA Resolver +// output and packaged binaries. They are exempt from the default filter size limit because they can exceed it. +// the exemption never adds a file to the zip. +var ScaFilePatterns = []string{ + // Manifests supported by the OSS realtime scanner (ossrealtime.IsSupportedManifestFile). + "pom.xml", + "package.json", + "bower.json", + "Directory.Packages.props", + "packages.config", + "*.csproj", + "go.mod", + "build.gradle", + "build.gradle.kts", + "libs.versions.toml", + "*.sbt", + "setup.cfg", + "setup.py", + "pyproject.toml", + "packages*.txt", + "constraint*.txt", + "Podfile", + "*.podspec", + "*.podspec.json", + "Cartfile", + "Cartfile.private", + "Gemfile", + "composer.json", + "pubspec.yaml", + "Package.swift", + "Package@swift-*.swift", + // Other manifests already in the default include filter. + "*requirement*.txt", + "build.sbt", + "Directory.Build.props", + // Lock files of the same package managers. + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "packages.lock.json", + "go.sum", + "poetry.lock", + "composer.lock", + "Gemfile.lock", + "Podfile.lock", + // SCA Resolver output. + "*.cxsca-results.json", + "*.cxsca-sast-results.json", + "*.cxsca-topology-results.json", + "*.cxsca.sig", + // Packaged binaries analysed by SCA. + "*.jar", +} + +// DefaultIncludeFilters returns the default include filter; the Linguist patterns are added only when LLM-based scanning is enabled. +func DefaultIncludeFilters(includeLinguist bool) []string { + if includeLinguist { + return BaseIncludeFilters + } + return SastIncludeFilters +} + +var sastIncludeFilters = []string{ "*.javasln", "*.project", "*.java", @@ -26,7 +97,6 @@ var BaseIncludeFilters = []string{ "*.cls", "*.dsr", "*.ctl", - "*.vb", "*.cpp", "*.c++", "*.cxx", @@ -52,7 +122,6 @@ var BaseIncludeFilters = []string{ "*.apexp", "*.page", "*.component", - "*.cls", "*.trigger", "*.tgr", "*.object", @@ -76,14 +145,9 @@ var BaseIncludeFilters = []string{ "*.pm", "*.plx", "*.psgi", - "*.java", - "*.kt", "*.m", - "*.h", "*.swift", "*.xib", - "*.html", - "*.htm", "*.pls", "*.sql", "*.pkh", @@ -112,12 +176,10 @@ var BaseIncludeFilters = []string{ "*.cpy", "*.aspx", "*.ascx", - "*.config", "*.xml", "*.cgi", "*.inc", "*.jar", - "*.js", "*.xsjs", "*.xsjslib", "*.dll", @@ -154,7 +216,6 @@ var BaseIncludeFilters = []string{ "*.mjs", "*.mts", "*.cts", - "*.html", "*.xhtml", "*.vm", "*.ac", diff --git a/internal/params/filters_linguist.go b/internal/params/filters_linguist.go new file mode 100644 index 00000000..ff527ab7 --- /dev/null +++ b/internal/params/filters_linguist.go @@ -0,0 +1,1165 @@ +package params + +// linguistExtensionFilters are GitHub Linguist programming and markup extensions not covered by sastIncludeFilters. +var linguistExtensionFilters = []string{ + // P1 - high enterprise usage + "*.abap", + "*.ashx", + "*.asmx", + "*.bash", + "*.bat", + "*.cmd", + "*.ipynb", + "*.j2", + "*.jinja", + "*.jinja2", + "*.ps1", + "*.psm1", + "*.rake", + "*.razor", + "*.sh", + + // P2 - medium enterprise usage + "*.4gl", + "*.al", + "*.asddls", + "*.astro", + "*.awk", + "*.bdy", + "*.builder", + "*.bzl", + "*.cds", + "*.cfc", + "*.cfm", + "*.cfml", + "*.clj", + "*.cljc", + "*.cljs", + "*.cobol", + "*.coffee", + "*.cppm", + "*.csx", + "*.cu", + "*.cuh", + "*.db2", + "*.dfm", + "*.dpr", + "*.dwl", + "*.ejs", + "*.es6", + "*.ex", + "*.exs", + "*.fnc", + "*.fs", + "*.fsi", + "*.fsx", + "*.gemspec", + "*.gohtml", + "*.gotmpl", + "*.gs", + "*.haml", + "*.hcl", + "*.hql", + "*.inl", + "*.ipp", + "*.jbuilder", + "*.jcl", + "*.ksh", + "*.liquid", + "*.mak", + "*.matlab", + "*.mk", + "*.mumps", + "*.njk", + "*.pas", + "*.pgsql", + "*.plb", + "*.plsql", + "*.podspec", + "*.pp", + "*.psd1", + "*.pyw", + "*.pyx", + "*.rego", + "*.rex", + "*.rexx", + "*.ru", + "*.sas", + "*.sbt", + "*.sls", + "*.spc", + "*.sra", + "*.sru", + "*.srw", + "*.ssjs", + "*.svelte", + "*.tftpl", + "*.tmpl", + "*.tofu", + "*.tpb", + "*.tpp", + "*.tps", + "*.trg", + "*.vba", + "*.vbhtml", + "*.vtl", + "*.vw", + "*.wsgi", + "*.zsh", + + // P3 - low enterprise usage + "*.6pl", + "*.6pm", + "*._coffee", + "*._js", + "*.ada", + "*.adb", + "*.ads", + "*.agda", + "*.ahk", + "*.aj", + "*.ak", + "*.ampl", + "*.apl", + "*.applescript", + "*.as", + "*.asm", + "*.au3", + "*.aug", + "*.auk", + "*.aw", + "*.axd", + "*.bal", + "*.baml", + "*.bats", + "*.bb", + "*.bi", + "*.blade", + "*.bones", + "*.boo", + "*.boot", + "*.bsl", + "*.c3", + "*.cairo", + "*.cake", + "*.carbon", + "*.cats", + "*.ccp", + "*.cdc", + "*.ceylon", + "*.chpl", + "*.circom", + "*.cj", + "*.cjsx", + "*.cl2", + "*.clar", + "*.cljs.hl", + "*.cljscm", + "*.cljx", + "*.clw", + "*.cmake.in", + "*.command", + "*.containerfile", + "*.cp", + "*.cql", + "*.cr", + "*.cs.pp", + "*.csh", + "*.cwl", + "*.cyp", + "*.cypher", + "*.dfy", + "*.dhall", + "*.di", + "*.do", + "*.ebuild", + "*.ect", + "*.edge", + "*.edgeql", + "*.ejs.t", + "*.elm", + "*.elv", + "*.erb.deface", + "*.erl", + "*.ets", + "*.eye", + "*.f03", + "*.f08", + "*.f77", + "*.f90", + "*.f95", + "*.factor", + "*.fcgi", + "*.feature", + "*.fish", + "*.fnl", + "*.fst", + "*.fth", + "*.ftlh", + "*.gawk", + "*.gjs", + "*.gleam", + "*.gms", + "*.god", + "*.grt", + "*.gtpl", + "*.gyp", + "*.gypi", + "*.h.in", + "*.ha", + "*.hack", + "*.haml.deface", + "*.hb", + "*.heex", + "*.hic", + "*.hip", + "*.hrl", + "*.hs", + "*.hta", + "*.html.eex", + "*.html.hl", + "*.html.tmpl", + "*.hx", + "*.hy", + "*.iced", + "*.idc", + "*.idr", + "*.ijs", + "*.imba", + "*.ino", + "*.ispc", + "*.iss", + "*.jac", + "*.jake", + "*.janet", + "*.jav", + "*.javascript", + "*.jl", + "*.jq", + "*.jsb", + "*.jscad", + "*.jsfl", + "*.jsh", + "*.jslib", + "*.jsm", + "*.jsonnet", + "*.jspre", + "*.jss", + "*.jst", + "*.jte", + "*.just", + "*.kid", + "*.kk", + "*.ktm", + "*.latte", + "*.lean", + "*.leex", + "*.leo", + "*.lfe", + "*.lhs", + "*.libsonnet", + "*.linq", + "*.lisp", + "*.litcoffee", + "*.lkml", + "*.ll", + "*.lmi", + "*.lpr", + "*.lsp", + "*.luau", + "*.make", + "*.makefile", + "*.mako", + "*.marko", + "*.mathematica", + "*.mawk", + "*.metal", + "*.mint", + "*.mkfile", + "*.ml", + "*.mli", + "*.moon", + "*.move", + "*.mspec", + "*.nasm", + "*.nawk", + "*.nc", + "*.nf", + "*.nim", + "*.nix", + "*.njs", + "*.nomad", + "*.nr", + "*.nse", + "*.nsi", + "*.nu", + "*.nut", + "*.ob2", + "*.opa", + "*.oxygene", + "*.p6", + "*.p8", + "*.pac", + "*.pacscript", + "*.pan", + "*.pascal", + "*.pd_lua", + "*.perl", + "*.phps", + "*.phpt", + "*.pig", + "*.pike", + "*.pl6", + "*.pm6", + "*.polar", + "*.pony", + "*.pprx", + "*.pq", + "*.prawn", + "*.prg", + "*.purs", + "*.pxd", + "*.pxi", + "*.py3", + "*.pyde", + "*.pyi", + "*.pyp", + "*.pyt", + "*.qasm", + "*.ql", + "*.qml", + "*.qs", + "*.rabl", + "*.raku", + "*.rakumod", + "*.rbbas", + "*.rbi", + "*.rbuild", + "*.rbw", + "*.rbx", + "*.rbxs", + "*.rd", + "*.reb", + "*.red", + "*.rei", + "*.resi", + "*.riot", + "*.rkt", + "*.robot", + "*.roc", + "*.rockspec", + "*.rpy", + "*.rs.in", + "*.rsx", + "*.ruby", + "*.sage", + "*.sbatch", + "*.scaml", + "*.sci", + "*.sed", + "*.sh.in", + "*.sjs", + "*.slim", + "*.slurm", + "*.smk", + "*.sml", + "*.soy", + "*.stan", + "*.surql", + "*.tac", + "*.tcc", + "*.tcl", + "*.tcsh", + "*.tea", + "*.templ", + "*.thor", + "*.thy", + "*.tm", + "*.tmux", + "*.tool", + "*.txx", + "*.upc", + "*.vala", + "*.vcl", + "*.vto", + "*.vy", + "*.wast", + "*.wat", + "*.watchr", + "*.wdl", + "*.wlua", + "*.wren", + "*.xht", + "*.xojo_code", + "*.xpy", + "*.xquery", + "*.xtend", + "*.yar", + "*.yul", + "*.zeek", + "*.zep", + "*.zig", + "*.zsh-theme", + + // P4 - niche, newly in scope + "*._ls", + "*.a51", + "*.ado", + "*.adp", + "*.agc", + "*.ah1", + "*.ah2", + "*.ahkl", + "*.aidl", + "*.aleo", + "*.alg", + "*.als", + "*.angelscript", + "*.app.src", + "*.arc", + "*.arr", + "*.asd", + "*.ash", + "*.asl", + "*.axi", + "*.axs", + "*.bbappend", + "*.bbclass", + "*.be", + "*.befunge", + "*.bend", + "*.bf", + "*.bison", + "*.blp", + "*.bmx", + "*.bpl", + "*.bqn", + "*.bro", + "*.brs", + "*.bru", + "*.bs", + "*.bsv", + "*.capnp", + "*.cdf", + "*.cginc", + "*.ch", + "*.chs", + "*.cirru", + "*.ck", + "*.cl", + "*.click", + "*.clp", + "*.clue", + "*.cnc", + "*.cocci", + "*.coffee.md", + "*.coq", + "*.cps", + "*.csc", + "*.csd", + "*.curry", + "*.cw", + "*.cy", + "*.das", + "*.dats", + "*.dcl", + "*.decls", + "*.djs", + "*.dlm", + "*.dm", + "*.doh", + "*.druby", + "*.dsl", + "*.duby", + "*.dyalog", + "*.dyl", + "*.dylan", + "*.ecl", + "*.eclass", + "*.eclxml", + "*.ecr", + "*.eh", + "*.el", + "*.eliom", + "*.eliomi", + "*.em", + "*.emberscript", + "*.eq", + "*.es", + "*.escript", + "*.esdl", + "*.fan", + "*.fancypack", + "*.fir", + "*.flex", + "*.flix", + "*.flux", + "*.forth", + "*.fp", + "*.fpp", + "*.fppi", + "*.frag", + "*.frg", + "*.frt", + "*.fsh", + "*.fshader", + "*.fsti", + "*.fun", + "*.fut", + "*.fx", + "*.fxh", + "*.fy", + "*.g4", + "*.gaml", + "*.gap", + "*.gco", + "*.gcode", + "*.gd", + "*.gdb", + "*.gdbinit", + "*.gdshader", + "*.gdshaderinc", + "*.geo", + "*.geom", + "*.gf", + "*.gi", + "*.glf", + "*.glsl", + "*.glslf", + "*.glslv", + "*.gno", + "*.gnu", + "*.gnuplot", + "*.golo", + "*.gp", + "*.grace", + "*.gsc", + "*.gshader", + "*.gsx", + "*.hats", + "*.hc", + "*.hexpat", + "*.hhi", + "*.hlean", + "*.hlsl", + "*.hlsli", + "*.hoon", + "*.hqf", + "*.hs-boot", + "*.hsc", + "*.hurl", + "*.hxsl", + "*.i3", + "*.i7x", + "*.icl", + "*.ig", + "*.ihlp", + "*.ijm", + "*.ik", + "*.il", + "*.ily", + "*.ink", + "*.intr", + "*.io", + "*.iol", + "*.ipf", + "*.isl", + "*.jai", + "*.jflex", + "*.jison", + "*.jisonlex", + "*.kak", + "*.kit", + "*.kojo", + "*.krl", + "*.ksy", + "*.kv", + "*.lagda", + "*.langium", + "*.las", + "*.lasso", + "*.lasso8", + "*.lasso9", + "*.ld", + "*.lds", + "*.lex", + "*.lgt", + "*.lid", + "*.lidr", + "*.ligo", + "*.liq", + "*.livecodescript", + "*.lobster", + "*.logtalk", + "*.lol", + "*.lookml", + "*.lp", + "*.ls", + "*.lsl", + "*.lslp", + "*.lvclass", + "*.lvlib", + "*.lvproj", + "*.ly", + "*.m2", + "*.m3", + "*.m4", + "*.ma", + "*.mao", + "*.mata", + "*.matah", + "*.maxhelp", + "*.maxpat", + "*.maxproj", + "*.mbt", + "*.mcfunction", + "*.mch", + "*.mcr", + "*.mdx", + "*.metta", + "*.mg", + "*.minid", + "*.mirah", + "*.ml4", + "*.mligo", + "*.mlir", + "*.mll", + "*.mly", + "*.mmk", + "*.mms", + "*.monkey", + "*.monkey2", + "*.moo", + "*.mpl", + "*.mps", + "*.mq4", + "*.mq5", + "*.mqh", + "*.mrc", + "*.ms", + "*.msd", + "*.mss", + "*.mt", + "*.mtml", + "*.mud", + "*.muf", + "*.mxt", + "*.myt", + "*.mzn", + "*.nas", + "*.nasl", + "*.nbp", + "*.ne", + "*.nearley", + "*.ned", + "*.ni", + "*.nim.cfg", + "*.nimble", + "*.nimrod", + "*.nims", + "*.nit", + "*.nlogo", + "*.nqp", + "*.nsh", + "*.nss", + "*.numpy", + "*.numpyw", + "*.numsc", + "*.ny", + "*.omgrofl", + "*.ooc", + "*.opal", + "*.opencl", + "*.opy", + "*.orc", + "*.os", + "*.overpassql", + "*.ox", + "*.oxh", + "*.oxo", + "*.oz", + "*.p4", + "*.p6l", + "*.p6m", + "*.pact", + "*.parrot", + "*.pasm", + "*.pat", + "*.pbi", + "*.pddl", + "*.pde", + "*.peggy", + "*.pegjs", + "*.pep", + "*.per", + "*.pir", + "*.plot", + "*.plt", + "*.pml", + "*.pmod", + "*.podsl", + "*.pogo", + "*.por", + "*.pov", + "*.praat", + "*.pri", + "*.prolog", + "*.prw", + "*.psc", + "*.pwn", + "*.qbs", + "*.qc", + "*.qll", + "*.qnt", + "*.r2", + "*.r3", + "*.rascript", + "*.rbfrm", + "*.rbmnu", + "*.rbres", + "*.rbtbar", + "*.rbuistate", + "*.rchit", + "*.re", + "*.rebol", + "*.reds", + "*.religo", + "*.resource", + "*.rg", + "*.rhai", + "*.ring", + "*.rktd", + "*.rktl", + "*.rl", + "*.rmiss", + "*.rsc", + "*.rsh", + "*.sagews", + "*.sail", + "*.sats", + "*.scad", + "*.sce", + "*.scenic", + "*.sco", + "*.scpt", + "*.scrbl", + "*.sdc", + "*.self", + "*.sexp", + "*.sh-session", + "*.shader", + "*.shen", + "*.sieve", + "*.sj", + "*.sl", + "*.slang", + "*.sld", + "*.slint", + "*.sma", + "*.smali", + "*.smithy", + "*.smt", + "*.smt2", + "*.snakefile", + "*.spin", + "*.sps", + "*.sqf", + "*.ss", + "*.st", + "*.sthlp", + "*.story", + "*.sv", + "*.svh", + "*.swg", + "*.swig", + "*.talon", + "*.tape", + "*.tcl.in", + "*.td", + "*.tesc", + "*.tese", + "*.thrift", + "*.tla", + "*.tlv", + "*.toit", + "*.tolk", + "*.tst", + "*.tu", + "*.txl", + "*.uc", + "*.udo", + "*.uno", + "*.uplc", + "*.ur", + "*.urs", + "*.vapi", + "*.vark", + "*.veo", + "*.verse", + "*.vert", + "*.vh", + "*.vhd", + "*.vhdl", + "*.vhf", + "*.vhi", + "*.vho", + "*.vhs", + "*.vht", + "*.vhw", + "*.vim", + "*.vmb", + "*.volt", + "*.vrx", + "*.vs", + "*.vsh", + "*.vshader", + "*.webidl", + "*.wgsl", + "*.whiley", + "*.wisp", + "*.wl", + "*.wlk", + "*.wls", + "*.wlt", + "*.ws", + "*.x10", + "*.x68", + "*.xc", + "*.xdc", + "*.xi", + "*.xm", + "*.xojo_menu", + "*.xojo_report", + "*.xojo_script", + "*.xojo_toolbar", + "*.xojo_window", + "*.xpl", + "*.xproc", + "*.xq", + "*.xql", + "*.xqm", + "*.xqy", + "*.xrl", + "*.xs", + "*.xsh", + "*.xsl", + "*.xslt", + "*.xzap", + "*.yacc", + "*.yap", + "*.yara", + "*.yrl", + "*.z3", + "*.zap", + "*.zig.zon", + "*.zil", + "*.zimpl", + "*.zmpl", + "*.zpl", + "*.zs", + + // Ambiguous: extension is also used by binary formats + "*.mo", + "*.pb", + "*.8xp", + "*.8xp.txt", + "*.com", + + // Ambiguous: extension or name is shared with non-code formats + "*.mm", + "*.ddl", + "*.pbt", + "*.prc", + "*.sol", + "*.cue", + "*.for", + "*.gts", + "*.mojo", + "*.odin", + "*.pkl", + "*.pluginspec", + "*.res", + "*.scm", + "*.star", + "*.sw", + "*.tact", + "*.4dm", + "*.4th", + "*.asc", + "*.asy", + "*.b", + "*.bst", + "*.d", + "*.dsc", + "*.dsp", + "*.e", + "*.f", + "*.fr", + "*.g", + "*.gml", + "*.gst", + "*.i", + "*.ice", + "*.j", + "*.k", + "*.ks", + "*.l", + "*.mask", + "*.mc", + "*.md", + "*.msg", + "*.mu", + "*.nb", + "*.ncl", + "*.nl", + "*.ol", + "*.p", + "*.q", + "*.r", + "*.s", + "*.scd", + "*.sch", + "*.sig", + "*.sip", + "*.sp", + "*.spec", + "*.svx", + "*.t", + "*.tl", + "*.tsp", + "*.typ", + "*.v", + "*.w", + "*.x", + "*.y", + "*.yy", + + // History files, mainly useful for secret detection + "*.rhistory", + + // Documentation and stylesheet markup + "*.apib", + "*.aux", + "*.bbcode", + "*.bbx", + "*.bib", + "*.bibtex", + "*.cbx", + "*.chem", + "*.cook", + "*.css", + "*.d2", + "*.dtx", + "*.epsi", + "*.ins", + "*.lbx", + "*.less", + "*.ltx", + "*.me", + "*.mermaid", + "*.mkii", + "*.mkiv", + "*.mkvi", + "*.mmd", + "*.pcss", + "*.pfa", + "*.pic", + "*.postcss", + "*.raml", + "*.rnh", + "*.rno", + "*.roff", + "*.rtf", + "*.sass", + "*.scss", + "*.snip", + "*.snippet", + "*.snippets", + "*.sss", + "*.sty", + "*.styl", + "*.tmac", + "*.yasnippet", + "*.1in", + "*.1m", + "*.1x", + "*.3in", + "*.3m", + "*.3p", + "*.3pm", + "*.3qt", + "*.3x", + "*.eps", + "*.man", + "*.mdoc", + "*.ps", + "*.srt", + "*.toc", +} + +// linguistFilenameFilters are GitHub Linguist programming and markup file names not covered by sastIncludeFilters. +var linguistFilenameFilters = []string{ + // P1 - high enterprise usage + "containerfile", + "gnumakefile", + "jenkinsfile", + "makefile", + + // P2 - medium enterprise usage + "bsdmakefile", + "buck", + "build.bazel", + "capfile", + "dangerfile", + "fastfile", + "guardfile", + "meson.build", + "module.bazel", + "rakefile", + "sconscript", + "sconstruct", + "workspace.bazel", + + // P3 - low enterprise usage + ".gclient", + ".irbrc", + ".justfile", + ".latexmkrc", + ".luacheckrc", + ".php_cs", + ".php_cs.dist", + ".pryrc", + ".rprofile", + ".simplecov", + "9fs", + "ack", + "apkbuild", + "appraisals", + "buildfile", + "cakefile", + "deliverfile", + "deps", + "earthfile", + "emakefile", + "expr-dist", + "jakefile", + "justfile", + "kbuild", + "latexmkrc", + "m3makefile", + "m3overrides", + "makefile.in", + "man", + "meson_options.txt", + "mkfile", + "mvnw", + "notebook", + "nukefile", + "phakefile", + "pkgbuild", + "rexfile", + "slakefile", + "snapfile", + "steepfile", + "thorfile", + "tiltfile", + "vagrantfile", + "workspace.bzlmod", + "wscript", + + // P4 - niche, newly in scope + ".abbrev_defs", + ".bash_aliases", + ".bash_functions", + ".bash_logout", + ".bash_profile", + ".bashrc", + ".cshrc", + ".emacs", + ".emacs.desktop", + ".envrc", + ".exrc", + ".factor-boot-rc", + ".factor-rc", + ".flaskenv", + ".gnus", + ".gvimrc", + ".kshrc", + ".login", + ".nvimrc", + ".profile", + ".spacemacs", + ".vimrc", + ".viper", + ".xinitrc", + ".xsession", + ".zlogin", + ".zlogout", + ".zprofile", + ".zshenv", + ".zshrc", + "_emacs", + "_vimrc", + "abbrev_defs", + "bash_aliases", + "bash_logout", + "bash_profile", + "bashrc", + "berksfile", + "brewfile", + "cask", + "cpanfile", + "cshrc", + "dune-project", + "eask", + "fakefile", + "gvimrc", + "jarfile", + "kakrc", + "kcl.mod", + "kcl.mod.lock", + "kshrc", + "ld.script", + "login", + "mavenfile", + "mix.lock", + "modulefile", + "moon.mod", + "moon.pkg", + "nim.cfg", + "nvimrc", + "owh", + "procfile", + "profile", + "project.ede", + "puppetfile", + "rebar.config.lock", + "rebar.lock", + "root", + "singularity", + "snakefile", + "starfield", + "vimrc", + "xinitrc", + "xsession", + "zlogin", + "zlogout", + "zprofile", + "zshenv", + "zshrc", + + // Ambiguous: extension or name is shared with non-code formats + "build", + "workspace", + + // History files, mainly useful for secret detection + ".bash_history", + ".rapp.history", + + // Documentation and stylesheet markup + "eqnrc", + "mmn", + "mmt", + "troffrc", + "troffrc-end", +} diff --git a/internal/params/filters_linguist_test.go b/internal/params/filters_linguist_test.go new file mode 100644 index 00000000..f3600d04 --- /dev/null +++ b/internal/params/filters_linguist_test.go @@ -0,0 +1,93 @@ +package params + +import ( + "slices" + "strings" + "testing" + + "gotest.tools/assert" +) + +// unsafeFilterChars are glob, negation and path characters that must not appear in a Linguist filter. +const unsafeFilterChars = "*?[" + `\` + "!/ " + +func TestLinguistExtensionFiltersFormat(t *testing.T) { + assert.Assert(t, len(linguistExtensionFilters) > 0) + for _, f := range linguistExtensionFilters { + assert.Assert(t, strings.HasPrefix(f, "*."), "extension filter %q must start with \"*.\"", f) + assert.Assert(t, len(f) > len("*."), "extension filter %q has no extension", f) + assert.Equal(t, strings.ToLower(f), f, "extension filter %q must be lower-case", f) + assert.Assert(t, !strings.ContainsAny(f[1:], unsafeFilterChars), "extension filter %q has unexpected glob or path characters", f) + } +} + +func TestLinguistFilenameFiltersFormat(t *testing.T) { + assert.Assert(t, len(linguistFilenameFilters) > 0) + for _, f := range linguistFilenameFilters { + assert.Assert(t, f != "", "filename filters must not contain empty entries") + assert.Equal(t, strings.ToLower(f), f, "filename filter %q must be lower-case", f) + assert.Assert(t, !strings.ContainsAny(f, unsafeFilterChars), "filename filter %q must be a plain file name", f) + } +} + +func TestLinguistFiltersHaveNoDuplicates(t *testing.T) { + for name, list := range map[string][]string{ + "linguistExtensionFilters": linguistExtensionFilters, + "linguistFilenameFilters": linguistFilenameFilters, + } { + seen := map[string]struct{}{} + for _, f := range list { + _, dup := seen[f] + assert.Assert(t, !dup, "%s contains duplicate %q", name, f) + seen[f] = struct{}{} + } + } +} + +func TestLinguistFiltersDoNotOverlapSastFilters(t *testing.T) { + sast := map[string]struct{}{} + for _, f := range sastIncludeFilters { + sast[strings.ToLower(f)] = struct{}{} + } + for _, list := range [][]string{linguistExtensionFilters, linguistFilenameFilters} { + for _, f := range list { + _, overlap := sast[f] + assert.Assert(t, !overlap, "%q is already in sastIncludeFilters", f) + } + } +} + +func TestBaseIncludeFiltersCombinesAllLists(t *testing.T) { + want := len(sastIncludeFilters) + len(linguistExtensionFilters) + len(linguistFilenameFilters) + assert.Equal(t, want, len(BaseIncludeFilters)) + assert.Equal(t, want, cap(BaseIncludeFilters), "BaseIncludeFilters should be clipped so appends never alias") + assert.Assert(t, slices.Equal(sastIncludeFilters, BaseIncludeFilters[:len(sastIncludeFilters)]), "SAST filters must stay first") + for _, list := range [][]string{sastIncludeFilters, linguistExtensionFilters, linguistFilenameFilters} { + for _, f := range list { + assert.Assert(t, slices.Contains(BaseIncludeFilters, f), "%q missing from BaseIncludeFilters", f) + } + } +} + +func TestBaseIncludeFiltersAppendDoesNotAlias(t *testing.T) { + // Appending to the shared slice without reassigning it is what this test checks. + a := append(BaseIncludeFilters, "*.first") //nolint:gocritic + b := append(BaseIncludeFilters, "*.second") //nolint:gocritic + assert.Equal(t, "*.first", a[len(a)-1]) + assert.Equal(t, "*.second", b[len(b)-1]) +} + +func TestLinguistFiltersSamples(t *testing.T) { + for _, f := range []string{"*.sh", "*.ps1", "*.bat", "*.ipynb", "*.j2", "jenkinsfile", "makefile", "containerfile", "gnumakefile", ".bash_history"} { + assert.Assert(t, slices.Contains(BaseIncludeFilters, f), "%q should be in BaseIncludeFilters", f) + } +} + +func TestDefaultIncludeFilters(t *testing.T) { + assert.Assert(t, slices.Equal(BaseIncludeFilters, DefaultIncludeFilters(true)), "LLM-based scanning on uses the full default filter") + assert.Assert(t, slices.Equal(sastIncludeFilters, DefaultIncludeFilters(false)), "LLM-based scanning off uses the SAST patterns only") + assert.Equal(t, len(SastIncludeFilters), cap(SastIncludeFilters), "SastIncludeFilters should be clipped so appends never alias") + for _, f := range linguistExtensionFilters { + assert.Assert(t, !slices.Contains(DefaultIncludeFilters(false), f), "%q must not be in the SAST-only filter", f) + } +} diff --git a/internal/wrappers/mock/projects-mock.go b/internal/wrappers/mock/projects-mock.go index f289399a..8a2c2bc9 100644 --- a/internal/wrappers/mock/projects-mock.go +++ b/internal/wrappers/mock/projects-mock.go @@ -9,6 +9,9 @@ import ( type ProjectsMockWrapper struct{} +// ProjectConfiguration is returned by GetConfiguration; tests set it to simulate project-level settings. +var ProjectConfiguration []wrappers.ProjectConfiguration + func (p *ProjectsMockWrapper) Create(model *wrappers.Project) ( *wrappers.ProjectResponseModel, *wrappers.ErrorModel, @@ -37,6 +40,12 @@ func (p *ProjectsMockWrapper) UpdateConfiguration(projectID string, configuratio return nil, nil } +// GetConfiguration returns the mock project configuration for the given project ID. +func (p *ProjectsMockWrapper) GetConfiguration(projectID string) ([]wrappers.ProjectConfiguration, *wrappers.ErrorModel, error) { + fmt.Println("Called Get Configuration for project", projectID, "in ProjectsMockWrapper") + return ProjectConfiguration, nil, nil +} + func (p *ProjectsMockWrapper) Get(params map[string]string) ( *wrappers.ProjectsCollectionResponseModel, *wrappers.ErrorModel, diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index e618b506..f0e15437 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -14,6 +14,8 @@ import ( commonParams "github.com/checkmarx/ast-cli/internal/params" ) +const failedToParseConfig = "Failed to parse project configuration response" + type ProjectsHTTPWrapper struct { path string } @@ -101,6 +103,46 @@ func (p *ProjectsHTTPWrapper) UpdateConfiguration(projectID string, configuratio return handleProjectResponseWithNoBody(resp, err, http.StatusNoContent) } +// GetConfiguration returns the effective configuration of a project; settings it does not override come from the tenant. +func (p *ProjectsHTTPWrapper) GetConfiguration(projectID string) ([]ProjectConfiguration, *ErrorModel, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + params := map[string]string{ + commonParams.ProjectIDFlag: projectID, + } + + fn := func() (*http.Response, error) { + return SendHTTPRequestWithQueryParams(http.MethodGet, "api/configuration/project", params, nil, clientTimeout) + } + resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay*time.Millisecond) + if err != nil { + return nil, nil, err + } + defer func() { + _ = resp.Body.Close() + }() + + decoder := json.NewDecoder(resp.Body) + + switch resp.StatusCode { + case http.StatusBadRequest, http.StatusInternalServerError: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return nil, nil, errors.Wrapf(err, failedToParseConfig) + } + return nil, &errorModel, nil + case http.StatusOK: + var configuration []ProjectConfiguration + err = decoder.Decode(&configuration) + if err != nil { + return nil, nil, errors.Wrapf(err, failedToParseConfig) + } + return configuration, nil, nil + default: + return nil, nil, errors.Errorf("response status code %d", resp.StatusCode) + } +} + func (p *ProjectsHTTPWrapper) Get(params map[string]string) ( *ProjectsCollectionResponseModel, *ErrorModel, error) { diff --git a/internal/wrappers/projects.go b/internal/wrappers/projects.go index deef8ce1..4189b1ac 100644 --- a/internal/wrappers/projects.go +++ b/internal/wrappers/projects.go @@ -56,4 +56,5 @@ type ProjectsWrapper interface { Delete(projectID string) (*ErrorModel, error) Tags() (map[string][]string, *ErrorModel, error) UpdateConfiguration(projectID string, configuration []ProjectConfiguration) (*ErrorModel, error) + GetConfiguration(projectID string) ([]ProjectConfiguration, *ErrorModel, error) } diff --git a/test/integration/filters_test.go b/test/integration/filters_test.go index b1a88d17..0b751c63 100644 --- a/test/integration/filters_test.go +++ b/test/integration/filters_test.go @@ -3,7 +3,11 @@ package integration import ( + "bytes" + "log" + "os" "path/filepath" + "strings" "testing" "github.com/checkmarx/ast-cli/internal/params" @@ -60,3 +64,68 @@ func TestScanWithConservativeFiltering_VerifyGenericPatternsExcluded(t *testing. err, _ := executeCommand(t, args...) assert.NilError(t, err, "Scan with conservative filtering should succeed") } + +// TestScanWithLinguistFilters_VerifyFilesUploaded asserts that Linguist extensions and filenames reach the source zip +// only when the tenant or project has LLM-based scanning enabled, that SAST files are always uploaded, and that files +// outside the filter are still left out. +func TestScanWithLinguistFilters_VerifyFilesUploaded(t *testing.T) { + sourceDir := t.TempDir() + files := map[string]string{ + "deploy.sh": "#!/bin/sh\necho deploy\n", + "build.ps1": "Write-Host 'build'\n", + "Jenkinsfile": "pipeline { agent any }\n", + "Makefile": "all:\n\techo build\n", + "Main.java": "public class Main {}\n", + "logo.png": "not a real image", + "notes.bak": "backup", + } + for name, content := range files { + assert.NilError(t, os.WriteFile(filepath.Join(sourceDir, name), []byte(content), 0o600)) + } + + args := []string{ + scanCommand, "create", + flag(params.ProjectName), GenerateRandomProjectNameForScan(), + flag(params.BranchFlag), "test-linguist-filters", + flag(params.SourcesFlag), sourceDir, + flag(params.ScanTypes), params.SastType, + flag(params.AsyncFlag), + flag(params.DebugFlag), + } + + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + + cmd := createASTIntegrationTestCommand(t) + err := execute(cmd, args...) + assert.NilError(t, err, "Scan with Linguist files should succeed") + + logText := buf.String() + assert.Assert(t, hasZipLogLine(logText, "Included: ", "Main.java"), "SAST files should always be included by the default filter") + + llmEnabled := strings.Contains(logText, "LLM-based scanning is enabled: adding Linguist patterns") + linguistPrefix := "Excluded: " + if llmEnabled { + linguistPrefix = "Included: " + } + for _, name := range []string{"deploy.sh", "build.ps1", "Jenkinsfile", "Makefile"} { + assert.Assert(t, hasZipLogLine(logText, linguistPrefix, name), "%s should be %s (LLM-based scanning enabled: %v)", name, linguistPrefix, llmEnabled) + } + for _, name := range []string{"logo.png", "notes.bak"} { + assert.Assert(t, hasZipLogLine(logText, "Excluded: ", name), "%s should be excluded by the default filter", name) + } +} + +// hasZipLogLine reports whether the zip walk logged prefix for a file whose path ends with name. +func hasZipLogLine(logText, prefix, name string) bool { + for _, line := range strings.Split(logText, "\n") { + line = strings.TrimSpace(line) + if strings.Contains(line, prefix) && (strings.HasSuffix(line, "/"+name) || strings.HasSuffix(line, `\`+name)) { + return true + } + } + return false +}