From 9819b976f5ab4acc579ef8e41e18e03e37e3b582 Mon Sep 17 00:00:00 2001 From: John Breault Date: Mon, 21 Sep 2026 19:52:53 -0400 Subject: [PATCH 1/2] Add --os-include, --os-exclude, --compression-format flags for selective release-asset syncing Allows GHES admins to select which CodeQL bundle OS assets to sync and which compression format (gz or zst) to use, reducing load on the GHES VM and avoiding syncing unnecessary release assets. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .gitignore | 1 + README.md | 6 +++ cmd/pull.go | 54 +++++++++++++++++-- cmd/sync.go | 5 +- internal/pull/pull.go | 77 +++++++++++++++++++++++---- internal/pull/pull_test.go | 106 +++++++++++++++++++++++++++++++++++++ 6 files changed, 234 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 5c3e848..15a860c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /codeql-action-sync /dist/ /pkged.go +/releases/ diff --git a/README.md b/README.md index c4c6ccd..e7389ad 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t * `--actions-admin-user` - The name of the Actions admin user, which will be used if you are updating the bundled CodeQL Action. If not specified `actions-admin` will be used. * `--force` - By default the tool will not overwrite existing repositories. Providing this flag will allow it to. * `--push-ssh` - Push Git contents over SSH rather than HTTPS. To use this option you must have SSH access to your GitHub Enterprise instance configured. +* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced. +* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`. +* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced. ### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server. From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder. @@ -36,6 +39,9 @@ From a machine with access to GitHub.com use the `./codeql-action-sync pull` com **Optional Arguments:** * `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used. * `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. The token does not need to have any scopes. +* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced. +* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`. +* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced. Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server. diff --git a/cmd/pull.go b/cmd/pull.go index d69f786..d282dae 100644 --- a/cmd/pull.go +++ b/cmd/pull.go @@ -1,6 +1,9 @@ package cmd import ( + "strings" + + usererrors "errors" "github.com/github/codeql-action-sync/internal/cachedirectory" "github.com/github/codeql-action-sync/internal/pull" "github.com/github/codeql-action-sync/internal/version" @@ -12,20 +15,65 @@ var pullCmd = &cobra.Command{ Short: "Pull the CodeQL Action from GitHub to a local cache.", RunE: func(cmd *cobra.Command, args []string) error { version.LogVersion() + if err := pullFlags.Validate(); err != nil { + return err + } cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir) - return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL) + return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat) }, } type pullFlagFields struct { - sourceToken string - sourceURL string + sourceToken string + sourceURL string + osInclude string + osExclude string + compressionFormat string } var pullFlags = pullFlagFields{} +const errorOSIncludeAndExclude = "You cannot specify both --os-include and --os-exclude at the same time. Please use only one of these flags." +const errorInvalidCompressionFormat = "Invalid --compression-format value. Valid values are \"gz\" or \"zst\"." + func (f *pullFlagFields) Init(cmd *cobra.Command) { cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.") cmd.Flags().StringVar(&f.sourceURL, "source-url", "", "Use a custom Git URL for fetching the Action repository contents from. The CodeQL bundles will still be fetched from GitHub.com.") cmd.Flags().MarkHidden("source-url") + cmd.Flags().StringVar(&f.osInclude, "os-include", "", "A comma-separated list of operating systems (e.g. \"linux64,win64\") to include CodeQL bundle release assets for. Cannot be used together with --os-exclude. If neither is specified, assets for all operating systems are synced.") + cmd.Flags().StringVar(&f.osExclude, "os-exclude", "", "A comma-separated list of operating systems (e.g. \"win64,osx64\") to exclude CodeQL bundle release assets for. Cannot be used together with --os-include.") + cmd.Flags().StringVar(&f.compressionFormat, "compression-format", "", "The compression format of CodeQL bundle release assets to sync, either \"gz\" or \"zst\". If not specified, both compression formats are synced.") +} + +func splitCommaSeparatedList(value string) []string { + if value == "" { + return []string{} + } + parts := strings.Split(value, ",") + result := make([]string, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +func (f *pullFlagFields) assetOSIncludes() []string { + return splitCommaSeparatedList(f.osInclude) +} + +func (f *pullFlagFields) assetOSExcludes() []string { + return splitCommaSeparatedList(f.osExclude) +} + +func (f *pullFlagFields) Validate() error { + if f.osInclude != "" && f.osExclude != "" { + return usererrors.New(errorOSIncludeAndExclude) + } + if f.compressionFormat != "" && f.compressionFormat != "gz" && f.compressionFormat != "zst" { + return usererrors.New(errorInvalidCompressionFormat) + } + return nil } diff --git a/cmd/sync.go b/cmd/sync.go index 7aec7b7..2ae2954 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -13,8 +13,11 @@ var syncCmd = &cobra.Command{ Short: "Sync the CodeQL Action from GitHub to a GitHub Enterprise Server installation.", RunE: func(cmd *cobra.Command, args []string) error { version.LogVersion() + if err := pullFlags.Validate(); err != nil { + return err + } cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir) - err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL) + err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat) if err != nil { return err } diff --git a/internal/pull/pull.go b/internal/pull/pull.go index 34bd0c6..8c3bb27 100644 --- a/internal/pull/pull.go +++ b/internal/pull/pull.go @@ -36,12 +36,60 @@ var relevantReferences = regexp.MustCompile("^refs/(heads|tags)/(main|v\\d+)$") const defaultConfigurationPath = "src/defaults.json" +// Matches release asset names like "codeql-bundle-linux64.tar.gz", +// "codeql-bundle-linux-arm64.tar.zst" or "codeql-bundle-osx64.tar.gz.checksum.txt". +// The first capture group is the OS identifier and the second is the compression format. +var releaseAssetNameRegexp = regexp.MustCompile(`^codeql-bundle-([a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)*)\.tar\.(gz|zst)(?:\.checksum\.txt)?$`) + type pullService struct { - ctx context.Context - cacheDirectory cachedirectory.CacheDirectory - gitCloneURL string - githubDotComClient *github.Client - sourceToken string + ctx context.Context + cacheDirectory cachedirectory.CacheDirectory + gitCloneURL string + githubDotComClient *github.Client + sourceToken string + assetOSIncludes []string + assetOSExcludes []string + assetCompressionFormat string +} + +// shouldDownloadAsset determines whether a release asset should be downloaded, based on the +// OS include/exclude lists and compression format configured on the pullService. Assets whose +// name does not match the "codeql-bundle-.tar." naming convention (for example +// "cli-version-X.txt") are not OS- or compression-specific, and are always downloaded. +func (pullService *pullService) shouldDownloadAsset(assetName string) bool { + matches := releaseAssetNameRegexp.FindStringSubmatch(assetName) + if matches == nil { + return true + } + assetOS := matches[1] + assetCompressionFormat := matches[2] + + if len(pullService.assetOSIncludes) > 0 { + included := false + for _, os := range pullService.assetOSIncludes { + if os == assetOS { + included = true + break + } + } + if !included { + return false + } + } + + if len(pullService.assetOSExcludes) > 0 { + for _, os := range pullService.assetOSExcludes { + if os == assetOS { + return false + } + } + } + + if pullService.assetCompressionFormat != "" && pullService.assetCompressionFormat != assetCompressionFormat { + return false + } + + return true } func (pullService *pullService) pullGit(fresh bool) error { @@ -215,6 +263,10 @@ func (pullService *pullService) pullReleases() error { return errors.Wrap(err, "Error creating assets directory.") } for _, asset := range release.Assets { + if !pullService.shouldDownloadAsset(asset.GetName()) { + log.Debugf("Skipping asset %s due to OS/compression format filters.", asset.GetName()) + continue + } log.Debugf("Downloading asset %s...", asset.GetName()) downloadPath := pullService.cacheDirectory.AssetPath(releaseTag, asset.GetName()) downloadPathStat, err := os.Stat(downloadPath) @@ -260,7 +312,7 @@ func (pullService *pullService) pullReleases() error { return nil } -func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string) error { +func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string, assetOSIncludes []string, assetOSExcludes []string, assetCompressionFormat string) error { err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version()) if err != nil { return err @@ -283,11 +335,14 @@ func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sou } pullService := pullService{ - ctx: ctx, - cacheDirectory: cacheDirectory, - gitCloneURL: sourceURL, - githubDotComClient: github.NewClient(tokenClient), - sourceToken: sourceToken, + ctx: ctx, + cacheDirectory: cacheDirectory, + gitCloneURL: sourceURL, + githubDotComClient: github.NewClient(tokenClient), + sourceToken: sourceToken, + assetOSIncludes: assetOSIncludes, + assetOSExcludes: assetOSExcludes, + assetCompressionFormat: assetCompressionFormat, } err = pullService.pullGit(false) diff --git a/internal/pull/pull_test.go b/internal/pull/pull_test.go index 1465078..d6648c9 100644 --- a/internal/pull/pull_test.go +++ b/internal/pull/pull_test.go @@ -46,6 +46,38 @@ var releaseSomeCodeQLVersionOnV1AndV2 = github.RepositoryRelease{ }, } +const releaseWithMultipleOSAssetsLinux64GzContent = "linux64 gz content" +const releaseWithMultipleOSAssetsLinux64ZstContent = "linux64 zst content" +const releaseWithMultipleOSAssetsWin64GzContent = "win64 gz content" +const releaseWithMultipleOSAssetsCliVersionContent = "1.2.3" + +var releaseWithMultipleOSAssets = github.RepositoryRelease{ + TagName: github.String("some-codeql-version-on-main"), + Name: github.String("some-codeql-version-on-main"), + Assets: []*github.ReleaseAsset{ + &github.ReleaseAsset{ + ID: github.Int64(10), + Name: github.String("codeql-bundle-linux64.tar.gz"), + Size: github.Int(len(releaseWithMultipleOSAssetsLinux64GzContent)), + }, + &github.ReleaseAsset{ + ID: github.Int64(11), + Name: github.String("codeql-bundle-linux64.tar.zst"), + Size: github.Int(len(releaseWithMultipleOSAssetsLinux64ZstContent)), + }, + &github.ReleaseAsset{ + ID: github.Int64(12), + Name: github.String("codeql-bundle-win64.tar.gz"), + Size: github.Int(len(releaseWithMultipleOSAssetsWin64GzContent)), + }, + &github.ReleaseAsset{ + ID: github.Int64(13), + Name: github.String("cli-version-1.2.3.txt"), + Size: github.Int(len(releaseWithMultipleOSAssetsCliVersionContent)), + }, + }, +} + func getTestPullService(t *testing.T, temporaryDirectory string, gitCloneURL string, githubURL string) pullService { cacheDirectory := cachedirectory.NewCacheDirectory(temporaryDirectory) var githubDotComClient *github.Client @@ -132,6 +164,44 @@ func TestPullGitNotFreshWithChanges(t *testing.T) { }) } +func TestShouldDownloadAsset(t *testing.T) { + cases := []struct { + name string + assetName string + assetOSIncludes []string + assetOSExcludes []string + assetCompressionFormat string + expected bool + }{ + {"no filters, gz asset", "codeql-bundle-linux64.tar.gz", nil, nil, "", true}, + {"no filters, zst asset", "codeql-bundle-linux64.tar.zst", nil, nil, "", true}, + {"no filters, non-OS asset", "cli-version-1.2.3.txt", nil, nil, "", true}, + {"os-include matches", "codeql-bundle-linux64.tar.gz", []string{"linux64", "win64"}, nil, "", true}, + {"os-include does not match", "codeql-bundle-osx64.tar.gz", []string{"linux64", "win64"}, nil, "", false}, + {"os-include ignores non-OS asset", "cli-version-1.2.3.txt", []string{"linux64"}, nil, "", true}, + {"os-exclude matches", "codeql-bundle-win64.tar.gz", nil, []string{"win64"}, "", false}, + {"os-exclude does not match", "codeql-bundle-linux64.tar.gz", nil, []string{"win64"}, "", true}, + {"os-exclude ignores non-OS asset", "cli-version-1.2.3.txt", nil, []string{"linux64"}, "", true}, + {"compression format matches", "codeql-bundle-linux64.tar.zst", nil, nil, "zst", true}, + {"compression format does not match", "codeql-bundle-linux64.tar.gz", nil, nil, "zst", false}, + {"compression format ignores non-OS asset", "cli-version-1.2.3.txt", nil, nil, "zst", true}, + {"checksum file follows same rules as its asset", "codeql-bundle-linux64.tar.gz.checksum.txt", nil, nil, "zst", false}, + {"os-include and compression format combined", "codeql-bundle-linux-arm64.tar.gz", []string{"linux-arm64"}, nil, "gz", true}, + {"os-include and compression format combined, format mismatch", "codeql-bundle-linux-arm64.tar.zst", []string{"linux-arm64"}, nil, "gz", false}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + pullService := pullService{ + assetOSIncludes: testCase.assetOSIncludes, + assetOSExcludes: testCase.assetOSExcludes, + assetCompressionFormat: testCase.assetCompressionFormat, + } + require.Equal(t, testCase.expected, pullService.shouldDownloadAsset(testCase.assetName)) + }) + } +} + func TestFindRelevantReleases(t *testing.T) { temporaryDirectory := test.CreateTemporaryDirectory(t) pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, "") @@ -190,3 +260,39 @@ func TestPullReleases(t *testing.T) { test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnMainContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle.tar.gz")) test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnV1AndV2Content, pullService.cacheDirectory.AssetPath("some-codeql-version-on-v1-and-v2", "codeql-bundle.tar.gz")) } + +func TestPullReleasesWithOSAndCompressionFilters(t *testing.T) { + temporaryDirectory := test.CreateTemporaryDirectory(t) + githubTestServer, githubURL := test.GetTestHTTPServer(t) + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-main", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromObject(t, releaseWithMultipleOSAssets, response) + }).Methods("GET") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/10", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsLinux64GzContent, response) + }).Methods("GET").Headers("accept", "application/octet-stream") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/13", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsCliVersionContent, response) + }).Methods("GET").Headers("accept", "application/octet-stream") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-v1-and-v2", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromObject(t, releaseSomeCodeQLVersionOnV1AndV2, response) + }).Methods("GET") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/2", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromString(t, releaseSomeCodeQLVersionOnV1AndV2Content, response) + }).Methods("GET").Headers("accept", "application/octet-stream") + + pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, githubURL) + pullService.assetOSIncludes = []string{"linux64"} + pullService.assetCompressionFormat = "gz" + err := pullService.pullGit(true) + require.NoError(t, err) + err = pullService.pullReleases() + require.NoError(t, err) + + // The included OS + compression format asset, and the non-OS-specific asset, should be downloaded. + test.RequireFileHasContent(t, releaseWithMultipleOSAssetsLinux64GzContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.gz")) + test.RequireFileHasContent(t, releaseWithMultipleOSAssetsCliVersionContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "cli-version-1.2.3.txt")) + + // The other OS/compression format combinations should have been skipped entirely. + require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.zst")) + require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-win64.tar.gz")) +} From efe3362aa6dab9a45ba014da3da4948813090fc8 Mon Sep 17 00:00:00 2001 From: John Breault Date: Mon, 21 Sep 2026 19:57:22 -0400 Subject: [PATCH 2/2] Fix import ordering to satisfy goimports lint check Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- cmd/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pull.go b/cmd/pull.go index d282dae..aec44e8 100644 --- a/cmd/pull.go +++ b/cmd/pull.go @@ -1,9 +1,9 @@ package cmd import ( + usererrors "errors" "strings" - usererrors "errors" "github.com/github/codeql-action-sync/internal/cachedirectory" "github.com/github/codeql-action-sync/internal/pull" "github.com/github/codeql-action-sync/internal/version"