Add selective release-asset syncing (--os-include, --os-exclude, --compression-format)

.gitignore

@@ -2,3 +2,4 @@
 /codeql-action-sync
 /dist/
 /pkged.go
+/releases/

README.md

@@ -29,13 +29,19 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t
 * `--actions-admin-user` - The name of the Actions admin user, which will be used if you are updating the bundled CodeQL Action. If not specified `actions-admin` will be used.
 * `--force` - By default the tool will not overwrite existing repositories. Providing this flag will allow it to.
 * `--push-ssh` - Push Git contents over SSH rather than HTTPS. To use this option you must have SSH access to your GitHub Enterprise instance configured.
+* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced.
+* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`.
+* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced.
 
 ### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server.
 From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder.
 
 **Optional Arguments:**
 * `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used.
 * `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. The token does not need to have any scopes.
+* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced.
+* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`.
+* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced.
 
 Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server.
 

cmd/pull.go

@@ -1,6 +1,9 @@
 package cmd
 
 import (
+	usererrors "errors"
+	"strings"
+
 	"github.com/github/codeql-action-sync/internal/cachedirectory"
 	"github.com/github/codeql-action-sync/internal/pull"
 	"github.com/github/codeql-action-sync/internal/version"
@@ -12,20 +15,65 @@ var pullCmd = &cobra.Command{
 	Short: "Pull the CodeQL Action from GitHub to a local cache.",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
+		if err := pullFlags.Validate(); err != nil {
+			return err
+		}
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
+		return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat)
 	},
 }
 
 type pullFlagFields struct {
-	sourceToken string
-	sourceURL   string
+	sourceToken       string
+	sourceURL         string
+	osInclude         string
+	osExclude         string
+	compressionFormat string
 }
 
 var pullFlags = pullFlagFields{}
 
+const errorOSIncludeAndExclude = "You cannot specify both --os-include and --os-exclude at the same time. Please use only one of these flags."
+const errorInvalidCompressionFormat = "Invalid --compression-format value. Valid values are \"gz\" or \"zst\"."
+
 func (f *pullFlagFields) Init(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.")
 	cmd.Flags().StringVar(&f.sourceURL, "source-url", "", "Use a custom Git URL for fetching the Action repository contents from. The CodeQL bundles will still be fetched from GitHub.com.")
 	cmd.Flags().MarkHidden("source-url")
+	cmd.Flags().StringVar(&f.osInclude, "os-include", "", "A comma-separated list of operating systems (e.g. \"linux64,win64\") to include CodeQL bundle release assets for. Cannot be used together with --os-exclude. If neither is specified, assets for all operating systems are synced.")
+	cmd.Flags().StringVar(&f.osExclude, "os-exclude", "", "A comma-separated list of operating systems (e.g. \"win64,osx64\") to exclude CodeQL bundle release assets for. Cannot be used together with --os-include.")
+	cmd.Flags().StringVar(&f.compressionFormat, "compression-format", "", "The compression format of CodeQL bundle release assets to sync, either \"gz\" or \"zst\". If not specified, both compression formats are synced.")
+}
+
+func splitCommaSeparatedList(value string) []string {
+	if value == "" {
+		return []string{}
+	}
+	parts := strings.Split(value, ",")
+	result := make([]string, 0, len(parts))
+	for _, part := range parts {
+		trimmed := strings.TrimSpace(part)
+		if trimmed != "" {
+			result = append(result, trimmed)
+		}
+	}
+	return result
+}
+
+func (f *pullFlagFields) assetOSIncludes() []string {
+	return splitCommaSeparatedList(f.osInclude)
+}
+
+func (f *pullFlagFields) assetOSExcludes() []string {
+	return splitCommaSeparatedList(f.osExclude)
+}
+
+func (f *pullFlagFields) Validate() error {
+	if f.osInclude != "" && f.osExclude != "" {
+		return usererrors.New(errorOSIncludeAndExclude)
+	}
+	if f.compressionFormat != "" && f.compressionFormat != "gz" && f.compressionFormat != "zst" {
+		return usererrors.New(errorInvalidCompressionFormat)
+	}
+	return nil
 }

cmd/sync.go

@@ -13,8 +13,11 @@ var syncCmd = &cobra.Command{
 	Short: "Sync the CodeQL Action from GitHub to a GitHub Enterprise Server installation.",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
+		if err := pullFlags.Validate(); err != nil {
+			return err
+		}
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
+		err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat)
 		if err != nil {
 			return err
 		}

internal/pull/pull.go

@@ -36,12 +36,60 @@ var relevantReferences = regexp.MustCompile("^refs/(heads|tags)/(main|v\\d+)$")
 
 const defaultConfigurationPath = "src/defaults.json"
 
+// Matches release asset names like "codeql-bundle-linux64.tar.gz",
+// "codeql-bundle-linux-arm64.tar.zst" or "codeql-bundle-osx64.tar.gz.checksum.txt".
+// The first capture group is the OS identifier and the second is the compression format.
+var releaseAssetNameRegexp = regexp.MustCompile(`^codeql-bundle-([a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)*)\.tar\.(gz|zst)(?:\.checksum\.txt)?$`)
+
 type pullService struct {
-	ctx                context.Context
-	cacheDirectory     cachedirectory.CacheDirectory
-	gitCloneURL        string
-	githubDotComClient *github.Client
-	sourceToken        string
+	ctx                    context.Context
+	cacheDirectory         cachedirectory.CacheDirectory
+	gitCloneURL            string
+	githubDotComClient     *github.Client
+	sourceToken            string
+	assetOSIncludes        []string
+	assetOSExcludes        []string
+	assetCompressionFormat string
+}
+
+// shouldDownloadAsset determines whether a release asset should be downloaded, based on the
+// OS include/exclude lists and compression format configured on the pullService. Assets whose
+// name does not match the "codeql-bundle-<os>.tar.<gz|zst>" naming convention (for example
+// "cli-version-X.txt") are not OS- or compression-specific, and are always downloaded.
+func (pullService *pullService) shouldDownloadAsset(assetName string) bool {
+	matches := releaseAssetNameRegexp.FindStringSubmatch(assetName)
+	if matches == nil {
+		return true
+	}
+	assetOS := matches[1]
+	assetCompressionFormat := matches[2]
+
+	if len(pullService.assetOSIncludes) > 0 {
+		included := false
+		for _, os := range pullService.assetOSIncludes {
+			if os == assetOS {
+				included = true
+				break
+			}
+		}
+		if !included {
+			return false
+		}
+	}
+
+	if len(pullService.assetOSExcludes) > 0 {
+		for _, os := range pullService.assetOSExcludes {
+			if os == assetOS {
+				return false
+			}
+		}
+	}
+
+	if pullService.assetCompressionFormat != "" && pullService.assetCompressionFormat != assetCompressionFormat {
+		return false
+	}
+
+	return true
 }
 
 func (pullService *pullService) pullGit(fresh bool) error {
@@ -215,6 +263,10 @@ func (pullService *pullService) pullReleases() error {
 			return errors.Wrap(err, "Error creating assets directory.")
 		}
 		for _, asset := range release.Assets {
+			if !pullService.shouldDownloadAsset(asset.GetName()) {
+				log.Debugf("Skipping asset %s due to OS/compression format filters.", asset.GetName())
+				continue
+			}
 			log.Debugf("Downloading asset %s...", asset.GetName())
 			downloadPath := pullService.cacheDirectory.AssetPath(releaseTag, asset.GetName())
 			downloadPathStat, err := os.Stat(downloadPath)
@@ -260,7 +312,7 @@ func (pullService *pullService) pullReleases() error {
 	return nil
 }
 
-func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string) error {
+func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string, assetOSIncludes []string, assetOSExcludes []string, assetCompressionFormat string) error {
 	err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version())
 	if err != nil {
 		return err
@@ -283,11 +335,14 @@ func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sou
 	}
 
 	pullService := pullService{
-		ctx:                ctx,
-		cacheDirectory:     cacheDirectory,
-		gitCloneURL:        sourceURL,
-		githubDotComClient: github.NewClient(tokenClient),
-		sourceToken:        sourceToken,
+		ctx:                    ctx,
+		cacheDirectory:         cacheDirectory,
+		gitCloneURL:            sourceURL,
+		githubDotComClient:     github.NewClient(tokenClient),
+		sourceToken:            sourceToken,
+		assetOSIncludes:        assetOSIncludes,
+		assetOSExcludes:        assetOSExcludes,
+		assetCompressionFormat: assetCompressionFormat,
 	}
 
 	err = pullService.pullGit(false)

internal/pull/pull_test.go

@@ -46,6 +46,38 @@ var releaseSomeCodeQLVersionOnV1AndV2 = github.RepositoryRelease{
 	},
 }
 
+const releaseWithMultipleOSAssetsLinux64GzContent = "linux64 gz content"
+const releaseWithMultipleOSAssetsLinux64ZstContent = "linux64 zst content"
+const releaseWithMultipleOSAssetsWin64GzContent = "win64 gz content"
+const releaseWithMultipleOSAssetsCliVersionContent = "1.2.3"
+
+var releaseWithMultipleOSAssets = github.RepositoryRelease{
+	TagName: github.String("some-codeql-version-on-main"),
+	Name:    github.String("some-codeql-version-on-main"),
+	Assets: []*github.ReleaseAsset{
+		&github.ReleaseAsset{
+			ID:   github.Int64(10),
+			Name: github.String("codeql-bundle-linux64.tar.gz"),
+			Size: github.Int(len(releaseWithMultipleOSAssetsLinux64GzContent)),
+		},
+		&github.ReleaseAsset{
+			ID:   github.Int64(11),
+			Name: github.String("codeql-bundle-linux64.tar.zst"),
+			Size: github.Int(len(releaseWithMultipleOSAssetsLinux64ZstContent)),
+		},
+		&github.ReleaseAsset{
+			ID:   github.Int64(12),
+			Name: github.String("codeql-bundle-win64.tar.gz"),
+			Size: github.Int(len(releaseWithMultipleOSAssetsWin64GzContent)),
+		},
+		&github.ReleaseAsset{
+			ID:   github.Int64(13),
+			Name: github.String("cli-version-1.2.3.txt"),
+			Size: github.Int(len(releaseWithMultipleOSAssetsCliVersionContent)),
+		},
+	},
+}
+
 func getTestPullService(t *testing.T, temporaryDirectory string, gitCloneURL string, githubURL string) pullService {
 	cacheDirectory := cachedirectory.NewCacheDirectory(temporaryDirectory)
 	var githubDotComClient *github.Client
@@ -132,6 +164,44 @@ func TestPullGitNotFreshWithChanges(t *testing.T) {
 	})
 }
 
+func TestShouldDownloadAsset(t *testing.T) {
+	cases := []struct {
+		name                   string
+		assetName              string
+		assetOSIncludes        []string
+		assetOSExcludes        []string
+		assetCompressionFormat string
+		expected               bool
+	}{
+		{"no filters, gz asset", "codeql-bundle-linux64.tar.gz", nil, nil, "", true},
+		{"no filters, zst asset", "codeql-bundle-linux64.tar.zst", nil, nil, "", true},
+		{"no filters, non-OS asset", "cli-version-1.2.3.txt", nil, nil, "", true},
+		{"os-include matches", "codeql-bundle-linux64.tar.gz", []string{"linux64", "win64"}, nil, "", true},
+		{"os-include does not match", "codeql-bundle-osx64.tar.gz", []string{"linux64", "win64"}, nil, "", false},
+		{"os-include ignores non-OS asset", "cli-version-1.2.3.txt", []string{"linux64"}, nil, "", true},
+		{"os-exclude matches", "codeql-bundle-win64.tar.gz", nil, []string{"win64"}, "", false},
+		{"os-exclude does not match", "codeql-bundle-linux64.tar.gz", nil, []string{"win64"}, "", true},
+		{"os-exclude ignores non-OS asset", "cli-version-1.2.3.txt", nil, []string{"linux64"}, "", true},
+		{"compression format matches", "codeql-bundle-linux64.tar.zst", nil, nil, "zst", true},
+		{"compression format does not match", "codeql-bundle-linux64.tar.gz", nil, nil, "zst", false},
+		{"compression format ignores non-OS asset", "cli-version-1.2.3.txt", nil, nil, "zst", true},
+		{"checksum file follows same rules as its asset", "codeql-bundle-linux64.tar.gz.checksum.txt", nil, nil, "zst", false},
+		{"os-include and compression format combined", "codeql-bundle-linux-arm64.tar.gz", []string{"linux-arm64"}, nil, "gz", true},
+		{"os-include and compression format combined, format mismatch", "codeql-bundle-linux-arm64.tar.zst", []string{"linux-arm64"}, nil, "gz", false},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.name, func(t *testing.T) {
+			pullService := pullService{
+				assetOSIncludes:        testCase.assetOSIncludes,
+				assetOSExcludes:        testCase.assetOSExcludes,
+				assetCompressionFormat: testCase.assetCompressionFormat,
+			}
+			require.Equal(t, testCase.expected, pullService.shouldDownloadAsset(testCase.assetName))
+		})
+	}
+}
+
 func TestFindRelevantReleases(t *testing.T) {
 	temporaryDirectory := test.CreateTemporaryDirectory(t)
 	pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, "")
@@ -190,3 +260,39 @@ func TestPullReleases(t *testing.T) {
 	test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnMainContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle.tar.gz"))
 	test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnV1AndV2Content, pullService.cacheDirectory.AssetPath("some-codeql-version-on-v1-and-v2", "codeql-bundle.tar.gz"))
 }
+
+func TestPullReleasesWithOSAndCompressionFilters(t *testing.T) {
+	temporaryDirectory := test.CreateTemporaryDirectory(t)
+	githubTestServer, githubURL := test.GetTestHTTPServer(t)
+	githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-main", func(response http.ResponseWriter, request *http.Request) {
+		test.ServeHTTPResponseFromObject(t, releaseWithMultipleOSAssets, response)
+	}).Methods("GET")
+	githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/10", func(response http.ResponseWriter, request *http.Request) {
+		test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsLinux64GzContent, response)
+	}).Methods("GET").Headers("accept", "application/octet-stream")
+	githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/13", func(response http.ResponseWriter, request *http.Request) {
+		test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsCliVersionContent, response)
+	}).Methods("GET").Headers("accept", "application/octet-stream")
+	githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-v1-and-v2", func(response http.ResponseWriter, request *http.Request) {
+		test.ServeHTTPResponseFromObject(t, releaseSomeCodeQLVersionOnV1AndV2, response)
+	}).Methods("GET")
+	githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/2", func(response http.ResponseWriter, request *http.Request) {
+		test.ServeHTTPResponseFromString(t, releaseSomeCodeQLVersionOnV1AndV2Content, response)
+	}).Methods("GET").Headers("accept", "application/octet-stream")
+
+	pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, githubURL)
+	pullService.assetOSIncludes = []string{"linux64"}
+	pullService.assetCompressionFormat = "gz"
+	err := pullService.pullGit(true)
+	require.NoError(t, err)
+	err = pullService.pullReleases()
+	require.NoError(t, err)
+
+	// The included OS + compression format asset, and the non-OS-specific asset, should be downloaded.
+	test.RequireFileHasContent(t, releaseWithMultipleOSAssetsLinux64GzContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.gz"))
+	test.RequireFileHasContent(t, releaseWithMultipleOSAssetsCliVersionContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "cli-version-1.2.3.txt"))
+
+	// The other OS/compression format combinations should have been skipped entirely.
+	require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.zst"))
+	require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-win64.tar.gz"))
+}