Skip to content

Commit fd1c187

Browse files
JohnBreaultJohnBreault
authored
Revert "Add selective release-asset syncing (--os-include, --os-exclude, --compression-format)"
1 parent 27598c1 commit fd1c187

6 files changed

Lines changed: 15 additions & 234 deletions

File tree

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,3 @@
22
/codeql-action-sync
33
/dist/
44
/pkged.go
5-
/releases/

‎README.md‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,19 +29,13 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t
2929
* `--actions-admin-user` - The name of the Actions admin user, which will be used if you are updating the bundled CodeQL Action. If not specified `actions-admin` will be used.
3030
* `--force` - By default the tool will not overwrite existing repositories. Providing this flag will allow it to.
3131
* `--push-ssh` - Push Git contents over SSH rather than HTTPS. To use this option you must have SSH access to your GitHub Enterprise instance configured.
32-
* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced.
33-
* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`.
34-
* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced.
3532

3633
### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server.
3734
From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder.
3835

3936
**Optional Arguments:**
4037
* `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used.
4138
* `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. The token does not need to have any scopes.
42-
* `--os-include` - A comma-separated list of operating systems (e.g. `linux64,win64`) to include CodeQL bundle release assets for. Cannot be used together with `--os-exclude`. If neither is specified, assets for all operating systems are synced.
43-
* `--os-exclude` - A comma-separated list of operating systems (e.g. `win64,osx64`) to exclude CodeQL bundle release assets for. Cannot be used together with `--os-include`.
44-
* `--compression-format` - The compression format of CodeQL bundle release assets to sync, either `gz` or `zst`. If not specified, both compression formats are synced.
4539

4640
Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server.
4741

‎cmd/pull.go‎

Lines changed: 3 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package cmd
22

33
import (
4-
usererrors "errors"
5-
"strings"
6-
74
"github.com/github/codeql-action-sync/internal/cachedirectory"
85
"github.com/github/codeql-action-sync/internal/pull"
96
"github.com/github/codeql-action-sync/internal/version"
@@ -15,65 +12,20 @@ var pullCmd = &cobra.Command{
1512
Short: "Pull the CodeQL Action from GitHub to a local cache.",
1613
RunE: func(cmd *cobra.Command, args []string) error {
1714
version.LogVersion()
18-
if err := pullFlags.Validate(); err != nil {
19-
return err
20-
}
2115
cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
22-
return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat)
16+
return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
2317
},
2418
}
2519

2620
type pullFlagFields struct {
27-
sourceToken string
28-
sourceURL string
29-
osInclude string
30-
osExclude string
31-
compressionFormat string
21+
sourceToken string
22+
sourceURL string
3223
}
3324

3425
var pullFlags = pullFlagFields{}
3526

36-
const errorOSIncludeAndExclude = "You cannot specify both --os-include and --os-exclude at the same time. Please use only one of these flags."
37-
const errorInvalidCompressionFormat = "Invalid --compression-format value. Valid values are \"gz\" or \"zst\"."
38-
3927
func (f *pullFlagFields) Init(cmd *cobra.Command) {
4028
cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.")
4129
cmd.Flags().StringVar(&f.sourceURL, "source-url", "", "Use a custom Git URL for fetching the Action repository contents from. The CodeQL bundles will still be fetched from GitHub.com.")
4230
cmd.Flags().MarkHidden("source-url")
43-
cmd.Flags().StringVar(&f.osInclude, "os-include", "", "A comma-separated list of operating systems (e.g. \"linux64,win64\") to include CodeQL bundle release assets for. Cannot be used together with --os-exclude. If neither is specified, assets for all operating systems are synced.")
44-
cmd.Flags().StringVar(&f.osExclude, "os-exclude", "", "A comma-separated list of operating systems (e.g. \"win64,osx64\") to exclude CodeQL bundle release assets for. Cannot be used together with --os-include.")
45-
cmd.Flags().StringVar(&f.compressionFormat, "compression-format", "", "The compression format of CodeQL bundle release assets to sync, either \"gz\" or \"zst\". If not specified, both compression formats are synced.")
46-
}
47-
48-
func splitCommaSeparatedList(value string) []string {
49-
if value == "" {
50-
return []string{}
51-
}
52-
parts := strings.Split(value, ",")
53-
result := make([]string, 0, len(parts))
54-
for _, part := range parts {
55-
trimmed := strings.TrimSpace(part)
56-
if trimmed != "" {
57-
result = append(result, trimmed)
58-
}
59-
}
60-
return result
61-
}
62-
63-
func (f *pullFlagFields) assetOSIncludes() []string {
64-
return splitCommaSeparatedList(f.osInclude)
65-
}
66-
67-
func (f *pullFlagFields) assetOSExcludes() []string {
68-
return splitCommaSeparatedList(f.osExclude)
69-
}
70-
71-
func (f *pullFlagFields) Validate() error {
72-
if f.osInclude != "" && f.osExclude != "" {
73-
return usererrors.New(errorOSIncludeAndExclude)
74-
}
75-
if f.compressionFormat != "" && f.compressionFormat != "gz" && f.compressionFormat != "zst" {
76-
return usererrors.New(errorInvalidCompressionFormat)
77-
}
78-
return nil
7931
}

‎cmd/sync.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,8 @@ var syncCmd = &cobra.Command{
1313
Short: "Sync the CodeQL Action from GitHub to a GitHub Enterprise Server installation.",
1414
RunE: func(cmd *cobra.Command, args []string) error {
1515
version.LogVersion()
16-
if err := pullFlags.Validate(); err != nil {
17-
return err
18-
}
1916
cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
20-
err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.assetOSIncludes(), pullFlags.assetOSExcludes(), pullFlags.compressionFormat)
17+
err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
2118
if err != nil {
2219
return err
2320
}

‎internal/pull/pull.go‎

Lines changed: 11 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -36,60 +36,12 @@ var relevantReferences = regexp.MustCompile("^refs/(heads|tags)/(main|v\\d+)$")
3636

3737
const defaultConfigurationPath = "src/defaults.json"
3838

39-
// Matches release asset names like "codeql-bundle-linux64.tar.gz",
40-
// "codeql-bundle-linux-arm64.tar.zst" or "codeql-bundle-osx64.tar.gz.checksum.txt".
41-
// The first capture group is the OS identifier and the second is the compression format.
42-
var releaseAssetNameRegexp = regexp.MustCompile(`^codeql-bundle-([a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)*)\.tar\.(gz|zst)(?:\.checksum\.txt)?$`)
43-
4439
type pullService struct {
45-
ctx context.Context
46-
cacheDirectory cachedirectory.CacheDirectory
47-
gitCloneURL string
48-
githubDotComClient *github.Client
49-
sourceToken string
50-
assetOSIncludes []string
51-
assetOSExcludes []string
52-
assetCompressionFormat string
53-
}
54-
55-
// shouldDownloadAsset determines whether a release asset should be downloaded, based on the
56-
// OS include/exclude lists and compression format configured on the pullService. Assets whose
57-
// name does not match the "codeql-bundle-<os>.tar.<gz|zst>" naming convention (for example
58-
// "cli-version-X.txt") are not OS- or compression-specific, and are always downloaded.
59-
func (pullService *pullService) shouldDownloadAsset(assetName string) bool {
60-
matches := releaseAssetNameRegexp.FindStringSubmatch(assetName)
61-
if matches == nil {
62-
return true
63-
}
64-
assetOS := matches[1]
65-
assetCompressionFormat := matches[2]
66-
67-
if len(pullService.assetOSIncludes) > 0 {
68-
included := false
69-
for _, os := range pullService.assetOSIncludes {
70-
if os == assetOS {
71-
included = true
72-
break
73-
}
74-
}
75-
if !included {
76-
return false
77-
}
78-
}
79-
80-
if len(pullService.assetOSExcludes) > 0 {
81-
for _, os := range pullService.assetOSExcludes {
82-
if os == assetOS {
83-
return false
84-
}
85-
}
86-
}
87-
88-
if pullService.assetCompressionFormat != "" && pullService.assetCompressionFormat != assetCompressionFormat {
89-
return false
90-
}
91-
92-
return true
40+
ctx context.Context
41+
cacheDirectory cachedirectory.CacheDirectory
42+
gitCloneURL string
43+
githubDotComClient *github.Client
44+
sourceToken string
9345
}
9446

9547
func (pullService *pullService) pullGit(fresh bool) error {
@@ -263,10 +215,6 @@ func (pullService *pullService) pullReleases() error {
263215
return errors.Wrap(err, "Error creating assets directory.")
264216
}
265217
for _, asset := range release.Assets {
266-
if !pullService.shouldDownloadAsset(asset.GetName()) {
267-
log.Debugf("Skipping asset %s due to OS/compression format filters.", asset.GetName())
268-
continue
269-
}
270218
log.Debugf("Downloading asset %s...", asset.GetName())
271219
downloadPath := pullService.cacheDirectory.AssetPath(releaseTag, asset.GetName())
272220
downloadPathStat, err := os.Stat(downloadPath)
@@ -312,7 +260,7 @@ func (pullService *pullService) pullReleases() error {
312260
return nil
313261
}
314262

315-
func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string, assetOSIncludes []string, assetOSExcludes []string, assetCompressionFormat string) error {
263+
func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string) error {
316264
err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version())
317265
if err != nil {
318266
return err
@@ -335,14 +283,11 @@ func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sou
335283
}
336284

337285
pullService := pullService{
338-
ctx: ctx,
339-
cacheDirectory: cacheDirectory,
340-
gitCloneURL: sourceURL,
341-
githubDotComClient: github.NewClient(tokenClient),
342-
sourceToken: sourceToken,
343-
assetOSIncludes: assetOSIncludes,
344-
assetOSExcludes: assetOSExcludes,
345-
assetCompressionFormat: assetCompressionFormat,
286+
ctx: ctx,
287+
cacheDirectory: cacheDirectory,
288+
gitCloneURL: sourceURL,
289+
githubDotComClient: github.NewClient(tokenClient),
290+
sourceToken: sourceToken,
346291
}
347292

348293
err = pullService.pullGit(false)

‎internal/pull/pull_test.go‎

Lines changed: 0 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -46,38 +46,6 @@ var releaseSomeCodeQLVersionOnV1AndV2 = github.RepositoryRelease{
4646
},
4747
}
4848

49-
const releaseWithMultipleOSAssetsLinux64GzContent = "linux64 gz content"
50-
const releaseWithMultipleOSAssetsLinux64ZstContent = "linux64 zst content"
51-
const releaseWithMultipleOSAssetsWin64GzContent = "win64 gz content"
52-
const releaseWithMultipleOSAssetsCliVersionContent = "1.2.3"
53-
54-
var releaseWithMultipleOSAssets = github.RepositoryRelease{
55-
TagName: github.String("some-codeql-version-on-main"),
56-
Name: github.String("some-codeql-version-on-main"),
57-
Assets: []*github.ReleaseAsset{
58-
&github.ReleaseAsset{
59-
ID: github.Int64(10),
60-
Name: github.String("codeql-bundle-linux64.tar.gz"),
61-
Size: github.Int(len(releaseWithMultipleOSAssetsLinux64GzContent)),
62-
},
63-
&github.ReleaseAsset{
64-
ID: github.Int64(11),
65-
Name: github.String("codeql-bundle-linux64.tar.zst"),
66-
Size: github.Int(len(releaseWithMultipleOSAssetsLinux64ZstContent)),
67-
},
68-
&github.ReleaseAsset{
69-
ID: github.Int64(12),
70-
Name: github.String("codeql-bundle-win64.tar.gz"),
71-
Size: github.Int(len(releaseWithMultipleOSAssetsWin64GzContent)),
72-
},
73-
&github.ReleaseAsset{
74-
ID: github.Int64(13),
75-
Name: github.String("cli-version-1.2.3.txt"),
76-
Size: github.Int(len(releaseWithMultipleOSAssetsCliVersionContent)),
77-
},
78-
},
79-
}
80-
8149
func getTestPullService(t *testing.T, temporaryDirectory string, gitCloneURL string, githubURL string) pullService {
8250
cacheDirectory := cachedirectory.NewCacheDirectory(temporaryDirectory)
8351
var githubDotComClient *github.Client
@@ -164,44 +132,6 @@ func TestPullGitNotFreshWithChanges(t *testing.T) {
164132
})
165133
}
166134

167-
func TestShouldDownloadAsset(t *testing.T) {
168-
cases := []struct {
169-
name string
170-
assetName string
171-
assetOSIncludes []string
172-
assetOSExcludes []string
173-
assetCompressionFormat string
174-
expected bool
175-
}{
176-
{"no filters, gz asset", "codeql-bundle-linux64.tar.gz", nil, nil, "", true},
177-
{"no filters, zst asset", "codeql-bundle-linux64.tar.zst", nil, nil, "", true},
178-
{"no filters, non-OS asset", "cli-version-1.2.3.txt", nil, nil, "", true},
179-
{"os-include matches", "codeql-bundle-linux64.tar.gz", []string{"linux64", "win64"}, nil, "", true},
180-
{"os-include does not match", "codeql-bundle-osx64.tar.gz", []string{"linux64", "win64"}, nil, "", false},
181-
{"os-include ignores non-OS asset", "cli-version-1.2.3.txt", []string{"linux64"}, nil, "", true},
182-
{"os-exclude matches", "codeql-bundle-win64.tar.gz", nil, []string{"win64"}, "", false},
183-
{"os-exclude does not match", "codeql-bundle-linux64.tar.gz", nil, []string{"win64"}, "", true},
184-
{"os-exclude ignores non-OS asset", "cli-version-1.2.3.txt", nil, []string{"linux64"}, "", true},
185-
{"compression format matches", "codeql-bundle-linux64.tar.zst", nil, nil, "zst", true},
186-
{"compression format does not match", "codeql-bundle-linux64.tar.gz", nil, nil, "zst", false},
187-
{"compression format ignores non-OS asset", "cli-version-1.2.3.txt", nil, nil, "zst", true},
188-
{"checksum file follows same rules as its asset", "codeql-bundle-linux64.tar.gz.checksum.txt", nil, nil, "zst", false},
189-
{"os-include and compression format combined", "codeql-bundle-linux-arm64.tar.gz", []string{"linux-arm64"}, nil, "gz", true},
190-
{"os-include and compression format combined, format mismatch", "codeql-bundle-linux-arm64.tar.zst", []string{"linux-arm64"}, nil, "gz", false},
191-
}
192-
193-
for _, testCase := range cases {
194-
t.Run(testCase.name, func(t *testing.T) {
195-
pullService := pullService{
196-
assetOSIncludes: testCase.assetOSIncludes,
197-
assetOSExcludes: testCase.assetOSExcludes,
198-
assetCompressionFormat: testCase.assetCompressionFormat,
199-
}
200-
require.Equal(t, testCase.expected, pullService.shouldDownloadAsset(testCase.assetName))
201-
})
202-
}
203-
}
204-
205135
func TestFindRelevantReleases(t *testing.T) {
206136
temporaryDirectory := test.CreateTemporaryDirectory(t)
207137
pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, "")
@@ -260,39 +190,3 @@ func TestPullReleases(t *testing.T) {
260190
test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnMainContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle.tar.gz"))
261191
test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnV1AndV2Content, pullService.cacheDirectory.AssetPath("some-codeql-version-on-v1-and-v2", "codeql-bundle.tar.gz"))
262192
}
263-
264-
func TestPullReleasesWithOSAndCompressionFilters(t *testing.T) {
265-
temporaryDirectory := test.CreateTemporaryDirectory(t)
266-
githubTestServer, githubURL := test.GetTestHTTPServer(t)
267-
githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-main", func(response http.ResponseWriter, request *http.Request) {
268-
test.ServeHTTPResponseFromObject(t, releaseWithMultipleOSAssets, response)
269-
}).Methods("GET")
270-
githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/10", func(response http.ResponseWriter, request *http.Request) {
271-
test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsLinux64GzContent, response)
272-
}).Methods("GET").Headers("accept", "application/octet-stream")
273-
githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/13", func(response http.ResponseWriter, request *http.Request) {
274-
test.ServeHTTPResponseFromString(t, releaseWithMultipleOSAssetsCliVersionContent, response)
275-
}).Methods("GET").Headers("accept", "application/octet-stream")
276-
githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-v1-and-v2", func(response http.ResponseWriter, request *http.Request) {
277-
test.ServeHTTPResponseFromObject(t, releaseSomeCodeQLVersionOnV1AndV2, response)
278-
}).Methods("GET")
279-
githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/2", func(response http.ResponseWriter, request *http.Request) {
280-
test.ServeHTTPResponseFromString(t, releaseSomeCodeQLVersionOnV1AndV2Content, response)
281-
}).Methods("GET").Headers("accept", "application/octet-stream")
282-
283-
pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, githubURL)
284-
pullService.assetOSIncludes = []string{"linux64"}
285-
pullService.assetCompressionFormat = "gz"
286-
err := pullService.pullGit(true)
287-
require.NoError(t, err)
288-
err = pullService.pullReleases()
289-
require.NoError(t, err)
290-
291-
// The included OS + compression format asset, and the non-OS-specific asset, should be downloaded.
292-
test.RequireFileHasContent(t, releaseWithMultipleOSAssetsLinux64GzContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.gz"))
293-
test.RequireFileHasContent(t, releaseWithMultipleOSAssetsCliVersionContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "cli-version-1.2.3.txt"))
294-
295-
// The other OS/compression format combinations should have been skipped entirely.
296-
require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.zst"))
297-
require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-win64.tar.gz"))
298-
}

0 commit comments

Comments
 (0)