diff --git a/.gitignore b/.gitignore index 7f2500e..c679217 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ vendor bin tmp artifacts +**/*.code-workspace diff --git a/VERSION b/VERSION index d8f327b..c78e2a1 100644 --- a/VERSION +++ b/VERSION @@ -1,7 +1,8 @@ -v1.9.0 +v1.10.0 +Support repos with no launch.yaml Move k8s deploys to platform events Previously: +- Move k8s deploys to platform events - Add SyncEntity call to k8s deployApps command - Support configurable deployment branches -- Add new deploy-apps command diff --git a/cmd/goci/README.md b/cmd/goci/README.md index 2e8ef4e..25141f4 100644 --- a/cmd/goci/README.md +++ b/cmd/goci/README.md @@ -4,11 +4,18 @@ ## Configuration -goci accepts very limited arguments which merely change the mode it runs in. The rest of the configuration is entirely through environment variables and launch config settings. See the[environment](../../internal/environment/environment.go) package for detailed documentation of environment variables for configuration. goci reads it's configuration from the `build` section of the launch config of each application. See the [build section](https://github.com/Clever/catapult/blob/master/swagger.yml#L1773) of the launch yaml to learn about the various parameters which configure goci. +goci accepts very limited arguments which merely change the mode it runs in. The rest of the configuration is entirely through environment variables and app config files. See the [environment](../../internal/environment/environment.go) package for detailed documentation of environment variables for configuration. + +goci supports two config paths: + +- **Kubernetes apps** (preferred): `config//stack.yaml` — goci reads the `build` block and determines run type from `stack.helm.chart` (`clever-lambda` → lambda, anything else → docker). +- **Catapult apps** (legacy): `launch/.yml` — goci reads from the `build` section of the launch config of each application. See the [build section](https://github.com/Clever/catapult/blob/master/swagger.yml#L1773) of the launch yaml to learn about the various parameters which configure goci. + +goci reads from **both** paths and merges the results. If the same app appears in both `config/` and `launch/`, the Kubernetes config wins. ## Modes -1. `goci detect` detects any changed applications according to their launch configuration. This can be used to pass a name of apps to another script. +1. `goci detect` detects any changed applications according to their stack.yaml or launch.yaml configuration. This can be used to pass a name of apps to another script. 2. `goci artifact-build-publish-deploy` builds, publishes and deploys any application artifacts. 3. `goci validate` validates an applications go version, while also checking for compatible branch naming conventions for catapult. 4. `goci publish-utility` publishes catalog-info.yaml to the service catalog. @@ -16,8 +23,7 @@ goci accepts very limited arguments which merely change the mode it runs in. The ## Multi-app Support -goci will automatically detect all launch configs in the `launch` -directory, then perform the following actions as needed. +goci will automatically detect all applications in the `config` directory (falling back to `launch` for legacy repos), then perform the following actions as needed. 1. detect the run type of the application 2. Run any configured build commands @@ -25,8 +31,8 @@ directory, then perform the following actions as needed. 4. publish all built docker images to all ECR regions 5. publish all lambdas to s3 in all regions. 6. Sync all changed apps with catalog config -7. Publish new application versions to catapult -8. Deploy any changed applications. +7. [launch.yaml only]Publish new application versions to catapult +8. [launch.yaml only] Deploy any changed applications. ## Development diff --git a/cmd/goci/main.go b/cmd/goci/main.go index eb01be1..8a3d55d 100644 --- a/cmd/goci/main.go +++ b/cmd/goci/main.go @@ -9,7 +9,6 @@ import ( "regexp" "strings" - "github.com/Clever/catapult/gen-go/models" "github.com/Clever/ci-scripts/internal/backstage" "github.com/Clever/ci-scripts/internal/catalogsync" "github.com/Clever/ci-scripts/internal/catapult" @@ -53,13 +52,13 @@ func main() { } func run(mode string) error { - var apps map[string]*models.LaunchConfig + var apps map[string]*repo.AppConfig var appIDs []string var err error // Only discover applications for specific modes if mode == "validate" || mode == "detect" || mode == "artifact-build-publish-deploy" || mode == "deploy-apps" { - apps, err = repo.DiscoverApplications("./launch") + apps, err = repo.DiscoverApplications() if err != nil { return err } @@ -91,7 +90,7 @@ func run(mode string) error { if len(apps) == 0 { fmt.Println("No applications have buildable changes. If this is unexpected, " + - "double check your artifact dependency configuration in the launch yaml.") + "double check your artifact dependency configuration in the launch.yaml or stack.yaml.") return nil } @@ -143,16 +142,25 @@ func run(mode string) error { } } } - cp := catapult.New() - - if err = cp.Publish(ctx, artifacts); err != nil { - return err + catapultArtifacts := make([]*catapult.Artifact, 0, len(artifacts)) + catapultAppIDs := make([]string, 0, len(appIDs)) + for _, art := range artifacts { + if ac, ok := apps[art.ID]; ok && !ac.IsKubernetes { + catapultArtifacts = append(catapultArtifacts, art) + catapultAppIDs = append(catapultAppIDs, art.ID) + } } - if environment.Branch() == "master" { - if err := cp.Deploy(ctx, appIDs); err != nil { + if len(catapultArtifacts) > 0 { + cp := catapult.New() + if err = cp.Publish(ctx, catapultArtifacts); err != nil { return err } + if environment.Branch() == "master" { + if err := cp.Deploy(ctx, catapultAppIDs); err != nil { + return err + } + } } // We want to validate on every run, not just when the mode is "validate". @@ -254,7 +262,7 @@ func fetchLatestGoVersion() (string, string, error) { } // allAppsBuilt returns an error if any apps are missing a build artifact. -func allAppsBuilt(discoveredApps map[string]*models.LaunchConfig, builtApps []*catapult.Artifact) error { +func allAppsBuilt(discoveredApps map[string]*repo.AppConfig, builtApps []*catapult.Artifact) error { if len(discoveredApps) == len(builtApps) { return nil } @@ -303,7 +311,7 @@ func publishUtility() error { func deployApps(appIds []string) error { if len(appIds) == 0 { fmt.Println("No applications have buildable changes. If this is unexpected, " + - "double check your artifact dependency configuration in the launch yaml.") + "double check your artifact dependency configuration in the launch yaml or stack.yaml.") return nil } ctx := context.Background() diff --git a/go.mod b/go.mod index 1171d7f..b5fb581 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 github.com/docker/docker v23.0.2+incompatible + github.com/getkin/kin-openapi v0.139.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/moby/buildkit v0.11.5 + github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.19.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -40,11 +42,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect github.com/aws/smithy-go v1.26.0 // indirect github.com/containerd/containerd v1.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect - github.com/getkin/kin-openapi v0.139.0 // indirect github.com/go-openapi/analysis v0.24.1 // indirect github.com/go-openapi/errors v0.22.4 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect @@ -83,6 +85,7 @@ require ( github.com/opencontainers/runc v1.1.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/speakeasy-api/jsonpath v0.6.3 // indirect diff --git a/internal/docker/targets.go b/internal/docker/targets.go index 6c39e8c..9d2084d 100644 --- a/internal/docker/targets.go +++ b/internal/docker/targets.go @@ -3,7 +3,6 @@ package docker import ( "fmt" - "github.com/Clever/catapult/gen-go/models" "github.com/Clever/ci-scripts/internal/catapult" "github.com/Clever/ci-scripts/internal/environment" "github.com/Clever/ci-scripts/internal/repo" @@ -24,7 +23,7 @@ type DockerTarget struct { // Dockerfile and its set of tags will be in the final list. This is an // optimization so we do not build multiple copies of the same // Dockerfile which only differ at runtime. -func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget, []*catapult.Artifact) { +func BuildTargets(apps map[string]*repo.AppConfig) (map[string]DockerTarget, []*catapult.Artifact) { var ( targets = map[string]DockerTarget{} done = map[string]struct{}{} @@ -36,9 +35,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget continue } - artifact := repo.ArtifactName(name, launch) + artifact := launch.ArtifactName artifacts = append(artifacts, &catapult.Artifact{ - RunType: string(models.RunTypeDocker), + RunType: launch.RunType, ID: name, Branch: environment.Branch(), Source: fmt.Sprintf("github:Clever/%s@%s", environment.Repo(), environment.FullSHA1()), @@ -63,9 +62,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget ) tags = append(tags, tag) - targets[repo.Dockerfile(launch)] = DockerTarget{ + targets[launch.Dockerfile] = DockerTarget{ Tags: tags, - Command: repo.BuildCommand(launch), + Command: launch.BuildCommand, } } return targets, artifacts diff --git a/internal/lambda/targets.go b/internal/lambda/targets.go index 78403ee..11e73cb 100644 --- a/internal/lambda/targets.go +++ b/internal/lambda/targets.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/Clever/catapult/gen-go/models" "github.com/Clever/ci-scripts/internal/catapult" "github.com/Clever/ci-scripts/internal/environment" "github.com/Clever/ci-scripts/internal/repo" @@ -26,7 +25,7 @@ type LambdaTarget struct { // destination zip file in the value struct. Any apps with a shared // artifact will have only one entry in the map, but will still have // individual entries in the catapult build artifacts -func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget, []*catapult.Artifact) { +func BuildTargets(apps map[string]*repo.AppConfig) (map[string]LambdaTarget, []*catapult.Artifact) { var ( targets = map[string]LambdaTarget{} done = map[string]struct{}{} @@ -38,9 +37,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget continue } - artifact := repo.ArtifactName(name, launch) + artifact := launch.ArtifactName artifacts = append(artifacts, &catapult.Artifact{ - RunType: string(models.RunTypeLambda), + RunType: launch.RunType, ID: name, Branch: environment.Branch(), Source: fmt.Sprintf("github:Clever/%s@%s", environment.Repo(), environment.FullSHA1()), @@ -54,7 +53,7 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget done[artifact] = struct{}{} targets[artifact] = LambdaTarget{ Zip: fmt.Sprintf("./bin/%s.zip", artifact), - Command: repo.BuildCommand(launch), + Command: launch.BuildCommand, } } return targets, artifacts diff --git a/internal/repo/app_config.go b/internal/repo/app_config.go index 587ac22..535e749 100644 --- a/internal/repo/app_config.go +++ b/internal/repo/app_config.go @@ -5,14 +5,47 @@ import ( "os" "github.com/ghodss/yaml" + + "github.com/Clever/catapult/gen-go/models" ) const ( appStackConfigPath = "config/%s/stack.yaml" + + RunTypeDocker = "docker" + RunTypeLambda = "lambda" ) +type appHelmYAML struct { + Chart string `json:"chart"` +} + +type appStackBlockYAML struct { + Helm appHelmYAML `json:"helm"` +} + +type appBuildYAML struct { + Command string `json:"command"` + ArtifactName string `json:"artifactName"` + Dockerfile string `json:"dockerfile"` + Dependencies []string `json:"dependencies"` +} + type appStackYAML struct { - AutoDeployEnvs []string `json:"autoDeployEnvs"` + AutoDeployEnvs []string `json:"autoDeployEnvs"` + Stack appStackBlockYAML `json:"stack"` + Build appBuildYAML `json:"build"` +} + +// AppConfig holds build and runtime configuration +type AppConfig struct { + Name string + RunType string + ArtifactName string + BuildCommand string + Dockerfile string + Dependencies []string + IsKubernetes bool } // ReadAppStackAutoDeployEnvs reads autoDeployEnvs from config//stack.yaml. @@ -34,3 +67,67 @@ func AutoDeployEnvs(app string) ([]string, error) { } return stack.AutoDeployEnvs, nil } + +func appConfigForCatapult(appName string, lc *models.LaunchConfig) *AppConfig { + runType := RunTypeDocker + if lc.Run != nil && lc.Run.Type == models.RunTypeLambda { + runType = RunTypeLambda + } + + artifactName := appName + var buildCommand, dockerfile string + var dependencies []string + if lc.Build != nil { + if lc.Build.Artifact != nil { + if lc.Build.Artifact.Name != "" { + artifactName = lc.Build.Artifact.Name + } + buildCommand = lc.Build.Artifact.Command + dependencies = lc.Build.Artifact.Dependencies + } + if lc.Build.Docker != nil { + dockerfile = lc.Build.Docker.File + } + } + + return &AppConfig{ + Name: appName, + RunType: runType, + ArtifactName: artifactName, + BuildCommand: buildCommand, + Dockerfile: dockerfile, + Dependencies: dependencies, + } +} + +func appConfigForKubernetes(appName string) (*AppConfig, error) { + stackPath := fmt.Sprintf(appStackConfigPath, appName) + b, err := os.ReadFile(stackPath) + if err != nil { + return nil, fmt.Errorf("failed to read stack.yaml for %s: %w", appName, err) + } + var stack appStackYAML + if err := yaml.Unmarshal(b, &stack); err != nil { + return nil, fmt.Errorf("failed to unmarshal stack.yaml for %s: %w", stackPath, err) + } + + runType := RunTypeDocker + if stack.Stack.Helm.Chart == "clever-lambda" { + runType = RunTypeLambda + } + + artifactName := appName + if stack.Build.ArtifactName != "" { + artifactName = stack.Build.ArtifactName + } + + return &AppConfig{ + Name: appName, + RunType: runType, + ArtifactName: artifactName, + BuildCommand: stack.Build.Command, + Dockerfile: stack.Build.Dockerfile, + Dependencies: stack.Build.Dependencies, + IsKubernetes: true, + }, nil +} diff --git a/internal/repo/app_config_test.go b/internal/repo/app_config_test.go new file mode 100644 index 0000000..7ca5258 --- /dev/null +++ b/internal/repo/app_config_test.go @@ -0,0 +1,409 @@ +package repo + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Clever/catapult/gen-go/models" + "github.com/stretchr/testify/assert" +) + +func TestAppConfigForCatapult(t *testing.T) { + tests := []struct { + name string + appName string + lc *models.LaunchConfig + expected *AppConfig + }{ + { + name: "no build or run block defaults to docker with app name as artifact", + appName: "my-service", + lc: &models.LaunchConfig{}, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + }, + }, + { + name: "explicit docker run type", + appName: "my-service", + lc: &models.LaunchConfig{ + Run: &models.LaunchRun{Type: models.RunTypeDocker}, + }, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + }, + }, + { + name: "lambda run type", + appName: "my-lambda", + lc: &models.LaunchConfig{ + Run: &models.LaunchRun{Type: models.RunTypeLambda}, + }, + expected: &AppConfig{ + Name: "my-lambda", + RunType: RunTypeLambda, + ArtifactName: "my-lambda", + }, + }, + { + name: "artifact name override", + appName: "sso-my-service", + lc: &models.LaunchConfig{ + Build: &models.LaunchBuild{ + Artifact: &models.BuildArtifact{Name: "my-service"}, + }, + }, + expected: &AppConfig{ + Name: "sso-my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + }, + }, + { + name: "build command", + appName: "my-service", + lc: &models.LaunchConfig{ + Build: &models.LaunchBuild{ + Artifact: &models.BuildArtifact{Command: "make build"}, + }, + }, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + BuildCommand: "make build", + }, + }, + { + name: "dockerfile override", + appName: "my-service", + lc: &models.LaunchConfig{ + Build: &models.LaunchBuild{ + Docker: &models.BuildDocker{File: "Dockerfile.api"}, + }, + }, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + Dockerfile: "Dockerfile.api", + }, + }, + { + name: "dependencies", + appName: "my-service", + lc: &models.LaunchConfig{ + Build: &models.LaunchBuild{ + Artifact: &models.BuildArtifact{ + Dependencies: []string{"*.go", "go.mod", "go.sum"}, + }, + }, + }, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + Dependencies: []string{"*.go", "go.mod", "go.sum"}, + }, + }, + { + name: "all fields set", + appName: "sso-my-service", + lc: &models.LaunchConfig{ + Run: &models.LaunchRun{Type: models.RunTypeDocker}, + Build: &models.LaunchBuild{ + Artifact: &models.BuildArtifact{ + Name: "my-service", + Command: "make build", + Dependencies: []string{"*.go", "Makefile"}, + }, + Docker: &models.BuildDocker{File: "Dockerfile.sso"}, + }, + }, + expected: &AppConfig{ + Name: "sso-my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + BuildCommand: "make build", + Dockerfile: "Dockerfile.sso", + Dependencies: []string{"*.go", "Makefile"}, + }, + }, + { + name: "build block present but artifact is nil", + appName: "my-service", + lc: &models.LaunchConfig{ + Build: &models.LaunchBuild{ + Docker: &models.BuildDocker{File: "Dockerfile.two"}, + }, + }, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + Dockerfile: "Dockerfile.two", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := appConfigForCatapult(tt.appName, tt.lc) + assert.Equal(t, tt.expected, got) + }) + } +} + +// writeStackYAML creates config//stack.yaml under dir with the given content. +func writeStackYAML(t *testing.T, dir, appName, content string) { + t.Helper() + appDir := filepath.Join(dir, "config", appName) + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatalf("mkdir %s: %v", appDir, err) + } + if err := os.WriteFile(filepath.Join(appDir, "stack.yaml"), []byte(content), 0644); err != nil { + t.Fatalf("write stack.yaml: %v", err) + } +} + +func TestAppConfigForKubernetes(t *testing.T) { + tests := []struct { + name string + appName string + stackYAML string + expected *AppConfig + expectError bool + }{ + { + name: "no build block defaults to docker with app name as artifact", + appName: "my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +`, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + }, + }, + { + name: "clever-lambda chart sets lambda run type", + appName: "my-lambda", + stackYAML: ` +stack: + helm: + chart: clever-lambda +`, + expected: &AppConfig{ + Name: "my-lambda", + RunType: RunTypeLambda, + ArtifactName: "my-lambda", + }, + }, + { + name: "artifact name override", + appName: "sso-my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +build: + artifactName: my-service +`, + expected: &AppConfig{ + Name: "sso-my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + }, + }, + { + name: "build command", + appName: "my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +build: + command: make build +`, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + BuildCommand: "make build", + }, + }, + { + name: "dockerfile override", + appName: "my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +build: + dockerfile: Dockerfile.api +`, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + Dockerfile: "Dockerfile.api", + }, + }, + { + name: "dependencies", + appName: "my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +build: + dependencies: + - "*.go" + - go.mod + - go.sum +`, + expected: &AppConfig{ + Name: "my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + Dependencies: []string{"*.go", "go.mod", "go.sum"}, + }, + }, + { + name: "all fields set", + appName: "sso-my-service", + stackYAML: ` +stack: + helm: + chart: clever-application +build: + artifactName: my-service + command: make build + dockerfile: Dockerfile.sso + dependencies: + - "*.go" + - Makefile +`, + expected: &AppConfig{ + Name: "sso-my-service", + RunType: RunTypeDocker, + ArtifactName: "my-service", + BuildCommand: "make build", + Dockerfile: "Dockerfile.sso", + Dependencies: []string{"*.go", "Makefile"}, + }, + }, + { + name: "missing stack.yaml returns error", + appName: "nonexistent-app", + stackYAML: "", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + if tt.stackYAML != "" { + writeStackYAML(t, dir, tt.appName, tt.stackYAML) + } + + got, err := appConfigForKubernetes(tt.appName) + if tt.expectError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestDiscoverKubernetesApplications(t *testing.T) { + t.Run("discovers all apps from config directory", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + writeStackYAML(t, dir, "app-one", ` +stack: + helm: + chart: clever-application +build: + artifactName: app-one +`) + writeStackYAML(t, dir, "app-two", ` +stack: + helm: + chart: clever-lambda +build: + command: make build +`) + + apps, err := discoverKubernetesApplications("config") + assert.NoError(t, err) + assert.Len(t, apps, 2) + assert.Equal(t, &AppConfig{ + Name: "app-one", + RunType: RunTypeDocker, + ArtifactName: "app-one", + }, apps["app-one"]) + assert.Equal(t, &AppConfig{ + Name: "app-two", + RunType: RunTypeLambda, + ArtifactName: "app-two", + BuildCommand: "make build", + }, apps["app-two"]) + }) + + t.Run("non-directory entries are skipped", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + writeStackYAML(t, dir, "my-app", ` +stack: + helm: + chart: clever-application +`) + // Write a non-directory file directly under config/ + configDir := filepath.Join(dir, "config") + if err := os.WriteFile(filepath.Join(configDir, "not-an-app.yaml"), []byte("foo: bar"), 0644); err != nil { + t.Fatalf("write stray file: %v", err) + } + + apps, err := discoverKubernetesApplications("config") + assert.NoError(t, err) + assert.Len(t, apps, 1) + assert.Contains(t, apps, "my-app") + }) + + t.Run("empty config directory returns empty map", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + if err := os.MkdirAll(filepath.Join(dir, "config"), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + apps, err := discoverKubernetesApplications("config") + assert.NoError(t, err) + assert.Empty(t, apps) + }) + + t.Run("missing config directory returns error", func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + _, err := discoverKubernetesApplications("config") + assert.Error(t, err) + }) +} diff --git a/internal/repo/detect_change.go b/internal/repo/detect_change.go index 11c40eb..1944db0 100644 --- a/internal/repo/detect_change.go +++ b/internal/repo/detect_change.go @@ -4,18 +4,14 @@ import ( "fmt" "os/exec" - "github.com/Clever/catapult/gen-go/models" "github.com/Clever/ci-scripts/internal/environment" ) -// DetectArtifactDependencyChange checks if the artifact dependency -// globs defined in the launch config have changed by using git diff for -// only the specified file globs. The dependencies are always checked -// against the primary branch. More advanced dependency checking is -// hard and involves persisted caching of some sort which should be -// left to a build system later on. -func DetectArtifactDependencyChange(lc *models.LaunchConfig) (bool, error) { - if lc.Build == nil || lc.Build.Artifact == nil || lc.Build.Artifact.Dependencies == nil { +// DetectArtifactDependencyChange checks if any of the given file globs have +// changed since the primary branch. An empty slice means no filtering — +// the app is always considered changed. +func DetectArtifactDependencyChange(dependencies []string) (bool, error) { + if len(dependencies) == 0 { return true, nil } @@ -24,7 +20,7 @@ func DetectArtifactDependencyChange(lc *models.LaunchConfig) (bool, error) { compareRange = environment.PreviousPipelineCompare() } - args := append([]string{"diff", "--name-only", compareRange, "--"}, lc.Build.Artifact.Dependencies...) + args := append([]string{"diff", "--name-only", compareRange, "--"}, dependencies...) gitCmd := exec.Command("git", args...) fmt.Println("Checking for changes with:", gitCmd.String()) diff --git a/internal/repo/detect_change_test.go b/internal/repo/detect_change_test.go index 52918e2..8fe8135 100644 --- a/internal/repo/detect_change_test.go +++ b/internal/repo/detect_change_test.go @@ -2,8 +2,6 @@ package repo import ( "testing" - - "github.com/Clever/catapult/gen-go/models" ) func TestDetectArtifactDependencyChange(t *testing.T) { @@ -11,15 +9,8 @@ func TestDetectArtifactDependencyChange(t *testing.T) { // shell commands are running within this directory which limits // find from seeing all files in this repo. t.Skip("skipping test") - lc := &models.LaunchConfig{ - Build: &models.LaunchBuild{ - Artifact: &models.BuildArtifact{ - Dependencies: []string{"*.go"}, - }, - }, - } - changed, err := DetectArtifactDependencyChange(lc) + changed, err := DetectArtifactDependencyChange([]string{"*.go"}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 9120cc6..9fb1d57 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -13,13 +13,45 @@ import ( "github.com/Clever/catapult/gen-go/models" ) -// DiscoverApplications finds any launch config files in the specified -// directory and returns a map with the application name as the key and -// the corresponding launch config file as the value. DB launch configs -// are ignored. Any applications that do not have changes detected -// according to the launch config dependencies are filtered from the -// result set. -func DiscoverApplications(dir string) (map[string]*models.LaunchConfig, error) { +// DiscoverApplications returns buildable apps with detected changes. It reads +// from both ./config (Kubernetes) and ./launch (Catapult), merging results with +// the Kubernetes config winning when the same app appears in both. +func DiscoverApplications() (map[string]*AppConfig, error) { + catapultApps, catapultErr := discoverCatapultApplications("./launch") + if catapultErr != nil && !errors.Is(catapultErr, os.ErrNotExist) { + return nil, catapultErr + } + fmt.Println("Discovered Catapult apps:", len(catapultApps)) + + + kubernetesApps, kubernetesErr := discoverKubernetesApplications("./config") + if kubernetesErr != nil && !errors.Is(kubernetesErr, os.ErrNotExist) { + return nil, kubernetesErr + } + fmt.Println("Discovered Kubernetes apps:", len(kubernetesApps)) + + if errors.Is(catapultErr, os.ErrNotExist) && errors.Is(kubernetesErr, os.ErrNotExist) { + return nil, fmt.Errorf("no app configs found: expected a launch/ or config/ directory at the repo root") + } + + apps := catapultApps + if apps == nil { + apps = map[string]*AppConfig{} + } + for name, kubernetesAC := range kubernetesApps { + if catapultAC, exists := apps[name]; exists { + if kubernetesAC.BuildCommand == "" && catapultAC.BuildCommand != "" { + return nil, fmt.Errorf("%s has no build.command in config/%s/stack.yaml but one exists in launch/%s.yml — add build.command to stack.yaml to complete the migration", name, name, name) + } + } + apps[name] = kubernetesAC + } + return apps, nil +} + +// discoverCatapultApplications reads launch/*.yml files, skips DB configs, filters +// by change detection, and converts each to AppConfig. +func discoverCatapultApplications(dir string) (map[string]*AppConfig, error) { fe, err := os.ReadDir(dir) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -28,7 +60,7 @@ func DiscoverApplications(dir string) (map[string]*models.LaunchConfig, error) { return nil, fmt.Errorf("failed to read launch directory: %v", err) } - m := map[string]*models.LaunchConfig{} + apps := map[string]*AppConfig{} for _, f := range fe { if f.IsDir() { continue @@ -52,81 +84,65 @@ func DiscoverApplications(dir string) (map[string]*models.LaunchConfig, error) { continue } - if changed, err := DetectArtifactDependencyChange(&lc); err != nil { + appName := strings.TrimSuffix(f.Name(), ".yml") + ac := appConfigForCatapult(appName, &lc) + if changed, err := DetectArtifactDependencyChange(ac.Dependencies); err != nil { return nil, fmt.Errorf("failed to detect artifact dependency change for %s: %v", f.Name(), err) } else if !changed { continue } - m[strings.TrimSuffix(f.Name(), ".yml")] = &lc + apps[appName] = ac } - - return m, nil + return apps, nil } -// Dockerfile returns the dockerfile name specified in the launch config -// if any is present, otherwise it returns an empty string. -func Dockerfile(lc *models.LaunchConfig) string { - if lc.Build != nil && lc.Build.Docker != nil { - return lc.Build.Docker.File +// discoverKubernetesApplications reads config//stack.yaml for each app +// directory under the given dir, filters out apps with no detected changes, +// and returns a map of app name to AppConfig. +func discoverKubernetesApplications(dir string) (map[string]*AppConfig, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("directory %s not found: %w", dir, err) + } + return nil, fmt.Errorf("failed to read config directory: %v", err) } - return "" -} -// IsDockerRunType returns true if the launch config specifies a run -// type of docker. -func IsDockerRunType(lc *models.LaunchConfig) bool { - if r := lc.Run; r != nil { - switch r.Type { - case models.RunTypeDocker: - return true - // for legacy support reasons, an empty run type is treated as a - // run type of docker. - case "": - return true - default: - return false + apps := map[string]*AppConfig{} + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + appName := entry.Name() + ac, err := appConfigForKubernetes(appName) + if err != nil { + return nil, fmt.Errorf("failed to read app config for %s: %v", appName, err) } - } - // no run object also counts as docker. - return true -} -// IsLambdaRunType returns true if the launch config specifies a run -// type of lambda. -func IsLambdaRunType(lc *models.LaunchConfig) bool { - if r := lc.Run; r != nil { - switch r.Type { - case models.RunTypeLambda: - return true - default: - return false + changed, err := DetectArtifactDependencyChange(ac.Dependencies) + if err != nil { + return nil, fmt.Errorf("failed to detect artifact dependency change for %s: %v", appName, err) + } + if !changed { + continue } + + apps[appName] = ac } - return false + return apps, nil } -// ArtifactName returns the correct artifact name for the application. -// The default pattern is the app name. There is an optional launch -// config override in order to enable sharing one artifact between -// multiple applications. This may happen with for example, sso and -// non-sso, where the application is the same or only differs at run -// time based on environmental configuration. -func ArtifactName(appName string, lc *models.LaunchConfig) string { - artifactName := appName - if lc.Build != nil && lc.Build.Artifact != nil && lc.Build.Artifact.Name != "" { - artifactName = lc.Build.Artifact.Name - } - return artifactName +// IsDockerRunType returns true if the app should be built as a Docker image. +// An empty RunType defaults to docker. +func IsDockerRunType(ac *AppConfig) bool { + return ac.RunType == RunTypeDocker || ac.RunType == "" } -// BuildCommand returns the build command for the application artifact, -// if any is specified. Otherwise it returns an empty string. -func BuildCommand(lc *models.LaunchConfig) string { - if lc.Build == nil || lc.Build.Artifact == nil { - return "" - } - return lc.Build.Artifact.Command +// IsLambdaRunType returns true if the app should be built as a Lambda artifact. +func IsLambdaRunType(ac *AppConfig) bool { + return ac.RunType == RunTypeLambda } // ExecBuild runs the build command for the application artifact, if any