[INFRA-421] support repos with no launch.yaml

.gitignore

@@ -3,3 +3,4 @@ vendor
 bin
 tmp
 artifacts
+**/*.code-workspace

VERSION

@@ -1,7 +1,8 @@
-v1.9.0
+v1.10.0
+Support repos with no launch.yaml
 Move k8s deploys to platform events
 
 Previously:
+- Move k8s deploys to platform events
 - Add SyncEntity call to k8s deployApps command
 - Support configurable deployment branches
-- Add new deploy-apps command

cmd/goci/README.md

@@ -4,29 +4,35 @@
 
 ## Configuration
 
-goci accepts very limited arguments which merely change the mode it runs in. The rest of the configuration is entirely through environment variables and launch config settings. See the[environment](../../internal/environment/environment.go) package for detailed documentation of environment variables for configuration. goci reads it's configuration from the `build` section of the launch config of each application. See the [build section](https://github.com/Clever/catapult/blob/master/swagger.yml#L1773) of the launch yaml to learn about the various parameters which configure goci.
+goci accepts very limited arguments which merely change the mode it runs in. The rest of the configuration is entirely through environment variables and app config files. See the [environment](../../internal/environment/environment.go) package for detailed documentation of environment variables for configuration.
+
+goci supports two config paths:
+
+- **Kubernetes apps** (preferred): `config/<app>/stack.yaml` — goci reads the `build` block and determines run type from `stack.helm.chart` (`clever-lambda` → lambda, anything else → docker).
+- **Catapult apps** (legacy): `launch/<app>.yml` — goci reads from the `build` section of the launch config of each application. See the [build section](https://github.com/Clever/catapult/blob/master/swagger.yml#L1773) of the launch yaml to learn about the various parameters which configure goci. 
+
+goci reads from **both** paths and merges the results. If the same app appears in both `config/` and `launch/`, the Kubernetes config wins.
 
 ## Modes
 
-1. `goci detect` detects any changed applications according to their launch configuration. This can be used to pass a name of apps to another script.
+1. `goci detect` detects any changed applications according to their stack.yaml or launch.yaml configuration. This can be used to pass a name of apps to another script.
 2. `goci artifact-build-publish-deploy` builds, publishes and deploys any application artifacts.
 3. `goci validate` validates an applications go version, while also checking for compatible branch naming conventions for catapult.
 4. `goci publish-utility` publishes catalog-info.yaml to the service catalog.
 
 
 ## Multi-app Support
 
-goci will automatically detect all launch configs in the `launch`
-directory, then perform the following actions as needed.
+goci will automatically detect all applications in the `config` directory (falling back to `launch` for legacy repos), then perform the following actions as needed.
 
 1. detect the run type of the application
 2. Run any configured build commands
 3. build any docker images
 4. publish all built docker images to all ECR regions
 5. publish all lambdas to s3 in all regions.
 6. Sync all changed apps with catalog config
-7. Publish new application versions to catapult
-8. Deploy any changed applications.
+7. [launch.yaml only]Publish new application versions to catapult
+8. [launch.yaml only] Deploy any changed applications.
 
 ## Development
 

cmd/goci/main.go

@@ -9,7 +9,6 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/Clever/catapult/gen-go/models"
 	"github.com/Clever/ci-scripts/internal/backstage"
 	"github.com/Clever/ci-scripts/internal/catalogsync"
 	"github.com/Clever/ci-scripts/internal/catapult"
@@ -53,13 +52,13 @@ func main() {
 }
 
 func run(mode string) error {
-	var apps map[string]*models.LaunchConfig
+	var apps map[string]*repo.AppConfig
 	var appIDs []string
 	var err error
 
 	// Only discover applications for specific modes
 	if mode == "validate" || mode == "detect" || mode == "artifact-build-publish-deploy" || mode == "deploy-apps" {
-		apps, err = repo.DiscoverApplications("./launch")
+		apps, err = repo.DiscoverApplications()
 		if err != nil {
 			return err
 		}
@@ -91,7 +90,7 @@ func run(mode string) error {
 
 	if len(apps) == 0 {
 		fmt.Println("No applications have buildable changes. If this is unexpected, " +
-			"double check your artifact dependency configuration in the launch yaml.")
+			"double check your artifact dependency configuration in the launch.yaml or stack.yaml.")
 		return nil
 	}
 
@@ -143,16 +142,25 @@ func run(mode string) error {
 			}
 		}
 	}
-	cp := catapult.New()
-
-	if err = cp.Publish(ctx, artifacts); err != nil {
-		return err
+	catapultArtifacts := make([]*catapult.Artifact, 0, len(artifacts))
+	catapultAppIDs := make([]string, 0, len(appIDs))
+	for _, art := range artifacts {
+		if ac, ok := apps[art.ID]; ok && !ac.IsKubernetes {
+			catapultArtifacts = append(catapultArtifacts, art)
+			catapultAppIDs = append(catapultAppIDs, art.ID)
+		}
 	}
 
-	if environment.Branch() == "master" {
-		if err := cp.Deploy(ctx, appIDs); err != nil {
+	if len(catapultArtifacts) > 0 {
+		cp := catapult.New()
+		if err = cp.Publish(ctx, catapultArtifacts); err != nil {
 			return err
 		}
+		if environment.Branch() == "master" {
+			if err := cp.Deploy(ctx, catapultAppIDs); err != nil {
+				return err
+			}
+		}
 	}
 
 	// We want to validate on every run, not just when the mode is "validate".
@@ -254,7 +262,7 @@ func fetchLatestGoVersion() (string, string, error) {
 }
 
 // allAppsBuilt returns an error if any apps are missing a build artifact.
-func allAppsBuilt(discoveredApps map[string]*models.LaunchConfig, builtApps []*catapult.Artifact) error {
+func allAppsBuilt(discoveredApps map[string]*repo.AppConfig, builtApps []*catapult.Artifact) error {
 	if len(discoveredApps) == len(builtApps) {
 		return nil
 	}
@@ -303,7 +311,7 @@ func publishUtility() error {
 func deployApps(appIds []string) error {
 	if len(appIds) == 0 {
 		fmt.Println("No applications have buildable changes. If this is unexpected, " +
-			"double check your artifact dependency configuration in the launch yaml.")
+			"double check your artifact dependency configuration in the launch yaml or stack.yaml.")
 		return nil
 	}
 	ctx := context.Background()

go.mod

@@ -15,8 +15,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0
 	github.com/docker/docker v23.0.2+incompatible
+	github.com/getkin/kin-openapi v0.139.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/moby/buildkit v0.11.5
+	github.com/stretchr/testify v1.11.1
 	golang.org/x/sync v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -40,11 +42,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
 	github.com/aws/smithy-go v1.26.0 // indirect
 	github.com/containerd/containerd v1.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect
-	github.com/getkin/kin-openapi v0.139.0 // indirect
 	github.com/go-openapi/analysis v0.24.1 // indirect
 	github.com/go-openapi/errors v0.22.4 // indirect
 	github.com/go-openapi/jsonpointer v0.22.4 // indirect
@@ -83,6 +85,7 @@ require (
 	github.com/opencontainers/runc v1.1.4 // indirect
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/speakeasy-api/jsonpath v0.6.3 // indirect

internal/docker/targets.go

@@ -3,7 +3,6 @@ package docker
 import (
 	"fmt"
 
-	"github.com/Clever/catapult/gen-go/models"
 	"github.com/Clever/ci-scripts/internal/catapult"
 	"github.com/Clever/ci-scripts/internal/environment"
 	"github.com/Clever/ci-scripts/internal/repo"
@@ -24,7 +23,7 @@ type DockerTarget struct {
 // Dockerfile and its set of tags will be in the final list. This is an
 // optimization so we do not build multiple copies of the same
 // Dockerfile which only differ at runtime.
-func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget, []*catapult.Artifact) {
+func BuildTargets(apps map[string]*repo.AppConfig) (map[string]DockerTarget, []*catapult.Artifact) {
 	var (
 		targets   = map[string]DockerTarget{}
 		done      = map[string]struct{}{}
@@ -36,9 +35,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget
 			continue
 		}
 
-		artifact := repo.ArtifactName(name, launch)
+		artifact := launch.ArtifactName
 		artifacts = append(artifacts, &catapult.Artifact{
-			RunType:   string(models.RunTypeDocker),
+			RunType:   launch.RunType,
 			ID:        name,
 			Branch:    environment.Branch(),
 			Source:    fmt.Sprintf("github:Clever/%s@%s", environment.Repo(), environment.FullSHA1()),
@@ -63,9 +62,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]DockerTarget
 		)
 		tags = append(tags, tag)
 
-		targets[repo.Dockerfile(launch)] = DockerTarget{
+		targets[launch.Dockerfile] = DockerTarget{
 			Tags:    tags,
-			Command: repo.BuildCommand(launch),
+			Command: launch.BuildCommand,
 		}
 	}
 	return targets, artifacts

internal/lambda/targets.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Clever/catapult/gen-go/models"
 	"github.com/Clever/ci-scripts/internal/catapult"
 	"github.com/Clever/ci-scripts/internal/environment"
 	"github.com/Clever/ci-scripts/internal/repo"
@@ -26,7 +25,7 @@ type LambdaTarget struct {
 // destination zip file in the value struct. Any apps with a shared
 // artifact will have only one entry in the map, but will still have
 // individual entries in the catapult build artifacts
-func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget, []*catapult.Artifact) {
+func BuildTargets(apps map[string]*repo.AppConfig) (map[string]LambdaTarget, []*catapult.Artifact) {
 	var (
 		targets   = map[string]LambdaTarget{}
 		done      = map[string]struct{}{}
@@ -38,9 +37,9 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget
 			continue
 		}
 
-		artifact := repo.ArtifactName(name, launch)
+		artifact := launch.ArtifactName
 		artifacts = append(artifacts, &catapult.Artifact{
-			RunType:   string(models.RunTypeLambda),
+			RunType:   launch.RunType,
 			ID:        name,
 			Branch:    environment.Branch(),
 			Source:    fmt.Sprintf("github:Clever/%s@%s", environment.Repo(), environment.FullSHA1()),
@@ -54,7 +53,7 @@ func BuildTargets(apps map[string]*models.LaunchConfig) (map[string]LambdaTarget
 		done[artifact] = struct{}{}
 		targets[artifact] = LambdaTarget{
 			Zip:     fmt.Sprintf("./bin/%s.zip", artifact),
-			Command: repo.BuildCommand(launch),
+			Command: launch.BuildCommand,
 		}
 	}
 	return targets, artifacts

internal/repo/app_config.go

@@ -5,14 +5,47 @@ import (
 	"os"
 
 	"github.com/ghodss/yaml"
+
+	"github.com/Clever/catapult/gen-go/models"
 )
 
 const (
 	appStackConfigPath = "config/%s/stack.yaml"
+
+	RunTypeDocker = "docker"
+	RunTypeLambda = "lambda"
 )
 
+type appHelmYAML struct {
+	Chart string `json:"chart"`
+}
+
+type appStackBlockYAML struct {
+	Helm appHelmYAML `json:"helm"`
+}
+
+type appBuildYAML struct {
+	Command      string   `json:"command"`
+	ArtifactName string   `json:"artifactName"`
+	Dockerfile   string   `json:"dockerfile"`
+	Dependencies []string `json:"dependencies"`
+}
+
 type appStackYAML struct {
-	AutoDeployEnvs []string `json:"autoDeployEnvs"`
+	AutoDeployEnvs []string          `json:"autoDeployEnvs"`
+	Stack          appStackBlockYAML `json:"stack"`
+	Build          appBuildYAML      `json:"build"`
+}
+
+// AppConfig holds build and runtime configuration
+type AppConfig struct {
+	Name         string
+	RunType      string
+	ArtifactName string
+	BuildCommand string
+	Dockerfile   string
+	Dependencies []string
+	IsKubernetes bool
 }
 
 // ReadAppStackAutoDeployEnvs reads autoDeployEnvs from config/<app>/stack.yaml.
@@ -34,3 +67,67 @@ func AutoDeployEnvs(app string) ([]string, error) {
 	}
 	return stack.AutoDeployEnvs, nil
 }
+
+func appConfigForCatapult(appName string, lc *models.LaunchConfig) *AppConfig {
+	runType := RunTypeDocker
+	if lc.Run != nil && lc.Run.Type == models.RunTypeLambda {
+		runType = RunTypeLambda
+	}
+
+	artifactName := appName
+	var buildCommand, dockerfile string
+	var dependencies []string
+	if lc.Build != nil {
+		if lc.Build.Artifact != nil {
+			if lc.Build.Artifact.Name != "" {
+				artifactName = lc.Build.Artifact.Name
+			}
+			buildCommand = lc.Build.Artifact.Command
+			dependencies = lc.Build.Artifact.Dependencies
+		}
+		if lc.Build.Docker != nil {
+			dockerfile = lc.Build.Docker.File
+		}
+	}
+
+	return &AppConfig{
+		Name:         appName,
+		RunType:      runType,
+		ArtifactName: artifactName,
+		BuildCommand: buildCommand,
+		Dockerfile:   dockerfile,
+		Dependencies: dependencies,
+	}
+}
+
+func appConfigForKubernetes(appName string) (*AppConfig, error) {
+	stackPath := fmt.Sprintf(appStackConfigPath, appName)
+	b, err := os.ReadFile(stackPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read stack.yaml for %s: %w", appName, err)
+	}
+	var stack appStackYAML
+	if err := yaml.Unmarshal(b, &stack); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal stack.yaml for %s: %w", stackPath, err)
+	}
+
+	runType := RunTypeDocker
+	if stack.Stack.Helm.Chart == "clever-lambda" {
+		runType = RunTypeLambda
+	}
+
+	artifactName := appName
+	if stack.Build.ArtifactName != "" {
+		artifactName = stack.Build.ArtifactName
+	}
+
+	return &AppConfig{
+		Name:         appName,
+		RunType:      runType,
+		ArtifactName: artifactName,
+		BuildCommand: stack.Build.Command,
+		Dockerfile:   stack.Build.Dockerfile,
+		Dependencies: stack.Build.Dependencies,
+		IsKubernetes: true,
+	}, nil
+}