From 467cebb04314597b76d834c9bad0230da805607e Mon Sep 17 00:00:00 2001 From: Fernando Davis Date: Thu, 6 Aug 2026 19:57:24 -0400 Subject: [PATCH 1/5] test validation --- .github/workflows/deploy-applicationset.yaml | 323 ++++++++++--------- 1 file changed, 162 insertions(+), 161 deletions(-) diff --git a/.github/workflows/deploy-applicationset.yaml b/.github/workflows/deploy-applicationset.yaml index ef5d14e..97ddd56 100644 --- a/.github/workflows/deploy-applicationset.yaml +++ b/.github/workflows/deploy-applicationset.yaml @@ -9,6 +9,7 @@ on: push: branches: + - dev - main paths: - '*/*/applicationset.yaml' @@ -366,164 +367,164 @@ jobs: PY - # ========================================================= - # DEPLOYMENT - # ========================================================= - deploy: - name: Deploy ApplicationSet - runs-on: ubuntu-latest - - needs: - - validate - - # Deploy ONLY after merge/push to main. - # Never deploy from a Pull Request. - if: | - github.event_name == 'push' && - github.ref == 'refs/heads/main' - - steps: - # ----------------------------------------------------- - # Checkout - # ----------------------------------------------------- - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # ----------------------------------------------------- - # Configure Kubernetes authentication - # - # GitHub Secret: - # - # KUBECONFIG_B64 - # - # ----------------------------------------------------- - - name: Configure Kubernetes - env: - KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} - run: | - set -e - - mkdir -p ~/.kube - - echo "$KUBECONFIG_B64" \ - | base64 --decode \ - > ~/.kube/config - - chmod 600 ~/.kube/config - - # ----------------------------------------------------- - # Verify Kubernetes connectivity - # ----------------------------------------------------- - - name: Verify Kubernetes connection - run: | - kubectl cluster-info - - echo - echo "Current context:" - kubectl config current-context - - # ----------------------------------------------------- - # Make sure ApplicationSet CRD exists - # ----------------------------------------------------- - - name: Verify ApplicationSet CRD - run: | - kubectl get crd applicationsets.argoproj.io - - # ----------------------------------------------------- - # Find ApplicationSets changed by this push - # ----------------------------------------------------- - - name: Find changed ApplicationSets - run: | - set -e - - BASE_SHA="${{ github.event.before }}" - HEAD_SHA="${{ github.sha }}" - - echo "Base SHA: ${BASE_SHA}" - echo "Head SHA: ${HEAD_SHA}" - - if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then - - find . \ - -mindepth 3 \ - -maxdepth 3 \ - -type f \ - -name applicationset.yaml \ - | sed 's#^\./##' \ - | sort \ - > /tmp/appset-files.txt - - else - - git diff \ - --name-only \ - --diff-filter=ACMR \ - "${BASE_SHA}" \ - "${HEAD_SHA}" \ - -- '*/*/applicationset.yaml' \ - > /tmp/appset-files.txt - - fi - - echo - echo "ApplicationSets to deploy:" - echo "------------------------------------------" - - cat /tmp/appset-files.txt - - echo "------------------------------------------" - - # ----------------------------------------------------- - # Kubernetes + ApplicationSet CRD validation - # - # Nothing is actually created here. - # ----------------------------------------------------- - - name: Server-side ApplicationSet validation - run: | - set -e - - while read -r file; do - - [ -z "$file" ] && continue - - echo - echo "==========================================" - echo "Server-side validating: $file" - echo "==========================================" - - kubectl apply \ - --dry-run=server \ - -f "$file" - - done < /tmp/appset-files.txt - - # ----------------------------------------------------- - # Deploy ApplicationSets - # ----------------------------------------------------- - - name: Deploy ApplicationSets - run: | - set -e - - while read -r file; do - - [ -z "$file" ] && continue - - echo - echo "==========================================" - echo "Deploying: $file" - echo "==========================================" - - kubectl apply \ - -f "$file" - - done < /tmp/appset-files.txt - - # ----------------------------------------------------- - # Show resulting ApplicationSets - # ----------------------------------------------------- - - name: Show deployed ApplicationSets - run: | - kubectl get applicationsets \ - -n argocd \ No newline at end of file + # # ========================================================= + # # DEPLOYMENT + # # ========================================================= + # deploy: + # name: Deploy ApplicationSet + # runs-on: ubuntu-latest + + # needs: + # - validate + + # # Deploy ONLY after merge/push to main. + # # Never deploy from a Pull Request. + # if: | + # github.event_name == 'push' && + # github.ref == 'refs/heads/main' + + # steps: + # # ----------------------------------------------------- + # # Checkout + # # ----------------------------------------------------- + # - name: Checkout repository + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # # ----------------------------------------------------- + # # Configure Kubernetes authentication + # # + # # GitHub Secret: + # # + # # KUBECONFIG_B64 + # # + # # ----------------------------------------------------- + # - name: Configure Kubernetes + # env: + # KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} + # run: | + # set -e + + # mkdir -p ~/.kube + + # echo "$KUBECONFIG_B64" \ + # | base64 --decode \ + # > ~/.kube/config + + # chmod 600 ~/.kube/config + + # # ----------------------------------------------------- + # # Verify Kubernetes connectivity + # # ----------------------------------------------------- + # - name: Verify Kubernetes connection + # run: | + # kubectl cluster-info + + # echo + # echo "Current context:" + # kubectl config current-context + + # # ----------------------------------------------------- + # # Make sure ApplicationSet CRD exists + # # ----------------------------------------------------- + # - name: Verify ApplicationSet CRD + # run: | + # kubectl get crd applicationsets.argoproj.io + + # # ----------------------------------------------------- + # # Find ApplicationSets changed by this push + # # ----------------------------------------------------- + # - name: Find changed ApplicationSets + # run: | + # set -e + + # BASE_SHA="${{ github.event.before }}" + # HEAD_SHA="${{ github.sha }}" + + # echo "Base SHA: ${BASE_SHA}" + # echo "Head SHA: ${HEAD_SHA}" + + # if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + + # find . \ + # -mindepth 3 \ + # -maxdepth 3 \ + # -type f \ + # -name applicationset.yaml \ + # | sed 's#^\./##' \ + # | sort \ + # > /tmp/appset-files.txt + + # else + + # git diff \ + # --name-only \ + # --diff-filter=ACMR \ + # "${BASE_SHA}" \ + # "${HEAD_SHA}" \ + # -- '*/*/applicationset.yaml' \ + # > /tmp/appset-files.txt + + # fi + + # echo + # echo "ApplicationSets to deploy:" + # echo "------------------------------------------" + + # cat /tmp/appset-files.txt + + # echo "------------------------------------------" + + # # ----------------------------------------------------- + # # Kubernetes + ApplicationSet CRD validation + # # + # # Nothing is actually created here. + # # ----------------------------------------------------- + # - name: Server-side ApplicationSet validation + # run: | + # set -e + + # while read -r file; do + + # [ -z "$file" ] && continue + + # echo + # echo "==========================================" + # echo "Server-side validating: $file" + # echo "==========================================" + + # kubectl apply \ + # --dry-run=server \ + # -f "$file" + + # done < /tmp/appset-files.txt + + # # ----------------------------------------------------- + # # Deploy ApplicationSets + # # ----------------------------------------------------- + # - name: Deploy ApplicationSets + # run: | + # set -e + + # while read -r file; do + + # [ -z "$file" ] && continue + + # echo + # echo "==========================================" + # echo "Deploying: $file" + # echo "==========================================" + + # kubectl apply \ + # -f "$file" + + # done < /tmp/appset-files.txt + + # # ----------------------------------------------------- + # # Show resulting ApplicationSets + # # ----------------------------------------------------- + # - name: Show deployed ApplicationSets + # run: | + # kubectl get applicationsets \ + # -n argocd From efba70677251a974f6e7b89ade8e40819f7d79e4 Mon Sep 17 00:00:00 2001 From: Fernando Davis Date: Thu, 6 Aug 2026 20:00:43 -0400 Subject: [PATCH 2/5] add NL at the end of the appsets --- dfa/dev/applicationset.yaml | 2 +- ige/dev/applicationset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dfa/dev/applicationset.yaml b/dfa/dev/applicationset.yaml index 91954ab..d4a3eb8 100644 --- a/dfa/dev/applicationset.yaml +++ b/dfa/dev/applicationset.yaml @@ -34,4 +34,4 @@ spec: syncOptions: - CreateNamespace=true - - PruneLast=true \ No newline at end of file + - PruneLast=true diff --git a/ige/dev/applicationset.yaml b/ige/dev/applicationset.yaml index 91954ab..d4a3eb8 100644 --- a/ige/dev/applicationset.yaml +++ b/ige/dev/applicationset.yaml @@ -34,4 +34,4 @@ spec: syncOptions: - CreateNamespace=true - - PruneLast=true \ No newline at end of file + - PruneLast=true From 62eed26dc5768f7043b99693e6c6635f3146023d Mon Sep 17 00:00:00 2001 From: Fernando Davis Date: Thu, 6 Aug 2026 20:03:12 -0400 Subject: [PATCH 3/5] correct project names --- dfa/dev/applicationset.yaml | 2 +- ige/dev/applicationset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dfa/dev/applicationset.yaml b/dfa/dev/applicationset.yaml index d4a3eb8..1425111 100644 --- a/dfa/dev/applicationset.yaml +++ b/dfa/dev/applicationset.yaml @@ -16,7 +16,7 @@ spec: name: '{{app}}' spec: - project: default + project: dfa source: repoURL: https://github.com/ferchdav/argocd-example-apps diff --git a/ige/dev/applicationset.yaml b/ige/dev/applicationset.yaml index d4a3eb8..aa25083 100644 --- a/ige/dev/applicationset.yaml +++ b/ige/dev/applicationset.yaml @@ -16,7 +16,7 @@ spec: name: '{{app}}' spec: - project: default + project: ige source: repoURL: https://github.com/ferchdav/argocd-example-apps From 4f7fb4bd6ab99f2fe35a5d11e9240c19b2f83262 Mon Sep 17 00:00:00 2001 From: Fernando Davis Date: Fri, 7 Aug 2026 14:35:28 -0400 Subject: [PATCH 4/5] update wklfs ver 3 --- .github/config/tribe-projects.yaml | 81 ++ .github/scripts/select_appsets.py | 272 ++++++ .github/scripts/validate_applicationset.py | 311 +++++++ .../workflows/deploy-applicationset-v1-yaml | 530 ++++++++++++ .github/workflows/deploy-applicationset.yaml | 796 ++++++++---------- {pay => opintel}/dev/applicationset.yaml | 4 +- 6 files changed, 1555 insertions(+), 439 deletions(-) create mode 100644 .github/config/tribe-projects.yaml create mode 100644 .github/scripts/select_appsets.py create mode 100644 .github/scripts/validate_applicationset.py create mode 100644 .github/workflows/deploy-applicationset-v1-yaml rename {pay => opintel}/dev/applicationset.yaml (92%) diff --git a/.github/config/tribe-projects.yaml b/.github/config/tribe-projects.yaml new file mode 100644 index 0000000..115dc4c --- /dev/null +++ b/.github/config/tribe-projects.yaml @@ -0,0 +1,81 @@ +tribes: + + admin-portal: + enabled: false + allowed_projects: + - dobsy + - core-sre + + catalog: + enabled: false + allowed_projects: + - pipeline + - core-sre + + commerce: + enabled: false + allowed_projects: + - commerce + + core-sre: + enabled: false + allowed_projects: + - core-sre + + data-pipelines: + enabled: false + allowed_projects: + - pipeline + - core-sre + + dataprep: + enabled: false + allowed_projects: + - pipeline + - core-sre + + dfa: + enabled: true + allowed_projects: + - dfa + + dobsygpt: + enabled: false + allowed_projects: + - dobsy + - core-sre + + doc: + enabled: false + allowed_projects: + - doc + - core-sre + + dqm: + enabled: false + allowed_projects: + - dobsy + - core-sre + + ige: + enabled: true + allowed_projects: + - ige + + keying-gkeapps: + enabled: false + allowed_projects: + - knl + - core-sre + + opintel: + enabled: false + allowed_projects: + - doc + - core-sre + + opintel-platform: + enabled: false + allowed_projects: + - dobsy + - core-sre \ No newline at end of file diff --git a/.github/scripts/select_appsets.py b/.github/scripts/select_appsets.py new file mode 100644 index 0000000..208fba4 --- /dev/null +++ b/.github/scripts/select_appsets.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path +import sys + +import yaml + + +def load_config(filename): + if not filename or not Path(filename).exists(): + return {} + + with open(filename, "r", encoding="utf-8") as f: + document = yaml.safe_load(f) or {} + + tribes = document.get("tribes", {}) + + if not isinstance(tribes, dict): + raise ValueError("'tribes' must be a YAML mapping") + + result = {} + + for tribe, config in tribes.items(): + + if not isinstance(config, dict): + raise ValueError( + f"Configuration for tribe '{tribe}' must be a mapping" + ) + + enabled = config.get("enabled", False) + allowed_projects = config.get("allowed_projects", []) + + if not isinstance(enabled, bool): + raise ValueError( + f"'enabled' for tribe '{tribe}' must be true or false" + ) + + if not isinstance(allowed_projects, list): + raise ValueError( + f"'allowed_projects' for tribe '{tribe}' must be a list" + ) + + if not allowed_projects: + raise ValueError( + f"Tribe '{tribe}' must contain at least one allowed project" + ) + + result[tribe] = { + "enabled": enabled, + "allowed_projects": sorted( + str(project) for project in allowed_projects + ), + } + + return result + + +def load_changed_files(filename): + if not Path(filename).exists(): + return [] + + with open(filename, "r", encoding="utf-8") as f: + return [ + line.strip().lstrip("./") + for line in f + if line.strip() + ] + + +def application_sets_for_tribe(tribe): + tribe_path = Path(tribe) + + if not tribe_path.exists(): + return [] + + return [ + str(path) + for path in tribe_path.glob("*/applicationset.yaml") + if path.is_file() + ] + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--config", + required=True, + help="Current tribe/project configuration", + ) + + parser.add_argument( + "--base-config", + required=False, + help="Configuration from the base commit", + ) + + parser.add_argument( + "--changed-files", + required=True, + help="File containing changed ApplicationSet paths", + ) + + parser.add_argument( + "--output", + required=True, + help="Output file containing ApplicationSets to process", + ) + + parser.add_argument( + "--all-enabled", + action="store_true", + help="Select all ApplicationSets belonging to enabled tribes", + ) + + args = parser.parse_args() + + try: + current_config = load_config(args.config) + base_config = load_config(args.base_config) + + except Exception as exc: + print(f"ERROR: Invalid tribe configuration: {exc}") + sys.exit(1) + + enabled_tribes = { + tribe + for tribe, config in current_config.items() + if config["enabled"] + } + + print() + print("============================================================") + print("Enabled tribes") + print("============================================================") + + for tribe in sorted(enabled_tribes): + projects = ", ".join( + current_config[tribe]["allowed_projects"] + ) + + print(f"{tribe}: {projects}") + + selected = set() + + # -------------------------------------------------------- + # workflow_dispatch + # + # Process ALL ApplicationSets from enabled tribes. + # -------------------------------------------------------- + + if args.all_enabled: + + for tribe in enabled_tribes: + + for filename in application_sets_for_tribe(tribe): + selected.add(filename) + + else: + + # ---------------------------------------------------- + # Normal PR / push behavior: + # + # Select changed ApplicationSets only if their tribe + # is currently enabled. + # ---------------------------------------------------- + + changed_files = load_changed_files( + args.changed_files + ) + + print() + print("============================================================") + print("Changed ApplicationSets") + print("============================================================") + + for filename in changed_files: + + path = Path(filename) + + if len(path.parts) != 3: + print( + f"SKIPPED: {filename} " + "(unexpected directory structure)" + ) + continue + + tribe = path.parts[0] + + if tribe not in current_config: + print( + f"SKIPPED: {filename} " + f"(tribe '{tribe}' is not configured)" + ) + continue + + if not current_config[tribe]["enabled"]: + print( + f"SKIPPED: {filename} " + f"(tribe '{tribe}' is disabled)" + ) + continue + + print(f"SELECTED: {filename}") + selected.add(filename) + + # ---------------------------------------------------- + # Configuration changes + # + # If an enabled tribe's configuration changed, + # process ALL of its ApplicationSets. + # + # This handles: + # + # enabled: false -> true + # + # and changes to allowed_projects. + # ---------------------------------------------------- + + print() + print("============================================================") + print("Tribe configuration changes") + print("============================================================") + + for tribe in sorted(enabled_tribes): + + current = current_config.get(tribe) + previous = base_config.get(tribe) + + if current != previous: + + print( + f"Configuration changed for enabled tribe: {tribe}" + ) + + for filename in application_sets_for_tribe(tribe): + + print( + f"SELECTED due to config change: {filename}" + ) + + selected.add(filename) + + # -------------------------------------------------------- + # Write result + # -------------------------------------------------------- + + selected = sorted(selected) + + with open(args.output, "w", encoding="utf-8") as f: + + for filename in selected: + f.write(filename + "\n") + + print() + print("============================================================") + print("ApplicationSets selected") + print("============================================================") + + if not selected: + print("None") + + for filename in selected: + print(filename) + + print() + print(f"Total: {len(selected)}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/scripts/validate_applicationset.py b/.github/scripts/validate_applicationset.py new file mode 100644 index 0000000..63e6636 --- /dev/null +++ b/.github/scripts/validate_applicationset.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path +import sys + +import yaml + + +def load_config(filename): + + with open(filename, "r", encoding="utf-8") as f: + document = yaml.safe_load(f) or {} + + tribes = document.get("tribes", {}) + + if not isinstance(tribes, dict): + raise ValueError( + "'tribes' must be a YAML mapping" + ) + + return tribes + + +def load_files(filename): + + with open(filename, "r", encoding="utf-8") as f: + return [ + line.strip() + for line in f + if line.strip() + ] + + +def main(): + + parser = argparse.ArgumentParser() + + parser.add_argument( + "--config", + required=True, + help="Tribe/project configuration", + ) + + parser.add_argument( + "--files", + required=True, + help="ApplicationSets to validate", + ) + + args = parser.parse_args() + + config = load_config(args.config) + filenames = load_files(args.files) + + if not filenames: + print("No enabled ApplicationSets to validate.") + return + + errors = [] + + for filename in filenames: + + path = Path(filename) + + print() + print("============================================================") + print(f"Validating: {filename}") + print("============================================================") + + # ---------------------------------------------------- + # Validate directory structure + # ---------------------------------------------------- + + if len(path.parts) != 3: + + errors.append( + f"{filename}: invalid path. Expected " + "//applicationset.yaml" + ) + + continue + + tribe = path.parts[0] + environment = path.parts[1] + + print(f"Tribe: {tribe}") + print(f"Environment: {environment}") + + # ---------------------------------------------------- + # Validate tribe exists + # ---------------------------------------------------- + + if tribe not in config: + + errors.append( + f"{filename}: tribe '{tribe}' is not configured" + ) + + continue + + tribe_config = config[tribe] + + # ---------------------------------------------------- + # Verify tribe is enabled + # ---------------------------------------------------- + + if not tribe_config.get("enabled", False): + + errors.append( + f"{filename}: tribe '{tribe}' is not enabled" + ) + + continue + + allowed_projects = tribe_config.get( + "allowed_projects", + [], + ) + + if not allowed_projects: + + errors.append( + f"{filename}: no allowed projects configured " + f"for tribe '{tribe}'" + ) + + continue + + print( + "Allowed projects: " + + ", ".join(allowed_projects) + ) + + # ---------------------------------------------------- + # Parse ApplicationSet + # ---------------------------------------------------- + + try: + + with open( + filename, + "r", + encoding="utf-8", + ) as f: + + manifest = yaml.safe_load(f) + + except Exception as exc: + + errors.append( + f"{filename}: YAML parsing error: {exc}" + ) + + continue + + if not isinstance(manifest, dict): + + errors.append( + f"{filename}: YAML document must contain an object" + ) + + continue + + # ---------------------------------------------------- + # apiVersion + # ---------------------------------------------------- + + api_version = manifest.get("apiVersion") + + if api_version != "argoproj.io/v1alpha1": + + errors.append( + f"{filename}: apiVersion must be " + "'argoproj.io/v1alpha1'" + ) + + # ---------------------------------------------------- + # kind + # ---------------------------------------------------- + + kind = manifest.get("kind") + + if kind != "ApplicationSet": + + errors.append( + f"{filename}: kind must be 'ApplicationSet'" + ) + + # ---------------------------------------------------- + # metadata + # ---------------------------------------------------- + + metadata = manifest.get("metadata") + + if not isinstance(metadata, dict): + + errors.append( + f"{filename}: metadata is required" + ) + + elif not metadata.get("name"): + + errors.append( + f"{filename}: metadata.name is required" + ) + + # ---------------------------------------------------- + # spec + # ---------------------------------------------------- + + spec = manifest.get("spec") + + if not isinstance(spec, dict): + + errors.append( + f"{filename}: spec is required" + ) + + continue + + if not spec.get("generators"): + + errors.append( + f"{filename}: spec.generators is required" + ) + + # ---------------------------------------------------- + # template + # ---------------------------------------------------- + + template = spec.get("template") + + if not isinstance(template, dict): + + errors.append( + f"{filename}: spec.template is required" + ) + + continue + + template_spec = template.get("spec") + + if not isinstance(template_spec, dict): + + errors.append( + f"{filename}: spec.template.spec is required" + ) + + continue + + # ---------------------------------------------------- + # Argo CD project + # ---------------------------------------------------- + + project = template_spec.get("project") + + if not project: + + errors.append( + f"{filename}: " + "spec.template.spec.project is required" + ) + + continue + + print(f"Actual project: {project}") + + # ---------------------------------------------------- + # Validate project against mapping + # ---------------------------------------------------- + + if project not in allowed_projects: + + errors.append( + f"{filename}: Argo CD project '{project}' " + f"is not allowed for tribe '{tribe}'. " + f"Allowed projects: " + f"{', '.join(allowed_projects)}" + ) + + print("Result: FAIL") + + else: + + print("Result: PASS") + + # -------------------------------------------------------- + # Final result + # -------------------------------------------------------- + + if errors: + + print() + print("============================================================") + print("APPLICATIONSET VALIDATION FAILED") + print("============================================================") + + for error in errors: + print(f"ERROR: {error}") + + sys.exit(1) + + print() + print("============================================================") + print("All ApplicationSets passed validation.") + print("============================================================") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/deploy-applicationset-v1-yaml b/.github/workflows/deploy-applicationset-v1-yaml new file mode 100644 index 0000000..97ddd56 --- /dev/null +++ b/.github/workflows/deploy-applicationset-v1-yaml @@ -0,0 +1,530 @@ +name: Validate and Deploy ApplicationSet + +on: + pull_request: + branches: + - main + paths: + - '*/*/applicationset.yaml' + + push: + branches: + - dev + - main + paths: + - '*/*/applicationset.yaml' + + workflow_dispatch: + +permissions: + contents: read + +jobs: + # ========================================================= + # VALIDATION + # ========================================================= + validate: + name: Validate ApplicationSet + runs-on: ubuntu-latest + + steps: + # ----------------------------------------------------- + # Checkout repository + # ----------------------------------------------------- + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # ----------------------------------------------------- + # Install validation dependencies + # ----------------------------------------------------- + - name: Install YAML validation tools + run: | + python -m pip install --upgrade pip + pip install yamllint pyyaml + + # ----------------------------------------------------- + # Find changed ApplicationSet files + # ----------------------------------------------------- + - name: Find ApplicationSet files + run: | + set -e + + echo "GitHub event: ${{ github.event_name }}" + + # Pull Request + if [ "${{ github.event_name }}" = "pull_request" ]; then + + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + + echo "PR base: ${BASE_SHA}" + echo "PR head: ${HEAD_SHA}" + + git diff --name-only --diff-filter=ACMR "${BASE_SHA}" "${HEAD_SHA}" -- '*/*/applicationset.yaml' > /tmp/appset-files.txt + + # Push to main + elif [ "${{ github.event_name }}" = "push" ]; then + + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + + echo "Push base: ${BASE_SHA}" + echo "Push head: ${HEAD_SHA}" + + # github.event.before can be all zeros on the first push + if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + find . -mindepth 3 -maxdepth 3 -type f -name applicationset.yaml | sed 's#^\./##' | sort > /tmp/appset-files.txt + else + git diff --name-only --diff-filter=ACMR "${BASE_SHA}" "${HEAD_SHA}" -- '*/*/applicationset.yaml' > /tmp/appset-files.txt + fi + + # workflow_dispatch + else + find . -mindepth 3 -maxdepth 3 -type f -name applicationset.yaml | sed 's#^\./##' | sort > /tmp/appset-files.txt + fi + + echo + echo "ApplicationSet files:" + echo "------------------------------------------" + + cat /tmp/appset-files.txt + + echo "------------------------------------------" + + if [ ! -s /tmp/appset-files.txt ]; then + echo "No ApplicationSet files found." + exit 0 + fi + + # ----------------------------------------------------- + # Validate YAML syntax / formatting + # ----------------------------------------------------- + - name: Validate YAML syntax + run: | + set -e + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "Validating YAML: $file" + echo "==========================================" + + yamllint \ + -d '{ + extends: default, + rules: { + line-length: disable, + document-start: disable, + truthy: disable + } + }' \ + "$file" + + done < /tmp/appset-files.txt + + # ----------------------------------------------------- + # Validate basic ApplicationSet structure + # ----------------------------------------------------- + - name: Validate ApplicationSet structure + run: | + python <<'PY' + import sys + import yaml + + errors = [] + + with open("/tmp/appset-files.txt") as files: + filenames = [ + line.strip() + for line in files + if line.strip() + ] + + for filename in filenames: + + print() + print("=" * 60) + print(f"Validating: {filename}") + print("=" * 60) + + try: + with open(filename) as f: + manifest = yaml.safe_load(f) + + except Exception as exc: + errors.append( + f"{filename}: YAML parsing error: {exc}" + ) + continue + + if not isinstance(manifest, dict): + errors.append( + f"{filename}: YAML document must contain an object" + ) + continue + + # apiVersion + if manifest.get("apiVersion") != "argoproj.io/v1alpha1": + errors.append( + f"{filename}: apiVersion must be " + f"'argoproj.io/v1alpha1'" + ) + + # kind + if manifest.get("kind") != "ApplicationSet": + errors.append( + f"{filename}: kind must be 'ApplicationSet'" + ) + + # metadata + metadata = manifest.get("metadata") + + if not isinstance(metadata, dict): + errors.append( + f"{filename}: metadata is required" + ) + else: + if not metadata.get("name"): + errors.append( + f"{filename}: metadata.name is required" + ) + + # spec + spec = manifest.get("spec") + + if not isinstance(spec, dict): + errors.append( + f"{filename}: spec is required" + ) + continue + + # generators + generators = spec.get("generators") + + if not generators: + errors.append( + f"{filename}: spec.generators is required" + ) + + # template + template = spec.get("template") + + if not isinstance(template, dict): + errors.append( + f"{filename}: spec.template is required" + ) + continue + + template_spec = template.get("spec") + + if not isinstance(template_spec, dict): + errors.append( + f"{filename}: spec.template.spec is required" + ) + + if errors: + + print() + print("==========================================") + print("APPLICATIONSET STRUCTURE VALIDATION FAILED") + print("==========================================") + + for error in errors: + print(f"ERROR: {error}") + + sys.exit(1) + + print() + print("ApplicationSet structure validation passed.") + + PY + + # ----------------------------------------------------- + # Validate: + # + # //applicationset.yaml + # + # tribe MUST equal: + # + # spec.template.spec.project + # + # Example: + # + # ige/dev/applicationset.yaml + # project: ige + # + # dfa/prod/applicationset.yaml + # project: dfa + # ----------------------------------------------------- + - name: Validate tribe matches Argo CD project + run: | + python <<'PY' + import sys + import yaml + from pathlib import Path + + errors = [] + + with open("/tmp/appset-files.txt") as files: + filenames = [ + line.strip() + for line in files + if line.strip() + ] + + for filename in filenames: + + path = Path(filename) + + print() + print("=" * 60) + print(f"Checking tribe/project: {filename}") + print("=" * 60) + + # Expected: + # + # tribe/environment/applicationset.yaml + + if len(path.parts) != 3: + errors.append( + f"{filename}: invalid path. Expected " + f"//applicationset.yaml" + ) + continue + + tribe = path.parts[0] + environment = path.parts[1] + + print(f"Tribe: {tribe}") + print(f"Environment: {environment}") + + try: + with open(filename) as f: + manifest = yaml.safe_load(f) + + except Exception as exc: + errors.append( + f"{filename}: cannot parse YAML: {exc}" + ) + continue + + try: + project = ( + manifest["spec"] + ["template"] + ["spec"] + ["project"] + ) + + except (KeyError, TypeError): + + errors.append( + f"{filename}: missing " + f"spec.template.spec.project" + ) + + continue + + print(f"Project: {project}") + + if project != tribe: + + errors.append( + f"{filename}: Argo CD project " + f"'{project}' does not match " + f"tribe '{tribe}'" + ) + + print("RESULT: FAIL") + + else: + + print("RESULT: PASS") + print( + f"Tribe '{tribe}' matches " + f"Argo CD project '{project}'" + ) + + if errors: + + print() + print("==========================================") + print("TRIBE / PROJECT VALIDATION FAILED") + print("==========================================") + + for error in errors: + print(f"ERROR: {error}") + + sys.exit(1) + + print() + print("All tribe/project validations passed.") + + PY + + # # ========================================================= + # # DEPLOYMENT + # # ========================================================= + # deploy: + # name: Deploy ApplicationSet + # runs-on: ubuntu-latest + + # needs: + # - validate + + # # Deploy ONLY after merge/push to main. + # # Never deploy from a Pull Request. + # if: | + # github.event_name == 'push' && + # github.ref == 'refs/heads/main' + + # steps: + # # ----------------------------------------------------- + # # Checkout + # # ----------------------------------------------------- + # - name: Checkout repository + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # # ----------------------------------------------------- + # # Configure Kubernetes authentication + # # + # # GitHub Secret: + # # + # # KUBECONFIG_B64 + # # + # # ----------------------------------------------------- + # - name: Configure Kubernetes + # env: + # KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} + # run: | + # set -e + + # mkdir -p ~/.kube + + # echo "$KUBECONFIG_B64" \ + # | base64 --decode \ + # > ~/.kube/config + + # chmod 600 ~/.kube/config + + # # ----------------------------------------------------- + # # Verify Kubernetes connectivity + # # ----------------------------------------------------- + # - name: Verify Kubernetes connection + # run: | + # kubectl cluster-info + + # echo + # echo "Current context:" + # kubectl config current-context + + # # ----------------------------------------------------- + # # Make sure ApplicationSet CRD exists + # # ----------------------------------------------------- + # - name: Verify ApplicationSet CRD + # run: | + # kubectl get crd applicationsets.argoproj.io + + # # ----------------------------------------------------- + # # Find ApplicationSets changed by this push + # # ----------------------------------------------------- + # - name: Find changed ApplicationSets + # run: | + # set -e + + # BASE_SHA="${{ github.event.before }}" + # HEAD_SHA="${{ github.sha }}" + + # echo "Base SHA: ${BASE_SHA}" + # echo "Head SHA: ${HEAD_SHA}" + + # if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + + # find . \ + # -mindepth 3 \ + # -maxdepth 3 \ + # -type f \ + # -name applicationset.yaml \ + # | sed 's#^\./##' \ + # | sort \ + # > /tmp/appset-files.txt + + # else + + # git diff \ + # --name-only \ + # --diff-filter=ACMR \ + # "${BASE_SHA}" \ + # "${HEAD_SHA}" \ + # -- '*/*/applicationset.yaml' \ + # > /tmp/appset-files.txt + + # fi + + # echo + # echo "ApplicationSets to deploy:" + # echo "------------------------------------------" + + # cat /tmp/appset-files.txt + + # echo "------------------------------------------" + + # # ----------------------------------------------------- + # # Kubernetes + ApplicationSet CRD validation + # # + # # Nothing is actually created here. + # # ----------------------------------------------------- + # - name: Server-side ApplicationSet validation + # run: | + # set -e + + # while read -r file; do + + # [ -z "$file" ] && continue + + # echo + # echo "==========================================" + # echo "Server-side validating: $file" + # echo "==========================================" + + # kubectl apply \ + # --dry-run=server \ + # -f "$file" + + # done < /tmp/appset-files.txt + + # # ----------------------------------------------------- + # # Deploy ApplicationSets + # # ----------------------------------------------------- + # - name: Deploy ApplicationSets + # run: | + # set -e + + # while read -r file; do + + # [ -z "$file" ] && continue + + # echo + # echo "==========================================" + # echo "Deploying: $file" + # echo "==========================================" + + # kubectl apply \ + # -f "$file" + + # done < /tmp/appset-files.txt + + # # ----------------------------------------------------- + # # Show resulting ApplicationSets + # # ----------------------------------------------------- + # - name: Show deployed ApplicationSets + # run: | + # kubectl get applicationsets \ + # -n argocd diff --git a/.github/workflows/deploy-applicationset.yaml b/.github/workflows/deploy-applicationset.yaml index 97ddd56..2ba26ad 100644 --- a/.github/workflows/deploy-applicationset.yaml +++ b/.github/workflows/deploy-applicationset.yaml @@ -1,11 +1,15 @@ name: Validate and Deploy ApplicationSet on: + pull_request: branches: - main paths: - '*/*/applicationset.yaml' + - '.github/config/tribe-projects.yaml' + - '.github/scripts/**' + - '.github/workflows/deploy-applicationset.yaml' push: branches: @@ -13,97 +17,202 @@ on: - main paths: - '*/*/applicationset.yaml' + - '.github/config/tribe-projects.yaml' + - '.github/scripts/**' + - '.github/workflows/deploy-applicationset.yaml' workflow_dispatch: + permissions: contents: read + jobs: + # ========================================================= - # VALIDATION + # 1. VALIDATION + # + # Runs automatically. + # + # This validates: + # + # - YAML syntax + # - ApplicationSet structure + # - tribe/project mapping + # - enabled tribe rollout rules + # + # It does NOT require Kubernetes access. # ========================================================= + validate: + name: Validate ApplicationSet + runs-on: ubuntu-latest + outputs: + appset_count: ${{ steps.select.outputs.count }} + steps: - # ----------------------------------------------------- - # Checkout repository - # ----------------------------------------------------- + - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - # ----------------------------------------------------- - # Install validation dependencies - # ----------------------------------------------------- - - name: Install YAML validation tools + + - name: Install validation tools run: | python -m pip install --upgrade pip - pip install yamllint pyyaml + pip install pyyaml yamllint - # ----------------------------------------------------- - # Find changed ApplicationSet files - # ----------------------------------------------------- - - name: Find ApplicationSet files - run: | - set -e - echo "GitHub event: ${{ github.event_name }}" + - name: Determine base commit + id: base + shell: bash + run: | - # Pull Request if [ "${{ github.event_name }}" = "pull_request" ]; then BASE_SHA="${{ github.event.pull_request.base.sha }}" - HEAD_SHA="${{ github.event.pull_request.head.sha }}" - - echo "PR base: ${BASE_SHA}" - echo "PR head: ${HEAD_SHA}" - - git diff --name-only --diff-filter=ACMR "${BASE_SHA}" "${HEAD_SHA}" -- '*/*/applicationset.yaml' > /tmp/appset-files.txt - # Push to main elif [ "${{ github.event_name }}" = "push" ]; then BASE_SHA="${{ github.event.before }}" - HEAD_SHA="${{ github.sha }}" - echo "Push base: ${BASE_SHA}" - echo "Push head: ${HEAD_SHA}" + else + + BASE_SHA="" + + fi + + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" - # github.event.before can be all zeros on the first push - if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then - find . -mindepth 3 -maxdepth 3 -type f -name applicationset.yaml | sed 's#^\./##' | sort > /tmp/appset-files.txt - else - git diff --name-only --diff-filter=ACMR "${BASE_SHA}" "${HEAD_SHA}" -- '*/*/applicationset.yaml' > /tmp/appset-files.txt - fi + echo "Base SHA: ${BASE_SHA}" + + + - name: Find changed ApplicationSets + shell: bash + run: | + + set -e + + BASE_SHA="${{ steps.base.outputs.sha }}" + + touch /tmp/appset-files-all.txt + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + + echo "Manual workflow execution." + + elif [ -z "${BASE_SHA}" ] || \ + [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + + find . \ + -mindepth 3 \ + -maxdepth 3 \ + -type f \ + -name applicationset.yaml \ + | sed 's#^\./##' \ + | sort \ + > /tmp/appset-files-all.txt - # workflow_dispatch else - find . -mindepth 3 -maxdepth 3 -type f -name applicationset.yaml | sed 's#^\./##' | sort > /tmp/appset-files.txt + + git diff \ + --name-only \ + --diff-filter=ACMR \ + "${BASE_SHA}" \ + "${{ github.sha }}" \ + -- '*/*/applicationset.yaml' \ + > /tmp/appset-files-all.txt + fi echo - echo "ApplicationSet files:" - echo "------------------------------------------" + echo "Changed ApplicationSets:" + cat /tmp/appset-files-all.txt || true - cat /tmp/appset-files.txt - echo "------------------------------------------" + - name: Load previous tribe configuration + shell: bash + run: | + + BASE_SHA="${{ steps.base.outputs.sha }}" + + echo "tribes: {}" \ + > /tmp/tribe-projects-base.yaml + + if [ -z "${BASE_SHA}" ]; then + exit 0 + fi - if [ ! -s /tmp/appset-files.txt ]; then - echo "No ApplicationSet files found." + if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then exit 0 fi - # ----------------------------------------------------- - # Validate YAML syntax / formatting - # ----------------------------------------------------- + if git cat-file \ + -e "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + 2>/dev/null; then + + git show \ + "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + > /tmp/tribe-projects-base.yaml + + fi + + + - name: Select ApplicationSets + id: select + shell: bash + run: | + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + + python \ + .github/scripts/select_appsets.py \ + --config .github/config/tribe-projects.yaml \ + --changed-files /tmp/appset-files-all.txt \ + --output /tmp/appset-files.txt \ + --all-enabled + + else + + python \ + .github/scripts/select_appsets.py \ + --config .github/config/tribe-projects.yaml \ + --base-config /tmp/tribe-projects-base.yaml \ + --changed-files /tmp/appset-files-all.txt \ + --output /tmp/appset-files.txt + + fi + + COUNT=$( + grep -cve '^[[:space:]]*$' \ + /tmp/appset-files.txt \ + 2>/dev/null || true + ) + + echo "count=${COUNT}" >> "$GITHUB_OUTPUT" + + echo + echo "Selected ApplicationSets:" + cat /tmp/appset-files.txt || true + + + - name: Upload selected ApplicationSet list + uses: actions/upload-artifact@v4 + with: + name: selected-applicationsets + path: /tmp/appset-files.txt + if-no-files-found: error + + - name: Validate YAML syntax + if: steps.select.outputs.count != '0' + shell: bash run: | - set -e while read -r file; do @@ -111,7 +220,7 @@ jobs: echo echo "==========================================" - echo "Validating YAML: $file" + echo "yamllint: $file" echo "==========================================" yamllint \ @@ -127,404 +236,217 @@ jobs: done < /tmp/appset-files.txt - # ----------------------------------------------------- - # Validate basic ApplicationSet structure - # ----------------------------------------------------- - - name: Validate ApplicationSet structure + + - name: Validate ApplicationSets + if: steps.select.outputs.count != '0' run: | - python <<'PY' - import sys - import yaml - - errors = [] - - with open("/tmp/appset-files.txt") as files: - filenames = [ - line.strip() - for line in files - if line.strip() - ] - - for filename in filenames: - - print() - print("=" * 60) - print(f"Validating: {filename}") - print("=" * 60) - - try: - with open(filename) as f: - manifest = yaml.safe_load(f) - - except Exception as exc: - errors.append( - f"{filename}: YAML parsing error: {exc}" - ) - continue - - if not isinstance(manifest, dict): - errors.append( - f"{filename}: YAML document must contain an object" - ) - continue - - # apiVersion - if manifest.get("apiVersion") != "argoproj.io/v1alpha1": - errors.append( - f"{filename}: apiVersion must be " - f"'argoproj.io/v1alpha1'" - ) - - # kind - if manifest.get("kind") != "ApplicationSet": - errors.append( - f"{filename}: kind must be 'ApplicationSet'" - ) - - # metadata - metadata = manifest.get("metadata") - - if not isinstance(metadata, dict): - errors.append( - f"{filename}: metadata is required" - ) - else: - if not metadata.get("name"): - errors.append( - f"{filename}: metadata.name is required" - ) - - # spec - spec = manifest.get("spec") - - if not isinstance(spec, dict): - errors.append( - f"{filename}: spec is required" - ) - continue - - # generators - generators = spec.get("generators") - - if not generators: - errors.append( - f"{filename}: spec.generators is required" - ) - - # template - template = spec.get("template") - - if not isinstance(template, dict): - errors.append( - f"{filename}: spec.template is required" - ) - continue - - template_spec = template.get("spec") - - if not isinstance(template_spec, dict): - errors.append( - f"{filename}: spec.template.spec is required" - ) - - if errors: - - print() - print("==========================================") - print("APPLICATIONSET STRUCTURE VALIDATION FAILED") - print("==========================================") - - for error in errors: - print(f"ERROR: {error}") - - sys.exit(1) - - print() - print("ApplicationSet structure validation passed.") - - PY - - # ----------------------------------------------------- - # Validate: - # - # //applicationset.yaml - # - # tribe MUST equal: - # - # spec.template.spec.project - # - # Example: - # - # ige/dev/applicationset.yaml - # project: ige - # - # dfa/prod/applicationset.yaml - # project: dfa - # ----------------------------------------------------- - - name: Validate tribe matches Argo CD project + + python \ + .github/scripts/validate_applicationset.py \ + --config .github/config/tribe-projects.yaml \ + --files /tmp/appset-files.txt + + + - name: Nothing to validate + if: steps.select.outputs.count == '0' run: | - python <<'PY' - import sys - import yaml - from pathlib import Path + echo "No ApplicationSets from enabled tribes require validation." - errors = [] - with open("/tmp/appset-files.txt") as files: - filenames = [ - line.strip() - for line in files - if line.strip() - ] + # ========================================================= + # 2. KUBERNETES SERVER-SIDE DRY-RUN + # + # Runs automatically AFTER validation. + # + # This checks the real Kubernetes API server and the + # installed ApplicationSet CRD. + # + # This job does NOT wait for manual approval. + # + # It runs only for: + # + # - push to main + # - workflow_dispatch + # + # It does not run on pull_request to avoid exposing + # Kubernetes credentials during PR validation. + # ========================================================= - for filename in filenames: + server-dry-run: - path = Path(filename) + name: Kubernetes Server-Side Dry Run - print() - print("=" * 60) - print(f"Checking tribe/project: {filename}") - print("=" * 60) + runs-on: ubuntu-latest + + needs: + - validate - # Expected: - # - # tribe/environment/applicationset.yaml + if: | + needs.validate.outputs.appset_count != '0' && + ( + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'push' && + github.ref == 'refs/heads/main' + ) + ) - if len(path.parts) != 3: - errors.append( - f"{filename}: invalid path. Expected " - f"//applicationset.yaml" - ) - continue - - tribe = path.parts[0] - environment = path.parts[1] - - print(f"Tribe: {tribe}") - print(f"Environment: {environment}") - - try: - with open(filename) as f: - manifest = yaml.safe_load(f) - - except Exception as exc: - errors.append( - f"{filename}: cannot parse YAML: {exc}" - ) - continue - - try: - project = ( - manifest["spec"] - ["template"] - ["spec"] - ["project"] - ) - - except (KeyError, TypeError): - - errors.append( - f"{filename}: missing " - f"spec.template.spec.project" - ) - - continue - - print(f"Project: {project}") - - if project != tribe: - - errors.append( - f"{filename}: Argo CD project " - f"'{project}' does not match " - f"tribe '{tribe}'" - ) - - print("RESULT: FAIL") - - else: - - print("RESULT: PASS") - print( - f"Tribe '{tribe}' matches " - f"Argo CD project '{project}'" - ) - - if errors: - - print() - print("==========================================") - print("TRIBE / PROJECT VALIDATION FAILED") - print("==========================================") - - for error in errors: - print(f"ERROR: {error}") - - sys.exit(1) - - print() - print("All tribe/project validations passed.") - - PY - - # # ========================================================= - # # DEPLOYMENT - # # ========================================================= - # deploy: - # name: Deploy ApplicationSet - # runs-on: ubuntu-latest - - # needs: - # - validate - - # # Deploy ONLY after merge/push to main. - # # Never deploy from a Pull Request. - # if: | - # github.event_name == 'push' && - # github.ref == 'refs/heads/main' - - # steps: - # # ----------------------------------------------------- - # # Checkout - # # ----------------------------------------------------- - # - name: Checkout repository - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # # ----------------------------------------------------- - # # Configure Kubernetes authentication - # # - # # GitHub Secret: - # # - # # KUBECONFIG_B64 - # # - # # ----------------------------------------------------- - # - name: Configure Kubernetes - # env: - # KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} - # run: | - # set -e - - # mkdir -p ~/.kube - - # echo "$KUBECONFIG_B64" \ - # | base64 --decode \ - # > ~/.kube/config - - # chmod 600 ~/.kube/config - - # # ----------------------------------------------------- - # # Verify Kubernetes connectivity - # # ----------------------------------------------------- - # - name: Verify Kubernetes connection - # run: | - # kubectl cluster-info + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + + - name: Download selected ApplicationSet list + uses: actions/download-artifact@v4 + with: + name: selected-applicationsets + path: /tmp - # echo - # echo "Current context:" - # kubectl config current-context - - # # ----------------------------------------------------- - # # Make sure ApplicationSet CRD exists - # # ----------------------------------------------------- - # - name: Verify ApplicationSet CRD - # run: | - # kubectl get crd applicationsets.argoproj.io - - # # ----------------------------------------------------- - # # Find ApplicationSets changed by this push - # # ----------------------------------------------------- - # - name: Find changed ApplicationSets - # run: | - # set -e - - # BASE_SHA="${{ github.event.before }}" - # HEAD_SHA="${{ github.sha }}" - - # echo "Base SHA: ${BASE_SHA}" - # echo "Head SHA: ${HEAD_SHA}" - - # if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then - - # find . \ - # -mindepth 3 \ - # -maxdepth 3 \ - # -type f \ - # -name applicationset.yaml \ - # | sed 's#^\./##' \ - # | sort \ - # > /tmp/appset-files.txt - - # else - - # git diff \ - # --name-only \ - # --diff-filter=ACMR \ - # "${BASE_SHA}" \ - # "${HEAD_SHA}" \ - # -- '*/*/applicationset.yaml' \ - # > /tmp/appset-files.txt - - # fi - - # echo - # echo "ApplicationSets to deploy:" - # echo "------------------------------------------" - - # cat /tmp/appset-files.txt - - # echo "------------------------------------------" - - # # ----------------------------------------------------- - # # Kubernetes + ApplicationSet CRD validation - # # - # # Nothing is actually created here. - # # ----------------------------------------------------- - # - name: Server-side ApplicationSet validation - # run: | - # set -e - - # while read -r file; do - - # [ -z "$file" ] && continue - - # echo - # echo "==========================================" - # echo "Server-side validating: $file" - # echo "==========================================" - - # kubectl apply \ - # --dry-run=server \ - # -f "$file" - - # done < /tmp/appset-files.txt - - # # ----------------------------------------------------- - # # Deploy ApplicationSets - # # ----------------------------------------------------- - # - name: Deploy ApplicationSets - # run: | - # set -e - - # while read -r file; do - - # [ -z "$file" ] && continue - - # echo - # echo "==========================================" - # echo "Deploying: $file" - # echo "==========================================" - - # kubectl apply \ - # -f "$file" - - # done < /tmp/appset-files.txt - - # # ----------------------------------------------------- - # # Show resulting ApplicationSets - # # ----------------------------------------------------- - # - name: Show deployed ApplicationSets - # run: | - # kubectl get applicationsets \ - # -n argocd + + - name: Configure Kubernetes for dry-run + env: + KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64_DRY_RUN }} + shell: bash + run: | + + mkdir -p ~/.kube + + echo "$KUBECONFIG_B64" \ + | base64 --decode \ + > ~/.kube/config + + chmod 600 ~/.kube/config + + + - name: Verify Kubernetes connection + run: | + + kubectl cluster-info + + echo + kubectl config current-context + + + - name: Verify ApplicationSet CRD + run: | + kubectl get crd applicationsets.argoproj.io + + + - name: Server-side ApplicationSet dry-run + shell: bash + run: | + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "Server-side dry-run: $file" + echo "==========================================" + + kubectl apply \ + --dry-run=server \ + -f "$file" + + done < /tmp/appset-files.txt + + + # ========================================================= + # 3. DEPLOY + # + # This job waits for manual approval. + # + # The approval happens AFTER: + # + # - validate + # - kubectl dry-run + # + # Once approved, this job performs: + # + # kubectl apply -f ... + # ========================================================= + + deploy: + + name: Deploy ApplicationSet CD + + runs-on: ubuntu-latest + + needs: + - validate + - server-dry-run + + if: | + needs.validate.outputs.appset_count != '0' && + ( + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'push' && + github.ref == 'refs/heads/main' + ) + ) + + environment: + name: applicationset-cd + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + + - name: Download selected ApplicationSet list + uses: actions/download-artifact@v4 + with: + name: selected-applicationsets + path: /tmp + + + - name: Configure Kubernetes for deployment + env: + KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64_DEPLOY }} + shell: bash + run: | + + mkdir -p ~/.kube + + echo "$KUBECONFIG_B64" \ + | base64 --decode \ + > ~/.kube/config + + chmod 600 ~/.kube/config + + + - name: Verify Kubernetes connection + run: | + + kubectl cluster-info + + echo + kubectl config current-context + + + - name: Deploy ApplicationSets + shell: bash + run: | + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "Deploying: $file" + echo "==========================================" + + kubectl apply \ + -f "$file" + + done < /tmp/appset-files.txt + + + - name: Show ApplicationSets + run: | + kubectl get applicationsets -A \ No newline at end of file diff --git a/pay/dev/applicationset.yaml b/opintel/dev/applicationset.yaml similarity index 92% rename from pay/dev/applicationset.yaml rename to opintel/dev/applicationset.yaml index 91954ab..ec0d226 100644 --- a/pay/dev/applicationset.yaml +++ b/opintel/dev/applicationset.yaml @@ -16,7 +16,7 @@ spec: name: '{{app}}' spec: - project: default + project: doc source: repoURL: https://github.com/ferchdav/argocd-example-apps @@ -34,4 +34,4 @@ spec: syncOptions: - CreateNamespace=true - - PruneLast=true \ No newline at end of file + - PruneLast=true From c05f7e07409fff0aa4eeecc6e7d1fc1e7ba79c55 Mon Sep 17 00:00:00 2001 From: Fernando Davis Date: Fri, 7 Aug 2026 14:38:01 -0400 Subject: [PATCH 5/5] updload v2 --- .../workflows/deploy-applicationset-v2-yaml | 479 ++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 .github/workflows/deploy-applicationset-v2-yaml diff --git a/.github/workflows/deploy-applicationset-v2-yaml b/.github/workflows/deploy-applicationset-v2-yaml new file mode 100644 index 0000000..ecd3696 --- /dev/null +++ b/.github/workflows/deploy-applicationset-v2-yaml @@ -0,0 +1,479 @@ +name: Validate and Deploy ApplicationSet + +on: + + pull_request: + branches: + - main + paths: + - '*/*/applicationset.yaml' + - '.github/config/tribe-projects.yaml' + - '.github/scripts/**' + - '.github/workflows/deploy-applicationset.yaml' + + push: + branches: + - main + paths: + - '*/*/applicationset.yaml' + - '.github/config/tribe-projects.yaml' + - '.github/scripts/**' + - '.github/workflows/deploy-applicationset.yaml' + + workflow_dispatch: + +permissions: + contents: read + + +jobs: + + # ========================================================= + # VALIDATION + # ========================================================= + + validate: + + name: Validate ApplicationSet + + runs-on: ubuntu-latest + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + + # ----------------------------------------------------- + # Dependencies + # ----------------------------------------------------- + + - name: Install validation tools + run: | + python -m pip install --upgrade pip + pip install pyyaml yamllint + + + # ----------------------------------------------------- + # Determine base SHA + # ----------------------------------------------------- + + - name: Determine base commit + id: base + shell: bash + run: | + + if [ "${{ github.event_name }}" = "pull_request" ]; then + + BASE_SHA="${{ github.event.pull_request.base.sha }}" + + elif [ "${{ github.event_name }}" = "push" ]; then + + BASE_SHA="${{ github.event.before }}" + + else + + BASE_SHA="" + + fi + + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" + + echo "Base SHA: ${BASE_SHA}" + + + # ----------------------------------------------------- + # Find changed ApplicationSets + # ----------------------------------------------------- + + - name: Find changed ApplicationSets + shell: bash + run: | + + set -e + + BASE_SHA="${{ steps.base.outputs.sha }}" + + touch /tmp/appset-files-all.txt + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + + echo "Manual workflow execution." + + elif [ -z "${BASE_SHA}" ] || \ + [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + + find . \ + -mindepth 3 \ + -maxdepth 3 \ + -type f \ + -name applicationset.yaml \ + | sed 's#^\./##' \ + | sort \ + > /tmp/appset-files-all.txt + + else + + git diff \ + --name-only \ + --diff-filter=ACMR \ + "${BASE_SHA}" \ + "${{ github.sha }}" \ + -- '*/*/applicationset.yaml' \ + > /tmp/appset-files-all.txt + + fi + + echo + echo "Changed ApplicationSets:" + cat /tmp/appset-files-all.txt || true + + + # ----------------------------------------------------- + # Get tribe configuration from base commit + # ----------------------------------------------------- + + - name: Load previous tribe configuration + shell: bash + run: | + + BASE_SHA="${{ steps.base.outputs.sha }}" + + echo "tribes: {}" \ + > /tmp/tribe-projects-base.yaml + + if [ -z "${BASE_SHA}" ]; then + exit 0 + fi + + if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + exit 0 + fi + + if git cat-file \ + -e "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + 2>/dev/null; then + + git show \ + "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + > /tmp/tribe-projects-base.yaml + + fi + + + # ----------------------------------------------------- + # Select enabled tribes + # ----------------------------------------------------- + + - name: Select ApplicationSets + id: select + shell: bash + run: | + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + + python \ + .github/scripts/select_appsets.py \ + --config .github/config/tribe-projects.yaml \ + --changed-files /tmp/appset-files-all.txt \ + --output /tmp/appset-files.txt \ + --all-enabled + + else + + python \ + .github/scripts/select_appsets.py \ + --config .github/config/tribe-projects.yaml \ + --base-config /tmp/tribe-projects-base.yaml \ + --changed-files /tmp/appset-files-all.txt \ + --output /tmp/appset-files.txt + + fi + + COUNT=$( + grep -cve '^[[:space:]]*$' \ + /tmp/appset-files.txt \ + 2>/dev/null || true + ) + + echo "count=${COUNT}" >> "$GITHUB_OUTPUT" + + + # ----------------------------------------------------- + # YAML lint + # ----------------------------------------------------- + + - name: Validate YAML syntax + if: steps.select.outputs.count != '0' + shell: bash + run: | + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "yamllint: $file" + echo "==========================================" + + yamllint \ + -d '{ + extends: default, + rules: { + line-length: disable, + document-start: disable, + truthy: disable + } + }' \ + "$file" + + done < /tmp/appset-files.txt + + + # ----------------------------------------------------- + # ApplicationSet validation + # ----------------------------------------------------- + + - name: Validate ApplicationSets + if: steps.select.outputs.count != '0' + run: | + + python \ + .github/scripts/validate_applicationset.py \ + --config .github/config/tribe-projects.yaml \ + --files /tmp/appset-files.txt + + + - name: Nothing to validate + if: steps.select.outputs.count == '0' + run: | + echo "No ApplicationSets from enabled tribes require validation." + + + # ========================================================= + # DEPLOY + # ========================================================= + + deploy: + + name: Deploy ApplicationSet + + runs-on: ubuntu-latest + + needs: + - validate + + if: | + github.event_name == 'push' && + github.ref == 'refs/heads/main' + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + + - name: Install Python dependency + run: | + python -m pip install --upgrade pip + pip install pyyaml + + + # ----------------------------------------------------- + # Previous config + # ----------------------------------------------------- + + - name: Load previous tribe configuration + shell: bash + run: | + + BASE_SHA="${{ github.event.before }}" + + echo "tribes: {}" \ + > /tmp/tribe-projects-base.yaml + + if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + exit 0 + fi + + if git cat-file \ + -e "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + 2>/dev/null; then + + git show \ + "${BASE_SHA}:.github/config/tribe-projects.yaml" \ + > /tmp/tribe-projects-base.yaml + + fi + + + # ----------------------------------------------------- + # Changed ApplicationSets + # ----------------------------------------------------- + + - name: Find changed ApplicationSets + shell: bash + run: | + + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + + touch /tmp/appset-files-all.txt + + if [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + + find . \ + -mindepth 3 \ + -maxdepth 3 \ + -type f \ + -name applicationset.yaml \ + | sed 's#^\./##' \ + | sort \ + > /tmp/appset-files-all.txt + + else + + git diff \ + --name-only \ + --diff-filter=ACMR \ + "${BASE_SHA}" \ + "${HEAD_SHA}" \ + -- '*/*/applicationset.yaml' \ + > /tmp/appset-files-all.txt + + fi + + + # ----------------------------------------------------- + # Select enabled ApplicationSets + # ----------------------------------------------------- + + - name: Select ApplicationSets + id: select + shell: bash + run: | + + python \ + .github/scripts/select_appsets.py \ + --config .github/config/tribe-projects.yaml \ + --base-config /tmp/tribe-projects-base.yaml \ + --changed-files /tmp/appset-files-all.txt \ + --output /tmp/appset-files.txt + + COUNT=$( + grep -cve '^[[:space:]]*$' \ + /tmp/appset-files.txt \ + 2>/dev/null || true + ) + + echo "count=${COUNT}" >> "$GITHUB_OUTPUT" + + + # ----------------------------------------------------- + # Kubernetes authentication + # ----------------------------------------------------- + + - name: Configure Kubernetes + if: steps.select.outputs.count != '0' + env: + KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} + shell: bash + run: | + + mkdir -p ~/.kube + + echo "$KUBECONFIG_B64" \ + | base64 --decode \ + > ~/.kube/config + + chmod 600 ~/.kube/config + + + # ----------------------------------------------------- + # Connection + # ----------------------------------------------------- + + - name: Verify Kubernetes connection + if: steps.select.outputs.count != '0' + run: | + + kubectl cluster-info + + echo + kubectl config current-context + + + # ----------------------------------------------------- + # ApplicationSet CRD + # ----------------------------------------------------- + + - name: Verify ApplicationSet CRD + if: steps.select.outputs.count != '0' + run: | + kubectl get crd applicationsets.argoproj.io + + + # ----------------------------------------------------- + # Server-side dry-run + # ----------------------------------------------------- + + - name: Server-side validation + if: steps.select.outputs.count != '0' + shell: bash + run: | + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "Server-side validation: $file" + echo "==========================================" + + kubectl apply \ + --dry-run=server \ + -f "$file" + + done < /tmp/appset-files.txt + + + # ----------------------------------------------------- + # Deployment + # ----------------------------------------------------- + + - name: Deploy ApplicationSets + if: steps.select.outputs.count != '0' + shell: bash + run: | + + while read -r file; do + + [ -z "$file" ] && continue + + echo + echo "==========================================" + echo "Deploying: $file" + echo "==========================================" + + kubectl apply \ + -f "$file" + + done < /tmp/appset-files.txt + + + - name: Show ApplicationSets + if: steps.select.outputs.count != '0' + run: | + kubectl get applicationsets -A + + + - name: Nothing to deploy + if: steps.select.outputs.count == '0' + run: | + echo "No enabled ApplicationSets require deployment." \ No newline at end of file