benchmarks.yml

name: Benchmarks
on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      mode:
        description: Run harness contracts or the full database suite
        type: choice
        options: [contracts, full]
        default: full
permissions: {contents: read, actions: read}
concurrency:
  group: keyload-benchmarks-${{ github.ref }}
  cancel-in-progress: false
env:
  NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages
jobs:
  comparison-images:
    name: Build Docker images
    if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    timeout-minutes: 150
    outputs:
      databases: ${{ steps.plan.outputs.database_matrices }}
      bundle-artifact-id: ${{ steps.images-bundle.outputs.artifact-id }}
      runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }}
    env:
      GH_TOKEN: ${{ github.token }}
      KEYLOAD_COMPARISON_JOB_NAME: Build Docker images
    steps:
      - name: Download source code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      - name: Set up .NET
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
        with:
          global-json-file: global.json
      - name: Restore .NET packages once
        run: dotnet restore KeyLoad.slnx
      - name: Build the complete solution once
        run: dotnet build KeyLoad.slnx --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned
      - name: Build current Debug analyzer for native formatting
        run: dotnet build src/KeyLoad.Analyzers/KeyLoad.Analyzers.csproj --no-restore --configuration Debug
      - name: Check code formatting
        run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore
      - name: Check repository rules
        run: node scripts/Features/RepositoryGovernance/verify.mjs
      - name: Test benchmark contracts
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/*/*' --KeyLoadTests:ResultsDirectory=TestResults/benchmark-contracts --KeyLoadTests:ReportTrx=true
      - name: Test canonical benchmark worker selection
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/(BenchmarkSessionContractTests|SessionCohortContractTests|CompiledBenchmarkRuntimeBundleTests)/*' --KeyLoadTests:ResultsDirectory=TestResults/session-contracts --KeyLoadTests:ReportTrx=true
      - name: Test prompt termination on Aspire resource failures
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/AspireFailure*/*' --KeyLoadTests:ResultsDirectory=TestResults/aspire-failure --KeyLoadTests:ReportTrx=true
      - name: Plan the shared database sessions
        id: plan
        run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT"
      - name: Save the canonical session plan
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-plan
          path: ${{ runner.temp }}/session-plan.json
          if-no-files-found: error
          retention-days: 90
      - name: Check Docker
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: docker version
      - name: Check bounded Docker registry readiness
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageRegistryReadinessTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/registry-readiness
      - name: Build KeyLoad server and benchmark Docker images
        id: images
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/BenchmarkComparisons/prepare-images.mjs
      - name: Configure Docker image references
        if: (github.event_name == 'push' || inputs.mode == 'full')
        env:
          KEYLOAD_SERVER_IMAGE: ${{ steps.images.outputs.server-image }}
          KEYLOAD_RUNNER_IMAGE: ${{ steps.images.outputs.load-generator-image }}
        run: |
          test -n "$KEYLOAD_SERVER_IMAGE" && test -n "$KEYLOAD_RUNNER_IMAGE"
          printf 'KeyLoad__ContainerImages__Server=%s\nBenchmarks__ContainerImages__LoadGenerator=%s\nKEYLOAD_IMAGE_RECEIPT=%s\n' "$KEYLOAD_SERVER_IMAGE" "$KEYLOAD_RUNNER_IMAGE" "$RUNNER_TEMP/keyload-images/image-receipt.json" >> "$GITHUB_ENV"
      - name: Check native benchmark preparation contracts
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: ./.github/workflows/Features/BenchmarkComparisons/SessionContracts
      - name: Check Docker image export and import
        id: roundtrip
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageBundleRealTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/image-roundtrip
      - name: Verify GitHub job identity
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedGitHubCurrentJobTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/job-identity
      - name: Save Docker images
        id: images-bundle
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-image-bundle
          path: ${{ runner.temp }}/keyload-image-bundle/*
          if-no-files-found: error
          compression-level: 0
          retention-days: 90
      - name: Pack the compiled benchmark runtime
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack
      - name: Save compiled runtime identities
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-compiled-runtime-identity
          path: |
            artifacts/benchmark-runtime-bundle/benchmark-runtime.json
            artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json
          if-no-files-found: error
          retention-days: 90
      - name: Save the compiled benchmark runtime
        id: runtime-bundle
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-compiled-test-bundle
          path: artifacts/benchmark-runtime-bundle/*
          if-no-files-found: error
          compression-level: 0
          retention-days: 90
      - name: Clean up Docker registry after image failure
        if: ${{ always() && steps.roundtrip.outcome != 'success' && (steps.images.outcome == 'success' || steps.images.outcome == 'failure' || steps.images.outcome == 'cancelled') }}
        run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs
      - name: Save original preparation diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-images-qualification
          path: |
            TestResults/**
            artifacts/qualification/**
            artifacts/code-quality/**
            ${{ runner.temp }}/keyload-images/**
            ${{ runner.temp }}/keyload-images-build/**
            ${{ runner.temp }}/keyload-cell-github/**
          if-no-files-found: error
          retention-days: 90
  comparison-keyload:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).keyload }}
    env: &database-environment
      GH_TOKEN: ${{ github.token }}
      KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }}
      KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }}
      Benchmarks__Target: ${{ matrix.target }}
      Benchmarks__NodeCount: ${{ matrix.nodeCount }}
      Benchmarks__Session: 'true'
      Benchmarks__SessionId: ${{ matrix.id }}
      Benchmarks__EvidenceProfile: session-v1
    steps: &database-steps
      - name: Download source code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      - name: Prepare the native database session
        id: images
        uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup
        with:
          image-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }}
          runtime-artifact-id: ${{ needs.comparison-images.outputs.runtime-artifact-id }}
      - name: Run database workload
        id: workload
        run: dotnet tests/KeyLoad.ComparisonTests/bin/Release/net10.0/KeyLoad.ComparisonTests.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m
      - name: Record benchmark availability
        if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }}
        env:
          KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }}
        run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs
      - name: Save benchmark results
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: ${{ matrix.artifactName }}
          path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/**
          if-no-files-found: error
          retention-days: 90
      - name: End this job and retain native diagnostics
        if: always()
        uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown
        with:
          setup-outcome: ${{ steps.images.outcome }}
          artifact-name: comparison-case-qualification-session-${{ matrix.id }}
  comparison-postgresql:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).postgresql }}
    env: *database-environment
    steps: *database-steps
  comparison-qdrant:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).qdrant }}
    env: *database-environment
    steps: *database-steps
  comparison-rabbitmq:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).rabbitmq }}
    env: *database-environment
    steps: *database-steps
  comparison-redis:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).redis }}
    env: *database-environment
    steps: *database-steps
  comparison-neo4j:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).neo4j }}
    env: *database-environment
    steps: *database-steps
  comparison-mongodb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).mongodb }}
    env: *database-environment
    steps: *database-steps
  comparison-opensearch:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).opensearch }}
    env: *database-environment
    steps: *database-steps
  comparison-kurrentdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).kurrentdb }}
    env: *database-environment
    steps: *database-steps
  comparison-surrealdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).surrealdb }}
    env: *database-environment
    steps: *database-steps
  comparison-helixdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).helixdb }}
    env: *database-environment
    steps: *database-steps
  keyload-functional-heavy-load:
    name: KeyLoad functional RF3 million-record ingestion
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-images]
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Download source code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      - name: Prepare real functional RF3 ingestion
        uses: ./.github/workflows/Features/BenchmarkComparisons/FunctionalHeavyLoadSetup
        with:
          bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }}
      - name: Test exclusive heavy-load admission
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=rf3 '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/rf3-heavy-admission --KeyLoadTests:ReportTrx=true
      - name: Test ingestion while SDK and MCP reads are active
        run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=rf3 '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadRf3Tests/*' --KeyLoadTests:HeavyLoad:Enabled=true --KeyLoadTests:TimeoutMinutes=140 --KeyLoadTests:ResultsDirectory=TestResults/rf3-heavy-load --KeyLoadTests:ReportTrx=true
      - name: Clean up this job's Docker registry
        if: always()
        run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs
      - name: Save original functional load results and image receipts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: keyload-functional-heavy-load
          path: |
            TestResults/rf3-heavy-admission/**
            TestResults/rf3-heavy-load/**
            ${{ runner.temp }}/keyload-images/**
            artifacts/code-quality/**
          if-no-files-found: error
          retention-days: 90
  comparison-aggregate:
    name: Combine benchmark results
    needs: [comparison-images, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load]
    if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-images.result == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 150
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Download source code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      - name: Download original authenticated session results
        run: |
          mkdir -p artifacts/comparisons/sessions
          node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider"
      - name: Check complete session accounting
        run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate"
      - name: Save combined benchmark results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-session-suite
          path: artifacts/comparisons/sessions/aggregate/**
          if-no-files-found: error
          retention-days: 90
      - name: Save GitHub result verification
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: comparison-session-provider-evidence
          path: |
            artifacts/comparisons/sessions/provider/github/**
            artifacts/comparisons/sessions/provider/proof.json
          if-no-files-found: error
          retention-days: 90
  website-trigger:
    name: Trigger Website
    needs: [comparison-images, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load, comparison-aggregate]
    if: >-
      always() && !cancelled() &&
      (github.event_name == 'push' || inputs.mode == 'full') &&
      needs.comparison-aggregate.result == 'success' &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'skipped') &&
      github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 &&
      github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions: {actions: write}
    env:
      GH_TOKEN: ${{ github.token }}
      GH_REPO: ${{ github.repository }}
      BENCHMARK_RUN_ID: ${{ github.run_id }}
    steps:
      - name: Start the separate Website workflow
        run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID"