Whole lot of ansible clean up

.github/actions/nix-install-ephemeral/action.yml

@@ -1,14 +1,22 @@
 name: 'Install Nix on ephemeral runners'
 description: 'Installs Nix and sets up AWS credentials to push to the Nix binary cache'
 inputs:
+  aws-region:
+    description: 'AWS region for the Nix binary cache S3 bucket'
+    required: false
+    default: 'us-east-1'
+  force-clean:
+    description: Whether to delete pre-existing nix paths
+    required: false
+    default: 'false'
   push-to-cache:
     description: 'Whether to push build outputs to the Nix binary cache'
     required: false
     default: 'false'
-  aws-region:
-    description: 'AWS region for the Nix binary cache S3 bucket'
+  multi-user:
+    description: Whether to install nix in multi-user mode (default true) or single-user mode
     required: false
-    default: 'us-east-1'
+    default: 'true'
 runs:
   using: 'composite'
   steps:
@@ -20,6 +28,13 @@ runs:
         aws-region: ${{ inputs.aws-region }}
         output-credentials: true
         role-duration-seconds: 7200
+    - name: Force Clean Pre-Existing Nix Paths
+      if: inputs.force-clean == 'true'
+      shell: bash
+      run: sudo rm -rf /nix /etc/nix "$HOME/.nix-profile" "$HOME/.nix-channels" "$HOME/.config/nix"
+    - name: Ensure /etc/nix exists
+      shell: bash
+      run: sudo mkdir -p /etc/nix
     - name: Setup AWS credentials for Nix
       if: ${{ inputs.push-to-cache == 'true' }}
       shell: bash
@@ -28,7 +43,6 @@ runs:
         sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
         sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN
         sudo -H aws configure set region ${{ inputs.aws-region }}
-        sudo mkdir -p /etc/nix
         sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
         cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null
         #!/usr/bin/env bash
@@ -44,34 +58,50 @@ runs:
     - name: Install Nix
       shell: bash
       run: |
-        sudo tee /tmp/nix-extra.conf > /dev/null <<'NIXCONF'
+        if [[ "${{ inputs.multi-user }}" == true ]]; then
+          daemon=--daemon
+          nixconfdir=/etc/nix
+          path=/nix/var/nix/profiles/default/bin
+          tmpconf=/tmp/nix-extra.conf
+
+          echo 'access-tokens = github.com=${{ github.token }}' | sudo tee $nixconfdir/github.nix.conf >/dev/null
+          sudo chmod 400 $nixconfdir/github.nix.conf
+        else
+          daemon=--no-daemon
+          nixconfdir=$HOME/.config/nix
+          path=$HOME/.nix-profile/bin
+          tmpconf=$nixconfdir/nix.conf # --nix-extra-conf-files is ignored for single-user installs
+
+          mkdir -p "$(dirname $tmpconf)"
+          echo 'access-tokens = github.com=${{ github.token }}' | tee $nixconfdir/github.nix.conf >/dev/null
+          chmod 400 $nixconfdir/github.nix.conf
+        fi
+
+        cat >$tmpconf <<'NIXCONF'
         always-allow-substitutes = true
-        extra-experimental-features = nix-command flakes
-        extra-system-features = kvm
+        extra-experimental-features = flakes nix-command
+        extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com
+        extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=
         max-jobs = 4
-        substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-        trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
         NIXCONF
 
-        if [ "${{ inputs.push-to-cache }}" = "true" ]; then
-          echo "post-build-hook = /etc/nix/upload-to-cache.sh" | sudo tee -a /tmp/nix-extra.conf > /dev/null
+
+        if [[ -e /dev/kvm ]]; then
+          echo 'extra-system-features = kvm' >>$tmpconf
         fi
 
-        curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf
+        if [[ "${{ inputs.push-to-cache }}" == true ]]; then
+          echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>$tmpconf
+        fi
 
-        cat /etc/nix/nix.conf
+        echo "!include $nixconfdir/github.nix.conf" >>$tmpconf
+
+        curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file $tmpconf
+
+        sudo cat $nixconfdir/nix.conf
 
         # Add nix to PATH for subsequent steps
-        echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
-        # Source the daemon profile so nix works in this step too
-        . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
-    - name: Configure Nix with GitHub token
-      shell: bash
-      env:
-        GH_TOKEN: ${{ github.token }}
-      run: |
-        echo "access-tokens = github.com=$GH_TOKEN" | sudo tee -a /etc/nix/nix.conf > /dev/null
-        sudo systemctl restart nix-daemon || true
+        echo "$path" >> "$GITHUB_PATH"
     - name: Print Nix version
       shell: bash
       run: nix --version

.github/workflows/qemu-image-build.yml

@@ -3,11 +3,11 @@ name: Build QEMU image
 on:
   push:
     paths:
-      - '.github/workflows/qemu-image-build.yml'
-      - 'qemu-arm64-nix.pkr.hcl'
-      - 'common-nix.vars.pkr.hcl'
-      - 'ansible/vars.yml'
-      - 'scripts/*'
+      - .github/workflows/qemu-image-build.yml
+      - ansible/vars.yml
+      - nix/packages/build-qemu-image/*
+      - qemu.pkr.hcl
+      - scripts/*
   workflow_dispatch:
 
 permissions:
@@ -64,35 +64,20 @@ jobs:
           echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
           echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
 
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq
-          PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils
-          sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils
+      - name: Install Nix
+        uses: ./.github/actions/nix-install-ephemeral
+        with:
+          force-clean: "true"
+          multi-user: "false"
 
       - name: Build QEMU artifact
-        run: |
-          make init
-          GIT_SHA=${{github.sha}}
-          export PACKER_LOG=1
-          packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl
+        run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          echo "version=$PG_VERSION" >> $GITHUB_OUTPUT
 
       - name: configure aws credentials - staging
         if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }}

.gitignore

@@ -12,6 +12,10 @@
 # ansible related
 ansible/image-manifest*.json
 
+# packer releasted
+output-cloudimg/
+packer-qemu-working/
+
 # python related
 *$py.class
 *.py[cod]
@@ -21,5 +25,4 @@ venv/
 #nix related
 common-nix.vars.pkr.hcl
 db/schema.sql
-nixos.qcow2
 result*

ansible/files/adminapi.service.j2

@@ -14,7 +14,7 @@ Restart=always
 RestartSec=3
 TimeoutStopSec=10
 Environment="AWS_USE_DUALSTACK_ENDPOINT=true"
-{% if qemu_mode is defined and qemu_mode %}
+{% if qemu %}
 Environment="AWS_SDK_LOAD_CONFIG=true"
 {% endif %}
 StandardOutput=journal

ansible/files/gotrue.service.j2

@@ -90,7 +90,7 @@ Environment=GOTRUE_RELOADING_POLLER_ENABLED=false
 # ensures only 1 reload operation occurs during a burst of config updates.
 Environment=GOTRUE_RELOADING_GRACE_PERIOD_INTERVAL=2s
 
-{% if qemu_mode is defined and qemu_mode %}
+{% if qemu %}
 # v3 does not use filesystem notifications for config reloads.
 Environment=GOTRUE_RELOADING_NOTIFY_ENABLED=false
 {% else %}

ansible/files/postgres_exporter.service.j2

@@ -6,7 +6,7 @@ StartLimitBurst=0
 
 [Service]
 Type=simple
-ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal {% if qemu_mode is defined and qemu_mode %}--no-collector.database {% endif %}
+ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal {% if qemu %}--no-collector.database {% endif %}
 
 User=postgres
 Group=postgres

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2

@@ -6,7 +6,7 @@ Requires=local-fs.target
 
 [Service]
 Type=oneshot
-{% if qemu_mode is defined and qemu_mode %}
+{% if qemu %}
 ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml salt --apply --store-result --targets-key  targetsv3-main.yaml
 User=root
 Group=root

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2

@@ -4,7 +4,7 @@ Requires=supabase-admin-agent_salt.service
 
 [Timer]
 {# We're using a significantly lower frequency for triggering this agent on qemu images for the moment. Once we've performed additional validations re: the aggregate impact of running it more frequently, the frequency can be increased. #}
-{% if qemu_mode is defined and qemu_mode %}
+{% if qemu %}
 OnCalendar=*-*-* 0,6,12,18:00:00
 RandomizedDelaySec={{ supabase_admin_agent_splay }}
 AccuracySec=1h