From 079096df7efe5a611712884f9348d806e1408b7f Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 1/9] ci/nix-install-ephemeral: Clean up action's nix-extra.conf * No need for sudo to write to /tmp * Put the github token into a separate file to avoid leaking by mistake * Do not set extra-system-features = kvm until we know we have a kvm device * Drop sourcing the shell env since its not actually used in this step --- .../actions/nix-install-ephemeral/action.yml | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 638111853a..83bd67100e 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -44,34 +44,35 @@ runs: - name: Install Nix shell: bash run: | - sudo tee /tmp/nix-extra.conf > /dev/null <<'NIXCONF' + sudo tee /etc/nix/github.nix.conf >/dev/null <<'EOF' + access-tokens = github.com=${{ github.token }} + EOF + chmod 400 /etc/nix/github.nix.conf + cat >/tmp/nix-extra.conf <<'NIXCONF' always-allow-substitutes = true - extra-experimental-features = nix-command flakes - extra-system-features = kvm + extra-experimental-features = flakes nix-command + extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com + extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= max-jobs = 4 - substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com - trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= NIXCONF - if [ "${{ inputs.push-to-cache }}" = "true" ]; then - echo "post-build-hook = /etc/nix/upload-to-cache.sh" | sudo tee -a /tmp/nix-extra.conf > /dev/null + + if [[ -e /dev/kvm ]]; then + echo 'extra-system-features = kvm' >>/tmp/nix-extra.conf + fi + + if [[ "${{ inputs.push-to-cache }}" == true ]]; then + echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>/tmp/nix-extra.conf fi + echo '!include /etc/nix/github.nix.conf' >>/tmp/nix-extra.conf + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf cat /etc/nix/nix.conf # Add nix to PATH for subsequent steps echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" - # Source the daemon profile so nix works in this step too - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - - name: Configure Nix with GitHub token - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "access-tokens = github.com=$GH_TOKEN" | sudo tee -a /etc/nix/nix.conf > /dev/null - sudo systemctl restart nix-daemon || true - name: Print Nix version shell: bash run: nix --version From d4ed77cbeaa2fb2619ec65ea1dae2b711f341047 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 2/9] ci/nix-install-ephemeral: Add multi-user input and single-user install support This way we can use this action in the qemu image build workflow that runs in dind and requires single-user install. --- .../actions/nix-install-ephemeral/action.yml | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 83bd67100e..a0557554dc 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -1,14 +1,18 @@ name: 'Install Nix on ephemeral runners' description: 'Installs Nix and sets up AWS credentials to push to the Nix binary cache' inputs: + aws-region: + description: 'AWS region for the Nix binary cache S3 bucket' + required: false + default: 'us-east-1' push-to-cache: description: 'Whether to push build outputs to the Nix binary cache' required: false default: 'false' - aws-region: - description: 'AWS region for the Nix binary cache S3 bucket' + multi-user: + description: Whether to install nix in multi-user mode (default true) or single-user mode required: false - default: 'us-east-1' + default: 'true' runs: using: 'composite' steps: @@ -44,11 +48,26 @@ runs: - name: Install Nix shell: bash run: | - sudo tee /etc/nix/github.nix.conf >/dev/null <<'EOF' - access-tokens = github.com=${{ github.token }} - EOF - chmod 400 /etc/nix/github.nix.conf - cat >/tmp/nix-extra.conf <<'NIXCONF' + if [[ "${{ inputs.multi-user }}" == true ]]; then + daemon=--daemon + nixconfdir=/etc/nix + path=/nix/var/nix/profiles/default/bin + tmpconf=/tmp/nix-extra.conf + + echo 'access-tokens = github.com=${{ github.token }}' | sudo tee $nixconfdir/github.nix.conf >/dev/null + sudo chmod 400 $nixconfdir/github.nix.conf + else + daemon=--no-daemon + nixconfdir=$HOME/.config/nix + path=$HOME/.nix-profile/bin + tmpconf=$nixconfdir/nix.conf # --nix-extra-conf-files is ignored for single-user installs + + mkdir -p "$(dirname $tmpconf)" + echo 'access-tokens = github.com=${{ github.token }}' | tee $nixconfdir/github.nix.conf >/dev/null + chmod 400 $nixconfdir/github.nix.conf + fi + + cat >$tmpconf <<'NIXCONF' always-allow-substitutes = true extra-experimental-features = flakes nix-command extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com @@ -58,21 +77,21 @@ runs: if [[ -e /dev/kvm ]]; then - echo 'extra-system-features = kvm' >>/tmp/nix-extra.conf + echo 'extra-system-features = kvm' >>$tmpconf fi if [[ "${{ inputs.push-to-cache }}" == true ]]; then - echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>/tmp/nix-extra.conf + echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>$tmpconf fi - echo '!include /etc/nix/github.nix.conf' >>/tmp/nix-extra.conf + echo "!include $nixconfdir/github.nix.conf" >>$tmpconf - curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file $tmpconf - cat /etc/nix/nix.conf + sudo cat $nixconfdir/nix.conf # Add nix to PATH for subsequent steps - echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" + echo "$path" >> "$GITHUB_PATH" - name: Print Nix version shell: bash run: nix --version From 95e101ecc81dbfa301f211c433894072dbf1fba2 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 3/9] ci/nix-install-ephemeral: Add force-clean input Signals to force clean known nix paths. This is temporarily useful for native arm runners that have a pre-existing detsys nix install. --- .github/actions/nix-install-ephemeral/action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index a0557554dc..af4c2292d4 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -5,6 +5,10 @@ inputs: description: 'AWS region for the Nix binary cache S3 bucket' required: false default: 'us-east-1' + force-clean: + description: Whether to delete pre-existing nix paths + required: false + default: 'false' push-to-cache: description: 'Whether to push build outputs to the Nix binary cache' required: false @@ -24,6 +28,13 @@ runs: aws-region: ${{ inputs.aws-region }} output-credentials: true role-duration-seconds: 7200 + - name: Force Clean Pre-Existing Nix Paths + if: inputs.force-clean == 'true' + shell: bash + run: sudo rm -rf /nix /etc/nix "$HOME/.nix-profile" "$HOME/.nix-channels" "$HOME/.config/nix" + - name: Ensure /etc/nix exists + shell: bash + run: sudo mkdir -p /etc/nix - name: Setup AWS credentials for Nix if: ${{ inputs.push-to-cache == 'true' }} shell: bash @@ -32,7 +43,6 @@ runs: sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN sudo -H aws configure set region ${{ inputs.aws-region }} - sudo mkdir -p /etc/nix sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null #!/usr/bin/env bash From 2e0a33964b7874be3671c03b6d5432c72c3b84e5 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 14 Jul 2026 22:05:53 -0400 Subject: [PATCH 4/9] packer/qemu: Wrap image build in nix --- .github/workflows/qemu-image-build.yml | 41 ++++++------------- .gitignore | 5 ++- .../build-qemu-image/build-qemu-image.sh | 37 +++++++++++++++++ nix/packages/build-qemu-image/default.nix | 23 +++++++++++ nix/packages/default.nix | 1 + qemu-arm64-nix.pkr.hcl | 40 ++++++++---------- 6 files changed, 96 insertions(+), 51 deletions(-) create mode 100644 nix/packages/build-qemu-image/build-qemu-image.sh create mode 100644 nix/packages/build-qemu-image/default.nix diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index f3f9e417a8..2ab5ce8504 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -3,11 +3,11 @@ name: Build QEMU image on: push: paths: - - '.github/workflows/qemu-image-build.yml' - - 'qemu-arm64-nix.pkr.hcl' - - 'common-nix.vars.pkr.hcl' - - 'ansible/vars.yml' - - 'scripts/*' + - .github/workflows/qemu-image-build.yml + - ansible/vars.yml + - nix/packages/build-qemu-image/* + - qemu-arm64-nix.pkr.hcl + - scripts/* workflow_dispatch: permissions: @@ -64,35 +64,20 @@ jobs: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl - run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq - PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) - PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes - echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file - echo "" >> common-nix.vars.pkr.hcl - - # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + - name: Install Nix + uses: ./.github/actions/nix-install-ephemeral + with: + force-clean: "true" + multi-user: "false" - name: Build QEMU artifact - run: | - make init - GIT_SHA=${{github.sha}} - export PACKER_LOG=1 - packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl + run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" - name: Grab release version id: process_release_version run: | - VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - echo "version=$VERSION" >> $GITHUB_OUTPUT + PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + echo "version=$PG_VERSION" >> $GITHUB_OUTPUT - name: configure aws credentials - staging if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} diff --git a/.gitignore b/.gitignore index f24b95b1b3..15429d4357 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ # ansible related ansible/image-manifest*.json +# packer releasted +output-cloudimg/ +packer-qemu-working/ + # python related *$py.class *.py[cod] @@ -21,5 +25,4 @@ venv/ #nix related common-nix.vars.pkr.hcl db/schema.sql -nixos.qcow2 result* diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh new file mode 100644 index 0000000000..23e43b659e --- /dev/null +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -0,0 +1,37 @@ +# shellcheck shell=bash + +# nix-built qemu resolves its libraries via RPATH; an inherited +# LD_LIBRARY_PATH (e.g. from a devshell) can inject a mismatched glibc and +# break `qemu-system-* --version`, so drop it. +unset LD_LIBRARY_PATH + +postgres_major_version=${1:-} +if [ -z "$postgres_major_version" ]; then + echo "Usage: build-qemu-image [arch]" >&2 + exit 1 +fi + +qemu_bin=$(which qemu-system-aarch64) +fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-aarch64-code.fd +fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-arm-vars.fd + +mkdir -p packer-qemu-working +# qemu writes to the vars pflash during boot, so make a local writable copy. +cp "$fw_vars" packer-qemu-working/fw-vars.fd +chmod u+w packer-qemu-working/fw-vars.fd + +git_sha=${GIT_SHA:-$(git rev-parse HEAD)} +postgres_version=$(yq -r ".postgres_release[\"postgres$postgres_major_version\"]" ansible/vars.yml) + +# Build the cloud-init seed ISO before invoking packer. +cloud-localds packer-qemu-working/seeds-cloudimg.iso user-data-cloudimg meta-data + +packer init qemu-nix-arm64.pkr.hcl +PACKER_LOG=${PACKER_LOG:-1} packer build \ + -var "fw_code=$fw_code" \ + -var "fw_vars=packer-qemu-working/fw-vars.fd" \ + -var "git_sha=$git_sha" \ + -var "postgres-version=$postgres_version" \ + -var "postgres_major_version=$postgres_major_version" \ + -var "seeds_iso=packer-qemu-working/seeds-cloudimg.iso" \ + qemu-nix-arm64.pkr.hcl diff --git a/nix/packages/build-qemu-image/default.nix b/nix/packages/build-qemu-image/default.nix new file mode 100644 index 0000000000..94762f6f47 --- /dev/null +++ b/nix/packages/build-qemu-image/default.nix @@ -0,0 +1,23 @@ +{ + cloud-utils, + coreutils, + gitMinimal, + packer, + qemu, + writeShellApplication, + yq, +}: +writeShellApplication { + name = "build-qemu-image"; + + runtimeInputs = [ + cloud-utils + coreutils + gitMinimal + packer + qemu + yq + ]; + + text = builtins.readFile ./build-qemu-image.sh; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index a452a9b784..c3b1ae012e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -44,6 +44,7 @@ packages = ( { build-ami = pkgs.callPackage ./build-ami.nix { packer = self'.packages.packer; }; + build-qemu-image = pkgs.callPackage ./build-qemu-image { packer = self'.packages.packer; }; build-test-ami = pkgs.callPackage ./build-test-ami.nix { packer = self'.packages.packer; }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 61c0bdc4e5..82b7539b3f 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -12,6 +12,21 @@ variable "postgres_major_version" { default = "" } +# Path to the UEFI firmware code image. Provided from /nix by build-qemu-image. +variable "fw_code" { + type = string +} + +# Path to a writable UEFI firmware vars image created by build-qemu-image before packer runs. +variable "fw_vars" { + type = string +} + +# Path to the cloud-init seed ISO created by build-qemu-image before packer runs. +variable "seeds_iso" { + type = string +} + packer { required_plugins { qemu = { @@ -21,25 +36,6 @@ packer { } } -source "null" "dependencies" { - communicator = "none" -} - -build { - name = "cloudimg.deps" - sources = [ - "source.null.dependencies" - ] - - provisioner "shell-local" { - inline = [ - "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd", - "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data" - ] - inline_shebang = "/bin/bash -e" - } -} - source "qemu" "cloudimg" { boot_wait = "2s" cpus = 8 @@ -59,10 +55,10 @@ source "qemu" "cloudimg" { ["-machine", "virt,gic-version=3"], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], - ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], - ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], + ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.fw_code}"], + ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.fw_vars}"], ["-drive", "file=output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"], + ["-drive", "file=${var.seeds_iso},format=raw"], ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" From 8dce8c9ce6b480b1f5973832acfcaf1f1e79e578 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 15 Jul 2026 09:07:36 -0400 Subject: [PATCH 5/9] packer/qemu: Support x86_64 Same as before where host must match target. This is nice to test x86_64 specific playbook tasks. --- .github/workflows/qemu-image-build.yml | 4 +-- .../build-qemu-image/build-qemu-image.sh | 28 +++++++++++++++---- qemu-arm64-nix.pkr.hcl => qemu.pkr.hcl | 19 +++++++++++-- 3 files changed, 40 insertions(+), 11 deletions(-) rename qemu-arm64-nix.pkr.hcl => qemu.pkr.hcl (90%) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 2ab5ce8504..589c3e8c47 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -6,7 +6,7 @@ on: - .github/workflows/qemu-image-build.yml - ansible/vars.yml - nix/packages/build-qemu-image/* - - qemu-arm64-nix.pkr.hcl + - qemu.pkr.hcl - scripts/* workflow_dispatch: @@ -71,7 +71,7 @@ jobs: multi-user: "false" - name: Build QEMU artifact - run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" + run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64 - name: Grab release version id: process_release_version diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index 23e43b659e..b97ea98327 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -11,12 +11,25 @@ if [ -z "$postgres_major_version" ]; then exit 1 fi -qemu_bin=$(which qemu-system-aarch64) -fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-aarch64-code.fd -fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-arm-vars.fd +case ${2:-} in +arm64) + qemu_bin=$(which qemu-system-aarch64) + fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-aarch64-code.fd + fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-arm-vars.fd + image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img + machine=virt,gic-version=3 + ;; +amd64) + qemu_bin=$(which qemu-system-x86_64) + fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-x86_64-code.fd + fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-i386-vars.fd + image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img + machine=q35 + ;; +*) echo "Error: Invalid arch '${2:-}'. Must be 'arm64' or 'amd64'" >&2 && exit 1 ;; +esac mkdir -p packer-qemu-working -# qemu writes to the vars pflash during boot, so make a local writable copy. cp "$fw_vars" packer-qemu-working/fw-vars.fd chmod u+w packer-qemu-working/fw-vars.fd @@ -26,12 +39,15 @@ postgres_version=$(yq -r ".postgres_release[\"postgres$postgres_major_version\"] # Build the cloud-init seed ISO before invoking packer. cloud-localds packer-qemu-working/seeds-cloudimg.iso user-data-cloudimg meta-data -packer init qemu-nix-arm64.pkr.hcl +packer init qemu.pkr.hcl PACKER_LOG=${PACKER_LOG:-1} packer build \ -var "fw_code=$fw_code" \ -var "fw_vars=packer-qemu-working/fw-vars.fd" \ -var "git_sha=$git_sha" \ + -var "iso_url=$image" \ + -var "machine=$machine" \ -var "postgres-version=$postgres_version" \ -var "postgres_major_version=$postgres_major_version" \ + -var "qemu_binary=$qemu_bin" \ -var "seeds_iso=packer-qemu-working/seeds-cloudimg.iso" \ - qemu-nix-arm64.pkr.hcl + qemu.pkr.hcl diff --git a/qemu-arm64-nix.pkr.hcl b/qemu.pkr.hcl similarity index 90% rename from qemu-arm64-nix.pkr.hcl rename to qemu.pkr.hcl index 82b7539b3f..2f2ef8f8b4 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu.pkr.hcl @@ -27,6 +27,19 @@ variable "seeds_iso" { type = string } +# Arch-specific values supplied by build-qemu-image (arm64 or amd64). +variable "iso_url" { + type = string +} + +variable "machine" { + type = string +} + +variable "qemu_binary" { + type = string +} + packer { required_plugins { qemu = { @@ -45,14 +58,14 @@ source "qemu" "cloudimg" { headless = true http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" - iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img" + iso_url = var.iso_url memory = 40000 qemu_img_args { convert = ["-o", "compression_type=zstd"] } - qemu_binary = "qemu-system-aarch64" + qemu_binary = var.qemu_binary qemuargs = [ - ["-machine", "virt,gic-version=3"], + ["-machine", var.machine], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.fw_code}"], From 881ef96a9463a67e7b792622f5e0ea90d7fe0671 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 13:03:20 -0400 Subject: [PATCH 6/9] packer/qemu: Support building on Darwin --- nix/packages/build-qemu-image/build-qemu-image.sh | 9 +++++++-- qemu.pkr.hcl | 4 +--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index b97ea98327..dcf8af940e 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -11,20 +11,25 @@ if [ -z "$postgres_major_version" ]; then exit 1 fi +case $(uname -o) in +Darwin) accel=hvf ;; +GNU/Linux) accel=kvm ;; +*) echo "Error: Invalid os '$(uname -o)'. Must be Darwin or Linux" >&2 && exit 1 ;; +esac case ${2:-} in arm64) qemu_bin=$(which qemu-system-aarch64) fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-aarch64-code.fd fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-arm-vars.fd image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img - machine=virt,gic-version=3 + machine=virt,accel=$accel,gic-version=max,highmem=on ;; amd64) qemu_bin=$(which qemu-system-x86_64) fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-x86_64-code.fd fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-i386-vars.fd image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img - machine=q35 + machine=q35,accel=$accel ;; *) echo "Error: Invalid arch '${2:-}'. Must be 'arm64' or 'amd64'" >&2 && exit 1 ;; esac diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 2f2ef8f8b4..9e935499de 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -59,7 +59,7 @@ source "qemu" "cloudimg" { http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" iso_url = var.iso_url - memory = 40000 + memory = 4096 # in MiB qemu_img_args { convert = ["-o", "compression_type=zstd"] } @@ -72,7 +72,6 @@ source "qemu" "cloudimg" { ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.fw_vars}"], ["-drive", "file=output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], ["-drive", "file=${var.seeds_iso},format=raw"], - ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" ssh_handshake_attempts = 500 @@ -81,7 +80,6 @@ source "qemu" "cloudimg" { ssh_username = "ubuntu" ssh_wait_timeout = "1h" use_backing_file = false - accelerator = "kvm" } build { From 1650510f5c3246cc9fdd6cd397a3b27d661cb3e6 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sat, 18 Jul 2026 16:01:04 -0400 Subject: [PATCH 7/9] packer/qemu: Upgrade to 1.1.6 Fixes packer-qemu-plugin 100% cpu usage across all cores (similar to seen with the amazon ebssurrogate plugin). CPU usage seen with gnutime -v, before: ``` User time (seconds): 4703.36 System time (seconds): 259.53 Percent of CPU this job got: 781% Elapsed (wall clock) time (h:mm:ss or m:ss): 10:35.18 ``` after: ``` User time (seconds): 732.49 System time (seconds): 310.12 Percent of CPU this job got: 209% Elapsed (wall clock) time (h:mm:ss or m:ss): 8:17.33 ``` --- qemu.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 9e935499de..3d4a2d6e64 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -44,7 +44,7 @@ packer { required_plugins { qemu = { source = "github.com/hashicorp/qemu" - version = "1.1.5" + version = "1.1.6" } } } From a99cdf3df632ea4bf0d8804bf730bc983adcfab5 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sat, 18 Jul 2026 16:01:04 -0400 Subject: [PATCH 8/9] packer/qemu: Disable vnc We don't need it. --- qemu.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 3d4a2d6e64..b9cfb2fe8c 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -52,6 +52,7 @@ packer { source "qemu" "cloudimg" { boot_wait = "2s" cpus = 8 + disable_vnc = true disk_image = true disk_size = "15G" format = "qcow2" From ea336cfb596fb2643524acace98efd8ef82f6386 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 9/9] packer/qemu: Install qemu specific packages in qemu-bootstrap-nix.sh We already have a pre-ansible step to install packages that are missing compared to the ebs setup so let move these 2 there and avoid some weird qemu only package install tasks (not to mention that I'm not sure installing gpg in supabase-admin-agent file made much sense anyway). --- ansible/tasks/internal/supabase-admin-agent.yml | 7 ------- ansible/tasks/setup-system.yml | 8 -------- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 15 ++++++++++++++- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 69e7861291..c5ab847362 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -68,13 +68,6 @@ arch: "arm64" when: platform == "arm64" -- name: install gpg explicitly for qemu artifacts - become: yes - apt: - pkg: - - gpg - when: qemu_mode is defined - - name: Download supabase-admin-agent archive get_url: url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-{{ arch }}.tar.gz" diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 8b961516aa..631e5f3b28 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -76,14 +76,6 @@ owner: 'root' group: 'root' -- name: Install other useful tools - ansible.builtin.apt: - pkg: - - less - update_cache: true - when: - - qemu_mode is defined - - name: Set the platform arch as a fact ansible.builtin.set_fact: platform: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else 'arm64' }}" diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 8a52260ee8..944d34d978 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -22,7 +22,20 @@ function waitfor_boot_finished { } function install_packages { - apt-get update && sudo apt-get install software-properties-common e2fsprogs nfs-common locales iptables arptables ebtables ufw logrotate -y + apt-get update + apt-get install -y \ + arptables \ + e2fsprogs \ + ebtables \ + gpg \ + iptables \ + less \ + locales \ + logrotate \ + nfs-common \ + software-properties-common \ + ufw \ + ; # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out # add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y