diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 638111853a..af4c2292d4 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -1,14 +1,22 @@ name: 'Install Nix on ephemeral runners' description: 'Installs Nix and sets up AWS credentials to push to the Nix binary cache' inputs: + aws-region: + description: 'AWS region for the Nix binary cache S3 bucket' + required: false + default: 'us-east-1' + force-clean: + description: Whether to delete pre-existing nix paths + required: false + default: 'false' push-to-cache: description: 'Whether to push build outputs to the Nix binary cache' required: false default: 'false' - aws-region: - description: 'AWS region for the Nix binary cache S3 bucket' + multi-user: + description: Whether to install nix in multi-user mode (default true) or single-user mode required: false - default: 'us-east-1' + default: 'true' runs: using: 'composite' steps: @@ -20,6 +28,13 @@ runs: aws-region: ${{ inputs.aws-region }} output-credentials: true role-duration-seconds: 7200 + - name: Force Clean Pre-Existing Nix Paths + if: inputs.force-clean == 'true' + shell: bash + run: sudo rm -rf /nix /etc/nix "$HOME/.nix-profile" "$HOME/.nix-channels" "$HOME/.config/nix" + - name: Ensure /etc/nix exists + shell: bash + run: sudo mkdir -p /etc/nix - name: Setup AWS credentials for Nix if: ${{ inputs.push-to-cache == 'true' }} shell: bash @@ -28,7 +43,6 @@ runs: sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN sudo -H aws configure set region ${{ inputs.aws-region }} - sudo mkdir -p /etc/nix sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null #!/usr/bin/env bash @@ -44,34 +58,50 @@ runs: - name: Install Nix shell: bash run: | - sudo tee /tmp/nix-extra.conf > /dev/null <<'NIXCONF' + if [[ "${{ inputs.multi-user }}" == true ]]; then + daemon=--daemon + nixconfdir=/etc/nix + path=/nix/var/nix/profiles/default/bin + tmpconf=/tmp/nix-extra.conf + + echo 'access-tokens = github.com=${{ github.token }}' | sudo tee $nixconfdir/github.nix.conf >/dev/null + sudo chmod 400 $nixconfdir/github.nix.conf + else + daemon=--no-daemon + nixconfdir=$HOME/.config/nix + path=$HOME/.nix-profile/bin + tmpconf=$nixconfdir/nix.conf # --nix-extra-conf-files is ignored for single-user installs + + mkdir -p "$(dirname $tmpconf)" + echo 'access-tokens = github.com=${{ github.token }}' | tee $nixconfdir/github.nix.conf >/dev/null + chmod 400 $nixconfdir/github.nix.conf + fi + + cat >$tmpconf <<'NIXCONF' always-allow-substitutes = true - extra-experimental-features = nix-command flakes - extra-system-features = kvm + extra-experimental-features = flakes nix-command + extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com + extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= max-jobs = 4 - substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com - trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= NIXCONF - if [ "${{ inputs.push-to-cache }}" = "true" ]; then - echo "post-build-hook = /etc/nix/upload-to-cache.sh" | sudo tee -a /tmp/nix-extra.conf > /dev/null + + if [[ -e /dev/kvm ]]; then + echo 'extra-system-features = kvm' >>$tmpconf fi - curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf + if [[ "${{ inputs.push-to-cache }}" == true ]]; then + echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>$tmpconf + fi - cat /etc/nix/nix.conf + echo "!include $nixconfdir/github.nix.conf" >>$tmpconf + + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file $tmpconf + + sudo cat $nixconfdir/nix.conf # Add nix to PATH for subsequent steps - echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" - # Source the daemon profile so nix works in this step too - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - - name: Configure Nix with GitHub token - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "access-tokens = github.com=$GH_TOKEN" | sudo tee -a /etc/nix/nix.conf > /dev/null - sudo systemctl restart nix-daemon || true + echo "$path" >> "$GITHUB_PATH" - name: Print Nix version shell: bash run: nix --version diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index f3f9e417a8..589c3e8c47 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -3,11 +3,11 @@ name: Build QEMU image on: push: paths: - - '.github/workflows/qemu-image-build.yml' - - 'qemu-arm64-nix.pkr.hcl' - - 'common-nix.vars.pkr.hcl' - - 'ansible/vars.yml' - - 'scripts/*' + - .github/workflows/qemu-image-build.yml + - ansible/vars.yml + - nix/packages/build-qemu-image/* + - qemu.pkr.hcl + - scripts/* workflow_dispatch: permissions: @@ -64,35 +64,20 @@ jobs: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl - run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq - PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) - PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes - echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file - echo "" >> common-nix.vars.pkr.hcl - - # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + - name: Install Nix + uses: ./.github/actions/nix-install-ephemeral + with: + force-clean: "true" + multi-user: "false" - name: Build QEMU artifact - run: | - make init - GIT_SHA=${{github.sha}} - export PACKER_LOG=1 - packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl + run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64 - name: Grab release version id: process_release_version run: | - VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - echo "version=$VERSION" >> $GITHUB_OUTPUT + PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + echo "version=$PG_VERSION" >> $GITHUB_OUTPUT - name: configure aws credentials - staging if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} diff --git a/.gitignore b/.gitignore index f24b95b1b3..15429d4357 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ # ansible related ansible/image-manifest*.json +# packer releasted +output-cloudimg/ +packer-qemu-working/ + # python related *$py.class *.py[cod] @@ -21,5 +25,4 @@ venv/ #nix related common-nix.vars.pkr.hcl db/schema.sql -nixos.qcow2 result* diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 69e7861291..c5ab847362 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -68,13 +68,6 @@ arch: "arm64" when: platform == "arm64" -- name: install gpg explicitly for qemu artifacts - become: yes - apt: - pkg: - - gpg - when: qemu_mode is defined - - name: Download supabase-admin-agent archive get_url: url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-{{ arch }}.tar.gz" diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 8b961516aa..631e5f3b28 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -76,14 +76,6 @@ owner: 'root' group: 'root' -- name: Install other useful tools - ansible.builtin.apt: - pkg: - - less - update_cache: true - when: - - qemu_mode is defined - - name: Set the platform arch as a fact ansible.builtin.set_fact: platform: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else 'arm64' }}" diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 8a52260ee8..944d34d978 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -22,7 +22,20 @@ function waitfor_boot_finished { } function install_packages { - apt-get update && sudo apt-get install software-properties-common e2fsprogs nfs-common locales iptables arptables ebtables ufw logrotate -y + apt-get update + apt-get install -y \ + arptables \ + e2fsprogs \ + ebtables \ + gpg \ + iptables \ + less \ + locales \ + logrotate \ + nfs-common \ + software-properties-common \ + ufw \ + ; # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out # add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh new file mode 100644 index 0000000000..dcf8af940e --- /dev/null +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -0,0 +1,58 @@ +# shellcheck shell=bash + +# nix-built qemu resolves its libraries via RPATH; an inherited +# LD_LIBRARY_PATH (e.g. from a devshell) can inject a mismatched glibc and +# break `qemu-system-* --version`, so drop it. +unset LD_LIBRARY_PATH + +postgres_major_version=${1:-} +if [ -z "$postgres_major_version" ]; then + echo "Usage: build-qemu-image [arch]" >&2 + exit 1 +fi + +case $(uname -o) in +Darwin) accel=hvf ;; +GNU/Linux) accel=kvm ;; +*) echo "Error: Invalid os '$(uname -o)'. Must be Darwin or Linux" >&2 && exit 1 ;; +esac +case ${2:-} in +arm64) + qemu_bin=$(which qemu-system-aarch64) + fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-aarch64-code.fd + fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-arm-vars.fd + image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img + machine=virt,accel=$accel,gic-version=max,highmem=on + ;; +amd64) + qemu_bin=$(which qemu-system-x86_64) + fw_code=${qemu_bin%/bin/*}/share/qemu/edk2-x86_64-code.fd + fw_vars=${qemu_bin%/bin/*}/share/qemu/edk2-i386-vars.fd + image=https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img + machine=q35,accel=$accel + ;; +*) echo "Error: Invalid arch '${2:-}'. Must be 'arm64' or 'amd64'" >&2 && exit 1 ;; +esac + +mkdir -p packer-qemu-working +cp "$fw_vars" packer-qemu-working/fw-vars.fd +chmod u+w packer-qemu-working/fw-vars.fd + +git_sha=${GIT_SHA:-$(git rev-parse HEAD)} +postgres_version=$(yq -r ".postgres_release[\"postgres$postgres_major_version\"]" ansible/vars.yml) + +# Build the cloud-init seed ISO before invoking packer. +cloud-localds packer-qemu-working/seeds-cloudimg.iso user-data-cloudimg meta-data + +packer init qemu.pkr.hcl +PACKER_LOG=${PACKER_LOG:-1} packer build \ + -var "fw_code=$fw_code" \ + -var "fw_vars=packer-qemu-working/fw-vars.fd" \ + -var "git_sha=$git_sha" \ + -var "iso_url=$image" \ + -var "machine=$machine" \ + -var "postgres-version=$postgres_version" \ + -var "postgres_major_version=$postgres_major_version" \ + -var "qemu_binary=$qemu_bin" \ + -var "seeds_iso=packer-qemu-working/seeds-cloudimg.iso" \ + qemu.pkr.hcl diff --git a/nix/packages/build-qemu-image/default.nix b/nix/packages/build-qemu-image/default.nix new file mode 100644 index 0000000000..94762f6f47 --- /dev/null +++ b/nix/packages/build-qemu-image/default.nix @@ -0,0 +1,23 @@ +{ + cloud-utils, + coreutils, + gitMinimal, + packer, + qemu, + writeShellApplication, + yq, +}: +writeShellApplication { + name = "build-qemu-image"; + + runtimeInputs = [ + cloud-utils + coreutils + gitMinimal + packer + qemu + yq + ]; + + text = builtins.readFile ./build-qemu-image.sh; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index a452a9b784..c3b1ae012e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -44,6 +44,7 @@ packages = ( { build-ami = pkgs.callPackage ./build-ami.nix { packer = self'.packages.packer; }; + build-qemu-image = pkgs.callPackage ./build-qemu-image { packer = self'.packages.packer; }; build-test-ami = pkgs.callPackage ./build-test-ami.nix { packer = self'.packages.packer; }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; diff --git a/qemu-arm64-nix.pkr.hcl b/qemu.pkr.hcl similarity index 69% rename from qemu-arm64-nix.pkr.hcl rename to qemu.pkr.hcl index 61c0bdc4e5..b9cfb2fe8c 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu.pkr.hcl @@ -12,58 +12,67 @@ variable "postgres_major_version" { default = "" } -packer { - required_plugins { - qemu = { - source = "github.com/hashicorp/qemu" - version = "1.1.5" - } - } +# Path to the UEFI firmware code image. Provided from /nix by build-qemu-image. +variable "fw_code" { + type = string } -source "null" "dependencies" { - communicator = "none" +# Path to a writable UEFI firmware vars image created by build-qemu-image before packer runs. +variable "fw_vars" { + type = string } -build { - name = "cloudimg.deps" - sources = [ - "source.null.dependencies" - ] +# Path to the cloud-init seed ISO created by build-qemu-image before packer runs. +variable "seeds_iso" { + type = string +} - provisioner "shell-local" { - inline = [ - "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd", - "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data" - ] - inline_shebang = "/bin/bash -e" +# Arch-specific values supplied by build-qemu-image (arm64 or amd64). +variable "iso_url" { + type = string +} + +variable "machine" { + type = string +} + +variable "qemu_binary" { + type = string +} + +packer { + required_plugins { + qemu = { + source = "github.com/hashicorp/qemu" + version = "1.1.6" + } } } source "qemu" "cloudimg" { boot_wait = "2s" cpus = 8 + disable_vnc = true disk_image = true disk_size = "15G" format = "qcow2" headless = true http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" - iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img" - memory = 40000 + iso_url = var.iso_url + memory = 4096 # in MiB qemu_img_args { convert = ["-o", "compression_type=zstd"] } - qemu_binary = "qemu-system-aarch64" + qemu_binary = var.qemu_binary qemuargs = [ - ["-machine", "virt,gic-version=3"], + ["-machine", var.machine], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], - ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], - ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], + ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.fw_code}"], + ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.fw_vars}"], ["-drive", "file=output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"], - ["--enable-kvm"] + ["-drive", "file=${var.seeds_iso},format=raw"], ] shutdown_command = "sudo -S shutdown -P now" ssh_handshake_attempts = 500 @@ -72,7 +81,6 @@ source "qemu" "cloudimg" { ssh_username = "ubuntu" ssh_wait_timeout = "1h" use_backing_file = false - accelerator = "kvm" } build {