From c0bbbb3119ff4be8d9c543f8ef410fc24b07659d Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Thu, 2 Jul 2026 09:59:33 +0530 Subject: [PATCH 1/8] Enhancement: Parallelize plugin builds --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 790dc87c..60b39779 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ mkdir -p ${OUTPUT_DIR}/web ${OUTPUT_DIR}/plugins (cd ${WORKING_DIR} && for item in ${COMMANDS}; do go build -ldflags "${LDFLAGS}" -o dist/$item cmd/$item/$item.go; done) # build plugins -(cd ${WORKING_DIR} && for item in ${PLUGINS}; do go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/$item.so plugins/$item/$item.go; done) +(cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 0 -I {} sh -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') # build web (cd ${WORKING_DIR}/web && rm -rf node_modules > /dev/null 2>&1 && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) From a6bb0b835974383f7fd08fe77f2dc7038c2b0123 Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Thu, 2 Jul 2026 10:42:26 +0530 Subject: [PATCH 2/8] Changed script to use bash --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 60b39779..432e03b6 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ mkdir -p ${OUTPUT_DIR}/web ${OUTPUT_DIR}/plugins (cd ${WORKING_DIR} && for item in ${COMMANDS}; do go build -ldflags "${LDFLAGS}" -o dist/$item cmd/$item/$item.go; done) # build plugins -(cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 0 -I {} sh -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') +(cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 0 -I {} bash -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') # build web (cd ${WORKING_DIR}/web && rm -rf node_modules > /dev/null 2>&1 && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) From 8deadb5d6d49aafb33fcb5882924ccd075ccdbce Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Thu, 2 Jul 2026 13:01:23 +0530 Subject: [PATCH 3/8] Updated push action, cache reuse --- .github/workflows/docker-image.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c1fa5c11..9bd30dc4 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -45,9 +45,14 @@ jobs: fi - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_SUMMARY: false + DOCKER_BUILD_RECORD_UPLOAD: false with: context: . push: true tags: patternoss/heimdall:${{ env.DOCKER_TAG }} platforms: linux/amd64,linux/arm64 + cache-from: type=registry,ref=patternoss/heimdall:buildcache + cache-to: type=registry,ref=patternoss/heimdall:buildcache,mode=max From 5633273beab9c90374eec76d747c3bb8706f24ae Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Wed, 8 Jul 2026 14:25:31 +0530 Subject: [PATCH 4/8] More enhancements --- .github/workflows/build.yml | 24 ++++++++++++++++- Dockerfile | 53 ++++++++++++++++++++++++++++++++----- build.sh | 44 +++++++++++++++++++++--------- 3 files changed, 101 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3311f80b..546bbebb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,5 +13,27 @@ jobs: uses: actions/setup-go@v5 with: go-version: 1.24.5 + # setup-go caches the module + build cache keyed on go.sum by default. + - name: Cache pnpm store + uses: actions/cache@v4 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-${{ hashFiles('web/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- - name: Build and Test - run: ./build.sh + run: ./build.sh --go --ui --test + + # Verify the image builds end-to-end (no push) so Dockerfile breakage is caught on PRs. + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build image (no push) + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: false + platforms: linux/amd64,linux/arm64 + cache-from: type=registry,ref=patternoss/heimdall:buildcache + cache-to: type=registry,ref=patternoss/heimdall:buildcache,mode=max diff --git a/Dockerfile b/Dockerfile index af76abbe..009c6201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,54 @@ -FROM golang:1.24.6 +FROM golang:1.24.6 AS go-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + binutils \ + binutils-gold \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y nodejs awscli jq -RUN apt-get update && apt-get install -y --no-install-recommends build-essential binutils binutils-gold pkg-config && rm -rf /var/lib/apt/lists/* WORKDIR /go/src/github.com/patterninc/heimdall -COPY . . +COPY go.mod go.sum ./ +RUN go mod download + +COPY build.sh ./ +COPY cmd ./cmd +COPY internal ./internal +COPY pkg ./pkg +COPY plugins ./plugins + +# Declared right before its only consumer so it invalidates as little as possible. +ARG BUILD_VERSION=dev +RUN BUILD_VERSION="${BUILD_VERSION}" ./build.sh --go + +FROM node:20-bookworm AS web-builder + +WORKDIR /app + +COPY web/pnpm-lock.yaml web/pnpm-workspace.yaml ./web/ +RUN corepack enable && corepack prepare pnpm@10.28.2 --activate && (cd web && pnpm fetch) + +COPY build.sh ./ +COPY web/ ./web/ +RUN ./build.sh --ui + +FROM node:20-bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + awscli \ + jq \ + && corepack enable && corepack prepare pnpm@10.28.2 --activate \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /go/src/github.com/patterninc/heimdall -# set config file +# Rarely-changing runtime files first, frequently-changing build outputs last. COPY configs/local.yaml /etc/heimdall/heimdall.yaml COPY entrypoint.sh /usr/local/bin/entrypoint.sh -# build executables -RUN ./build.sh +COPY assets ./assets +COPY --from=go-builder /go/src/github.com/patterninc/heimdall/dist ./dist +COPY --from=web-builder /app/web ./web CMD [ "/usr/local/bin/entrypoint.sh" ] \ No newline at end of file diff --git a/build.sh b/build.sh index 432e03b6..64f9c8c6 100755 --- a/build.sh +++ b/build.sh @@ -7,25 +7,45 @@ set -ex BUILD_VERSION=$(git describe --tags --abbrev=0 --match "v*" | cut -dv -f2) WORKING_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) OUTPUT_DIR="${WORKING_DIR}/dist" -COMMANDS=$(ls ${WORKING_DIR}/cmd) -PLUGINS=$(ls ${WORKING_DIR}/plugins) LDFLAGS="-X main.Build=${BUILD_VERSION}" +BUILD_GO=false +BUILD_WEB=false +RUN_TESTS=false +for arg in "$@"; do + case "${arg}" in + --go) BUILD_GO=true ;; + --ui) BUILD_WEB=true ;; + --test) RUN_TESTS=true ;; + *) echo "unknown flag: ${arg} (use --go, --ui, --test)" >&2; exit 1 ;; + esac +done # reset output dir rm -rf ${OUTPUT_DIR} > /dev/null 2>&1 mkdir -p ${OUTPUT_DIR}/web ${OUTPUT_DIR}/plugins -# go environment -(cd ${WORKING_DIR} && go version) +if [ "${RUN_TESTS}" = "true" ]; then + echo "running tests" + (cd ${WORKING_DIR} && go test ./internal/pkg/... && go test ./pkg/...) +fi -# run tests -(cd ${WORKING_DIR} && go test ./internal/pkg/... && go test ./pkg/...) +if [ "${BUILD_GO}" = "true" ]; then + COMMANDS=$(ls ${WORKING_DIR}/cmd) + PLUGINS=$(ls ${WORKING_DIR}/plugins) -# build commands -(cd ${WORKING_DIR} && for item in ${COMMANDS}; do go build -ldflags "${LDFLAGS}" -o dist/$item cmd/$item/$item.go; done) -# build plugins -(cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 0 -I {} bash -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') -# build web -(cd ${WORKING_DIR}/web && rm -rf node_modules > /dev/null 2>&1 && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) + # go environment + (cd ${WORKING_DIR} && go version) + + # build commands + (cd ${WORKING_DIR} && echo "${COMMANDS}" | xargs -P 4 -I {} bash -c 'echo "building command: {}" && go build -ldflags "${LDFLAGS}" -o dist/{} cmd/{}/{}.go') + + # build plugins + (cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 4 -I {} bash -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') +fi + +if [ "${BUILD_WEB}" = "true" ]; then + # build web + (cd ${WORKING_DIR}/web && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) +fi From ddf63d3b0b7e95cfe13bee76f26a13a92899ce97 Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Wed, 8 Jul 2026 14:26:36 +0530 Subject: [PATCH 5/8] Added readme --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 4ba0327f..553d320a 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,21 @@ GET /api/v1/job//stderr --- +## 🛠️ Building from Source + +`build.sh` compiles the Go binaries + plugins and builds the web UI. It builds **only what you ask for** via flags (no defaults): + +```bash +./build.sh --go # build Go binaries (cmd/*) + plugins (plugins/*.so) +./build.sh --ui # build the Next.js web UI +./build.sh --test # run the Go test suites +./build.sh --go --ui --test # do everything (what CI runs) +``` + +Passing no flag builds nothing. Unknown flags fail fast. Outputs land in `dist/` (binaries in `dist/`, plugins in `dist/plugins/`). + +--- + ## 🔌 Supported Plugins Heimdall supports a growing set of pluggable command types: From c04a2814222a66253d534d88bc0dadec53268722 Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Wed, 8 Jul 2026 17:28:15 +0530 Subject: [PATCH 6/8] Trying cache from gha runner --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 546bbebb..50836bde 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,5 +35,5 @@ jobs: push: false load: false platforms: linux/amd64,linux/arm64 - cache-from: type=registry,ref=patternoss/heimdall:buildcache - cache-to: type=registry,ref=patternoss/heimdall:buildcache,mode=max + cache-from: type=gha + cache-to: type=gha,mode=max From f68e2a156ef36d6367fe6687bc18fb040eeca213 Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Wed, 8 Jul 2026 20:16:27 +0530 Subject: [PATCH 7/8] Trying matrix based approach --- .github/workflows/build.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50836bde..5b1966b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,19 @@ jobs: run: ./build.sh --go --ui --test # Verify the image builds end-to-end (no push) so Dockerfile breakage is caught on PRs. - + # Each arch builds on its OWN native runner (no QEMU emulation), in parallel. + docker-build: + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v5 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build image (no push) @@ -34,6 +46,6 @@ jobs: context: . push: false load: false - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + cache-from: type=gha,scope=${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=${{ matrix.platform }} From c38d998d08531badbee7ed6a0c822128b1a52d9e Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Thu, 9 Jul 2026 16:10:30 +0530 Subject: [PATCH 8/8] Testing multi arch manifest merge --- .github/workflows/build.yml | 71 +++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b1966b6..bdbb1a2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,9 @@ on: workflow_dispatch: pull_request: +env: + IMAGE: patternoss/heimdall + jobs: build: runs-on: ubuntu-latest @@ -33,19 +36,81 @@ jobs: include: - platform: linux/amd64 runner: ubuntu-latest + arch: amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm + arch: arm64 runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v5 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build image (no push) + + - name: Log in to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_SUMMARY: false + DOCKER_BUILD_RECORD_UPLOAD: false with: context: . - push: false - load: false platforms: ${{ matrix.platform }} + build-args: | + BUILD_VERSION=ci + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Stitch: collect the per-arch digests and assemble a single throwaway multi-arch tag. + merge: + runs-on: ubuntu-latest + needs: docker-build + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Determine throwaway tag + id: tagger + run: echo "tag=ci-${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT" + + - name: Create and push manifest list + working-directory: /tmp/digests + run: | + docker buildx imagetools create -t ${{ env.IMAGE }}:${{ steps.tagger.outputs.tag }} \ + $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect + run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ steps.tagger.outputs.tag }}