diff --git a/.github/workflows/build-4.1.yml b/.github/workflows/build-4.1.yml new file mode 100644 index 00000000000..2040b8aeb1f --- /dev/null +++ b/.github/workflows/build-4.1.yml @@ -0,0 +1,498 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build (4.1) + +on: + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + schedule: + - cron: '*/30 * * * *' + +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + prerelease: + name: Prerelease + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + outputs: + should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + repository: 'apache/doris' + ref: 'branch-4.1' + fetch-depth: 0 + + - name: Check Diff + id: check_diff + run: | + tag_name='automation-4.1' + title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + max_attempts=3 + + if [[ -z "$(gh release list)" ]] || + ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then + gh release create -t "${title}" "${tag_name}" + fi + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view "${tag_name}")" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" + current_version="$(git log -1 --format='%H')" + + echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" + echo "Current Version: ${current_version}" + + should_release=false + attempt=1 + if [[ -z "${last_version}" ]]; then + echo "The first release was detected." + should_release=true + elif [[ "${last_version}" != "${current_version}" ]]; then + cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'" + echo "Execute: ${cmd}" + content="$(eval "${cmd}")" || true + if [[ -n "${content}" ]]; then + echo -e "Detect changes:\n${content}" + should_release=true + fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park the branch until the next thirdparty/ + # commit lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true + fi + + if "${should_release}"; then + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md + else + gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ + s/Update Time:.*/Update Time: *$(date)*/ + s/Doris Version:.*/Doris Version: *${current_version}*/ + }" >release_note.md + fi + gh release edit -F release_note.md "${tag_name}" + + echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT + + - name: Download Source and Upload + if: steps.check_diff.outputs.should_release == 'true' + run: | + tag_name='automation-4.1' + + cd thirdparty + # Pre-download packages whose default mirror is blocked or gone on GH Actions runners: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is blocked from GH Actions + mkdir -p src + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - + + tar -zcvf doris-thirdparty-source.tgz src + + gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true + gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz + + build: + name: Build + needs: prerelease + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') + strategy: + fail-fast: false + matrix: + config: + - name: macOS-arm64 + os: macos-14 + packages: >- + 'm4' + 'automake' + 'autoconf' + 'libtool' + 'pkg-config' + 'texinfo' + 'coreutils' + 'gnu-getopt' + 'python@3' + 'ninja' + 'ccache' + 'bison' + 'byacc' + 'gettext' + 'wget' + 'pcre' + 'openjdk@11' + 'maven' + 'node' + 'llvm@20' + + - name: Linux + os: ubuntu-22.04 + packages: >- + 'build-essential' + 'automake' + 'autoconf' + 'libtool-bin' + 'pkg-config' + 'cmake=3.22.1-1ubuntu1.22.04.2' + 'ninja-build' + 'ccache' + 'python-is-python3' + 'bison' + 'byacc' + 'flex' + 'binutils-dev' + 'libiberty-dev' + 'curl' + 'git' + 'zip' + 'unzip' + 'autopoint' + 'openjdk-8-jdk' + 'openjdk-8-jdk-headless' + 'maven' + + runs-on: ${{ matrix.config.os }} + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout easimon/maximize-build-space + if: ${{ matrix.config.name == 'Linux' }} + run: | + git clone -b v7 https://github.com/easimon/maximize-build-space + + #- name: Maximize build space + # if: ${{ matrix.config.name == 'Linux' }} + # uses: ./maximize-build-space + # with: + # root-reserve-mb: 4096 + # swap-size-mb: 8192 + # remove-dotnet: 'true' + # remove-android: 'true' + # remove-haskell: 'true' + # remove-codeql: 'true' + # remove-docker-images: 'true' + + - name: Checkout + uses: actions/checkout@v5 + with: + repository: 'apache/doris' + ref: 'branch-4.1' + + - name: Download + run: | + tag_name='automation-4.1' + + cd thirdparty + curl -L "https://github.com/${{ github.repository }}/releases/download/${tag_name}/doris-thirdparty-source.tgz" \ + -o doris-thirdparty-source.tgz + tar -zxvf doris-thirdparty-source.tgz + + - name: Prepare for ${{ matrix.config.os }} + run: | + if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then + # Install packages except cmake + brew install ${{ matrix.config.packages }} || true + # Install specific version of cmake + brew unlink cmake || true + wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz + tar -xzf cmake-3.22.1-macos-universal.tar.gz + sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + cmake --version + else + export DEFAULT_DIR='/opt/doris' + export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" + + sudo apt update + sudo apt-cache policy cmake + sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }} + + mkdir -p "${DEFAULT_DIR}" + wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.25/ldb_toolchain_gen.sh \ + -q -O /tmp/ldb_toolchain_gen.sh + bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" + fi + + - name: Build and Upload + run: | + tag_name='automation-4.1' + + if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then + export DEFAULT_DIR='/opt/doris' + export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" + export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" + export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" + export DORIS_TOOLCHAIN=clang + else + export MACOSX_DEPLOYMENT_TARGET=12.0 + fi + + export CMAKE_POLICY_VERSION_MINIMUM=3.10 + export CUSTOM_CMAKE="/usr/local/bin/cmake" + cd thirdparty + # Fix packages that may be empty if the source tarball was built before the mirror fixes: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) blocked from GH Actions + if [[ ! -s src/lzo-2.10.tar.gz ]]; then + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + fi + if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + fi + ./build-thirdparty.sh -j "$(nproc)" + + kernel="$(uname -s | awk '{print tolower($0)}')" + arch="$(uname -m)" + rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # Delete existing asset first to avoid gh CLI --clobber race-condition bug + # (gh release upload --clobber sometimes returns "release not found" exit 1 + # when deleting an existing asset, even though the upload itself succeeds). + # --yes is required: without it gh cannot confirm the prompt on a runner + # and the delete silently does nothing, which then made the plain upload + # fail because the asset was still there. + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true + gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + + update-docker: + name: Update Docker Image + needs: [prerelease, build] + if: | + always() && ( + (needs.prerelease.outputs.should_release == 'true' && needs.build.result == 'success') || + (github.event_name == 'workflow_dispatch' && needs.build.result != 'failure') + ) + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout docker/setup-buildx-action + run: | + rm -rf ./.github/actions/setup-buildx-action + git clone https://github.com/docker/setup-buildx-action .github/actions/setup-buildx-action + + pushd .github/actions/setup-buildx-action &>/dev/null + git checkout 8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + popd &>/dev/null + + - name: Checkout docker/login-action + run: | + rm -rf ./.github/actions/login-action + git clone https://github.com/docker/login-action .github/actions/login-action + + pushd .github/actions/login-action &>/dev/null + git checkout c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + popd &>/dev/null + + - name: Checkout docker/build-push-action + run: | + rm -rf ./.github/actions/build-push-action + git clone https://github.com/docker/build-push-action .github/actions/build-push-action + + pushd .github/actions/build-push-action &>/dev/null + git checkout ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + popd &>/dev/null + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx-action + + - name: Login to Docker Hub + uses: ./.github/actions/login-action + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Prepare Dockerfiles + run: | + # Pull the upstream Dockerfile from apache/doris branch-4.1 to stay in sync with + # any toolchain / dependency changes they make. + curl -fsSL https://raw.githubusercontent.com/apache/doris/branch-4.1/docker/compilation/Dockerfile \ + -o Dockerfile.upstream + + python3 - << 'PYEOF' + import sys, re + + with open('Dockerfile.upstream') as f: + content = f.read() + + # Patch 1: fix epel metalink. + # The upstream RUN line installs epel-release then immediately runs yum + # install/clean/makecache, but epel.repo's metalink may be unreliable. + # Insert a sed fix between epel-release install and the next yum install. + old_epel = 'yum install epel-release -y && yum install https://packages.endpointdev.com' + new_epel = ('yum install epel-release -y \\\n' + ' && sed -i \\\n' + ' -e \'s/^metalink=/#metalink=/\' \\\n' + ' -e \'s|^#baseurl=http://download.fedoraproject.org/pub/epel/7|baseurl=https://mirrors.aliyun.com/epel/7|\' \\\n' + ' /etc/yum.repos.d/epel*.repo \\\n' + ' && yum install https://packages.endpointdev.com') + assert old_epel in content, f"Patch 1 failed: target string not found in upstream Dockerfile" + patched = content.replace(old_epel, new_epel, 1) + assert patched != content, "Patch 1 was a no-op: upstream Dockerfile may have changed" + + # Patch 2: replace "clone & build thirdparty" block with downloading + # our prebuilt artifact. The block starts with "# clone lastest source + # code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris". + prebuilt_block = ( + '# Download prebuilt thirdparty from GitHub Release (built by doris-thirdparty automation)\n' + 'ARG GITHUB_REPOSITORY\n' + 'RUN mkdir -p /var/local/thirdparty \\\n' + ' && wget -q "https://github.com/${GITHUB_REPOSITORY}/releases/download/automation-4.1/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \\\n' + ' -O /tmp/prebuilt.tar.xz \\\n' + ' && tar -xf /tmp/prebuilt.tar.xz -C /var/local/thirdparty \\\n' + ' && rm /tmp/prebuilt.tar.xz\n' + ) + patched_2 = re.sub( + r'# clone lastest source code.*?rm -rf \$\{DEFAULT_DIR\}/doris\n', + prebuilt_block, + patched, flags=re.DOTALL + ) + assert patched_2 != patched, "Patch 2 was a no-op: 'clone lastest source code' block not found in upstream Dockerfile" + patched = patched_2 + + # Write normal image Dockerfile + with open('Dockerfile.patched', 'w') as f: + f.write(patched) + + # Patch 3 (no-avx2): add USE_AVX2=0 to the ENV block in builder stage. + # The ENV block ends with PATH=... just before "# install ccache". + old_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH"\n # USE_AVX2=0' + new_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH" \\\n USE_AVX2=0' + assert old_avx2 in patched, "Patch 3 failed: '# USE_AVX2=0' comment not found; upstream Dockerfile may have changed" + noavx2 = patched.replace(old_avx2, new_avx2) + assert noavx2 != patched, "Patch 3 was a no-op: no-avx2 Dockerfile is identical to normal Dockerfile" + with open('Dockerfile.patched-noavx2', 'w') as f: + f.write(noavx2) + + print("=== normal: check ===") + for line in open('Dockerfile.patched'): + if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): + print(line, end='') + print("=== noavx2: check ===") + for line in open('Dockerfile.patched-noavx2'): + if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): + print(line, end='') + PYEOF + + - name: Build and Push Docker Image (normal) + uses: ./.github/actions/build-push-action + with: + context: . + file: Dockerfile.patched + build-args: | + GITHUB_REPOSITORY=${{ github.repository }} + push: true + tags: ${{ vars.DOCKER_TAGS_4_1 || 'apache/doris:build-env-ldb-toolchain-4.1-latest' }} + platforms: linux/amd64 + + - name: Build and Push Docker Image (no-avx2) + uses: ./.github/actions/build-push-action + with: + context: . + file: Dockerfile.patched-noavx2 + build-args: | + GITHUB_REPOSITORY=${{ github.repository }} + push: true + tags: ${{ vars.DOCKER_TAGS_NOAVX2_4_1 || 'apache/doris:build-env-ldb-toolchain-4.1-no-avx2-latest' }} + platforms: linux/amd64 + + success: + name: Success + needs: [prerelease, build, update-docker] + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + permissions: + contents: write + steps: + - name: Update Checksums + run: | + tag_name='automation-4.1' + + gh release download "${tag_name}" + + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + gh release edit -F release_note.md "${tag_name}" + + failure: + name: Failure + needs: [prerelease, build, update-docker] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} + permissions: + contents: write + steps: + - name: Update Checksums + run: | + tag_name='automation-4.1' + + gh release download "${tag_name}" + + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + gh release edit -F release_note.md "${tag_name}"