From 4734aa552baeae97ea5132844129996421cc6c59 Mon Sep 17 00:00:00 2001 From: Dale Lane Date: Fri, 26 Jun 2026 09:57:12 +0100 Subject: [PATCH 1/4] feat: add nightly snapshot releases to GHA pipeline Adding two jobs to the nightly GHA workflow that matches the cron_snapshot_deploment_binary and cron_snapshot_deployment_maven jobs that are in tools/azure-pipelines/build-nightly-dist.yml Both of these are using JDK 11 as the minimum supported version described in build-apache-repo.yml Signed-off-by: Dale Lane --- .github/workflows/nightly.yml | 116 ++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 70c6f8d09e606..ea7d5240590b9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -77,6 +77,122 @@ jobs: s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }} s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }} + snapshot_binary: + name: "Snapshot Binary Release" + runs-on: ubuntu-24.04 + container: + image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy + options: --init --privileged + env: + MOUNTED_WORKING_DIR: /__w/flink/flink + CONTAINER_LOCAL_WORKING_DIR: /root/flink + MAVEN_REPO_FOLDER: /root/.m2/repository + MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + steps: + - name: "Flink Checkout" + uses: actions/checkout@v5 + with: + persist-credentials: false + - name: "Initialize job" + uses: "./.github/actions/job_init" + with: + jdk_version: 11 + maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }} + source_directory: ${{ env.MOUNTED_WORKING_DIR }} + target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + - name: "Build snapshot binary release" + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available + set +o errexit + source ./tools/ci/maven-utils.sh + set -o errexit + run_mvn -version + export MVN="run_mvn" + export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "Determined RELEASE_VERSION as '$RELEASE_VERSION'" + cd tools + MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh + echo "Created files:" + find ./releasing/release + cd .. + - name: "Upload artifacts to S3" + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + source ./tools/ci/deploy_nightly_to_s3.sh + upload_to_s3 ./tools/releasing/release + env: + ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }} + + snapshot_maven: + name: "Snapshot Maven Deploy" + runs-on: ubuntu-24.04 + timeout-minutes: 240 + container: + image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy + options: --init --privileged + env: + MOUNTED_WORKING_DIR: /__w/flink/flink + CONTAINER_LOCAL_WORKING_DIR: /root/flink + MAVEN_REPO_FOLDER: /root/.m2/repository + MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + steps: + - name: "Flink Checkout" + uses: actions/checkout@v5 + with: + persist-credentials: false + - name: "Initialize job" + uses: "./.github/actions/job_init" + with: + jdk_version: 11 + maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }} + source_directory: ${{ env.MOUNTED_WORKING_DIR }} + target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + - name: "Deploy Maven snapshot" + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available + set +o errexit + source ./tools/ci/maven-utils.sh + set -o errexit + run_mvn -version + + cd tools + cat << EOF > deploy-settings.xml + + + + apache.snapshots.https + ${MAVEN_DEPLOY_USER} + ${MAVEN_DEPLOY_PASS} + + + + + google-maven-central + GCS Maven Central mirror + https://maven-central-eu.storage-download.googleapis.com/maven2/ + central + + + + EOF + + export CUSTOM_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} -Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml" + export MVN_RUN_VERBOSE=true + ./releasing/deploy_staging_jars.sh + env: + MAVEN_DEPLOY_USER: ${{ secrets.MAVEN_DEPLOY_USER }} + MAVEN_DEPLOY_PASS: ${{ secrets.MAVEN_DEPLOY_PASS }} + build_python_wheels: name: "Build Python Wheels on ${{ matrix.os_name }}" runs-on: ${{ matrix.os }} From 37aeda87d6ed23495bb1bdf2210f7a2d63d233a6 Mon Sep 17 00:00:00 2001 From: Dale Lane Date: Thu, 9 Jul 2026 14:12:19 +0100 Subject: [PATCH 2/4] fix: make python wheels inclusion in binary releases clearer The create_binary_release.sh script is used in two ways: 1) manually, as part of creating a release 2) automated, as part of a nightly pipeline When run manually, there is a documented expectation (in the wiki) that a release manager will put the wheels in place ahead of running the script. When run as part of the pipeline, the wheels won't be there, and this causes the script to exit with an error. The wheels aren't needed for nightly builds, so this error is ignored. This commit changes that behaviour to be more intentional. It explicitly stops the automated build from requiring wheels, and starts checking exit codes so any other errors in future will not be ignored. Signed-off-by: Dale Lane --- .github/workflows/nightly.yml | 2 +- tools/azure-pipelines/build-nightly-dist.yml | 5 ++++- tools/releasing/create_binary_release.sh | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ea7d5240590b9..43f06847ecd7e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -113,7 +113,7 @@ jobs: export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) echo "Determined RELEASE_VERSION as '$RELEASE_VERSION'" cd tools - MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh + MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh echo "Created files:" find ./releasing/release cd .. diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml index c3c7849a70ac2..8315db6c03739 100644 --- a/tools/azure-pipelines/build-nightly-dist.yml +++ b/tools/azure-pipelines/build-nightly-dist.yml @@ -51,7 +51,10 @@ jobs: export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) echo "Determined RELEASE_VERSION as '$RELEASE_VERSION' " cd tools - MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh + if ! MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh; then + echo "create_binary_release.sh failed" 1>&2 + exit 1 + fi echo "Created files:" find ./releasing/release cd .. diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index fff640920207b..b9ad6dab80e29 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -21,6 +21,11 @@ ## Variables with defaults (if not overwritten by environment) ## SKIP_GPG=${SKIP_GPG:-false} +# Set to true for automated builds (e.g. nightly snapshots) which +# do not try to include Python wheels in the binary release. +# Defaults to false for manual use, when the release manager will +# first download platform wheels to flink-python/dist +SKIP_PYTHON_WHEELS=${SKIP_PYTHON_WHEELS:-false} MVN=${MVN:-mvn} if [ -z "${RELEASE_VERSION:-}" ]; then @@ -135,6 +140,8 @@ make_python_release() { # py39,py310,py311,py312 for mac 10.9, 11.0 and linux (12 wheel packages) EXPECTED_WHEEL_PACKAGES_NUM=12 # Need to move the downloaded wheel packages from Azure CI to the directory flink-python/dist manually. + # (update glob behaviour to expand to nothing if there are no matching files) + shopt -s nullglob for wheel_file in *.whl; do if [[ ! ${wheel_file} =~ ^apache_flink-$PYFLINK_VERSION- ]]; then echo -e "\033[31;1mThe file name of the python package: ${wheel_file} is not consistent with given release version: ${PYFLINK_VERSION}!\033[0m" @@ -143,9 +150,14 @@ make_python_release() { cp ${wheel_file} "${PYTHON_RELEASE_DIR}/${wheel_file}" wheel_packages_num=$((wheel_packages_num+1)) done + shopt -u nullglob if [[ ${wheel_packages_num} != ${EXPECTED_WHEEL_PACKAGES_NUM} ]]; then - echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m" - exit 1 + if [[ "$SKIP_PYTHON_WHEELS" == "true" ]]; then + echo "SKIP_PYTHON_WHEELS is set: found ${wheel_packages_num} wheel(s) instead of the expected ${EXPECTED_WHEEL_PACKAGES_NUM}; continuing without them" + else + echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m" + exit 1 + fi fi cd ${PYTHON_RELEASE_DIR} From 497655fc5eb6194778f495798a62d0fae6c83be7 Mon Sep 17 00:00:00 2001 From: Dale Lane Date: Fri, 10 Jul 2026 11:45:02 +0100 Subject: [PATCH 3/4] fix: avoid AZP and GHA nightly pipelines overwriting each other If the deploy_nightly_to_s3.sh script is used by both overnight pipelines, then they need to write to separate locations to avoid them clobbering each other. I've left the AZP destination as-is, and added a prefix to the new GHA pipeline. If/when the AZP pipeline is retired, that prefix can be removed from the GHA call. Signed-off-by: Dale Lane --- .github/workflows/nightly.yml | 4 +++- tools/ci/deploy_nightly_to_s3.sh | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 43f06847ecd7e..b8400ca191892 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -122,7 +122,9 @@ jobs: shell: bash run: | source ./tools/ci/deploy_nightly_to_s3.sh - upload_to_s3 ./tools/releasing/release + # while AZP snapshot release builds are still being uploaded + # upload to a separate GHA-specific prefix to avoid overwrites + upload_to_s3 ./tools/releasing/release gha-trial/ env: ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }} diff --git a/tools/ci/deploy_nightly_to_s3.sh b/tools/ci/deploy_nightly_to_s3.sh index b6d1101ffaf85..3c1e5068bb70d 100755 --- a/tools/ci/deploy_nightly_to_s3.sh +++ b/tools/ci/deploy_nightly_to_s3.sh @@ -23,6 +23,7 @@ set -e -x function upload_to_s3() { local FILES_DIR=$1 + local PREFIX=${2:-} # The AWS CLI reads credentials from the environment (AWS_ACCESS_KEY_ID / # AWS_SECRET_ACCESS_KEY), so they are never placed on a command line and @@ -41,5 +42,5 @@ function upload_to_s3() { # Mirrors the previous uploader's behaviour: copy the directory contents to # the bucket root using the bucket's default (private) ACL. - aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/" --recursive --no-progress + aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/${PREFIX}" --recursive --no-progress } From 5053bd5a230428c146e1915858a6aea4d513d8a9 Mon Sep 17 00:00:00 2001 From: Dale Lane Date: Tue, 28 Jul 2026 11:41:26 +0100 Subject: [PATCH 4/4] feat: support for migrating one branch at a time Signed-off-by: Dale Lane --- .github/workflows/nightly.yml | 81 ++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b8400ca191892..c98a48c6a09d4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,38 @@ concurrency: permissions: read-all +env: + # Branches whose nightly snapshots are published by this pipeline (binaries + # to S3, jars to repository.apache.org). Branches not listed here will build + # snapshots, and keep them as build artifacts of the workflow run only. + # This allows the migration from Azure Pipelines to GitHub Actions to happen + # one release branch at a time, without both pipelines publishing over each + # other: add a branch here once its snapshots are no longer published by + # the Azure pipeline, or while it is in the shadow phase described under + # SHADOW_SNAPSHOT_BRANCHES below. + # + # Adding a branch here needs a matching change to that branch's copy of + # tools/azure-pipelines/build-apache-repo.yml, ideally in the same commit. + # Remove the "cron_snapshot_deployment" build-nightly-dist.yml template from + # the "cron_build" stage in tools/azure-pipelines/build-apache-repo.yml + # so that the Azure pipeline stops publishing snapshots for the branch. + # Removing that template will stop snapshots being published by AZP + PUBLISH_SNAPSHOT_BRANCHES: '["master"]' + + # Branches in the shadow phase, where this pipeline and the Azure pipeline both + # publish the same branch so that their output can be compared. Binaries for + # these branches are uploaded to a GHA-specific prefix in the S3 bucket instead + # of the bucket root, so that the two pipelines don't overwrite each other. + # (Snapshot jars in Maven don't need an equivalent, as they are timestamped.) + # + # A branch belongs here only while it is *also* still being published by the + # Azure pipeline. Remove it from this list in the same commit that removes the + # branch's "cron_snapshot_deployment" template from build-apache-repo.yml, so + # that it starts publishing to the bucket root as the Azure pipeline stops. + # Leaving a branch here after AZP has stopped means nothing is publishing its + # snapshots to the location that consumers read from. + SHADOW_SNAPSHOT_BRANCHES: '["master"]' + jobs: pre-compile-checks: name: "Pre-compile Checks" @@ -88,9 +120,10 @@ jobs: CONTAINER_LOCAL_WORKING_DIR: /root/flink MAVEN_REPO_FOLDER: /root/.m2/repository MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + RELEASE_DIR: /root/flink/tools/releasing/release steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false - name: "Initialize job" @@ -117,15 +150,26 @@ jobs: echo "Created files:" find ./releasing/release cd .. + # Uploaded for every branch so that the binaries can be inspected and + # compared against the ones the Azure pipeline produces. + - name: "Upload snapshot binary release as build artifact" + uses: actions/upload-artifact@v7 + with: + name: snapshot-binary-release-${{ github.run_number }} + path: ${{ env.RELEASE_DIR }} + if-no-files-found: error + retention-days: 5 - name: "Upload artifacts to S3" + if: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} shell: bash run: | source ./tools/ci/deploy_nightly_to_s3.sh - # while AZP snapshot release builds are still being uploaded - # upload to a separate GHA-specific prefix to avoid overwrites - upload_to_s3 ./tools/releasing/release gha-trial/ + # branches in the shadow phase upload to a GHA-specific prefix, so that + # they don't overwrite the builds AZP is still uploading + upload_to_s3 ./tools/releasing/release "${S3_PREFIX}" env: + S3_PREFIX: ${{ contains(fromJSON(env.SHADOW_SNAPSHOT_BRANCHES), github.ref_name) && 'gha-trial/' || '' }} ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }} @@ -142,9 +186,11 @@ jobs: CONTAINER_LOCAL_WORKING_DIR: /root/flink MAVEN_REPO_FOLDER: /root/.m2/repository MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + STAGING_JARS_DIR: /root/staging-jars + STAGING_JARS_TARBALL: /root/staging-jars.tar.gz steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false - name: "Initialize job" @@ -189,11 +235,36 @@ jobs: EOF export CUSTOM_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} -Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml" + + if [ "${PUBLISH_SNAPSHOTS}" != "true" ]; then + # this branch's snapshot jars are published by the Azure pipeline, so + # deploy into a local repository that is kept as a build artifact + # instead of pushing to repository.apache.org + echo "'${GITHUB_REF_NAME}' does not publish snapshots - deploying to ${STAGING_JARS_DIR} instead of repository.apache.org" + export CUSTOM_OPTIONS="${CUSTOM_OPTIONS} -DaltDeploymentRepository=staging::file://${STAGING_JARS_DIR}" + fi + export MVN_RUN_VERBOSE=true ./releasing/deploy_staging_jars.sh env: MAVEN_DEPLOY_USER: ${{ secrets.MAVEN_DEPLOY_USER }} MAVEN_DEPLOY_PASS: ${{ secrets.MAVEN_DEPLOY_PASS }} + PUBLISH_SNAPSHOTS: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + - name: "Archive locally deployed snapshot jars" + if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + shell: bash + run: | + du -sh "${STAGING_JARS_DIR}" + tar --create --gzip --file "${STAGING_JARS_TARBALL}" -C "${STAGING_JARS_DIR}" . + ls -lh "${STAGING_JARS_TARBALL}" + - name: "Upload snapshot jars as build artifact" + if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + uses: actions/upload-artifact@v7 + with: + name: snapshot-maven-jars-${{ github.run_number }} + path: ${{ env.STAGING_JARS_TARBALL }} + if-no-files-found: error + retention-days: 5 build_python_wheels: name: "Build Python Wheels on ${{ matrix.os_name }}"