[FLINK-40248][ci] Add nightly snapshot publishing to GHA pipeline

.github/workflows/nightly.yml

@@ -27,6 +27,38 @@ concurrency:
 
 permissions: read-all
 
+env:
+  # Branches whose nightly snapshots are published by this pipeline (binaries
+  # to S3, jars to repository.apache.org). Branches not listed here will build
+  # snapshots, and keep them as build artifacts of the workflow run only.
+  # This allows the migration from Azure Pipelines to GitHub Actions to happen
+  # one release branch at a time, without both pipelines publishing over each
+  # other: add a branch here once its snapshots are no longer published by
+  # the Azure pipeline, or while it is in the shadow phase described under
+  # SHADOW_SNAPSHOT_BRANCHES below.
+  #
+  # Adding a branch here needs a matching change to that branch's copy of
+  # tools/azure-pipelines/build-apache-repo.yml, ideally in the same commit.
+  # Remove the "cron_snapshot_deployment" build-nightly-dist.yml template from
+  # the "cron_build" stage in tools/azure-pipelines/build-apache-repo.yml
+  # so that the Azure pipeline stops publishing snapshots for the branch.
+  # Removing that template will stop snapshots being published by AZP
+  PUBLISH_SNAPSHOT_BRANCHES: '["master"]'
+
+  # Branches in the shadow phase, where this pipeline and the Azure pipeline both
+  # publish the same branch so that their output can be compared. Binaries for
+  # these branches are uploaded to a GHA-specific prefix in the S3 bucket instead
+  # of the bucket root, so that the two pipelines don't overwrite each other.
+  # (Snapshot jars in Maven don't need an equivalent, as they are timestamped.)
+  #
+  # A branch belongs here only while it is *also* still being published by the
+  # Azure pipeline. Remove it from this list in the same commit that removes the
+  # branch's "cron_snapshot_deployment" template from build-apache-repo.yml, so
+  # that it starts publishing to the bucket root as the Azure pipeline stops.
+  # Leaving a branch here after AZP has stopped means nothing is publishing its
+  # snapshots to the location that consumers read from.
+  SHADOW_SNAPSHOT_BRANCHES: '["master"]'
+
 jobs:
   pre-compile-checks:
     name: "Pre-compile Checks"
@@ -77,6 +109,163 @@ jobs:
       s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
       s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
 
+  snapshot_binary:
+    name: "Snapshot Binary Release"
+    runs-on: ubuntu-24.04
+    container:
+      image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
+      options: --init --privileged
+    env:
+      MOUNTED_WORKING_DIR: /__w/flink/flink
+      CONTAINER_LOCAL_WORKING_DIR: /root/flink
+      MAVEN_REPO_FOLDER: /root/.m2/repository
+      MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
+      RELEASE_DIR: /root/flink/tools/releasing/release
+    steps:
+      - name: "Flink Checkout"
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: "Initialize job"
+        uses: "./.github/actions/job_init"
+        with:
+          jdk_version: 11
+          maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }}
+          source_directory: ${{ env.MOUNTED_WORKING_DIR }}
+          target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+      - name: "Build snapshot binary release"
+        working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+        shell: bash
+        run: |
+          # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available
+          set +o errexit
+          source ./tools/ci/maven-utils.sh
+          set -o errexit
+          run_mvn -version
+          export MVN="run_mvn"
+          export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+          echo "Determined RELEASE_VERSION as '$RELEASE_VERSION'"
+          cd tools
+          MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh
+          echo "Created files:"
+          find ./releasing/release
+          cd ..
+      # Uploaded for every branch so that the binaries can be inspected and
+      # compared against the ones the Azure pipeline produces.
+      - name: "Upload snapshot binary release as build artifact"
+        uses: actions/upload-artifact@v7
+        with:
+          name: snapshot-binary-release-${{ github.run_number }}
+          path: ${{ env.RELEASE_DIR }}
+          if-no-files-found: error
+          retention-days: 5
+      - name: "Upload artifacts to S3"
+        if: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }}
+        working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+        shell: bash
+        run: |
+          source ./tools/ci/deploy_nightly_to_s3.sh
+          # branches in the shadow phase upload to a GHA-specific prefix, so that
+          #  they don't overwrite the builds AZP is still uploading
+          upload_to_s3 ./tools/releasing/release "${S3_PREFIX}"
+        env:
+          S3_PREFIX: ${{ contains(fromJSON(env.SHADOW_SNAPSHOT_BRANCHES), github.ref_name) && 'gha-trial/' || '' }}
+          ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }}
+
+  snapshot_maven:
+    name: "Snapshot Maven Deploy"
+    runs-on: ubuntu-24.04
+    timeout-minutes: 240
+    container:
+      image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy
+      options: --init --privileged
+    env:
+      MOUNTED_WORKING_DIR: /__w/flink/flink
+      CONTAINER_LOCAL_WORKING_DIR: /root/flink
+      MAVEN_REPO_FOLDER: /root/.m2/repository
+      MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
+      STAGING_JARS_DIR: /root/staging-jars
+      STAGING_JARS_TARBALL: /root/staging-jars.tar.gz
+    steps:
+      - name: "Flink Checkout"
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: "Initialize job"
+        uses: "./.github/actions/job_init"
+        with:
+          jdk_version: 11
+          maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }}
+          source_directory: ${{ env.MOUNTED_WORKING_DIR }}
+          target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+      - name: "Deploy Maven snapshot"
+        working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
+        shell: bash
+        run: |
+          # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available
+          set +o errexit
+          source ./tools/ci/maven-utils.sh
+          set -o errexit
+          run_mvn -version
+
+          cd tools
+          cat << EOF > deploy-settings.xml
+          <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                    xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                                        https://maven.apache.org/xsd/settings-1.0.0.xsd">
+            <servers>
+              <server>
+                <id>apache.snapshots.https</id>
+                <username>${MAVEN_DEPLOY_USER}</username>
+                <password>${MAVEN_DEPLOY_PASS}</password>
+              </server>
+            </servers>
+            <mirrors>
+              <mirror>
+                <id>google-maven-central</id>
+                <name>GCS Maven Central mirror</name>
+                <url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
+                <mirrorOf>central</mirrorOf>
+              </mirror>
+            </mirrors>
+          </settings>
+          EOF
+
+          export CUSTOM_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} -Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml"
+
+          if [ "${PUBLISH_SNAPSHOTS}" != "true" ]; then
+            # this branch's snapshot jars are published by the Azure pipeline, so
+            #  deploy into a local repository that is kept as a build artifact
+            #  instead of pushing to repository.apache.org
+            echo "'${GITHUB_REF_NAME}' does not publish snapshots - deploying to ${STAGING_JARS_DIR} instead of repository.apache.org"
+            export CUSTOM_OPTIONS="${CUSTOM_OPTIONS} -DaltDeploymentRepository=staging::file://${STAGING_JARS_DIR}"
+          fi
+
+          export MVN_RUN_VERBOSE=true
+          ./releasing/deploy_staging_jars.sh
+        env:
+          MAVEN_DEPLOY_USER: ${{ secrets.MAVEN_DEPLOY_USER }}
+          MAVEN_DEPLOY_PASS: ${{ secrets.MAVEN_DEPLOY_PASS }}
+          PUBLISH_SNAPSHOTS: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }}
+      - name: "Archive locally deployed snapshot jars"
+        if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }}
+        shell: bash
+        run: |
+          du -sh "${STAGING_JARS_DIR}"
+          tar --create --gzip --file "${STAGING_JARS_TARBALL}" -C "${STAGING_JARS_DIR}" .
+          ls -lh "${STAGING_JARS_TARBALL}"
+      - name: "Upload snapshot jars as build artifact"
+        if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: snapshot-maven-jars-${{ github.run_number }}
+          path: ${{ env.STAGING_JARS_TARBALL }}
+          if-no-files-found: error
+          retention-days: 5
+
   build_python_wheels:
     name: "Build Python Wheels on ${{ matrix.os_name }}"
     runs-on: ${{ matrix.os }}

tools/azure-pipelines/build-nightly-dist.yml

@@ -51,7 +51,10 @@ jobs:
             export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
             echo "Determined RELEASE_VERSION as '$RELEASE_VERSION' "
             cd tools
-            MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh
+            if ! MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh; then
+              echo "create_binary_release.sh failed" 1>&2
+              exit 1
+            fi
             echo "Created files:"
             find ./releasing/release
             cd ..

tools/ci/deploy_nightly_to_s3.sh

@@ -23,6 +23,7 @@ set -e -x
 
 function upload_to_s3() {
 	local FILES_DIR=$1
+	local PREFIX=${2:-}
 
 	# The AWS CLI reads credentials from the environment (AWS_ACCESS_KEY_ID /
 	# AWS_SECRET_ACCESS_KEY), so they are never placed on a command line and
@@ -41,5 +42,5 @@ function upload_to_s3() {
 
 	# Mirrors the previous uploader's behaviour: copy the directory contents to
 	# the bucket root using the bucket's default (private) ACL.
-	aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/" --recursive --no-progress
+	aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/${PREFIX}" --recursive --no-progress
 }

tools/releasing/create_binary_release.sh

@@ -21,6 +21,11 @@
 ## Variables with defaults (if not overwritten by environment)
 ##
 SKIP_GPG=${SKIP_GPG:-false}
+# Set to true for automated builds (e.g. nightly snapshots) which
+#  do not try to include Python wheels in the binary release.
+# Defaults to false for manual use, when the release manager will
+#  first download platform wheels to flink-python/dist
+SKIP_PYTHON_WHEELS=${SKIP_PYTHON_WHEELS:-false}
 MVN=${MVN:-mvn}
 
 if [ -z "${RELEASE_VERSION:-}" ]; then
@@ -135,6 +140,8 @@ make_python_release() {
   # py39,py310,py311,py312 for mac 10.9, 11.0 and linux (12 wheel packages)
   EXPECTED_WHEEL_PACKAGES_NUM=12
   # Need to move the downloaded wheel packages from Azure CI to the directory flink-python/dist manually.
+  # (update glob behaviour to expand to nothing if there are no matching files)
+  shopt -s nullglob
   for wheel_file in *.whl; do
     if [[ ! ${wheel_file} =~ ^apache_flink-$PYFLINK_VERSION- ]]; then
         echo -e "\033[31;1mThe file name of the python package: ${wheel_file} is not consistent with given release version: ${PYFLINK_VERSION}!\033[0m"
@@ -143,9 +150,14 @@ make_python_release() {
     cp ${wheel_file} "${PYTHON_RELEASE_DIR}/${wheel_file}"
     wheel_packages_num=$((wheel_packages_num+1))
   done
+  shopt -u nullglob
   if [[ ${wheel_packages_num} != ${EXPECTED_WHEEL_PACKAGES_NUM} ]]; then
-    echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m"
-    exit 1
+    if [[ "$SKIP_PYTHON_WHEELS" == "true" ]]; then
+      echo "SKIP_PYTHON_WHEELS is set: found ${wheel_packages_num} wheel(s) instead of the expected ${EXPECTED_WHEEL_PACKAGES_NUM}; continuing without them"
+    else
+      echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m"
+      exit 1
+    fi
   fi
 
   cd ${PYTHON_RELEASE_DIR}