diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 70c6f8d09e606e..c98a48c6a09d4c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,38 @@ concurrency: permissions: read-all +env: + # Branches whose nightly snapshots are published by this pipeline (binaries + # to S3, jars to repository.apache.org). Branches not listed here will build + # snapshots, and keep them as build artifacts of the workflow run only. + # This allows the migration from Azure Pipelines to GitHub Actions to happen + # one release branch at a time, without both pipelines publishing over each + # other: add a branch here once its snapshots are no longer published by + # the Azure pipeline, or while it is in the shadow phase described under + # SHADOW_SNAPSHOT_BRANCHES below. + # + # Adding a branch here needs a matching change to that branch's copy of + # tools/azure-pipelines/build-apache-repo.yml, ideally in the same commit. + # Remove the "cron_snapshot_deployment" build-nightly-dist.yml template from + # the "cron_build" stage in tools/azure-pipelines/build-apache-repo.yml + # so that the Azure pipeline stops publishing snapshots for the branch. + # Removing that template will stop snapshots being published by AZP + PUBLISH_SNAPSHOT_BRANCHES: '["master"]' + + # Branches in the shadow phase, where this pipeline and the Azure pipeline both + # publish the same branch so that their output can be compared. Binaries for + # these branches are uploaded to a GHA-specific prefix in the S3 bucket instead + # of the bucket root, so that the two pipelines don't overwrite each other. + # (Snapshot jars in Maven don't need an equivalent, as they are timestamped.) + # + # A branch belongs here only while it is *also* still being published by the + # Azure pipeline. Remove it from this list in the same commit that removes the + # branch's "cron_snapshot_deployment" template from build-apache-repo.yml, so + # that it starts publishing to the bucket root as the Azure pipeline stops. + # Leaving a branch here after AZP has stopped means nothing is publishing its + # snapshots to the location that consumers read from. + SHADOW_SNAPSHOT_BRANCHES: '["master"]' + jobs: pre-compile-checks: name: "Pre-compile Checks" @@ -77,6 +109,163 @@ jobs: s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }} s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }} + snapshot_binary: + name: "Snapshot Binary Release" + runs-on: ubuntu-24.04 + container: + image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy + options: --init --privileged + env: + MOUNTED_WORKING_DIR: /__w/flink/flink + CONTAINER_LOCAL_WORKING_DIR: /root/flink + MAVEN_REPO_FOLDER: /root/.m2/repository + MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + RELEASE_DIR: /root/flink/tools/releasing/release + steps: + - name: "Flink Checkout" + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: "Initialize job" + uses: "./.github/actions/job_init" + with: + jdk_version: 11 + maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }} + source_directory: ${{ env.MOUNTED_WORKING_DIR }} + target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + - name: "Build snapshot binary release" + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available + set +o errexit + source ./tools/ci/maven-utils.sh + set -o errexit + run_mvn -version + export MVN="run_mvn" + export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "Determined RELEASE_VERSION as '$RELEASE_VERSION'" + cd tools + MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh + echo "Created files:" + find ./releasing/release + cd .. + # Uploaded for every branch so that the binaries can be inspected and + # compared against the ones the Azure pipeline produces. + - name: "Upload snapshot binary release as build artifact" + uses: actions/upload-artifact@v7 + with: + name: snapshot-binary-release-${{ github.run_number }} + path: ${{ env.RELEASE_DIR }} + if-no-files-found: error + retention-days: 5 + - name: "Upload artifacts to S3" + if: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + source ./tools/ci/deploy_nightly_to_s3.sh + # branches in the shadow phase upload to a GHA-specific prefix, so that + # they don't overwrite the builds AZP is still uploading + upload_to_s3 ./tools/releasing/release "${S3_PREFIX}" + env: + S3_PREFIX: ${{ contains(fromJSON(env.SHADOW_SNAPSHOT_BRANCHES), github.ref_name) && 'gha-trial/' || '' }} + ARTIFACTS_S3_BUCKET: ${{ secrets.ARTIFACTS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }} + + snapshot_maven: + name: "Snapshot Maven Deploy" + runs-on: ubuntu-24.04 + timeout-minutes: 240 + container: + image: apache/flink-ci-docker:java_8_11_17_21_25_maven_386_jammy + options: --init --privileged + env: + MOUNTED_WORKING_DIR: /__w/flink/flink + CONTAINER_LOCAL_WORKING_DIR: /root/flink + MAVEN_REPO_FOLDER: /root/.m2/repository + MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository + STAGING_JARS_DIR: /root/staging-jars + STAGING_JARS_TARBALL: /root/staging-jars.tar.gz + steps: + - name: "Flink Checkout" + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: "Initialize job" + uses: "./.github/actions/job_init" + with: + jdk_version: 11 + maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }} + source_directory: ${{ env.MOUNTED_WORKING_DIR }} + target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + - name: "Deploy Maven snapshot" + working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} + shell: bash + run: | + # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available + set +o errexit + source ./tools/ci/maven-utils.sh + set -o errexit + run_mvn -version + + cd tools + cat << EOF > deploy-settings.xml + + + + apache.snapshots.https + ${MAVEN_DEPLOY_USER} + ${MAVEN_DEPLOY_PASS} + + + + + google-maven-central + GCS Maven Central mirror + https://maven-central-eu.storage-download.googleapis.com/maven2/ + central + + + + EOF + + export CUSTOM_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} -Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml" + + if [ "${PUBLISH_SNAPSHOTS}" != "true" ]; then + # this branch's snapshot jars are published by the Azure pipeline, so + # deploy into a local repository that is kept as a build artifact + # instead of pushing to repository.apache.org + echo "'${GITHUB_REF_NAME}' does not publish snapshots - deploying to ${STAGING_JARS_DIR} instead of repository.apache.org" + export CUSTOM_OPTIONS="${CUSTOM_OPTIONS} -DaltDeploymentRepository=staging::file://${STAGING_JARS_DIR}" + fi + + export MVN_RUN_VERBOSE=true + ./releasing/deploy_staging_jars.sh + env: + MAVEN_DEPLOY_USER: ${{ secrets.MAVEN_DEPLOY_USER }} + MAVEN_DEPLOY_PASS: ${{ secrets.MAVEN_DEPLOY_PASS }} + PUBLISH_SNAPSHOTS: ${{ contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + - name: "Archive locally deployed snapshot jars" + if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + shell: bash + run: | + du -sh "${STAGING_JARS_DIR}" + tar --create --gzip --file "${STAGING_JARS_TARBALL}" -C "${STAGING_JARS_DIR}" . + ls -lh "${STAGING_JARS_TARBALL}" + - name: "Upload snapshot jars as build artifact" + if: ${{ !contains(fromJSON(env.PUBLISH_SNAPSHOT_BRANCHES), github.ref_name) }} + uses: actions/upload-artifact@v7 + with: + name: snapshot-maven-jars-${{ github.run_number }} + path: ${{ env.STAGING_JARS_TARBALL }} + if-no-files-found: error + retention-days: 5 + build_python_wheels: name: "Build Python Wheels on ${{ matrix.os_name }}" runs-on: ${{ matrix.os }} diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml index c3c7849a70ac23..8315db6c03739a 100644 --- a/tools/azure-pipelines/build-nightly-dist.yml +++ b/tools/azure-pipelines/build-nightly-dist.yml @@ -51,7 +51,10 @@ jobs: export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) echo "Determined RELEASE_VERSION as '$RELEASE_VERSION' " cd tools - MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh + if ! MVN_RUN_VERBOSE=true SKIP_GPG=true SKIP_PYTHON_WHEELS=true ./releasing/create_binary_release.sh; then + echo "create_binary_release.sh failed" 1>&2 + exit 1 + fi echo "Created files:" find ./releasing/release cd .. diff --git a/tools/ci/deploy_nightly_to_s3.sh b/tools/ci/deploy_nightly_to_s3.sh index b6d1101ffaf85f..3c1e5068bb70d0 100755 --- a/tools/ci/deploy_nightly_to_s3.sh +++ b/tools/ci/deploy_nightly_to_s3.sh @@ -23,6 +23,7 @@ set -e -x function upload_to_s3() { local FILES_DIR=$1 + local PREFIX=${2:-} # The AWS CLI reads credentials from the environment (AWS_ACCESS_KEY_ID / # AWS_SECRET_ACCESS_KEY), so they are never placed on a command line and @@ -41,5 +42,5 @@ function upload_to_s3() { # Mirrors the previous uploader's behaviour: copy the directory contents to # the bucket root using the bucket's default (private) ACL. - aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/" --recursive --no-progress + aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/${PREFIX}" --recursive --no-progress } diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index fff640920207bd..b9ad6dab80e29e 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -21,6 +21,11 @@ ## Variables with defaults (if not overwritten by environment) ## SKIP_GPG=${SKIP_GPG:-false} +# Set to true for automated builds (e.g. nightly snapshots) which +# do not try to include Python wheels in the binary release. +# Defaults to false for manual use, when the release manager will +# first download platform wheels to flink-python/dist +SKIP_PYTHON_WHEELS=${SKIP_PYTHON_WHEELS:-false} MVN=${MVN:-mvn} if [ -z "${RELEASE_VERSION:-}" ]; then @@ -135,6 +140,8 @@ make_python_release() { # py39,py310,py311,py312 for mac 10.9, 11.0 and linux (12 wheel packages) EXPECTED_WHEEL_PACKAGES_NUM=12 # Need to move the downloaded wheel packages from Azure CI to the directory flink-python/dist manually. + # (update glob behaviour to expand to nothing if there are no matching files) + shopt -s nullglob for wheel_file in *.whl; do if [[ ! ${wheel_file} =~ ^apache_flink-$PYFLINK_VERSION- ]]; then echo -e "\033[31;1mThe file name of the python package: ${wheel_file} is not consistent with given release version: ${PYFLINK_VERSION}!\033[0m" @@ -143,9 +150,14 @@ make_python_release() { cp ${wheel_file} "${PYTHON_RELEASE_DIR}/${wheel_file}" wheel_packages_num=$((wheel_packages_num+1)) done + shopt -u nullglob if [[ ${wheel_packages_num} != ${EXPECTED_WHEEL_PACKAGES_NUM} ]]; then - echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m" - exit 1 + if [[ "$SKIP_PYTHON_WHEELS" == "true" ]]; then + echo "SKIP_PYTHON_WHEELS is set: found ${wheel_packages_num} wheel(s) instead of the expected ${EXPECTED_WHEEL_PACKAGES_NUM}; continuing without them" + else + echo -e "\033[31;1mThe number of wheel packages ${wheel_packages_num} is not equal to the expected number ${EXPECTED_WHEEL_PACKAGES_NUM}!\033[0m" + exit 1 + fi fi cd ${PYTHON_RELEASE_DIR}