From 93a62ba3735257e08c8a4badaf2dde46863867dc Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Mon, 7 Sep 2026 13:53:04 +0200 Subject: [PATCH 1/3] ci: retry the network fetches every job starts with rosdep update reads the distribution index from raw.githubusercontent.com and apt-get update reads the package indexes from the mirrors. Both run before anything of ours is built, and a reset connection on either fails the job for a reason that has nothing to do with the change under test. rosdep update now gets three attempts, and apt-get update uses apt's own Acquire::Retries. --- .github/workflows/ci.yml | 26 +++++++++++++------------- .github/workflows/docs.yml | 2 +- .github/workflows/opcua-plugin.yml | 14 +++++++------- .github/workflows/quality.yml | 30 +++++++++++++++--------------- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c7888b0e..71f6b19d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -68,13 +68,13 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs if [ "${{ matrix.ros_distro }}" = "humble" ]; then apt-get install -y ros-humble-rmw-cyclonedds-cpp fi source /opt/ros/${{ matrix.ros_distro }}/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done # Linters (clang-tidy / clang-format) only run in the Jazzy quality # job (quality.yml). Multiple package.xml files still list them as # , so we skip the rosdep keys here to keep build-and- @@ -200,7 +200,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -232,13 +232,13 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs if [ "${{ matrix.ros_distro }}" = "humble" ] || [ "${{ matrix.ros_distro }}" = "jazzy" ]; then apt-get install -y ros-${{ matrix.ros_distro }}-rmw-cyclonedds-cpp fi source /opt/ros/${{ matrix.ros_distro }}/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done # Same skip-keys as build-and-test: the linters run only in the Jazzy quality job. rosdep install --from-paths src --ignore-src -y \ --skip-keys "ament_cmake_clang_tidy ament_cmake_clang_format" @@ -332,7 +332,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -366,12 +366,12 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y \ ros-jazzy-test-msgs \ ros-jazzy-rmw-cyclonedds-cpp source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Build packages @@ -500,7 +500,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -532,11 +532,11 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update # gpg is required by codecov/codecov-action@v5 dependency check apt-get install -y lcov ros-jazzy-test-msgs gpg source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -r -y - name: Build packages with coverage @@ -663,7 +663,7 @@ jobs: steps: - name: Install tools run: | - apt-get update + apt-get -o Acquire::Retries=3 update # gpg and curl are both required by the codecov action's dependency # check, which fails the step rather than skipping the upload. apt-get install -y git lcov gpg curl diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 565e0cdcc..8f1451957 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | - sudo apt-get update + sudo apt-get -o Acquire::Retries=3 update sudo apt-get install -y graphviz plantuml doxygen pip install -e docs/.[dev] diff --git a/.github/workflows/opcua-plugin.yml b/.github/workflows/opcua-plugin.yml index 43cf4a39d..8184c9f1d 100644 --- a/.github/workflows/opcua-plugin.yml +++ b/.github/workflows/opcua-plugin.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -82,10 +82,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs libyaml-cpp-dev libssl-dev source /opt/ros/${{ matrix.ros_distro }}/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done # Skip nav2_msgs (vda5050_agent declares it; not available on all # distros) and the linter rosdep keys. Linters only run in the Jazzy # quality job, so we don't need them installed here. The upstream @@ -242,7 +242,7 @@ jobs: - name: Install jq + asyncua (smoke test prerequisite) run: | - sudo apt-get update + sudo apt-get -o Acquire::Retries=3 update sudo apt-get install -y jq python3-pip pip3 install --break-system-packages asyncua @@ -276,7 +276,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -303,12 +303,12 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update # openssl: cert generation. libssl-dev: encryption backend for both # the plugin's open62541pp and the fixture's static open62541. apt-get install -y ros-jazzy-test-msgs libyaml-cpp-dev libssl-dev openssl source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y \ --skip-keys='nav2_msgs ament_cmake_clang_format ament_cmake_clang_tidy' diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 3f5d3300d..81b635504 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -49,10 +49,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y clang-format ccache source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Cache ccache @@ -113,7 +113,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -160,10 +160,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Build (compile database only, no clang-tidy) @@ -265,7 +265,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -292,10 +292,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Redirect heavy build output to /mnt @@ -419,7 +419,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -446,10 +446,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Redirect heavy build output to /mnt @@ -566,7 +566,7 @@ jobs: steps: - name: Install Git run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y git - name: Checkout repository @@ -598,10 +598,10 @@ jobs: - name: Install dependencies run: | - apt-get update + apt-get -o Acquire::Retries=3 update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash - rosdep update + for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done rosdep install --from-paths src --ignore-src -y - name: Redirect heavy build output to /mnt From b7054f401edfc3503bd89a6debf5ea22011ad06f Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Tue, 8 Sep 2026 09:41:34 +0200 Subject: [PATCH 2/3] ci: keep the retry loop to the fetches that do not have one apt already retries a failed index fetch by itself. The default for Acquire::Retries has been 3 since apt 2.3.2, and on the three images this repository builds in - ubuntu:jammy (apt 2.4.14), ubuntu:noble (2.8.3) and ubuntu:resolute (3.2.0) - a fetch against a dead endpoint makes four attempts with the flag and four attempts without it, and one attempt with Acquire::Retries=0. The calls go back to plain apt-get update. The ctcache install in the clang-tidy job does need a loop. It is a git+https URL, so pip runs git clone in a subprocess to fetch it, and pip's own --retries only covers its HTTP session. --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/docs.yml | 2 +- .github/workflows/opcua-plugin.yml | 10 +++++----- .github/workflows/quality.yml | 22 +++++++++++----------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71f6b19d2..0d2d83ae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -68,7 +68,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs if [ "${{ matrix.ros_distro }}" = "humble" ]; then apt-get install -y ros-humble-rmw-cyclonedds-cpp @@ -200,7 +200,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -232,7 +232,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs if [ "${{ matrix.ros_distro }}" = "humble" ] || [ "${{ matrix.ros_distro }}" = "jazzy" ]; then apt-get install -y ros-${{ matrix.ros_distro }}-rmw-cyclonedds-cpp @@ -332,7 +332,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -366,7 +366,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y \ ros-jazzy-test-msgs \ ros-jazzy-rmw-cyclonedds-cpp @@ -500,7 +500,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -532,7 +532,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update # gpg is required by codecov/codecov-action@v5 dependency check apt-get install -y lcov ros-jazzy-test-msgs gpg source /opt/ros/jazzy/setup.bash @@ -663,7 +663,7 @@ jobs: steps: - name: Install tools run: | - apt-get -o Acquire::Retries=3 update + apt-get update # gpg and curl are both required by the codecov action's dependency # check, which fails the step rather than skipping the upload. apt-get install -y git lcov gpg curl diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f1451957..565e0cdcc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | - sudo apt-get -o Acquire::Retries=3 update + sudo apt-get update sudo apt-get install -y graphviz plantuml doxygen pip install -e docs/.[dev] diff --git a/.github/workflows/opcua-plugin.yml b/.github/workflows/opcua-plugin.yml index 8184c9f1d..633206379 100644 --- a/.github/workflows/opcua-plugin.yml +++ b/.github/workflows/opcua-plugin.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -82,7 +82,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs libyaml-cpp-dev libssl-dev source /opt/ros/${{ matrix.ros_distro }}/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done @@ -242,7 +242,7 @@ jobs: - name: Install jq + asyncua (smoke test prerequisite) run: | - sudo apt-get -o Acquire::Retries=3 update + sudo apt-get update sudo apt-get install -y jq python3-pip pip3 install --break-system-packages asyncua @@ -276,7 +276,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -303,7 +303,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update # openssl: cert generation. libssl-dev: encryption backend for both # the plugin's open62541pp and the fixture's static open62541. apt-get install -y ros-jazzy-test-msgs libyaml-cpp-dev libssl-dev openssl diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 81b635504..d29fa2489 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y clang-format ccache source /opt/ros/jazzy/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done @@ -113,7 +113,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -146,7 +146,7 @@ jobs: - name: Install clang-tidy-cache (ctcache) run: | apt-get install -y python3-pip - pip3 install --break-system-packages git+https://github.com/matus-chochlik/ctcache.git@3880541496c7ce0d2e2830d2f525375078fe0b1f + for attempt in 1 2 3; do pip3 install --break-system-packages git+https://github.com/matus-chochlik/ctcache.git@3880541496c7ce0d2e2830d2f525375078fe0b1f && break; [ "$attempt" = 3 ] && exit 1; echo "ctcache install attempt $attempt failed, retrying" >&2; sleep 5; done printf '#!/bin/sh\nexec clang-tidy-cache /usr/bin/clang-tidy "$@"\n' > /usr/local/bin/ctcache-wrapper chmod +x /usr/local/bin/ctcache-wrapper @@ -160,7 +160,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done @@ -265,7 +265,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -292,7 +292,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done @@ -419,7 +419,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -446,7 +446,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done @@ -566,7 +566,7 @@ jobs: steps: - name: Install Git run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y git - name: Checkout repository @@ -598,7 +598,7 @@ jobs: - name: Install dependencies run: | - apt-get -o Acquire::Retries=3 update + apt-get update apt-get install -y ros-jazzy-test-msgs source /opt/ros/jazzy/setup.bash for attempt in 1 2 3; do rosdep update && break; [ "$attempt" = 3 ] && exit 1; echo "rosdep update attempt $attempt failed, retrying" >&2; sleep 5; done From 6b4a66e8b437c0631fcb61604ed359f344a0a10f Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Tue, 8 Sep 2026 13:28:19 +0200 Subject: [PATCH 3/3] ci: give the ros-apt-source download a retry window that outlasts an outage The release asset is the one fetch in this action with nothing to fall back to. The version lookup above it can use the pinned value when the API is unreachable, but a 5xx on the .deb ends the step, and the step failing takes the whole job with it: setup-ros then has no apt source and the build stops at "colcon: command not found". Measured against a server that answers 500 every time: the old flags make 6 attempts over 18 seconds, the new ones make 9 over 131 seconds. A short outage on the release host is now waited out instead of failing every job that starts during it. --- .github/actions/ros-apt-source/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/ros-apt-source/action.yml b/.github/actions/ros-apt-source/action.yml index ea8e2fd45..f47583db6 100644 --- a/.github/actions/ros-apt-source/action.yml +++ b/.github/actions/ros-apt-source/action.yml @@ -49,7 +49,13 @@ runs: url="https://github.com/ros-infrastructure/ros-apt-source/releases/download/${version}/${deb}" echo "Installing ${deb} (ros-apt-source ${version}, ${codename})" - curl --fail --location --retry 5 --retry-delay 3 -o "/tmp/${deb}" "${url}" + # This download has no fallback: the version lookup above can fall back to + # a pin, but a 5xx on the release asset itself ends the job. The retry + # window is therefore minutes rather than seconds, so a short outage on + # the release host is waited out instead of failing every job that starts + # during it. + curl --fail --location --retry 8 --retry-delay 15 --retry-max-time 180 \ + -o "/tmp/${deb}" "${url}" $SUDO dpkg -i "/tmp/${deb}" rm -f "/tmp/${deb}" $SUDO apt-get update