ci: reduce iOS functional test flakiness

.github/workflows/functional-test.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: [ master ]
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -15,6 +18,7 @@ env:
   APPIUM_TEST_SERVER_HOST: '127.0.0.1'
   PYTHONUNBUFFERED: 1
   UV_LOCKED: 'true'
+  UV_VERSION: '0.11.32'
 
 jobs:
   ios_test:
@@ -49,6 +53,21 @@ jobs:
       with:
         xcode-version: ${{ env.XCODE_VERSION }}
 
+    - name: Set up Python
+      uses: actions/setup-python@v7
+      with:
+        python-version: '3.14'
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@v10.3.3
+      with:
+        enable-cache: true
+        python-version: '3.14'
+        version: ${{ env.UV_VERSION }}
+
+    - name: Install Python dependencies
+      run: uv sync
+
     - name: Install Appium and drivers
       run: |
         npm install -g appium
@@ -62,11 +81,8 @@ jobs:
         host: ${{ env.APPIUM_TEST_SERVER_HOST }}
         server_args: '--relaxed-security'
 
-    - name: Start iOS Simulator UI
-      run: |
-        defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false
-        open -Fn "$(xcode-select --print-path)/Applications/Simulator.app"
     - name: Prepare iOS simulator
+      timeout-minutes: 20
       uses: futureware-tech/simulator-action@v6
       with:
         model: ${{ env.IPHONE_MODEL }}
@@ -77,16 +93,27 @@ jobs:
     - name: Finalize iOS simulator boot
       run: xcrun --sdk iphonesimulator --show-sdk-version
 
-    - name: Set up Python
-      uses: actions/setup-python@v7
-      with:
-        python-version: 3.14
+    - name: Start iOS Simulator UI
+      run: |
+        defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false
+        open -Fn "$(xcode-select --print-path)/Applications/Simulator.app"
 
-    - name: Set up uv
-      uses: astral-sh/setup-uv@v10.3.3
-      with:
-        enable-cache: true
-        python-version: '3.14'
+    - name: Warm up iOS simulator
+      # The first app launch on a fresh simulator is often slow enough to make
+      # the first Appium session time out, so do it once before the tests.
+      run: |
+        xcrun simctl install booted "$PREBUILT_WDA_PATH"
+        xcrun simctl launch booted com.apple.mobilesafari
+        sleep 15
+        xcrun simctl terminate booted com.apple.mobilesafari || true
+
+    - name: Warm up WebDriverAgent
+      # Start and quit one Appium session so WDA's first launch happens here, not in a test.
+      timeout-minutes: 12
+      continue-on-error: true
+      run: uv run python -m test.functional.ios.helper.warm_up
+      env:
+        LOCAL_PREBUILT_WDA: ${{ env.PREBUILT_WDA_PATH }}
 
     - name: Run Tests
       run: |
@@ -187,13 +214,14 @@ jobs:
     - name: Set up Python 3.14
       uses: actions/setup-python@v7
       with:
-        python-version: 3.14
+        python-version: '3.14'
 
     - name: Set up uv
       uses: astral-sh/setup-uv@v10.3.3
       with:
         enable-cache: true
         python-version: '3.14'
+        version: ${{ env.UV_VERSION }}
 
     - name: Run tests
       uses: reactivecircus/android-emulator-runner@v2
@@ -269,7 +297,17 @@ jobs:
     - name: Set up Python 3.14
       uses: actions/setup-python@v7
       with:
-        python-version: 3.14
+        python-version: '3.14'
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@v10.3.3
+      with:
+        enable-cache: true
+        python-version: '3.14'
+        version: ${{ env.UV_VERSION }}
+
+    - name: Install Python dependencies
+      run: uv sync
 
     - name: Install Node.js
       uses: actions/setup-node@v7
@@ -305,12 +343,6 @@ jobs:
         server_args: '--relaxed-security'
         log_file: appium-${{ matrix.e2e-tests }}.log
 
-    - name: Set up uv
-      uses: astral-sh/setup-uv@v10.3.3
-      with:
-        enable-cache: true
-        python-version: '3.14'
-
     - name: Run Android tests
       if: matrix.e2e-tests == 'flutter-android'
       uses: reactivecircus/android-emulator-runner@v2
@@ -322,14 +354,10 @@ jobs:
         disable-spellchecker: true
         disable-animations: true
 
-    - name: Start iOS Simulator UI
-      if: matrix.e2e-tests == 'flutter-ios'
-      run: |
-        defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false
-        open -Fn "$(xcode-select --print-path)/Applications/Simulator.app"
-
-    - uses: futureware-tech/simulator-action@v6
+    - name: Prepare iOS simulator
+      uses: futureware-tech/simulator-action@v6
       if: matrix.e2e-tests == 'flutter-ios'
+      timeout-minutes: 20
       with:
         # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md
         model: ${{ env.IPHONE_MODEL }}
@@ -341,6 +369,31 @@ jobs:
       if: matrix.e2e-tests == 'flutter-ios'
       run: xcrun --sdk iphonesimulator --show-sdk-version
 
+    - name: Start iOS Simulator UI
+      if: matrix.e2e-tests == 'flutter-ios'
+      run: |
+        defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false
+        open -Fn "$(xcode-select --print-path)/Applications/Simulator.app"
+
+    - name: Warm up iOS simulator
+      if: matrix.e2e-tests == 'flutter-ios'
+      # The first app launch on a fresh simulator is often slow enough to make
+      # the first Appium session time out, so do it once before the tests.
+      run: |
+        xcrun simctl install booted "$PREBUILT_WDA_PATH"
+        xcrun simctl launch booted com.apple.mobilesafari
+        sleep 15
+        xcrun simctl terminate booted com.apple.mobilesafari || true
+
+    - name: Warm up WebDriverAgent
+      if: matrix.e2e-tests == 'flutter-ios'
+      # Start and quit one Appium session so WDA's first launch happens here, not in a test.
+      timeout-minutes: 12
+      continue-on-error: true
+      run: uv run python -m test.functional.ios.helper.warm_up
+      env:
+        LOCAL_PREBUILT_WDA: ${{ env.PREBUILT_WDA_PATH }}
+
     - name: Run IOS tests
       if: matrix.e2e-tests == 'flutter-ios'
       run: |

test/functional/flutter_integration/conftest.py

@@ -20,6 +20,7 @@
 from appium import webdriver
 from appium.webdriver.client_config import AppiumClientConfig
 from appium.webdriver.extensions.flutter_integration.flutter_commands import FlutterCommand
+from test.functional.test_helper import create_session_with_wda_retry
 from test.helpers.constants import SERVER_URL_BASE
 
 from .helper.options import make_options
@@ -36,7 +37,7 @@ def driver() -> Generator['WebDriver', None, None]:
     client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE)
     client_config.timeout = 600
 
-    driver = webdriver.Remote(options=options, client_config=client_config)
+    driver = create_session_with_wda_retry(lambda: webdriver.Remote(options=options, client_config=client_config))
 
     yield driver
 

test/functional/ios/helper/options.py

@@ -28,6 +28,8 @@ def make_options(app: str | None = None) -> XCUITestOptions:
     options.allow_touch_id_enroll = True
     options.wda_local_port = get_wda_port()
     options.simple_is_visible_check = True
+    # CI simulators are often slow to launch WDA right after boot.
+    options.wda_launch_timeout = 240000
 
     if app is not None:
         options.app = app

test/functional/ios/helper/warm_up.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Start and quit one Appium session so WebDriverAgent's first, slowest launch happens before the tests.
+
+Used by CI only. Failures are reported but don't fail the step: the tests retry on their own.
+It makes a single attempt, so that it always finishes within the step's 12-minute timeout.
+"""
+
+import sys
+
+from appium import webdriver
+from appium.webdriver.client_config import AppiumClientConfig
+from test.helpers.constants import SERVER_URL_BASE
+
+from .options import make_options
+
+
+def main() -> int:
+    client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE)
+    # Match the tests' timeout: giving up earlier would leave the server still creating this
+    # session, so the first test's session request would queue behind it.
+    client_config.timeout = 600
+    try:
+        driver = webdriver.Remote(options=make_options(), client_config=client_config)
+    except Exception as e:  # noqa: BLE001
+        print(f'Warm-up session failed: {e}', file=sys.stderr)
+        return 0
+    try:
+        driver.quit()
+    except Exception as e:  # noqa: BLE001
+        print(f'Warm-up session started but failed to quit: {e}', file=sys.stderr)
+        return 0
+    print('Warm-up session started and quit successfully')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

test/functional/ios/safari_tests.py

@@ -20,6 +20,7 @@
 
 from appium import webdriver
 from appium.webdriver.client_config import AppiumClientConfig
+from test.functional.test_helper import create_session_with_wda_retry
 from test.helpers.constants import SERVER_URL_BASE
 
 from .helper.options import make_options
@@ -28,6 +29,7 @@
     from appium.webdriver.webdriver import WebDriver
 
 
+# One session per test: if a session fails to start, the next test still gets a fresh attempt.
 @pytest.fixture
 def driver() -> Generator['WebDriver', None, None]:
     """Create and configure Safari driver for testing."""
@@ -39,7 +41,7 @@ def driver() -> Generator['WebDriver', None, None]:
 
     client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE)
     client_config.timeout = 600
-    driver = webdriver.Remote(options=options, client_config=client_config)
+    driver = create_session_with_wda_retry(lambda: webdriver.Remote(options=options, client_config=client_config))
 
     # Fresh iOS 17.4 simulator may not show up the webview context with "safari"
     # after a fresh simlator instance creation.

test/functional/test_helper.py

@@ -6,6 +6,7 @@
 from time import sleep
 from typing import TYPE_CHECKING, Any
 
+from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
@@ -141,3 +142,23 @@ def get_wda_port() -> int:
     """
     worker_info = get_worker_info()
     return 8100 + (worker_info.worker_number or 0)
+
+
+# Errors the XCUITest driver raises after it has already cleaned up a failed WDA startup,
+# so creating a new session afterwards is safe.
+_WDA_STARTUP_ERRORS = ('Unable to launch WebDriverAgent', 'Unable to start WebDriverAgent session')
+
+
+def create_session_with_wda_retry(create: Callable[[], 'WebDriver'], interval_sec: float = 10) -> 'WebDriver':
+    """Create a session, retrying once when WebDriverAgent fails to start.
+
+    The XCUITest driver does not retry WDA startup itself when `usePreinstalledWDA` is set,
+    and the first launch on a freshly booted CI simulator often fails.
+    """
+    try:
+        return create()
+    except WebDriverException as e:
+        if not any(msg in str(e) for msg in _WDA_STARTUP_ERRORS):
+            raise
+    sleep(interval_sec)
+    return create()