Issue 3289 semaphore ci f1

.semaphore/semaphore.yml

@@ -2,7 +2,7 @@ version: v1.0
 name: FlowCrypt Android App
 agent:
   machine:
-    type: e2-standard-2
+    type: f1-standard-2
     os_image: ubuntu2404
 execution_time_limit:
   minutes: 75
@@ -25,45 +25,52 @@ global_job_config:
       value: /home/semaphore/Android/Sdk
   prologue:
     commands:
-      - sudo apt-get update
       # use JAVA 21 by default
       - sem-version java 21
       # general settings
       - export PATH=${ANDROID_HOME}/emulator:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/platform-tools:${PATH}
       - sudo rm -rf ~/.rbenv ~/.phpbrew
       - checkout
-      # fix DNS to ping *.localhost and *.flowcrypt.test
-      - ./script/ci-setup-DNS.sh
       # init environment variables
       - export GRADLE_SUM=$(checksum build.gradle.kts)-$(checksum FlowCrypt/build.gradle.kts)
       - export SDK_SUM=$(checksum ./script/ci-install-android-sdk.sh)
       - export APP_GRADLE_CACHE=project-gradle-cache-$GRADLE_SUM # project .gradle cache, per Gradle config hash
-      - export ANDROID_SDK_CACHE=android-sdk-$SDK_SUM # Android SDK cache, per SDK installer hash
+      - if [ "${INSTALL_EMULATOR:-1}" = "0" ]; then export ANDROID_SDK_CACHE=android-sdk-slim-$SDK_SUM; cache restore $ANDROID_SDK_CACHE; else export ANDROID_SDK_CACHE=android-sdk-full-$SDK_SUM; cache restore $ANDROID_SDK_CACHE,android-sdk-d839d0895100d9722fb794bbb9677f92,android-sdk-$SDK_SUM; fi
       # restore app caches
       - cache restore $APP_GRADLE_CACHE
       # restore global caches
-      - cache restore $ANDROID_SDK_CACHE
       - cache restore gradle-wrapper
       - cache restore gradle-cache-$GRADLE_SUM
       # Install Android dependencies if needed
       - ./script/ci-install-android-sdk.sh
 blocks:
   - name: 'Build'
+    dependencies: []
     execution_time_limit:
       minutes: 20
     task:
+      env_vars:
+        - name: INSTALL_EMULATOR
+          value: "0"
+        - name: INSTALL_KVM_DEPS
+          value: "0"
+        - name: RUN_KVM_CHECK
+          value: "0"
       jobs:
         - name: 'Build Project'
           commands:
             # print debug info
             - cat /proc/cpuinfo
             # print Java version
             - java -version
-            # compile project
-            - ./gradlew --console=plain --no-daemon --build-cache assembleConsumerUiTests
+            # compile project and test artifacts to prime build cache
+            - ./gradlew --console=plain --no-daemon --build-cache assembleConsumerUiTests packageConsumerUiTestsAndroidTest packageEnterpriseUiTestsAndroidTest
       epilogue:
         on_pass:
           commands:
+            # push workflow artifacts for test jobs
+            - ./script/ci-push-build-artifacts.sh
+
             # store app cache
             - echo "Store the app cache"
             - cache has_key $APP_GRADLE_CACHE || cache store $APP_GRADLE_CACHE .gradle
@@ -75,8 +82,16 @@ blocks:
             - cache has_key gradle-wrapper || cache store gradle-wrapper ~/.gradle/wrapper
             - cache has_key gradle-cache-$GRADLE_SUM || cache store gradle-cache-$GRADLE_SUM ~/.gradle/caches
 
-  - name: 'Testing'
+  - name: 'Code Quality & Unit Tests'
+    dependencies: []
     task:
+      env_vars:
+        - name: INSTALL_EMULATOR
+          value: "0"
+        - name: INSTALL_KVM_DEPS
+          value: "0"
+        - name: RUN_KVM_CHECK
+          value: "0"
       jobs:
         - name: 'Lint(structural quality)'
           execution_time_limit:
@@ -91,20 +106,37 @@ blocks:
           commands:
             # run JUnit tests
             - ./script/ci-junit-tests.sh
+      epilogue:
+        always:
+          commands:
+            - ./script/ci-publish-test-results.sh
+        on_fail:
+          commands:
+            - ./script/ci-get-and-publish-debug-info-as-artifact.sh
 
+  - name: 'Instrumentation Tests'
+    dependencies: [ "Build" ]
+    task:
+      env_vars:
+        - name: INSTALL_EMULATOR
+          value: "1"
+      prologue:
+        commands:
+          - ./script/ci-pull-build-artifacts.sh
+      jobs:
         - name: 'Instrumentation tests(No email server)'
           execution_time_limit:
             minutes: 60
           matrix:
             - env_var: EMULATOR
-              values: [ "0", "1", "2", "3" ]
+              values: [ "0", "1", "2", "3", "4", "5" ]
           commands:
             # Setup and run an emulator
             - ./script/ci-setup-and-run-emulator.sh
             # wait until ready
             - ./script/ci-wait-for-emulator.sh
             # Run instrumentation tests with live progress and a final summary
-            - ./script/ci-stream-instrumentation-test-progress.sh ./script/ci-instrumentation-tests-without-mailserver.sh 4 $EMULATOR
+            - ./script/ci-stream-instrumentation-test-progress.sh ./script/ci-instrumentation-tests-without-mailserver.sh 6 $EMULATOR
 
         - name: 'Instrumentation tests(with email server)'
           execution_time_limit:
@@ -147,6 +179,9 @@ blocks:
             # export tests result if needed
             - ./script/ci-publish-test-results.sh
 
+            # store full android sdk cache if not present
+            - cache has_key $ANDROID_SDK_CACHE || cache store $ANDROID_SDK_CACHE $ANDROID_HOME || true
+
             # do additional actions if needed
             - ./script/ci-after-success-actions-for-instrumentation-tests.sh
 
@@ -168,3 +203,5 @@ after_pipeline:
           - artifact pull workflow "test-results/${SEMAPHORE_PIPELINE_ID}.json" -d pipeline-test-results.json
           - ./script/ci-generate-test-report.sh pipeline-test-results.json test-report.md
           - artifact push workflow -f -d .semaphore/REPORT.md test-report.md
+          - artifact yank workflow build-cache.tar || true
+          - artifact yank workflow apks.tar || true

gradle.properties

@@ -19,6 +19,6 @@
 # org.gradle.parallel=true
 #more details here https://youtu.be/7ll-rkLCtyk?t=1219
 org.gradle.caching=true
-org.gradle.jvmargs=-Xmx2g
+org.gradle.jvmargs=-Xmx3072m -XX:+UseParallelGC
 android.useAndroidX=true
 android.enableJetifier=false

gradle/wrapper/gradle-wrapper.properties

@@ -2,6 +2,8 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.0-bin.zip
 networkTimeout=10000
+retries=0
+retryBackOffMs=500
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

script/ci-after-success-actions-for-instrumentation-tests.sh

@@ -22,8 +22,11 @@ if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then
   echo "Store screenshots"
   if adb shell test -d /sdcard/Pictures; then
     rm -rf Pictures
-    adb pull "/sdcard/Pictures" Pictures
-    artifact push job Pictures
+    if adb pull "/sdcard/Pictures" Pictures 2>/dev/null && [[ -d Pictures && "$(ls -A Pictures 2>/dev/null)" ]]; then
+      artifact push job Pictures || true
+    else
+      echo "No screenshots found in /sdcard/Pictures, skipping"
+    fi
   else
     echo "No /sdcard/Pictures directory found, skipping"
   fi

script/ci-get-and-publish-debug-info-as-artifact.sh

@@ -28,22 +28,26 @@ else
 fi
 
 if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then
-  # store full logcat log
-  echo "Collect logcat logs as logcat.txt.gz for $SEMAPHORE_JOB_NAME"
-  if adb logcat -d | gzip > "$HOME/logcat.txt.gz"; then
-    artifact push job "$HOME/logcat.txt.gz"
-  else
-    echo "Could not collect logcat within $ADB_COMMAND_TIMEOUT, skipping"
-  fi
+  if adb get-state 2>/dev/null | grep -q "device"; then
+    # store full logcat log
+    echo "Collect logcat logs as logcat.txt.gz for $SEMAPHORE_JOB_NAME"
+    if adb logcat -d | gzip > "$HOME/logcat.txt.gz"; then
+      artifact push job "$HOME/logcat.txt.gz"
+    else
+      echo "Could not collect logcat within $ADB_COMMAND_TIMEOUT, skipping"
+    fi
 
-  echo "Store the device's screenshot for $SEMAPHORE_JOB_NAME"
-  if adb shell screencap -p /sdcard/screencap.png; then
-    if adb pull "/sdcard/screencap.png"; then
-      artifact push job screencap.png
+    echo "Store the device's screenshot for $SEMAPHORE_JOB_NAME"
+    if adb shell screencap -p /sdcard/screencap.png; then
+      if adb pull "/sdcard/screencap.png"; then
+        artifact push job screencap.png
+      else
+        echo "Could not pull screencap.png"
+      fi
     else
-      echo "Could not pull screencap.png"
+      echo "Could not create screencap.png"
     fi
   else
-    echo "Could not create screencap.png"
+    echo "No connected device found for $SEMAPHORE_JOB_NAME, skipping logcat and screenshot."
   fi
 fi

script/ci-install-android-sdk.sh

@@ -22,12 +22,18 @@ fi
 # -----------------------------
 export ANDROID_HOME="${ANDROID_HOME:-$HOME/Android/Sdk}"
 export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}"
+INSTALL_EMULATOR="${INSTALL_EMULATOR:-1}"
 INSTALL_KVM_DEPS="${INSTALL_KVM_DEPS:-1}"
 RUN_KVM_CHECK="${RUN_KVM_CHECK:-1}"
 ANDROID_PLATFORM="${ANDROID_PLATFORM:-android-36}"
 ANDROID_SYSTEM_IMAGE="${ANDROID_SYSTEM_IMAGE:-system-images;android-36;google_apis;x86_64}"
 ANDROID_BUILD_TOOLS="${ANDROID_BUILD_TOOLS:-}"
 
+if [[ "$INSTALL_EMULATOR" == "0" ]]; then
+  INSTALL_KVM_DEPS=0
+  RUN_KVM_CHECK=0
+fi
+
 # -----------------------------
 # Pin cmdline-tools archive here
 # -----------------------------
@@ -92,7 +98,10 @@ check_cmdline_tools_latest_or_fail
 # KVM deps (as in your script)
 # -----------------------------
 if [[ "$INSTALL_KVM_DEPS" == "1" ]]; then
-  sudo apt-get -qq install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils > /dev/null
+  if ! dpkg -s qemu-kvm >/dev/null 2>&1; then
+    sudo apt-get update -qq
+    sudo apt-get -qq install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils > /dev/null
+  fi
 fi
 if [[ "$RUN_KVM_CHECK" == "1" ]]; then
   sudo kvm-ok
@@ -102,7 +111,15 @@ fi
 # Install SDK if ~/Android doesn't exist (as in your script)
 # -----------------------------
 if [[ -x "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" ]]; then
-  echo "$ANDROID_HOME already exists, skipping installation"
+  if [[ "$INSTALL_EMULATOR" != "0" ]] && [[ ! -d "$ANDROID_HOME/system-images" ]]; then
+    echo "$ANDROID_HOME exists, but emulator components are missing. Installing emulator and system images..."
+    ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "emulator" | grep -v = || true)
+    if [[ -n "$ANDROID_SYSTEM_IMAGE" ]]; then
+      (echo "y" | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "${ANDROID_SYSTEM_IMAGE}" > /dev/null | grep -v = || true)
+    fi
+  else
+    echo "$ANDROID_HOME already exists, skipping installation"
+  fi
 else
   echo "$ANDROID_HOME does not exist, installing"
   mkdir -p "$ANDROID_HOME"
@@ -133,11 +150,15 @@ else
   (echo "yes" | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses > /dev/null | grep -v = || true)
   ( sleep 5; echo "y" ) | ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "platforms;${ANDROID_PLATFORM}" > /dev/null | grep -v = || true)
   ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "platform-tools" | grep -v = || true)
-  ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "emulator" | grep -v = || true)
+  if [[ "$INSTALL_EMULATOR" != "0" ]]; then
+    ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "emulator" | grep -v = || true)
+    if [[ -n "$ANDROID_SYSTEM_IMAGE" ]]; then
+      (echo "y" | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "${ANDROID_SYSTEM_IMAGE}" > /dev/null | grep -v = || true)
+    fi
+  fi
   if [[ -n "$ANDROID_BUILD_TOOLS" ]]; then
     ("${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "build-tools;${ANDROID_BUILD_TOOLS}" | grep -v = || true)
   fi
-  (echo "y" | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "${ANDROID_SYSTEM_IMAGE}" > /dev/null | grep -v = || true)
 fi
 
 # Uncomment this for debug

script/ci-pull-build-artifacts.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+#
+# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com
+# Contributors: denbond7
+#
+
+set -euo pipefail
+
+echo "==> Pulling pre-built artifacts from Semaphore workflow storage..."
+
+mkdir -p "$HOME/.gradle/caches"
+if artifact pull workflow build-cache.tar -d /tmp/build-cache.tar 2>/dev/null; then
+  echo "Extracting Gradle build cache into ~/.gradle/caches/..."
+  tar -xf /tmp/build-cache.tar -C "$HOME/.gradle/caches"
+  rm -f /tmp/build-cache.tar
+  echo "Build cache restored successfully."
+else
+  echo "Notice: Workflow build-cache artifact not available; will compile incrementally."
+fi
+
+mkdir -p "FlowCrypt/build/outputs"
+if artifact pull workflow apks.tar -d /tmp/apks.tar 2>/dev/null; then
+  echo "Extracting pre-built APKs into FlowCrypt/build/outputs/..."
+  tar -xf /tmp/apks.tar -C "FlowCrypt/build/outputs"
+  rm -f /tmp/apks.tar
+  echo "APKs restored successfully."
+else
+  echo "Notice: Workflow APKs artifact not available; will build as needed."
+fi
+
+echo "==> Build artifacts preparation finished."

script/ci-push-build-artifacts.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#
+# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com
+# Contributors: denbond7
+#
+
+set -euo pipefail
+
+echo "==> Pushing build artifacts to Semaphore workflow storage..."
+
+if [[ -d "$HOME/.gradle/caches/build-cache-1" ]]; then
+  echo "Packaging Gradle build cache (uncompressed tar for speed)..."
+  tar -cf /tmp/build-cache.tar -C "$HOME/.gradle/caches" build-cache-1
+  echo "Uploading build-cache.tar ($(du -h /tmp/build-cache.tar | cut -f1))..."
+  artifact push workflow /tmp/build-cache.tar -d build-cache.tar
+  rm -f /tmp/build-cache.tar
+  echo "Build cache pushed successfully."
+else
+  echo "Warning: ~/.gradle/caches/build-cache-1 not found, skipping build cache push."
+fi
+
+if [[ -d "FlowCrypt/build/outputs/apk" ]]; then
+  echo "Packaging built APKs..."
+  tar -cf /tmp/apks.tar -C "FlowCrypt/build/outputs" apk
+  echo "Uploading apks.tar ($(du -h /tmp/apks.tar | cut -f1))..."
+  artifact push workflow /tmp/apks.tar -d apks.tar
+  rm -f /tmp/apks.tar
+  echo "APKs pushed successfully."
+else
+  echo "Warning: FlowCrypt/build/outputs/apk not found, skipping APKs push."
+fi
+
+echo "==> Build artifacts published to workflow storage."

script/ci-setup-DNS.sh

@@ -7,8 +7,15 @@
 
 set -euo pipefail
 
-echo "Installing DNS tools..."
-sudo apt install -y dnsmasq dnsutils
+if systemctl is-active --quiet dnsmasq && [[ -f /etc/dnsmasq.d/flowcrypt.conf ]]; then
+  echo "dnsmasq is already configured and running."
+  exit 0
+fi
+
+if ! dpkg -s dnsmasq >/dev/null 2>&1 || ! command -v dig >/dev/null 2>&1; then
+  echo "Installing DNS tools..."
+  sudo apt-get install -y --no-install-recommends dnsmasq dnsutils
+fi
 
 echo "Configuring dnsmasq..."
 sudo tee /etc/dnsmasq.d/flowcrypt.conf >/dev/null <<'EOF'

script/ci-setup-and-run-emulator.sh

@@ -7,6 +7,7 @@
 
 set -euo pipefail
 
+./script/ci-setup-DNS.sh
 AVD_RAM_SIZE=2048 ./script/create-avd.sh
 EMULATOR_GPU_MODE=auto \
 EMULATOR_READ_ONLY=1 \