ci: Run tests for both snapshot and stable selenium versions (#2394)

Author: mykola-mokhnach

.github/workflows/ci.yml

@@ -23,10 +23,10 @@ env:
   ANDROID_SDK_VERSION: "28"
   ANDROID_EMU_NAME: test
   ANDROID_EMU_TARGET: default
-  # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md
-  XCODE_VERSION: "15.4"
-  IOS_DEVICE_NAME: iPhone 15
-  IOS_PLATFORM_VERSION: "17.5"
+  # https://github.com/actions/runner-images/blob/main/images/macos (macos-15 for Xcode 16 / iOS 18.5)
+  XCODE_VERSION: "16.4"
+  IOS_DEVICE_NAME: iPhone 16
+  IOS_PLATFORM_VERSION: "18.5"
   FLUTTER_ANDROID_APP: "https://github.com/AppiumTestDistribution/appium-flutter-server/releases/latest/download/app-debug.apk"
   FLUTTER_IOS_APP: "https://github.com/AppiumTestDistribution/appium-flutter-server/releases/latest/download/ios.zip"
   PREBUILT_WDA_PATH: ${{ github.workspace }}/wda/WebDriverAgentRunner-Runner.app
@@ -38,33 +38,49 @@ jobs:
       matrix:
         include:
         - java: 17
-          # Need to use specific (not `-latest`) version of macOS to be sure the required version of Xcode/simulator is available
-          platform: macos-14
+          platform: macos-15
           e2e-tests: ios
+          selenium: stable
         - java: 17
-          # Need to use specific (not `-latest`) version of macOS to be sure the required version of Xcode/simulator is available
-          platform: macos-14
+          platform: macos-15
           e2e-tests: flutter-ios
+          selenium: stable
         - java: 17
           platform: ubuntu-latest
           e2e-tests: android
+          selenium: stable
         - java: 17
           platform: ubuntu-latest
           e2e-tests: flutter-android
+          selenium: stable
         - java: 21
           platform: ubuntu-latest
+          selenium: stable
         - java: 25
           platform: ubuntu-latest
+          selenium: stable
+        # Snapshot: compile-only (no E2E) to verify compatibility with latest Selenium
+        - java: 17
+          platform: ubuntu-latest
+          selenium: snapshot
+        - java: 21
+          platform: ubuntu-latest
+          selenium: snapshot
+        - java: 25
+          platform: ubuntu-latest
+          selenium: snapshot
       fail-fast: false
 
     runs-on: ${{ matrix.platform }}
 
-    name: JDK ${{ matrix.java }} - ${{ matrix.platform }} ${{ matrix.e2e-tests }}
+    continue-on-error: ${{ matrix.selenium == 'snapshot' }}
+
+    name: JDK ${{ matrix.java }} - ${{ matrix.platform }} ${{ matrix.e2e-tests || 'build' }} (${{ matrix.selenium }})
     steps:
     - uses: actions/checkout@v6
 
     - name: Enable KVM group perms
-      if: matrix.e2e-tests == 'android' || matrix.e2e-tests == 'flutter-android'
+      if: (matrix.e2e-tests == 'android' || matrix.e2e-tests == 'flutter-android') && matrix.selenium == 'stable'
       run: |
         echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
         sudo udevadm control --reload-rules
@@ -79,80 +95,84 @@ jobs:
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@v5
 
-    - name: Build with Gradle against Selenium nightly build
+    - name: Resolve Selenium snapshot version
+      if: matrix.selenium == 'snapshot'
       run: |
         latest_snapshot=$(curl -sf https://raw.githubusercontent.com/SeleniumHQ/selenium/refs/heads/trunk/java/version.bzl | grep 'SE_VERSION' | sed 's/.*"\(.*\)".*/\1/')
         echo ">>> $latest_snapshot"
         echo "latest_snapshot=$latest_snapshot" >> "$GITHUB_ENV"
-        ./gradlew clean build -PisCI -Pselenium.version=$latest_snapshot
 
-    - name: Build with Gradle against stable Selenium version
-      run: |
-        ./gradlew clean build -PisCI
+    - name: Build with Gradle (stable Selenium)
+      if: matrix.selenium == 'stable'
+      run: ./gradlew clean build
+
+    - name: Build with Gradle (snapshot Selenium)
+      if: matrix.selenium == 'snapshot'
+      run: ./gradlew clean build -Pselenium.version=$latest_snapshot
 
     - name: Install Node.js
-      if: ${{ matrix.e2e-tests }}
+      if: ${{ matrix.e2e-tests && matrix.selenium == 'stable' }}
       uses: actions/setup-node@v6
       with:
         node-version: 'lts/*'
 
     - name: Install Appium
-      if: ${{ matrix.e2e-tests }}
+      if: ${{ matrix.e2e-tests && matrix.selenium == 'stable' }}
       run: npm install --location=global appium
 
     - name: Install UIA2 driver
-      if: matrix.e2e-tests == 'android' || matrix.e2e-tests == 'flutter-android'
+      if: (matrix.e2e-tests == 'android' || matrix.e2e-tests == 'flutter-android') && matrix.selenium == 'stable'
       run: appium driver install uiautomator2
 
     - name: Install Flutter Integration driver
-      if: matrix.e2e-tests == 'flutter-android' || matrix.e2e-tests == 'flutter-ios'
+      if: (matrix.e2e-tests == 'flutter-android' || matrix.e2e-tests == 'flutter-ios') && matrix.selenium == 'stable'
       run: appium driver install appium-flutter-integration-driver --source npm
 
     - name: Run Android E2E tests
-      if: matrix.e2e-tests == 'android'
+      if: matrix.e2e-tests == 'android' && matrix.selenium == 'stable'
       uses: reactivecircus/android-emulator-runner@v2
       with:
-        script: ./gradlew e2eAndroidTest -PisCI -Pselenium.version=$latest_snapshot
+        script: ./gradlew e2eAndroidTest
         api-level: ${{ env.ANDROID_SDK_VERSION }}
         avd-name: ${{ env.ANDROID_EMU_NAME }}
         disable-spellchecker: true
         disable-animations: true
         target: ${{ env.ANDROID_EMU_TARGET }}
 
     - name: Run Flutter Android E2E tests
-      if: matrix.e2e-tests == 'flutter-android'
+      if: matrix.e2e-tests == 'flutter-android' && matrix.selenium == 'stable'
       uses: reactivecircus/android-emulator-runner@v2
       with:
-        script: ./gradlew e2eFlutterTest -Pplatform="android" -Pselenium.version=$latest_snapshot -PisCI -PflutterApp=${{ env.FLUTTER_ANDROID_APP }}
+        script: ./gradlew e2eFlutterTest -Pplatform="android" -PflutterApp=${{ env.FLUTTER_ANDROID_APP }}
         api-level: ${{ env.ANDROID_SDK_VERSION }}
         avd-name: ${{ env.ANDROID_EMU_NAME }}
         disable-spellchecker: true
         disable-animations: true
         target: ${{ env.ANDROID_EMU_TARGET }}
 
     - name: Select Xcode
-      if: matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios'
+      if: (matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios') && matrix.selenium == 'stable'
       uses: maxim-lobanov/setup-xcode@v1
       with:
         xcode-version: "${{ env.XCODE_VERSION }}"
     - name: Prepare iOS simulator
-      if: matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios'
+      if: (matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios') && matrix.selenium == 'stable'
       uses: futureware-tech/simulator-action@v4
       with:
         model: "${{ env.IOS_DEVICE_NAME }}"
         os_version: "${{ env.IOS_PLATFORM_VERSION }}"
         wait_for_boot: true
         shutdown_after_job: false
     - name: Install XCUITest driver
-      if: matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios'
+      if: (matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios') && matrix.selenium == 'stable'
       run: appium driver install xcuitest
     - name: Download prebuilt WDA
-      if: matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios'
+      if: (matrix.e2e-tests == 'ios' || matrix.e2e-tests == 'flutter-ios') && matrix.selenium == 'stable'
       run: appium driver run xcuitest download-wda-sim  --platform=ios --outdir=$(dirname "$PREBUILT_WDA_PATH")
     - name: Run iOS E2E tests
-      if: matrix.e2e-tests == 'ios'
-      run: ./gradlew e2eIosTest -PisCI -Pselenium.version=$latest_snapshot
+      if: matrix.e2e-tests == 'ios' && matrix.selenium == 'stable'
+      run: ./gradlew e2eIosTest
 
     - name: Run Flutter iOS E2E tests
-      if: matrix.e2e-tests == 'flutter-ios'
-      run: ./gradlew e2eFlutterTest -Pplatform="ios" -Pselenium.version=$latest_snapshot -PisCI -PflutterApp=${{ env.FLUTTER_IOS_APP }}
+      if: matrix.e2e-tests == 'flutter-ios' && matrix.selenium == 'stable'
+      run: ./gradlew e2eFlutterTest -Pplatform="ios" -PflutterApp=${{ env.FLUTTER_IOS_APP }}

build.gradle

@@ -24,7 +24,9 @@ version = appiumClientVersion
 repositories {
     mavenCentral()
 
-    if (project.hasProperty("isCI")) {
+    // Only add Selenium snapshot repo when resolving a SNAPSHOT version.
+    // Release versions (e.g. from gradle.properties) resolve from Maven Central only.
+    if (project.property('selenium.version').toString().contains('SNAPSHOT')) {
         maven {
             name = 'Central Portal Snapshots'
             url = 'https://central.sonatype.com/repository/maven-snapshots/'
@@ -49,7 +51,8 @@ dependencies {
     compileOnly 'org.projectlombok:lombok:1.18.42'
     annotationProcessor 'org.projectlombok:lombok:1.18.42'
 
-    if (project.hasProperty("isCI")) {
+    // Pin exact Selenium version when using a SNAPSHOT (needs snapshot repo); otherwise use version range.
+    if (seleniumVersion.toString().contains('SNAPSHOT')) {
         api "org.seleniumhq.selenium:selenium-api:${seleniumVersion}"
         api "org.seleniumhq.selenium:selenium-remote-driver:${seleniumVersion}"
         api "org.seleniumhq.selenium:selenium-support:${seleniumVersion}"