Use Firebase Test Lab (PoC)

.github/workflows/integration-tests-ui-firebase-test-lab.yml

@@ -0,0 +1,75 @@
+name: 'Integration Tests (Firebase Test Lab)'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Ui tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    # we copy the secret to the env variable in order to access it in the workflow
+    env:
+      # The contents of service_account.json encoded using base64, to avoid any shell escaping issues
+      # base64 -i <file>.json
+      FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64 }}
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: 'Set up Java: 17'
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@707359876a764dbcdb9da0b0ed08291818310c3d # pin@v3
+        with:
+          gradle-home-cache-cleanup: true
+
+      - name: Make assembleUiTests
+        run: make assembleUiTests
+
+      - name: Install gcloud
+        uses: 'google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a' # pin 2.1.2
+        with:
+          version: '>= 363.0.0'
+
+      - name: Configure gcloud
+        run: |
+          echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | gcloud auth activate-service-account --key-file=-
+          project_id=$(echo "$FIREBASE_TEST_LAB_SERVICE_ACCOUNT_JSON_BASE64" | base64 -d | jq -r ".project_id")
+          gcloud config set project $project_id
+
+      - name: Firebase Test Lab
+        run: |
+          app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk
+          test_app_apk=./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk
+          attempts=4
+          type=instrumentation
+
+          gcloud firebase test android run \
+            --app=$app_apk --test=$test_app_apk \
+            --type=$type \
+            --num-flaky-test-attempts=$attempts \
+            --device=^:^model=MediumPhone.arm:version=34 \
+            --device=^:^model=bluejay:version=32 \
+            --device=^:^model=a14xm:version=34
+
+          status=$?
+          if [ $status -ne 0 ]; then
+              echo "Firebase Test Lab failed with status $status"
+              exit 1
+          fi

sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts

@@ -125,6 +125,7 @@ dependencies {
     androidTestImplementation(Config.TestLibs.mockWebserver)
     androidTestImplementation(Config.TestLibs.androidxJunit)
     androidTestImplementation(Config.TestLibs.leakCanaryInstrumentation)
+    androidTestImplementation("com.google.firebase:testlab-instr-lib:0.2")
     androidTestUtil(Config.TestLibs.androidxTestOrchestrator)
 }
 

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/AndroidManifest.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+  <!-- WRITE_EXTERNAL_STORAGE is not needed on Android 10 (API level 29) or higher. -->
+
+  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+  <uses-permission android:name="android.permission.INTERNET"/>
+
+  <instrumentation
+    android:name="androidx.test.runner.AndroidJUnitRunner"
+    android:targetPackage="io.sentry.uitest.android">
+
+    <meta-data
+      android:name="screenCaptureProcessors"
+      android:value="com.google.firebase.testlab.screenshot.FirebaseScreenCaptureProcessor" />
+  </instrumentation>
+
+</manifest>

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/AutomaticSpansTest.kt

@@ -90,7 +90,7 @@ class AutomaticSpansTest : BaseUiTest() {
                 // AGP matrix tests have no frames
                 Assume.assumeTrue(totalFrames > 0)
                 assertNotEquals(totalFrames, 0)
-                assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.")
+                assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.")
             }
             assertNoOtherEnvelopes()
         }
@@ -126,7 +126,7 @@ class AutomaticSpansTest : BaseUiTest() {
                 // AGP matrix tests have no frames
                 Assume.assumeTrue(totalFrames > 0)
                 assertNotEquals(totalFrames, 0)
-                assertTrue(totalFrames > slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.")
+                assertTrue(totalFrames >= slowFrames + frozenFrames, "Expected total frames ($totalFrames) to be higher than the sum of slow ($slowFrames) and frozen ($frozenFrames) frames.")
             }
             assertNoOtherEnvelopes()
         }