Fix e2e and mockapi tests failing

.github/workflows/android-app.yml

@@ -313,7 +313,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: relay-list
-          path: android/app/build/extraAssets/relays.json
+          path: android/app/build/extraAssets
 
       - name: Build app
         uses: burrunan/gradle-cache-action@v1
@@ -416,10 +416,9 @@ jobs:
           - test-type: app
             path: android/app/build/outputs/apk
             test-repeat: 1
-          # Disabled (test-repeat='0') due to flakiness unless overridden by input.
           - test-type: mockapi
             path: android/test/mockapi/build/outputs/apk
-            test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 0 }}
+            test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 1 }}
     steps:
       - name: Prepare report dir
         if: ${{ matrix.test-repeat != 0 }}
@@ -470,7 +469,8 @@ jobs:
 
   instrumented-e2e-tests:
     name: Run instrumented e2e tests
-    runs-on: [self-hosted, android-device]
+    # Temporary workaround for targeting the runner android-runner-v1
+    runs-on: [self-hosted, android-device, android-emulator]
     if: github.event_name == 'schedule' || github.event.inputs.run_e2e_tests == 'true'
     timeout-minutes: 30
     needs: [build-app, build-instrumented-tests]
@@ -507,6 +507,7 @@ jobs:
           INFRA_FLAVOR: prod
           VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }}
           INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000'
+          ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }}
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests.sh
 
@@ -521,6 +522,7 @@ jobs:
         clearPackageData=true,\
         runnerBuilder=de.mannodermaus.junit5.AndroidJUnit5Builder,\
         invalid_test_account_number=0000000000000000,\
+        enable_highly_rate_limited_tests=${{ github.event_name == 'schedule' && 'true' || 'false' }},\
         partner_auth=${{ secrets.STAGEMOLE_PARTNER_AUTH }}"
     strategy:
       fail-fast: false

android/BuildInstructions.md

@@ -238,3 +238,6 @@ the `ENABLE_IN_APP_VERSION_NOTIFICATIONS` property can be set in `local.properti
 ```
 ENABLE_IN_APP_VERSION_NOTIFICATIONS=false
 ```
+
+### Run tests highly affected by rate limiting
+To avoid being rate limited we avoid running tests sending requests that are highly rate limited too often. If you want to run these tests you can set `enable_highly_rate_limited_tests=true` in `local.properties`. The default value is `false`.

android/gradle/libs.versions.toml

@@ -71,7 +71,7 @@ android-gradle-aapt = { module = "com.android.tools.build:aapt2" }
 android-volley = { module = "com.android.volley:volley", version.ref = "android-volley" }
 
 # AndroidX
-androidx-activity-Compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" }
+androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" }
 androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidx-appcompat" }
 androidx-coresplashscreen = { module = "androidx.core:core-splashscreen", version.ref = "androidx-coresplashscreen" }
 androidx-espresso = { module = "androidx.test.espresso:espresso-core", version.ref = "androidx-espresso" }

android/gradle/verification-metadata.xml

@@ -208,9 +208,6 @@
          </artifact>
       </component>
       <component group="androidx.annotation" name="annotation-experimental" version="1.1.0">
-         <artifact name="annotation-experimental-1.1.0.aar">
-            <sha256 value="0157de61a2064047896a058080f3fd67ba57ad9a94857b3f7a363660243e3f90" origin="Generated by Gradle"/>
-         </artifact>
          <artifact name="annotation-experimental-1.1.0.module">
             <sha256 value="0361d1526a4d7501255e19779e09e93cdbd07fee0e2f5c50b7a137432d510119" origin="Generated by Gradle"/>
          </artifact>
@@ -5453,6 +5450,14 @@
             <sha256 value="68516559e6f84a621b9783cd892a64630ccd7875843588ddb3f0501425e33f15" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-android" version="1.7.1">
+         <artifact name="kotlinx-coroutines-android-1.7.1.jar">
+            <sha256 value="107313760c18f8da174e8d8103504a468e806e88f7b55a84bd1c0eaeea118e9a" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="kotlinx-coroutines-android-1.7.1.module">
+            <sha256 value="beb7ff0f5ebc63a0b30af2ae1214e0b622a7b7e408240e64a8ea5b213c4d5334" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-android" version="1.7.3">
          <artifact name="kotlinx-coroutines-android-1.7.3.jar">
             <sha256 value="59fffb26bee12c32dadcfa5d420c2a7db85d3253518128b170efda726613256d" origin="Generated by Gradle"/>

android/scripts/run-instrumented-tests.sh

@@ -16,6 +16,7 @@ TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/te
 PARTNER_AUTH="${PARTNER_AUTH:-}"
 VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}"
 INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}"
+ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}"
 REPORT_DIR="${REPORT_DIR:-}"
 
 while [[ "$#" -gt 0 ]]; do
@@ -130,6 +131,7 @@ case "$TEST_TYPE" in
         echo "Error: The variable PARTNER_AUTH or VALID_TEST_ACCOUNT_NUMBER must be set."
         exit 1
     fi
+    OPTIONAL_TEST_ARGUMENTS+=" -e enable_highly_rate_limited_tests $ENABLE_HIGHLY_RATE_LIMITED_TESTS"
     USE_ORCHESTRATOR="true"
     PACKAGE_NAME="net.mullvad.mullvadvpn"
     if [[ "$INFRA_FLAVOR" =~ ^(devmole|stagemole)$ ]]; then

android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt

@@ -16,7 +16,6 @@ import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
-import net.mullvad.mullvadvpn.lib.common.constant.BuildTypes
 import net.mullvad.mullvadvpn.lib.common.constant.GRPC_SOCKET_FILE_NAMED_ARGUMENT
 import net.mullvad.mullvadvpn.lib.common.constant.KEY_CONNECT_ACTION
 import net.mullvad.mullvadvpn.lib.common.constant.KEY_DISCONNECT_ACTION
@@ -150,7 +149,7 @@ class MullvadVpnService : TalpidVpnService() {
 
     private fun startDaemon() {
         val apiEndpointConfiguration =
-            if (Build.TYPE == BuildTypes.DEBUG) {
+            if (BuildConfig.DEBUG) {
                 intentProvider.getLatestIntent()?.getApiEndpointConfigurationExtras()
                     ?: apiEndpointConfiguration
             } else {

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt

@@ -24,13 +24,11 @@ fun UiDevice.findObjectWithTimeout(
 
     wait(Until.hasObject(selector), timeout)
 
-    return try {
-        findObject(selector)
-    } catch (e: NullPointerException) {
-        throw IllegalArgumentException(
-            "No matches for selector within timeout ($timeout): $selector"
-        )
-    }
+    val foundObject = findObject(selector)
+
+    require(foundObject != null) { "No matches for selector within timeout ($timeout): $selector" }
+
+    return foundObject
 }
 
 fun UiDevice.clickAgreeOnPrivacyDisclaimer() {

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt

@@ -52,15 +52,6 @@ class AppInteractor(
         ensureLoggedIn()
     }
 
-    fun launchAndCreateAccount() {
-        launch()
-        device.clickAgreeOnPrivacyDisclaimer()
-        device.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove()
-        waitForLoginPrompt()
-        attemptCreateAccount()
-        ensureAccountCreated()
-    }
-
     fun attemptLogin(accountNumber: String) {
         val loginObject =
             device.findObjectWithTimeout(By.clazz("android.widget.EditText")).apply {

android/test/e2e/build.gradle.kts

@@ -46,6 +46,7 @@ android {
                 put("clearPackageData", "true")
                 addOptionalPropertyAsArgument("valid_test_account_number")
                 addOptionalPropertyAsArgument("invalid_test_account_number")
+                addOptionalPropertyAsArgument("enable_highly_rate_limited_tests")
             }
     }
 
@@ -142,4 +143,12 @@ dependencies {
     implementation(libs.kotlin.stdlib)
 
     androidTestUtil(libs.androidx.test.orchestrator)
+
+    // Needed or else the app crashes when launched
+    implementation(Dependencies.junit5AndroidTestCompose)
+    implementation(libs.compose.material3)
+
+    // Need these for forcing later versions of dependencies
+    implementation(libs.compose.ui)
+    implementation(libs.androidx.activity.compose)
 }

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt

@@ -5,8 +5,8 @@ import net.mullvad.mullvadvpn.test.common.constant.LOGIN_FAILURE_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.extension.clickAgreeOnPrivacyDisclaimer
 import net.mullvad.mullvadvpn.test.common.extension.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove
 import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout
+import net.mullvad.mullvadvpn.test.e2e.annotations.HighlyRateLimited
 import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
 
@@ -27,7 +27,7 @@ class LoginTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
     }
 
     @Test
-    @Disabled("Disabled to avoid getting rate-limited.")
+    @HighlyRateLimited
     fun testLoginWithInvalidCredentials() {
         // Given
         val invalidDummyAccountNumber = accountTestRule.invalidAccountNumber

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt

@@ -23,17 +23,4 @@ class LogoutTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
         // Then
         assertNotNull(device.findObjectWithTimeout(By.text("Login")))
     }
-
-    @Test
-    fun testCreateAccountAndLogout() {
-        // Given
-        app.launchAndCreateAccount()
-
-        // When
-        app.clickAccountCog()
-        app.clickActionButtonByText("Log out")
-
-        // Then
-        assertNotNull(device.findObjectWithTimeout(By.text("Login")))
-    }
 }

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt

@@ -0,0 +1,37 @@
+package net.mullvad.mullvadvpn.test.e2e.annotations
+
+import androidx.test.platform.app.InstrumentationRegistry
+import net.mullvad.mullvadvpn.test.e2e.constant.ENABLE_HIGHLY_RATE_LIMITED
+import org.junit.jupiter.api.extension.ConditionEvaluationResult
+import org.junit.jupiter.api.extension.ExecutionCondition
+import org.junit.jupiter.api.extension.ExtendWith
+import org.junit.jupiter.api.extension.ExtensionContext
+
+/**
+ * Annotation for tests making use of API endpoints/requests that are highly rate limited such as
+ * failed login requests.
+ */
+@Retention(AnnotationRetention.RUNTIME)
+@ExtendWith(HighlyRateLimited.ShouldRunWhenSeverelyAffectedByRateLimiting::class)
+annotation class HighlyRateLimited {
+    class ShouldRunWhenSeverelyAffectedByRateLimiting : ExecutionCondition {
+        override fun evaluateExecutionCondition(
+            context: ExtensionContext?
+        ): ConditionEvaluationResult {
+            val enableHighlyRateLimited =
+                InstrumentationRegistry.getArguments()
+                    .getString(ENABLE_HIGHLY_RATE_LIMITED)
+                    ?.toBoolean() ?: false
+
+            if (enableHighlyRateLimited) {
+                return ConditionEvaluationResult.enabled(
+                    "Running test highly affected by rate limiting."
+                )
+            } else {
+                return ConditionEvaluationResult.disabled(
+                    "Skipping test highly affected by rate limiting."
+                )
+            }
+        }
+    }
+}

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt

@@ -4,3 +4,4 @@ const val LOG_TAG = "mullvad-e2e"
 const val PARTNER_AUTH = "partner_auth"
 const val VALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "valid_test_account_number"
 const val INVALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "invalid_test_account_number"
+const val ENABLE_HIGHLY_RATE_LIMITED = "enable_highly_rate_limited_tests"

android/test/firebase/e2e-play-stagemole.yml

@@ -6,13 +6,16 @@
   timeout: 10m
   use-orchestrator: true
   device:
     - {model: shiba, version: 34, locale: en, orientation: portrait} # pixel 8
     - {model: felix, version: 34, locale: en, orientation: portrait} # pixel fold
     - {model: tangorpro, version: 33, locale: en, orientation: portrait} # pixel tablet
     - {model: oriole, version: 32, locale: en, orientation: portrait} # pixel 6
     - {model: oriole, version: 31, locale: en, orientation: portrait} # pixel 6
     - {model: redfin, version: 30, locale: en, orientation: portrait} # pixel 5
     - {model: crownqlteue, version: 29, locale: en, orientation: portrait} # galaxy note9
     - {model: blueline, version: 28, locale: en, orientation: portrait} # pixel 3
     - {model: cactus, version: 27, locale: en, orientation: portrait} # redmi 6a
     - {model: starqlteue, version: 26, locale: en, orientation: portrait} # galaxy s9
+  environment-variables:
+    clearPackageData: "true"
+    runnerBuilder: "de.mannodermaus.junit5.AndroidJUnit5Builder"

android/test/mockapi/build.gradle.kts

@@ -88,4 +88,12 @@ dependencies {
     implementation(libs.mockkWebserver)
 
     androidTestUtil(libs.androidx.test.orchestrator)
+
+    // Needed or else the app crashes when launched
+    implementation(Dependencies.junit5AndroidTestCompose)
+    implementation(libs.compose.material3)
+
+    // Need these for forcing later versions of dependencies
+    implementation(libs.compose.ui)
+    implementation(libs.androidx.activity.compose)
 }