Allow control over instrumentation tests on unsupported devices via configuration parameter and DSL

instrumentation/CHANGELOG.md

@@ -8,6 +8,7 @@ Change Log
 - Update to Compose 1.10
 - Support instrumentation with JUnit 5 and 6 (the plugin will choose the correct runtime accordingly)
 - Avoid error when a client doesn't include junit-jupiter-params on the runtime classpath
+- New: Instead of silently skipping tests when running on unsupported devices, fail test execution via configuration parameter `de.mannodermaus.junit.unsupported.behavior`
 
 ## 1.9.0 (2025-10-10)
 

instrumentation/compose/build.gradle.kts

@@ -21,6 +21,10 @@ android {
 junitPlatform {
     // Using local dependency instead of Maven coordinates
     instrumentationTests.enabled = false
+
+    // Fail test execution when running on unsupported device
+    // (TODO: Change this to the proper instrumentationTests API once released as stable)
+    configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail")
 }
 
 dependencies {

instrumentation/core/build.gradle.kts

@@ -18,6 +18,10 @@ junitPlatform {
         // See TaggedTests.kt for usage of this tag
         excludeTags("nope")
     }
+
+    // Fail test execution when running on unsupported device
+    // (TODO: Change this to the proper instrumentationTests API once released as stable)
+    configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail")
 }
 
 // Use local project dependencies on android-test instrumentation libraries

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt

@@ -63,7 +63,7 @@ public open class AndroidJUnitFrameworkBuilder internal constructor() : RunnerBu
 
         try {
             return if (junitFrameworkAvailable) {
-                tryCreateJUnitFrameworkRunner(testClass) { params }
+                tryCreateJUnitFrameworkRunner(testClass, params)
             } else {
                 null
             }

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/ConfigurationParameters.kt

@@ -0,0 +1,10 @@
+package de.mannodermaus.junit5.internal
+
+public object ConfigurationParameters {
+    /**
+     * How to behave when executing instrumentation tests on an unsupported device (i.e. too old).
+     * Accepted values: "skip", "fail"
+     */
+    public const val BEHAVIOR_FOR_UNSUPPORTED_DEVICES: String =
+        "de.mannodermaus.junit.unsupported.behavior"
+}

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt

@@ -18,10 +18,10 @@ import org.junit.runner.notification.RunNotifier
 @VisibleForTesting(otherwise = VisibleForTesting.PACKAGE_PRIVATE)
 internal class AndroidJUnitFramework(
     private val testClass: Class<*>,
-    paramsSupplier: () -> JUnitFrameworkRunnerParams = JUnitFrameworkRunnerParams::create,
+    params: JUnitFrameworkRunnerParams,
 ) : Runner() {
     private val launcher = LauncherFactory.create()
-    private val testTree by lazy { generateTestTree(paramsSupplier()) }
+    private val testTree by lazy { generateTestTree(params) }
 
     override fun getDescription() = testTree.suiteDescription
 

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/DummyJUnitFramework.kt

@@ -2,10 +2,11 @@ package de.mannodermaus.junit5.internal.runners
 
 import android.os.Build
 import android.util.Log
+import de.mannodermaus.junit5.internal.ConfigurationParameters
 import de.mannodermaus.junit5.internal.JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION
 import de.mannodermaus.junit5.internal.LOG_TAG
 import de.mannodermaus.junit5.internal.dummy.JupiterTestMethodFinder
-import java.lang.reflect.Method
+import org.junit.platform.commons.JUnitException
 import org.junit.runner.Description
 import org.junit.runner.Runner
 import org.junit.runner.notification.RunNotifier
@@ -14,22 +15,28 @@ import org.junit.runner.notification.RunNotifier
  * Fake Runner that marks all JUnit Framework methods as ignored, used for old devices without the
  * required Java capabilities.
  */
-internal class DummyJUnitFramework(private val testClass: Class<*>) : Runner() {
+internal class DummyJUnitFramework(
+    private val testClass: Class<*>,
+    params: JUnitFrameworkRunnerParams,
+) : Runner() {
 
-    private val testMethods: Set<Method> = JupiterTestMethodFinder.find(testClass)
+    private val testMethods = JupiterTestMethodFinder.find(testClass)
+    private val behaviorForUnsupportedDevices = params.behaviorForUnsupportedDevices
 
     override fun run(notifier: RunNotifier) {
-        Log.w(
-            LOG_TAG,
-            "JUnit Framework is not supported on this device: " +
-                "API level ${Build.VERSION.SDK_INT} is less than " +
-                "${JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION}, the minimum requirement. " +
-                "All Jupiter tests for ${testClass.name} will be disabled.",
-        )
-
-        for (testMethod in testMethods) {
-            val description = Description.createTestDescription(testClass, testMethod.name)
-            notifier.fireTestIgnored(description)
+        when (behaviorForUnsupportedDevices) {
+            "skip" -> skipTests(notifier)
+            "fail" -> failExecution(unsupportedDeviceMessage)
+            else -> {
+                Log.w(
+                    LOG_TAG,
+                    "Unknown value found for configuration parameter " +
+                        "'${ConfigurationParameters.BEHAVIOR_FOR_UNSUPPORTED_DEVICES}': " +
+                        "$behaviorForUnsupportedDevices. Apply default behavior " +
+                        "and skip tests for this class.",
+                )
+                skipTests(notifier)
+            }
         }
     }
 
@@ -39,4 +46,24 @@ internal class DummyJUnitFramework(private val testClass: Class<*>) : Runner() {
                 it.addChild(Description.createTestDescription(testClass, method.name))
             }
         }
+
+    private val unsupportedDeviceMessage by lazy {
+        "JUnit Framework is not supported on this device: " +
+            "API level ${Build.VERSION.SDK_INT} is less than " +
+            "${JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION}, the minimum requirement. " +
+            "All Jupiter tests for ${testClass.name} will be disabled."
+    }
+
+    private fun skipTests(notifier: RunNotifier) {
+        Log.w(LOG_TAG, unsupportedDeviceMessage)
+
+        for (testMethod in testMethods) {
+            val description = Description.createTestDescription(testClass, testMethod.name)
+            notifier.fireTestIgnored(description)
+        }
+    }
+
+    private fun failExecution(message: String): Nothing {
+        throw JUnitException(message)
+    }
 }

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerFactory.kt

@@ -13,13 +13,13 @@ import org.junit.runner.Runner
  */
 internal fun tryCreateJUnitFrameworkRunner(
     klass: Class<*>,
-    paramsSupplier: () -> JUnitFrameworkRunnerParams,
+    params: JUnitFrameworkRunnerParams,
 ): Runner? {
     val runner =
         if (Build.VERSION.SDK_INT >= JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION) {
-            AndroidJUnitFramework(klass, paramsSupplier)
+            AndroidJUnitFramework(klass, params)
         } else {
-            DummyJUnitFramework(klass)
+            DummyJUnitFramework(klass, params)
         }
 
     // It's still possible for the runner to not be relevant to the test run,

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerParams.kt

@@ -2,6 +2,7 @@ package de.mannodermaus.junit5.internal.runners
 
 import android.os.Bundle
 import androidx.test.platform.app.InstrumentationRegistry
+import de.mannodermaus.junit5.internal.ConfigurationParameters
 import de.mannodermaus.junit5.internal.discovery.GeneratedFilters
 import de.mannodermaus.junit5.internal.discovery.ParsedSelectors
 import de.mannodermaus.junit5.internal.discovery.PropertiesParser
@@ -13,7 +14,7 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder
 
 internal data class JUnitFrameworkRunnerParams(
     private val arguments: Bundle = Bundle(),
-    private val filters: List<Filter<*>> = emptyList(),
+    private val filtersSupplier: () -> List<Filter<*>> = { emptyList() },
     val environmentVariables: Map<String, String> = emptyMap(),
     val systemProperties: Map<String, String> = emptyMap(),
     private val configurationParameters: Map<String, String> = emptyMap(),
@@ -25,14 +26,17 @@ internal data class JUnitFrameworkRunnerParams(
     fun createDiscoveryRequest(selectors: List<DiscoverySelector>): LauncherDiscoveryRequest {
         return LauncherDiscoveryRequestBuilder.request()
             .selectors(selectors)
-            .filters(*this.filters.toTypedArray())
+            .filters(*this.filtersSupplier().toTypedArray())
             .configurationParameters(this.configurationParameters)
             .build()
     }
 
     val isParallelExecutionEnabled: Boolean
         get() = configurationParameters["junit.jupiter.execution.parallel.enabled"] == "true"
 
+    val behaviorForUnsupportedDevices: String?
+        get() = configurationParameters[ConfigurationParameters.BEHAVIOR_FOR_UNSUPPORTED_DEVICES]
+
     val isUsingOrchestrator: Boolean
         get() = arguments.getString("orchestratorService") != null
 
@@ -64,16 +68,19 @@ internal data class JUnitFrameworkRunnerParams(
             // which aren't subject to the filtering imposed through adb.
             // A special resource file may be looked up at runtime, containing
             // the filters to apply by the AndroidJUnit5 runner.
-            val filters =
+            // This requires lazy access because it reaches into JUnit internals,
+            // which may need Java functionality not supported by the current device
+            val filtersSupplier = {
                 GeneratedFilters.fromContext(instrumentation.context) +
                     listOfNotNull(ShardingFilter.fromArguments(arguments))
+            }
 
             return JUnitFrameworkRunnerParams(
-                arguments,
-                filters,
-                environmentVariables,
-                systemProperties,
-                configurationParameters,
+                arguments = arguments,
+                filtersSupplier = filtersSupplier,
+                environmentVariables = environmentVariables,
+                systemProperties = systemProperties,
+                configurationParameters = configurationParameters,
             )
         }
     }

instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/dummy/JupiterTestMethodFinderTests.kt

@@ -96,8 +96,8 @@ class JupiterTestMethodFinderTests {
         val listener = CountingRunListener()
         notifier.addListener(listener)
 
-        val params = JUnitFrameworkRunnerParams(filters = listOfNotNull(filter))
-        AndroidJUnitFramework(cls) { params }.run(notifier)
+        val params = JUnitFrameworkRunnerParams(filtersSupplier = { listOfNotNull(filter) })
+        AndroidJUnitFramework(cls, params).run(notifier)
 
         return listener
     }

instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFrameworkTests.kt

@@ -81,7 +81,11 @@ class AndroidJUnitFrameworkTests {
         val resultRef = AtomicReference<CollectingRunListener.Results>()
         val args = buildArgs(shardingConfig)
         withMockedInstrumentation(args) {
-            val runner = AndroidJUnitFramework(Sample_NormalTests::class.java)
+            val runner =
+                AndroidJUnitFramework(
+                    testClass = Sample_NormalTests::class.java,
+                    params = JUnitFrameworkRunnerParams(),
+                )
             val listener = CollectingRunListener()
             val notifier = RunNotifier().also { it.addListener(listener) }
             runner.run(notifier)

instrumentation/sample/build.gradle.kts

@@ -40,6 +40,10 @@ junitPlatform {
 
     // Using local dependency instead of Maven coordinates
     instrumentationTests.enabled = false
+
+    // Fail test execution when running on unsupported device
+    // (TODO: Change this to the proper instrumentationTests API once released as stable)
+    configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail")
 }
 
 dependencies {

plugin/CHANGELOG.md

@@ -4,6 +4,11 @@ Change Log
 ## Unreleased
 - Update to Kotlin 2.3
 - Internal: Replace deprecated `OutputDirectoryProvider` with its correct replacement
+- Support instrumentation with JUnit 5 and 6 (the plugin will choose the correct runtime accordingly)
+- Introduce `de.mannodermaus.android-junit-framework` as the new plugin ID
+- New: Control behavior of test execution on unsupported devices via `instrumentationTests.behaviorForUnsupportedDevices`
+  - "Skip": Skip tests and mark them as ignored (default and equal to previous behavior)
+  - "Fail": Throw an exception and fail test execution
 
 ## 1.14.0.0 (2025-10-10)
 - JUnit 5.14.0