Skip to content

Commit e107b41

Browse files
authored
Improve reporting for parallel instrumentation tests (#307)
Wrap the default RunNotifier with a parallel-aware variant if JUnit Jupiter's parallel test execution is enabled. It injects itself into the default AndroidX instrumentation and reorders the emission of test events as necessary. This requires a hefty bit of reflective inspection, therefore guard this access via a helper class. As for the sample app, enable parallelism for it and update the tests to demonstrate the effect of it further. Finally, improve the error message when trying to launch a UI test (e.g. Espresso) in parallel, since that doesn't work. Resolves #295.
1 parent 80f30de commit e107b41

File tree

13 files changed

+325
-22
lines changed

13 files changed

+325
-22
lines changed

instrumentation/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ Change Log
77
- Add support for inherited tests (#288)
88
- Only autoconfigure JUnit 5 for instrumentation tests when the user explicitly adds junit-jupiter-api as a dependency
99
- Prevent noisy logs in Logcat complaining about unresolvable annotation classes (#306)
10+
- Add support for parallel execution of non-UI instrumentation tests (#295)
1011

1112
## 1.3.0 (2021-09-17)
1213

instrumentation/build.gradle.kts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ apiValidation {
3131
ignoredPackages.add("de.mannodermaus.junit5.internal")
3232
ignoredPackages.add("de.mannodermaus.junit5.compose.internal")
3333
ignoredProjects.add("sample")
34+
ignoredProjects.add("testutil")
35+
ignoredProjects.add("testutil-reflect")
3436
}

instrumentation/core/src/main/java/de/mannodermaus/junit5/ActivityScenarioExtension.kt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.extension.ExtensionContext
1212
import org.junit.jupiter.api.extension.ParameterContext
1313
import org.junit.jupiter.api.extension.ParameterResolver
1414
import org.junit.jupiter.api.extension.RegisterExtension
15+
import org.junit.jupiter.api.parallel.ExecutionMode
1516
import java.lang.reflect.ParameterizedType
1617

1718
/**
@@ -153,6 +154,13 @@ private constructor(private val scenarioSupplier: () -> ActivityScenario<A>) : B
153154
/* Methods */
154155

155156
override fun beforeEach(context: ExtensionContext) {
157+
require(context.executionMode == ExecutionMode.SAME_THREAD) {
158+
"UI tests using ActivityScenarioExtension cannot be executed in ${context.executionMode} mode. " +
159+
"Please change it to ${ExecutionMode.SAME_THREAD}, e.g. via the @Execution annotation! " +
160+
"For more information, you can consult the JUnit 5 User Guide at " +
161+
"https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution-synchronization."
162+
}
163+
156164
_scenario = scenarioSupplier()
157165
}
158166

instrumentation/runner/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ configurations.all {
8383

8484
dependencies {
8585
implementation(libs.androidXTestMonitor)
86+
implementation(libs.androidXTestRunner)
8687
implementation(libs.kotlinStdLib)
8788
implementation(libs.junit4)
8889

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnit5.kt

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,27 @@ import android.util.Log
55
import androidx.annotation.VisibleForTesting
66
import de.mannodermaus.junit5.internal.LOG_TAG
77
import de.mannodermaus.junit5.internal.LibcoreAccess
8+
import de.mannodermaus.junit5.internal.runners.notification.ParallelRunNotifier
89
import org.junit.platform.launcher.core.LauncherFactory
910
import org.junit.runner.Runner
1011
import org.junit.runner.notification.RunNotifier
1112

1213
/**
1314
* JUnit Runner implementation using the JUnit Platform as its backbone.
1415
* Serves as an intermediate solution to writing JUnit 5-based instrumentation tests
15-
* until official support arrives for this. This is in Java because we require access to package-private data,
16-
* and Kotlin is more strict about that: https://youtrack.jetbrains.com/issue/KT-15315
17-
*
18-
*
19-
* Replacement For:
20-
* AndroidJUnit4
16+
* until official support arrives for this.
2117
*
2218
* @see org.junit.platform.runner.JUnitPlatform
2319
*/
2420
@SuppressLint("NewApi")
2521
@VisibleForTesting(otherwise = VisibleForTesting.PACKAGE_PRIVATE)
26-
internal class AndroidJUnit5
27-
@JvmOverloads constructor(
22+
internal class AndroidJUnit5(
2823
private val testClass: Class<*>,
29-
private val runnerParams: AndroidJUnit5RunnerParams = createRunnerParams(testClass)
24+
private val runnerParams: AndroidJUnit5RunnerParams = createRunnerParams(testClass),
3025
) : Runner() {
3126

3227
private val launcher = LauncherFactory.create()
33-
private val testTree = generateTestTree(runnerParams)
28+
private val testTree by lazy { generateTestTree(runnerParams) }
3429

3530
override fun getDescription() =
3631
testTree.suiteDescription
@@ -41,7 +36,10 @@ internal class AndroidJUnit5
4136
registerSystemProperties()
4237

4338
// Finally, launch the test plan on the JUnit Platform
44-
launcher.execute(testTree.testPlan, AndroidJUnitPlatformRunnerListener(testTree, notifier))
39+
launcher.execute(
40+
testTree.testPlan,
41+
AndroidJUnitPlatformRunnerListener(testTree, createNotifier(notifier)),
42+
)
4543
}
4644

4745
/* Private */
@@ -66,9 +64,19 @@ internal class AndroidJUnit5
6664
}
6765
}
6866

69-
private fun generateTestTree(params: AndroidJUnit5RunnerParams): AndroidJUnitPlatformTestTree {
70-
val discoveryRequest = params.createDiscoveryRequest()
71-
val testPlan = launcher.discover(discoveryRequest)
72-
return AndroidJUnitPlatformTestTree(testPlan, testClass, params.isIsolatedMethodRun())
73-
}
67+
private fun generateTestTree(params: AndroidJUnit5RunnerParams) =
68+
AndroidJUnitPlatformTestTree(
69+
testPlan = launcher.discover(params.createDiscoveryRequest()),
70+
testClass = testClass,
71+
isIsolatedMethodRun = params.isIsolatedMethodRun,
72+
isParallelExecutionEnabled = params.isParallelExecutionEnabled,
73+
)
74+
75+
private fun createNotifier(nextNotifier: RunNotifier) =
76+
if (testTree.isParallelExecutionEnabled) {
77+
// Wrap the default notifier with a special handler for parallel test execution
78+
ParallelRunNotifier(nextNotifier)
79+
} else {
80+
nextNotifier
81+
}
7482
}

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnit5RunnerParams.kt

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@ internal data class AndroidJUnit5RunnerParams(
2626
.configurationParameters(this.configurationParameters)
2727
.build()
2828

29-
fun isIsolatedMethodRun(): Boolean {
30-
return selectors.size == 1 && selectors.first() is MethodSelector
31-
}
29+
val isIsolatedMethodRun: Boolean
30+
get() = selectors.size == 1 && selectors.first() is MethodSelector
31+
32+
val isParallelExecutionEnabled: Boolean
33+
get() = configurationParameters["junit.jupiter.execution.parallel.enabled"] == "true"
3234
}
3335

3436
private const val ARG_ENVIRONMENT_VARIABLES = "environmentVariables"

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitPlatformTestTree.kt

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import java.util.function.Predicate
2222
internal class AndroidJUnitPlatformTestTree(
2323
testPlan: TestPlan,
2424
testClass: Class<*>,
25-
private val isIsolatedMethodRun: Boolean
25+
private val isIsolatedMethodRun: Boolean,
26+
val isParallelExecutionEnabled: Boolean,
2627
) {
2728

2829
private val descriptions = mutableMapOf<TestIdentifier, Description>()
@@ -146,7 +147,8 @@ internal class AndroidJUnitPlatformTestTree(
146147
.map(nameExtractor)
147148
.orElse("<unrooted>"),
148149
/* name = */ name,
149-
/* uniqueId = */ identifier.uniqueId
150+
// Used to distinguish JU5 from other frameworks (e.g. for parallel execution)
151+
/* ...annotations = */ org.junit.jupiter.api.Test(),
150152
)
151153
} else {
152154
Description.createSuiteDescription(name, identifier.uniqueId)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package de.mannodermaus.junit5.internal.runners.notification
2+
3+
import org.junit.runner.Description
4+
import org.junit.runner.notification.Failure
5+
import org.junit.runner.notification.RunListener
6+
7+
/**
8+
* A wrapper implementation around JUnit's [RunListener] class
9+
* which only works selectively. In other words, this implementation only delegates
10+
* to its parameter for test descriptors that pass the given [filter].
11+
*/
12+
internal class FilteredRunListener(
13+
private val delegate: RunListener,
14+
private val filter: (Description) -> Boolean,
15+
) : RunListener() {
16+
override fun testStarted(description: Description) {
17+
if (filter(description)) {
18+
delegate.testStarted(description)
19+
}
20+
}
21+
22+
override fun testIgnored(description: Description) {
23+
if (filter(description)) {
24+
delegate.testIgnored(description)
25+
}
26+
}
27+
28+
override fun testFailure(failure: Failure) {
29+
if (filter(failure.description)) {
30+
delegate.testFailure(failure)
31+
}
32+
}
33+
34+
override fun testAssumptionFailure(failure: Failure) {
35+
if (filter(failure.description)) {
36+
delegate.testAssumptionFailure(failure)
37+
}
38+
}
39+
40+
override fun testFinished(description: Description) {
41+
if (filter(description)) {
42+
delegate.testFinished(description)
43+
}
44+
}
45+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
package de.mannodermaus.junit5.internal.runners.notification
2+
3+
import android.util.Log
4+
import androidx.test.internal.runner.listener.InstrumentationResultPrinter
5+
import de.mannodermaus.junit5.internal.LOG_TAG
6+
import org.junit.runner.Description
7+
import org.junit.runner.Result
8+
import org.junit.runner.notification.Failure
9+
import org.junit.runner.notification.RunListener
10+
import org.junit.runner.notification.RunNotifier
11+
12+
/**
13+
* Wrapping implementation of JUnit 4's run notifier for parallel test execution
14+
* (i.e. when "junit.jupiter.execution.parallel.enabled" is active during the run).
15+
* It unpacks the singular 'instrumentation result printer' assigned by Android
16+
* into using one instance per test, preventing its mutable internals from being
17+
* modified by concurrent threads at the same time.
18+
*/
19+
internal class ParallelRunNotifier(private val delegate: RunNotifier) : RunNotifier() {
20+
companion object {
21+
// Reflective access is available via companion object
22+
// to allow for shared storage of data across notifiers
23+
private val reflection by lazy {
24+
try {
25+
Reflection()
26+
} catch (e: Throwable) {
27+
Log.e(LOG_TAG, "FATAL: Cannot initialize reflective access", e)
28+
null
29+
}
30+
}
31+
}
32+
33+
private val states = mutableMapOf<String, InstrumentationResultPrinter?>()
34+
35+
// Original printer registered via Android instrumentation
36+
private val printer = reflection?.initialize(delegate)
37+
38+
override fun fireTestSuiteStarted(description: Description) {
39+
delegate.fireTestSuiteStarted(description)
40+
}
41+
42+
override fun fireTestRunStarted(description: Description) {
43+
delegate.fireTestRunStarted(description)
44+
}
45+
46+
override fun fireTestStarted(description: Description) {
47+
synchronized(this) {
48+
delegate.fireTestStarted(description)
49+
50+
// Notify original printer immediately,
51+
// then freeze its state for the current method for later
52+
printer?.testStarted(description)
53+
states[description] = reflection?.copy(printer)
54+
}
55+
}
56+
57+
override fun fireTestIgnored(description: Description) {
58+
synchronized(this) {
59+
delegate.fireTestIgnored(description)
60+
61+
printer?.testIgnored(description)
62+
}
63+
}
64+
65+
override fun fireTestFailure(failure: Failure) {
66+
delegate.fireTestFailure(failure)
67+
68+
states[failure.description]?.testFailure(failure)
69+
}
70+
71+
override fun fireTestAssumptionFailed(failure: Failure) {
72+
delegate.fireTestAssumptionFailed(failure)
73+
74+
states[failure.description]?.testAssumptionFailure(failure)
75+
}
76+
77+
override fun fireTestFinished(description: Description) {
78+
synchronized(this) {
79+
delegate.fireTestFinished(description)
80+
81+
states[description]?.testFinished(description)
82+
states.remove(description)
83+
}
84+
}
85+
86+
override fun fireTestRunFinished(result: Result) {
87+
delegate.fireTestRunFinished(result)
88+
}
89+
90+
override fun fireTestSuiteFinished(description: Description) {
91+
delegate.fireTestSuiteFinished(description)
92+
}
93+
94+
/* Private */
95+
96+
private operator fun <T> Map<String, T>.get(key: Description): T? {
97+
return get(key.displayName)
98+
}
99+
100+
private operator fun <T> MutableMap<String, T>.set(key: Description, value: T) {
101+
put(key.displayName, value)
102+
}
103+
104+
private fun <T> MutableMap<String, T>.remove(key: Description) {
105+
remove(key.displayName)
106+
}
107+
108+
@Suppress("UNCHECKED_CAST")
109+
private class Reflection {
110+
private val synchronizedRunListenerClass =
111+
Class.forName("org.junit.runner.notification.SynchronizedRunListener")
112+
private val synchronizedListenerDelegateField = synchronizedRunListenerClass
113+
.getDeclaredField("listener").also { it.isAccessible = true }
114+
private val runNotifierListenersField = RunNotifier::class.java
115+
.getDeclaredField("listeners").also { it.isAccessible = true }
116+
117+
private var cached: InstrumentationResultPrinter? = null
118+
119+
fun initialize(notifier: RunNotifier): InstrumentationResultPrinter? {
120+
try {
121+
// The printer needs to be retrieved only once per test run
122+
cached?.let { return it }
123+
124+
// The Android system registers a global listener
125+
// for communicating status events back to the instrumentation.
126+
// In parallel mode, this communication must be piped through
127+
// a custom piece of logic in order to not lose any mutable data
128+
// from concurrent method invocations
129+
val listeners = runNotifierListenersField.get(notifier) as? List<RunListener>
130+
131+
// The Android instrumentation may wrap the printer inside another JUnit listener,
132+
// so make sure to search for the result inside its toString() representation
133+
// (rather than through an 'it is X' check)
134+
val candidate = listeners?.firstOrNull {
135+
InstrumentationResultPrinter::class.java.name in it.toString()
136+
}
137+
138+
if (candidate != null) {
139+
// Replace the original listener with a wrapped version of itself,
140+
// which will allow all non-JUnit 5 tests through the normal pipeline
141+
// (tests that actually _are_ JUnit 5 will be handled differently)
142+
notifier.removeListener(candidate)
143+
notifier.addListener(FilteredRunListener(candidate, Description::isNotJUnit5))
144+
}
145+
146+
// The Android instrumentation may wrap the printer inside another JUnit listener,
147+
// so make sure to search for the result inside its toString() representation
148+
// (rather than through an 'it is X' check)
149+
val result = if (synchronizedRunListenerClass.isInstance(candidate)) {
150+
synchronizedListenerDelegateField.get(candidate) as? InstrumentationResultPrinter
151+
} else {
152+
candidate as? InstrumentationResultPrinter
153+
}
154+
155+
cached = result
156+
return result
157+
} catch (e: Throwable) {
158+
e.printStackTrace()
159+
return null
160+
}
161+
}
162+
163+
fun copy(original: InstrumentationResultPrinter?): InstrumentationResultPrinter? = try {
164+
if (original != null) {
165+
InstrumentationResultPrinter().also { copy ->
166+
copy.instrumentation = original.instrumentation
167+
168+
InstrumentationResultPrinter::class.java.declaredFields.forEach { field ->
169+
field.isAccessible = true
170+
field.set(copy, field.get(original))
171+
}
172+
}
173+
} else {
174+
null
175+
}
176+
} catch (e: Throwable) {
177+
e.printStackTrace()
178+
null
179+
}
180+
}
181+
}
182+
183+
private val Description.isNotJUnit5: Boolean
184+
get() = getAnnotation(org.junit.jupiter.api.Test::class.java) == null

instrumentation/sample/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ android {
2222
// Make sure to use the AndroidJUnitRunner (or a sub-class) in order to hook in the JUnit 5 Test Builder
2323
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
2424
testInstrumentationRunnerArguments["runnerBuilder"] = "de.mannodermaus.junit5.AndroidJUnit5Builder"
25+
testInstrumentationRunnerArguments["configurationParameters"] = "junit.jupiter.execution.parallel.enabled=true,junit.jupiter.execution.parallel.mode.default=concurrent"
2526

2627
buildConfigField("boolean", "MY_VALUE", "true")
2728
}

0 commit comments

Comments
 (0)