1 /*
<lambda>null2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.graph
18 
19 import android.content.Context
20 import android.graphics.ImageFormat
21 import android.hardware.camera2.CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL
22 import android.hardware.camera2.CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL
23 import android.media.ImageReader
24 import android.os.Build
25 import android.util.Size
26 import androidx.camera.camera2.pipe.CameraBackendFactory
27 import androidx.camera.camera2.pipe.CameraGraph
28 import androidx.camera.camera2.pipe.CameraGraphId
29 import androidx.camera.camera2.pipe.CameraStream
30 import androidx.camera.camera2.pipe.CameraSurfaceManager
31 import androidx.camera.camera2.pipe.Request
32 import androidx.camera.camera2.pipe.StreamFormat
33 import androidx.camera.camera2.pipe.internal.CameraBackendsImpl
34 import androidx.camera.camera2.pipe.internal.CameraGraphParametersImpl
35 import androidx.camera.camera2.pipe.internal.FrameCaptureQueue
36 import androidx.camera.camera2.pipe.internal.FrameDistributor
37 import androidx.camera.camera2.pipe.internal.ImageSourceMap
38 import androidx.camera.camera2.pipe.media.ImageReaderImageSources
39 import androidx.camera.camera2.pipe.testing.CameraControllerSimulator
40 import androidx.camera.camera2.pipe.testing.FakeAudioRestrictionController
41 import androidx.camera.camera2.pipe.testing.FakeCameraBackend
42 import androidx.camera.camera2.pipe.testing.FakeCameraMetadata
43 import androidx.camera.camera2.pipe.testing.FakeGraphProcessor
44 import androidx.camera.camera2.pipe.testing.FakeThreads
45 import androidx.camera.camera2.pipe.testing.RobolectricCameraPipeTestRunner
46 import androidx.test.core.app.ApplicationProvider
47 import androidx.testutils.assertThrows
48 import com.google.common.truth.Truth.assertThat
49 import kotlinx.coroutines.CancellationException
50 import kotlinx.coroutines.CoroutineScope
51 import kotlinx.coroutines.ExperimentalCoroutinesApi
52 import kotlinx.coroutines.Job
53 import kotlinx.coroutines.SupervisorJob
54 import kotlinx.coroutines.async
55 import kotlinx.coroutines.delay
56 import kotlinx.coroutines.isActive
57 import kotlinx.coroutines.runBlocking
58 import kotlinx.coroutines.test.TestScope
59 import kotlinx.coroutines.test.advanceUntilIdle
60 import kotlinx.coroutines.test.runTest
61 import kotlinx.coroutines.yield
62 import org.junit.Before
63 import org.junit.Test
64 import org.junit.runner.RunWith
65 import org.mockito.kotlin.eq
66 import org.mockito.kotlin.mock
67 import org.mockito.kotlin.times
68 import org.mockito.kotlin.verify
69 import org.robolectric.annotation.Config
70 import org.robolectric.annotation.internal.DoNotInstrument
71 
72 @OptIn(ExperimentalCoroutinesApi::class)
73 @RunWith(RobolectricCameraPipeTestRunner::class)
74 @DoNotInstrument
75 @Config(minSdk = Build.VERSION_CODES.LOLLIPOP)
76 internal class CameraGraphImplTest {
77     private val testScope = TestScope()
78 
79     private val context = ApplicationProvider.getApplicationContext() as Context
80     private val metadata =
81         FakeCameraMetadata(
82             mapOf(INFO_SUPPORTED_HARDWARE_LEVEL to INFO_SUPPORTED_HARDWARE_LEVEL_FULL),
83         )
84     private val fakeGraphProcessor = FakeGraphProcessor()
85     private val imageReader1 = ImageReader.newInstance(1280, 720, ImageFormat.YUV_420_888, 4)
86     private val imageReader2 = ImageReader.newInstance(1920, 1080, ImageFormat.YUV_420_888, 4)
87     private val fakeSurfaceListener: CameraSurfaceManager.SurfaceListener = mock()
88     private val cameraSurfaceManager = CameraSurfaceManager()
89 
90     private val stream1Config =
91         CameraStream.Config.create(Size(1280, 720), StreamFormat.YUV_420_888)
92     private val stream2Config =
93         CameraStream.Config.create(Size(1920, 1080), StreamFormat.YUV_420_888)
94 
95     private val graphId = CameraGraphId.nextId()
96     private val graphConfig =
97         CameraGraph.Config(
98             camera = metadata.camera,
99             streams = listOf(stream1Config, stream2Config),
100         )
101     private val threads = FakeThreads.fromTestScope(testScope)
102     private val backend = FakeCameraBackend(fakeCameras = mapOf(metadata.camera to metadata))
103     private val backends =
104         CameraBackendsImpl(
105             defaultBackendId = backend.id,
106             cameraBackends = mapOf(backend.id to CameraBackendFactory { backend }),
107             context,
108             threads
109         )
110     private val cameraContext = CameraBackendsImpl.CameraBackendContext(context, threads, backends)
111     private val imageSources = ImageReaderImageSources(threads)
112     private val frameCaptureQueue = FrameCaptureQueue()
113     private val cameraController =
114         CameraControllerSimulator(cameraContext, graphId, graphConfig, fakeGraphProcessor)
115     private val cameraControllerProvider: () -> CameraControllerSimulator = { cameraController }
116     private val streamGraph = StreamGraphImpl(metadata, graphConfig, cameraControllerProvider)
117     private val imageSourceMap = ImageSourceMap(graphConfig, streamGraph, imageSources)
118     private val frameDistributor =
119         FrameDistributor(imageSourceMap.imageSources, frameCaptureQueue) {}
120     private val surfaceGraph =
121         SurfaceGraph(streamGraph, cameraControllerProvider, cameraSurfaceManager, emptyMap())
122     private val audioRestriction = FakeAudioRestrictionController()
123     private val sessionLock = SessionLock()
124     private val cameraGraph =
125         CameraGraphImpl(
126             graphConfig,
127             metadata,
128             fakeGraphProcessor,
129             fakeGraphProcessor,
130             streamGraph,
131             surfaceGraph,
132             cameraController,
133             GraphState3A(),
134             Listener3A(),
135             frameDistributor,
136             frameCaptureQueue,
137             audioRestriction,
138             graphId,
139             CameraGraphParametersImpl(sessionLock, fakeGraphProcessor, testScope),
140             sessionLock
141         )
142     private val stream1: CameraStream =
143         checkNotNull(cameraGraph.streams[stream1Config]) {
144             "Failed to find stream for $stream1Config!"
145         }
146 
147     private val stream2 =
148         checkNotNull(cameraGraph.streams[stream2Config]) {
149             "Failed to find stream for $stream2Config!"
150         }
151 
152     init {
153         cameraSurfaceManager.addListener(fakeSurfaceListener)
154     }
155 
156     @Before
157     fun setUp() {
158         cameraController.streamGraph = streamGraph
159     }
160 
161     @Test fun createCameraGraphImpl() = testScope.runTest { assertThat(cameraGraph).isNotNull() }
162 
163     @Test
164     fun testAcquireSession() =
165         testScope.runTest {
166             val session = cameraGraph.acquireSession()
167             assertThat(session).isNotNull()
168         }
169 
170     @Test
171     fun testAcquireSessionOrNull() =
172         testScope.runTest {
173             val session = cameraGraph.acquireSessionOrNull()
174             assertThat(session).isNotNull()
175         }
176 
177     @Test
178     fun testAcquireSessionOrNullAfterAcquireSession() =
179         testScope.runTest {
180             val session = cameraGraph.acquireSession()
181             assertThat(session).isNotNull()
182 
183             // Since a session is already active, an attempt to acquire another session will fail.
184             val session1 = cameraGraph.acquireSessionOrNull()
185             assertThat(session1).isNull()
186 
187             // Closing an active session should allow a new session instance to be created.
188             session.close()
189 
190             val session2 = cameraGraph.acquireSessionOrNull()
191             assertThat(session2).isNotNull()
192         }
193 
194     @Test
195     fun sessionSubmitsRequestsToGraphProcessor() =
196         testScope.runTest {
197             val session = checkNotNull(cameraGraph.acquireSessionOrNull())
198             val request = Request(listOf())
199             session.submit(request)
200             advanceUntilIdle()
201 
202             assertThat(fakeGraphProcessor.requestQueue).contains(listOf(request))
203         }
204 
205     @Test
206     fun sessionSetsRepeatingRequestOnGraphProcessor() =
207         testScope.runTest {
208             val session = checkNotNull(cameraGraph.acquireSessionOrNull())
209             val request = Request(listOf())
210             session.startRepeating(request)
211             advanceUntilIdle()
212 
213             assertThat(fakeGraphProcessor.repeatingRequest).isSameInstanceAs(request)
214         }
215 
216     @Test
217     fun sessionAbortsRequestOnGraphProcessor() =
218         testScope.runTest {
219             val session = checkNotNull(cameraGraph.acquireSessionOrNull())
220             val request = Request(listOf())
221             session.submit(request)
222             session.abort()
223             advanceUntilIdle()
224 
225             assertThat(fakeGraphProcessor.requestQueue).isEmpty()
226         }
227 
228     @Test
229     fun closingSessionDoesNotCloseGraphProcessor() =
230         testScope.runTest {
231             val session = cameraGraph.acquireSessionOrNull()
232             checkNotNull(session).close()
233             advanceUntilIdle()
234 
235             assertThat(fakeGraphProcessor.closed).isFalse()
236         }
237 
238     @Test
239     fun closingCameraGraphClosesGraphProcessor() =
240         testScope.runTest {
241             cameraGraph.close()
242             assertThat(fakeGraphProcessor.closed).isTrue()
243         }
244 
245     @Test
246     fun stoppingCameraGraphStopsGraphProcessor() =
247         testScope.runTest {
248             assertThat(cameraController.started).isFalse()
249             assertThat(fakeGraphProcessor.closed).isFalse()
250             cameraGraph.start()
251             assertThat(cameraController.started).isTrue()
252             cameraGraph.stop()
253             assertThat(cameraController.started).isFalse()
254             assertThat(fakeGraphProcessor.closed).isFalse()
255             cameraGraph.start()
256             assertThat(cameraController.started).isTrue()
257             cameraGraph.close()
258             assertThat(cameraController.started).isFalse()
259             assertThat(fakeGraphProcessor.closed).isTrue()
260         }
261 
262     @Test
263     fun closingCameraGraphClosesAssociatedSurfaces() =
264         testScope.runTest {
265             cameraGraph.setSurface(stream1.id, imageReader1.surface)
266             cameraGraph.setSurface(stream2.id, imageReader2.surface)
267             cameraGraph.close()
268 
269             verify(fakeSurfaceListener, times(1)).onSurfaceActive(eq(imageReader1.surface))
270             verify(fakeSurfaceListener, times(1)).onSurfaceActive(eq(imageReader2.surface))
271             verify(fakeSurfaceListener, times(1)).onSurfaceInactive(eq(imageReader1.surface))
272             verify(fakeSurfaceListener, times(1)).onSurfaceInactive(eq(imageReader1.surface))
273         }
274 
275     @Test
276     fun useSessionInOperatesInOrder() =
277         testScope.runTest {
278             val events = mutableListOf<Int>()
279             val job1 =
280                 cameraGraph.useSessionIn(testScope) {
281                     yield()
282                     events += 2
283                 }
284             val job2 =
285                 cameraGraph.useSessionIn(testScope) {
286                     delay(100)
287                     events += 3
288                 }
289             val job3 =
290                 cameraGraph.useSessionIn(testScope) {
291                     yield()
292                     events += 4
293                 }
294 
295             events += 1
296             job1.join()
297             job2.join()
298             job3.join()
299 
300             assertThat(events).containsExactly(1, 2, 3, 4).inOrder()
301         }
302 
303     @Test
304     fun useSessionWithEarlyCloseAllowsInterleavedExecution() =
305         testScope.runTest {
306             val events = mutableListOf<Int>()
307             val job1 =
308                 cameraGraph.useSessionIn(testScope) { session ->
309                     yield()
310                     events += 2
311                     session.close()
312                     delay(1000)
313                     events += 5
314                 }
315             val job2 =
316                 cameraGraph.useSessionIn(testScope) {
317                     delay(100)
318                     events += 3
319                 }
320             val job3 =
321                 cameraGraph.useSessionIn(testScope) {
322                     yield()
323                     events += 4
324                 }
325 
326             events += 1
327             job1.join()
328             job2.join()
329             job3.join()
330 
331             assertThat(events).containsExactly(1, 2, 3, 4, 5).inOrder()
332         }
333 
334     @Test
335     fun useSessionInWithRunBlockingDoesNotStall() = runBlocking {
336         val deferred = cameraGraph.useSessionIn(this) { delay(1) }
337         deferred.await() // Make sure this does not block.
338     }
339 
340     @Test
341     fun coroutineScope_isCanceledWithException() =
342         testScope.runTest {
343             val scope = CoroutineScope(Job())
344 
345             val deferred = scope.async { throw RuntimeException() }
346             deferred.join()
347 
348             // Ensure the deferred is completed with an exception, and that the scope is NOT active.
349             assertThat(deferred.isCompleted).isTrue()
350             assertThat(deferred.getCompletionExceptionOrNull())
351                 .isInstanceOf(RuntimeException::class.java)
352             assertThrows<RuntimeException> { deferred.await() }
353             assertThat(scope.isActive).isFalse()
354         }
355 
356     @Test
357     fun coroutineSupervisorScope_isNotCanceledWithException() =
358         testScope.runTest {
359             val scope = CoroutineScope(SupervisorJob())
360 
361             val deferred = scope.async { throw RuntimeException() }
362             deferred.join()
363 
364             // Ensure the deferred is completed with an exception, and that the scope remains
365             // active.
366             assertThat(deferred.isCompleted).isTrue()
367             assertThat(deferred.getCompletionExceptionOrNull())
368                 .isInstanceOf(RuntimeException::class.java)
369             assertThrows<RuntimeException> { deferred.await() }
370             assertThat(scope.isActive).isTrue()
371         }
372 
373     @Test
374     fun useSessionIn_scopeIsCanceledWithException() =
375         testScope.runTest {
376             val scope = CoroutineScope(Job())
377 
378             val deferred = cameraGraph.useSessionIn(scope) { throw RuntimeException() }
379             deferred.join()
380 
381             assertThat(deferred.isCompleted).isTrue()
382             assertThat(deferred.getCompletionExceptionOrNull())
383                 .isInstanceOf(RuntimeException::class.java)
384             assertThrows<RuntimeException> { deferred.await() }
385             assertThat(scope.isActive).isFalse() // Regular scopes are canceled
386         }
387 
388     @Test
389     fun useSessionIn_supervisorScopeIsNotCanceledWithException() =
390         testScope.runTest {
391             val scope = CoroutineScope(SupervisorJob())
392             val deferred = cameraGraph.useSessionIn(scope) { throw RuntimeException() }
393             deferred.join()
394 
395             assertThat(deferred.isCompleted).isTrue()
396             assertThat(deferred.getCompletionExceptionOrNull())
397                 .isInstanceOf(RuntimeException::class.java)
398             assertThrows<RuntimeException> { deferred.await() }
399             assertThat(scope.isActive).isTrue() // Supervisor scopes are not canceled
400         }
401 
402     @Test
403     fun coroutineSupervisorTestScope_isNotCanceledWithException() =
404         testScope.runTest {
405             // This illustrates the correct way to create a scope that uses the testScope
406             // dispatcher, does delay skipping, but also does not fail the test if an exception
407             // occurs when doing scope.async. This is useful if, for example, in a real environment
408             // scope represents a supervisor job that will not crash if a coroutine fails and if
409             // some other system is handling the result of the deferred.
410             val scope = CoroutineScope(testScope.coroutineContext + Job())
411 
412             val deferred =
413                 scope.async {
414                     delay(100000) // Delay skipping
415                     throw RuntimeException()
416                 }
417             deferred.join()
418 
419             assertThat(deferred.isCompleted).isTrue()
420             assertThat(deferred.getCompletionExceptionOrNull())
421                 .isInstanceOf(RuntimeException::class.java)
422             assertThrows<RuntimeException> { deferred.await() }
423             assertThat(scope.isActive).isFalse()
424             assertThat(testScope.isActive).isTrue()
425         }
426 
427     @Test
428     fun useSessionIn_withSupervisorTestScopeDoesNotCancelTestScope() =
429         testScope.runTest {
430             // Create a scope that uses the testScope dispatcher and delaySkipping, but does not
431             // fail
432             // the test if an exception occurs in useSessionIn.
433             val scope = CoroutineScope(testScope.coroutineContext + SupervisorJob())
434 
435             // If you pass in a testScope to useSessionIn, any exception will cause the test to
436             // fail. If, instead, you want to test that the deferred handles the exception, you must
437             // pass in an independent CoroutineScope.
438             val deferred = cameraGraph.useSessionIn(scope) { throw RuntimeException() }
439             deferred.join()
440 
441             assertThat(deferred.isCompleted).isTrue()
442             assertThat(deferred.getCompletionExceptionOrNull())
443                 .isInstanceOf(RuntimeException::class.java)
444             assertThat(scope.isActive).isTrue() // Supervisor scopes are not canceled
445             assertThat(testScope.isActive).isTrue()
446         }
447 
448     @Test
449     fun useSessionIn_withCancellationDoesNotFailTest() =
450         testScope.runTest {
451             val deferred =
452                 cameraGraph.useSessionIn(testScope) {
453                     throw CancellationException() // Throwing cancellation does not cause the test
454                     // to fail.
455                 }
456             deferred.join()
457 
458             assertThat(deferred.isActive).isFalse()
459             assertThat(deferred.isCompleted).isTrue()
460             assertThat(deferred.isCancelled).isTrue()
461             assertThat(deferred.getCompletionExceptionOrNull())
462                 .isInstanceOf(CancellationException::class.java)
463             assertThat(testScope.isActive).isTrue()
464         }
465 
466     @Test
467     fun useSession_throwsExceptions() =
468         testScope.runTest {
469             assertThrows<RuntimeException> { cameraGraph.useSession { throw RuntimeException() } }
470         }
471 
472     @Test
473     fun testGetOutputLatency() =
474         testScope.runTest {
475             assertThat(cameraController.getOutputLatency(null)).isNull()
476             cameraController.simulateOutputLatency()
477             assertThat(cameraController.getOutputLatency(null)?.estimatedLatencyNs)
478                 .isEqualTo(cameraController.outputLatencySet?.estimatedLatencyNs)
479         }
480 }
481