1 /*
2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.testing
18 
19 import android.content.Context
20 import android.hardware.camera2.CameraCharacteristics
21 import android.hardware.camera2.CaptureResult
22 import android.os.Build
23 import android.util.Size
24 import androidx.camera.camera2.pipe.CameraError
25 import androidx.camera.camera2.pipe.CameraGraph
26 import androidx.camera.camera2.pipe.CameraStream
27 import androidx.camera.camera2.pipe.GraphState.GraphStateError
28 import androidx.camera.camera2.pipe.GraphState.GraphStateStarted
29 import androidx.camera.camera2.pipe.GraphState.GraphStateStarting
30 import androidx.camera.camera2.pipe.GraphState.GraphStateStopped
31 import androidx.camera.camera2.pipe.GraphState.GraphStateStopping
32 import androidx.camera.camera2.pipe.Request
33 import androidx.camera.camera2.pipe.StreamFormat
34 import androidx.test.core.app.ApplicationProvider
35 import com.google.common.truth.Truth.assertThat
36 import kotlinx.coroutines.Dispatchers
37 import kotlinx.coroutines.ExperimentalCoroutinesApi
38 import kotlinx.coroutines.delay
39 import kotlinx.coroutines.flow.drop
40 import kotlinx.coroutines.flow.first
41 import kotlinx.coroutines.flow.take
42 import kotlinx.coroutines.flow.toList
43 import kotlinx.coroutines.launch
44 import kotlinx.coroutines.test.TestScope
45 import kotlinx.coroutines.test.advanceUntilIdle
46 import kotlinx.coroutines.test.runTest
47 import kotlinx.coroutines.withContext
48 import kotlinx.coroutines.withTimeout
49 import kotlinx.coroutines.withTimeoutOrNull
50 import org.junit.Test
51 import org.junit.runner.RunWith
52 import org.robolectric.annotation.Config
53 
54 @OptIn(ExperimentalCoroutinesApi::class)
55 @RunWith(RobolectricCameraPipeTestRunner::class)
56 @Config(minSdk = Build.VERSION_CODES.LOLLIPOP)
57 class CameraGraphSimulatorTest {
58     private val testScope = TestScope()
59     private val metadata =
60         FakeCameraMetadata(
61             characteristics =
62                 mapOf(CameraCharacteristics.LENS_FACING to CameraCharacteristics.LENS_FACING_FRONT)
63         )
64 
65     private val streamConfig = CameraStream.Config.create(Size(640, 480), StreamFormat.YUV_420_888)
66 
67     private val graphConfig =
68         CameraGraph.Config(camera = metadata.camera, streams = listOf(streamConfig))
69 
70     private val context = ApplicationProvider.getApplicationContext() as Context
71     private val simulator = CameraGraphSimulator.create(testScope, context, metadata, graphConfig)
72 
73     @Test
simulatorCanSimulateRepeatingFramesnull74     fun simulatorCanSimulateRepeatingFrames() =
75         testScope.runTest {
76             val stream = simulator.streams[streamConfig]!!
77             val listener = FakeRequestListener()
78             val request = Request(streams = listOf(stream.id), listeners = listOf(listener))
79             simulator.acquireSession().use { it.startRepeating(request) }
80             simulator.start()
81             simulator.simulateCameraStarted()
82             simulator.initializeSurfaces()
83             advanceUntilIdle()
84 
85             val frame = simulator.simulateNextFrame()
86 
87             assertThat(frame.request).isSameInstanceAs(request)
88             assertThat(frame.frameNumber.value).isGreaterThan(0)
89             assertThat(frame.timestampNanos).isGreaterThan(0)
90 
91             val startEvent = listener.onStartedFlow.first()
92             assertThat(startEvent.frameNumber).isNotNull()
93             assertThat(startEvent.frameNumber).isEqualTo(frame.frameNumber)
94             assertThat(startEvent.timestamp).isNotNull()
95             assertThat(startEvent.timestamp.value).isGreaterThan(0)
96             assertThat(startEvent.requestMetadata.repeating).isTrue()
97             assertThat(startEvent.requestMetadata.request.streams).contains(stream.id)
98             assertThat(startEvent.requestMetadata.template).isEqualTo(graphConfig.defaultTemplate)
99 
100             val totalCaptureResultEvent =
101                 withContext(Dispatchers.IO) {
102                     withTimeoutOrNull(timeMillis = 50) { listener.onTotalCaptureResultFlow.first() }
103                 }
104 
105             assertThat(totalCaptureResultEvent).isNull()
106 
107             // Launch the callbacks in a coroutine job to test the behavior of the simulator.
108             val simulateCallbacks = launch {
109                 val resultMetadata = mutableMapOf<CaptureResult.Key<*>, Any>()
110                 // Simulate two partial capture results, and one total capture result.
111                 resultMetadata[CaptureResult.LENS_STATE] = CaptureResult.LENS_STATE_MOVING
112                 frame.simulatePartialCaptureResult(resultMetadata)
113                 delay(10)
114 
115                 resultMetadata[CaptureResult.LENS_APERTURE] = 2.0f
116                 frame.simulatePartialCaptureResult(resultMetadata)
117                 delay(10)
118 
119                 resultMetadata[CaptureResult.FLASH_STATE] = CaptureResult.FLASH_STATE_FIRED
120                 frame.simulateTotalCaptureResult(resultMetadata)
121                 delay(10)
122 
123                 frame.simulateComplete(
124                     resultMetadata,
125                     extraMetadata = mapOf(CaptureResult.LENS_APERTURE to 4.0f)
126                 )
127             }
128 
129             val partialEvent1 = listener.onPartialCaptureResultFlow.first()
130             assertThat(partialEvent1.frameNumber).isEqualTo(frame.frameNumber)
131             assertThat(partialEvent1.frameMetadata.camera).isEqualTo(metadata.camera)
132             assertThat(partialEvent1.frameMetadata[CaptureResult.LENS_STATE]).isEqualTo(1)
133             assertThat(partialEvent1.frameMetadata[CaptureResult.LENS_APERTURE]).isNull()
134             assertThat(partialEvent1.frameMetadata[CaptureResult.FLASH_STATE]).isNull()
135 
136             val partialEvent2 = listener.onPartialCaptureResultFlow.drop(1).first()
137             assertThat(partialEvent2.frameNumber).isEqualTo(frame.frameNumber)
138             assertThat(partialEvent2.frameMetadata.camera).isEqualTo(metadata.camera)
139             assertThat(partialEvent2.frameMetadata[CaptureResult.LENS_STATE]).isEqualTo(1)
140             assertThat(partialEvent2.frameMetadata[CaptureResult.LENS_APERTURE]).isEqualTo(2.0f)
141             assertThat(partialEvent2.frameMetadata[CaptureResult.FLASH_STATE]).isNull()
142 
143             val totalEvent = listener.onTotalCaptureResultFlow.first()
144             assertThat(totalEvent.frameNumber).isEqualTo(frame.frameNumber)
145             assertThat(totalEvent.frameInfo.camera).isEqualTo(metadata.camera)
146             assertThat(totalEvent.frameInfo.metadata[CaptureResult.LENS_STATE]).isEqualTo(1)
147             assertThat(totalEvent.frameInfo.metadata[CaptureResult.LENS_APERTURE]).isEqualTo(2.0f)
148             assertThat(totalEvent.frameInfo.metadata[CaptureResult.FLASH_STATE]).isEqualTo(3)
149 
150             val completedEvent = listener.onCompleteFlow.first()
151             assertThat(completedEvent.frameNumber).isEqualTo(frame.frameNumber)
152             assertThat(completedEvent.frameInfo.camera).isEqualTo(metadata.camera)
153             assertThat(completedEvent.frameInfo.metadata[CaptureResult.LENS_STATE]).isEqualTo(1)
154             assertThat(completedEvent.frameInfo.metadata[CaptureResult.LENS_APERTURE])
155                 .isEqualTo(4.0f)
156             assertThat(completedEvent.frameInfo.metadata[CaptureResult.FLASH_STATE]).isEqualTo(3)
157 
158             simulateCallbacks.join()
159         }
160 
161     @Test
simulatorAbortsRequestsnull162     fun simulatorAbortsRequests() =
163         testScope.runTest {
164             val stream = simulator.streams[streamConfig]!!
165             val listener = FakeRequestListener()
166             val request = Request(streams = listOf(stream.id), listeners = listOf(listener))
167 
168             simulator.acquireSession().use { it.submit(request = request) }
169             simulator.close()
170 
171             val abortedEvent = listener.onAbortedFlow.first()
172             assertThat(abortedEvent.request).isSameInstanceAs(request)
173         }
174 
175     @Test
simulatorCanIssueBufferLossnull176     fun simulatorCanIssueBufferLoss() =
177         testScope.runTest {
178             val stream = simulator.streams[streamConfig]!!
179             val listener = FakeRequestListener()
180             val request = Request(streams = listOf(stream.id), listeners = listOf(listener))
181 
182             simulator.acquireSession().use { it.submit(request = request) }
183 
184             simulator.start()
185             simulator.simulateCameraStarted()
186             simulator.initializeSurfaces()
187             advanceUntilIdle()
188 
189             val frame = simulator.simulateNextFrame()
190             assertThat(frame.request).isSameInstanceAs(request)
191 
192             frame.simulateBufferLoss(stream.id)
193             val lossEvent = listener.onBufferLostFlow.first()
194             assertThat(lossEvent.frameNumber).isEqualTo(frame.frameNumber)
195             assertThat(lossEvent.requestMetadata.request).isSameInstanceAs(request)
196             assertThat(lossEvent.streamId).isEqualTo(stream.id)
197         }
198 
199     @Test
simulatorCanIssueMultipleFramesnull200     fun simulatorCanIssueMultipleFrames() =
201         testScope.runTest {
202             val stream = simulator.streams[streamConfig]!!
203             val listener = FakeRequestListener()
204             val request = Request(streams = listOf(stream.id), listeners = listOf(listener))
205 
206             simulator.acquireSession().use { it.startRepeating(request = request) }
207             simulator.start()
208             simulator.simulateCameraStarted()
209             simulator.initializeSurfaces()
210             advanceUntilIdle()
211 
212             val frame1 = simulator.simulateNextFrame()
213             val frame2 = simulator.simulateNextFrame()
214             val frame3 = simulator.simulateNextFrame()
215 
216             assertThat(frame1).isNotEqualTo(frame2)
217             assertThat(frame2).isNotEqualTo(frame3)
218             assertThat(frame1.request).isSameInstanceAs(request)
219             assertThat(frame2.request).isSameInstanceAs(request)
220             assertThat(frame3.request).isSameInstanceAs(request)
221 
222             val simulateCallbacks = launch {
223                 val resultMetadata = mutableMapOf<CaptureResult.Key<*>, Any>()
224                 resultMetadata[CaptureResult.LENS_STATE] = CaptureResult.LENS_STATE_MOVING
225                 frame1.simulateTotalCaptureResult(resultMetadata)
226                 frame1.simulateComplete(resultMetadata)
227 
228                 delay(15)
229                 frame2.simulateTotalCaptureResult(resultMetadata)
230                 frame2.simulateComplete(resultMetadata)
231 
232                 delay(15)
233                 resultMetadata[CaptureResult.LENS_STATE] = CaptureResult.LENS_STATE_STATIONARY
234                 frame3.simulateTotalCaptureResult(resultMetadata)
235                 frame3.simulateComplete(resultMetadata)
236             }
237 
238             val startEvents =
239                 withTimeout(timeMillis = 250) { listener.onStartedFlow.take(3).toList() }
240             assertThat(startEvents).hasSize(3)
241 
242             val event1 = startEvents[0]
243             val event2 = startEvents[1]
244             val event3 = startEvents[2]
245 
246             // Frame numbers are not equal
247             assertThat(event1.frameNumber).isNotEqualTo(event2.frameNumber)
248             assertThat(event2.frameNumber).isNotEqualTo(event3.frameNumber)
249 
250             // Timestamps are in ascending order
251             assertThat(event3.timestamp.value).isGreaterThan(event2.timestamp.value)
252             assertThat(event2.timestamp.value).isGreaterThan(event1.timestamp.value)
253 
254             // Metadata references the same request.
255             assertThat(event1.requestMetadata.repeating).isTrue()
256             assertThat(event2.requestMetadata.repeating).isTrue()
257             assertThat(event3.requestMetadata.repeating).isTrue()
258             assertThat(event1.requestMetadata.request).isSameInstanceAs(request)
259             assertThat(event2.requestMetadata.request).isSameInstanceAs(request)
260             assertThat(event3.requestMetadata.request).isSameInstanceAs(request)
261 
262             val completeEvents =
263                 withTimeout(timeMillis = 250) { listener.onCompleteFlow.take(3).toList() }
264             assertThat(completeEvents).hasSize(3)
265 
266             val completeEvent1 = completeEvents[0]
267             val completeEvent2 = completeEvents[1]
268             val completeEvent3 = completeEvents[2]
269 
270             assertThat(completeEvent1.frameNumber).isEqualTo(event1.frameNumber)
271             assertThat(completeEvent2.frameNumber).isEqualTo(event2.frameNumber)
272             assertThat(completeEvent3.frameNumber).isEqualTo(event3.frameNumber)
273 
274             assertThat(completeEvent1.frameInfo.metadata[CaptureResult.LENS_STATE])
275                 .isEqualTo(CaptureResult.LENS_STATE_MOVING)
276             assertThat(completeEvent2.frameInfo.metadata[CaptureResult.LENS_STATE])
277                 .isEqualTo(CaptureResult.LENS_STATE_MOVING)
278             assertThat(completeEvent3.frameInfo.metadata[CaptureResult.LENS_STATE])
279                 .isEqualTo(CaptureResult.LENS_STATE_STATIONARY)
280 
281             simulateCallbacks.join()
282         }
283 
284     @Test
simulatorCanSimulateGraphStatenull285     fun simulatorCanSimulateGraphState() =
286         testScope.runTest {
287             assertThat(simulator.graphState.value).isEqualTo(GraphStateStopped)
288 
289             simulator.start()
290             assertThat(simulator.graphState.value).isEqualTo(GraphStateStarting)
291 
292             simulator.simulateCameraStarted()
293             assertThat(simulator.graphState.value).isEqualTo(GraphStateStarted)
294 
295             simulator.stop()
296             assertThat(simulator.graphState.value).isEqualTo(GraphStateStopping)
297 
298             simulator.simulateCameraStopped()
299             assertThat(simulator.graphState.value).isEqualTo(GraphStateStopped)
300         }
301 
302     @Test
simulatorCanSimulateGraphErrornull303     fun simulatorCanSimulateGraphError() =
304         testScope.runTest {
305             val error = GraphStateError(CameraError.ERROR_CAMERA_DEVICE, willAttemptRetry = true)
306 
307             simulator.simulateCameraError(error)
308             // The CameraGraph is stopped at this point, so the errors should be ignored.
309             assertThat(simulator.graphState.value).isEqualTo(GraphStateStopped)
310 
311             simulator.start()
312             simulator.simulateCameraError(error)
313             val graphState = simulator.graphState.value
314             assertThat(graphState).isInstanceOf(GraphStateError::class.java)
315             val graphStateError = graphState as GraphStateError
316             assertThat(graphStateError.cameraError).isEqualTo(error.cameraError)
317             assertThat(graphStateError.willAttemptRetry).isEqualTo(error.willAttemptRetry)
318 
319             simulator.simulateCameraStarted()
320             assertThat(simulator.graphState.value).isEqualTo(GraphStateStarted)
321 
322             simulator.stop()
323             simulator.simulateCameraStopped()
324             simulator.simulateCameraError(error)
325             assertThat(simulator.graphState.value).isEqualTo(GraphStateStopped)
326         }
327 }
328