1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.extensions.util
18 
19 import android.content.Context
20 import android.graphics.ImageFormat
21 import android.graphics.SurfaceTexture
22 import android.hardware.camera2.CameraAccessException
23 import android.hardware.camera2.CameraCharacteristics
24 import android.hardware.camera2.CameraDevice
25 import android.hardware.camera2.CameraExtensionCharacteristics
26 import android.hardware.camera2.CameraExtensionSession
27 import android.hardware.camera2.CameraManager
28 import android.hardware.camera2.CaptureRequest
29 import android.hardware.camera2.TotalCaptureResult
30 import android.hardware.camera2.params.ExtensionSessionConfiguration
31 import android.hardware.camera2.params.OutputConfiguration
32 import android.media.Image
33 import android.media.ImageReader
34 import android.os.Build
35 import android.os.Handler
36 import android.os.Looper
37 import android.view.Surface
38 import androidx.annotation.RequiresApi
39 import androidx.camera.core.impl.utils.executor.CameraXExecutors
40 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil
41 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.AVAILABLE_CAMERA2_EXTENSION_MODES
42 import androidx.camera.integration.extensions.utils.CameraIdExtensionModePair
43 import androidx.camera.testing.impl.CameraUtil
44 import androidx.camera.testing.impl.LabTestRule
45 import androidx.camera.testing.impl.SurfaceTextureProvider
46 import androidx.concurrent.futures.await
47 import androidx.test.core.app.ApplicationProvider
48 import com.google.common.truth.Truth.assertThat
49 import kotlinx.coroutines.CompletableDeferred
50 import org.junit.Assume.assumeTrue
51 
52 @RequiresApi(31)
53 object Camera2ExtensionsTestUtil {
54     private const val LAB_STRESS_TEST_OPERATION_REPEAT_COUNT = 10
55     private const val STRESS_TEST_OPERATION_REPEAT_COUNT = 3
56     const val EXTENSION_NOT_FOUND = -1
57 
58     /** Returns whether the target device is excluded for extensions test */
59     @JvmStatic
60     fun isTargetDeviceExcludedForExtensionsTest(): Boolean {
61         // Skips Cuttlefish device since actually it is not a real marketing device which supports
62         // extensions and it will cause pre-submit failures.
63         return !Build.MODEL.contains("Cuttlefish", true)
64     }
65 
66     @JvmStatic
67     fun getStressTestRepeatingCount() =
68         if (LabTestRule.isInLabTest()) {
69             LAB_STRESS_TEST_OPERATION_REPEAT_COUNT
70         } else {
71             STRESS_TEST_OPERATION_REPEAT_COUNT
72         }
73 
74     /** Gets a list of all camera id and extension mode combinations. */
75     @JvmStatic
76     fun getAllCameraIdExtensionModeCombinations(
77         context: Context = ApplicationProvider.getApplicationContext()
78     ): List<CameraIdExtensionModePair> =
79         CameraUtil.getBackwardCompatibleCameraIdListOrThrow()
80             .flatMap { cameraId ->
81                 AVAILABLE_CAMERA2_EXTENSION_MODES.map { extensionMode ->
82                     CameraIdExtensionModePair(cameraId, extensionMode)
83                 }
84             }
85             .filter {
86                 Camera2ExtensionsUtil.isCamera2ExtensionModeSupported(
87                     context,
88                     it.cameraId,
89                     it.extensionMode
90                 )
91             }
92 
93     suspend fun assertCanOpenExtensionsSession(
94         cameraManager: CameraManager,
95         cameraId: String,
96         extensionMode: Int,
97         verifyOutput: Boolean = false
98     ) {
99         val extensionsCharacteristics = cameraManager.getCameraExtensionCharacteristics(cameraId)
100         assumeCameraExtensionSupported(extensionMode, extensionsCharacteristics)
101 
102         // Preview surface
103         val previewSize =
104             extensionsCharacteristics
105                 .getExtensionSupportedSizes(extensionMode, SurfaceTexture::class.java)
106                 .maxBy { it.width * it.height }
107         val deferredPreviewFrame = CompletableDeferred<SurfaceTexture>()
108 
109         // Some OEM requires frames drain (updateTexImage being invoked) in SurfaceTexture,
110         // otherwise it might cause still capture to fail.
111         val surfaceTextureHolder =
112             SurfaceTextureProvider.createAutoDrainingSurfaceTextureAsync(
113                     previewSize.width,
114                     previewSize.height,
115                     {
116                         if (!deferredPreviewFrame.isCompleted) {
117                             deferredPreviewFrame.complete(it)
118                         }
119                     }
120                 )
121                 .await()
122         val previewSurface = Surface(surfaceTextureHolder.surfaceTexture)
123 
124         // Still capture surface
125         val imageReader = createCaptureImageReader(extensionsCharacteristics, extensionMode)
126         val captureSurface = imageReader.surface
127 
128         val cameraDevice = openCameraDevice(cameraManager, cameraId)
129         val outputConfigurationPreview = OutputConfiguration(previewSurface)
130         val outputConfigurationCapture = OutputConfiguration(captureSurface)
131         val extensionSession =
132             openExtensionSession(
133                 cameraDevice,
134                 extensionMode,
135                 listOf(outputConfigurationPreview, outputConfigurationCapture)
136             )
137         assertThat(extensionSession).isNotNull()
138 
139         val builder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
140         builder.addTarget(previewSurface)
141 
142         extensionSession.setRepeatingRequest(
143             builder.build(),
144             CameraXExecutors.ioExecutor(),
145             object : CameraExtensionSession.ExtensionCaptureCallback() {
146                 override fun onCaptureSequenceCompleted(
147                     session: CameraExtensionSession,
148                     sequenceId: Int
149                 ) {}
150 
151                 override fun onCaptureStarted(
152                     session: CameraExtensionSession,
153                     request: CaptureRequest,
154                     timestamp: Long
155                 ) {}
156 
157                 override fun onCaptureProcessStarted(
158                     session: CameraExtensionSession,
159                     request: CaptureRequest
160                 ) {}
161 
162                 override fun onCaptureFailed(
163                     session: CameraExtensionSession,
164                     request: CaptureRequest
165                 ) {}
166 
167                 override fun onCaptureSequenceAborted(
168                     session: CameraExtensionSession,
169                     sequenceId: Int
170                 ) {}
171 
172                 override fun onCaptureResultAvailable(
173                     session: CameraExtensionSession,
174                     request: CaptureRequest,
175                     result: TotalCaptureResult
176                 ) {}
177             }
178         )
179 
180         if (verifyOutput) {
181             deferredPreviewFrame.await()
182             val image = takePicture(cameraDevice, extensionSession, imageReader)
183             assertThat(image).isNotNull()
184             image!!.close()
185         }
186 
187         extensionSession.close()
188         cameraDevice.close()
189         imageReader.close()
190         previewSurface.release()
191         captureSurface.release()
192         surfaceTextureHolder.close()
193     }
194 
195     /**
196      * Check if the device supports the [extensionMode] and other extension specific characteristics
197      * required for testing. Halt the test if any criteria is not satisfied.
198      */
199     fun assumeCameraExtensionSupported(
200         extensionMode: Int,
201         extensionsCharacteristics: CameraExtensionCharacteristics
202     ) {
203         assumeTrue(extensionsCharacteristics.supportedExtensions.contains(extensionMode))
204         assumeTrue(
205             extensionsCharacteristics
206                 .getExtensionSupportedSizes(extensionMode, SurfaceTexture::class.java)
207                 .isNotEmpty()
208         )
209         assumeTrue(
210             extensionsCharacteristics
211                 .getExtensionSupportedSizes(extensionMode, ImageFormat.JPEG)
212                 .isNotEmpty()
213         )
214     }
215 
216     fun createCaptureImageReader(
217         extensionsCharacteristics: CameraExtensionCharacteristics,
218         extensionMode: Int
219     ): ImageReader {
220         val captureSize =
221             extensionsCharacteristics
222                 .getExtensionSupportedSizes(extensionMode, ImageFormat.JPEG)
223                 .maxBy { it.width * it.height }
224         return ImageReader.newInstance(captureSize.width, captureSize.height, ImageFormat.JPEG, 2)
225     }
226 
227     /** Open the camera device and return the [CameraDevice] instance. */
228     suspend fun openCameraDevice(cameraManager: CameraManager, cameraId: String): CameraDevice {
229         val deferred = CompletableDeferred<CameraDevice>()
230         cameraManager.openCamera(
231             cameraId,
232             CameraXExecutors.ioExecutor(),
233             object : CameraDevice.StateCallback() {
234                 override fun onOpened(cameraDevice: CameraDevice) {
235                     deferred.complete(cameraDevice)
236                 }
237 
238                 override fun onDisconnected(cameraDevice: CameraDevice) {
239                     deferred.completeExceptionally(RuntimeException("Camera Disconnected"))
240                 }
241 
242                 override fun onError(cameraDevice: CameraDevice, error: Int) {
243                     deferred.completeExceptionally(
244                         RuntimeException("Camera onError(error=$cameraDevice)")
245                     )
246                 }
247             }
248         )
249         return deferred.await()
250     }
251 
252     /** Open the [CameraExtensionSession] and return the instance. */
253     suspend fun openExtensionSession(
254         cameraDevice: CameraDevice,
255         extensionMode: Int,
256         outputConfigs: List<OutputConfiguration>
257     ): CameraExtensionSession {
258         val deferred = CompletableDeferred<CameraExtensionSession>()
259 
260         val extensionSessionConfiguration =
261             ExtensionSessionConfiguration(
262                 extensionMode,
263                 outputConfigs,
264                 CameraXExecutors.ioExecutor(),
265                 object : CameraExtensionSession.StateCallback() {
266                     override fun onConfigured(cameraExtensionSession: CameraExtensionSession) {
267                         deferred.complete(cameraExtensionSession)
268                     }
269 
270                     override fun onConfigureFailed(session: CameraExtensionSession) {
271                         deferred.completeExceptionally(RuntimeException("onConfigureFailed"))
272                     }
273 
274                     override fun onClosed(session: CameraExtensionSession) {}
275                 }
276             )
277         cameraDevice.createExtensionSession(extensionSessionConfiguration)
278         return deferred.await()
279     }
280 
281     /**
282      * Take a picture with the provided [session] and output the contents to the [imageReader]. The
283      * latest image written to the [imageReader] is returned.
284      */
285     suspend fun takePicture(
286         cameraDevice: CameraDevice,
287         session: CameraExtensionSession,
288         imageReader: ImageReader
289     ): Image? {
290         val builder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE)
291         builder.addTarget(imageReader.surface)
292         val deferredCapture = CompletableDeferred<Int>()
293         session.capture(
294             builder.build(),
295             CameraXExecutors.ioExecutor(),
296             object : CameraExtensionSession.ExtensionCaptureCallback() {
297                 override fun onCaptureSequenceCompleted(
298                     session: CameraExtensionSession,
299                     sequenceId: Int
300                 ) {
301                     deferredCapture.complete(sequenceId)
302                 }
303 
304                 override fun onCaptureStarted(
305                     session: CameraExtensionSession,
306                     request: CaptureRequest,
307                     timestamp: Long
308                 ) {}
309 
310                 override fun onCaptureProcessStarted(
311                     session: CameraExtensionSession,
312                     request: CaptureRequest
313                 ) {}
314 
315                 override fun onCaptureFailed(
316                     session: CameraExtensionSession,
317                     request: CaptureRequest
318                 ) {
319                     deferredCapture.completeExceptionally(RuntimeException("onCaptureFailed"))
320                 }
321 
322                 override fun onCaptureSequenceAborted(
323                     session: CameraExtensionSession,
324                     sequenceId: Int
325                 ) {
326                     deferredCapture.completeExceptionally(
327                         RuntimeException("onCaptureSequenceAborted")
328                     )
329                 }
330             }
331         )
332 
333         val deferredImage = CompletableDeferred<Image?>()
334         imageReader.setOnImageAvailableListener(
335             {
336                 val image = imageReader.acquireNextImage()
337                 deferredImage.complete(image)
338             },
339             Handler(Looper.getMainLooper())
340         )
341         deferredCapture.await()
342         return deferredImage.await()
343     }
344 
345     fun findNextSupportedCameraId(
346         context: Context,
347         currentCameraId: String,
348         extensionsMode: Int
349     ): String? {
350         val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
351         try {
352             val supportedCameraIdList =
353                 cameraManager.cameraIdList.filter {
354                     val characteristics = cameraManager.getCameraCharacteristics(it)
355                     val backwardCompatible =
356                         characteristics
357                             .get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES)!!
358                             .toList()
359                             .contains(
360                                 CameraCharacteristics
361                                     .REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE
362                             )
363                     if (!backwardCompatible) {
364                         return@filter false
365                     }
366                     val extCharacteristics = cameraManager.getCameraExtensionCharacteristics((it))
367                     return@filter extCharacteristics.supportedExtensions.contains(extensionsMode)
368                 }
369 
370             if (supportedCameraIdList.size <= 1) {
371                 return null
372             }
373             val currentIndex = supportedCameraIdList.indexOf(currentCameraId)
374             return supportedCameraIdList[(currentIndex + 1) % supportedCameraIdList.size]
375         } catch (e: CameraAccessException) {}
376         return null
377     }
378 
379     fun findNextEffectMode(context: Context, cameraId: String, extensionsMode: Int): Int {
380         val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
381         try {
382             val characteristics = cameraManager.getCameraExtensionCharacteristics((cameraId))
383             val supportedExtensions =
384                 ArrayList(characteristics.supportedExtensions).apply { sort() }
385             val currentIndex = supportedExtensions.indexOf(extensionsMode)
386             if (currentIndex >= 0 && supportedExtensions.size > 1) {
387                 return supportedExtensions[(currentIndex + 1) % supportedExtensions.size]
388             }
389         } catch (e: CameraAccessException) {}
390         return EXTENSION_NOT_FOUND
391     }
392 }
393