1 /*
<lambda>null2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.extensions.util
18 
19 import android.content.Context
20 import android.content.Intent
21 import android.graphics.ImageFormat
22 import android.hardware.camera2.CameraCharacteristics
23 import android.os.Build
24 import android.util.Size
25 import androidx.camera.camera2.Camera2Config
26 import androidx.camera.camera2.pipe.integration.CameraPipeConfig
27 import androidx.camera.core.CameraInfo
28 import androidx.camera.core.CameraXConfig
29 import androidx.camera.core.ImageCapture
30 import androidx.camera.core.Preview
31 import androidx.camera.core.impl.CameraInfoInternal
32 import androidx.camera.extensions.ExtensionMode
33 import androidx.camera.extensions.ExtensionsManager
34 import androidx.camera.extensions.impl.AutoImageCaptureExtenderImpl
35 import androidx.camera.extensions.impl.AutoPreviewExtenderImpl
36 import androidx.camera.extensions.impl.BeautyImageCaptureExtenderImpl
37 import androidx.camera.extensions.impl.BeautyPreviewExtenderImpl
38 import androidx.camera.extensions.impl.BokehImageCaptureExtenderImpl
39 import androidx.camera.extensions.impl.BokehPreviewExtenderImpl
40 import androidx.camera.extensions.impl.HdrImageCaptureExtenderImpl
41 import androidx.camera.extensions.impl.HdrPreviewExtenderImpl
42 import androidx.camera.extensions.impl.ImageCaptureExtenderImpl
43 import androidx.camera.extensions.impl.NightImageCaptureExtenderImpl
44 import androidx.camera.extensions.impl.NightPreviewExtenderImpl
45 import androidx.camera.extensions.impl.PreviewExtenderImpl
46 import androidx.camera.extensions.impl.advanced.AdvancedExtenderImpl
47 import androidx.camera.extensions.impl.advanced.AutoAdvancedExtenderImpl
48 import androidx.camera.extensions.impl.advanced.BeautyAdvancedExtenderImpl
49 import androidx.camera.extensions.impl.advanced.BokehAdvancedExtenderImpl
50 import androidx.camera.extensions.impl.advanced.HdrAdvancedExtenderImpl
51 import androidx.camera.extensions.impl.advanced.NightAdvancedExtenderImpl
52 import androidx.camera.extensions.internal.ExtensionVersion
53 import androidx.camera.extensions.internal.ExtensionsUtils
54 import androidx.camera.extensions.internal.Version
55 import androidx.camera.integration.extensions.CameraExtensionsActivity
56 import androidx.camera.integration.extensions.CameraExtensionsActivity.CAMERA2_IMPLEMENTATION_OPTION
57 import androidx.camera.integration.extensions.CameraExtensionsActivity.CAMERA_PIPE_IMPLEMENTATION_OPTION
58 import androidx.camera.integration.extensions.IntentExtraKey
59 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_VIDEO_CAPTURE_ENABLED
60 import androidx.camera.integration.extensions.utils.CameraSelectorUtil.createCameraSelectorById
61 import androidx.camera.integration.extensions.utils.ExtensionModeUtil.AVAILABLE_EXTENSION_MODES
62 import androidx.camera.lifecycle.ProcessCameraProvider
63 import androidx.camera.testing.impl.CameraUtil
64 import androidx.camera.testing.impl.LabTestRule
65 import androidx.test.core.app.ActivityScenario
66 import androidx.test.core.app.ApplicationProvider
67 import com.google.common.truth.Truth.assertThat
68 import java.util.concurrent.TimeUnit
69 import junit.framework.AssertionFailedError
70 import org.junit.Assume.assumeTrue
71 
72 object CameraXExtensionsTestUtil {
73 
74     data class CameraXExtensionTestParams(
75         val implName: String,
76         val cameraXConfig: CameraXConfig,
77         val cameraId: String,
78         val extensionMode: Int,
79     )
80 
81     /** Gets a list of all camera id and extension mode combinations. */
82     @JvmStatic
83     fun getAllCameraIdExtensionModeCombinations(
84         context: Context = ApplicationProvider.getApplicationContext()
85     ): List<CameraXExtensionTestParams> =
86         filterOutUnavailableMode(
87             context,
88             CameraUtil.getBackwardCompatibleCameraIdListOrThrow().flatMap { cameraId ->
89                 AVAILABLE_EXTENSION_MODES.flatMap { extensionMode ->
90                     CAMERAX_CONFIGS.map { config ->
91                         CameraXExtensionTestParams(
92                             config.first,
93                             config.second,
94                             cameraId,
95                             extensionMode
96                         )
97                     }
98                 }
99             }
100         )
101 
102     private fun filterOutUnavailableMode(
103         context: Context,
104         list: List<CameraXExtensionTestParams>
105     ): List<CameraXExtensionTestParams> {
106         var extensionsManager: ExtensionsManager? = null
107         var cameraProvider: ProcessCameraProvider? = null
108         try {
109             cameraProvider = ProcessCameraProvider.getInstance(context)[2, TimeUnit.SECONDS]
110             extensionsManager =
111                 ExtensionsManager.getInstanceAsync(context, cameraProvider)[2, TimeUnit.SECONDS]
112 
113             val result: MutableList<CameraXExtensionTestParams> = mutableListOf()
114             for (item in list) {
115                 val cameraSelector = createCameraSelectorById(item.cameraId)
116                 if (extensionsManager.isExtensionAvailable(cameraSelector, item.extensionMode)) {
117                     result.add(item)
118                 }
119             }
120             return result
121         } catch (e: Exception) {
122             return list
123         } finally {
124             try {
125                 cameraProvider?.shutdownAsync()?.get()
126                 extensionsManager?.shutdown()?.get()
127             } catch (e: Exception) {}
128         }
129     }
130 
131     /**
132      * Gets a list of all camera id and mode combinations. Normal mode and all extension modes will
133      * be included.
134      */
135     @JvmStatic
136     fun getAllCameraIdModeCombinations(): List<Array<Any>> =
137         arrayListOf<Array<Any>>().apply {
138             val allModes = mutableListOf<Int>()
139             allModes.add(0, ExtensionMode.NONE)
140             allModes.addAll(AVAILABLE_EXTENSION_MODES)
141             CameraUtil.getBackwardCompatibleCameraIdListOrThrow().forEach { cameraId ->
142                 allModes.forEach { mode -> add(arrayOf(cameraId, mode)) }
143             }
144         }
145 
146     /**
147      * Creates an [ImageCaptureExtenderImpl] object for specific [ExtensionMode] and camera id.
148      *
149      * @param extensionMode The extension mode for the created object.
150      * @param cameraId The target camera id.
151      * @param cameraCharacteristics The camera characteristics of the target camera.
152      * @return An [ImageCaptureExtenderImpl] object.
153      */
154     @JvmStatic
155     fun createImageCaptureExtenderImpl(
156         @ExtensionMode.Mode extensionMode: Int,
157         cameraId: String,
158         cameraCharacteristics: CameraCharacteristics
159     ): ImageCaptureExtenderImpl =
160         when (extensionMode) {
161             ExtensionMode.HDR -> HdrImageCaptureExtenderImpl()
162             ExtensionMode.BOKEH -> BokehImageCaptureExtenderImpl()
163             ExtensionMode.FACE_RETOUCH -> BeautyImageCaptureExtenderImpl()
164             ExtensionMode.NIGHT -> NightImageCaptureExtenderImpl()
165             ExtensionMode.AUTO -> AutoImageCaptureExtenderImpl()
166             else -> throw AssertionFailedError("No such ImageCapture extender implementation")
167         }.apply { init(cameraId, cameraCharacteristics) }
168 
169     /**
170      * Creates a [PreviewExtenderImpl] object for specific [ExtensionMode] and camera id.
171      *
172      * @param extensionMode The extension mode for the created object.
173      * @param cameraId The target camera id.
174      * @param cameraCharacteristics The camera characteristics of the target camera.
175      * @return A [PreviewExtenderImpl] object.
176      */
177     @JvmStatic
178     fun createPreviewExtenderImpl(
179         @ExtensionMode.Mode extensionMode: Int,
180         cameraId: String,
181         cameraCharacteristics: CameraCharacteristics
182     ): PreviewExtenderImpl =
183         when (extensionMode) {
184             ExtensionMode.HDR -> HdrPreviewExtenderImpl()
185             ExtensionMode.BOKEH -> BokehPreviewExtenderImpl()
186             ExtensionMode.FACE_RETOUCH -> BeautyPreviewExtenderImpl()
187             ExtensionMode.NIGHT -> NightPreviewExtenderImpl()
188             ExtensionMode.AUTO -> AutoPreviewExtenderImpl()
189             else -> throw AssertionFailedError("No such Preview extender implementation")
190         }.apply { init(cameraId, cameraCharacteristics) }
191 
192     /**
193      * Creates a [AdvancedExtenderImpl] object for specific [ExtensionMode] and camera id.
194      *
195      * @param extensionMode The extension mode for the created object.
196      * @param cameraId The target camera id.
197      * @param cameraCharacteristics The camera characteristics of the target camera.
198      * @return A [AdvancedExtenderImpl] object.
199      */
200     @JvmStatic
201     fun createAdvancedExtenderImpl(
202         @ExtensionMode.Mode extensionMode: Int,
203         cameraId: String,
204         cameraInfo: CameraInfo
205     ): AdvancedExtenderImpl =
206         when (extensionMode) {
207             ExtensionMode.HDR -> HdrAdvancedExtenderImpl()
208             ExtensionMode.BOKEH -> BokehAdvancedExtenderImpl()
209             ExtensionMode.FACE_RETOUCH -> BeautyAdvancedExtenderImpl()
210             ExtensionMode.NIGHT -> NightAdvancedExtenderImpl()
211             ExtensionMode.AUTO -> AutoAdvancedExtenderImpl()
212             else -> throw AssertionFailedError("No such Preview extender implementation")
213         }.apply {
214             val cameraCharacteristicsMap =
215                 ExtensionsUtils.getCameraCharacteristicsMap(cameraInfo as CameraInfoInternal)
216             init(cameraId, cameraCharacteristicsMap)
217         }
218 
219     /**
220      * Returns whether the target camera device can support the test for a specific extension mode.
221      */
222     @JvmStatic
223     fun isTargetDeviceAvailableForExtensions(): Boolean {
224         // Runtime version must be non-null if the device supports extensions.
225         if (ExtensionVersion.getRuntimeVersion() == null) {
226             return false
227         }
228 
229         // Skips Cuttlefish device since actually it is not a real marketing device which supports
230         // extensions and it will cause pre-submit failures.
231         return !Build.MODEL.contains("Cuttlefish", true)
232     }
233 
234     @JvmStatic
235     fun assumeExtensionModeSupported(
236         extensionsManager: ExtensionsManager,
237         cameraId: String,
238         extensionMode: Int
239     ) {
240         val cameraIdCameraSelector = createCameraSelectorById(cameraId)
241         assumeTrue(
242             "Extensions mode($extensionMode) not supported",
243             extensionsManager.isExtensionAvailable(cameraIdCameraSelector, extensionMode)
244         )
245     }
246 
247     @JvmStatic
248     fun assumeExtensionModeOutputFormatSupported(
249         cameraProvider: ProcessCameraProvider,
250         extensionsManager: ExtensionsManager,
251         cameraId: String,
252         extensionMode: Int,
253         outputFormat: Int
254     ) {
255         val cameraIdCameraSelector = createCameraSelectorById(cameraId)
256         val extensionsEnabledCameraSelector =
257             extensionsManager.getExtensionEnabledCameraSelector(
258                 cameraIdCameraSelector,
259                 extensionMode
260             )
261         val imageCaptureCapabilities =
262             ImageCapture.getImageCaptureCapabilities(
263                 cameraProvider.getCameraInfo(extensionsEnabledCameraSelector)
264             )
265         assumeTrue(
266             "Extensions mode($extensionMode) does not supported output format $outputFormat still" +
267                 " image capture",
268             imageCaptureCapabilities.supportedOutputFormats.contains(outputFormat)
269         )
270     }
271 
272     @JvmStatic
273     fun assumeAnyExtensionModeSupported(extensionsManager: ExtensionsManager, cameraId: String) {
274         val cameraIdCameraSelector = createCameraSelectorById(cameraId)
275         var anyExtensionModeSupported = false
276 
277         AVAILABLE_EXTENSION_MODES.forEach { mode ->
278             if (extensionsManager.isExtensionAvailable(cameraIdCameraSelector, mode)) {
279                 anyExtensionModeSupported = true
280                 return@forEach
281             }
282         }
283 
284         assumeTrue(anyExtensionModeSupported)
285     }
286 
287     @JvmStatic
288     fun getFirstSupportedExtensionMode(
289         extensionsManager: ExtensionsManager,
290         cameraId: String
291     ): Int {
292         val cameraIdCameraSelector = createCameraSelectorById(cameraId)
293 
294         AVAILABLE_EXTENSION_MODES.forEach { mode ->
295             if (extensionsManager.isExtensionAvailable(cameraIdCameraSelector, mode)) {
296                 return mode
297             }
298         }
299 
300         return ExtensionMode.NONE
301     }
302 
303     @JvmStatic
304     fun isAdvancedExtenderImplemented(): Boolean {
305         if (!isTargetDeviceAvailableForExtensions()) {
306             return false
307         }
308         if (ExtensionVersion.getRuntimeVersion()!! < Version.VERSION_1_2) {
309             return false
310         }
311 
312         return ExtensionVersion.isAdvancedExtenderSupported()
313     }
314 
315     @JvmStatic
316     fun launchCameraExtensionsActivity(
317         cameraId: String,
318         extensionMode: Int,
319         outputFormat: Int = ImageCapture.OUTPUT_FORMAT_JPEG,
320         videoCaptureEnabled: Boolean? = null,
321         deleteCapturedImages: Boolean = true,
322     ): ActivityScenario<CameraExtensionsActivity> {
323         val intent =
324             ApplicationProvider.getApplicationContext<Context>()
325                 .packageManager
326                 .getLaunchIntentForPackage(BASIC_SAMPLE_PACKAGE)
327                 ?.apply {
328                     putExtra(IntentExtraKey.INTENT_EXTRA_KEY_CAMERA_ID, cameraId)
329                     putExtra(IntentExtraKey.INTENT_EXTRA_KEY_EXTENSION_MODE, extensionMode)
330                     putExtra(IntentExtraKey.INTENT_EXTRA_KEY_OUTPUT_FORMAT, outputFormat)
331                     putExtra(
332                         IntentExtraKey.INTENT_EXTRA_KEY_DELETE_CAPTURED_IMAGE,
333                         deleteCapturedImages
334                     )
335                     videoCaptureEnabled?.let {
336                         putExtra(INTENT_EXTRA_KEY_VIDEO_CAPTURE_ENABLED, it)
337                     }
338                     flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK
339                 }
340 
341         val activityScenario: ActivityScenario<CameraExtensionsActivity> =
342             ActivityScenario.launch(intent)
343 
344         activityScenario.waitForInitializationIdle()
345 
346         // Ensure ActivityScenario is cleaned up properly
347         // Wait for PreviewView to become STREAMING state and its IdlingResource to become idle.
348         activityScenario.onActivity {
349             // Checks that CameraExtensionsActivity's current extension mode is correct.
350             assertThat(it.currentExtensionMode).isEqualTo(extensionMode)
351         }
352 
353         return activityScenario
354     }
355 
356     /**
357      * Obtains the ImageCapture supported resolutions according to the provided
358      * ImageCaptureExtenderImpl.
359      */
360     @JvmStatic
361     fun getImageCaptureSupportedResolutions(
362         impl: ImageCaptureExtenderImpl,
363         cameraCharacteristics: CameraCharacteristics
364     ): List<Size> {
365         // Returns the supported resolutions list from ImageCaptureExtenderImpl if it provides the
366         // info.
367         impl.supportedResolutions?.forEach {
368             // When there is no capture processor, the image format is JPEG.
369             // When there is capture processor for post-processing, the image format is YUV_420_888.
370             if (
371                 (impl.captureProcessor == null && it.first == ImageFormat.JPEG) ||
372                     (impl.captureProcessor != null && it.first == ImageFormat.YUV_420_888)
373             ) {
374                 return it.second.toList()
375             }
376         }
377 
378         // Returns the supported resolutions list from StreamConfigurationMap if
379         // ImageCaptureExtenderImpl doesn't provide the info.
380         val map = cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
381         return map!!.getOutputSizes(ImageFormat.JPEG).toList()
382     }
383 
384     /**
385      * Obtains the ImageCapture supported resolutions according to the provided
386      * AdvancedExtenderImpl.
387      */
388     @JvmStatic
389     fun getImageCaptureSupportedResolutions(
390         impl: AdvancedExtenderImpl,
391         cameraId: String,
392         cameraCharacteristics: CameraCharacteristics
393     ): List<Size> {
394         // Returns the supported resolutions list from AdvancedExtenderImpl if it provides the
395         // info.
396         impl.getSupportedCaptureOutputResolutions(cameraId).forEach {
397             if (it.key == ImageFormat.JPEG) {
398                 return it.value
399             }
400         }
401 
402         // Returns the supported resolutions list from StreamConfigurationMap if
403         // ImageCaptureExtenderImpl doesn't provide the info.
404         val map = cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
405         return map!!.getOutputSizes(ImageFormat.JPEG).toList()
406     }
407 
408     @JvmStatic
409     fun getStressTestRepeatingCount() =
410         if (LabTestRule.isInLabTest()) {
411             LAB_STRESS_TEST_OPERATION_REPEAT_COUNT
412         } else {
413             STRESS_TEST_OPERATION_REPEAT_COUNT
414         }
415 
416     /**
417      * Stress test target testing operation count.
418      *
419      * <p>The target testing operation might be:
420      * <ul>
421      * <li> Open and close camera
422      * <li> Open and close capture session
423      * <li> Bind and unbind use cases
424      * <li> Pause and resume lifecycle owner
425      * <li> Switch cameras
426      * <li> Switch extension modes
427      * </ul>
428      */
429     private const val LAB_STRESS_TEST_OPERATION_REPEAT_COUNT = 10
430     private const val STRESS_TEST_OPERATION_REPEAT_COUNT = 3
431 
432     /** Constant to specify that the verification target is [Preview]. */
433     const val VERIFICATION_TARGET_PREVIEW = 0x1
434 
435     /** Constant to specify that the verification target is [ImageCapture]. */
436     const val VERIFICATION_TARGET_IMAGE_CAPTURE = 0x2
437 
438     /** A list of supported implementation options and their respective [CameraXConfig]. */
439     private val CAMERAX_CONFIGS =
440         listOf(
441             Pair(CAMERA2_IMPLEMENTATION_OPTION, Camera2Config.defaultConfig()),
442             Pair(CAMERA_PIPE_IMPLEMENTATION_OPTION, CameraPipeConfig.defaultConfig())
443         )
444 }
445