1 /*
<lambda>null2  * Copyright 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.antelope.cameracontrollers
18 
19 import android.graphics.SurfaceTexture
20 import android.hardware.camera2.CameraCaptureSession
21 import android.hardware.camera2.CameraDevice
22 import android.util.Log
23 import android.view.Surface
24 import android.view.ViewGroup
25 import androidx.annotation.OptIn
26 import androidx.camera.camera2.interop.Camera2Interop
27 import androidx.camera.camera2.interop.ExperimentalCamera2Interop
28 import androidx.camera.core.CameraSelector
29 import androidx.camera.core.ImageCapture
30 import androidx.camera.core.Preview
31 import androidx.camera.core.impl.utils.executor.CameraXExecutors
32 import androidx.camera.integration.antelope.CameraParams
33 import androidx.camera.integration.antelope.CameraXImageAvailableListener
34 import androidx.camera.integration.antelope.CustomLifecycle
35 import androidx.camera.integration.antelope.FocusMode
36 import androidx.camera.integration.antelope.MainActivity
37 import androidx.camera.integration.antelope.MainActivity.Companion.logd
38 import androidx.camera.integration.antelope.PrefHelper
39 import androidx.camera.integration.antelope.TestConfig
40 import androidx.camera.integration.antelope.TestType
41 import androidx.camera.lifecycle.ProcessCameraProvider
42 import androidx.concurrent.futures.await
43 import androidx.core.util.Consumer
44 import androidx.lifecycle.LifecycleOwner
45 import kotlinx.coroutines.DelicateCoroutinesApi
46 import kotlinx.coroutines.Dispatchers
47 import kotlinx.coroutines.GlobalScope
48 import kotlinx.coroutines.launch
49 
50 /**
51  * Opens the camera using the Camera X API and starts the open counter. The open call will complete
52  * in the DeviceStateCallback asynchronously. For switch tests, the camera id will be swizzling so
53  * the original camera id is saved.
54  *
55  * CameraX manages its lifecycle internally, for the purpose of repeated testing, Antelope uses a
56  * custom lifecycle to allow for starting new tests cleanly which is started here.
57  *
58  * All the needed Cmaera X use cases should be bound before starting the lifecycle. Depending on the
59  * test, bind either the preview case, or both the preview and image capture case.
60  */
61 @kotlin.OptIn(DelicateCoroutinesApi::class)
62 internal fun cameraXOpenCamera(
63     activity: MainActivity,
64     params: CameraParams,
65     testConfig: TestConfig
66 ) {
67 
68     try {
69         // TODO make the switch test methodology more robust and handle physical cameras
70         // Currently we swap out the ids behind the scenes
71         // This requires to save the actual camera id for after the test
72         if (
73             (testConfig.currentRunningTest == TestType.SWITCH_CAMERA) ||
74                 (testConfig.currentRunningTest == TestType.MULTI_SWITCH)
75         ) {
76             testConfig.switchTestRealCameraId = params.id // Save the actual camera ID
77             params.id = testConfig.switchTestCurrentCamera
78         }
79 
80         params.cameraXDeviceStateCallback = CameraXDeviceStateCallback(params, activity, testConfig)
81         params.cameraXPreviewSessionStateCallback =
82             CameraXPreviewSessionStateCallback(activity, params, testConfig)
83 
84         if (
85             params.cameraXDeviceStateCallback != null &&
86                 params.cameraXPreviewSessionStateCallback != null
87         ) {
88             params.cameraXPreviewBuilder =
89                 cameraXPreviewUseCaseBuilder(
90                     testConfig.focusMode,
91                     params.cameraXDeviceStateCallback!!,
92                     params.cameraXPreviewSessionStateCallback!!
93                 )
94         }
95 
96         if (!params.cameraXLifecycle.isFinished()) {
97             logd("Lifecycle not finished, finishing it.")
98             params.cameraXLifecycle.pauseAndStop()
99             params.cameraXLifecycle.finish()
100         }
101         params.cameraXLifecycle = CustomLifecycle()
102 
103         val lifecycleOwner: LifecycleOwner = params.cameraXLifecycle
104         val previewUseCase = params.cameraXPreviewBuilder.build()
105 
106         // Set preview to observe the surface texture
107         activity.runOnUiThread {
108             previewUseCase.setSurfaceProvider { surfaceRequest ->
109                 // Create the SurfaceTexture and Surface
110                 val surfaceTexture = SurfaceTexture(0)
111                 surfaceTexture.setDefaultBufferSize(
112                     surfaceRequest.resolution.width,
113                     surfaceRequest.resolution.height
114                 )
115                 surfaceTexture.detachFromGLContext()
116                 val surface = Surface(surfaceTexture)
117 
118                 // Attach the SurfaceTexture on the TextureView
119                 if (!isCameraSurfaceTextureReleased(surfaceTexture)) {
120                     val viewGroup = params.cameraXPreviewTexture?.parent as ViewGroup
121                     viewGroup.removeView(params.cameraXPreviewTexture)
122                     viewGroup.addView(params.cameraXPreviewTexture)
123                     params.cameraXPreviewTexture?.setSurfaceTexture(surfaceTexture)
124                 }
125 
126                 // Surface provided to camera for producing buffers into and
127                 // Release the SurfaceTexture and Surface once camera is done with it
128                 surfaceRequest.provideSurface(
129                     surface,
130                     CameraXExecutors.directExecutor(),
131                     Consumer {
132                         surface.release()
133                         surfaceTexture.release()
134                     }
135                 )
136             }
137         }
138 
139         // TODO: As of 0.3.0 CameraX can only use front and back cameras.
140         //  Update in future versions
141         val cameraProviderFuture = ProcessCameraProvider.getInstance(activity)
142         val cameraXcameraID =
143             if (params.id == "0") {
144                 CameraSelector.LENS_FACING_BACK
145             } else {
146                 CameraSelector.LENS_FACING_FRONT
147             }
148         val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraXcameraID).build()
149         when (testConfig.currentRunningTest) {
150             //  Only the preview is required
151             TestType.PREVIEW,
152             TestType.SWITCH_CAMERA,
153             TestType.MULTI_SWITCH -> {
154                 params.timer.openStart = System.currentTimeMillis()
155                 GlobalScope.launch(Dispatchers.Main) {
156                     val cameraProvider = cameraProviderFuture.await()
157                     cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, previewUseCase)
158                     params.cameraXLifecycle.start()
159                 }
160             }
161             else -> {
162                 // Both preview and image capture are needed
163                 params.cameraXCaptureSessionCallback =
164                     CameraXCaptureSessionCallback(activity, params, testConfig)
165 
166                 if (
167                     params.cameraXDeviceStateCallback != null &&
168                         params.cameraXCaptureSessionCallback != null
169                 ) {
170                     params.cameraXCaptureBuilder =
171                         cameraXImageCaptureUseCaseBuilder(
172                             testConfig.focusMode,
173                             params.cameraXDeviceStateCallback!!,
174                             params.cameraXCaptureSessionCallback!!
175                         )
176                 }
177 
178                 params.cameraXImageCaptureUseCase = params.cameraXCaptureBuilder.build()
179 
180                 params.timer.openStart = System.currentTimeMillis()
181 
182                 GlobalScope.launch(Dispatchers.Main) {
183                     val cameraProvider = cameraProviderFuture.await()
184                     cameraProvider.bindToLifecycle(
185                         lifecycleOwner,
186                         cameraSelector,
187                         previewUseCase,
188                         params.cameraXImageCaptureUseCase
189                     )
190                     params.cameraXLifecycle.start()
191                 }
192             }
193         }
194     } catch (e: Exception) {
195         MainActivity.logd("cameraXOpenCamera exception: " + params.id)
196         e.printStackTrace()
197     }
198 }
199 
200 /** End Camera X custom lifecycle, unbind use cases, and start timing the camera close. */
201 @kotlin.OptIn(DelicateCoroutinesApi::class)
closeCameraXnull202 internal fun closeCameraX(activity: MainActivity, params: CameraParams, testConfig: TestConfig) {
203     logd("In closecameraX, camera: " + params.id + ",  test: " + testConfig.currentRunningTest)
204 
205     params.timer.cameraCloseStart = System.currentTimeMillis()
206 
207     if (!params.cameraXLifecycle.isFinished()) {
208         params.cameraXLifecycle.pauseAndStop()
209         params.cameraXLifecycle.finish()
210 
211         // CameraX calls need to be on the main thread
212         val cameraProviderFuture = ProcessCameraProvider.getInstance(activity)
213         GlobalScope.launch(Dispatchers.Main) {
214             val cameraProvider = cameraProviderFuture.await()
215             cameraProvider.unbindAll()
216         }
217     }
218     if (
219         (testConfig.currentRunningTest == TestType.SWITCH_CAMERA) ||
220             (testConfig.currentRunningTest == TestType.MULTI_SWITCH)
221     ) {
222         params.id = testConfig.switchTestRealCameraId // Restore the actual camera ID
223     }
224 
225     params.isOpen = false
226 }
227 
228 /** Proceed to take and measure a still image capture. */
cameraXTakePicturenull229 internal fun cameraXTakePicture(
230     activity: MainActivity,
231     params: CameraParams,
232     testConfig: TestConfig
233 ) {
234     if (params.cameraXLifecycle.isFinished()) {
235         cameraXAbort(activity, params, testConfig)
236         return
237     }
238 
239     logd("CameraX TakePicture: capture start.")
240 
241     // Pause in multi-captures to make sure HDR routines don't get overloaded
242     logd(
243         "CameraX TakePicture. Pausing for " +
244             PrefHelper.getPreviewBuffer(activity) +
245             "ms to let preview run."
246     )
247 
248     params.timer.previewFillStart = System.currentTimeMillis()
249     Thread.sleep(PrefHelper.getPreviewBuffer(activity))
250     params.timer.previewFillEnd = System.currentTimeMillis()
251 
252     params.timer.captureStart = System.currentTimeMillis()
253     params.timer.autofocusStart = System.currentTimeMillis()
254     params.timer.autofocusEnd = System.currentTimeMillis()
255 
256     logd("Capture timer started: " + params.timer.captureStart)
257     activity.runOnUiThread {
258         params.cameraXImageCaptureUseCase.takePicture(
259             CameraXExecutors.mainThreadExecutor(),
260             CameraXImageAvailableListener(activity, params, testConfig)
261         )
262     }
263 }
264 
265 /** An abort request has been received. Abandon everything */
cameraXAbortnull266 internal fun cameraXAbort(activity: MainActivity, params: CameraParams, testConfig: TestConfig) {
267     closeCameraX(activity, params, testConfig)
268     return
269 }
270 
271 /**
272  * Try to determine if a SurfaceTexture is released.
273  *
274  * Prior to SDK 26 there was not built in mechanism for this. This method relies on expected
275  * exceptions being thrown if a released SurfaceTexture is updated.
276  */
isCameraSurfaceTextureReleasednull277 private fun isCameraSurfaceTextureReleased(texture: SurfaceTexture): Boolean {
278     var released = false
279 
280     if (26 <= android.os.Build.VERSION.SDK_INT) {
281         released = texture.isReleased
282     } else {
283         // WARNING: This relies on some implementation details of the SurfaceTexture native code.
284         // If the SurfaceTexture is released, we should get a RuntimeException. If not, we should
285         // get an IllegalStateException since we are not in the same EGL context as the camera.
286         var exception: Exception? = null
287         try {
288             texture.updateTexImage()
289         } catch (e: IllegalStateException) {
290             logd("in isCameraSurfaceTextureReleased: IllegalStateException: " + e.message)
291             exception = e
292             released = false
293         } catch (e: RuntimeException) {
294             logd("in isCameraSurfaceTextureReleased: RuntimeException: " + e.message)
295             exception = e
296             released = true
297         }
298 
299         if (!released && exception == null) {
300             throw RuntimeException("Unable to determine if SurfaceTexture is released")
301         }
302     }
303 
304     logd("The camera texture is: " + if (released) "RELEASED" else "NOT RELEASED")
305 
306     return released
307 }
308 
309 /** Setup the Camera X preview use case */
310 @OptIn(ExperimentalCamera2Interop::class)
cameraXPreviewUseCaseBuildernull311 private fun cameraXPreviewUseCaseBuilder(
312     focusMode: FocusMode,
313     deviceStateCallback: CameraDevice.StateCallback,
314     sessionCaptureStateCallback: CameraCaptureSession.StateCallback
315 ): Preview.Builder {
316 
317     val configBuilder = Preview.Builder()
318     Camera2Interop.Extender(configBuilder)
319         .setDeviceStateCallback(deviceStateCallback)
320         .setSessionStateCallback(sessionCaptureStateCallback)
321     // TODO(b/142915154): Enables focusMode when CameraX support direct AF mode setting.
322 
323     // Prints a log to suppress "fix Parameter 'focusMode' is never used" build error"
324     Log.d("Antelope", "focusMode($focusMode) Not enabled.")
325     return configBuilder
326 }
327 
328 /** Setup the Camera X image capture use case */
329 @OptIn(ExperimentalCamera2Interop::class)
cameraXImageCaptureUseCaseBuildernull330 private fun cameraXImageCaptureUseCaseBuilder(
331     focusMode: FocusMode,
332     deviceStateCallback: CameraDevice.StateCallback,
333     sessionCaptureCallback: CameraCaptureSession.CaptureCallback
334 ): ImageCapture.Builder {
335 
336     val configBuilder =
337         ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY)
338     Camera2Interop.Extender(configBuilder)
339         .setDeviceStateCallback(deviceStateCallback)
340         .setSessionCaptureCallback(sessionCaptureCallback)
341     // TODO(b/142915154): Enables focusMode when CameraX support direct AF mode setting.
342 
343     // Prints a log to suppress "fix Parameter 'focusMode' is never used" build error"
344     Log.d("Antelope", "focusMode($focusMode) Not enabled.")
345     return configBuilder
346 }
347