1 /*
2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.compat
18 
19 import android.hardware.camera2.CameraCaptureSession
20 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession
21 import android.hardware.camera2.CaptureRequest
22 import android.hardware.camera2.params.OutputConfiguration
23 import android.os.Build
24 import android.os.Handler
25 import android.view.Surface
26 import androidx.annotation.RequiresApi
27 import androidx.camera.camera2.pipe.UnsafeWrapper
28 import androidx.camera.camera2.pipe.core.Debug
29 import androidx.camera.camera2.pipe.core.Log
30 import androidx.camera.camera2.pipe.internal.CameraErrorListener
31 import kotlin.reflect.KClass
32 import kotlinx.atomicfu.atomic
33 
34 /**
35  * Interface shim for [CameraCaptureSession] with minor modifications.
36  *
37  * This interface has been modified to correct nullness, adjust exceptions, and to return or produce
38  * wrapper interfaces instead of the native Camera2 types.
39  */
40 internal interface CameraCaptureSessionWrapper : UnsafeWrapper, AutoCloseable {
41 
42     /**
43      * @return The [CameraDeviceWrapper] that created this CameraCaptureSession
44      * @see [CameraCaptureSession.getDevice]
45      */
46     val device: CameraDeviceWrapper
47 
48     /**
49      * @return True if the application can submit reprocess capture requests with this camera
50      *   capture session. false otherwise.
51      * @see [CameraCaptureSession.isReprocessable].
52      */
53     val isReprocessable: Boolean
54 
55     /**
56      * @return The Surface where reprocessing capture requests get the input images from.
57      * @see [CameraCaptureSession.getInputSurface]
58      */
59     val inputSurface: Surface?
60 
61     /** @see [CameraCaptureSession.abortCaptures]. */
abortCapturesnull62     fun abortCaptures(): Boolean
63 
64     /**
65      * @param request The settings for this exposure
66      * @param listener The callback object to notify once this request has been processed.
67      * @return An unique capture sequence id.
68      * @see [CameraCaptureSession.capture].
69      */
70     fun capture(request: CaptureRequest, listener: CameraCaptureSession.CaptureCallback): Int?
71 
72     /**
73      * @param requests A list of CaptureRequest(s) for this sequence of exposures
74      * @param listener A callback object to notify each time one of the requests in the burst has
75      *   been processed.
76      * @return An unique capture sequence id.
77      * @see [CameraCaptureSession.captureBurst].
78      */
79     fun captureBurst(
80         requests: List<CaptureRequest>,
81         listener: CameraCaptureSession.CaptureCallback
82     ): Int?
83 
84     /**
85      * @param requests A list of settings to cycle through indefinitely.
86      * @param listener A callback object to notify each time one of the requests in the repeating
87      *   bursts has finished processing.
88      * @return An unique capture sequence ID.
89      * @see [CameraCaptureSession.setRepeatingBurst]
90      */
91     fun setRepeatingBurst(
92         requests: List<CaptureRequest>,
93         listener: CameraCaptureSession.CaptureCallback
94     ): Int?
95 
96     /**
97      * @param request The request to repeat indefinitely.
98      * @param listener The callback object to notify every time the request finishes processing.
99      * @return An unique capture sequence ID.
100      * @see [CameraCaptureSession.setRepeatingRequest].
101      */
102     fun setRepeatingRequest(
103         request: CaptureRequest,
104         listener: CameraCaptureSession.CaptureCallback
105     ): Int?
106 
107     /** @see [CameraCaptureSession.stopRepeating]. */
108     fun stopRepeating(): Boolean
109 
110     /** Forwards to CameraCaptureSession#finalizeOutputConfigurations */
111     fun finalizeOutputConfigurations(outputConfigs: List<OutputConfigurationWrapper>): Boolean
112 
113     /** @see CameraCaptureSession.StateCallback */
114     interface StateCallback : SessionStateCallback {
115         /** @see CameraCaptureSession.StateCallback.onActive */
116         fun onActive(session: CameraCaptureSessionWrapper)
117 
118         /** @see CameraCaptureSession.StateCallback.onClosed */
119         fun onClosed(session: CameraCaptureSessionWrapper)
120 
121         /** @see CameraCaptureSession.StateCallback.onConfigureFailed */
122         fun onConfigureFailed(session: CameraCaptureSessionWrapper)
123 
124         /** @see CameraCaptureSession.StateCallback.onConfigured */
125         fun onConfigured(session: CameraCaptureSessionWrapper)
126 
127         /** @see CameraCaptureSession.StateCallback.onReady */
128         fun onReady(session: CameraCaptureSessionWrapper)
129 
130         /** @see CameraCaptureSession.StateCallback.onReady */
131         fun onCaptureQueueEmpty(session: CameraCaptureSessionWrapper)
132     }
133 }
134 
135 internal interface CameraConstrainedHighSpeedCaptureSessionWrapper : CameraCaptureSessionWrapper {
136     /**
137      * Forwards to [CameraConstrainedHighSpeedCaptureSession.createHighSpeedRequestList]
138      *
139      * @param request A capture list.
140      * @return A list of high speed requests.
141      */
createHighSpeedRequestListnull142     fun createHighSpeedRequestList(request: CaptureRequest): List<CaptureRequest>?
143 }
144 
145 internal class AndroidCaptureSessionStateCallback(
146     private val device: CameraDeviceWrapper,
147     private val stateCallback: CameraCaptureSessionWrapper.StateCallback,
148     lastStateCallback: SessionStateCallback?,
149     private val cameraErrorListener: CameraErrorListener,
150     private val interopSessionStateCallback: CameraCaptureSession.StateCallback? = null,
151     private val callbackHandler: Handler
152 ) : CameraCaptureSession.StateCallback() {
153     private val _lastStateCallback = atomic(lastStateCallback)
154     private val captureSession = atomic<CameraCaptureSessionWrapper?>(null)
155 
156     override fun onConfigured(session: CameraCaptureSession) {
157         stateCallback.onConfigured(getWrapped(session, cameraErrorListener))
158 
159         // b/249258992 - This is a workaround to ensure previous CameraCaptureSession.StateCallback
160         //   instances receive some kind of "finalization" signal if onClosed is not fired by the
161         //   framework after a subsequent session has been configured.
162         finalizeLastSession()
163         interopSessionStateCallback?.onConfigured(session)
164     }
165 
166     override fun onConfigureFailed(session: CameraCaptureSession) {
167         stateCallback.onConfigureFailed(getWrapped(session, cameraErrorListener))
168         finalizeSession()
169         interopSessionStateCallback?.onConfigureFailed(session)
170     }
171 
172     override fun onReady(session: CameraCaptureSession) {
173         stateCallback.onReady(getWrapped(session, cameraErrorListener))
174         interopSessionStateCallback?.onReady(session)
175     }
176 
177     override fun onActive(session: CameraCaptureSession) {
178         stateCallback.onActive(getWrapped(session, cameraErrorListener))
179         interopSessionStateCallback?.onActive(session)
180     }
181 
182     override fun onClosed(session: CameraCaptureSession) {
183         stateCallback.onClosed(getWrapped(session, cameraErrorListener))
184         finalizeSession()
185         interopSessionStateCallback?.onClosed(session)
186     }
187 
188     override fun onCaptureQueueEmpty(session: CameraCaptureSession) {
189         stateCallback.onCaptureQueueEmpty(getWrapped(session, cameraErrorListener))
190         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
191             Api26Compat.onCaptureQueueEmpty(interopSessionStateCallback, session)
192         }
193     }
194 
195     private fun getWrapped(
196         session: CameraCaptureSession,
197         cameraErrorListener: CameraErrorListener,
198     ): CameraCaptureSessionWrapper {
199         var local = captureSession.value
200         if (local != null) {
201             return local
202         }
203 
204         local = wrapSession(session, cameraErrorListener)
205         if (captureSession.compareAndSet(null, local)) {
206             return local
207         }
208         return captureSession.value!!
209     }
210 
211     private fun wrapSession(
212         session: CameraCaptureSession,
213         cameraErrorListener: CameraErrorListener,
214     ): CameraCaptureSessionWrapper {
215         // Starting in Android P, it's possible for the standard "createCaptureSession" method to
216         // return a CameraConstrainedHighSpeedCaptureSession depending on the configuration. If
217         // this happens, several methods are not allowed, the behavior is different, and interacting
218         // with the session requires several behavior changes for these interactions to work well.
219         return if (
220             Build.VERSION.SDK_INT >= 23 && session is CameraConstrainedHighSpeedCaptureSession
221         ) {
222             AndroidCameraConstrainedHighSpeedCaptureSession(
223                 device,
224                 session,
225                 cameraErrorListener,
226                 callbackHandler
227             )
228         } else {
229             AndroidCameraCaptureSession(device, session, cameraErrorListener, callbackHandler)
230         }
231     }
232 
233     private fun finalizeSession() {
234         finalizeLastSession()
235         stateCallback.onSessionFinalized()
236     }
237 
238     private fun finalizeLastSession() {
239         // Clear out the reference to the previous session, if one was set.
240         val previousSession = _lastStateCallback.getAndSet(null)
241         previousSession?.let { previousSession.onSessionFinalized() }
242     }
243 }
244 
245 internal open class AndroidCameraCaptureSession(
246     override val device: CameraDeviceWrapper,
247     private val cameraCaptureSession: CameraCaptureSession,
248     private val cameraErrorListener: CameraErrorListener,
249     private val callbackHandler: Handler
250 ) : CameraCaptureSessionWrapper {
abortCapturesnull251     override fun abortCaptures(): Boolean =
252         instrumentAndCatch("abortCaptures") { cameraCaptureSession.abortCaptures() } != null
253 
capturenull254     override fun capture(
255         request: CaptureRequest,
256         listener: CameraCaptureSession.CaptureCallback
257     ): Int? =
258         instrumentAndCatch("capture") {
259             cameraCaptureSession.capture(request, listener, callbackHandler)
260         }
261 
captureBurstnull262     override fun captureBurst(
263         requests: List<CaptureRequest>,
264         listener: CameraCaptureSession.CaptureCallback
265     ): Int? =
266         instrumentAndCatch("captureBurst") {
267             cameraCaptureSession.captureBurst(requests, listener, callbackHandler)
268         }
269 
setRepeatingBurstnull270     override fun setRepeatingBurst(
271         requests: List<CaptureRequest>,
272         listener: CameraCaptureSession.CaptureCallback
273     ): Int? =
274         instrumentAndCatch("setRepeatingBurst") {
275             cameraCaptureSession.setRepeatingBurst(requests, listener, callbackHandler)
276         }
277 
setRepeatingRequestnull278     override fun setRepeatingRequest(
279         request: CaptureRequest,
280         listener: CameraCaptureSession.CaptureCallback
281     ): Int? =
282         instrumentAndCatch("setRepeatingRequest") {
283             cameraCaptureSession.setRepeatingRequest(request, listener, callbackHandler)
284         }
285 
stopRepeatingnull286     override fun stopRepeating(): Boolean =
287         instrumentAndCatch("stopRepeating") { cameraCaptureSession.stopRepeating() } != null
288 
289     override val isReprocessable: Boolean
290         get() {
291             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
292                 return Api23Compat.isReprocessable(cameraCaptureSession)
293             }
294             // Reprocessing is not supported  prior to Android M
295             return false
296         }
297 
298     override val inputSurface: Surface?
299         get() {
300             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
301                 return Api23Compat.getInputSurface(cameraCaptureSession)
302             }
303             // Reprocessing is not supported prior to Android M, and a CaptureSession that does not
304             // support reprocessing will have a null input surface on M and beyond.
305             return null
306         }
307 
308     @RequiresApi(26)
finalizeOutputConfigurationsnull309     override fun finalizeOutputConfigurations(
310         outputConfigs: List<OutputConfigurationWrapper>
311     ): Boolean {
312         check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
313             "Attempting to call finalizeOutputConfigurations before O is not supported and may " +
314                 "lead to to unexpected behavior if an application is expects this call to " +
315                 "succeed."
316         }
317 
318         return instrumentAndCatch("finalizeOutputConfigurations") {
319             Api26Compat.finalizeOutputConfigurations(
320                 cameraCaptureSession,
321                 outputConfigs.map { it.unwrapAs(OutputConfiguration::class) }
322             )
323         } != null
324     }
325 
326     @Suppress("UNCHECKED_CAST")
unwrapAsnull327     override fun <T : Any> unwrapAs(type: KClass<T>): T? =
328         when (type) {
329             CameraCaptureSession::class -> cameraCaptureSession as T?
330             else -> null
331         }
332 
closenull333     override fun close() {
334         return cameraCaptureSession.close()
335     }
336 
337     /** Utility function to trace, measure, and suppress exceptions for expensive method calls. */
338     @Throws(ObjectUnavailableException::class)
instrumentAndCatchnull339     internal inline fun <T> instrumentAndCatch(fnName: String, crossinline block: () -> T) =
340         Debug.instrument("CXCP#$fnName-${device.cameraId.value}") {
341             catchAndReportCameraExceptions(device.cameraId, cameraErrorListener, block)
342         }
343 }
344 
345 /**
346  * An implementation of [CameraConstrainedHighSpeedCaptureSessionWrapper] forwards calls to a real
347  * [CameraConstrainedHighSpeedCaptureSession].
348  */
349 @RequiresApi(23)
350 internal class AndroidCameraConstrainedHighSpeedCaptureSession
351 internal constructor(
352     device: CameraDeviceWrapper,
353     private val session: CameraConstrainedHighSpeedCaptureSession,
354     cameraErrorListener: CameraErrorListener,
355     callbackHandler: Handler
356 ) :
357     AndroidCameraCaptureSession(device, session, cameraErrorListener, callbackHandler),
358     CameraConstrainedHighSpeedCaptureSessionWrapper {
createHighSpeedRequestListnull359     override fun createHighSpeedRequestList(request: CaptureRequest): List<CaptureRequest>? =
360         try {
361             // This converts a single CaptureRequest into a list of CaptureRequest(s) that must be
362             // submitted together during high speed recording.
363             Debug.trace("CXCP#createHighSpeedRequestList") {
364                 session.createHighSpeedRequestList(request)
365             }
366         } catch (e: IllegalStateException) {
367 
368             // b/111749845: If the camera device is closed before calling
369             // createHighSpeedRequestList it may throw an [IllegalStateException]. Since this can
370             // happen during normal operation of the camera, log and rethrow the error as a standard
371             // exception that can be ignored.
<lambda>null372             Log.warn { "Failed to createHighSpeedRequestList. $device may be closed." }
373             null
374         } catch (e: IllegalArgumentException) {
375 
376             // b/111749845: If the surface (such as the viewfinder) is destroyed before calling
377             // createHighSpeedRequestList it may throw an [IllegalArgumentException]. Since this can
378             // happen during normal operation of the camera, log and rethrow the error as a standard
379             // exception that can be ignored.
<lambda>null380             Log.warn {
381                 "Failed to createHighSpeedRequestList from $device because the output surface" +
382                     " was destroyed before calling createHighSpeedRequestList."
383             }
384             null
385         } catch (e: UnsupportedOperationException) {
386 
387             // b/358592149: When a high speed session is closed, and then another high speed session
388             // is opened, the resources from the previous session might not be available yet.
389             // Since Camera2CaptureSequenceProcessor will try to create the session again, log
390             // and rethrow the error as a standard exception that can be ignored.
<lambda>null391             Log.warn {
392                 "Failed to createHighSpeedRequestList from $device because the output surface" +
393                     " was not available."
394             }
395             null
396         }
397 
398     @Suppress("UNCHECKED_CAST")
unwrapAsnull399     override fun <T : Any> unwrapAs(type: KClass<T>): T? =
400         when (type) {
401             CameraConstrainedHighSpeedCaptureSession::class -> session as T?
402             else -> super.unwrapAs(type)
403         }
404 }
405