1 /*
<lambda>null2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.compat
18 
19 import android.hardware.camera2.CameraCaptureSession
20 import android.hardware.camera2.CameraExtensionSession
21 import android.hardware.camera2.CaptureRequest
22 import android.hardware.camera2.TotalCaptureResult
23 import android.os.Build
24 import android.view.Surface
25 import androidx.annotation.RequiresApi
26 import androidx.camera.camera2.pipe.FrameNumber
27 import androidx.camera.camera2.pipe.UnsafeWrapper
28 import androidx.camera.camera2.pipe.core.Log
29 import androidx.camera.camera2.pipe.internal.CameraErrorListener
30 import java.util.LinkedList
31 import java.util.Queue
32 import java.util.concurrent.Executor
33 import kotlin.reflect.KClass
34 import kotlinx.atomicfu.AtomicLong
35 import kotlinx.atomicfu.atomic
36 
37 /**
38  * Interface shim for [CameraExtensionSession] with minor modifications.
39  *
40  * This interface has been modified to correct nullness, adjust exceptions, and to return or produce
41  * wrapper interfaces instead of the native Camera2 types.
42  */
43 internal interface CameraExtensionSessionWrapper :
44     CameraCaptureSessionWrapper, UnsafeWrapper, AutoCloseable {
45 
46     /**
47      * @return The [CameraDeviceWrapper] that created this CameraExtensionSession
48      * @see [CameraExtensionSession.getDevice]
49      */
50     override val device: CameraDeviceWrapper
51 
52     /** @see [CameraExtensionSession.stopRepeating]. */
53     override fun stopRepeating(): Boolean
54 
55     /** @see CameraExtensionSession.StateCallback */
56     interface StateCallback : SessionStateCallback {
57         /** @see CameraExtensionSession.StateCallback.onClosed */
58         fun onClosed(session: CameraExtensionSessionWrapper)
59 
60         /** @see CameraExtensionSession.StateCallback.onConfigureFailed */
61         fun onConfigureFailed(session: CameraExtensionSessionWrapper)
62 
63         /** @see CameraExtensionSession.StateCallback.onConfigured */
64         fun onConfigured(session: CameraExtensionSessionWrapper)
65     }
66 
67     fun getRealTimeCaptureLatency(): CameraExtensionSession.StillCaptureLatency?
68 }
69 
70 @RequiresApi(31)
71 internal class AndroidExtensionSessionStateCallback(
72     private val device: CameraDeviceWrapper,
73     private val stateCallback: CameraExtensionSessionWrapper.StateCallback,
74     lastStateCallback: SessionStateCallback?,
75     private val cameraErrorListener: CameraErrorListener,
76     private val interopSessionStateCallback: CameraExtensionSession.StateCallback? = null,
77     private val callbackExecutor: Executor
78 ) : CameraExtensionSession.StateCallback() {
79     private val _lastStateCallback = atomic(lastStateCallback)
80     private val extensionSession = atomic<CameraExtensionSessionWrapper?>(null)
81 
onConfigurednull82     override fun onConfigured(session: CameraExtensionSession) {
83         stateCallback.onConfigured(getWrapped(session, cameraErrorListener))
84 
85         // b/249258992 - This is a workaround to ensure previous
86         // CameraExtensionSession.StateCallback instances receive some kind of "finalization"
87         // signal if onClosed is not fired by the framework after a subsequent session
88         // has been configured.
89         finalizeLastSession()
90         interopSessionStateCallback?.onConfigured(session)
91     }
92 
onConfigureFailednull93     override fun onConfigureFailed(session: CameraExtensionSession) {
94         stateCallback.onConfigureFailed(getWrapped(session, cameraErrorListener))
95         finalizeSession()
96         interopSessionStateCallback?.onConfigureFailed(session)
97     }
98 
onClosednull99     override fun onClosed(session: CameraExtensionSession) {
100         stateCallback.onClosed(getWrapped(session, cameraErrorListener))
101         finalizeSession()
102         interopSessionStateCallback?.onClosed(session)
103     }
104 
getWrappednull105     private fun getWrapped(
106         session: CameraExtensionSession,
107         cameraErrorListener: CameraErrorListener,
108     ): CameraExtensionSessionWrapper {
109         var local = extensionSession.value
110         if (local != null) {
111             return local
112         }
113 
114         local = wrapSession(session, cameraErrorListener)
115         if (extensionSession.compareAndSet(null, local)) {
116             return local
117         }
118         return extensionSession.value!!
119     }
120 
wrapSessionnull121     private fun wrapSession(
122         session: CameraExtensionSession,
123         cameraErrorListener: CameraErrorListener,
124     ): CameraExtensionSessionWrapper {
125         return AndroidCameraExtensionSession(device, session, cameraErrorListener, callbackExecutor)
126     }
127 
finalizeSessionnull128     private fun finalizeSession() {
129         finalizeLastSession()
130         stateCallback.onSessionFinalized()
131     }
132 
finalizeLastSessionnull133     private fun finalizeLastSession() {
134         // Clear out the reference to the previous session, if one was set.
135         val previousSession = _lastStateCallback.getAndSet(null)
136         previousSession?.let { previousSession.onSessionFinalized() }
137     }
138 }
139 
140 @RequiresApi(31)
141 internal open class AndroidCameraExtensionSession(
142     override val device: CameraDeviceWrapper,
143     private val cameraExtensionSession: CameraExtensionSession,
144     private val cameraErrorListener: CameraErrorListener,
145     private val callbackExecutor: Executor
146 ) : CameraExtensionSessionWrapper {
147 
148     private val frameNumbers: AtomicLong = atomic(0L)
149     private val extensionSessionMap: MutableMap<CameraExtensionSession, Long> = HashMap()
150 
capturenull151     override fun capture(
152         request: CaptureRequest,
153         listener: CameraCaptureSession.CaptureCallback
154     ): Int? =
155         catchAndReportCameraExceptions(device.cameraId, cameraErrorListener) {
156             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
157                 cameraExtensionSession.capture(
158                     request,
159                     callbackExecutor,
160                     Camera2CaptureSessionCallbackToExtensionCaptureCallback(
161                         listener as Camera2CaptureCallback,
162                         LinkedList()
163                     )
164                 )
165             } else {
166                 cameraExtensionSession.capture(
167                     request,
168                     callbackExecutor,
169                     Camera2CaptureSessionCallbackToExtensionCaptureCallbackAndroidS(
170                         listener as Camera2CaptureCallback,
171                         mutableMapOf()
172                     )
173                 )
174             }
175         }
176 
setRepeatingRequestnull177     override fun setRepeatingRequest(
178         request: CaptureRequest,
179         listener: CameraCaptureSession.CaptureCallback,
180     ): Int? =
181         catchAndReportCameraExceptions(device.cameraId, cameraErrorListener) {
182             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
183                 cameraExtensionSession.setRepeatingRequest(
184                     request,
185                     callbackExecutor,
186                     Camera2CaptureSessionCallbackToExtensionCaptureCallback(
187                         listener as Camera2CaptureCallback,
188                         LinkedList()
189                     )
190                 )
191             } else {
192                 cameraExtensionSession.setRepeatingRequest(
193                     request,
194                     callbackExecutor,
195                     Camera2CaptureSessionCallbackToExtensionCaptureCallbackAndroidS(
196                         listener as Camera2CaptureCallback,
197                         mutableMapOf()
198                     )
199                 )
200             }
201         }
202 
stopRepeatingnull203     override fun stopRepeating(): Boolean =
204         catchAndReportCameraExceptions(device.cameraId, cameraErrorListener) {
205             cameraExtensionSession.stopRepeating()
206         } != null
207 
208     override val isReprocessable: Boolean
209         get() = false
210 
211     override val inputSurface: Surface?
212         get() = null
213 
abortCapturesnull214     override fun abortCaptures(): Boolean = false
215 
216     override fun captureBurst(
217         requests: List<CaptureRequest>,
218         listener: CameraCaptureSession.CaptureCallback
219     ): Int? {
220         requests.forEach { captureRequest -> capture(captureRequest, listener) }
221         return null
222     }
223 
setRepeatingBurstnull224     override fun setRepeatingBurst(
225         requests: List<CaptureRequest>,
226         listener: CameraCaptureSession.CaptureCallback
227     ): Int? {
228         check(requests.size == 1) {
229             "CameraExtensionSession does not support setRepeatingBurst for more than one" +
230                 "CaptureRequest"
231         }
232         return setRepeatingRequest(requests.single(), listener)
233     }
234 
finalizeOutputConfigurationsnull235     override fun finalizeOutputConfigurations(
236         outputConfigs: List<OutputConfigurationWrapper>
237     ): Boolean {
238         Log.warn { "CameraExtensionSession does not support finalizeOutputConfigurations()" }
239         return false
240     }
241 
242     @Suppress("UNCHECKED_CAST")
unwrapAsnull243     override fun <T : Any> unwrapAs(type: KClass<T>): T? =
244         when (type) {
245             CameraExtensionSession::class -> cameraExtensionSession as T?
246             else -> null
247         }
248 
closenull249     override fun close() {
250         return cameraExtensionSession.close()
251     }
252 
getRealTimeCaptureLatencynull253     override fun getRealTimeCaptureLatency(): CameraExtensionSession.StillCaptureLatency? {
254         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
255             return cameraExtensionSession.realtimeStillCaptureLatency
256         }
257         return null
258     }
259 
260     inner class Camera2CaptureSessionCallbackToExtensionCaptureCallback(
261         private val captureCallback: Camera2CaptureCallback,
262         private val frameQueue: Queue<Long>
263     ) : CameraExtensionSession.ExtensionCaptureCallback() {
264 
onCaptureStartednull265         override fun onCaptureStarted(
266             session: CameraExtensionSession,
267             request: CaptureRequest,
268             timestamp: Long
269         ) {
270             val frameNumber = frameNumbers.incrementAndGet()
271             extensionSessionMap[session] = frameNumber
272             frameQueue.add(frameNumber)
273             captureCallback.onCaptureStarted(request, frameNumber, timestamp)
274         }
275 
onCaptureProcessStartednull276         override fun onCaptureProcessStarted(
277             session: CameraExtensionSession,
278             request: CaptureRequest
279         ) {}
280 
onCaptureProcessProgressednull281         override fun onCaptureProcessProgressed(
282             session: CameraExtensionSession,
283             request: CaptureRequest,
284             progress: Int
285         ) {
286             captureCallback.onCaptureProcessProgressed(request, progress)
287         }
288 
onCaptureFailednull289         override fun onCaptureFailed(session: CameraExtensionSession, request: CaptureRequest) {
290             val frameNumber = frameQueue.remove()
291             captureCallback.onCaptureFailed(request, FrameNumber(frameNumber))
292         }
293 
onCaptureSequenceCompletednull294         override fun onCaptureSequenceCompleted(session: CameraExtensionSession, sequenceId: Int) {
295             val frameNumber = extensionSessionMap[session]
296             captureCallback.onCaptureSequenceCompleted(sequenceId, frameNumber!!)
297         }
298 
onCaptureSequenceAbortednull299         override fun onCaptureSequenceAborted(session: CameraExtensionSession, sequenceId: Int) {
300             captureCallback.onCaptureSequenceAborted(sequenceId)
301         }
302 
onCaptureResultAvailablenull303         override fun onCaptureResultAvailable(
304             session: CameraExtensionSession,
305             request: CaptureRequest,
306             result: TotalCaptureResult
307         ) {
308             val frameNumber = frameQueue.remove()
309             captureCallback.onCaptureCompleted(request, result, FrameNumber(frameNumber))
310         }
311     }
312 
313     /**
314      * [CameraExtensionSession.ExtensionCaptureCallback]'s onCaptureResultAvailable is gated behind
315      * Android T, so any devices on Android S or older will not see onCaptureResultAvailable being
316      * triggered. This implementation calls onCaptureCompleted in onCaptureStarted and does not keep
317      * track of completed or failed frames for repeating requests.
318      */
319     inner class Camera2CaptureSessionCallbackToExtensionCaptureCallbackAndroidS(
320         private val captureCallback: Camera2CaptureCallback,
321         private val captureRequestMap: MutableMap<CaptureRequest, MutableList<Long>>
322     ) : CameraExtensionSession.ExtensionCaptureCallback() {
323 
onCaptureStartednull324         override fun onCaptureStarted(
325             session: CameraExtensionSession,
326             request: CaptureRequest,
327             timestamp: Long
328         ) {
329             val frameNumber = frameNumbers.incrementAndGet()
330             extensionSessionMap[session] = frameNumber
331             captureRequestMap.getOrPut(request) { mutableListOf() }.add(frameNumber)
332             captureCallback.onCaptureStarted(request, frameNumber, timestamp)
333         }
334 
onCaptureProcessStartednull335         override fun onCaptureProcessStarted(
336             session: CameraExtensionSession,
337             request: CaptureRequest
338         ) {}
339 
onCaptureFailednull340         override fun onCaptureFailed(session: CameraExtensionSession, request: CaptureRequest) {
341             if (captureRequestMap[request]!!.size == 1) {
342                 val frameNumber = captureRequestMap[request]!![0]
343                 captureCallback.onCaptureFailed(request, FrameNumber(frameNumber))
344             } else {
345                 Log.info {
346                     "onCaptureFailed is not triggered for repeating requests. Request " +
347                         "frame numbers: " +
348                         captureRequestMap[request]!!.stream()
349                 }
350             }
351         }
352 
onCaptureProcessProgressednull353         override fun onCaptureProcessProgressed(
354             session: CameraExtensionSession,
355             request: CaptureRequest,
356             progress: Int
357         ) {
358             captureCallback.onCaptureProcessProgressed(request, progress)
359         }
360 
onCaptureSequenceCompletednull361         override fun onCaptureSequenceCompleted(session: CameraExtensionSession, sequenceId: Int) {
362             val frameNumber = extensionSessionMap[session]
363             captureCallback.onCaptureSequenceCompleted(sequenceId, frameNumber!!)
364         }
365 
onCaptureSequenceAbortednull366         override fun onCaptureSequenceAborted(session: CameraExtensionSession, sequenceId: Int) {
367             captureCallback.onCaptureSequenceAborted(sequenceId)
368         }
369     }
370 }
371