1 /*
<lambda>null2  * Copyright 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.integration.impl
18 
19 import android.hardware.camera2.CameraDevice
20 import android.hardware.camera2.CaptureRequest
21 import android.view.Surface
22 import androidx.annotation.GuardedBy
23 import androidx.camera.camera2.pipe.CameraStream
24 import androidx.camera.camera2.pipe.core.Log
25 import androidx.camera.camera2.pipe.integration.adapter.RequestProcessorAdapter
26 import androidx.camera.camera2.pipe.integration.adapter.SessionConfigAdapter
27 import androidx.camera.camera2.pipe.integration.interop.CaptureRequestOptions
28 import androidx.camera.camera2.pipe.integration.interop.ExperimentalCamera2Interop
29 import androidx.camera.core.ImageAnalysis
30 import androidx.camera.core.ImageCapture
31 import androidx.camera.core.Preview
32 import androidx.camera.core.UseCase
33 import androidx.camera.core.impl.CameraInfoInternal
34 import androidx.camera.core.impl.CaptureConfig
35 import androidx.camera.core.impl.DeferrableSurface
36 import androidx.camera.core.impl.DeferrableSurfaces
37 import androidx.camera.core.impl.OutputSurface
38 import androidx.camera.core.impl.OutputSurfaceConfiguration
39 import androidx.camera.core.impl.SessionConfig
40 import androidx.camera.core.impl.SessionProcessor
41 import androidx.camera.core.impl.SessionProcessor.CaptureCallback
42 import androidx.camera.core.impl.TagBundle
43 import androidx.camera.core.impl.utils.executor.CameraXExecutors
44 import androidx.camera.core.impl.utils.futures.Futures
45 import androidx.camera.core.streamsharing.StreamSharing
46 import androidx.concurrent.futures.await
47 import kotlinx.coroutines.CoroutineScope
48 import kotlinx.coroutines.isActive
49 import kotlinx.coroutines.launch
50 import kotlinx.coroutines.withTimeoutOrNull
51 
52 @OptIn(ExperimentalCamera2Interop::class)
53 public class SessionProcessorManager(
54     private val sessionProcessor: SessionProcessor,
55     private val cameraInfoInternal: CameraInfoInternal,
56     private val scope: CoroutineScope,
57 ) {
58     private val lock = Any()
59 
60     public enum class State {
61         /**
62          * [CREATED] is the initial state, and indicates that the [SessionProcessorManager] has been
63          * created but not initialized yet.
64          */
65         CREATED,
66 
67         /**
68          * [INITIALIZED] indicates that the [SessionProcessor] has been initialized and we've
69          * received the updated session configurations. See also: [SessionProcessor.deInitSession].
70          */
71         INITIALIZED,
72 
73         /**
74          * [STARTED] indicates that we've provided our [androidx.camera.core.impl.RequestProcessor]
75          * implementation to [SessionProcessor]. See also [SessionProcessor.onCaptureSessionStart].
76          */
77         STARTED,
78 
79         /**
80          * [CLOSING] indicates that we're ending our capture session, and we'll no longer accept any
81          * further capture requests. See also: [SessionProcessor.onCaptureSessionEnd].
82          */
83         CLOSING,
84 
85         /**
86          * [CLOSED] indicates that the underlying capture session has been completely closed and
87          * we've de-initialized the session. See also: [SessionProcessor.deInitSession].
88          */
89         CLOSED,
90     }
91 
92     @GuardedBy("lock") private var state: State = State.CREATED
93 
94     @GuardedBy("lock") private var sessionOptions = CaptureRequestOptions.Builder().build()
95 
96     @GuardedBy("lock") private var stillCaptureOptions = CaptureRequestOptions.Builder().build()
97 
98     @GuardedBy("lock") private var requestProcessor: RequestProcessorAdapter? = null
99 
100     @GuardedBy("lock") private var pendingCaptureConfigs: List<CaptureConfig>? = null
101 
102     @GuardedBy("lock") private var pendingCaptureCallbacks: List<CaptureCallback>? = null
103 
104     @GuardedBy("lock")
105     internal var sessionConfig: SessionConfig? = null
106         set(value) =
107             synchronized(lock) {
108                 field = checkNotNull(value)
109                 if (state != State.STARTED) return
110                 checkNotNull(requestProcessor).sessionConfig = value
111                 sessionOptions =
112                     CaptureRequestOptions.Builder.from(value.implementationOptions).build()
113                 updateOptions()
114             }
115 
116     internal fun isClosed() = synchronized(lock) { state == State.CLOSED || state == State.CLOSING }
117 
118     internal fun initialize(
119         useCaseManager: UseCaseManager,
120         useCases: List<UseCase>,
121         timeoutMillis: Long = 5_000L,
122         configure: (UseCaseManager.Companion.UseCaseManagerConfig?) -> Unit,
123     ) =
124         scope.launch {
125             val sessionConfigAdapter = SessionConfigAdapter(useCases, null)
126             val deferrableSurfaces = sessionConfigAdapter.deferrableSurfaces
127             val surfaces = getSurfaces(deferrableSurfaces, timeoutMillis)
128             if (!isActive) return@launch configure(null)
129             if (surfaces.isEmpty()) {
130                 Log.error {
131                     "Cannot initialize ${this@SessionProcessorManager}: Surface list is empty"
132                 }
133                 return@launch configure(null)
134             }
135             if (surfaces.contains(null)) {
136                 Log.error {
137                     "Cannot initialize ${this@SessionProcessorManager}: Some Surfaces are invalid!"
138                 }
139                 sessionConfigAdapter.reportSurfaceInvalid(
140                     deferrableSurfaces[surfaces.indexOf(null)]
141                 )
142                 return@launch configure(null)
143             }
144             var previewOutputSurface: OutputSurface? = null
145             var captureOutputSurface: OutputSurface? = null
146             var analysisOutputSurface: OutputSurface? = null
147             var postviewOutputSurface: OutputSurface? = null
148             for ((deferrableSurface, surface) in deferrableSurfaces.zip(surfaces)) {
149                 when (deferrableSurface.containerClass) {
150                     Preview::class.java ->
151                         previewOutputSurface = createOutputSurface(deferrableSurface, surface!!)
152                     StreamSharing::class.java ->
153                         previewOutputSurface = createOutputSurface(deferrableSurface, surface!!)
154                     ImageCapture::class.java ->
155                         captureOutputSurface = createOutputSurface(deferrableSurface, surface!!)
156                     ImageAnalysis::class.java ->
157                         analysisOutputSurface = createOutputSurface(deferrableSurface, surface!!)
158                 }
159             }
160             val postviewOutputConfig =
161                 sessionConfigAdapter.getValidSessionConfigOrNull()?.postviewOutputConfig
162             var postviewDeferrableSurface: DeferrableSurface? = null
163             if (postviewOutputConfig != null) {
164                 postviewDeferrableSurface = postviewOutputConfig.surface
165                 postviewOutputSurface =
166                     createOutputSurface(
167                         postviewDeferrableSurface,
168                         postviewDeferrableSurface.surface.get()!!
169                     )
170             }
171             Log.debug {
172                 "SessionProcessorSurfaceManager: Identified surfaces: " +
173                     "previewOutputSurface = $previewOutputSurface, " +
174                     "captureOutputSurface = $captureOutputSurface, " +
175                     "analysisOutputSurface = $analysisOutputSurface, " +
176                     "postviewOutputSurface = $postviewOutputSurface"
177             }
178 
179             // IMPORTANT: The critical section (covered by synchronized) is intentionally expanded
180             // to cover the sections where we increment and decrement (on failure) the use count on
181             // the DeferrableSurfaces. This is needed because the SessionProcessorManager could be
182             // closed while we're still initializing, and we need to make sure we either initialize
183             // to a point where all the lifetimes of Surfaces are setup or we don't initialize at
184             // all beyond this point.
185             val processorSessionConfig =
186                 synchronized(lock) {
187                     if (isClosed()) return@synchronized null
188                     try {
189                         val surfacesToIncrement = ArrayList(deferrableSurfaces)
190                         postviewDeferrableSurface?.let { surfacesToIncrement.add(it) }
191                         DeferrableSurfaces.incrementAll(surfacesToIncrement)
192                     } catch (exception: DeferrableSurface.SurfaceClosedException) {
193                         sessionConfigAdapter.reportSurfaceInvalid(exception.deferrableSurface)
194                         return@synchronized null
195                     }
196                     try {
197                         Log.debug { "Invoking $sessionProcessor SessionProcessor#initSession" }
198                         sessionProcessor
199                             .initSession(
200                                 cameraInfoInternal,
201                                 OutputSurfaceConfiguration.create(
202                                     previewOutputSurface!!,
203                                     captureOutputSurface!!,
204                                     analysisOutputSurface,
205                                     postviewOutputSurface,
206                                 ),
207                             )
208                             .also { state = State.INITIALIZED }
209                     } catch (throwable: Throwable) {
210                         Log.error(throwable) { "initSession() failed" }
211                         DeferrableSurfaces.decrementAll(deferrableSurfaces)
212                         postviewDeferrableSurface?.decrementUseCount()
213                         throw throwable
214                     }
215                 } ?: return@launch configure(null)
216 
217             // DecrementAll the output surfaces when ProcessorSurface terminates.
218             processorSessionConfig.surfaces
219                 .first()
220                 .terminationFuture
221                 .addListener(
222                     {
223                         DeferrableSurfaces.decrementAll(deferrableSurfaces)
224                         postviewDeferrableSurface?.decrementUseCount()
225                     },
226                     CameraXExecutors.directExecutor()
227                 )
228 
229             val processorSessionConfigAdapter =
230                 SessionConfigAdapter(useCases, processorSessionConfig)
231 
232             val streamConfigMap = mutableMapOf<CameraStream.Config, DeferrableSurface>()
233             val cameraGraphConfig =
234                 useCaseManager.createCameraGraphConfig(
235                     processorSessionConfigAdapter,
236                     streamConfigMap,
237                     isExtensions = true,
238                 )
239 
240             val useCaseManagerConfig =
241                 UseCaseManager.Companion.UseCaseManagerConfig(
242                     useCases,
243                     processorSessionConfigAdapter,
244                     cameraGraphConfig,
245                     streamConfigMap,
246                 )
247 
248             return@launch configure(useCaseManagerConfig)
249         }
250 
251     internal fun onCaptureSessionStart(requestProcessor: RequestProcessorAdapter) {
252         var captureConfigsToIssue: List<CaptureConfig>?
253         var captureCallbacksToIssue: List<CaptureCallback>?
254         synchronized(lock) {
255             if (state != State.INITIALIZED) {
256                 Log.warn { "onCaptureSessionStart called on an uninitialized extensions session" }
257                 return
258             }
259             requestProcessor.sessionConfig = sessionConfig
260             this.requestProcessor = requestProcessor
261 
262             captureConfigsToIssue = pendingCaptureConfigs
263             captureCallbacksToIssue = pendingCaptureCallbacks
264             pendingCaptureConfigs = null
265             pendingCaptureCallbacks = null
266 
267             Log.debug { "Invoking SessionProcessor#onCaptureSessionStart" }
268             sessionProcessor.onCaptureSessionStart(requestProcessor)
269 
270             state = State.STARTED
271         }
272         val tagBundle = sessionConfig?.repeatingCaptureConfig?.tagBundle ?: TagBundle.emptyBundle()
273         startRepeating(tagBundle)
274         captureConfigsToIssue?.let { captureConfigs ->
275             submitCaptureConfigs(captureConfigs, checkNotNull(captureCallbacksToIssue))
276         }
277     }
278 
279     internal fun startRepeating(
280         tagBundle: TagBundle,
281         captureCallback: CaptureCallback = object : CaptureCallback {}
282     ) {
283         synchronized(lock) {
284             if (state != State.STARTED) return
285             Log.debug { "Invoking SessionProcessor#startRepeating" }
286             sessionProcessor.startRepeating(tagBundle, captureCallback)
287         }
288     }
289 
290     internal fun stopRepeating() {
291         synchronized(lock) {
292             if (state != State.STARTED) return
293             Log.debug { "Invoking SessionProcessor#stopRepeating" }
294             sessionProcessor.stopRepeating()
295         }
296     }
297 
298     internal fun submitCaptureConfigs(
299         captureConfigs: List<CaptureConfig>,
300         captureCallbacks: List<CaptureCallback>,
301     ) =
302         synchronized(lock) {
303             check(captureConfigs.size == captureCallbacks.size)
304             if (state != State.STARTED) {
305                 // The lifetime of image capture requests is separate from the extensions lifetime.
306                 // It is therefore possible for capture requests to be issued when the capture
307                 // session hasn't yet started (before invoking
308                 // SessionProcessor.onCaptureSessionStart). This is a copy of camera-camera2's
309                 // behavior where it stores the last capture configs that weren't submitted.
310                 Log.info {
311                     "SessionProcessor#submitCaptureConfigs: Session not yet started. " +
312                         "The capture requests will be submitted later"
313                 }
314                 pendingCaptureConfigs = captureConfigs
315                 pendingCaptureCallbacks = captureCallbacks
316                 return
317             }
318             for ((config, callback) in captureConfigs.zip(captureCallbacks)) {
319                 if (config.templateType == CameraDevice.TEMPLATE_STILL_CAPTURE) {
320                     val builder = CaptureRequestOptions.Builder.from(config.implementationOptions)
321                     if (
322                         config.implementationOptions.containsOption(CaptureConfig.OPTION_ROTATION)
323                     ) {
324                         builder.setCaptureRequestOption(
325                             CaptureRequest.JPEG_ORIENTATION,
326                             config.implementationOptions.retrieveOption(
327                                 CaptureConfig.OPTION_ROTATION
328                             )
329                         )
330                     }
331                     if (
332                         config.implementationOptions.containsOption(
333                             CaptureConfig.OPTION_JPEG_QUALITY
334                         )
335                     ) {
336                         builder.setCaptureRequestOption(
337                             CaptureRequest.JPEG_QUALITY,
338                             config.implementationOptions
339                                 .retrieveOption(CaptureConfig.OPTION_JPEG_QUALITY)!!
340                                 .toByte()
341                         )
342                     }
343                     synchronized(lock) {
344                         stillCaptureOptions = builder.build()
345                         updateOptions()
346                     }
347                     Log.debug { "Invoking SessionProcessor.startCapture()" }
348                     sessionProcessor.startCapture(
349                         config.isPostviewEnabled,
350                         config.tagBundle,
351                         callback
352                     )
353                 } else {
354                     val options =
355                         CaptureRequestOptions.Builder.from(config.implementationOptions).build()
356                     Log.debug { "Invoking SessionProcessor.startTrigger()" }
357                     sessionProcessor.startTrigger(options, config.tagBundle, callback)
358                 }
359             }
360         }
361 
362     internal fun prepareClose() =
363         synchronized(lock) {
364             if (state == State.STARTED) {
365                 sessionProcessor.onCaptureSessionEnd()
366             }
367             // If we have an initialized SessionProcessor session (i.e., initSession was called), we
368             // need to make sure close() invokes deInitSession and only does it when necessary.
369             if (state == State.INITIALIZED || state == State.STARTED) {
370                 state = State.CLOSING
371             } else {
372                 state = State.CLOSED
373             }
374         }
375 
376     internal fun close() =
377         synchronized(lock) {
378             // These states indicate that we had previously initialized a session (but not yet
379             // de-initialized), and thus we need to de-initialize the session here.
380             if (state == State.INITIALIZED || state == State.STARTED || state == State.CLOSING) {
381                 Log.debug { "Invoking $sessionProcessor SessionProcessor#deInitSession" }
382                 sessionProcessor.deInitSession()
383             }
384             state = State.CLOSED
385         }
386 
387     @GuardedBy("lock")
388     private fun updateOptions() {
389         val builder =
390             Camera2ImplConfig.Builder().apply {
391                 insertAllOptions(sessionOptions)
392                 insertAllOptions(stillCaptureOptions)
393             }
394         sessionProcessor.setParameters(builder.build())
395     }
396 
397     public companion object {
398         private suspend fun getSurfaces(
399             deferrableSurfaces: List<DeferrableSurface>,
400             timeoutMillis: Long,
401         ): List<Surface?> {
402             return withTimeoutOrNull(timeMillis = timeoutMillis) {
403                     Futures.successfulAsList(
404                             deferrableSurfaces.map {
405                                 Futures.nonCancellationPropagating(it.surface)
406                             }
407                         )
408                         .await()
409                 }
410                 .orEmpty()
411         }
412 
413         private fun createOutputSurface(deferrableSurface: DeferrableSurface, surface: Surface) =
414             OutputSurface.create(
415                 surface,
416                 deferrableSurface.prescribedSize,
417                 deferrableSurface.prescribedStreamFormat,
418             )
419     }
420 }
421