1 /*
<lambda>null2  * Copyright 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.integration.impl
18 
19 import android.content.Context
20 import android.graphics.ImageFormat
21 import android.hardware.camera2.CameraCharacteristics
22 import android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW
23 import android.hardware.camera2.CaptureRequest
24 import android.hardware.camera2.params.OutputConfiguration
25 import android.hardware.camera2.params.SessionConfiguration.SESSION_HIGH_SPEED
26 import android.hardware.camera2.params.SessionConfiguration.SESSION_REGULAR
27 import android.media.MediaCodec
28 import android.os.Build
29 import android.util.Pair
30 import androidx.annotation.GuardedBy
31 import androidx.annotation.VisibleForTesting
32 import androidx.camera.camera2.pipe.CameraDevices
33 import androidx.camera.camera2.pipe.CameraGraph
34 import androidx.camera.camera2.pipe.CameraGraph.OperatingMode
35 import androidx.camera.camera2.pipe.CameraGraph.RepeatingRequestRequirementsBeforeCapture.CompletionBehavior.AT_LEAST
36 import androidx.camera.camera2.pipe.CameraId
37 import androidx.camera.camera2.pipe.CameraMetadata
38 import androidx.camera.camera2.pipe.CameraMetadata.Companion.supportsLowLightBoost
39 import androidx.camera.camera2.pipe.CameraPipe
40 import androidx.camera.camera2.pipe.CameraStream
41 import androidx.camera.camera2.pipe.InputStream
42 import androidx.camera.camera2.pipe.OutputStream
43 import androidx.camera.camera2.pipe.OutputStream.DynamicRangeProfile
44 import androidx.camera.camera2.pipe.RequestTemplate
45 import androidx.camera.camera2.pipe.StreamFormat
46 import androidx.camera.camera2.pipe.StreamId
47 import androidx.camera.camera2.pipe.compat.CameraPipeKeys
48 import androidx.camera.camera2.pipe.core.Log
49 import androidx.camera.camera2.pipe.integration.adapter.CameraStateAdapter
50 import androidx.camera.camera2.pipe.integration.adapter.SessionConfigAdapter
51 import androidx.camera.camera2.pipe.integration.adapter.SupportedSurfaceCombination
52 import androidx.camera.camera2.pipe.integration.adapter.ZslControl
53 import androidx.camera.camera2.pipe.integration.compat.DynamicRangeProfilesCompat
54 import androidx.camera.camera2.pipe.integration.compat.quirk.CameraQuirks
55 import androidx.camera.camera2.pipe.integration.compat.quirk.CaptureSessionStuckQuirk
56 import androidx.camera.camera2.pipe.integration.compat.quirk.CloseCameraDeviceOnCameraGraphCloseQuirk
57 import androidx.camera.camera2.pipe.integration.compat.quirk.CloseCaptureSessionOnDisconnectQuirk
58 import androidx.camera.camera2.pipe.integration.compat.quirk.CloseCaptureSessionOnVideoQuirk
59 import androidx.camera.camera2.pipe.integration.compat.quirk.DeviceQuirks
60 import androidx.camera.camera2.pipe.integration.compat.quirk.DisableAbortCapturesOnStopQuirk
61 import androidx.camera.camera2.pipe.integration.compat.quirk.DisableAbortCapturesOnStopWithSessionProcessorQuirk
62 import androidx.camera.camera2.pipe.integration.compat.quirk.FinalizeSessionOnCloseQuirk
63 import androidx.camera.camera2.pipe.integration.compat.quirk.QuickSuccessiveImageCaptureFailsRepeatingRequestQuirk
64 import androidx.camera.camera2.pipe.integration.compat.workaround.TemplateParamsOverride
65 import androidx.camera.camera2.pipe.integration.config.CameraConfig
66 import androidx.camera.camera2.pipe.integration.config.CameraScope
67 import androidx.camera.camera2.pipe.integration.config.UseCaseCameraComponent
68 import androidx.camera.camera2.pipe.integration.config.UseCaseCameraConfig
69 import androidx.camera.camera2.pipe.integration.config.UseCaseGraphConfig
70 import androidx.camera.camera2.pipe.integration.internal.DynamicRangeConversions.dynamicRangeToFirstSupportedProfile
71 import androidx.camera.camera2.pipe.integration.internal.DynamicRangeResolver
72 import androidx.camera.camera2.pipe.integration.interop.Camera2CameraControl
73 import androidx.camera.camera2.pipe.integration.interop.ExperimentalCamera2Interop
74 import androidx.camera.core.DynamicRange
75 import androidx.camera.core.ImageCapture
76 import androidx.camera.core.MirrorMode
77 import androidx.camera.core.Preview
78 import androidx.camera.core.UseCase
79 import androidx.camera.core.concurrent.CameraCoordinator
80 import androidx.camera.core.impl.AttachedSurfaceInfo
81 import androidx.camera.core.impl.CameraInfoInternal
82 import androidx.camera.core.impl.CameraInternal
83 import androidx.camera.core.impl.CameraMode
84 import androidx.camera.core.impl.CaptureConfig
85 import androidx.camera.core.impl.DeferrableSurface
86 import androidx.camera.core.impl.EncoderProfilesProvider
87 import androidx.camera.core.impl.MutableOptionsBundle
88 import androidx.camera.core.impl.SessionConfig
89 import androidx.camera.core.impl.SessionConfig.OutputConfig.SURFACE_GROUP_ID_NONE
90 import androidx.camera.core.impl.SessionConfig.ValidatingBuilder
91 import androidx.camera.core.impl.SessionProcessor
92 import androidx.camera.core.impl.StreamSpec.FRAME_RATE_RANGE_UNSPECIFIED
93 import androidx.camera.core.impl.SurfaceConfig
94 import androidx.camera.core.impl.stabilization.StabilizationMode
95 import androidx.camera.core.streamsharing.StreamSharing
96 import androidx.camera.core.streamsharing.StreamSharingConfig
97 import javax.inject.Inject
98 import javax.inject.Provider
99 import kotlinx.coroutines.Deferred
100 import kotlinx.coroutines.Job
101 import kotlinx.coroutines.joinAll
102 import kotlinx.coroutines.runBlocking
103 import org.jetbrains.annotations.TestOnly
104 
105 /**
106  * This class keeps track of the currently attached and active [UseCase]'s for a specific camera. A
107  * [UseCase] during its lifetime, can be:
108  * - Attached: This happens when a use case is bound to a CameraX Lifecycle, and signals that the
109  *   camera should be opened, and a camera capture session should be created to include the stream
110  *   corresponding to the use case. In the integration layer here, we'll recreate a CameraGraph when
111  *   a use case is attached.
112  * - Detached: This happens when a use case is unbound from a CameraX Lifecycle, and signals that we
113  *   no longer need this specific use case and therefore its corresponding stream in our current
114  *   capture session. In the integration layer, we'll also recreate a CameraGraph when a use case is
115  *   detached, though it might not be strictly necessary.
116  * - Active: This happens when the use case is considered "ready", meaning that the use case is
117  *   ready to have frames delivered to it. In the case of the integration layer, this means we can
118  *   start submitting the capture requests corresponding to the use case. An important note here is
119  *   that a use case can actually become "active" before it is "attached", and thus we should only
120  *   take action when a use case is both "attached" and "active".
121  * - Inactive: This happens when use case no longer needs frames delivered to it. This is can be
122  *   seen as an optimization signal, as we technically are allowed to continue submitting capture
123  *   requests, but we no longer need to. An example of this is when you clear the analyzer during
124  *   ImageAnalysis.
125  *
126  * In this class, we also define a new term - "Running". A use case is considered running when it's
127  * both "attached" and "active". This means we should have a camera opened, a capture session with
128  * the streams created and have capture requests submitting.
129  */
130 @OptIn(ExperimentalCamera2Interop::class)
131 @CameraScope
132 public class UseCaseManager
133 @Inject
134 constructor(
135     private val cameraPipe: CameraPipe,
136     private val cameraDevices: CameraDevices,
137     private val cameraMetadata: CameraMetadata?,
138     @GuardedBy("lock") private val cameraCoordinator: CameraCoordinator,
139     private val callbackMap: CameraCallbackMap,
140     private val requestListener: ComboRequestListener,
141     private val cameraConfig: CameraConfig,
142     private val builder: UseCaseCameraComponent.Builder,
143     private val zslControl: ZslControl,
144     private val lowLightBoostControl: LowLightBoostControl,
145     @Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") // Java version required for Dagger
146     private val controls: java.util.Set<UseCaseCameraControl>,
147     private val camera2CameraControl: Camera2CameraControl,
148     private val cameraStateAdapter: CameraStateAdapter,
149     private val cameraQuirks: CameraQuirks,
150     private val cameraInternal: Provider<CameraInternal>,
151     private val useCaseThreads: Provider<UseCaseThreads>,
152     private val cameraInfoInternal: Provider<CameraInfoInternal>,
153     private val templateParamsOverride: TemplateParamsOverride,
154     private val encoderProfilesProvider: EncoderProfilesProvider,
155     private val cameraProperties: CameraProperties,
156     context: Context,
157     displayInfoManager: DisplayInfoManager,
158 ) {
159     private val lock = Any()
160 
161     internal var sessionProcessor: SessionProcessor? = null
162         get() =
163             synchronized(lock) {
164                 return field
165             }
166         set(value) = synchronized(lock) { field = value }
167 
168     @GuardedBy("lock") private var sessionProcessorManager: SessionProcessorManager? = null
169 
170     @GuardedBy("lock") private val attachedUseCases = mutableSetOf<UseCase>()
171 
172     @GuardedBy("lock") private val activeUseCases = mutableSetOf<UseCase>()
173 
174     @GuardedBy("lock") private var activeResumeEnabled = false
175 
176     @GuardedBy("lock") private var shouldCreateCameraGraphImmediately = true
177 
178     @GuardedBy("lock") private var deferredUseCaseManagerConfig: UseCaseManagerConfig? = null
179 
180     @GuardedBy("lock") private var pendingSessionProcessorInitialization = false
181 
182     @GuardedBy("lock") private var isPrimary = true
183 
184     @GuardedBy("lock")
185     private val pendingUseCasesToNotifyCameraControlReady = mutableSetOf<UseCase>()
186 
187     private val meteringRepeating by lazy {
188         MeteringRepeating.Builder(cameraProperties, displayInfoManager).build()
189     }
190 
191     private val supportedSurfaceCombination by lazy {
192         SupportedSurfaceCombination(context, cameraProperties.metadata, encoderProfilesProvider)
193     }
194 
195     private val dynamicRangeResolver = DynamicRangeResolver(cameraProperties.metadata)
196 
197     @Volatile private var _activeComponent: UseCaseCameraComponent? = null
198     public val camera: UseCaseCamera?
199         get() = _activeComponent?.getUseCaseCamera()
200 
201     public val useCaseGraphConfig: UseCaseGraphConfig?
202         get() = _activeComponent?.getUseCaseGraphConfig()
203 
204     private val closingCameraJobs = mutableListOf<Job>()
205 
206     private val allControls = controls.toMutableSet().apply { add(camera2CameraControl) }
207 
208     internal fun setCameraGraphCreationMode(createImmediately: Boolean) =
209         synchronized(lock) {
210             shouldCreateCameraGraphImmediately = createImmediately
211             if (shouldCreateCameraGraphImmediately) {
212                 // Clear the UseCaseManager configuration that haven't been "resumed" when we return
213                 // to single camera operating mode early.
214                 deferredUseCaseManagerConfig = null
215             }
216         }
217 
218     internal fun getDeferredCameraGraphConfig() =
219         synchronized(lock) { deferredUseCaseManagerConfig?.cameraGraphConfig }
220 
221     /**
222      * This attaches the specified [useCases] to the current set of attached use cases. When any
223      * changes are identified (i.e., a new use case is added), the subsequent actions would trigger
224      * a recreation of the current CameraGraph if there is one.
225      */
226     public fun attach(useCases: List<UseCase>): Unit =
227         synchronized(lock) {
228             if (useCases.isEmpty()) {
229                 Log.warn { "Attach [] from $this (Ignored)" }
230                 return
231             }
232             Log.debug { "Attaching $useCases from $this" }
233 
234             val unattachedUseCases =
235                 useCases.filter { useCase -> !attachedUseCases.contains(useCase) }
236 
237             // Notify state attached to use cases
238             for (useCase in unattachedUseCases) {
239                 useCase.onStateAttached()
240             }
241 
242             if (attachedUseCases.addAll(useCases)) {
243                 if (!addOrRemoveRepeatingUseCase(getRunningUseCases())) {
244                     updateZslDisabledByUseCaseConfigStatus()
245                     updateLowLightBoostDisabledByUseCaseSessionConfigStatus()
246                     refreshAttachedUseCases(attachedUseCases)
247                 }
248             }
249 
250             if (sessionProcessor != null || !shouldCreateCameraGraphImmediately) {
251                 pendingUseCasesToNotifyCameraControlReady.addAll(unattachedUseCases)
252             } else {
253                 unattachedUseCases.forEach { useCase ->
254                     // Notify CameraControl is ready after the UseCaseCamera is created
255                     useCase.onCameraControlReady()
256                 }
257             }
258         }
259 
260     /**
261      * This detaches the specified [useCases] from the current set of attached use cases. When any
262      * changes are identified (i.e., an existing use case is removed), the subsequent actions would
263      * trigger a recreation of the current CameraGraph.
264      */
265     public fun detach(useCases: List<UseCase>): Unit =
266         synchronized(lock) {
267             if (useCases.isEmpty()) {
268                 Log.warn { "Detaching [] from $this (Ignored)" }
269                 return
270             }
271             Log.debug { "Detaching $useCases from $this" }
272 
273             // When use cases are detached, they should be considered inactive as well. Also note
274             // that
275             // we remove the use cases from our set directly because the subsequent cleanup actions
276             // from
277             // detaching the use cases should suffice here.
278             activeUseCases.removeAll(useCases)
279 
280             // Notify state detached to use cases
281             for (useCase in useCases) {
282                 if (attachedUseCases.contains(useCase)) {
283                     useCase.onStateDetached()
284                 }
285             }
286 
287             // TODO: We might only want to tear down when the number of attached use cases goes to
288             //  zero. If a single UseCase is removed, we could deactivate it?
289             if (attachedUseCases.removeAll(useCases)) {
290                 if (addOrRemoveRepeatingUseCase(getRunningUseCases())) {
291                     return
292                 }
293 
294                 if (attachedUseCases.isEmpty()) {
295                     zslControl.setZslDisabledByUserCaseConfig(false)
296                     lowLightBoostControl.setLowLightBoostDisabledByUseCaseSessionConfig(false)
297                 } else {
298                     updateZslDisabledByUseCaseConfigStatus()
299                     updateLowLightBoostDisabledByUseCaseSessionConfigStatus()
300                 }
301                 refreshAttachedUseCases(attachedUseCases)
302             }
303             pendingUseCasesToNotifyCameraControlReady.removeAll(useCases)
304         }
305 
306     /**
307      * This marks the specified [useCase] as active ("activate"). This refreshes the current set of
308      * active use cases, and if any changes are identified, we update [UseCaseCamera] with the
309      * latest set of "running" (attached and active) use cases, which will in turn trigger actions
310      * for SessionConfig updates.
311      */
312     public fun activate(useCase: UseCase): Unit =
313         synchronized(lock) {
314             if (activeUseCases.add(useCase)) {
315                 refreshRunningUseCases()
316             }
317         }
318 
319     /**
320      * This marks the specified [useCase] as inactive ("deactivate"). This refreshes the current set
321      * of active use cases, and if any changes are identified, we update [UseCaseCamera] with the
322      * latest set of "running" (attached and active) use cases, which will in turn trigger actions
323      * for SessionConfig updates.
324      */
325     public fun deactivate(useCase: UseCase): Unit =
326         synchronized(lock) {
327             if (activeUseCases.remove(useCase)) {
328                 refreshRunningUseCases()
329             }
330         }
331 
332     public fun update(useCase: UseCase): Unit =
333         synchronized(lock) {
334             if (attachedUseCases.contains(useCase)) {
335                 refreshRunningUseCases()
336             }
337         }
338 
339     public fun reset(useCase: UseCase): Unit =
340         synchronized(lock) {
341             if (attachedUseCases.contains(useCase)) {
342                 refreshAttachedUseCases(attachedUseCases)
343             }
344         }
345 
346     public fun setPrimary(isPrimary: Boolean) {
347         synchronized(lock) { this.isPrimary = isPrimary }
348     }
349 
350     public fun setActiveResumeMode(enabled: Boolean): Unit? =
351         synchronized(lock) {
352             activeResumeEnabled = enabled
353             camera?.setActiveResumeMode(enabled)
354         }
355 
356     public suspend fun close() {
357         val closingJobs =
358             synchronized(lock) {
359                 closeCurrentUseCases()
360                 meteringRepeating.onUnbind()
361                 cameraDevices.disconnectAsync(cameraConfig.cameraId).also {
362                     closingCameraJobs.add(it)
363                 }
364                 closingCameraJobs.toList()
365             }
366         closingJobs.joinAll()
367     }
368 
369     override fun toString(): String = "UseCaseManager<${cameraConfig.cameraId}>"
370 
371     @GuardedBy("lock")
372     private fun refreshRunningUseCases() {
373         // refreshRunningUseCases() is called after we activate, deactivate, update or have finished
374         // attaching use cases. If the SessionProcessor is still being initialized, we cannot
375         // refresh the current set of running use cases (we don't have a UseCaseCamera), but we
376         // can safely abort here, because once the SessionProcessor is initialized, we'll resume
377         // the process of creating UseCaseCamera components, finish attaching use cases and finally
378         // invoke refreshingRunningUseCases().
379         if (pendingSessionProcessorInitialization) return
380         val runningUseCases = getRunningUseCases()
381         when {
382             shouldAddRepeatingUseCase(runningUseCases) -> addRepeatingUseCase()
383             shouldRemoveRepeatingUseCase(runningUseCases) -> removeRepeatingUseCase()
384             else -> {
385                 camera?.let {
386                     it.updateRepeatingRequests(isPrimary, runningUseCases)
387                     for (control in allControls) {
388                         if (control is RunningUseCasesChangeListener) {
389                             control.onRunningUseCasesChanged(runningUseCases)
390                         }
391                     }
392                 }
393             }
394         }
395     }
396 
397     private fun UseCaseCamera.updateRepeatingRequests(
398         isPrimary: Boolean,
399         runningUseCases: Set<UseCase>
400     ) {
401         // Note: This may be called with the same set of values that was previously set. This
402         // is used as a signal to indicate the properties of the UseCase may have changed.
403         SessionConfigAdapter(runningUseCases, isPrimary = isPrimary)
404             .getValidSessionConfigOrNull()
405             ?.let { requestControl.setSessionConfigAsync(it) }
406             ?: run {
407                 Log.debug { "Unable to reset the session due to invalid config" }
408                 requestControl.setSessionConfigAsync(
409                     SessionConfig.Builder().apply { setTemplateType(defaultTemplate) }.build()
410                 )
411             }
412     }
413 
414     private fun UseCaseCameraRequestControl.setSessionConfigAsync(
415         sessionConfig: SessionConfig
416     ): Deferred<Unit> =
417         setConfigAsync(
418             type = UseCaseCameraRequestControl.Type.SESSION_CONFIG,
419             config = sessionConfig.implementationOptions,
420             tags = sessionConfig.repeatingCaptureConfig.tagBundle.toMap(),
421             listeners =
422                 setOf(
423                     CameraCallbackMap.createFor(
424                         sessionConfig.repeatingCameraCaptureCallbacks,
425                         useCaseThreads.get().backgroundExecutor
426                     )
427                 ),
428             template = RequestTemplate(sessionConfig.repeatingCaptureConfig.templateType),
429             streams =
430                 useCaseGraphConfig?.getStreamIdsFromSurfaces(
431                     sessionConfig.repeatingCaptureConfig.surfaces
432                 ),
433             sessionConfig = sessionConfig,
434         )
435 
436     @GuardedBy("lock")
437     private fun refreshAttachedUseCases(newUseCases: Set<UseCase>) {
438         closeCurrentUseCases()
439 
440         val useCases = newUseCases.toList()
441 
442         // Update list of active useCases
443         if (useCases.isEmpty()) {
444             for (control in allControls) {
445                 control.requestControl = null
446                 control.reset()
447             }
448             return
449         }
450 
451         val usingLegacyExtensions =
452             sessionProcessor?.implementationType?.first == SessionProcessor.TYPE_VENDOR_LIBRARY
453 
454         if (usingLegacyExtensions || !shouldCreateCameraGraphImmediately) {
455             // We will need to set the UseCaseCamera to null since the new UseCaseCamera along with
456             // its respective CameraGraph configurations won't be ready until:
457             //
458             // - SessionProcessorManager finishes the initialization, _acquires the lock_, and
459             //    resume UseCaseManager successfully
460             // - And/or, the UseCaseManager is ready to be resumed under concurrent camera settings.
461             for (control in allControls) {
462                 control.requestControl = null
463             }
464         }
465 
466         // Enables extensions with the vendor library approach if extension mode is requested but
467         // Camera2 Extensions doesn't support it.
468         if (sessionProcessor != null) {
469             if (usingLegacyExtensions) {
470                 Log.debug { "Setting up UseCaseManager with SessionProcessorManager" }
471                 sessionProcessorManager =
472                     SessionProcessorManager(
473                             sessionProcessor!!,
474                             cameraInfoInternal.get(),
475                             useCaseThreads.get().scope,
476                         )
477                         .also { manager ->
478                             pendingSessionProcessorInitialization = true
479                             manager.initialize(this, useCases) { config ->
480                                 synchronized(lock) {
481                                     if (manager.isClosed()) {
482                                         // We've been cancelled by other use case transactions. This
483                                         // means the attached set of use cases have been updated in
484                                         // the meantime, and the UseCaseManagerConfig we have here
485                                         // is obsolete, so we can simply abort here.
486                                         return@initialize
487                                     }
488                                     if (config == null) {
489                                         Log.error { "Failed to initialize SessionProcessor" }
490                                         manager.close()
491                                         sessionProcessorManager = null
492                                         return@initialize
493                                     }
494                                     pendingSessionProcessorInitialization = false
495                                     this@UseCaseManager.tryResumeUseCaseManager(config)
496                                 }
497                             }
498                         }
499             } else {
500                 Log.debug { "Setting up UseCaseManager with OperatingMode.EXTENSION" }
501                 val sessionConfigAdapter = SessionConfigAdapter(useCases, isPrimary = isPrimary)
502                 val streamConfigMap = mutableMapOf<CameraStream.Config, DeferrableSurface>()
503                 val graphConfig =
504                     createCameraGraphConfig(
505                         OperatingMode.EXTENSION,
506                         sessionConfigAdapter,
507                         streamConfigMap,
508                         callbackMap,
509                         requestListener,
510                         cameraConfig,
511                         cameraQuirks,
512                         zslControl,
513                         templateParamsOverride,
514                         cameraMetadata,
515                         camera2ExtensionMode = sessionProcessor?.implementationType?.second,
516                         isExtensions = true,
517                         enableStreamUseCase = false
518                     )
519 
520                 sessionProcessor!!.initSession(cameraInfoInternal.get(), null)
521 
522                 val useCaseManagerConfig =
523                     UseCaseManagerConfig(
524                         useCases,
525                         sessionConfigAdapter,
526                         graphConfig,
527                         streamConfigMap
528                     )
529                 this.tryResumeUseCaseManager(useCaseManagerConfig)
530             }
531             return
532         } else {
533             val sessionConfigAdapter = SessionConfigAdapter(useCases, isPrimary = isPrimary)
534             val streamConfigMap = mutableMapOf<CameraStream.Config, DeferrableSurface>()
535             val graphConfig = createCameraGraphConfig(sessionConfigAdapter, streamConfigMap)
536 
537             val useCaseManagerConfig =
538                 UseCaseManagerConfig(useCases, sessionConfigAdapter, graphConfig, streamConfigMap)
539             this.tryResumeUseCaseManager(useCaseManagerConfig)
540         }
541     }
542 
543     @GuardedBy("lock")
544     private fun closeCurrentUseCases() {
545         // Close prior camera graph
546         camera.let { useCaseCamera ->
547             _activeComponent = null
548             useCaseCamera?.close()?.let { closingJob ->
549                 if (sessionProcessorManager != null) {
550                     // If the current session was created for extensions. We need to make sure
551                     // the closing procedures are done. This is needed because the same
552                     // SessionProcessor instance may be reused in the next extensions session, and
553                     // we need to make sure we de-initialize the current SessionProcessor session.
554                     runBlocking { closingJob.join() }
555                 } else {
556                     closingCameraJobs.add(closingJob)
557                     closingJob.invokeOnCompletion {
558                         synchronized(lock) { closingCameraJobs.remove(closingJob) }
559                     }
560                 }
561             }
562         }
563         sessionProcessor?.let { sessionProcessor ->
564             val usingLegacyExtensions =
565                 sessionProcessor.implementationType.first == SessionProcessor.TYPE_VENDOR_LIBRARY
566             if (usingLegacyExtensions) {
567                 sessionProcessorManager?.let {
568                     it.close()
569                     sessionProcessorManager = null
570                     pendingSessionProcessorInitialization = false
571                 }
572             } else {
573                 sessionProcessor.setCaptureSessionRequestProcessor(null)
574                 sessionProcessor.deInitSession()
575             }
576         }
577     }
578 
579     @VisibleForTesting
580     @GuardedBy("lock")
581     internal fun tryResumeUseCaseManager(useCaseManagerConfig: UseCaseManagerConfig) {
582         if (!shouldCreateCameraGraphImmediately) {
583             deferredUseCaseManagerConfig = useCaseManagerConfig
584             return
585         }
586         val cameraGraph = cameraPipe.create(useCaseManagerConfig.cameraGraphConfig)
587         beginComponentCreation(useCaseManagerConfig, cameraGraph)
588     }
589 
590     internal fun resumeDeferredComponentCreation(cameraGraph: CameraGraph) =
591         synchronized(lock) {
592             beginComponentCreation(checkNotNull(deferredUseCaseManagerConfig), cameraGraph)
593         }
594 
595     @GuardedBy("lock")
596     private fun beginComponentCreation(
597         useCaseManagerConfig: UseCaseManagerConfig,
598         cameraGraph: CameraGraph
599     ) {
600         val sessionProcessorEnabled =
601             useCaseManagerConfig.sessionConfigAdapter.isSessionProcessorEnabled
602         with(useCaseManagerConfig) {
603             if (sessionProcessorEnabled) {
604                 for ((streamConfig, deferrableSurface) in streamConfigMap) {
605                     cameraGraph.streams[streamConfig]?.let {
606                         cameraGraph.setSurface(it.id, deferrableSurface.surface.get())
607                     }
608                 }
609             }
610 
611             // Create and configure the new camera component.
612             _activeComponent =
613                 builder
614                     .config(
615                         UseCaseCameraConfig(
616                             useCases,
617                             sessionConfigAdapter,
618                             cameraStateAdapter,
619                             cameraGraph,
620                             streamConfigMap,
621                             sessionProcessorManager,
622                         )
623                     )
624                     .build()
625 
626             for (control in allControls) {
627                 control.requestControl = camera?.requestControl
628             }
629 
630             setCaptureSessionRequestProcessor(sessionConfigAdapter, cameraGraph)
631 
632             camera?.setActiveResumeMode(activeResumeEnabled)
633 
634             refreshRunningUseCases()
635         }
636 
637         Log.debug { "Notifying $pendingUseCasesToNotifyCameraControlReady camera control ready" }
638         for (useCase in pendingUseCasesToNotifyCameraControlReady) {
639             useCase.onCameraControlReady()
640         }
641         pendingUseCasesToNotifyCameraControlReady.clear()
642     }
643 
644     private fun setCaptureSessionRequestProcessor(
645         sessionConfigAdapter: SessionConfigAdapter,
646         cameraGraph: CameraGraph
647     ) {
648         val useCamera2Extension =
649             sessionProcessor?.implementationType?.first == SessionProcessor.TYPE_CAMERA2_EXTENSION
650         if (useCamera2Extension) {
651             val stillCaptureStreamId: StreamId? =
652                 sessionConfigAdapter.getValidSessionConfigOrNull()?.let { sessionConfig ->
653                     val repeatingSurfaces = sessionConfig.repeatingCaptureConfig.surfaces
654                     sessionConfig.surfaces
655                         .find { surface ->
656                             surface !in repeatingSurfaces
657                         } // Find the first non-repeating surface (nullable)
658                         ?.let { surface -> // If found...
659                             useCaseGraphConfig?.getStreamIdsFromSurfaces(
660                                 listOf(surface)
661                             ) // Get its StreamIds (nullable list)
662                         }
663                         ?.firstOrNull() // Get the first StreamId or null
664                 }
665 
666             sessionProcessor?.setCaptureSessionRequestProcessor(
667                 object : SessionProcessor.CaptureSessionRequestProcessor {
668                     override fun getRealtimeStillCaptureLatency(): Pair<Long, Long>? {
669                         val outputLatency =
670                             cameraGraph.streams.getOutputLatency(stillCaptureStreamId!!)
671                                 ?: return null
672                         val captureLatencyMs =
673                             outputLatency.estimatedCaptureLatencyNs.div(1_000_000)
674                         val processingLatencyMs =
675                             outputLatency.estimatedProcessingLatencyNs.div(1_000_000)
676                         return Pair.create(captureLatencyMs, processingLatencyMs)
677                     }
678 
679                     override fun setExtensionStrength(strength: Int) {
680                         if (Build.VERSION.SDK_INT >= 34) {
681                             camera
682                                 ?.requestControl
683                                 ?.setParametersAsync(
684                                     values =
685                                         mutableMapOf(CaptureRequest.EXTENSION_STRENGTH to strength)
686                                 )
687                         }
688                     }
689                 }
690             )
691         }
692     }
693 
694     @GuardedBy("lock")
695     private fun getRunningUseCases(): Set<UseCase> {
696         return attachedUseCases.intersect(activeUseCases)
697     }
698 
699     @TestOnly
700     @VisibleForTesting
701     public fun getRunningUseCasesForTest(): Set<UseCase> =
702         synchronized(lock) {
703             return getRunningUseCases()
704         }
705 
706     /**
707      * Adds or removes repeating use case if needed.
708      *
709      * @param runningUseCases the set of currently running use cases
710      * @return true if repeating use cases is added or removed, false otherwise
711      */
712     @GuardedBy("lock")
713     private fun addOrRemoveRepeatingUseCase(runningUseCases: Set<UseCase>): Boolean {
714         if (shouldAddRepeatingUseCase(runningUseCases)) {
715             addRepeatingUseCase()
716             return true
717         }
718         if (shouldRemoveRepeatingUseCase(runningUseCases)) {
719             removeRepeatingUseCase()
720             return true
721         }
722         return false
723     }
724 
725     @GuardedBy("lock")
726     private fun shouldAddRepeatingUseCase(runningUseCases: Set<UseCase>): Boolean {
727         val meteringRepeatingEnabled = attachedUseCases.contains(meteringRepeating)
728         if (!meteringRepeatingEnabled) {
729             val activeSurfaces = runningUseCases.withoutMetering().surfaceCount()
730             return activeSurfaces > 0 &&
731                 with(attachedUseCases.withoutMetering()) {
732                     (onlyVideoCapture() || requireMeteringRepeating()) &&
733                         isMeteringCombinationSupported()
734                 }
735         }
736         return false
737     }
738 
739     @GuardedBy("lock")
740     private fun addRepeatingUseCase() {
741         meteringRepeating.bindToCamera(cameraInternal.get(), null, null, null)
742         meteringRepeating.setupSession()
743         attach(listOf(meteringRepeating))
744         activate(meteringRepeating)
745     }
746 
747     @GuardedBy("lock")
748     private fun shouldRemoveRepeatingUseCase(runningUseCases: Set<UseCase>): Boolean {
749         val meteringRepeatingEnabled = runningUseCases.contains(meteringRepeating)
750         if (meteringRepeatingEnabled) {
751             val activeSurfaces = runningUseCases.withoutMetering().surfaceCount()
752             return activeSurfaces == 0 ||
753                 with(attachedUseCases.withoutMetering()) {
754                     !(onlyVideoCapture() || requireMeteringRepeating()) ||
755                         !isMeteringCombinationSupported()
756                 }
757         }
758         return false
759     }
760 
761     @GuardedBy("lock")
762     private fun removeRepeatingUseCase() {
763         deactivate(meteringRepeating)
764         detach(listOf(meteringRepeating))
765         meteringRepeating.unbindFromCamera(cameraInternal.get())
766     }
767 
768     internal fun createCameraGraphConfig(
769         sessionConfigAdapter: SessionConfigAdapter,
770         streamConfigMap: MutableMap<CameraStream.Config, DeferrableSurface>,
771         isExtensions: Boolean = false,
772     ): CameraGraph.Config {
773         return createCameraGraphConfig(
774             sessionConfigAdapter.getValidSessionConfigOrNull()?.let { sessionConfig ->
775                 when (sessionConfig.sessionType) {
776                     SESSION_REGULAR -> OperatingMode.NORMAL
777                     SESSION_HIGH_SPEED -> OperatingMode.HIGH_SPEED
778                     else -> OperatingMode.custom(sessionConfig.sessionType)
779                 }
780             } ?: OperatingMode.NORMAL,
781             sessionConfigAdapter,
782             streamConfigMap,
783             callbackMap,
784             requestListener,
785             cameraConfig,
786             cameraQuirks,
787             zslControl,
788             templateParamsOverride,
789             cameraMetadata,
790             camera2ExtensionMode = null,
791             isExtensions = isExtensions,
792         )
793     }
794 
795     private fun Collection<UseCase>.onlyVideoCapture(): Boolean {
796         return isNotEmpty() &&
797             checkSurfaces { _, sessionSurfaces ->
798                 sessionSurfaces.isNotEmpty() &&
799                     sessionSurfaces.all { it.containerClass == MediaCodec::class.java }
800             }
801     }
802 
803     private fun Collection<UseCase>.isMeteringCombinationSupported(): Boolean {
804         if (meteringRepeating.attachedSurfaceResolution == null) {
805             meteringRepeating.setupSession()
806         }
807 
808         val attachedSurfaceInfoList = getAttachedSurfaceInfoList()
809 
810         if (attachedSurfaceInfoList.isEmpty()) {
811             return false
812         }
813 
814         val sessionSurfacesConfigs = getSessionSurfacesConfigs()
815 
816         return supportedSurfaceCombination
817             .checkSupported(
818                 SupportedSurfaceCombination.FeatureSettings(
819                     getCameraMode(),
820                     getRequiredMaxBitDepth(attachedSurfaceInfoList),
821                     isPreviewStabilizationOn(),
822                     isUltraHdrOn()
823                 ),
824                 mutableListOf<SurfaceConfig>().apply {
825                     addAll(sessionSurfacesConfigs)
826                     add(createMeteringRepeatingSurfaceConfig())
827                 }
828             )
829             .also {
830                 Log.debug {
831                     "Combination of $sessionSurfacesConfigs + $meteringRepeating is supported: $it"
832                 }
833             }
834     }
835 
836     private fun getCameraMode(): Int {
837         synchronized(lock) {
838             if (
839                 cameraCoordinator.cameraOperatingMode ==
840                     CameraCoordinator.CAMERA_OPERATING_MODE_CONCURRENT
841             ) {
842                 return CameraMode.CONCURRENT_CAMERA
843             }
844         }
845 
846         return CameraMode.DEFAULT
847     }
848 
849     private fun getRequiredMaxBitDepth(attachedSurfaceInfoList: List<AttachedSurfaceInfo>): Int {
850         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
851             dynamicRangeResolver
852                 .resolveAndValidateDynamicRanges(
853                     attachedSurfaceInfoList,
854                     listOf(meteringRepeating.currentConfig),
855                     listOf(0)
856                 )
857                 .forEach { (_, u) ->
858                     if (u.bitDepth == DynamicRange.BIT_DEPTH_10_BIT) {
859                         return DynamicRange.BIT_DEPTH_10_BIT
860                     }
861                 }
862         }
863 
864         return DynamicRange.BIT_DEPTH_8_BIT
865     }
866 
867     private fun Collection<UseCase>.getAttachedSurfaceInfoList(): List<AttachedSurfaceInfo> =
868         mutableListOf<AttachedSurfaceInfo>().apply {
869             this@getAttachedSurfaceInfoList.forEach { useCase ->
870                 val surfaceResolution = useCase.attachedSurfaceResolution
871                 val streamSpec = useCase.attachedStreamSpec
872 
873                 // When collecting the info, the UseCases might be unbound to make these info
874                 // become null.
875                 if (surfaceResolution == null || streamSpec == null) {
876                     Log.warn { "Invalid surface resolution or stream spec is found." }
877                     clear()
878                     return@apply
879                 }
880 
881                 val surfaceConfig =
882                     supportedSurfaceCombination.transformSurfaceConfig(
883                         getCameraMode(),
884                         useCase.currentConfig.inputFormat,
885                         surfaceResolution
886                     )
887                 add(
888                     AttachedSurfaceInfo.create(
889                         surfaceConfig,
890                         useCase.currentConfig.inputFormat,
891                         surfaceResolution,
892                         streamSpec.dynamicRange,
893                         useCase.getCaptureTypes(),
894                         streamSpec.implementationOptions ?: MutableOptionsBundle.create(),
895                         useCase.currentConfig.getTargetFrameRate(null),
896                         checkNotNull(
897                             useCase.currentConfig.getTargetHighSpeedFrameRate(
898                                 FRAME_RATE_RANGE_UNSPECIFIED
899                             )
900                         ),
901                     )
902                 )
903             }
904         }
905 
906     private fun UseCase.getCaptureTypes() =
907         if (this is StreamSharing) {
908             (currentConfig as StreamSharingConfig).captureTypes
909         } else {
910             listOf(currentConfig.captureType)
911         }
912 
913     private fun Collection<UseCase>.isPreviewStabilizationOn() =
914         filterIsInstance<Preview>().firstOrNull()?.currentConfig?.previewStabilizationMode ==
915             StabilizationMode.ON
916 
917     private fun Collection<UseCase>.isUltraHdrOn() =
918         filterIsInstance<ImageCapture>().firstOrNull()?.currentConfig?.inputFormat ==
919             ImageFormat.JPEG_R
920 
921     private fun Collection<UseCase>.getSessionSurfacesConfigs(): List<SurfaceConfig> =
922         mutableListOf<SurfaceConfig>().apply {
923             this@getSessionSurfacesConfigs.forEach { useCase ->
924                 useCase.sessionConfig.surfaces.forEach { deferrableSurface ->
925                     add(
926                         supportedSurfaceCombination.transformSurfaceConfig(
927                             getCameraMode(),
928                             useCase.currentConfig.inputFormat,
929                             deferrableSurface.prescribedSize
930                         )
931                     )
932                 }
933             }
934         }
935 
936     private fun createMeteringRepeatingSurfaceConfig() =
937         supportedSurfaceCombination.transformSurfaceConfig(
938             getCameraMode(),
939             meteringRepeating.imageFormat,
940             meteringRepeating.attachedSurfaceResolution!!
941         )
942 
943     private fun Collection<UseCase>.surfaceCount(): Int =
944         ValidatingBuilder().let { validatingBuilder ->
945             forEach { useCase -> validatingBuilder.add(useCase.sessionConfig) }
946             return validatingBuilder.build().surfaces.size
947         }
948 
949     private fun Collection<UseCase>.withoutMetering(): Collection<UseCase> = filterNot {
950         it is MeteringRepeating
951     }
952 
953     private fun Collection<UseCase>.requireMeteringRepeating(): Boolean {
954         return isNotEmpty() &&
955             checkSurfaces { repeatingSurfaces, sessionSurfaces ->
956                 // There is no repeating UseCases
957                 sessionSurfaces.isNotEmpty() && repeatingSurfaces.isEmpty()
958             }
959     }
960 
961     private fun Collection<UseCase>.checkSurfaces(
962         predicate:
963             (
964                 repeatingSurfaces: List<DeferrableSurface>, sessionSurfaces: List<DeferrableSurface>
965             ) -> Boolean
966     ): Boolean =
967         ValidatingBuilder().let { validatingBuilder ->
968             forEach { useCase -> validatingBuilder.add(useCase.sessionConfig) }
969             val sessionConfig = validatingBuilder.build()
970             val captureConfig = sessionConfig.repeatingCaptureConfig
971             return predicate(captureConfig.surfaces, sessionConfig.surfaces)
972         }
973 
974     private fun updateZslDisabledByUseCaseConfigStatus() {
975         val disableZsl = attachedUseCases.any { it.currentConfig.isZslDisabled(false) }
976         zslControl.setZslDisabledByUserCaseConfig(disableZsl)
977     }
978 
979     private fun updateLowLightBoostDisabledByUseCaseSessionConfigStatus() {
980         if (!cameraProperties.metadata.supportsLowLightBoost) {
981             return
982         }
983 
984         // Low-light boost should be disabled when expected frame rate range exceeds 30.
985         if (attachedUseCases.getSessionConfig().expectedFrameRateRange.upper > 30) {
986             lowLightBoostControl.setLowLightBoostDisabledByUseCaseSessionConfig(true)
987             return
988         }
989 
990         // HDR 10-bit can be supported since API level 33
991         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
992             return
993         }
994 
995         // Low-light boost should be disabled when dynamic range setting is not 8-bit.
996         val attachedSurfaceInfoList = attachedUseCases.getAttachedSurfaceInfoList()
997         if (getRequiredMaxBitDepth(attachedSurfaceInfoList) != DynamicRange.BIT_DEPTH_8_BIT) {
998             lowLightBoostControl.setLowLightBoostDisabledByUseCaseSessionConfig(true)
999             return
1000         }
1001 
1002         lowLightBoostControl.setLowLightBoostDisabledByUseCaseSessionConfig(false)
1003     }
1004 
1005     private fun Collection<UseCase>.getSessionConfig(): SessionConfig =
1006         ValidatingBuilder().apply { forEach { useCase -> add(useCase.sessionConfig) } }.build()
1007 
1008     /**
1009      * This interface defines a listener that is notified when the set of running UseCases changes.
1010      *
1011      * A "running" UseCase is one that is both attached and active, meaning it's bound to the
1012      * lifecycle and ready to receive camera frames.
1013      *
1014      * Classes implementing this interface can take action when the active UseCase configuration
1015      * changes.
1016      */
1017     public interface RunningUseCasesChangeListener {
1018 
1019         /**
1020          * Invoked when the set of running UseCases has been modified (added, removed, or updated).
1021          *
1022          * @param runningUseCases The updated set of UseCases that are currently running.
1023          */
1024         public fun onRunningUseCasesChanged(runningUseCases: Set<UseCase>)
1025     }
1026 
1027     public companion object {
1028         internal data class UseCaseManagerConfig(
1029             val useCases: List<UseCase>,
1030             val sessionConfigAdapter: SessionConfigAdapter,
1031             val cameraGraphConfig: CameraGraph.Config,
1032             val streamConfigMap: MutableMap<CameraStream.Config, DeferrableSurface>
1033         )
1034 
1035         public fun SessionConfig.toCamera2ImplConfig(): Camera2ImplConfig {
1036             return Camera2ImplConfig(implementationOptions)
1037         }
1038 
1039         // return video stabilization mode. null indicate mode unspecified.
1040         public fun getVideoStabilizationModeFromCaptureConfig(captureConfig: CaptureConfig): Int? {
1041             val isPreviewStabilizationMode = captureConfig.previewStabilizationMode
1042             val isVideoStabilizationMode = captureConfig.videoStabilizationMode
1043 
1044             return if (
1045                 isPreviewStabilizationMode == StabilizationMode.OFF ||
1046                     isVideoStabilizationMode == StabilizationMode.OFF
1047             ) {
1048                 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_OFF
1049             } else if (isPreviewStabilizationMode == StabilizationMode.ON) {
1050                 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION
1051             } else if (isVideoStabilizationMode == StabilizationMode.ON) {
1052                 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON
1053             } else {
1054                 null
1055             }
1056         }
1057 
1058         public fun createCameraGraphConfig(
1059             operatingMode: OperatingMode,
1060             sessionConfigAdapter: SessionConfigAdapter,
1061             streamConfigMap: MutableMap<CameraStream.Config, DeferrableSurface>,
1062             callbackMap: CameraCallbackMap,
1063             requestListener: ComboRequestListener,
1064             cameraConfig: CameraConfig,
1065             cameraQuirks: CameraQuirks,
1066             zslControl: ZslControl,
1067             templateParamsOverride: TemplateParamsOverride,
1068             cameraMetadata: CameraMetadata?,
1069             camera2ExtensionMode: Int? = null,
1070             isExtensions: Boolean = false,
1071             enableStreamUseCase: Boolean = true,
1072         ): CameraGraph.Config {
1073             var containsVideo = false
1074             val streamGroupMap = mutableMapOf<Int, MutableList<CameraStream.Config>>()
1075             val inputStreams = mutableListOf<InputStream.Config>()
1076             var sessionTemplate = RequestTemplate(TEMPLATE_PREVIEW)
1077             val sessionParameters: MutableMap<Any, Any> = mutableMapOf()
1078             sessionConfigAdapter.getValidSessionConfigOrNull()?.let { sessionConfig ->
1079                 if (sessionConfig.templateType != CaptureConfig.TEMPLATE_TYPE_NONE) {
1080                     sessionTemplate = RequestTemplate(sessionConfig.templateType)
1081                 }
1082                 sessionParameters.putAll(templateParamsOverride.getOverrideParams(sessionTemplate))
1083                 sessionParameters.putAll(sessionConfig.implementationOptions.toParameters())
1084                 if (operatingMode == OperatingMode.EXTENSION) {
1085                     // camera2ExtensionMode must be non-null when operatingMode is EXTENSION
1086                     sessionParameters[CameraPipeKeys.camera2ExtensionMode] = camera2ExtensionMode!!
1087                 }
1088 
1089                 val physicalCameraIdForAllStreams =
1090                     sessionConfig.toCamera2ImplConfig().getPhysicalCameraId(null)
1091                 var zslStream: CameraStream.Config? = null
1092                 for (outputConfig in sessionConfig.outputConfigs) {
1093                     val deferrableSurface = outputConfig.surface
1094                     val physicalCameraId =
1095                         physicalCameraIdForAllStreams ?: outputConfig.physicalCameraId
1096                     val dynamicRange = outputConfig.dynamicRange
1097                     val mirrorMode = outputConfig.mirrorMode
1098                     val outputStreamConfig =
1099                         OutputStream.Config.create(
1100                             dynamicRangeProfile =
1101                                 dynamicRange.toDynamicRangeProfiles(cameraMetadata),
1102                             size = deferrableSurface.prescribedSize,
1103                             format = StreamFormat(deferrableSurface.prescribedStreamFormat),
1104                             camera =
1105                                 if (physicalCameraId == null) {
1106                                     null
1107                                 } else {
1108                                     CameraId.fromCamera2Id(physicalCameraId)
1109                                 },
1110                             // No need to map MIRROR_MODE_ON_FRONT_ONLY to MIRROR_MODE_AUTO
1111                             // since its default value in framework
1112                             mirrorMode =
1113                                 when (mirrorMode) {
1114                                     MirrorMode.MIRROR_MODE_OFF ->
1115                                         OutputStream.MirrorMode(
1116                                             OutputConfiguration.MIRROR_MODE_NONE
1117                                         )
1118                                     MirrorMode.MIRROR_MODE_ON ->
1119                                         OutputStream.MirrorMode(OutputConfiguration.MIRROR_MODE_H)
1120                                     else -> null
1121                                 },
1122                             streamUseCase =
1123                                 if (enableStreamUseCase) {
1124                                     getStreamUseCase(
1125                                         deferrableSurface,
1126                                         sessionConfigAdapter.surfaceToStreamUseCaseMap,
1127                                         cameraMetadata,
1128                                     )
1129                                 } else {
1130                                     null
1131                                 },
1132                             streamUseHint =
1133                                 if (enableStreamUseCase) {
1134                                     getStreamUseHint(
1135                                         deferrableSurface,
1136                                         sessionConfigAdapter.surfaceToStreamUseHintMap
1137                                     )
1138                                 } else {
1139                                     null
1140                                 },
1141                         )
1142                     val surfaces = outputConfig.sharedSurfaces + deferrableSurface
1143                     for (surface in surfaces) {
1144                         val stream = CameraStream.Config.create(outputStreamConfig)
1145                         streamConfigMap[stream] = surface
1146                         if (outputConfig.surfaceGroupId != SURFACE_GROUP_ID_NONE) {
1147                             val streamList = streamGroupMap[outputConfig.surfaceGroupId]
1148                             if (streamList == null) {
1149                                 streamGroupMap[outputConfig.surfaceGroupId] = mutableListOf(stream)
1150                             } else {
1151                                 streamList.add(stream)
1152                             }
1153                         }
1154                         if (surface.containerClass == MediaCodec::class.java) {
1155                             containsVideo = true
1156                         }
1157                         if (surface != deferrableSurface) continue
1158                         if (zslControl.isZslSurface(surface, sessionConfig)) {
1159                             zslStream = stream
1160                         }
1161                     }
1162                 }
1163                 if (sessionConfig.inputConfiguration != null) {
1164                     zslStream?.let {
1165                         inputStreams.add(
1166                             InputStream.Config(
1167                                 stream = it,
1168                                 maxImages = 1,
1169                                 streamFormat = it.outputs.single().format,
1170                             )
1171                         )
1172                     }
1173                 }
1174             }
1175 
1176             val combinedFlags = createCameraGraphFlags(cameraQuirks, containsVideo, isExtensions)
1177 
1178             // Set video stabilization mode to capture request
1179             var videoStabilizationMode: Int? = null
1180             if (sessionConfigAdapter.getValidSessionConfigOrNull() != null) {
1181                 val config =
1182                     sessionConfigAdapter.getValidSessionConfigOrNull()!!.repeatingCaptureConfig
1183                 videoStabilizationMode = getVideoStabilizationModeFromCaptureConfig(config)
1184             }
1185 
1186             // Set fps range to capture request
1187             val targetFpsRange = sessionConfigAdapter.getExpectedFrameRateRange()
1188             val defaultParameters =
1189                 buildMap<Any, Any?> {
1190                     if (isExtensions) {
1191                         set(CameraPipeKeys.ignore3ARequiredParameters, true)
1192                     }
1193                     videoStabilizationMode?.let {
1194                         set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, it)
1195                     }
1196                     set(
1197                         CameraPipeKeys.camera2CaptureRequestTag,
1198                         "android.hardware.camera2.CaptureRequest.setTag.CX"
1199                     )
1200                     targetFpsRange?.let {
1201                         set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, targetFpsRange)
1202                     }
1203                 }
1204             targetFpsRange?.let {
1205                 sessionParameters[CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE] = targetFpsRange
1206             }
1207 
1208             val postviewStream =
1209                 sessionConfigAdapter.getValidSessionConfigOrNull()?.let { sessionConfig ->
1210                     val physicalCameraIdForAllStreams =
1211                         sessionConfig.toCamera2ImplConfig().getPhysicalCameraId(null)
1212                     sessionConfig.postviewOutputConfig?.let { postviewOutputConfig ->
1213                         createPostviewStream(postviewOutputConfig, physicalCameraIdForAllStreams)
1214                             ?.also { streamConfigMap[it] = postviewOutputConfig.surface }
1215                     }
1216                 }
1217 
1218             // TODO: b/327517884 - Add a quirk to not abort captures on stop for certain OEMs during
1219             //   extension sessions.
1220 
1221             // Build up a config (using TEMPLATE_PREVIEW by default)
1222             return CameraGraph.Config(
1223                 camera = cameraConfig.cameraId,
1224                 streams = streamConfigMap.keys.toList(),
1225                 exclusiveStreamGroups = streamGroupMap.values.toList(),
1226                 input = if (inputStreams.isEmpty()) null else inputStreams,
1227                 postviewStream = postviewStream,
1228                 sessionTemplate = sessionTemplate,
1229                 sessionParameters = sessionParameters,
1230                 sessionMode = operatingMode,
1231                 defaultListeners = listOf(callbackMap, requestListener),
1232                 defaultParameters = defaultParameters,
1233                 flags = combinedFlags,
1234             )
1235         }
1236 
1237         private fun createPostviewStream(
1238             postviewConfig: SessionConfig.OutputConfig,
1239             physicalCameraIdForAllStreams: String?
1240         ): CameraStream.Config? {
1241             val deferrableSurface = postviewConfig.surface
1242             val physicalCameraId = physicalCameraIdForAllStreams ?: postviewConfig.physicalCameraId
1243             val mirrorMode = postviewConfig.mirrorMode
1244             val outputStreamConfig =
1245                 OutputStream.Config.create(
1246                     size = deferrableSurface.prescribedSize,
1247                     format = StreamFormat(deferrableSurface.prescribedStreamFormat),
1248                     camera =
1249                         if (physicalCameraId == null) {
1250                             null
1251                         } else {
1252                             CameraId.fromCamera2Id(physicalCameraId)
1253                         },
1254                     // No need to map MIRROR_MODE_ON_FRONT_ONLY to MIRROR_MODE_AUTO
1255                     // since its default value in framework
1256                     mirrorMode =
1257                         when (mirrorMode) {
1258                             MirrorMode.MIRROR_MODE_OFF ->
1259                                 OutputStream.MirrorMode(OutputConfiguration.MIRROR_MODE_NONE)
1260                             MirrorMode.MIRROR_MODE_ON ->
1261                                 OutputStream.MirrorMode(OutputConfiguration.MIRROR_MODE_H)
1262                             else -> null
1263                         },
1264                 )
1265             return CameraStream.Config.create(outputStreamConfig)
1266         }
1267 
1268         private fun getStreamUseCase(
1269             deferrableSurface: DeferrableSurface,
1270             mapping: Map<DeferrableSurface, Long>,
1271             cameraMetadata: CameraMetadata?,
1272         ): OutputStream.StreamUseCase? {
1273             val expectedStreamUseCase =
1274                 mapping[deferrableSurface]?.let { OutputStream.StreamUseCase(it) }
1275             return if (
1276                 Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
1277                     expectedStreamUseCase != null &&
1278                     cameraMetadata
1279                         ?.get(CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES)
1280                         ?.contains(expectedStreamUseCase.value) == true
1281             ) {
1282                 expectedStreamUseCase
1283             } else {
1284                 Log.warn {
1285                     "Expected stream use case for $deferrableSurface, " +
1286                         "$expectedStreamUseCase cannot be set!"
1287                 }
1288                 null
1289             }
1290         }
1291 
1292         private fun getStreamUseHint(
1293             deferrableSurface: DeferrableSurface,
1294             mapping: Map<DeferrableSurface, Long>
1295         ): OutputStream.StreamUseHint? {
1296             return mapping[deferrableSurface]?.let { OutputStream.StreamUseHint(it) }
1297         }
1298 
1299         private fun createCameraGraphFlags(
1300             cameraQuirks: CameraQuirks,
1301             containsVideo: Boolean,
1302             isExtensions: Boolean,
1303         ): CameraGraph.Flags {
1304             if (cameraQuirks.quirks.contains(CaptureSessionStuckQuirk::class.java)) {
1305                 Log.debug { "CameraPipe should be enabling CaptureSessionStuckQuirk by default" }
1306             }
1307             // TODO(b/276354253): Set quirkWaitForRepeatingRequestOnDisconnect flag for overrides.
1308 
1309             // TODO(b/277310425): When creating a CameraGraph, this flag should be turned OFF when
1310             //  this behavior is not needed based on the use case interaction and the device on
1311             //  which the test is running.
1312             val shouldFinalizeSessionOnCloseBehavior = FinalizeSessionOnCloseQuirk.getBehavior()
1313 
1314             val shouldCloseCaptureSessionOnDisconnect =
1315                 when {
1316                     isExtensions -> true
1317                     // If we can release Surfaces immediately, we'll finalize the session when the
1318                     // camera graph is closed (through FinalizeSessionOnCloseQuirk), and thus we
1319                     // won't need to explicitly close the capture session.
1320                     CameraQuirks.isImmediateSurfaceReleaseAllowed() -> false
1321                     cameraQuirks.quirks.contains(CloseCaptureSessionOnVideoQuirk::class.java) &&
1322                         containsVideo -> true
1323                     DeviceQuirks[CloseCaptureSessionOnDisconnectQuirk::class.java] != null -> true
1324                     else -> false
1325                 }
1326 
1327             val shouldCloseCameraDeviceOnClose =
1328                 DeviceQuirks[CloseCameraDeviceOnCameraGraphCloseQuirk::class.java] != null
1329 
1330             val shouldAbortCapturesOnStop =
1331                 when {
1332                     isExtensions &&
1333                         DeviceQuirks[
1334                             DisableAbortCapturesOnStopWithSessionProcessorQuirk::class.java] !=
1335                             null -> false
1336                     DeviceQuirks[DisableAbortCapturesOnStopQuirk::class.java] != null -> false
1337                     /** @see [CameraGraph.Flags.abortCapturesOnStop] */
1338                     Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> true
1339                     else -> false
1340                 }
1341 
1342             val repeatingRequestsToCompleteBeforeNonRepeatingCapture =
1343                 if (
1344                     cameraQuirks.quirks.contains(
1345                         QuickSuccessiveImageCaptureFailsRepeatingRequestQuirk::class.java
1346                     )
1347                 ) {
1348                     1u
1349                 } else {
1350                     0u
1351                 }
1352 
1353             return CameraGraph.Flags(
1354                 abortCapturesOnStop = shouldAbortCapturesOnStop,
1355                 awaitRepeatingRequestBeforeCapture =
1356                     CameraGraph.RepeatingRequestRequirementsBeforeCapture(
1357                         repeatingFramesToComplete =
1358                             repeatingRequestsToCompleteBeforeNonRepeatingCapture,
1359                         // TODO: b/364491700 - use CompletionBehavior.EXACT to disable CameraPipe
1360                         //  internal workaround when not required. See
1361                         //  Camera2Quirks.getRepeatingRequestFrameCountForCapture for details.
1362                         completionBehavior = AT_LEAST,
1363                     ),
1364                 closeCaptureSessionOnDisconnect = shouldCloseCaptureSessionOnDisconnect,
1365                 closeCameraDeviceOnClose = shouldCloseCameraDeviceOnClose,
1366                 finalizeSessionOnCloseBehavior = shouldFinalizeSessionOnCloseBehavior,
1367                 enableRestartDelays = true,
1368             )
1369         }
1370 
1371         private fun DynamicRange.toDynamicRangeProfiles(
1372             cameraMetadata: CameraMetadata?
1373         ): DynamicRangeProfile? {
1374             var dynamicRangeProfile: DynamicRangeProfile? = null
1375 
1376             if (Build.VERSION.SDK_INT >= 33) {
1377                 dynamicRangeProfile = DynamicRangeProfile.STANDARD
1378 
1379                 val dynamicRangeProfilesCompat =
1380                     cameraMetadata?.let { metadata ->
1381                         DynamicRangeProfilesCompat.fromCameraMetaData(metadata)
1382                     }
1383                 val supportedProfiles = dynamicRangeProfilesCompat?.toDynamicRangeProfiles()
1384 
1385                 if (supportedProfiles != null) {
1386                     val firstSupportedProfile =
1387                         dynamicRangeToFirstSupportedProfile(this, supportedProfiles)
1388                     if (firstSupportedProfile != null) {
1389                         dynamicRangeProfile = DynamicRangeProfile(firstSupportedProfile)
1390                     } else {
1391                         Log.error {
1392                             "Requested dynamic range is not supported. Defaulting to STANDARD" +
1393                                 " dynamic range profile.\nRequested dynamic range:\n $this"
1394                         }
1395                     }
1396                 }
1397             }
1398 
1399             return dynamicRangeProfile
1400         }
1401     }
1402 }
1403