1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.integration.impl
18 
19 import android.view.Surface
20 import androidx.annotation.GuardedBy
21 import androidx.camera.camera2.pipe.CameraGraph
22 import androidx.camera.camera2.pipe.CameraPipe
23 import androidx.camera.camera2.pipe.CameraSurfaceManager
24 import androidx.camera.camera2.pipe.StreamId
25 import androidx.camera.camera2.pipe.core.Log
26 import androidx.camera.camera2.pipe.integration.adapter.SessionConfigAdapter
27 import androidx.camera.camera2.pipe.integration.compat.workaround.InactiveSurfaceCloser
28 import androidx.camera.camera2.pipe.integration.config.UseCaseCameraScope
29 import androidx.camera.core.impl.DeferrableSurface
30 import androidx.camera.core.impl.DeferrableSurface.SurfaceClosedException
31 import androidx.camera.core.impl.DeferrableSurfaces
32 import androidx.camera.core.impl.utils.futures.Futures
33 import androidx.concurrent.futures.await
34 import javax.inject.Inject
35 import kotlinx.coroutines.CancellationException
36 import kotlinx.coroutines.CompletableDeferred
37 import kotlinx.coroutines.Deferred
38 import kotlinx.coroutines.TimeoutCancellationException
39 import kotlinx.coroutines.async
40 import kotlinx.coroutines.isActive
41 import kotlinx.coroutines.launch
42 import kotlinx.coroutines.withTimeoutOrNull
43 
44 private const val TIMEOUT_GET_SURFACE_IN_MS = 5_000L
45 
46 /** Configure the [DeferrableSurface]s to the [CameraGraph] and monitor the usage. */
47 @UseCaseCameraScope
48 public open class UseCaseSurfaceManager
49 @Inject
50 constructor(
51     private val threads: UseCaseThreads,
52     private val cameraPipe: CameraPipe,
53     private val inactiveSurfaceCloser: InactiveSurfaceCloser,
54     private val sessionConfigAdapter: SessionConfigAdapter
55 ) : CameraSurfaceManager.SurfaceListener {
56 
57     private val lock = Any()
58 
59     @GuardedBy("lock") private var setupDeferred: Deferred<Boolean>? = null
60 
61     @GuardedBy("lock") private val activeSurfaceMap = mutableMapOf<Surface, DeferrableSurface>()
62 
63     @GuardedBy("lock") private var configuredSurfaceMap: Map<Surface, DeferrableSurface>? = null
64 
65     @GuardedBy("lock") private var stopDeferred: CompletableDeferred<Unit>? = null
66 
67     /** Async set up the Surfaces to the [CameraGraph] */
68     public fun setupAsync(
69         graph: CameraGraph,
70         sessionConfigAdapter: SessionConfigAdapter,
71         surfaceToStreamMap: Map<DeferrableSurface, StreamId>,
72         timeoutMillis: Long = TIMEOUT_GET_SURFACE_IN_MS,
73     ): Deferred<Boolean> =
74         synchronized(lock) {
75             check(setupDeferred == null) { "Surfaces should only be set up once!" }
76             check(stopDeferred == null) { "Surfaces being setup after stopped!" }
77             check(configuredSurfaceMap == null)
78 
79             val deferrableSurfaces = sessionConfigAdapter.deferrableSurfaces
80             try {
81                 DeferrableSurfaces.incrementAll(deferrableSurfaces)
82             } catch (e: SurfaceClosedException) {
83                 Log.error { "Failed to increment DeferrableSurfaces: Surfaces closed" }
84                 // Report Surface invalid by launching a coroutine to avoid cyclic Dagger injection.
85                 threads.scope.launch {
86                     sessionConfigAdapter.reportSurfaceInvalid(e.deferrableSurface)
87                 }
88                 return@synchronized CompletableDeferred(false)
89             }
90 
91             val deferred =
92                 threads.scope
93                     .async {
94                         check(sessionConfigAdapter.isSessionConfigValid())
95 
96                         val surfaces =
97                             try {
98                                 getSurfaces(deferrableSurfaces, timeoutMillis)
99                             } catch (e: SurfaceClosedException) {
100                                 Log.error(e) { "Failed to get Surfaces: Surfaces closed" }
101                                 sessionConfigAdapter.reportSurfaceInvalid(e.deferrableSurface)
102                                 return@async false
103                             } catch (e: TimeoutCancellationException) {
104                                 Log.error(e) { "Failed to get Surfaces within $timeoutMillis ms" }
105                                 return@async false
106                             }
107                         if (!isActive || surfaces.isEmpty()) {
108                             Log.error {
109                                 "Failed to get Surfaces: isActive=$isActive, surfaces=$surfaces"
110                             }
111                             return@async false
112                         }
113                         if (surfaces.areValid()) {
114                             synchronized(lock) {
115                                 configuredSurfaceMap =
116                                     deferrableSurfaces.associateBy { deferrableSurface ->
117                                         checkNotNull(
118                                             surfaces[deferrableSurfaces.indexOf(deferrableSurface)]
119                                         )
120                                     }
121                                 setSurfaceListener()
122                             }
123 
124                             surfaceToStreamMap.forEach {
125                                 val stream = it.value
126                                 val surface = surfaces[deferrableSurfaces.indexOf(it.key)]
127                                 Log.debug { "Configured $surface for $stream" }
128                                 graph.setSurface(stream = stream, surface = surface)
129                                 inactiveSurfaceCloser.configure(stream, it.key, graph)
130                             }
131                             Log.info { "Surface setup complete" }
132                             return@async true
133                         } else {
134                             Log.error { "Surface setup failed: Some Surfaces are invalid" }
135                             // Only handle the first failed Surface since subsequent calls to
136                             // CameraInternal#onUseCaseReset() will handle the other failed Surfaces
137                             // if there are any.
138                             sessionConfigAdapter.reportSurfaceInvalid(
139                                 deferrableSurfaces[surfaces.indexOf(null)]
140                             )
141                             return@async false
142                         }
143                     }
144                     .apply {
145                         // When setup is done or cancelled, decrement the DeferrableSurfaces.
146                         invokeOnCompletion { DeferrableSurfaces.decrementAll(deferrableSurfaces) }
147                     }
148             setupDeferred = deferred
149             return@synchronized deferred
150         }
151 
152     /** Cancel the Surface set up and stop the monitoring of Surface usage. */
153     public fun stopAsync(): Deferred<Unit> =
154         synchronized(lock) {
155             val currentStopDeferred = stopDeferred
156             if (currentStopDeferred != null) {
157                 Log.warn { "UseCaseSurfaceManager is already stopping!" }
158                 return@synchronized currentStopDeferred
159             }
160             setupDeferred?.cancel()
161             inactiveSurfaceCloser.closeAll()
162             configuredSurfaceMap = null
163 
164             val deferred = CompletableDeferred<Unit>()
165             this.stopDeferred = deferred
166             // This may complete stopDeferred immediately
167             tryClearSurfaceListener()
168 
169             return@synchronized deferred
170         }
171 
172     /**
173      * Waits for any ongoing [setupAsync] to be completed and returns a boolean value to indicate if
174      * a successful setup exists.
175      *
176      * If [stopAsync] is called after a successful setup, this function returns false since the
177      * setup was terminated.
178      */
179     public open suspend fun awaitSetupCompletion(): Boolean {
180         if (sessionConfigAdapter.isSessionProcessorEnabled) {
181             // The SessionProcessor flow does not use the setupAsync flow of this class and the
182             // whole UseCaseCamera layer is created only after SessionProcessor setup is completed
183             // successfully.
184             return true
185         }
186 
187         val setupDeferred =
188             synchronized(lock) {
189                 val setupDeferredSnapshot = setupDeferred
190 
191                 if (setupDeferredSnapshot == null || stopDeferred != null) {
192                     return false
193                 }
194 
195                 setupDeferredSnapshot
196             }
197 
198         try {
199             return setupDeferred.await()
200         } catch (e: CancellationException) {
201             Log.warn(e) { "Surface setup was cancelled" }
202             return false
203         }
204     }
205 
206     override fun onSurfaceActive(surface: Surface) {
207         synchronized(lock) {
208             configuredSurfaceMap?.get(surface)?.let {
209                 if (!activeSurfaceMap.containsKey(surface)) {
210                     Log.debug { "SurfaceActive $it in ${this@UseCaseSurfaceManager}" }
211                     activeSurfaceMap[surface] = it
212                     try {
213                         it.incrementUseCount()
214                     } catch (e: SurfaceClosedException) {
215                         Log.error(e) { "Error when $surface going to increase the use count." }
216                         sessionConfigAdapter.reportSurfaceInvalid(e.deferrableSurface)
217                     }
218                 }
219             }
220         }
221     }
222 
223     override fun onSurfaceInactive(surface: Surface) {
224         synchronized(lock) {
225             activeSurfaceMap.remove(surface)?.let {
226                 Log.debug { "SurfaceInactive $it in ${this@UseCaseSurfaceManager}" }
227                 inactiveSurfaceCloser.onSurfaceInactive(it)
228                 try {
229                     it.decrementUseCount()
230                 } catch (e: IllegalStateException) {
231                     Log.error(e) { "Error when $surface going to decrease the use count." }
232                 }
233                 tryClearSurfaceListener()
234             }
235         }
236     }
237 
238     @GuardedBy("lock")
239     private fun setSurfaceListener() {
240         cameraPipe.cameraSurfaceManager().addListener(this)
241     }
242 
243     @GuardedBy("lock")
244     private fun tryClearSurfaceListener() {
245         synchronized(lock) {
246             if (activeSurfaceMap.isEmpty() && configuredSurfaceMap == null) {
247                 Log.debug { "${this@UseCaseSurfaceManager} remove surface listener" }
248                 cameraPipe.cameraSurfaceManager().removeListener(this)
249                 stopDeferred?.complete(Unit)
250             }
251         }
252     }
253 
254     private suspend fun getSurfaces(
255         deferrableSurfaces: List<DeferrableSurface>,
256         timeoutMillis: Long,
257     ): List<Surface?> {
258         return withTimeoutOrNull(timeMillis = timeoutMillis) {
259                 Futures.successfulAsList(
260                         deferrableSurfaces.map { Futures.nonCancellationPropagating(it.surface) }
261                     )
262                     .await()
263             }
264             .orEmpty()
265     }
266 
267     private fun List<Surface?>.areValid(): Boolean {
268         // If a Surface in configuredSurfaces is null it means the
269         // Surface was not retrieved from the ListenableFuture.
270         return isNotEmpty() && !contains(null)
271     }
272 }
273