1 /*
<lambda>null2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.camera2.pipe.graph
18 
19 import android.os.Build
20 import android.view.Surface
21 import androidx.camera.camera2.pipe.AudioRestrictionMode
22 import androidx.camera.camera2.pipe.CameraController
23 import androidx.camera.camera2.pipe.CameraGraph
24 import androidx.camera.camera2.pipe.CameraGraphId
25 import androidx.camera.camera2.pipe.CameraMetadata
26 import androidx.camera.camera2.pipe.GraphState
27 import androidx.camera.camera2.pipe.StreamGraph
28 import androidx.camera.camera2.pipe.StreamId
29 import androidx.camera.camera2.pipe.compat.AudioRestrictionController
30 import androidx.camera.camera2.pipe.config.CameraGraphScope
31 import androidx.camera.camera2.pipe.core.Debug
32 import androidx.camera.camera2.pipe.core.Log
33 import androidx.camera.camera2.pipe.core.Token
34 import androidx.camera.camera2.pipe.core.acquireToken
35 import androidx.camera.camera2.pipe.core.acquireTokenAndSuspend
36 import androidx.camera.camera2.pipe.core.tryAcquireToken
37 import androidx.camera.camera2.pipe.internal.CameraGraphParametersImpl
38 import androidx.camera.camera2.pipe.internal.FrameCaptureQueue
39 import androidx.camera.camera2.pipe.internal.FrameDistributor
40 import javax.inject.Inject
41 import kotlinx.atomicfu.atomic
42 import kotlinx.coroutines.CoroutineScope
43 import kotlinx.coroutines.CoroutineStart
44 import kotlinx.coroutines.Deferred
45 import kotlinx.coroutines.Job
46 import kotlinx.coroutines.async
47 import kotlinx.coroutines.coroutineScope
48 import kotlinx.coroutines.ensureActive
49 import kotlinx.coroutines.flow.StateFlow
50 import kotlinx.coroutines.sync.Mutex
51 
52 @CameraGraphScope
53 internal class CameraGraphImpl
54 @Inject
55 constructor(
56     graphConfig: CameraGraph.Config,
57     metadata: CameraMetadata,
58     private val graphProcessor: GraphProcessor,
59     private val graphListener: GraphListener,
60     private val streamGraph: StreamGraphImpl,
61     private val surfaceGraph: SurfaceGraph,
62     private val cameraController: CameraController,
63     private val graphState3A: GraphState3A,
64     private val listener3A: Listener3A,
65     private val frameDistributor: FrameDistributor,
66     private val frameCaptureQueue: FrameCaptureQueue,
67     private val audioRestrictionController: AudioRestrictionController,
68     override val id: CameraGraphId,
69     override val parameters: CameraGraphParametersImpl,
70     private val sessionLock: SessionLock
71 ) : CameraGraph {
72     private val controller3A = Controller3A(graphProcessor, metadata, graphState3A, listener3A)
73     private val closed = atomic(false)
74 
75     init {
76         // Log out the configuration of the camera graph when it is created.
77         Log.info { Debug.formatCameraGraphProperties(metadata, graphConfig, this) }
78 
79         // Enforce preview and video stream use cases for high speed sessions
80         if (graphConfig.sessionMode == CameraGraph.OperatingMode.HIGH_SPEED) {
81             require(streamGraph.outputs.isNotEmpty()) {
82                 "Cannot create a HIGH_SPEED CameraGraph without outputs."
83             }
84             require(streamGraph.outputs.size <= 2) {
85                 "Cannot create a HIGH_SPEED CameraGraph with more than two outputs. " +
86                     "Configured outputs are ${streamGraph.outputs}"
87             }
88 
89             // Streams must be preview and/or video for high speed sessions
90             val allStreamsValidForHighSpeedOperatingMode =
91                 this.streamGraph.outputs.all { it.isValidForHighSpeedOperatingMode() }
92 
93             require(allStreamsValidForHighSpeedOperatingMode) {
94                 "HIGH_SPEED CameraGraph must only contain Preview and/or Video " +
95                     "streams. Configured outputs are ${streamGraph.outputs}"
96             }
97         }
98 
99         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
100             require(graphConfig.input == null) { "Reprocessing not supported under Android M" }
101         }
102         if (graphConfig.input != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
103             require(graphConfig.input.isNotEmpty()) {
104                 "At least one InputConfiguration is required for reprocessing"
105             }
106             if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
107                 require(graphConfig.input.size <= 1) {
108                     "Multi resolution reprocessing not supported under Android S"
109                 }
110             }
111         }
112     }
113 
114     override val streams: StreamGraph
115         get() = streamGraph
116 
117     override val graphState: StateFlow<GraphState>
118         get() = graphProcessor.graphState
119 
120     override var isForeground: Boolean = true
121         set(value) {
122             field = value
123             cameraController.isForeground = value
124         }
125 
126     override fun start() {
127         check(!closed.value) { "Cannot start $this after calling close()" }
128 
129         Debug.traceStart { "$this#start" }
130         Log.info { "Starting $this" }
131         graphListener.onGraphStarting()
132         cameraController.start()
133         Debug.traceStop()
134     }
135 
136     override fun stop() {
137         check(!closed.value) { "Cannot stop $this after calling close()" }
138 
139         Debug.traceStart { "$this#stop" }
140         Log.info { "Stopping $this" }
141         graphListener.onGraphStopping()
142         cameraController.stop()
143         Debug.traceStop()
144     }
145 
146     override suspend fun acquireSession(): CameraGraph.Session {
147         // Step 1: Acquire a lock on the session mutex, which returns a releasable token. This may
148         //         or may not suspend.
149         val token = sessionLock.acquireToken()
150 
151         // Step 2: Return a session that can be used to interact with the session. The session must
152         //         be closed when it is no longer needed.
153         return createSessionFromToken(token)
154     }
155 
156     override fun acquireSessionOrNull(): CameraGraph.Session? {
157         val token = sessionLock.tryAcquireToken() ?: return null
158         return createSessionFromToken(token)
159     }
160 
161     override suspend fun <T> useSession(
162         action: suspend CoroutineScope.(CameraGraph.Session) -> T
163     ): T =
164         acquireSession().use {
165             // Wrap the block in a coroutineScope to ensure all operations are completed before
166             // releasing the lock.
167             coroutineScope { action(it) }
168         }
169 
170     override fun <T> useSessionIn(
171         scope: CoroutineScope,
172         action: suspend CoroutineScope.(CameraGraph.Session) -> T
173     ): Deferred<T> {
174         return sessionLock.withTokenIn(scope) { token ->
175             // Create and use the session
176             createSessionFromToken(token).use { session ->
177                 // Wrap the block in a coroutineScope to ensure all operations are completed
178                 // before exiting and releasing the lock. The lock can be released early if the
179                 // calling action decides to call session.close() early.
180                 coroutineScope { action(session) }
181             }
182         }
183     }
184 
185     private fun createSessionFromToken(token: Token) =
186         CameraGraphSessionImpl(token, graphProcessor, controller3A, frameCaptureQueue, parameters)
187 
188     override fun setSurface(stream: StreamId, surface: Surface?) {
189         Debug.traceStart { "$stream#setSurface" }
190         if (surface != null && !surface.isValid) {
191             Log.warn { "$this#setSurface: $surface is invalid" }
192         }
193         surfaceGraph[stream] = surface
194         Debug.traceStop()
195     }
196 
197     override fun updateAudioRestrictionMode(mode: AudioRestrictionMode) {
198         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
199             audioRestrictionController.updateCameraGraphAudioRestrictionMode(this, mode)
200         }
201     }
202 
203     override fun close() {
204         if (closed.compareAndSet(expect = false, update = true)) {
205             Debug.traceStart { "$this#close" }
206             Log.info { "Closing $this" }
207             graphProcessor.close()
208             cameraController.close()
209             frameDistributor.close()
210             frameCaptureQueue.close()
211             surfaceGraph.close()
212             audioRestrictionController.removeCameraGraph(this)
213             Debug.traceStop()
214         }
215     }
216 
217     override fun toString(): String = id.toString()
218 }
219 
220 @CameraGraphScope
221 internal class SessionLock @Inject constructor() {
222     private val mutex = Mutex()
223 
acquireTokennull224     internal suspend fun acquireToken(): Token = mutex.acquireToken()
225 
226     internal fun tryAcquireToken(): Token? = mutex.tryAcquireToken()
227 
228     internal fun <T> withTokenIn(
229         scope: CoroutineScope,
230         action: suspend (token: Token) -> T
231     ): Deferred<T> {
232         // https://github.com/Kotlin/kotlinx.coroutines/issues/1578
233         // To handle `runBlocking` we need to use `job.complete()` in `result.invokeOnCompletion`.
234         // However, if we do this directly on the scope that is provided it will cause
235         // SupervisorScopes to block and never complete. To work around this, we create a childJob,
236         // propagate the existing context, and use that as the context for scope.async.
237         val childJob = Job(scope.coroutineContext[Job])
238         val context = scope.coroutineContext + childJob
239         val result =
240             scope.async(context = context, start = CoroutineStart.UNDISPATCHED) {
241                 ensureActive() // Exit early if the parent scope has been canceled.
242 
243                 // It is very important to acquire *and* suspend here. Invoking a coroutine using
244                 // UNDISPATCHED will execute on the current thread until the suspension point, and
245                 // this will force the execution to switch to the provided scope after ensuring the
246                 // lock is acquired or in the queue. This guarantees exclusion, ordering, and
247                 // execution within the correct scope.
248                 mutex.acquireTokenAndSuspend().use { token -> action(token) }
249             }
250         result.invokeOnCompletion { childJob.complete() }
251         return result
252     }
253 
usenull254     private suspend fun <T> Token.use(block: suspend (Token) -> T): T {
255         try {
256             return block(this)
257         } finally {
258             this.release()
259         }
260     }
261 }
262