1 /*
<lambda>null2  * Copyright 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.testing.impl.wrappers
18 
19 import android.graphics.Bitmap
20 import android.graphics.Matrix
21 import android.graphics.Rect
22 import android.util.Log
23 import androidx.annotation.VisibleForTesting
24 import androidx.camera.core.ImageCapture
25 import androidx.camera.core.ImageCapture.OutputFileOptions
26 import androidx.camera.core.ImageCapture.OutputFileResults
27 import androidx.camera.core.ImageProcessingUtil
28 import androidx.camera.core.ImageProxy
29 import androidx.camera.core.Logger
30 import androidx.camera.core.imagecapture.Bitmap2JpegBytes
31 import androidx.camera.core.imagecapture.ImageCaptureControl
32 import androidx.camera.core.imagecapture.ImagePipeline
33 import androidx.camera.core.imagecapture.JpegBytes2Disk
34 import androidx.camera.core.imagecapture.JpegBytes2Image
35 import androidx.camera.core.imagecapture.RequestWithCallback
36 import androidx.camera.core.imagecapture.TakePictureManager
37 import androidx.camera.core.imagecapture.TakePictureManagerImpl
38 import androidx.camera.core.imagecapture.TakePictureRequest
39 import androidx.camera.core.impl.utils.executor.CameraXExecutors
40 import androidx.camera.core.processing.Packet
41 import androidx.camera.testing.fakes.FakeCamera
42 import androidx.camera.testing.fakes.FakeCameraCaptureResult
43 import androidx.camera.testing.fakes.FakeCameraControl
44 import androidx.camera.testing.fakes.FakeCameraControl.CaptureSuccessListener
45 import androidx.camera.testing.impl.ExifUtil
46 import androidx.camera.testing.impl.TestImageUtil
47 import androidx.camera.testing.impl.TestImageUtil.createBitmap
48 import androidx.camera.testing.impl.fakes.FakeImageInfo
49 import androidx.camera.testing.impl.fakes.FakeImageProxy
50 import kotlinx.atomicfu.atomic
51 
52 /**
53  * A [TakePictureManager] implementation wrapped around the real implementation
54  * [TakePictureManagerImpl].
55  *
56  * It is used for fake cameras and provides fake image capture results when required from a camera.
57  */
58 public class TakePictureManagerWrapper(
59     imageCaptureControl: ImageCaptureControl,
60     private val fakeCameras: List<FakeCamera>
61 ) : TakePictureManager {
62     // Try to keep the fake as close to real as possible
63     private val managerDelegate = TakePictureManagerImpl(imageCaptureControl)
64 
65     private val bitmap2JpegBytes = Bitmap2JpegBytes()
66     private val jpegBytes2Disk = JpegBytes2Disk()
67     private val jpegBytes2Image = JpegBytes2Image()
68 
69     private val imageProxyQueue = ArrayDeque<ImageProxy>()
70     private val outputFileResultsQueue = ArrayDeque<OutputFileResults>()
71 
72     private val pendingRequestCount = atomic(0)
73 
74     private val captureCompleteListener = CaptureSuccessListener { _ ->
75         completeCapturingRequest()
76 
77         // TODO - handle CameraControl receiving more than one capture per takePictureRequest
78         if (pendingRequestCount.decrementAndGet() == 0) {
79             removeCaptureCompleteListener()
80         }
81     }
82 
83     private fun addCaptureCompleteListener() {
84         Logger.d(TAG, "addCaptureCompleteListener: fakeCameras = $fakeCameras")
85 
86         fakeCameras.forEach { camera ->
87             if (camera.cameraControlInternal is FakeCameraControl) {
88                 (camera.cameraControlInternal as FakeCameraControl).addCaptureSuccessListener(
89                     captureCompleteListener
90                 )
91             } else {
92                 Logger.w(
93                     TAG,
94                     "Ignoring ${camera.cameraControlInternal} as it's not FakeCameraControl!"
95                 )
96             }
97         }
98     }
99 
100     private fun removeCaptureCompleteListener() {
101         Logger.d(TAG, "removeCaptureCompleteListener: fakeCameras = $fakeCameras")
102 
103         fakeCameras.forEach { camera ->
104             if (camera.cameraControlInternal is FakeCameraControl) {
105                 (camera.cameraControlInternal as FakeCameraControl).removeCaptureSuccessListener(
106                     captureCompleteListener
107                 )
108             } else {
109                 Logger.w(
110                     TAG,
111                     "Ignoring ${camera.cameraControlInternal} as it's not FakeCameraControl!"
112                 )
113             }
114         }
115     }
116 
117     override fun setImagePipeline(imagePipeline: ImagePipeline) {
118         managerDelegate.imagePipeline = imagePipeline
119     }
120 
121     override fun offerRequest(takePictureRequest: TakePictureRequest) {
122         if (pendingRequestCount.getAndIncrement() == 0) {
123             addCaptureCompleteListener()
124         }
125 
126         managerDelegate.offerRequest(takePictureRequest)
127     }
128 
129     override fun pause() {
130         managerDelegate.pause()
131     }
132 
133     override fun resume() {
134         managerDelegate.resume()
135     }
136 
137     override fun abortRequests() {
138         managerDelegate.abortRequests()
139     }
140 
141     @VisibleForTesting
142     override fun hasCapturingRequest(): Boolean = managerDelegate.hasCapturingRequest()
143 
144     @VisibleForTesting
145     override fun getCapturingRequest(): RequestWithCallback? = managerDelegate.capturingRequest
146 
147     @VisibleForTesting
148     override fun getIncompleteRequests(): List<RequestWithCallback> =
149         managerDelegate.incompleteRequests
150 
151     @VisibleForTesting
152     override fun getImagePipeline(): ImagePipeline = managerDelegate.imagePipeline
153 
154     @VisibleForTesting
155     private fun completeCapturingRequest() {
156         Log.d(
157             TAG,
158             "completeCapturingRequest: capturingRequest = ${managerDelegate.capturingRequest}"
159         )
160         managerDelegate.capturingRequest?.apply {
161             runOnMainThread { // onCaptureStarted, onImageCaptured etc. are @MainThread annotated
162                 Logger.d(TAG, "completeCapturingRequest: runOnMainThread")
163                 onCaptureStarted()
164                 onImageCaptured()
165 
166                 // TODO: b/365519650 - Take FakeCameraCaptureResult as parameter to contain extra
167                 //  user-provided data like bitmap/image proxy and use that to complete capture.
168                 val bitmap = takePictureRequest.createBitmap()
169 
170                 val outputFileOptions = takePictureRequest.outputFileOptions // enables
171                 val secondaryOutputResults = takePictureRequest.secondaryOutputFileOptions
172 
173                 if (takePictureRequest.onDiskCallback != null && outputFileOptions != null) {
174                     if (outputFileResultsQueue.isEmpty()) {
175                         if (secondaryOutputResults != null) {
176                             Logger.w(
177                                 TAG,
178                                 "Simultaneous capture not supported, outputFileOptions = $outputFileOptions"
179                             )
180                         }
181                         onFinalResult(
182                             createOutputFileResults(takePictureRequest, outputFileOptions, bitmap)
183                         )
184                     } else {
185                         onFinalResult(outputFileResultsQueue.removeFirst())
186                     }
187                 } else {
188                     if (imageProxyQueue.isEmpty()) {
189                         onFinalResult(createImageProxy(takePictureRequest, bitmap))
190                     } else {
191                         onFinalResult(imageProxyQueue.removeFirst())
192                     }
193                 }
194             }
195         }
196     }
197 
198     /**
199      * Enqueues an [ImageProxy] to be used as result for the next image capture with
200      * [ImageCapture.OnImageCapturedCallback].
201      *
202      * Note that the provided [ImageProxy] is consumed by next image capture and is not available
203      * for following captures. If no result is available during a capture, CameraX will create a
204      * fake image by itself and provide result based on that.
205      */
206     public fun enqueueImageProxy(imageProxy: ImageProxy) {
207         imageProxyQueue.add(imageProxy)
208     }
209 
210     /**
211      * Enqueues an [OutputFileResults] to be used as result for the next image capture with
212      * [ImageCapture.OnImageSavedCallback].
213      *
214      * Note that the provided [OutputFileResults] is consumed by next image capture and is not
215      * available for following captures. If no result is available during a capture, CameraX will
216      * create a fake image by itself and provide result based on that.
217      */
218     public fun enqueueOutputFileResults(outputFileResults: OutputFileResults) {
219         outputFileResultsQueue.add(outputFileResults)
220     }
221 
222     private fun createOutputFileResults(
223         takePictureRequest: TakePictureRequest,
224         outputFileOptions: OutputFileOptions,
225         bitmap: Bitmap,
226     ): OutputFileResults {
227         // TODO - Take a bitmap as input and use that directly
228         val bytesPacket = takePictureRequest.convertBitmapToBytes(bitmap)
229         return jpegBytes2Disk.apply(JpegBytes2Disk.In.of(bytesPacket, outputFileOptions))
230     }
231 
232     private fun createImageProxy(
233         takePictureRequest: TakePictureRequest,
234         bitmap: Bitmap,
235     ): ImageProxy {
236         if (canLoadImageProcessingUtilJniLib()) {
237             try {
238                 val bytesPacket = takePictureRequest.convertBitmapToBytes(bitmap)
239                 return jpegBytes2Image.apply(bytesPacket).data
240             } catch (e: Exception) {
241                 // We have observed this kind of issue in Pixel 2 API 26 emulator, however this is
242                 // added as a general workaround as this may happen in any emulator/device and
243                 // similar to how not all resolutions are supported due to device capabilities even
244                 // in production code.
245                 Logger.e(
246                     TAG,
247                     "createImageProxy: failed for cropRect = ${takePictureRequest.cropRect}" +
248                         " which may happen due to a high resolution not being supported" +
249                         ", trying again with 640x480",
250                     e
251                 )
252 
253                 val bytesPacket =
254                     takePictureRequest.convertBitmapToBytes(
255                         createBitmap(640, 480),
256                         Rect(0, 0, 640, 480)
257                     )
258                 Logger.d(TAG, "createImageProxy: bytesPacket size = ${bytesPacket.size}")
259 
260                 return jpegBytes2Image.apply(bytesPacket).data
261             }
262         } else {
263             return bitmap.toFakeImageProxy()
264         }
265     }
266 
267     private fun Bitmap.toFakeImageProxy(): ImageProxy {
268         return FakeImageProxy(FakeImageInfo(), this)
269     }
270 
271     private fun TakePictureRequest.createBitmap() =
272         TestImageUtil.createBitmap(cropRect.width(), cropRect.height())
273 
274     private fun TakePictureRequest.convertBitmapToBytes(
275         bitmap: Bitmap,
276         cropRect: Rect = this.cropRect
277     ): Packet<ByteArray> {
278         val inputPacket =
279             Packet.of(
280                 bitmap,
281                 ExifUtil.createExif(
282                     TestImageUtil.createJpegBytes(cropRect.width(), cropRect.height())
283                 ),
284                 cropRect,
285                 rotationDegrees,
286                 Matrix(),
287                 FakeCameraCaptureResult()
288             )
289 
290         return bitmap2JpegBytes.apply(Bitmap2JpegBytes.In.of(inputPacket, jpegQuality))
291     }
292 
293     private fun canLoadImageProcessingUtilJniLib(): Boolean {
294         try {
295             System.loadLibrary(ImageProcessingUtil.JNI_LIB_NAME)
296             return true
297         } catch (e: UnsatisfiedLinkError) {
298             Logger.d(TAG, "canLoadImageProcessingUtilJniLib", e)
299             return false
300         }
301     }
302 
303     private fun runOnMainThread(block: () -> Any) {
304         CameraXExecutors.mainThreadExecutor().submit(block)
305     }
306 
307     private companion object {
308         private const val TAG = "TakePictureManagerWrap"
309     }
310 }
311