1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.extensions
18 
19 import android.annotation.SuppressLint
20 import android.content.ContentResolver
21 import android.content.Context
22 import android.content.Intent
23 import android.graphics.SurfaceTexture
24 import android.hardware.camera2.CameraCaptureSession
25 import android.hardware.camera2.CameraCaptureSession.CaptureCallback
26 import android.hardware.camera2.CameraCharacteristics
27 import android.hardware.camera2.CameraDevice
28 import android.hardware.camera2.CameraExtensionCharacteristics
29 import android.hardware.camera2.CameraExtensionSession
30 import android.hardware.camera2.CameraExtensionSession.ExtensionCaptureCallback
31 import android.hardware.camera2.CameraManager
32 import android.hardware.camera2.CameraMetadata
33 import android.hardware.camera2.CameraMetadata.CONTROL_AF_TRIGGER_CANCEL
34 import android.hardware.camera2.CameraMetadata.CONTROL_AF_TRIGGER_IDLE
35 import android.hardware.camera2.CaptureFailure
36 import android.hardware.camera2.CaptureRequest
37 import android.hardware.camera2.CaptureRequest.CONTROL_AE_MODE
38 import android.hardware.camera2.CaptureRequest.CONTROL_AE_MODE_ON
39 import android.hardware.camera2.CaptureRequest.CONTROL_AE_REGIONS
40 import android.hardware.camera2.CaptureRequest.CONTROL_AF_MODE
41 import android.hardware.camera2.CaptureRequest.CONTROL_AF_MODE_AUTO
42 import android.hardware.camera2.CaptureRequest.CONTROL_AF_REGIONS
43 import android.hardware.camera2.CaptureRequest.CONTROL_AF_TRIGGER
44 import android.hardware.camera2.CaptureRequest.CONTROL_AF_TRIGGER_START
45 import android.hardware.camera2.CaptureRequest.CONTROL_AWB_REGIONS
46 import android.hardware.camera2.CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE
47 import android.hardware.camera2.TotalCaptureResult
48 import android.hardware.camera2.params.ExtensionSessionConfiguration
49 import android.hardware.camera2.params.MeteringRectangle
50 import android.hardware.camera2.params.OutputConfiguration
51 import android.hardware.camera2.params.SessionConfiguration
52 import android.hardware.camera2.params.SessionConfiguration.SESSION_REGULAR
53 import android.media.ImageReader
54 import android.net.Uri
55 import android.os.Build
56 import android.os.Bundle
57 import android.os.Handler
58 import android.os.HandlerThread
59 import android.os.Looper
60 import android.util.Log
61 import android.util.Size
62 import android.view.Menu
63 import android.view.MenuItem
64 import android.view.ScaleGestureDetector
65 import android.view.Surface
66 import android.view.TextureView
67 import android.view.View
68 import android.view.ViewStub
69 import android.widget.Button
70 import android.widget.FrameLayout
71 import android.widget.ImageButton
72 import android.widget.Switch
73 import android.widget.TextView
74 import android.widget.Toast
75 import androidx.annotation.GuardedBy
76 import androidx.annotation.RequiresApi
77 import androidx.annotation.VisibleForTesting
78 import androidx.appcompat.app.AppCompatActivity
79 import androidx.camera.core.impl.utils.futures.Futures
80 import androidx.camera.integration.extensions.ExtensionTestType.TEST_TYPE_CAMERA2_EXTENSION_STREAM_CONFIG_LATENCY
81 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_CAMERA_ID
82 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_ERROR_CODE
83 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_EXTENSION_MODE
84 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_IMAGE_ROTATION_DEGREES
85 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_IMAGE_URI
86 import androidx.camera.integration.extensions.IntentExtraKey.INTENT_EXTRA_KEY_REQUEST_CODE
87 import androidx.camera.integration.extensions.TapToFocusDetector.CameraInfo
88 import androidx.camera.integration.extensions.TestResultType.TEST_RESULT_FAILED
89 import androidx.camera.integration.extensions.TestResultType.TEST_RESULT_NOT_TESTED
90 import androidx.camera.integration.extensions.TestResultType.TEST_RESULT_PASSED
91 import androidx.camera.integration.extensions.ValidationErrorCode.ERROR_CODE_EXTENSION_MODE_NOT_SUPPORT
92 import androidx.camera.integration.extensions.ValidationErrorCode.ERROR_CODE_NONE
93 import androidx.camera.integration.extensions.ValidationErrorCode.ERROR_CODE_SAVE_IMAGE_FAILED
94 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.getCamera2ExtensionModeStringFromId
95 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.getLensFacingCameraId
96 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.isCamera2ExtensionModeSupported
97 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.pickPreviewResolution
98 import androidx.camera.integration.extensions.utils.Camera2ExtensionsUtil.pickStillImageResolution
99 import androidx.camera.integration.extensions.utils.FileUtil
100 import androidx.camera.integration.extensions.utils.TransformUtil.calculateRelativeImageRotationDegrees
101 import androidx.camera.integration.extensions.utils.TransformUtil.surfaceRotationToRotationDegrees
102 import androidx.camera.integration.extensions.utils.TransformUtil.transformTextureView
103 import androidx.camera.integration.extensions.validation.CameraValidationResultActivity
104 import androidx.camera.integration.extensions.validation.TestResults
105 import androidx.concurrent.futures.CallbackToFutureAdapter
106 import androidx.concurrent.futures.CallbackToFutureAdapter.Completer
107 import androidx.core.util.Preconditions
108 import androidx.test.espresso.idling.CountingIdlingResource
109 import com.google.common.util.concurrent.ListenableFuture
110 import java.text.Format
111 import java.text.SimpleDateFormat
112 import java.util.Calendar
113 import java.util.Locale
114 import java.util.concurrent.Executors
115 import java.util.concurrent.TimeUnit
116 import java.util.concurrent.atomic.AtomicLong
117 import kotlin.coroutines.cancellation.CancellationException
118 import kotlinx.coroutines.CompletableDeferred
119 import kotlinx.coroutines.CoroutineScope
120 import kotlinx.coroutines.Deferred
121 import kotlinx.coroutines.Dispatchers
122 import kotlinx.coroutines.ExperimentalCoroutinesApi
123 import kotlinx.coroutines.SupervisorJob
124 import kotlinx.coroutines.asCoroutineDispatcher
125 import kotlinx.coroutines.asExecutor
126 import kotlinx.coroutines.async
127 import kotlinx.coroutines.launch
128 import kotlinx.coroutines.runBlocking
129 
130 private const val TAG = "Camera2ExtensionsAct~"
131 private const val FRAMES_UNTIL_VIEW_IS_READY = 10
132 private const val KEY_CAMERA2_LATENCY = "camera2"
133 private const val KEY_CAMERA_EXTENSION_LATENCY = "camera_extension"
134 private const val MAX_EXTENSION_LATENCY_MILLIS = 800
135 
136 // The states of the camera/capture session open/close flow
137 //  - STATE_CAMERA_CLOSED -> Open camera first. Open the capture session when camera is opened.
138 //  - STATE_CAPTURE_SESSION_CONFIGURED -> Close the capture session and camera first. If the target
139 //    camera is the same, directly open the capture session with the new target extension mode.
140 //    Otherwise, reopen the camera and then switch to the new extension mode
141 //  - Others -> Only update the target camera and extension mode info. When receiving camera
142 //    onOpened or capture session onConfigured events, reopen to the new target camera and
143 //    extension mode if it is mismatched.
144 private const val STATE_CAMERA_CLOSED = 0
145 private const val STATE_CAMERA_OPENING = 1
146 private const val STATE_CAMERA_OPENED = 2
147 private const val STATE_CAPTURE_SESSION_OPENING = 3
148 private const val STATE_CAPTURE_SESSION_CONFIGURED = 4
149 private const val STATE_CAPTURE_SESSION_CLOSING = 5
150 private const val STATE_CAPTURE_SESSION_CLOSED = 6
151 private const val STATE_CAMERA_CLOSING = 7
152 
153 @RequiresApi(31)
154 class Camera2ExtensionsActivity : AppCompatActivity() {
155 
156     // ===============================================================
157     // Fields that will be accessed on the camera thread
158     // ===============================================================
159 
160     private var currentState = STATE_CAMERA_CLOSED
161 
162     /** A reference to the opened [CameraDevice]. */
163     private var cameraDevice: CameraDevice? = null
164 
165     /**
166      * The current camera capture session. Use Any type to store it because it might be either a
167      * CameraCaptureSession instance if current is in normal mode, or, it might be a
168      * CameraExtensionSession instance if current is in Camera2 extension mode.
169      */
170     private var cameraCaptureSession: Any? = null
171 
172     private val focusMeteringControl = FocusMeteringControl(::startAfTrigger, ::cancelAfTrigger)
173     private var meteringRectangles: Array<MeteringRectangle?> = EMPTY_RECTANGLES
174 
175     // ===============================================================
176     // Fields that will be accessed on the camera thread
177     // ===============================================================
178 
179     private lateinit var backCameraId: String
180     private lateinit var frontCameraId: String
181 
182     private var activityStopped = false
183     private var currentCameraId = "0"
184     private var cameraSensorRotationDegrees = 0
185 
186     /** Camera extension characteristics for the current camera device. */
187     private lateinit var extensionCharacteristics: CameraExtensionCharacteristics
188 
189     /** Track current extension type and index. */
190     private var currentExtensionMode = EXTENSION_MODE_NONE
191     private var currentExtensionIdx = 0
192     private val supportedExtensionModes = mutableListOf<Int>()
193     private var extensionModeEnabled = false
194 
195     private lateinit var tapToFocusDetector: TapToFocusDetector
196 
197     // ===============================================================
198     // Fields that will be accessed under synchronization protection
199     // ===============================================================
200     private val lock = Object()
201     @GuardedBy("lock")
202     private var captureSessionClosedDeferred: CompletableDeferred<Unit> =
203         CompletableDeferred<Unit>().apply { complete(Unit) }
204 
205     private lateinit var cameraManager: CameraManager
206 
207     /**
208      * Tracks the stream configuration latency of camera extension and camera2. Each key is
209      * associated with a list of durations. This allows clients to run multiple invocations to
210      * measure the min, avg, and max latency.
211      */
212     private val streamConfigurationLatency =
213         mutableMapOf<String, MutableList<Long>>(
214             KEY_CAMERA2_LATENCY to mutableListOf(),
215             KEY_CAMERA_EXTENSION_LATENCY to mutableListOf()
216         )
217 
218     /** Still capture image reader */
219     private var stillImageReader: ImageReader? = null
220 
221     private lateinit var containerView: View
222 
223     private lateinit var textureView: TextureView
224     private lateinit var videoStabilizationToggleView: Switch
225     private lateinit var videoStabilizationModeView: TextView
226 
227     private var previewSurface: Surface? = null
228 
229     private val surfaceTextureListener =
230         object : TextureView.SurfaceTextureListener {
231 
232             override fun onSurfaceTextureAvailable(
233                 surfaceTexture: SurfaceTexture,
234                 with: Int,
235                 height: Int
236             ) {
237                 previewSurface = Surface(surfaceTexture)
238                 setupAndStartPreview()
239             }
240 
241             override fun onSurfaceTextureSizeChanged(
242                 surfaceTexture: SurfaceTexture,
243                 with: Int,
244                 height: Int
245             ) {}
246 
247             override fun onSurfaceTextureDestroyed(surfaceTexture: SurfaceTexture): Boolean {
248                 // Will release the surface texture after the camera is closed
249                 return false
250             }
251 
252             override fun onSurfaceTextureUpdated(surfaceTexture: SurfaceTexture) {
253                 if (
254                     captureProcessStartedIdlingResource.isIdleNow &&
255                         receivedPreviewFrameCount.getAndIncrement() >= FRAMES_UNTIL_VIEW_IS_READY &&
256                         !previewIdlingResource.isIdleNow
257                 ) {
258                     previewIdlingResource.decrement()
259                 }
260 
261                 if (measureStreamConfigurationLatency && lastSurfaceTextureTimestampNanos != 0L) {
262                     val duration =
263                         TimeUnit.NANOSECONDS.toMillis(
264                             surfaceTexture.timestamp - lastSurfaceTextureTimestampNanos
265                         )
266                     if (duration > 150) {
267                         if (!extensionModeEnabled) {
268                             streamConfigurationLatency[KEY_CAMERA2_LATENCY]?.add(duration)
269                         } else {
270                             streamConfigurationLatency[KEY_CAMERA_EXTENSION_LATENCY]?.add(duration)
271                         }
272                         measureStreamConfigurationLatency = false
273                     }
274                 }
275                 lastSurfaceTextureTimestampNanos = surfaceTexture.timestamp
276             }
277         }
278 
279     private val captureCallbackExtensionMode =
280         object : ExtensionCaptureCallback() {
281             override fun onCaptureProcessStarted(
282                 session: CameraExtensionSession,
283                 request: CaptureRequest
284             ) {
285                 handleCaptureStartedEvent()
286             }
287 
288             override fun onCaptureFailed(session: CameraExtensionSession, request: CaptureRequest) {
289                 Log.e(TAG, "onCaptureFailed!!")
290             }
291         }
292 
293     private val captureCallbackNormalMode =
294         object : CaptureCallback() {
295             override fun onCaptureStarted(
296                 session: CameraCaptureSession,
297                 request: CaptureRequest,
298                 timestamp: Long,
299                 frameNumber: Long
300             ) {
301                 handleCaptureStartedEvent()
302             }
303         }
304 
305     private fun handleCaptureStartedEvent() {
306         checkRunOnCameraThread()
307         if (
308             receivedCaptureProcessStartedCount.getAndIncrement() >= FRAMES_UNTIL_VIEW_IS_READY &&
309                 !captureProcessStartedIdlingResource.isIdleNow
310         ) {
311             captureProcessStartedIdlingResource.decrement()
312         }
313     }
314 
315     private val comboCaptureCallbackExtensionMode =
316         ComboCaptureCallbackExtensionMode().apply {
317             addCaptureCallback(captureCallbackExtensionMode)
318         }
319 
320     private val comboCaptureCallbackNormalMode =
321         ComboCaptureCallbackNormalMode().apply { addCaptureCallback(captureCallbackNormalMode) }
322 
323     private val cameraTaskDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
324     private lateinit var cameraThread: Thread
325 
326     private val coroutineScope = CoroutineScope(Dispatchers.Default + SupervisorJob())
327 
328     private var imageSaveTerminationFuture: ListenableFuture<Any?> = Futures.immediateFuture(null)
329 
330     /**
331      * Tracks the last timestamp of a surface texture rendered on to the TextureView. This is used
332      * to measure the configuration latency from the last preview frame received from the previous
333      * camera session until the first preview frame received of the new camera session.
334      */
335     private var lastSurfaceTextureTimestampNanos: Long = 0
336 
337     /**
338      * A flag which represents when to measure the stream configuration latency. This is triggered
339      * when the user toggles the camera extension mode.
340      */
341     private var measureStreamConfigurationLatency: Boolean = true
342 
343     /** Used to wait for the camera is closed. */
344     private val cameraClosedIdlingResource = CountingIdlingResource("cameraClosed")
345 
346     /** Used to wait for the capture session is configured. */
347     private val captureSessionConfiguredIdlingResource =
348         CountingIdlingResource("captureSessionConfigured").apply { increment() }
349     /**
350      * Used to wait for the ExtensionCaptureCallback#onCaptureProcessStarted is called which means
351      * an image is captured and extension processing is triggered.
352      */
353     private val captureProcessStartedIdlingResource =
354         CountingIdlingResource("captureProcessStarted").apply { increment() }
355 
356     /**
357      * Used to wait for the preview is ready. This will become idle after
358      * captureProcessStartedIdlingResource becomes idle and
359      * [SurfaceTextureListener#onSurfaceTextureUpdated()] is also called. It means that there has
360      * been images captured to trigger the extension processing and the preview's SurfaceTexture is
361      * also updated by [SurfaceTexture#updateTexImage()] calls.
362      */
363     private val previewIdlingResource = CountingIdlingResource("preview").apply { increment() }
364 
365     /** Used to trigger a picture taking action and waits for the image being saved. */
366     private val imageSavedIdlingResource = CountingIdlingResource("imageSaved")
367 
368     private val receivedCaptureProcessStartedCount: AtomicLong = AtomicLong(0)
369     private val receivedPreviewFrameCount: AtomicLong = AtomicLong(0)
370 
371     private lateinit var sessionImageUriSet: SessionMediaUriSet
372 
373     /** Stores the request code passed from the caller activity. */
374     private var requestCode = -1
375 
376     /**
377      * This will be true if the activity is called by other activity to request capturing an image.
378      */
379     private var isRequestMode = false
380 
381     /** The result intent that saves the image capture request results. */
382     private lateinit var result: Intent
383 
384     /** A [HandlerThread] used for normal mode camera capture operations */
385     private val normalModeCaptureThread = HandlerThread("CameraThread").apply { start() }
386 
387     /** [Handler] corresponding to [normalModeCaptureThread] */
388     private val normalModeCaptureHandler = Handler(normalModeCaptureThread.looper)
389 
390     /** A [HandlerThread] used for saving image files */
391     private val imageSaverThread = HandlerThread("ImageSaver").apply { start() }
392 
393     /** [Handler] corresponding to [normalModeCaptureThread] */
394     private val imageSaverHandler = Handler(imageSaverThread.looper)
395 
396     /**
397      * A toast is shown when an extension is enabled or disabled. Tracking this allows cancelling
398      * the toast before showing a new one. This is specifically for scenarios where toggling an
399      * extension quickly requires cancelling the last toast before showing the new one.
400      */
401     private var toast: Toast? = null
402 
403     private var zoomRatio: Float = 1.0f
404 
405     /**
406      * Define a scale gesture detector to respond to pinch events and call setZoom on
407      * Camera.Parameters.
408      */
409     private val scaleGestureListener =
410         object : ScaleGestureDetector.SimpleOnScaleGestureListener() {
411             override fun onScaleBegin(detector: ScaleGestureDetector): Boolean = hasZoomSupport()
412 
413             override fun onScale(detector: ScaleGestureDetector): Boolean {
414                 // Set the zoom level
415                 startZoom(detector.scaleFactor)
416                 return true
417             }
418         }
419 
420     override fun onCreate(savedInstanceState: Bundle?) {
421         super.onCreate(savedInstanceState)
422         Log.d(TAG, "onCreate()")
423         setContentView(R.layout.activity_camera_extensions)
424 
425         // Retrieves the cameraThread that will be used to check whether the code is correctly
426         // executed on the camera thread.
427         runBlocking {
428             coroutineScope
429                 .launch(cameraTaskDispatcher) { cameraThread = Thread.currentThread() }
430                 .join()
431         }
432 
433         cameraManager = getSystemService(Context.CAMERA_SERVICE) as CameraManager
434         backCameraId = getLensFacingCameraId(cameraManager, CameraCharacteristics.LENS_FACING_BACK)
435         frontCameraId =
436             getLensFacingCameraId(cameraManager, CameraCharacteristics.LENS_FACING_FRONT)
437 
438         currentCameraId =
439             if (isCameraSupportExtensions(backCameraId)) {
440                 backCameraId
441             } else if (isCameraSupportExtensions(frontCameraId)) {
442                 frontCameraId
443             } else {
444                 Toast.makeText(
445                         this,
446                         "Can't find camera supporting Camera2 extensions.",
447                         Toast.LENGTH_SHORT
448                     )
449                     .show()
450                 switchActivity(CameraExtensionsActivity::class.java.name)
451                 return
452             }
453 
454         sessionImageUriSet = SessionMediaUriSet(contentResolver)
455 
456         // Gets params from extra bundle
457         intent.extras?.let { bundle ->
458             currentCameraId = bundle.getString(INTENT_EXTRA_KEY_CAMERA_ID, currentCameraId)
459             currentExtensionMode =
460                 bundle.getInt(INTENT_EXTRA_KEY_EXTENSION_MODE, currentExtensionMode)
461             extensionModeEnabled = currentExtensionMode != EXTENSION_MODE_NONE
462 
463             requestCode = bundle.getInt(INTENT_EXTRA_KEY_REQUEST_CODE, -1)
464             isRequestMode = requestCode != -1
465 
466             if (isRequestMode) {
467                 setupForRequestMode()
468             }
469         }
470 
471         updateExtensionInfo()
472         setupTextureView()
473         enableUiControl(false)
474         setupUiControl()
475         setupVideoStabilizationModeView()
476         enableZoomAndTapToFocusGesture()
477     }
478 
479     private fun setupForRequestMode() {
480         checkRunOnMainThread()
481         result = Intent()
482         result.putExtra(INTENT_EXTRA_KEY_EXTENSION_MODE, currentExtensionMode)
483         result.putExtra(INTENT_EXTRA_KEY_ERROR_CODE, ERROR_CODE_NONE)
484         setResult(requestCode, result)
485 
486         if (!isCamera2ExtensionModeSupported(this, currentCameraId, currentExtensionMode)) {
487             result.putExtra(INTENT_EXTRA_KEY_ERROR_CODE, ERROR_CODE_EXTENSION_MODE_NOT_SUPPORT)
488             finish()
489             return
490         }
491 
492         val lensFacing =
493             cameraManager
494                 .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.LENS_FACING]
495 
496         supportActionBar?.title = resources.getString(R.string.camera2_extensions_validator)
497         supportActionBar!!.subtitle =
498             "Camera $currentCameraId [${getLensFacingString(lensFacing!!)}][${
499                 getCamera2ExtensionModeStringFromId(currentExtensionMode)
500             }]"
501 
502         findViewById<Button>(R.id.PhotoToggle).visibility = View.INVISIBLE
503         findViewById<Button>(R.id.Switch).visibility = View.INVISIBLE
504 
505         setExtensionToggleButtonResource()
506         findViewById<ImageButton>(R.id.ExtensionToggle).apply {
507             visibility = View.VISIBLE
508             setOnClickListener {
509                 measureStreamConfigurationLatency = true
510                 extensionModeEnabled = !extensionModeEnabled
511 
512                 // Close current capture session to re-create the new capture session for the
513                 // new settings
514                 closeCaptureSession()
515                 setExtensionToggleButtonResource()
516 
517                 val newToast =
518                     if (extensionModeEnabled) {
519                         Toast.makeText(
520                             this@Camera2ExtensionsActivity,
521                             "Extension is enabled!",
522                             Toast.LENGTH_SHORT
523                         )
524                     } else {
525                         Toast.makeText(
526                             this@Camera2ExtensionsActivity,
527                             "Extension is disabled!",
528                             Toast.LENGTH_SHORT
529                         )
530                     }
531                 toast?.cancel()
532                 newToast.show()
533                 toast = newToast
534             }
535         }
536     }
537 
538     @Suppress("DEPRECATION") // EXTENSION_BEAUTY
539     private fun setExtensionToggleButtonResource() {
540         checkRunOnMainThread()
541         val extensionToggleButton: ImageButton = findViewById(R.id.ExtensionToggle)
542 
543         if (!extensionModeEnabled) {
544             extensionToggleButton.setImageResource(R.drawable.outline_block)
545             return
546         }
547 
548         val resourceId =
549             when (currentExtensionMode) {
550                 CameraExtensionCharacteristics.EXTENSION_HDR -> R.drawable.outline_hdr_on
551                 CameraExtensionCharacteristics.EXTENSION_BOKEH -> R.drawable.outline_portrait
552                 CameraExtensionCharacteristics.EXTENSION_NIGHT -> R.drawable.outline_bedtime
553                 CameraExtensionCharacteristics.EXTENSION_BEAUTY ->
554                     R.drawable.outline_face_retouching_natural
555                 CameraExtensionCharacteristics.EXTENSION_AUTOMATIC ->
556                     R.drawable.outline_auto_awesome
557                 else -> throw IllegalArgumentException("Invalid extension mode!")
558             }
559 
560         extensionToggleButton.setImageResource(resourceId)
561     }
562 
563     private fun getLensFacingString(lensFacing: Int) =
564         when (lensFacing) {
565             CameraMetadata.LENS_FACING_BACK -> "BACK"
566             CameraMetadata.LENS_FACING_FRONT -> "FRONT"
567             CameraMetadata.LENS_FACING_EXTERNAL -> "EXTERNAL"
568             else -> throw IllegalArgumentException("Invalid lens facing!!")
569         }
570 
571     private fun isCameraSupportExtensions(cameraId: String): Boolean {
572         val characteristics = cameraManager.getCameraExtensionCharacteristics(cameraId)
573         return characteristics.supportedExtensions.isNotEmpty()
574     }
575 
576     private fun updateExtensionInfo() {
577         checkRunOnMainThread()
578         Log.d(
579             TAG,
580             "updateExtensionInfo() - camera Id: $currentCameraId, current extension mode: " +
581                 "$currentExtensionMode"
582         )
583         extensionCharacteristics = cameraManager.getCameraExtensionCharacteristics(currentCameraId)
584         supportedExtensionModes.clear()
585         supportedExtensionModes.add(EXTENSION_MODE_NONE)
586         supportedExtensionModes.addAll(extensionCharacteristics.supportedExtensions)
587 
588         cameraSensorRotationDegrees =
589             cameraManager
590                 .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.SENSOR_ORIENTATION]
591                 ?: 0
592 
593         currentExtensionIdx = getExtensionModeIndex(currentExtensionMode)
594         extensionModeEnabled = currentExtensionMode != EXTENSION_MODE_NONE
595     }
596 
597     private fun getExtensionModeIndex(extensionMode: Int): Int {
598         checkRunOnMainThread()
599         supportedExtensionModes.forEachIndexed { index, mode ->
600             if (extensionMode == mode) {
601                 return index
602             }
603         }
604         // This should happen only when switching camera. The new target camera might not support
605         // the original extensions mode.
606         return -1
607     }
608 
609     private fun setupTextureView() {
610         val viewFinderStub = findViewById<ViewStub>(R.id.viewFinderStub)
611         viewFinderStub.layoutResource = R.layout.full_textureview
612         containerView = viewFinderStub.inflate()
613         textureView = containerView.findViewById(R.id.textureView)
614         textureView.surfaceTextureListener = surfaceTextureListener
615     }
616 
617     private fun setupVideoStabilizationModeView() {
618         videoStabilizationToggleView = findViewById(R.id.videoStabilizationToggle)
619         videoStabilizationModeView = findViewById(R.id.videoStabilizationMode)
620 
621         val availableModes =
622             cameraManager
623                 .getCameraCharacteristics(currentCameraId)
624                 .get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES)
625                 ?: intArrayOf()
626 
627         if (
628             availableModes.contains(
629                 CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION
630             )
631         ) {
632             videoStabilizationToggleView.visibility = View.VISIBLE
633             videoStabilizationModeView.visibility = View.VISIBLE
634 
635             videoStabilizationToggleView.setOnCheckedChangeListener { _, isChecked ->
636                 val mode = if (isChecked) "Preview" else "Off"
637                 videoStabilizationModeView.text = "Video Stabilization Mode: $mode"
638 
639                 setRepeatingRequest()
640             }
641         } else {
642             videoStabilizationToggleView.visibility = View.GONE
643             videoStabilizationModeView.visibility = View.GONE
644         }
645     }
646 
647     private fun enableUiControl(enabled: Boolean) {
648         findViewById<Button>(R.id.PhotoToggle).isEnabled = enabled
649         findViewById<Button>(R.id.Switch).isEnabled = enabled
650         findViewById<Button>(R.id.Picture).isEnabled = enabled
651     }
652 
653     private fun enableZoomAndTapToFocusGesture() {
654         val scaleGestureDetector = ScaleGestureDetector(this, scaleGestureListener)
655         textureView.setOnTouchListener { _, event ->
656             scaleGestureDetector.onTouchEvent(event)
657             tapToFocusDetector.onTouchEvent(event)
658             true
659         }
660     }
661 
662     private fun setupUiControl() {
663         checkRunOnMainThread()
664         val extensionModeToggleButton = findViewById<Button>(R.id.PhotoToggle)
665         extensionModeToggleButton.text = getCamera2ExtensionModeStringFromId(currentExtensionMode)
666         extensionModeToggleButton.setOnClickListener {
667             enableUiControl(false)
668             currentExtensionIdx = (currentExtensionIdx + 1) % supportedExtensionModes.size
669             currentExtensionMode = supportedExtensionModes[currentExtensionIdx]
670             extensionModeEnabled = currentExtensionMode != EXTENSION_MODE_NONE
671             extensionModeToggleButton.text =
672                 getCamera2ExtensionModeStringFromId(currentExtensionMode)
673             closeCaptureSession()
674         }
675 
676         val cameraSwitchButton = findViewById<Button>(R.id.Switch)
677         cameraSwitchButton.setOnClickListener { switchCamera() }
678 
679         val captureButton = findViewById<Button>(R.id.Picture)
680         captureButton.setOnClickListener {
681             enableUiControl(false)
682             resetImageSavedIdlingResource()
683             takePicture()
684         }
685     }
686 
687     private fun determineNextStepOnUiThread(
688         state: Int,
689         cameraId: String,
690         extensionMode: Int? = null
691     ) {
692         coroutineScope.launch(Dispatchers.Main) {
693             when (state) {
694                 STATE_CAMERA_OPENED -> {
695                     if (activityStopped || currentCameraId != cameraId) {
696                         closeCamera()
697                     } else {
698                         updatePreviewSize()
699                         openCaptureSession(currentExtensionMode, extensionModeEnabled)
700                     }
701                 }
702                 STATE_CAMERA_CLOSED -> {
703                     if (!activityStopped) {
704                         openCamera(cameraManager, currentCameraId)
705                     }
706                 }
707                 STATE_CAPTURE_SESSION_CONFIGURED -> {
708                     if (
709                         activityStopped ||
710                             (extensionModeEnabled && currentExtensionMode != extensionMode) ||
711                             (!extensionModeEnabled && extensionMode != EXTENSION_MODE_NONE)
712                     ) {
713                         closeCaptureSession()
714                     } else {
715                         setRepeatingRequest()
716                         enableUiControl(true)
717                         if (!captureSessionConfiguredIdlingResource.isIdleNow) {
718                             captureSessionConfiguredIdlingResource.decrement()
719                         }
720                     }
721                 }
722                 STATE_CAPTURE_SESSION_CLOSED -> {
723                     if (activityStopped || currentCameraId != cameraId) {
724                         closeCamera()
725                     } else {
726                         updatePreviewSize()
727                         openCaptureSession(currentExtensionMode, extensionModeEnabled)
728                     }
729                 }
730             }
731         }
732     }
733 
734     @VisibleForTesting
735     fun switchCamera() {
736         checkRunOnMainThread()
737         val newCameraId = if (currentCameraId == backCameraId) frontCameraId else backCameraId
738 
739         if (!isCameraSupportExtensions(newCameraId)) {
740             Toast.makeText(
741                     this,
742                     "Camera of the other lens facing doesn't support Camera2 extensions.",
743                     Toast.LENGTH_SHORT
744                 )
745                 .show()
746             return
747         }
748 
749         enableUiControl(false)
750         currentCameraId = newCameraId
751         updateExtensionInfo()
752 
753         val extensionModeToggleButton = findViewById<Button>(R.id.PhotoToggle)
754         extensionModeToggleButton.text = getCamera2ExtensionModeStringFromId(currentExtensionMode)
755 
756         closeCaptureSession()
757     }
758 
759     override fun onStart() {
760         super.onStart()
761         Log.d(TAG, "onStart()")
762         activityStopped = false
763         if (textureView.isAvailable) {
764             setupAndStartPreview()
765         }
766     }
767 
768     override fun onStop() {
769         Log.d(TAG, "onStop()++")
770         super.onStop()
771         activityStopped = true
772         // Closes the capture session to shut down the whole pipeline.
773         closeCaptureSession()
774         lastSurfaceTextureTimestampNanos = 0L
775         Log.d(TAG, "onStop()--")
776     }
777 
778     override fun onDestroy() {
779         Log.d(TAG, "onDestroy()++")
780         super.onDestroy()
781         Log.d(TAG, "Waiting for capture session closed...")
782         synchronized(lock) { captureSessionClosedDeferred }
783             .asListenableFuture()
784             .addListener(
785                 {
786                     previewSurface?.release()
787                     textureView.surfaceTexture?.release()
788                     normalModeCaptureThread.quitSafely()
789                     Log.d(TAG, "Surface texture released. $previewSurface")
790                     imageSaveTerminationFuture.addListener(
791                         {
792                             stillImageReader?.close()
793                             Log.d(TAG, "stillImageReader closed. ${stillImageReader?.surface}")
794                             imageSaverThread.quitSafely()
795                         },
796                         mainExecutor
797                     )
798                 },
799                 mainExecutor
800             )
801 
802         streamConfigurationLatency[KEY_CAMERA2_LATENCY]?.also {
803             val min = "${it.minOrNull() ?: "n/a"}"
804             val max = "${it.maxOrNull() ?: "n/a"}"
805             val avg = it.average().format(2)
806 
807             Log.d(
808                 TAG,
809                 "Camera2 Stream Configuration Latency: min=${min}ms max=${max}ms avg=${avg}ms"
810             )
811         }
812         var testResultDetails = ""
813         streamConfigurationLatency[KEY_CAMERA_EXTENSION_LATENCY]?.also {
814             val min = "${it.minOrNull() ?: "n/a"}"
815             val max = "${it.maxOrNull() ?: "n/a"}"
816             val avg = it.average().format(2)
817             testResultDetails = "min=${min}ms max=${max}ms avg=${avg}ms"
818 
819             Log.d(TAG, "Camera Extensions Stream Configuration Latency: $testResultDetails")
820         }
821 
822         val durations = streamConfigurationLatency[KEY_CAMERA_EXTENSION_LATENCY] ?: emptyList()
823         val testResult =
824             if (durations.isNotEmpty()) {
825                 if (durations.average() > MAX_EXTENSION_LATENCY_MILLIS) {
826                     TEST_RESULT_FAILED
827                 } else {
828                     TEST_RESULT_PASSED
829                 }
830             } else {
831                 TEST_RESULT_NOT_TESTED
832             }
833 
834         val testResults = TestResults.getInstance(this@Camera2ExtensionsActivity)
835         testResults.updateTestResultAndSave(
836             TEST_TYPE_CAMERA2_EXTENSION_STREAM_CONFIG_LATENCY,
837             currentCameraId,
838             currentExtensionMode,
839             testResult,
840             testResultDetails
841         )
842 
843         Log.d(TAG, "onDestroy()--")
844     }
845 
846     private fun closeCaptureSession() =
847         coroutineScope.async(cameraTaskDispatcher) {
848             Log.d(TAG, "closeCaptureSession()++")
849             // Directly return here if no capture session is configured yet. If the newly created
850             // capture session should be closed, handleCaptureSessionOnConfiguredEvent will invoke
851             // this function again to close it when the capture session is configured.
852             if (getCurrentState() != STATE_CAPTURE_SESSION_CONFIGURED) {
853                 return@async
854             }
855             setCurrentState(STATE_CAPTURE_SESSION_CLOSING)
856             resetCaptureSessionConfiguredIdlingResource()
857 
858             try {
859                 if (cameraCaptureSession is CameraCaptureSession) {
860                     (cameraCaptureSession as CameraCaptureSession).close()
861                     Log.d(TAG, "closed CameraCaptureSession")
862                 } else {
863                     (cameraCaptureSession as CameraExtensionSession).close()
864                     Log.d(TAG, "closed CameraExtensionSession")
865                 }
866             } catch (e: Exception) {
867                 Log.e(TAG, e.toString())
868             }
869             Log.d(TAG, "closeCaptureSession()--")
870         }
871 
872     /**
873      * Sets up the UI layout settings for the specified camera and extension mode. And then,
874      * triggers to open the camera and capture session to start the preview with the extension mode
875      * enabled.
876      */
877     private fun setupAndStartPreview() {
878         checkRunOnMainThread()
879         if (!textureView.isAvailable) {
880             Toast.makeText(this, "TextureView is invalid!!", Toast.LENGTH_SHORT).show()
881             finish()
882             return
883         }
884 
885         updatePreviewSize()
886         openCamera(cameraManager, currentCameraId)
887     }
888 
889     @Suppress("DEPRECATION") /* defaultDisplay */
890     private fun updatePreviewSize() {
891         checkRunOnMainThread()
892         val previewResolution =
893             pickPreviewResolution(
894                 cameraManager,
895                 currentCameraId,
896                 resources.displayMetrics,
897                 if (extensionModeEnabled) currentExtensionMode else EXTENSION_MODE_NONE
898             )
899 
900         if (previewResolution == null) {
901             Toast.makeText(this, "Invalid preview extension sizes!.", Toast.LENGTH_SHORT).show()
902             finish()
903             return
904         }
905 
906         Log.d(TAG, "Set default buffer size to previewResolution: $previewResolution")
907 
908         textureView.surfaceTexture?.setDefaultBufferSize(
909             previewResolution.width,
910             previewResolution.height
911         )
912 
913         textureView.layoutParams =
914             FrameLayout.LayoutParams(previewResolution.width, previewResolution.height)
915 
916         val containerViewSize = Size(containerView.width, containerView.height)
917 
918         val lensFacing =
919             cameraManager
920                 .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.LENS_FACING]
921 
922         transformTextureView(
923             textureView,
924             containerViewSize,
925             previewResolution,
926             windowManager.defaultDisplay.rotation,
927             cameraSensorRotationDegrees,
928             lensFacing == CameraCharacteristics.LENS_FACING_BACK
929         )
930 
931         tapToFocusDetector =
932             TapToFocusDetector(this, textureView, getCameraInfo(), display!!.rotation, ::tapToFocus)
933     }
934 
935     private fun getCameraInfo(): CameraInfo {
936         checkRunOnMainThread()
937         val lensFacing =
938             cameraManager
939                 .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.LENS_FACING]
940         val sensorOrientation =
941             cameraManager
942                 .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.SENSOR_ORIENTATION]
943         val activeArraySize =
944             cameraManager
945                 .getCameraCharacteristics(currentCameraId)[
946                     CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE]
947         return CameraInfo(lensFacing!!, sensorOrientation!!.toFloat(), activeArraySize!!)
948     }
949 
950     private fun tapToFocus(meteringRectangles: Array<MeteringRectangle?>) {
951         coroutineScope.launch(cameraTaskDispatcher) {
952             focusMeteringControl.updateMeteringRectangles(meteringRectangles)
953         }
954     }
955 
956     private fun checkRunOnMainThread() {
957         if (Thread.currentThread() != Looper.getMainLooper().thread) {
958             val exception = IllegalStateException("Must run on the main thread!")
959             Log.e(TAG, exception.toString())
960             exception.printStackTrace()
961             throw exception
962         }
963     }
964 
965     private fun checkRunOnCameraThread() {
966         if (
967             Thread.currentThread() != cameraThread &&
968                 Thread.currentThread() != normalModeCaptureThread
969         ) {
970             val exception = IllegalStateException("Must run on the camera thread!")
971             Log.e(TAG, exception.toString())
972             exception.printStackTrace()
973             throw exception
974         }
975     }
976 
977     private fun getCurrentState(): Int {
978         checkRunOnCameraThread()
979         return currentState
980     }
981 
982     private fun setCurrentState(state: Int) {
983         checkRunOnCameraThread()
984         Log.d(
985             TAG,
986             "Old state: ${getStateString(currentState)}, new state: ${getStateString(state)}"
987         )
988         currentState = state
989     }
990 
991     private fun getStateString(state: Int) =
992         when (state) {
993             STATE_CAMERA_CLOSED -> "STATE_CAMERA_CLOSED"
994             STATE_CAMERA_OPENING -> "STATE_CAMERA_OPENING"
995             STATE_CAMERA_OPENED -> "STATE_CAMERA_OPENED"
996             STATE_CAPTURE_SESSION_OPENING -> "STATE_CAPTURE_SESSION_OPENING"
997             STATE_CAPTURE_SESSION_CONFIGURED -> "STATE_CAPTURE_SESSION_CONFIGURED"
998             STATE_CAPTURE_SESSION_CLOSING -> "STATE_CAPTURE_SESSION_CLOSING"
999             STATE_CAPTURE_SESSION_CLOSED -> "STATE_CAPTURE_SESSION_CLOSED"
1000             STATE_CAMERA_CLOSING -> "STATE_CAMERA_CLOSING"
1001             else -> throw IllegalArgumentException("Invalid state value!")
1002         }
1003 
1004     /** Opens and returns the camera (as the result of the suspend coroutine) */
1005     @SuppressLint("MissingPermission")
1006     fun openCamera(
1007         manager: CameraManager,
1008         cameraId: String,
1009     ) =
1010         coroutineScope.async(cameraTaskDispatcher) {
1011             Log.d(TAG, "openCamera()++: $cameraId")
1012             if (getCurrentState() != STATE_CAMERA_CLOSED) {
1013                 return@async
1014             }
1015             setCurrentState(STATE_CAMERA_OPENING)
1016             resetCameraClosedIdlingResource()
1017             manager.openCamera(
1018                 cameraId,
1019                 cameraTaskDispatcher.asExecutor(),
1020                 object : CameraDevice.StateCallback() {
1021                     override fun onOpened(device: CameraDevice) {
1022                         Log.d(TAG, "Camera ${device.id} - onOpened")
1023                         cameraDevice = device
1024                         setCurrentState(STATE_CAMERA_OPENED)
1025                         determineNextStepOnUiThread(STATE_CAMERA_OPENED, device.id)
1026                     }
1027 
1028                     override fun onDisconnected(device: CameraDevice) {
1029                         Log.d(TAG, "Camera ${device.id} - onDisconnected")
1030                         // Closes camera when onDisconnected event is received
1031                         setCurrentState(STATE_CAMERA_CLOSING)
1032                         closeCamera()
1033                     }
1034 
1035                     override fun onClosed(device: CameraDevice) {
1036                         Log.d(TAG, "Camera ${device.id} - onClosed")
1037                         cameraDevice = null
1038                         setCurrentState(STATE_CAMERA_CLOSED)
1039                         if (!cameraClosedIdlingResource.isIdleNow) {
1040                             cameraClosedIdlingResource.decrement()
1041                         }
1042                         determineNextStepOnUiThread(STATE_CAMERA_CLOSED, device.id)
1043                     }
1044 
1045                     override fun onError(device: CameraDevice, error: Int) {
1046                         Log.d(TAG, "Camera ${device.id} - onError, error code: $error")
1047                         val msg =
1048                             when (error) {
1049                                 ERROR_CAMERA_DEVICE -> "Fatal (device)"
1050                                 ERROR_CAMERA_DISABLED -> "Device policy"
1051                                 ERROR_CAMERA_IN_USE -> "Camera in use"
1052                                 ERROR_CAMERA_SERVICE -> "Fatal (service)"
1053                                 ERROR_MAX_CAMERAS_IN_USE -> "Maximum cameras in use"
1054                                 else -> "Unknown"
1055                             }
1056                         val exc = RuntimeException("Camera $cameraId error: ($error) $msg")
1057                         Log.e(TAG, exc.message, exc)
1058                         // Closes camera when an error occurs
1059                         setCurrentState(STATE_CAMERA_CLOSING)
1060                         closeCamera()
1061                     }
1062                 }
1063             )
1064             Log.d(TAG, "openCamera()--: $cameraId")
1065         }
1066 
1067     private fun closeCamera() =
1068         coroutineScope.async(cameraTaskDispatcher) {
1069             val cameraId = cameraDevice?.id
1070             Log.d(TAG, "closeCamera()++: $cameraId")
1071             setCurrentState(STATE_CAMERA_CLOSING)
1072             cameraDevice?.close()
1073             Log.d(TAG, "closeCamera()--: $cameraId")
1074         }
1075 
1076     /** Opens and returns the extensions session (as the result of the suspend coroutine) */
1077     private fun openCaptureSession(extensionMode: Int, extensionModeEnabled: Boolean) =
1078         coroutineScope.async(cameraTaskDispatcher) {
1079             Log.d(TAG, "openCaptureSession")
1080             // Resets the metering rectangles
1081             meteringRectangles = EMPTY_RECTANGLES
1082             setCurrentState(STATE_CAPTURE_SESSION_OPENING)
1083 
1084             if (stillImageReader != null) {
1085                 val imageReaderToClose = stillImageReader!!
1086                 imageSaveTerminationFuture.addListener({ imageReaderToClose.close() }, mainExecutor)
1087             }
1088 
1089             stillImageReader = setupImageReader(cameraDevice!!.id, extensionMode)
1090 
1091             val outputConfigs = arrayListOf<OutputConfiguration>()
1092             outputConfigs.add(OutputConfiguration(stillImageReader!!.surface))
1093             outputConfigs.add(OutputConfiguration(previewSurface!!))
1094 
1095             synchronized(lock) { captureSessionClosedDeferred = CompletableDeferred() }
1096 
1097             if (extensionModeEnabled) {
1098                 createCameraExtensionSession(outputConfigs, extensionMode)
1099             } else {
1100                 createCameraCaptureSession(outputConfigs)
1101             }
1102         }
1103 
1104     /** Creates normal mode CameraCaptureSession */
1105     private fun createCameraCaptureSession(outputConfigs: ArrayList<OutputConfiguration>) {
1106         checkRunOnCameraThread()
1107         Log.d(TAG, "createCameraCaptureSession++")
1108         val sessionConfiguration =
1109             SessionConfiguration(
1110                 SESSION_REGULAR,
1111                 outputConfigs,
1112                 cameraTaskDispatcher.asExecutor(),
1113                 object : CameraCaptureSession.StateCallback() {
1114                     override fun onClosed(session: CameraCaptureSession) {
1115                         Log.d(TAG, "CaptureSession - onClosed: $session")
1116                         handleCaptureSessionOnClosedEvent(session.device.id, EXTENSION_MODE_NONE)
1117                     }
1118 
1119                     override fun onConfigured(session: CameraCaptureSession) {
1120                         Log.d(TAG, "CaptureSession - onConfigured: $session")
1121                         handleCaptureSessionOnConfiguredEvent(
1122                             session,
1123                             session.device.id,
1124                             EXTENSION_MODE_NONE
1125                         )
1126                     }
1127 
1128                     override fun onConfigureFailed(session: CameraCaptureSession) {
1129                         Log.e(TAG, "CaptureSession - onConfigureFailed: $session")
1130                         handleCaptureSessionOnConfigureFailedEvent()
1131                     }
1132                 }
1133             )
1134         cameraDevice!!.createCaptureSession(sessionConfiguration)
1135         Log.d(TAG, "createCameraCaptureSession--")
1136     }
1137 
1138     /** Creates extension mode CameraExtensionSession */
1139     private fun createCameraExtensionSession(
1140         outputConfigs: ArrayList<OutputConfiguration>,
1141         extensionMode: Int
1142     ) {
1143         checkRunOnCameraThread()
1144         Log.d(TAG, "createCameraExtensionSession++, extensionMode=$extensionMode")
1145         val extensionConfiguration =
1146             ExtensionSessionConfiguration(
1147                 extensionMode,
1148                 outputConfigs,
1149                 cameraTaskDispatcher.asExecutor(),
1150                 object : CameraExtensionSession.StateCallback() {
1151                     override fun onClosed(session: CameraExtensionSession) {
1152                         Log.d(TAG, "Extension CaptureSession - onClosed: $session")
1153                         handleCaptureSessionOnClosedEvent(cameraDevice!!.id, extensionMode)
1154                     }
1155 
1156                     override fun onConfigured(session: CameraExtensionSession) {
1157                         Log.d(TAG, "Extension CaptureSession - onConfigured: $session")
1158                         handleCaptureSessionOnConfiguredEvent(
1159                             session,
1160                             cameraDevice!!.id,
1161                             extensionMode
1162                         )
1163                     }
1164 
1165                     override fun onConfigureFailed(session: CameraExtensionSession) {
1166                         Log.e(TAG, "Extension CaptureSession - onConfigureFailed: $session")
1167                         handleCaptureSessionOnConfigureFailedEvent()
1168                     }
1169                 }
1170             )
1171         Log.d(TAG, "createCameraExtensionSession########")
1172         cameraDevice!!.createExtensionSession(extensionConfiguration)
1173         Log.d(TAG, "createCameraExtensionSession--")
1174     }
1175 
1176     private fun handleCaptureSessionOnClosedEvent(cameraId: String, mode: Int) {
1177         checkRunOnCameraThread()
1178         setCurrentState(STATE_CAPTURE_SESSION_CLOSED)
1179         cameraCaptureSession = null
1180         synchronized(lock) { captureSessionClosedDeferred.complete(Unit) }
1181         determineNextStepOnUiThread(STATE_CAPTURE_SESSION_CLOSED, cameraId, mode)
1182     }
1183 
1184     private fun handleCaptureSessionOnConfiguredEvent(session: Any, cameraId: String, mode: Int) {
1185         checkRunOnCameraThread()
1186         setCurrentState(STATE_CAPTURE_SESSION_CONFIGURED)
1187         cameraCaptureSession = session
1188         determineNextStepOnUiThread(STATE_CAPTURE_SESSION_CONFIGURED, cameraId, mode)
1189     }
1190 
1191     private fun handleCaptureSessionOnConfigureFailedEvent() {
1192         checkRunOnCameraThread()
1193         // CLoses the camera to restart the whole pipe line
1194         setCurrentState(STATE_CAMERA_CLOSING)
1195         closeCamera()
1196     }
1197 
1198     private fun startAfTrigger(meteringRectangles: Array<MeteringRectangle?>) {
1199         coroutineScope.launch(cameraTaskDispatcher) {
1200             this@Camera2ExtensionsActivity.meteringRectangles = meteringRectangles
1201             addFocusMeteringCaptureCallback()
1202 
1203             val captureBuilder = getCaptureRequestBuilder()
1204 
1205             captureBuilder.set(CONTROL_AF_TRIGGER, CONTROL_AF_TRIGGER_START)
1206 
1207             setRepeatingRequest(captureBuilder.build())
1208         }
1209     }
1210 
1211     private fun cancelAfTrigger(afTriggerType: Int) {
1212         coroutineScope.launch(cameraTaskDispatcher) {
1213             if (afTriggerType == CONTROL_AF_TRIGGER_CANCEL) {
1214                 this@Camera2ExtensionsActivity.meteringRectangles = EMPTY_RECTANGLES
1215             }
1216 
1217             removeFocusMeteringCaptureCallback()
1218 
1219             val captureBuilder = getCaptureRequestBuilder()
1220 
1221             if (afTriggerType == CONTROL_AF_TRIGGER_IDLE) {
1222                 captureBuilder.set(CONTROL_AF_TRIGGER, CONTROL_AF_TRIGGER_IDLE)
1223             } else {
1224                 captureBuilder.set(CONTROL_AF_TRIGGER, CONTROL_AF_TRIGGER_CANCEL)
1225             }
1226 
1227             setRepeatingRequest(captureBuilder.build())
1228         }
1229     }
1230 
1231     private fun addFocusMeteringCaptureCallback() {
1232         checkRunOnCameraThread()
1233         val captureCallback =
1234             focusMeteringControl.getCaptureCallback(cameraCaptureSession is CameraExtensionSession)
1235         if (cameraCaptureSession is CameraExtensionSession) {
1236             comboCaptureCallbackExtensionMode.addCaptureCallback(
1237                 captureCallback as ExtensionCaptureCallback
1238             )
1239         } else {
1240             comboCaptureCallbackNormalMode.addCaptureCallback(captureCallback as CaptureCallback)
1241         }
1242     }
1243 
1244     private fun removeFocusMeteringCaptureCallback() {
1245         checkRunOnCameraThread()
1246         val captureCallback =
1247             focusMeteringControl.getCaptureCallback(cameraCaptureSession is CameraExtensionSession)
1248         if (cameraCaptureSession is CameraExtensionSession) {
1249             comboCaptureCallbackExtensionMode.removeCaptureCallback(
1250                 captureCallback as ExtensionCaptureCallback
1251             )
1252         } else {
1253             comboCaptureCallbackNormalMode.removeCaptureCallback(captureCallback as CaptureCallback)
1254         }
1255     }
1256 
1257     private fun setRepeatingRequest(captureRequest: CaptureRequest? = null) {
1258         coroutineScope.launch(cameraTaskDispatcher) {
1259             if (cameraCaptureSession is CameraCaptureSession) {
1260                 (cameraCaptureSession as CameraCaptureSession).setRepeatingRequest(
1261                     captureRequest ?: getCaptureRequestBuilder().build(),
1262                     comboCaptureCallbackNormalMode,
1263                     normalModeCaptureHandler
1264                 )
1265             } else {
1266                 (cameraCaptureSession as CameraExtensionSession).setRepeatingRequest(
1267                     captureRequest ?: getCaptureRequestBuilder().build(),
1268                     cameraTaskDispatcher.asExecutor(),
1269                     comboCaptureCallbackExtensionMode
1270                 )
1271             }
1272         }
1273     }
1274 
1275     private fun getCaptureRequestBuilder(): CaptureRequest.Builder {
1276         checkRunOnCameraThread()
1277         val captureBuilder = cameraDevice!!.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
1278         captureBuilder.addTarget(previewSurface!!)
1279         val videoStabilizationMode =
1280             if (videoStabilizationToggleView.isChecked) {
1281                 CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION
1282             } else {
1283                 CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_OFF
1284             }
1285 
1286         captureBuilder.set(CONTROL_VIDEO_STABILIZATION_MODE, videoStabilizationMode)
1287 
1288         captureBuilder.set(CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio)
1289 
1290         if (!meteringRectangles.contentEquals(EMPTY_RECTANGLES)) {
1291             captureBuilder.set(CONTROL_AF_MODE, CONTROL_AF_MODE_AUTO)
1292             captureBuilder.set(CONTROL_AF_REGIONS, meteringRectangles)
1293             captureBuilder.set(CONTROL_AE_MODE, CONTROL_AE_MODE_ON)
1294             captureBuilder.set(CONTROL_AE_REGIONS, meteringRectangles)
1295             captureBuilder.set(CONTROL_AWB_REGIONS, meteringRectangles)
1296         }
1297 
1298         return captureBuilder
1299     }
1300 
1301     private fun setupImageReader(cameraId: String, extensionMode: Int): ImageReader {
1302         val (size, format) =
1303             pickStillImageResolution(
1304                 cameraManager.getCameraCharacteristics(cameraId),
1305                 extensionCharacteristics,
1306                 extensionMode
1307             )
1308 
1309         Log.d(TAG, "Setup image reader - size: $size, format: $format")
1310 
1311         return ImageReader.newInstance(size.width, size.height, format, 1)
1312     }
1313 
1314     /** Takes a picture. */
1315     private fun takePicture() {
1316         checkRunOnMainThread()
1317         val (fileName, suffix) = generateFileName(currentCameraId, currentExtensionMode)
1318         val rotationDegrees: Int
1319         try {
1320             val lensFacing =
1321                 cameraManager
1322                     .getCameraCharacteristics(currentCameraId)[CameraCharacteristics.LENS_FACING]
1323 
1324             rotationDegrees =
1325                 calculateRelativeImageRotationDegrees(
1326                     (surfaceRotationToRotationDegrees(display!!.rotation)),
1327                     cameraSensorRotationDegrees,
1328                     lensFacing == CameraCharacteristics.LENS_FACING_BACK
1329                 )
1330         } catch (e: Exception) {
1331             Log.e(TAG, e.toString())
1332             return
1333         }
1334 
1335         var takePictureCompleter: Completer<Any>? = null
1336 
1337         imageSaveTerminationFuture =
1338             CallbackToFutureAdapter.getFuture<Any> {
1339                 takePictureCompleter = it
1340                 "imageSaveTerminationFuture"
1341             }
1342 
1343         stillImageReader!!.setOnImageAvailableListener(
1344             { reader: ImageReader ->
1345                 val imageUri = acquireImageAndSave(reader, fileName, suffix, rotationDegrees)
1346 
1347                 imageUri?.let { sessionImageUriSet.add(it) }
1348 
1349                 stillImageReader!!.setOnImageAvailableListener(null, null)
1350                 takePictureCompleter?.set(null)
1351 
1352                 if (!imageSavedIdlingResource.isIdleNow) {
1353                     imageSavedIdlingResource.decrement()
1354                 }
1355 
1356                 coroutineScope.launch(Dispatchers.Main) {
1357                     if (isRequestMode) {
1358                         if (imageUri == null) {
1359                             result.putExtra(
1360                                 INTENT_EXTRA_KEY_ERROR_CODE,
1361                                 ERROR_CODE_SAVE_IMAGE_FAILED
1362                             )
1363                         } else {
1364                             result.putExtra(INTENT_EXTRA_KEY_IMAGE_URI, imageUri)
1365                             result.putExtra(
1366                                 INTENT_EXTRA_KEY_IMAGE_ROTATION_DEGREES,
1367                                 rotationDegrees
1368                             )
1369                         }
1370                         finish()
1371                     } else {
1372                         enableUiControl(true)
1373                     }
1374                 }
1375             },
1376             imageSaverHandler
1377         )
1378         submitStillImageCaptureRequest(takePictureCompleter!!)
1379     }
1380 
1381     /** Acquires the latest image from the image reader and save it to the Pictures folder */
1382     private fun acquireImageAndSave(
1383         imageReader: ImageReader,
1384         fileName: String,
1385         suffix: String,
1386         rotationDegrees: Int
1387     ): Uri? {
1388         var uri: Uri?
1389 
1390         imageReader.acquireLatestImage().let { image ->
1391             try {
1392                 uri =
1393                     if (isRequestMode) {
1394                         // Saves as temp file if the activity is called by other validation activity
1395                         // to capture a image.
1396                         FileUtil.saveImageToTempFile(image, fileName, suffix, null, rotationDegrees)
1397                     } else {
1398                         FileUtil.saveImage(
1399                             image,
1400                             fileName,
1401                             suffix,
1402                             "Pictures/ExtensionsPictures",
1403                             contentResolver,
1404                             rotationDegrees
1405                         )
1406                     }
1407             } finally {
1408                 image.close()
1409             }
1410 
1411             val msg =
1412                 if (uri != null) {
1413                     "Saved image to $fileName.jpg"
1414                 } else {
1415                     "Failed to save image."
1416                 }
1417 
1418             if (!isRequestMode) {
1419                 coroutineScope.launch(Dispatchers.Main) {
1420                     Toast.makeText(this@Camera2ExtensionsActivity, msg, Toast.LENGTH_SHORT).show()
1421                 }
1422             }
1423         }
1424 
1425         return uri
1426     }
1427 
1428     private fun submitStillImageCaptureRequest(takePictureCompleter: Completer<Any>?) {
1429         coroutineScope.launch(cameraTaskDispatcher) {
1430             Preconditions.checkState(
1431                 cameraCaptureSession != null,
1432                 "take picture button is only enabled when session is configured successfully"
1433             )
1434 
1435             val captureBuilder =
1436                 cameraDevice!!.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE)
1437             captureBuilder.addTarget(stillImageReader!!.surface)
1438 
1439             if (cameraCaptureSession is CameraCaptureSession) {
1440                 (cameraCaptureSession as CameraCaptureSession).capture(
1441                     captureBuilder.build(),
1442                     object : CaptureCallback() {
1443                         override fun onCaptureFailed(
1444                             session: CameraCaptureSession,
1445                             request: CaptureRequest,
1446                             failure: CaptureFailure
1447                         ) {
1448                             takePictureCompleter?.set(null)
1449                             Log.e(TAG, "Failed to take picture.")
1450                         }
1451                     },
1452                     normalModeCaptureHandler
1453                 )
1454             } else {
1455                 (cameraCaptureSession as CameraExtensionSession).capture(
1456                     captureBuilder.build(),
1457                     cameraTaskDispatcher.asExecutor(),
1458                     object : ExtensionCaptureCallback() {
1459                         override fun onCaptureFailed(
1460                             session: CameraExtensionSession,
1461                             request: CaptureRequest
1462                         ) {
1463                             takePictureCompleter?.set(null)
1464                             Log.e(TAG, "Failed to take picture.")
1465                         }
1466 
1467                         override fun onCaptureSequenceCompleted(
1468                             session: CameraExtensionSession,
1469                             sequenceId: Int
1470                         ) {
1471                             Log.v(TAG, "onCaptureProcessSequenceCompleted: $sequenceId")
1472                         }
1473                     }
1474                 )
1475             }
1476         }
1477     }
1478 
1479     /**
1480      * Generate the output file name and suffix depending on whether the image is requested by the
1481      * validation activity.
1482      */
1483     private fun generateFileName(cameraId: String, extensionMode: Int): Pair<String, String> {
1484         val fileName: String
1485         val suffix: String
1486 
1487         if (isRequestMode) {
1488             val lensFacing =
1489                 cameraManager
1490                     .getCameraCharacteristics(cameraId)[CameraCharacteristics.LENS_FACING]!!
1491             fileName =
1492                 "[Camera2Extension][Camera-$cameraId][${getLensFacingStringFromInt(lensFacing)}][${
1493                     getCamera2ExtensionModeStringFromId(extensionMode)
1494                 }]${if (extensionModeEnabled) "[Enabled]" else "[Disabled]"}"
1495             suffix = ""
1496         } else {
1497             val formatter: Format = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS", Locale.US)
1498             fileName =
1499                 "[${formatter.format(Calendar.getInstance().time)}][Camera2]${
1500                     getCamera2ExtensionModeStringFromId(extensionMode)
1501                 }"
1502             suffix = ".jpg"
1503         }
1504 
1505         return Pair(fileName, suffix)
1506     }
1507 
1508     private fun getLensFacingStringFromInt(lensFacing: Int): String =
1509         when (lensFacing) {
1510             CameraMetadata.LENS_FACING_BACK -> "BACK"
1511             CameraMetadata.LENS_FACING_FRONT -> "FRONT"
1512             CameraMetadata.LENS_FACING_EXTERNAL -> "EXTERNAL"
1513             else -> throw IllegalArgumentException("Invalid lens facing!!")
1514         }
1515 
1516     override fun onCreateOptionsMenu(menu: Menu): Boolean {
1517         if (!isRequestMode) {
1518             val inflater = menuInflater
1519             inflater.inflate(R.menu.main_menu_camera2_extensions_activity, menu)
1520         }
1521 
1522         return true
1523     }
1524 
1525     override fun onOptionsItemSelected(item: MenuItem): Boolean {
1526         when (item.itemId) {
1527             R.id.menu_camerax_extensions -> {
1528                 switchActivity(CameraExtensionsActivity::class.java.name)
1529                 return true
1530             }
1531             R.id.menu_validation_tool -> {
1532                 switchActivity(CameraValidationResultActivity::class.java.name)
1533                 return true
1534             }
1535         }
1536         return super.onOptionsItemSelected(item)
1537     }
1538 
1539     private fun switchActivity(className: String) {
1540         // Set the activityStopped as true early because the onStop event might come late after the
1541         // target activity is launched. The camera might be re-opened in this activity to cause the
1542         // ERROR_CAMERA_IN_USE problem when the target activity tries to open the camera.
1543         activityStopped = true
1544         val intent = Intent()
1545         intent.flags = Intent.FLAG_ACTIVITY_CLEAR_TASK or Intent.FLAG_ACTIVITY_NEW_TASK
1546         intent.setClassName(this, className)
1547         startActivity(intent)
1548     }
1549 
1550     @VisibleForTesting
1551     fun getCameraClosedIdlingResource(): CountingIdlingResource = cameraClosedIdlingResource
1552 
1553     @VisibleForTesting
1554     fun getCaptureSessionConfiguredIdlingResource(): CountingIdlingResource =
1555         captureSessionConfiguredIdlingResource
1556 
1557     @VisibleForTesting
1558     fun getPreviewIdlingResource(): CountingIdlingResource = previewIdlingResource
1559 
1560     @VisibleForTesting
1561     fun getImageSavedIdlingResource(): CountingIdlingResource = imageSavedIdlingResource
1562 
1563     private fun resetCameraClosedIdlingResource() {
1564         if (cameraClosedIdlingResource.isIdleNow) {
1565             cameraClosedIdlingResource.increment()
1566         }
1567     }
1568 
1569     private fun resetCaptureSessionConfiguredIdlingResource() {
1570         if (captureSessionConfiguredIdlingResource.isIdleNow) {
1571             captureSessionConfiguredIdlingResource.increment()
1572         }
1573     }
1574 
1575     @VisibleForTesting
1576     fun resetPreviewIdlingResource() {
1577         receivedCaptureProcessStartedCount.set(0)
1578         receivedPreviewFrameCount.set(0)
1579 
1580         if (captureProcessStartedIdlingResource.isIdleNow) {
1581             captureProcessStartedIdlingResource.increment()
1582         }
1583 
1584         if (previewIdlingResource.isIdleNow) {
1585             previewIdlingResource.increment()
1586         }
1587     }
1588 
1589     private fun resetImageSavedIdlingResource() {
1590         if (imageSavedIdlingResource.isIdleNow) {
1591             imageSavedIdlingResource.increment()
1592         }
1593     }
1594 
1595     @VisibleForTesting
1596     fun deleteSessionImages() {
1597         sessionImageUriSet.deleteAllUris()
1598     }
1599 
1600     private class SessionMediaUriSet(val contentResolver: ContentResolver) {
1601         private val mSessionMediaUris: MutableSet<Uri> = mutableSetOf()
1602 
1603         fun add(uri: Uri) {
1604             synchronized(mSessionMediaUris) { mSessionMediaUris.add(uri) }
1605         }
1606 
1607         fun deleteAllUris() {
1608             synchronized(mSessionMediaUris) {
1609                 val it = mSessionMediaUris.iterator()
1610                 while (it.hasNext()) {
1611                     contentResolver.delete(it.next(), null, null)
1612                     it.remove()
1613                 }
1614             }
1615         }
1616     }
1617 
1618     private fun startZoom(scaleFactor: Float) {
1619         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return
1620 
1621         zoomRatio =
1622             (zoomRatio * scaleFactor).coerceIn(
1623                 ZoomUtil.minZoom(cameraManager.getCameraCharacteristics(currentCameraId)),
1624                 ZoomUtil.maxZoom(cameraManager.getCameraCharacteristics(currentCameraId))
1625             )
1626         Log.d(TAG, "onScale: $zoomRatio")
1627         setRepeatingRequest()
1628     }
1629 
1630     /** Not all cameras have zoom support. Returns true if zoom is supported otherwise false. */
1631     private fun hasZoomSupport(): Boolean {
1632         checkRunOnMainThread()
1633         return if (!extensionModeEnabled) {
1634             ZoomUtil.hasZoomSupport(currentCameraId, cameraManager)
1635         } else if (extensionModeEnabled && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
1636             ZoomUtilExtensions.hasZoomSupport(currentCameraId, cameraManager, currentExtensionMode)
1637         } else {
1638             false
1639         }
1640     }
1641 
1642     @RequiresApi(33)
1643     private object ZoomUtilExtensions {
1644         @JvmStatic
1645         fun hasZoomSupport(
1646             cameraId: String,
1647             cameraManager: CameraManager,
1648             extensionMode: Int
1649         ): Boolean =
1650             cameraManager
1651                 .getCameraExtensionCharacteristics(cameraId)
1652                 .getAvailableCaptureRequestKeys(extensionMode)
1653                 .contains(CaptureRequest.CONTROL_ZOOM_RATIO)
1654     }
1655 
1656     @RequiresApi(31)
1657     private object ZoomUtil {
1658         fun hasZoomSupport(cameraId: String, cameraManager: CameraManager): Boolean {
1659             val characteristics = cameraManager.getCameraCharacteristics(cameraId)
1660             val availableCaptureRequestKeys = characteristics.availableCaptureRequestKeys
1661             return availableCaptureRequestKeys.contains(CaptureRequest.CONTROL_ZOOM_RATIO)
1662         }
1663 
1664         fun minZoom(characteristics: CameraCharacteristics): Float =
1665             characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE)?.lower ?: 1.0f
1666 
1667         fun maxZoom(characteristics: CameraCharacteristics): Float =
1668             characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE)?.upper ?: 1.0f
1669     }
1670 
1671     /**
1672      * A combo ExtensionCaptureCallback implementation to receive to pass the events to the
1673      * underlying callbacks.
1674      */
1675     private class ComboCaptureCallbackExtensionMode : ExtensionCaptureCallback() {
1676         private val captureCallbacks: MutableList<ExtensionCaptureCallback> = mutableListOf()
1677 
1678         fun addCaptureCallback(captureCallback: ExtensionCaptureCallback) {
1679             if (!captureCallbacks.contains(captureCallback)) {
1680                 captureCallbacks.add(captureCallback)
1681             }
1682         }
1683 
1684         fun removeCaptureCallback(captureCallback: ExtensionCaptureCallback) {
1685             captureCallbacks.remove(captureCallback)
1686         }
1687 
1688         override fun onCaptureStarted(
1689             session: CameraExtensionSession,
1690             request: CaptureRequest,
1691             timestamp: Long
1692         ) {
1693             captureCallbacks.forEach { it.onCaptureStarted(session, request, timestamp) }
1694         }
1695 
1696         override fun onCaptureProcessStarted(
1697             session: CameraExtensionSession,
1698             request: CaptureRequest
1699         ) {
1700             captureCallbacks.forEach { it.onCaptureProcessStarted(session, request) }
1701         }
1702 
1703         @RequiresApi(Build.VERSION_CODES.TIRAMISU)
1704         override fun onCaptureResultAvailable(
1705             session: CameraExtensionSession,
1706             request: CaptureRequest,
1707             result: TotalCaptureResult
1708         ) {
1709             captureCallbacks.forEach { it.onCaptureResultAvailable(session, request, result) }
1710         }
1711 
1712         override fun onCaptureFailed(session: CameraExtensionSession, request: CaptureRequest) {
1713             captureCallbacks.forEach { it.onCaptureFailed(session, request) }
1714         }
1715     }
1716 
1717     /**
1718      * A combo CaptureCallback implementation to receive to pass the events to the underlying
1719      * callbacks.
1720      */
1721     private class ComboCaptureCallbackNormalMode : CaptureCallback() {
1722         private val captureCallbacks: MutableList<CaptureCallback> = mutableListOf()
1723 
1724         fun addCaptureCallback(captureCallback: CaptureCallback) {
1725             if (!captureCallbacks.contains(captureCallback)) {
1726                 captureCallbacks.add(captureCallback)
1727             }
1728         }
1729 
1730         fun removeCaptureCallback(captureCallback: CaptureCallback) {
1731             captureCallbacks.remove(captureCallback)
1732         }
1733 
1734         override fun onCaptureStarted(
1735             session: CameraCaptureSession,
1736             request: CaptureRequest,
1737             timestamp: Long,
1738             frameNumber: Long
1739         ) {
1740             captureCallbacks.forEach {
1741                 it.onCaptureStarted(session, request, timestamp, frameNumber)
1742             }
1743         }
1744 
1745         override fun onCaptureCompleted(
1746             session: CameraCaptureSession,
1747             request: CaptureRequest,
1748             result: TotalCaptureResult
1749         ) {
1750             captureCallbacks.forEach { it.onCaptureCompleted(session, request, result) }
1751         }
1752     }
1753 }
1754 
formatnull1755 fun Double.format(scale: Int): String = String.format("%.${scale}f", this)
1756 
1757 /** Convert a job into a ListenableFuture<T>. */
1758 @OptIn(ExperimentalCoroutinesApi::class)
1759 private fun <T> Deferred<T>.asListenableFuture(
1760     tag: Any? = "Deferred.asListenableFuture"
1761 ): ListenableFuture<T> {
1762     val resolver: CallbackToFutureAdapter.Resolver<T> =
1763         CallbackToFutureAdapter.Resolver<T> { completer ->
1764             this.invokeOnCompletion {
1765                 if (it != null) {
1766                     if (it is CancellationException) {
1767                         completer.setCancelled()
1768                     } else {
1769                         completer.setException(it)
1770                     }
1771                 } else {
1772                     // Ignore exceptions - This should never throw in this situation.
1773                     completer.set(this.getCompleted())
1774                 }
1775             }
1776             tag
1777         }
1778     return CallbackToFutureAdapter.getFuture(resolver)
1779 }
1780