1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.cts; 18 19 import static org.mockito.ArgumentMatchers.anyBoolean; 20 import static org.mockito.Matchers.any; 21 import static org.mockito.Matchers.anyInt; 22 import static org.mockito.Matchers.anyLong; 23 import static org.mockito.Matchers.eq; 24 import static org.mockito.Mockito.atLeastOnce; 25 import static org.mockito.Mockito.mock; 26 import static org.mockito.Mockito.reset; 27 import static org.mockito.Mockito.timeout; 28 import static org.mockito.Mockito.times; 29 import static org.mockito.Mockito.verify; 30 31 import com.android.compatibility.common.util.DeviceReportLog; 32 import com.android.compatibility.common.util.ResultType; 33 import com.android.compatibility.common.util.ResultUnit; 34 import com.android.compatibility.common.util.Stat; 35 import com.android.ex.camera2.blocking.BlockingSessionCallback; 36 import com.android.ex.camera2.blocking.BlockingExtensionSessionCallback; 37 import com.android.ex.camera2.blocking.BlockingStateCallback; 38 import com.android.ex.camera2.exceptions.TimeoutRuntimeException; 39 import com.android.ex.camera2.pos.AutoFocusStateMachine; 40 41 import android.graphics.ImageFormat; 42 import android.graphics.Rect; 43 import android.graphics.SurfaceTexture; 44 import android.hardware.HardwareBuffer; 45 import android.hardware.camera2.CameraCaptureSession; 46 import android.hardware.camera2.CameraDevice; 47 import android.hardware.camera2.CameraExtensionCharacteristics; 48 import android.hardware.camera2.CameraExtensionSession; 49 import android.hardware.camera2.CameraMetadata; 50 import android.hardware.camera2.CaptureRequest; 51 import android.hardware.camera2.CaptureResult; 52 import android.hardware.camera2.TotalCaptureResult; 53 import android.hardware.camera2.cts.helpers.CameraErrorCollector; 54 import android.hardware.camera2.cts.helpers.StaticMetadata; 55 import android.hardware.camera2.cts.testcases.Camera2AndroidTestRule; 56 import android.hardware.camera2.params.ExtensionSessionConfiguration; 57 import android.hardware.camera2.params.MeteringRectangle; 58 import android.hardware.camera2.params.OutputConfiguration; 59 import android.hardware.camera2.params.SessionConfiguration; 60 import android.media.ExifInterface; 61 import android.media.Image; 62 import android.media.ImageReader; 63 import android.os.SystemClock; 64 import android.util.Range; 65 import android.util.Size; 66 67 import static android.hardware.camera2.cts.CameraTestUtils.*; 68 import static android.hardware.cts.helpers.CameraUtils.*; 69 70 import android.util.Log; 71 import android.view.Surface; 72 import android.view.TextureView; 73 74 import androidx.test.platform.app.InstrumentationRegistry; 75 import androidx.test.rule.ActivityTestRule; 76 77 import org.junit.Rule; 78 import org.junit.Test; 79 import org.junit.runner.RunWith; 80 import org.junit.runners.Parameterized; 81 82 import java.io.IOException; 83 import java.util.ArrayList; 84 import java.util.Arrays; 85 import java.util.HashSet; 86 import java.util.List; 87 import java.util.Set; 88 import java.util.stream.Collectors; 89 90 @RunWith(Parameterized.class) 91 public class CameraExtensionSessionTest extends Camera2ParameterizedTestCase { 92 private static final String TAG = "CameraExtensionSessionTest"; 93 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 94 private static final long WAIT_FOR_COMMAND_TO_COMPLETE_MS = 5000; 95 private static final long REPEATING_REQUEST_TIMEOUT_MS = 5000; 96 private static final int MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS = 10000; 97 private static final float ZOOM_ERROR_MARGIN = 0.05f; 98 99 private SurfaceTexture mSurfaceTexture = null; 100 private Camera2AndroidTestRule mTestRule = null; 101 private CameraErrorCollector mCollector = null; 102 103 private DeviceReportLog mReportLog; 104 105 @Override setUp()106 public void setUp() throws Exception { 107 super.setUp(); 108 mTestRule = new Camera2AndroidTestRule(mContext); 109 mTestRule.before(); 110 mCollector = new CameraErrorCollector(); 111 } 112 113 @Override tearDown()114 public void tearDown() throws Exception { 115 if (mTestRule != null) { 116 mTestRule.after(); 117 } 118 if (mSurfaceTexture != null) { 119 mSurfaceTexture.release(); 120 mSurfaceTexture = null; 121 } 122 if (mCollector != null) { 123 try { 124 mCollector.verify(); 125 } catch (Throwable e) { 126 throw new Exception(e.getMessage()); 127 } 128 } 129 super.tearDown(); 130 } 131 132 @Rule 133 public ActivityTestRule<CameraExtensionTestActivity> mActivityRule = 134 new ActivityTestRule<>(CameraExtensionTestActivity.class); 135 updatePreviewSurfaceTexture()136 private void updatePreviewSurfaceTexture() { 137 if (mSurfaceTexture != null) { 138 return; 139 } 140 141 TextureView textureView = mActivityRule.getActivity().getTextureView(); 142 mSurfaceTexture = getAvailableSurfaceTexture(WAIT_FOR_COMMAND_TO_COMPLETE_MS, textureView); 143 assertNotNull("Failed to acquire valid preview surface texture!", mSurfaceTexture); 144 } 145 146 // Verify that camera extension sessions can be created and closed as expected. 147 @Test testBasicExtensionLifecycle()148 public void testBasicExtensionLifecycle() throws Exception { 149 for (String id : mCameraIdsUnderTest) { 150 StaticMetadata staticMeta = 151 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 152 if (!staticMeta.isColorOutputSupported()) { 153 continue; 154 } 155 updatePreviewSurfaceTexture(); 156 CameraExtensionCharacteristics extensionChars = 157 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 158 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 159 for (Integer extension : supportedExtensions) { 160 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 161 mSurfaceTexture.getClass()); 162 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 163 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight()); 164 OutputConfiguration outputConfig = new OutputConfiguration( 165 new Surface(mSurfaceTexture)); 166 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 167 outputConfigs.add(outputConfig); 168 169 BlockingExtensionSessionCallback sessionListener = 170 new BlockingExtensionSessionCallback( 171 mock(CameraExtensionSession.StateCallback.class)); 172 ExtensionSessionConfiguration configuration = 173 new ExtensionSessionConfiguration(extension, outputConfigs, 174 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 175 176 try { 177 mTestRule.openDevice(id); 178 CameraDevice camera = mTestRule.getCamera(); 179 camera.createExtensionSession(configuration); 180 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 181 SESSION_CONFIGURE_TIMEOUT_MS); 182 183 extensionSession.close(); 184 sessionListener.getStateWaiter().waitForState( 185 BlockingExtensionSessionCallback.SESSION_CLOSED, 186 SESSION_CLOSE_TIMEOUT_MS); 187 } finally { 188 mTestRule.closeDevice(id); 189 } 190 } 191 } 192 } 193 194 // Verify that regular camera sessions close as expected after creating a camera extension 195 // session. 196 @Test testCloseCaptureSession()197 public void testCloseCaptureSession() throws Exception { 198 for (String id : mCameraIdsUnderTest) { 199 StaticMetadata staticMeta = 200 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 201 if (!staticMeta.isColorOutputSupported()) { 202 continue; 203 } 204 updatePreviewSurfaceTexture(); 205 CameraExtensionCharacteristics extensionChars = 206 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 207 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 208 for (Integer extension : supportedExtensions) { 209 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 210 mSurfaceTexture.getClass()); 211 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 212 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight()); 213 Surface repeatingSurface = new Surface(mSurfaceTexture); 214 OutputConfiguration textureOutput = new OutputConfiguration(repeatingSurface); 215 List<OutputConfiguration> outputs = new ArrayList<>(); 216 outputs.add(textureOutput); 217 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback( 218 mock(CameraCaptureSession.StateCallback.class)); 219 SessionConfiguration regularConfiguration = new SessionConfiguration( 220 SessionConfiguration.SESSION_REGULAR, outputs, 221 new HandlerExecutor(mTestRule.getHandler()), regularSessionListener); 222 223 BlockingExtensionSessionCallback sessionListener = 224 new BlockingExtensionSessionCallback(mock( 225 CameraExtensionSession.StateCallback.class)); 226 ExtensionSessionConfiguration configuration = 227 new ExtensionSessionConfiguration(extension, outputs, 228 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 229 230 try { 231 mTestRule.openDevice(id); 232 mTestRule.getCamera().createCaptureSession(regularConfiguration); 233 234 CameraCaptureSession session = 235 regularSessionListener 236 .waitAndGetSession(SESSION_CONFIGURE_TIMEOUT_MS); 237 assertNotNull(session); 238 239 CameraDevice camera = mTestRule.getCamera(); 240 camera.createExtensionSession(configuration); 241 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 242 SESSION_CONFIGURE_TIMEOUT_MS); 243 assertNotNull(extensionSession); 244 245 regularSessionListener.getStateWaiter().waitForState( 246 BlockingExtensionSessionCallback.SESSION_CLOSED, 247 SESSION_CLOSE_TIMEOUT_MS); 248 249 extensionSession.close(); 250 sessionListener.getStateWaiter().waitForState( 251 BlockingExtensionSessionCallback.SESSION_CLOSED, 252 SESSION_CLOSE_TIMEOUT_MS); 253 } finally { 254 mTestRule.closeDevice(id); 255 } 256 } 257 } 258 } 259 260 // Verify that camera extension sessions close as expected when creating a regular capture 261 // session. 262 @Test testCloseExtensionSession()263 public void testCloseExtensionSession() throws Exception { 264 for (String id : mCameraIdsUnderTest) { 265 StaticMetadata staticMeta = 266 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 267 if (!staticMeta.isColorOutputSupported()) { 268 continue; 269 } 270 updatePreviewSurfaceTexture(); 271 CameraExtensionCharacteristics extensionChars = 272 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 273 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 274 for (Integer extension : supportedExtensions) { 275 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 276 mSurfaceTexture.getClass()); 277 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 278 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight()); 279 Surface surface = new Surface(mSurfaceTexture); 280 OutputConfiguration textureOutput = new OutputConfiguration(surface); 281 List<OutputConfiguration> outputs = new ArrayList<>(); 282 outputs.add(textureOutput); 283 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback( 284 mock(CameraCaptureSession.StateCallback.class)); 285 SessionConfiguration regularConfiguration = new SessionConfiguration( 286 SessionConfiguration.SESSION_REGULAR, outputs, 287 new HandlerExecutor(mTestRule.getHandler()), regularSessionListener); 288 289 BlockingExtensionSessionCallback sessionListener = 290 new BlockingExtensionSessionCallback(mock( 291 CameraExtensionSession.StateCallback.class)); 292 ExtensionSessionConfiguration configuration = 293 new ExtensionSessionConfiguration(extension, outputs, 294 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 295 296 try { 297 mTestRule.openDevice(id); 298 CameraDevice camera = mTestRule.getCamera(); 299 camera.createExtensionSession(configuration); 300 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 301 SESSION_CONFIGURE_TIMEOUT_MS); 302 assertNotNull(extensionSession); 303 304 mTestRule.getCamera().createCaptureSession(regularConfiguration); 305 sessionListener.getStateWaiter().waitForState( 306 BlockingExtensionSessionCallback.SESSION_CLOSED, 307 SESSION_CLOSE_TIMEOUT_MS); 308 309 CameraCaptureSession session = 310 regularSessionListener.waitAndGetSession( 311 SESSION_CONFIGURE_TIMEOUT_MS); 312 session.close(); 313 regularSessionListener.getStateWaiter().waitForState( 314 BlockingSessionCallback.SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 315 } finally { 316 mTestRule.closeDevice(id); 317 } 318 } 319 } 320 } 321 322 // Verify camera device query 323 @Test testGetDevice()324 public void testGetDevice() throws Exception { 325 for (String id : mCameraIdsUnderTest) { 326 StaticMetadata staticMeta = 327 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 328 if (!staticMeta.isColorOutputSupported()) { 329 continue; 330 } 331 updatePreviewSurfaceTexture(); 332 CameraExtensionCharacteristics extensionChars = 333 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 334 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 335 for (Integer extension : supportedExtensions) { 336 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 337 mSurfaceTexture.getClass()); 338 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 339 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight()); 340 OutputConfiguration privateOutput = new OutputConfiguration( 341 new Surface(mSurfaceTexture)); 342 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 343 outputConfigs.add(privateOutput); 344 345 BlockingExtensionSessionCallback sessionListener = 346 new BlockingExtensionSessionCallback( 347 mock(CameraExtensionSession.StateCallback.class)); 348 ExtensionSessionConfiguration configuration = 349 new ExtensionSessionConfiguration(extension, outputConfigs, 350 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 351 352 try { 353 mTestRule.openDevice(id); 354 CameraDevice camera = mTestRule.getCamera(); 355 camera.createExtensionSession(configuration); 356 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 357 SESSION_CONFIGURE_TIMEOUT_MS); 358 359 assertEquals("Unexpected/Invalid camera device", mTestRule.getCamera(), 360 extensionSession.getDevice()); 361 } finally { 362 mTestRule.closeDevice(id); 363 } 364 365 try { 366 sessionListener.getStateWaiter().waitForState( 367 BlockingExtensionSessionCallback.SESSION_CLOSED, 368 SESSION_CLOSE_TIMEOUT_MS); 369 fail("should get TimeoutRuntimeException due to previously closed camera " 370 + "device"); 371 } catch (TimeoutRuntimeException e) { 372 // Expected, per API spec we should not receive any further session callbacks 373 // besides the device state 'onClosed' callback. 374 } 375 } 376 } 377 } 378 379 // Test case for repeating/stopRepeating on all supported extensions and expected state/capture 380 // callbacks. 381 @Test testRepeatingCapture()382 public void testRepeatingCapture() throws Exception { 383 for (String id : mCameraIdsUnderTest) { 384 StaticMetadata staticMeta = 385 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 386 if (!staticMeta.isColorOutputSupported()) { 387 continue; 388 } 389 updatePreviewSurfaceTexture(); 390 CameraExtensionCharacteristics extensionChars = 391 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 392 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 393 for (Integer extension : supportedExtensions) { 394 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 395 mSurfaceTexture.getClass()); 396 Size maxSize = 397 CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 398 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), 399 maxSize.getHeight()); 400 Surface texturedSurface = new Surface(mSurfaceTexture); 401 402 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 403 outputConfigs.add(new OutputConfiguration(texturedSurface)); 404 405 BlockingExtensionSessionCallback sessionListener = 406 new BlockingExtensionSessionCallback(mock( 407 CameraExtensionSession.StateCallback.class)); 408 ExtensionSessionConfiguration configuration = 409 new ExtensionSessionConfiguration(extension, outputConfigs, 410 new HandlerExecutor(mTestRule.getHandler()), 411 sessionListener); 412 413 boolean captureResultsSupported = 414 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty(); 415 416 try { 417 mTestRule.openDevice(id); 418 CameraDevice camera = mTestRule.getCamera(); 419 camera.createExtensionSession(configuration); 420 CameraExtensionSession extensionSession = 421 sessionListener.waitAndGetSession( 422 SESSION_CONFIGURE_TIMEOUT_MS); 423 assertNotNull(extensionSession); 424 425 CaptureRequest.Builder captureBuilder = 426 mTestRule.getCamera().createCaptureRequest( 427 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 428 captureBuilder.addTarget(texturedSurface); 429 CameraExtensionSession.ExtensionCaptureCallback captureCallbackMock = 430 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 431 SimpleCaptureCallback simpleCaptureCallback = 432 new SimpleCaptureCallback(extension, captureCallbackMock, 433 extensionChars.getAvailableCaptureResultKeys(extension), 434 mCollector); 435 CaptureRequest request = captureBuilder.build(); 436 int sequenceId = extensionSession.setRepeatingRequest(request, 437 new HandlerExecutor(mTestRule.getHandler()), simpleCaptureCallback); 438 439 verify(captureCallbackMock, 440 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 441 .onCaptureStarted(eq(extensionSession), eq(request), anyLong()); 442 verify(captureCallbackMock, 443 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 444 .onCaptureProcessStarted(extensionSession, request); 445 if (captureResultsSupported) { 446 verify(captureCallbackMock, 447 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce()) 448 .onCaptureResultAvailable(eq(extensionSession), eq(request), 449 any(TotalCaptureResult.class)); 450 } 451 452 extensionSession.stopRepeating(); 453 454 verify(captureCallbackMock, 455 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 456 .onCaptureSequenceCompleted(extensionSession, sequenceId); 457 458 verify(captureCallbackMock, times(0)) 459 .onCaptureSequenceAborted(any(CameraExtensionSession.class), 460 anyInt()); 461 462 extensionSession.close(); 463 464 sessionListener.getStateWaiter().waitForState( 465 BlockingExtensionSessionCallback.SESSION_CLOSED, 466 SESSION_CLOSE_TIMEOUT_MS); 467 468 assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" + 469 " callbacks must be greater or equal than the number of calls" + 470 " to onCaptureStarted!", 471 simpleCaptureCallback.getTotalFramesArrived() + 472 simpleCaptureCallback.getTotalFramesFailed() >= 473 simpleCaptureCallback.getTotalFramesStarted()); 474 assertTrue(String.format("The last repeating request surface timestamp " + 475 "%d must be less than or equal to the last " + 476 "onCaptureStarted " + 477 "timestamp %d", mSurfaceTexture.getTimestamp(), 478 simpleCaptureCallback.getLastTimestamp()), 479 mSurfaceTexture.getTimestamp() <= 480 simpleCaptureCallback.getLastTimestamp()); 481 } finally { 482 mTestRule.closeDevice(id); 483 texturedSurface.release(); 484 } 485 } 486 } 487 } 488 489 // Test for postview of still capture on all supported extensions 490 @Test testPostviewAndCapture()491 public void testPostviewAndCapture() throws Exception { 492 final int IMAGE_COUNT = 10; 493 final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = { 494 ImageFormat.YUV_420_888, 495 ImageFormat.JPEG 496 }; 497 for (String id : mCameraIdsUnderTest) { 498 StaticMetadata staticMeta = 499 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 500 if (!staticMeta.isColorOutputSupported()) { 501 continue; 502 } 503 updatePreviewSurfaceTexture(); 504 CameraExtensionCharacteristics extensionChars = 505 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 506 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 507 for (Integer extension : supportedExtensions) { 508 if (!extensionChars.isPostviewAvailable(extension)) { 509 continue; 510 } 511 512 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 513 boolean captureProgressSupported = 514 extensionChars.isCaptureProcessProgressAvailable(extension); 515 516 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 517 captureFormat); 518 if (extensionSizes.isEmpty()) { 519 continue; 520 } 521 522 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 523 List<Size> postviewSizes = extensionChars.getPostviewSupportedSizes(extension, 524 maxSize, captureFormat); 525 if (postviewSizes.isEmpty()) { 526 continue; 527 } 528 529 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false, 530 1); 531 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize, 532 captureFormat, /*maxImages*/ 1, imageListener, 533 mTestRule.getHandler()); 534 Surface imageReaderSurface = extensionImageReader.getSurface(); 535 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 536 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 537 outputConfigs.add(readerOutput); 538 539 Size postviewSize = 540 CameraTestUtils.getMaxSize(postviewSizes.toArray(new Size[0])); 541 SimpleImageReaderListener imageListenerPostview = 542 new SimpleImageReaderListener(false, 1); 543 ImageReader postviewImageReader = CameraTestUtils.makeImageReader(postviewSize, 544 captureFormat, /*maxImages*/ 1, imageListenerPostview, 545 mTestRule.getHandler()); 546 Surface postviewImageReaderSurface = postviewImageReader.getSurface(); 547 OutputConfiguration postviewReaderOutput = 548 new OutputConfiguration(postviewImageReaderSurface); 549 550 BlockingExtensionSessionCallback sessionListener = 551 new BlockingExtensionSessionCallback(mock( 552 CameraExtensionSession.StateCallback.class)); 553 ExtensionSessionConfiguration configuration = 554 new ExtensionSessionConfiguration(extension, outputConfigs, 555 new HandlerExecutor(mTestRule.getHandler()), 556 sessionListener); 557 configuration.setPostviewOutputConfiguration(postviewReaderOutput); 558 assertNotNull(configuration.getPostviewOutputConfiguration()); 559 560 String streamName = "test_extension_postview_capture"; 561 mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName); 562 mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE); 563 mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL, 564 ResultUnit.NONE); 565 double[] captureTimes = new double[IMAGE_COUNT]; 566 double[] postviewCaptureTimes = new double[IMAGE_COUNT]; 567 boolean captureResultsSupported = 568 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty(); 569 570 try { 571 mTestRule.openDevice(id); 572 CameraDevice camera = mTestRule.getCamera(); 573 camera.createExtensionSession(configuration); 574 CameraExtensionSession extensionSession = 575 sessionListener.waitAndGetSession( 576 SESSION_CONFIGURE_TIMEOUT_MS); 577 assertNotNull(extensionSession); 578 579 CaptureRequest.Builder captureBuilder = 580 mTestRule.getCamera().createCaptureRequest( 581 CameraDevice.TEMPLATE_STILL_CAPTURE); 582 captureBuilder.addTarget(imageReaderSurface); 583 captureBuilder.addTarget(postviewImageReaderSurface); 584 CameraExtensionSession.ExtensionCaptureCallback captureMockCallback = 585 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 586 SimpleCaptureCallback captureCallback = 587 new SimpleCaptureCallback(extension, captureMockCallback, 588 extensionChars.getAvailableCaptureResultKeys(extension), 589 mCollector); 590 591 for (int i = 0; i < IMAGE_COUNT; i++) { 592 int jpegOrientation = (i * 90) % 360; // degrees [0..270] 593 if (captureFormat == ImageFormat.JPEG) { 594 captureBuilder.set(CaptureRequest.JPEG_ORIENTATION, 595 jpegOrientation); 596 } 597 CaptureRequest request = captureBuilder.build(); 598 long startTimeMs = SystemClock.elapsedRealtime(); 599 captureCallback.resetCaptureProgress(); 600 int sequenceId = extensionSession.capture(request, 601 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 602 603 Image imgPostview = 604 imageListenerPostview 605 .getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS); 606 postviewCaptureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs; 607 if (captureFormat == ImageFormat.JPEG) { 608 verifyJpegOrientation(imgPostview, postviewSize, jpegOrientation); 609 } else { 610 validateImage(imgPostview, postviewSize.getWidth(), 611 postviewSize.getHeight(), captureFormat, null); 612 } 613 Long imgTsPostview = imgPostview.getTimestamp(); 614 imgPostview.close(); 615 616 Image img = 617 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS); 618 captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs; 619 if (captureFormat == ImageFormat.JPEG) { 620 verifyJpegOrientation(img, maxSize, jpegOrientation); 621 } else { 622 validateImage(img, maxSize.getWidth(), maxSize.getHeight(), 623 captureFormat, null); 624 } 625 Long imgTs = img.getTimestamp(); 626 img.close(); 627 628 assertEquals("Still capture timestamp does not match its " 629 + "postview timestamp", imgTsPostview, imgTs); 630 631 verify(captureMockCallback, 632 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 633 .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs)); 634 verify(captureMockCallback, times(1)) 635 .onCaptureStarted(eq(extensionSession), eq(request), anyLong()); 636 verify(captureMockCallback, 637 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 638 .onCaptureProcessStarted(extensionSession, request); 639 verify(captureMockCallback, 640 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 641 .onCaptureSequenceCompleted(extensionSession, sequenceId); 642 if (captureResultsSupported) { 643 verify(captureMockCallback, 644 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 645 .onCaptureResultAvailable(eq(extensionSession), eq(request), 646 any(TotalCaptureResult.class)); 647 } 648 if (captureProgressSupported) { 649 verify(captureMockCallback, 650 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 651 .onCaptureProcessProgressed(eq(extensionSession), 652 eq(request), eq(100)); 653 } 654 } 655 656 mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL, 657 ResultUnit.NONE); 658 mReportLog.addValue("height", maxSize.getHeight(), 659 ResultType.NEUTRAL, ResultUnit.NONE); 660 mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL, 661 ResultUnit.NONE); 662 long avgPostviewLatency = (long) Stat.getAverage(postviewCaptureTimes); 663 mReportLog.addValue("avg_postview_latency", avgPostviewLatency, 664 ResultType.LOWER_BETTER, ResultUnit.MS); 665 long avgCaptureLatency = (long) Stat.getAverage(captureTimes); 666 mReportLog.addValue("avg_capture_latency", avgCaptureLatency, 667 ResultType.LOWER_BETTER, ResultUnit.MS); 668 669 verify(captureMockCallback, times(0)) 670 .onCaptureSequenceAborted(any(CameraExtensionSession.class), 671 anyInt()); 672 verify(captureMockCallback, times(0)) 673 .onCaptureFailed(any(CameraExtensionSession.class), 674 any(CaptureRequest.class)); 675 Range<Long> latencyRange = 676 extensionChars.getEstimatedCaptureLatencyRangeMillis(extension, 677 maxSize, captureFormat); 678 if (latencyRange != null) { 679 String msg = String.format("Camera [%s]: The measured average " 680 + "capture latency of %d ms. for extension type %d " 681 + "with image format: %d and size: %dx%d must be " 682 + "within the advertised range of [%d, %d] ms.", 683 id, avgCaptureLatency, extension, captureFormat, 684 maxSize.getWidth(), maxSize.getHeight(), 685 latencyRange.getLower(), latencyRange.getUpper()); 686 assertTrue(msg, latencyRange.contains(avgCaptureLatency)); 687 } 688 689 extensionSession.close(); 690 691 sessionListener.getStateWaiter().waitForState( 692 BlockingExtensionSessionCallback.SESSION_CLOSED, 693 SESSION_CLOSE_TIMEOUT_MS); 694 } finally { 695 mTestRule.closeDevice(id); 696 extensionImageReader.close(); 697 mReportLog.submit(InstrumentationRegistry.getInstrumentation()); 698 } 699 } 700 } 701 } 702 } 703 704 // Test case for multi-frame only capture on all supported extensions and expected state 705 // callbacks. Verify still frame output, measure the average capture latency and if possible 706 // ensure that the value is within the reported range. 707 @Test testMultiFrameCapture()708 public void testMultiFrameCapture() throws Exception { 709 final int IMAGE_COUNT = 10; 710 final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = { 711 ImageFormat.YUV_420_888, 712 ImageFormat.JPEG 713 }; 714 for (String id : mCameraIdsUnderTest) { 715 StaticMetadata staticMeta = 716 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 717 if (!staticMeta.isColorOutputSupported()) { 718 continue; 719 } 720 updatePreviewSurfaceTexture(); 721 CameraExtensionCharacteristics extensionChars = 722 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 723 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 724 for (Integer extension : supportedExtensions) { 725 boolean captureProgressSupported = extensionChars.isCaptureProcessProgressAvailable( 726 extension); 727 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 728 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 729 captureFormat); 730 if (extensionSizes.isEmpty()) { 731 continue; 732 } 733 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 734 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false, 735 1); 736 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize, 737 captureFormat, /*maxImages*/ 1, imageListener, 738 mTestRule.getHandler()); 739 Surface imageReaderSurface = extensionImageReader.getSurface(); 740 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 741 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 742 outputConfigs.add(readerOutput); 743 744 BlockingExtensionSessionCallback sessionListener = 745 new BlockingExtensionSessionCallback(mock( 746 CameraExtensionSession.StateCallback.class)); 747 ExtensionSessionConfiguration configuration = 748 new ExtensionSessionConfiguration(extension, outputConfigs, 749 new HandlerExecutor(mTestRule.getHandler()), 750 sessionListener); 751 String streamName = "test_extension_capture"; 752 mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName); 753 mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE); 754 mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL, 755 ResultUnit.NONE); 756 double[] captureTimes = new double[IMAGE_COUNT]; 757 boolean captureResultsSupported = 758 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty(); 759 760 try { 761 mTestRule.openDevice(id); 762 CameraDevice camera = mTestRule.getCamera(); 763 camera.createExtensionSession(configuration); 764 CameraExtensionSession extensionSession = 765 sessionListener.waitAndGetSession( 766 SESSION_CONFIGURE_TIMEOUT_MS); 767 assertNotNull(extensionSession); 768 769 CaptureRequest.Builder captureBuilder = 770 mTestRule.getCamera().createCaptureRequest( 771 CameraDevice.TEMPLATE_STILL_CAPTURE); 772 captureBuilder.addTarget(imageReaderSurface); 773 CameraExtensionSession.ExtensionCaptureCallback captureMockCallback = 774 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 775 SimpleCaptureCallback captureCallback = 776 new SimpleCaptureCallback(extension, captureMockCallback, 777 extensionChars.getAvailableCaptureResultKeys(extension), 778 mCollector); 779 780 for (int i = 0; i < IMAGE_COUNT; i++) { 781 int jpegOrientation = (i * 90) % 360; // degrees [0..270] 782 if (captureFormat == ImageFormat.JPEG) { 783 captureBuilder.set(CaptureRequest.JPEG_ORIENTATION, 784 jpegOrientation); 785 } 786 CaptureRequest request = captureBuilder.build(); 787 long startTimeMs = SystemClock.elapsedRealtime(); 788 captureCallback.resetCaptureProgress(); 789 int sequenceId = extensionSession.capture(request, 790 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 791 792 Image img = 793 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS); 794 captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs; 795 if (captureFormat == ImageFormat.JPEG) { 796 verifyJpegOrientation(img, maxSize, jpegOrientation); 797 } else { 798 validateImage(img, maxSize.getWidth(), maxSize.getHeight(), 799 captureFormat, null); 800 } 801 Long imgTs = img.getTimestamp(); 802 img.close(); 803 804 verify(captureMockCallback, 805 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 806 .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs)); 807 verify(captureMockCallback, times(1)) 808 .onCaptureStarted(eq(extensionSession), eq(request), anyLong()); 809 verify(captureMockCallback, 810 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 811 .onCaptureProcessStarted(extensionSession, request); 812 verify(captureMockCallback, 813 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 814 .onCaptureSequenceCompleted(extensionSession, sequenceId); 815 if (captureResultsSupported) { 816 verify(captureMockCallback, 817 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 818 .onCaptureResultAvailable(eq(extensionSession), eq(request), 819 any(TotalCaptureResult.class)); 820 } 821 if (captureProgressSupported) { 822 verify(captureMockCallback, 823 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 824 .onCaptureProcessProgressed(eq(extensionSession), 825 eq(request), eq(100)); 826 } 827 } 828 829 mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL, 830 ResultUnit.NONE); 831 mReportLog.addValue("height", maxSize.getHeight(), 832 ResultType.NEUTRAL, ResultUnit.NONE); 833 mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL, 834 ResultUnit.NONE); 835 long avgCaptureLatency = (long) Stat.getAverage(captureTimes); 836 mReportLog.addValue("avg_latency", avgCaptureLatency, 837 ResultType.LOWER_BETTER, ResultUnit.MS); 838 839 verify(captureMockCallback, times(0)) 840 .onCaptureSequenceAborted(any(CameraExtensionSession.class), 841 anyInt()); 842 verify(captureMockCallback, times(0)) 843 .onCaptureFailed(any(CameraExtensionSession.class), 844 any(CaptureRequest.class)); 845 Range<Long> latencyRange = 846 extensionChars.getEstimatedCaptureLatencyRangeMillis(extension, 847 maxSize, captureFormat); 848 if (latencyRange != null) { 849 String msg = String.format("Camera [%s]: The measured average " 850 + "capture latency of %d ms. for extension type %d " 851 + "with image format: %d and size: %dx%d must be " 852 + "within the advertised range of [%d, %d] ms.", 853 id, avgCaptureLatency, extension, captureFormat, 854 maxSize.getWidth(), maxSize.getHeight(), 855 latencyRange.getLower(), latencyRange.getUpper()); 856 assertTrue(msg, latencyRange.contains(avgCaptureLatency)); 857 } 858 859 extensionSession.close(); 860 861 sessionListener.getStateWaiter().waitForState( 862 BlockingExtensionSessionCallback.SESSION_CLOSED, 863 SESSION_CLOSE_TIMEOUT_MS); 864 } finally { 865 mTestRule.closeDevice(id); 866 extensionImageReader.close(); 867 mReportLog.submit(InstrumentationRegistry.getInstrumentation()); 868 } 869 } 870 } 871 } 872 } 873 874 // Verify concurrent extension sessions behavior 875 @Test testConcurrentSessions()876 public void testConcurrentSessions() throws Exception { 877 Set<Set<String>> concurrentCameraIdSet = 878 mTestRule.getCameraManager().getConcurrentCameraIds(); 879 if (concurrentCameraIdSet.isEmpty()) { 880 return; 881 } 882 883 for (String id : mCameraIdsUnderTest) { 884 StaticMetadata staticMeta = 885 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 886 if (!staticMeta.isColorOutputSupported()) { 887 continue; 888 } 889 CameraExtensionCharacteristics extensionChars = 890 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 891 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 892 if (supportedExtensions.isEmpty()) { 893 continue; 894 } 895 896 Set<String> concurrentCameraIds = null; 897 for (Set<String> entry : concurrentCameraIdSet) { 898 if (entry.contains(id)) { 899 concurrentCameraIds = entry; 900 break; 901 } 902 } 903 if (concurrentCameraIds == null) { 904 continue; 905 } 906 907 String concurrentCameraId = null; 908 CameraExtensionCharacteristics concurrentExtensionChars = null; 909 for (String entry : concurrentCameraIds) { 910 if (entry.equals(id)) { 911 continue; 912 } 913 if (!(new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics( 914 entry))).isColorOutputSupported()) { 915 continue; 916 } 917 CameraExtensionCharacteristics chars = 918 mTestRule.getCameraManager().getCameraExtensionCharacteristics(entry); 919 if (chars.getSupportedExtensions().isEmpty()) { 920 continue; 921 } 922 concurrentCameraId = entry; 923 concurrentExtensionChars = chars; 924 break; 925 } 926 if ((concurrentCameraId == null) || (concurrentExtensionChars == null)) { 927 continue; 928 } 929 930 updatePreviewSurfaceTexture(); 931 int extensionId = supportedExtensions.get(0); 932 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extensionId, 933 mSurfaceTexture.getClass()); 934 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 935 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight()); 936 OutputConfiguration outputConfig = new OutputConfiguration( 937 new Surface(mSurfaceTexture)); 938 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 939 outputConfigs.add(outputConfig); 940 941 BlockingExtensionSessionCallback sessionListener = 942 new BlockingExtensionSessionCallback( 943 mock(CameraExtensionSession.StateCallback.class)); 944 ExtensionSessionConfiguration configuration = 945 new ExtensionSessionConfiguration(extensionId, outputConfigs, 946 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 947 948 CameraDevice concurrentCameraDevice = null; 949 ImageReader extensionImageReader = null; 950 try { 951 mTestRule.openDevice(id); 952 concurrentCameraDevice = CameraTestUtils.openCamera(mTestRule.getCameraManager(), 953 concurrentCameraId, new BlockingStateCallback(), mTestRule.getHandler()); 954 CameraDevice camera = mTestRule.getCamera(); 955 camera.createExtensionSession(configuration); 956 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 957 SESSION_CONFIGURE_TIMEOUT_MS); 958 assertNotNull(extensionSession); 959 960 assertNotNull(concurrentCameraDevice); 961 int concurrentExtensionId = 962 concurrentExtensionChars.getSupportedExtensions().get(0); 963 List<Size> captureSizes = concurrentExtensionChars.getExtensionSupportedSizes( 964 concurrentExtensionId, mSurfaceTexture.getClass()); 965 assertFalse("No SurfaceTexture output supported", captureSizes.isEmpty()); 966 Size captureMaxSize = 967 CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0])); 968 969 extensionImageReader = ImageReader.newInstance( 970 captureMaxSize.getWidth(), captureMaxSize.getHeight(), ImageFormat.PRIVATE, 971 /*maxImages*/ 1, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); 972 Surface imageReaderSurface = extensionImageReader.getSurface(); 973 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 974 outputConfigs = new ArrayList<>(); 975 outputConfigs.add(readerOutput); 976 CameraExtensionSession.StateCallback mockSessionListener = 977 mock(CameraExtensionSession.StateCallback.class); 978 ExtensionSessionConfiguration concurrentConfiguration = 979 new ExtensionSessionConfiguration(concurrentExtensionId, outputConfigs, 980 new HandlerExecutor(mTestRule.getHandler()), 981 mockSessionListener); 982 concurrentCameraDevice.createExtensionSession(concurrentConfiguration); 983 // Trying to initialize multiple concurrent extension sessions is expected to fail 984 verify(mockSessionListener, timeout(SESSION_CONFIGURE_TIMEOUT_MS).times(1)) 985 .onConfigureFailed(any(CameraExtensionSession.class)); 986 verify(mockSessionListener, times(0)).onConfigured( 987 any(CameraExtensionSession.class)); 988 989 extensionSession.close(); 990 sessionListener.getStateWaiter().waitForState( 991 BlockingExtensionSessionCallback.SESSION_CLOSED, 992 SESSION_CLOSE_TIMEOUT_MS); 993 994 // Initialization of another extension session must now be possible 995 BlockingExtensionSessionCallback concurrentSessionListener = 996 new BlockingExtensionSessionCallback( 997 mock(CameraExtensionSession.StateCallback.class)); 998 concurrentConfiguration = new ExtensionSessionConfiguration(concurrentExtensionId, 999 outputConfigs, new HandlerExecutor(mTestRule.getHandler()), 1000 concurrentSessionListener); 1001 concurrentCameraDevice.createExtensionSession(concurrentConfiguration); 1002 extensionSession = concurrentSessionListener.waitAndGetSession( 1003 SESSION_CONFIGURE_TIMEOUT_MS); 1004 assertNotNull(extensionSession); 1005 extensionSession.close(); 1006 concurrentSessionListener.getStateWaiter().waitForState( 1007 BlockingExtensionSessionCallback.SESSION_CLOSED, 1008 SESSION_CLOSE_TIMEOUT_MS); 1009 } finally { 1010 mTestRule.closeDevice(id); 1011 if (concurrentCameraDevice != null) { 1012 concurrentCameraDevice.close(); 1013 } 1014 if (extensionImageReader != null) { 1015 extensionImageReader.close(); 1016 } 1017 } 1018 } 1019 } 1020 1021 // Test case combined repeating with multi frame capture on all supported extensions. 1022 // Verify still frame output. 1023 @Test testRepeatingAndCaptureCombined()1024 public void testRepeatingAndCaptureCombined() throws Exception { 1025 final double LATENCY_MARGIN = .1f; // Account for system load, capture call duration etc. 1026 for (String id : mCameraIdsUnderTest) { 1027 StaticMetadata staticMeta = 1028 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 1029 if (!staticMeta.isColorOutputSupported()) { 1030 continue; 1031 } 1032 updatePreviewSurfaceTexture(); 1033 CameraExtensionCharacteristics extensionChars = 1034 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 1035 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 1036 for (Integer extension : supportedExtensions) { 1037 1038 Set<CaptureRequest.Key> supportedRequestKeys = 1039 extensionChars.getAvailableCaptureRequestKeys(extension); 1040 boolean supportsStrengthControl = supportedRequestKeys.contains( 1041 CaptureRequest.EXTENSION_STRENGTH); 1042 1043 if (supportsStrengthControl) { 1044 Set<CaptureResult.Key> supportedResultKeys = 1045 extensionChars.getAvailableCaptureResultKeys(extension); 1046 assertTrue(supportedResultKeys.contains(CaptureResult.EXTENSION_STRENGTH)); 1047 } 1048 1049 int captureFormat = ImageFormat.JPEG; 1050 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension, 1051 captureFormat); 1052 assertFalse("No Jpeg output supported", captureSizes.isEmpty()); 1053 Size captureMaxSize = 1054 CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0])); 1055 1056 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false 1057 , 1); 1058 ImageReader extensionImageReader = CameraTestUtils.makeImageReader( 1059 captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener, 1060 mTestRule.getHandler()); 1061 Surface imageReaderSurface = extensionImageReader.getSurface(); 1062 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 1063 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 1064 outputConfigs.add(readerOutput); 1065 1066 // Pick a supported preview/repeating size with aspect ratio close to the 1067 // multi-frame capture size 1068 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension, 1069 mSurfaceTexture.getClass()); 1070 Size maxRepeatingSize = 1071 CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0])); 1072 List<Size> previewSizes = getSupportedPreviewSizes(id, 1073 mTestRule.getCameraManager(), 1074 getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND)); 1075 List<Size> supportedPreviewSizes = 1076 previewSizes.stream().filter(repeatingSizes::contains).collect( 1077 Collectors.toList()); 1078 if (!supportedPreviewSizes.isEmpty()) { 1079 float targetAr = 1080 ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight(); 1081 for (Size s : supportedPreviewSizes) { 1082 float currentAr = ((float) s.getWidth()) / s.getHeight(); 1083 if (Math.abs(targetAr - currentAr) < 0.01) { 1084 maxRepeatingSize = s; 1085 break; 1086 } 1087 } 1088 } 1089 1090 boolean captureResultsSupported = 1091 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty(); 1092 1093 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(), 1094 maxRepeatingSize.getHeight()); 1095 Surface texturedSurface = new Surface(mSurfaceTexture); 1096 outputConfigs.add(new OutputConfiguration(texturedSurface)); 1097 1098 BlockingExtensionSessionCallback sessionListener = 1099 new BlockingExtensionSessionCallback(mock( 1100 CameraExtensionSession.StateCallback.class)); 1101 ExtensionSessionConfiguration configuration = 1102 new ExtensionSessionConfiguration(extension, outputConfigs, 1103 new HandlerExecutor(mTestRule.getHandler()), 1104 sessionListener); 1105 try { 1106 mTestRule.openDevice(id); 1107 CameraDevice camera = mTestRule.getCamera(); 1108 camera.createExtensionSession(configuration); 1109 CameraExtensionSession extensionSession = 1110 sessionListener.waitAndGetSession( 1111 SESSION_CONFIGURE_TIMEOUT_MS); 1112 assertNotNull(extensionSession); 1113 1114 CaptureRequest.Builder captureBuilder = 1115 mTestRule.getCamera().createCaptureRequest( 1116 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 1117 captureBuilder.addTarget(texturedSurface); 1118 CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock = 1119 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 1120 SimpleCaptureCallback repeatingCaptureCallback = 1121 new SimpleCaptureCallback(extension, repeatingCallbackMock, 1122 extensionChars.getAvailableCaptureResultKeys(extension), 1123 mCollector); 1124 1125 if (supportsStrengthControl) { 1126 captureBuilder.set(CaptureRequest.EXTENSION_STRENGTH, 100); 1127 } 1128 1129 CaptureRequest repeatingRequest = captureBuilder.build(); 1130 int repeatingSequenceId = 1131 extensionSession.setRepeatingRequest(repeatingRequest, 1132 new HandlerExecutor(mTestRule.getHandler()), 1133 repeatingCaptureCallback); 1134 1135 Thread.sleep(REPEATING_REQUEST_TIMEOUT_MS); 1136 1137 verify(repeatingCallbackMock, atLeastOnce()) 1138 .onCaptureStarted(eq(extensionSession), eq(repeatingRequest), 1139 anyLong()); 1140 verify(repeatingCallbackMock, atLeastOnce()) 1141 .onCaptureProcessStarted(extensionSession, repeatingRequest); 1142 if (captureResultsSupported) { 1143 verify(repeatingCallbackMock, 1144 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce()) 1145 .onCaptureResultAvailable(eq(extensionSession), 1146 eq(repeatingRequest), any(TotalCaptureResult.class)); 1147 } 1148 verify(repeatingCallbackMock, times(0)).onCaptureProcessProgressed( 1149 any(CameraExtensionSession.class), any(CaptureRequest.class), anyInt()); 1150 1151 captureBuilder = mTestRule.getCamera().createCaptureRequest( 1152 android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE); 1153 captureBuilder.addTarget(imageReaderSurface); 1154 CameraExtensionSession.ExtensionCaptureCallback captureCallback = 1155 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 1156 1157 CameraExtensionSession.StillCaptureLatency stillCaptureLatency = 1158 extensionSession.getRealtimeStillCaptureLatency(); 1159 CaptureRequest captureRequest = captureBuilder.build(); 1160 long startTimeMs = SystemClock.elapsedRealtime(); 1161 int captureSequenceId = extensionSession.capture(captureRequest, 1162 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 1163 1164 Image img = 1165 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS); 1166 long captureTime = SystemClock.elapsedRealtime() - startTimeMs; 1167 validateImage(img, captureMaxSize.getWidth(), 1168 captureMaxSize.getHeight(), captureFormat, null); 1169 Long imgTs = img.getTimestamp(); 1170 img.close(); 1171 1172 if (stillCaptureLatency != null) { 1173 assertTrue("Still capture frame latency must be positive!", 1174 stillCaptureLatency.getCaptureLatency() > 0); 1175 assertTrue("Processing capture latency must be non-negative!", 1176 stillCaptureLatency.getProcessingLatency() >= 0); 1177 long estimatedTotalLatency = stillCaptureLatency.getCaptureLatency() + 1178 stillCaptureLatency.getProcessingLatency(); 1179 long estimatedTotalLatencyMin = 1180 (long) (estimatedTotalLatency * (1.f - LATENCY_MARGIN)); 1181 long estimatedTotalLatencyMax = 1182 (long) (estimatedTotalLatency * (1.f + LATENCY_MARGIN)); 1183 assertTrue(String.format("Camera %s: Measured still capture latency " + 1184 "doesn't match: %d ms, expected [%d,%d]ms.", id, 1185 captureTime, estimatedTotalLatencyMin, 1186 estimatedTotalLatencyMax), 1187 (captureTime <= estimatedTotalLatencyMax) && 1188 (captureTime >= estimatedTotalLatencyMin)); 1189 } 1190 1191 verify(captureCallback, times(1)) 1192 .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs)); 1193 verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 1194 .onCaptureProcessStarted(extensionSession, captureRequest); 1195 if (captureResultsSupported) { 1196 verify(captureCallback, 1197 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 1198 .onCaptureResultAvailable(eq(extensionSession), 1199 eq(captureRequest), any(TotalCaptureResult.class)); 1200 } 1201 verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 1202 .onCaptureSequenceCompleted(extensionSession, 1203 captureSequenceId); 1204 verify(captureCallback, times(0)) 1205 .onCaptureSequenceAborted(any(CameraExtensionSession.class), 1206 anyInt()); 1207 verify(captureCallback, times(0)) 1208 .onCaptureFailed(any(CameraExtensionSession.class), 1209 any(CaptureRequest.class)); 1210 1211 extensionSession.stopRepeating(); 1212 1213 verify(repeatingCallbackMock, 1214 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 1215 .onCaptureSequenceCompleted(extensionSession, repeatingSequenceId); 1216 1217 verify(repeatingCallbackMock, times(0)) 1218 .onCaptureSequenceAborted(any(CameraExtensionSession.class), 1219 anyInt()); 1220 1221 extensionSession.close(); 1222 1223 sessionListener.getStateWaiter().waitForState( 1224 BlockingExtensionSessionCallback.SESSION_CLOSED, 1225 SESSION_CLOSE_TIMEOUT_MS); 1226 1227 assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" + 1228 " callbacks must be greater or equal than the number of calls" + 1229 " to onCaptureStarted!", 1230 repeatingCaptureCallback.getTotalFramesArrived() + 1231 repeatingCaptureCallback.getTotalFramesFailed() >= 1232 repeatingCaptureCallback.getTotalFramesStarted()); 1233 assertTrue(String.format("The last repeating request surface timestamp " + 1234 "%d must be less than or equal to the last " + 1235 "onCaptureStarted " + 1236 "timestamp %d", mSurfaceTexture.getTimestamp(), 1237 repeatingCaptureCallback.getLastTimestamp()), 1238 mSurfaceTexture.getTimestamp() <= 1239 repeatingCaptureCallback.getLastTimestamp()); 1240 1241 } finally { 1242 mTestRule.closeDevice(id); 1243 texturedSurface.release(); 1244 extensionImageReader.close(); 1245 } 1246 } 1247 } 1248 } 1249 verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation)1250 private void verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation) 1251 throws IOException { 1252 byte[] blobBuffer = getDataFromImage(img); 1253 String blobFilename = mTestRule.getDebugFileNameBase() + "/verifyJpegKeys.jpeg"; 1254 dumpFile(blobFilename, blobBuffer); 1255 ExifInterface exif = new ExifInterface(blobFilename); 1256 int exifWidth = exif.getAttributeInt(ExifInterface.TAG_IMAGE_WIDTH, /*defaultValue*/0); 1257 int exifHeight = exif.getAttributeInt(ExifInterface.TAG_IMAGE_LENGTH, /*defaultValue*/0); 1258 Size exifSize = new Size(exifWidth, exifHeight); 1259 int exifOrientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, 1260 /*defaultValue*/ ExifInterface.ORIENTATION_UNDEFINED); 1261 final int ORIENTATION_MIN = ExifInterface.ORIENTATION_UNDEFINED; 1262 final int ORIENTATION_MAX = ExifInterface.ORIENTATION_ROTATE_270; 1263 assertTrue(String.format("Exif orientation must be in range of [%d, %d]", 1264 ORIENTATION_MIN, ORIENTATION_MAX), 1265 exifOrientation >= ORIENTATION_MIN && exifOrientation <= ORIENTATION_MAX); 1266 1267 /** 1268 * Device captured image doesn't respect the requested orientation, 1269 * which means it rotates the image buffer physically. Then we 1270 * should swap the exif width/height accordingly to compare. 1271 */ 1272 boolean deviceRotatedImage = exifOrientation == ExifInterface.ORIENTATION_UNDEFINED; 1273 1274 if (deviceRotatedImage) { 1275 // Case 1. 1276 boolean needSwap = (requestedOrientation % 180 == 90); 1277 if (needSwap) { 1278 exifSize = new Size(exifHeight, exifWidth); 1279 } 1280 } else { 1281 // Case 2. 1282 assertEquals("Exif orientation should match requested orientation", 1283 requestedOrientation, getExifOrientationInDegree(exifOrientation)); 1284 } 1285 1286 assertEquals("Exif size should match jpeg capture size", jpegSize, exifSize); 1287 } 1288 getExifOrientationInDegree(int exifOrientation)1289 private static int getExifOrientationInDegree(int exifOrientation) { 1290 switch (exifOrientation) { 1291 case ExifInterface.ORIENTATION_NORMAL: 1292 return 0; 1293 case ExifInterface.ORIENTATION_ROTATE_90: 1294 return 90; 1295 case ExifInterface.ORIENTATION_ROTATE_180: 1296 return 180; 1297 case ExifInterface.ORIENTATION_ROTATE_270: 1298 return 270; 1299 default: 1300 fail("It is impossible to get non 0, 90, 180, 270 degress exif" + 1301 "info based on the request orientation range"); 1302 return -1; 1303 } 1304 } 1305 1306 public static class SimpleCaptureCallback 1307 extends CameraExtensionSession.ExtensionCaptureCallback { 1308 private long mLastTimestamp = -1; 1309 private int mNumFramesArrived = 0; 1310 private int mNumFramesStarted = 0; 1311 private int mNumFramesFailed = 0; 1312 private int mLastProgressValue = -1; 1313 private boolean mNonIncreasingTimestamps = false; 1314 private HashSet<Long> mExpectedResultTimestamps = new HashSet<>(); 1315 private final CameraExtensionSession.ExtensionCaptureCallback mProxy; 1316 private final AutoFocusStateMachine mAFStateMachine; 1317 private final FlashStateListener mFlashStateListener; 1318 private final AutoExposureStateListener mAEStateListener; 1319 private final HashSet<CaptureResult.Key> mSupportedResultKeys; 1320 private final CameraErrorCollector mCollector; 1321 private final boolean mPerFrameControl; 1322 private final int mExtensionType; 1323 SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector)1324 public SimpleCaptureCallback(int extensionType, 1325 CameraExtensionSession.ExtensionCaptureCallback proxy, 1326 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector) { 1327 this(extensionType, proxy, supportedResultKeys, errorCollector, null /*afListener*/, 1328 null /*flashState*/, null /*aeState*/, false /*perFrameControl*/); 1329 } 1330 SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, AutoFocusStateMachine afState, FlashStateListener flashState, AutoExposureStateListener aeState, boolean perFrameControl)1331 public SimpleCaptureCallback(int extensionType, 1332 CameraExtensionSession.ExtensionCaptureCallback proxy, 1333 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, 1334 AutoFocusStateMachine afState, FlashStateListener flashState, 1335 AutoExposureStateListener aeState, boolean perFrameControl) { 1336 mProxy = proxy; 1337 mSupportedResultKeys = new HashSet<>(supportedResultKeys); 1338 mCollector = errorCollector; 1339 mAFStateMachine = afState; 1340 mFlashStateListener = flashState; 1341 mAEStateListener = aeState; 1342 mPerFrameControl = perFrameControl; 1343 mExtensionType = extensionType; 1344 } 1345 resetCaptureProgress()1346 public void resetCaptureProgress() { 1347 mLastProgressValue = -1; 1348 } 1349 1350 @Override onCaptureStarted(CameraExtensionSession session, CaptureRequest request, long timestamp)1351 public void onCaptureStarted(CameraExtensionSession session, 1352 CaptureRequest request, long timestamp) { 1353 mExpectedResultTimestamps.add(timestamp); 1354 if (timestamp < mLastTimestamp) { 1355 mNonIncreasingTimestamps = true; 1356 } 1357 mLastTimestamp = timestamp; 1358 mNumFramesStarted++; 1359 if (mProxy != null) { 1360 mProxy.onCaptureStarted(session, request, timestamp); 1361 } 1362 } 1363 1364 @Override onCaptureProcessStarted(CameraExtensionSession session, CaptureRequest request)1365 public void onCaptureProcessStarted(CameraExtensionSession session, 1366 CaptureRequest request) { 1367 mNumFramesArrived++; 1368 if (mProxy != null) { 1369 mProxy.onCaptureProcessStarted(session, request); 1370 } 1371 } 1372 1373 @Override onCaptureFailed(CameraExtensionSession session, CaptureRequest request)1374 public void onCaptureFailed(CameraExtensionSession session, 1375 CaptureRequest request) { 1376 mNumFramesFailed++; 1377 if (mProxy != null) { 1378 mProxy.onCaptureFailed(session, request); 1379 } 1380 } 1381 1382 @Override onCaptureSequenceAborted(CameraExtensionSession session, int sequenceId)1383 public void onCaptureSequenceAborted(CameraExtensionSession session, 1384 int sequenceId) { 1385 if (mProxy != null) { 1386 mProxy.onCaptureSequenceAborted(session, sequenceId); 1387 } 1388 } 1389 1390 @Override onCaptureSequenceCompleted(CameraExtensionSession session, int sequenceId)1391 public void onCaptureSequenceCompleted(CameraExtensionSession session, 1392 int sequenceId) { 1393 if (mProxy != null) { 1394 mProxy.onCaptureSequenceCompleted(session, sequenceId); 1395 } 1396 } 1397 1398 @Override onCaptureProcessProgressed(CameraExtensionSession session, CaptureRequest request, int progress)1399 public void onCaptureProcessProgressed(CameraExtensionSession session, 1400 CaptureRequest request, int progress) { 1401 if ((progress < 0) || (progress > 100)) { 1402 mCollector.addMessage("Capture progress invalid value: " + progress); 1403 return; 1404 } 1405 if (mLastProgressValue >= progress) { 1406 mCollector.addMessage("Unexpected progress value: " + progress + 1407 " last progress value: " + mLastProgressValue); 1408 return; 1409 } 1410 mLastProgressValue = progress; 1411 if (mProxy != null) { 1412 mProxy.onCaptureProcessProgressed(session, request, progress); 1413 } 1414 } 1415 1416 @Override onCaptureResultAvailable(CameraExtensionSession session, CaptureRequest request, TotalCaptureResult result)1417 public void onCaptureResultAvailable(CameraExtensionSession session, 1418 CaptureRequest request, TotalCaptureResult result) { 1419 final float METERING_REGION_ERROR_PERCENT_DELTA = 0.05f; 1420 if (mSupportedResultKeys.isEmpty()) { 1421 mCollector.addMessage("Capture results not supported, but " + 1422 "onCaptureResultAvailable still got triggered!"); 1423 return; 1424 } 1425 1426 List<CaptureResult.Key<?>> resultKeys = result.getKeys(); 1427 for (CaptureResult.Key<?> resultKey : resultKeys) { 1428 mCollector.expectTrue("Capture result " + resultKey + " is not among the" 1429 + " supported result keys!", mSupportedResultKeys.contains(resultKey)); 1430 } 1431 1432 Long timeStamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1433 assertNotNull(timeStamp); 1434 assertTrue("Capture result sensor timestamp: " + timeStamp + " must match " 1435 + " with the timestamp passed to onCaptureStarted!", 1436 mExpectedResultTimestamps.contains(timeStamp)); 1437 1438 Integer currentType = result.get(CaptureResult.EXTENSION_CURRENT_TYPE); 1439 if (currentType != null) { 1440 mCollector.expectNotEquals("The reported extension type cannot be set to AUTO!", 1441 CameraExtensionCharacteristics.EXTENSION_AUTOMATIC, currentType); 1442 if (mExtensionType == CameraExtensionCharacteristics.EXTENSION_AUTOMATIC) { 1443 Integer expectedValues[] = { 1444 CameraExtensionCharacteristics.EXTENSION_BOKEH, 1445 CameraExtensionCharacteristics.EXTENSION_HDR, 1446 CameraExtensionCharacteristics.EXTENSION_NIGHT, 1447 CameraExtensionCharacteristics.EXTENSION_FACE_RETOUCH}; 1448 mCollector.expectContains("Unexpected extension type result: " 1449 + currentType, expectedValues, currentType); 1450 } else { 1451 mCollector.expectEquals("Unexpected extension type result: " + currentType 1452 + " expected: " + mExtensionType, mExtensionType, currentType); 1453 } 1454 } 1455 1456 Integer strength = request.get(CaptureRequest.EXTENSION_STRENGTH); 1457 if (strength != null) { 1458 Integer resultStrength = result.get(CaptureResult.EXTENSION_STRENGTH); 1459 mCollector.expectTrue("Request extension strength: " + strength + 1460 " doesn't match with result: " + resultStrength, 1461 strength.equals(resultStrength)); 1462 } 1463 1464 Integer jpegOrientation = request.get(CaptureRequest.JPEG_ORIENTATION); 1465 if (jpegOrientation != null) { 1466 Integer resultJpegOrientation = result.get(CaptureResult.JPEG_ORIENTATION); 1467 mCollector.expectTrue("Request Jpeg orientation: " + jpegOrientation + 1468 " doesn't match with result: " + resultJpegOrientation, 1469 jpegOrientation.equals(resultJpegOrientation)); 1470 } 1471 1472 Byte jpegQuality = request.get(CaptureRequest.JPEG_QUALITY); 1473 if (jpegQuality != null) { 1474 Byte resultJpegQuality = result.get(CaptureResult.JPEG_QUALITY); 1475 mCollector.expectTrue("Request Jpeg quality: " + jpegQuality + 1476 " doesn't match with result: " + resultJpegQuality, 1477 jpegQuality.equals(resultJpegQuality)); 1478 } 1479 1480 if (resultKeys.contains(CaptureResult.CONTROL_ZOOM_RATIO) && mPerFrameControl) { 1481 Float zoomRatio = request.get(CaptureRequest.CONTROL_ZOOM_RATIO); 1482 if (zoomRatio != null) { 1483 Float resultZoomRatio = result.get(CaptureResult.CONTROL_ZOOM_RATIO); 1484 mCollector.expectTrue( 1485 String.format("Request and result zoom ratio should be similar " + 1486 "(requested = %f, result = %f", zoomRatio, resultZoomRatio), 1487 Math.abs(zoomRatio - resultZoomRatio) / zoomRatio <= ZOOM_ERROR_MARGIN); 1488 } 1489 } 1490 1491 if (mFlashStateListener != null) { 1492 Integer flashMode = request.get(CaptureRequest.FLASH_MODE); 1493 if ((flashMode != null) && mPerFrameControl) { 1494 Integer resultFlashMode = result.get(CaptureResult.FLASH_MODE); 1495 mCollector.expectTrue("Request flash mode: " + flashMode + 1496 " doesn't match with result: " + resultFlashMode, 1497 flashMode.equals(resultFlashMode)); 1498 } 1499 1500 Integer flashState = result.get(CaptureResult.FLASH_STATE); 1501 if (flashState != null) { 1502 switch (flashState) { 1503 case CaptureResult.FLASH_STATE_UNAVAILABLE: 1504 mFlashStateListener.onUnavailable(); 1505 break; 1506 case CaptureResult.FLASH_STATE_FIRED: 1507 mFlashStateListener.onFired(); 1508 break; 1509 case CaptureResult.FLASH_STATE_CHARGING: 1510 mFlashStateListener.onCharging(); 1511 break; 1512 case CaptureResult.FLASH_STATE_PARTIAL: 1513 mFlashStateListener.onPartial(); 1514 break; 1515 case CaptureResult.FLASH_STATE_READY: 1516 mFlashStateListener.onReady(); 1517 break; 1518 default: 1519 mCollector.addMessage("Unexpected flash state: " + flashState); 1520 } 1521 } 1522 } 1523 1524 if (mAEStateListener != null) { 1525 Integer aeMode = request.get(CaptureRequest.CONTROL_AE_MODE); 1526 if ((aeMode != null) && mPerFrameControl) { 1527 Integer resultAeMode = result.get(CaptureResult.CONTROL_AE_MODE); 1528 mCollector.expectTrue("Request AE mode: " + aeMode + 1529 " doesn't match with result: " + resultAeMode, 1530 aeMode.equals(resultAeMode)); 1531 } 1532 1533 Boolean aeLock = request.get(CaptureRequest.CONTROL_AE_LOCK); 1534 if ((aeLock != null) && mPerFrameControl) { 1535 Boolean resultAeLock = result.get(CaptureResult.CONTROL_AE_LOCK); 1536 mCollector.expectTrue("Request AE lock: " + aeLock + 1537 " doesn't match with result: " + resultAeLock, 1538 aeLock.equals(resultAeLock)); 1539 } 1540 1541 Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE); 1542 if (aeState != null) { 1543 switch (aeState) { 1544 case CaptureResult.CONTROL_AE_STATE_CONVERGED: 1545 mAEStateListener.onConverged(); 1546 break; 1547 case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED: 1548 mAEStateListener.onFlashRequired(); 1549 break; 1550 case CaptureResult.CONTROL_AE_STATE_INACTIVE: 1551 mAEStateListener.onInactive(); 1552 break; 1553 case CaptureResult.CONTROL_AE_STATE_LOCKED: 1554 mAEStateListener.onLocked(); 1555 break; 1556 case CaptureResult.CONTROL_AE_STATE_PRECAPTURE: 1557 mAEStateListener.onPrecapture(); 1558 break; 1559 case CaptureResult.CONTROL_AE_STATE_SEARCHING: 1560 mAEStateListener.onSearching(); 1561 break; 1562 default: 1563 mCollector.addMessage("Unexpected AE state: " + aeState); 1564 } 1565 } 1566 } 1567 1568 if (mAFStateMachine != null) { 1569 Integer afMode = request.get(CaptureRequest.CONTROL_AF_MODE); 1570 if ((afMode != null) && mPerFrameControl) { 1571 Integer resultAfMode = result.get(CaptureResult.CONTROL_AF_MODE); 1572 mCollector.expectTrue("Request AF mode: " + afMode + 1573 " doesn't match with result: " + resultAfMode, 1574 afMode.equals(resultAfMode)); 1575 } 1576 1577 MeteringRectangle[] afRegions = request.get(CaptureRequest.CONTROL_AF_REGIONS); 1578 if ((afRegions != null) && mPerFrameControl) { 1579 MeteringRectangle[] resultAfRegions = result.get( 1580 CaptureResult.CONTROL_AF_REGIONS); 1581 mCollector.expectMeteringRegionsAreSimilar( 1582 "AF regions in result and request should be similar", 1583 afRegions, resultAfRegions, METERING_REGION_ERROR_PERCENT_DELTA); 1584 } 1585 1586 Integer afTrigger = request.get(CaptureRequest.CONTROL_AF_TRIGGER); 1587 if ((afTrigger != null) && mPerFrameControl) { 1588 Integer resultAfTrigger = result.get(CaptureResult.CONTROL_AF_TRIGGER); 1589 mCollector.expectTrue("Request AF trigger: " + afTrigger + 1590 " doesn't match with result: " + resultAfTrigger, 1591 afTrigger.equals(resultAfTrigger)); 1592 } 1593 1594 mAFStateMachine.onCaptureCompleted(result); 1595 } 1596 1597 if (mProxy != null) { 1598 mProxy.onCaptureResultAvailable(session, request, result); 1599 } 1600 } 1601 getTotalFramesArrived()1602 public int getTotalFramesArrived() { 1603 return mNumFramesArrived; 1604 } 1605 getTotalFramesStarted()1606 public int getTotalFramesStarted() { 1607 return mNumFramesStarted; 1608 } 1609 getTotalFramesFailed()1610 public int getTotalFramesFailed() { 1611 return mNumFramesFailed; 1612 } 1613 getLastTimestamp()1614 public long getLastTimestamp() throws IllegalStateException { 1615 if (mNonIncreasingTimestamps) { 1616 throw new IllegalStateException("Non-monotonically increasing timestamps!"); 1617 } 1618 return mLastTimestamp; 1619 } 1620 } 1621 1622 public interface AutoFocusStateListener { onDone(boolean success)1623 void onDone(boolean success); onScan()1624 void onScan(); onInactive()1625 void onInactive(); 1626 } 1627 1628 private class TestAutoFocusProxy implements AutoFocusStateMachine.AutoFocusStateListener { 1629 private final AutoFocusStateListener mListener; 1630 TestAutoFocusProxy(AutoFocusStateListener listener)1631 TestAutoFocusProxy(AutoFocusStateListener listener) { 1632 mListener = listener; 1633 } 1634 1635 @Override onAutoFocusSuccess(CaptureResult result, boolean locked)1636 public void onAutoFocusSuccess(CaptureResult result, boolean locked) { 1637 mListener.onDone(true); 1638 } 1639 1640 @Override onAutoFocusFail(CaptureResult result, boolean locked)1641 public void onAutoFocusFail(CaptureResult result, boolean locked) { 1642 mListener.onDone(false); 1643 } 1644 1645 @Override onAutoFocusScan(CaptureResult result)1646 public void onAutoFocusScan(CaptureResult result) { 1647 mListener.onScan(); 1648 } 1649 1650 @Override onAutoFocusInactive(CaptureResult result)1651 public void onAutoFocusInactive(CaptureResult result) { 1652 mListener.onInactive(); 1653 } 1654 } 1655 1656 // Verify that camera extension sessions can support AF and AF metering controls. The test 1657 // goal is to check that AF related controls and results are supported and can be used to 1658 // lock the AF state and not to do an exhaustive check of the AF state transitions or manual AF. 1659 @Test testAFMetering()1660 public void testAFMetering() throws Exception { 1661 final CaptureRequest.Key[] FOCUS_CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_AF_MODE, 1662 CaptureRequest.CONTROL_AF_REGIONS, CaptureRequest.CONTROL_AF_TRIGGER}; 1663 final CaptureResult.Key[] FOCUS_CAPTURE_RESULT_SET = {CaptureResult.CONTROL_AF_MODE, 1664 CaptureResult.CONTROL_AF_REGIONS, CaptureResult.CONTROL_AF_TRIGGER, 1665 CaptureResult.CONTROL_AF_STATE}; 1666 final int METERING_REGION_SCALE_RATIO = 8; 1667 final int WAIT_FOR_FOCUS_DONE_TIMEOUT_MS = 6000; 1668 for (String id : mCameraIdsUnderTest) { 1669 StaticMetadata staticMeta = 1670 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 1671 if (!staticMeta.isColorOutputSupported()) { 1672 continue; 1673 } 1674 if (!staticMeta.hasFocuser()) { 1675 continue; 1676 } 1677 1678 boolean perFrameControl = staticMeta.isPerFrameControlSupported(); 1679 final Rect activeArraySize = staticMeta.getActiveArraySizeChecked(); 1680 int regionWidth = activeArraySize.width() / METERING_REGION_SCALE_RATIO - 1; 1681 int regionHeight = activeArraySize.height() / METERING_REGION_SCALE_RATIO - 1; 1682 int centerX = activeArraySize.width() / 2; 1683 int centerY = activeArraySize.height() / 2; 1684 1685 // Center region 1686 MeteringRectangle[] afMeteringRects = {new MeteringRectangle( 1687 centerX - regionWidth / 2, centerY - regionHeight / 2, 1688 regionWidth, regionHeight, 1689 MeteringRectangle.METERING_WEIGHT_MAX)}; 1690 1691 updatePreviewSurfaceTexture(); 1692 CameraExtensionCharacteristics extensionChars = 1693 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 1694 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 1695 for (Integer extension : supportedExtensions) { 1696 Set<CaptureRequest.Key> supportedRequestKeys = 1697 extensionChars.getAvailableCaptureRequestKeys(extension); 1698 if (!supportedRequestKeys.containsAll(Arrays.asList(FOCUS_CAPTURE_REQUEST_SET))) { 1699 continue; 1700 } 1701 Set<CaptureResult.Key> supportedResultKeys = 1702 extensionChars.getAvailableCaptureResultKeys(extension); 1703 assertTrue(supportedResultKeys.containsAll( 1704 Arrays.asList(FOCUS_CAPTURE_RESULT_SET))); 1705 1706 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 1707 mSurfaceTexture.getClass()); 1708 Size maxSize = 1709 CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 1710 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), 1711 maxSize.getHeight()); 1712 Surface texturedSurface = new Surface(mSurfaceTexture); 1713 1714 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 1715 outputConfigs.add(new OutputConfiguration(texturedSurface)); 1716 1717 BlockingExtensionSessionCallback sessionListener = 1718 new BlockingExtensionSessionCallback(mock( 1719 CameraExtensionSession.StateCallback.class)); 1720 ExtensionSessionConfiguration configuration = 1721 new ExtensionSessionConfiguration(extension, outputConfigs, 1722 new HandlerExecutor(mTestRule.getHandler()), 1723 sessionListener); 1724 1725 try { 1726 mTestRule.openDevice(id); 1727 CameraDevice camera = mTestRule.getCamera(); 1728 camera.createExtensionSession(configuration); 1729 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 1730 SESSION_CONFIGURE_TIMEOUT_MS); 1731 assertNotNull(extensionSession); 1732 1733 // Check passive AF 1734 AutoFocusStateListener mockAFListener = 1735 mock(AutoFocusStateListener.class); 1736 AutoFocusStateMachine afState = new AutoFocusStateMachine( 1737 new TestAutoFocusProxy(mockAFListener)); 1738 CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock = 1739 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 1740 SimpleCaptureCallback repeatingCaptureCallback = 1741 new SimpleCaptureCallback(extension, repeatingCallbackMock, 1742 extensionChars.getAvailableCaptureResultKeys(extension), 1743 mCollector, afState, null /*flashState*/, 1744 null /*aeState*/, perFrameControl); 1745 1746 CaptureRequest.Builder captureBuilder = 1747 mTestRule.getCamera().createCaptureRequest( 1748 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 1749 captureBuilder.addTarget(texturedSurface); 1750 afState.setPassiveAutoFocus(true /*picture*/, captureBuilder); 1751 captureBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects); 1752 CaptureRequest request = captureBuilder.build(); 1753 int passiveSequenceId = extensionSession.setRepeatingRequest(request, 1754 new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback); 1755 assertTrue(passiveSequenceId > 0); 1756 1757 verify(repeatingCallbackMock, 1758 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 1759 .onCaptureResultAvailable(eq(extensionSession), eq(request), 1760 any(TotalCaptureResult.class)); 1761 1762 verify(mockAFListener, 1763 timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS).atLeastOnce()) 1764 .onDone(anyBoolean()); 1765 1766 // Check active AF 1767 mockAFListener = mock(AutoFocusStateListener.class); 1768 CameraExtensionSession.ExtensionCaptureCallback callbackMock = mock( 1769 CameraExtensionSession.ExtensionCaptureCallback.class); 1770 AutoFocusStateMachine activeAFState = new AutoFocusStateMachine( 1771 new TestAutoFocusProxy(mockAFListener)); 1772 CameraExtensionSession.ExtensionCaptureCallback captureCallback = 1773 new SimpleCaptureCallback(extension, callbackMock, 1774 extensionChars.getAvailableCaptureResultKeys(extension), 1775 mCollector, activeAFState, null /*flashState*/, 1776 null /*aeState*/, perFrameControl); 1777 1778 CaptureRequest.Builder triggerBuilder = 1779 mTestRule.getCamera().createCaptureRequest( 1780 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 1781 triggerBuilder.addTarget(texturedSurface); 1782 afState.setActiveAutoFocus(captureBuilder, triggerBuilder); 1783 afState.unlockAutoFocus(captureBuilder, triggerBuilder); 1784 triggerBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects); 1785 request = captureBuilder.build(); 1786 int activeSequenceId = extensionSession.setRepeatingRequest(request, 1787 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 1788 assertTrue((activeSequenceId > 0) && 1789 (activeSequenceId != passiveSequenceId)); 1790 1791 verify(callbackMock, 1792 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 1793 .onCaptureResultAvailable(eq(extensionSession), eq(request), 1794 any(TotalCaptureResult.class)); 1795 1796 CaptureRequest triggerRequest = triggerBuilder.build(); 1797 reset(mockAFListener); 1798 int triggerSequenceId = extensionSession.capture(triggerRequest, 1799 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 1800 assertTrue((triggerSequenceId > 0) && 1801 (activeSequenceId != triggerSequenceId) && 1802 (triggerSequenceId != passiveSequenceId)); 1803 1804 verify(callbackMock, 1805 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 1806 .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest), 1807 any(TotalCaptureResult.class)); 1808 1809 afState.lockAutoFocus(captureBuilder, triggerBuilder); 1810 triggerRequest = triggerBuilder.build(); 1811 reset(mockAFListener); 1812 extensionSession.capture(triggerRequest, 1813 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 1814 1815 verify(callbackMock, 1816 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 1817 .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest), 1818 any(TotalCaptureResult.class)); 1819 1820 verify(mockAFListener, timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS) 1821 .atLeast(1)).onDone(anyBoolean()); 1822 1823 extensionSession.stopRepeating(); 1824 1825 extensionSession.close(); 1826 1827 sessionListener.getStateWaiter().waitForState( 1828 BlockingExtensionSessionCallback.SESSION_CLOSED, 1829 SESSION_CLOSE_TIMEOUT_MS); 1830 } finally { 1831 mTestRule.closeDevice(id); 1832 texturedSurface.release(); 1833 } 1834 } 1835 } 1836 } 1837 1838 // Verify that camera extension sessions can support the zoom ratio control. 1839 @Test testZoomRatio()1840 public void testZoomRatio() throws Exception { 1841 final CaptureRequest.Key[] ZOOM_CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_ZOOM_RATIO}; 1842 final CaptureResult.Key[] ZOOM_CAPTURE_RESULT_SET = {CaptureResult.CONTROL_ZOOM_RATIO}; 1843 final int ZOOM_RATIO_STEPS = 10; 1844 for (String id : mCameraIdsUnderTest) { 1845 StaticMetadata staticMeta = 1846 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 1847 if (!staticMeta.isColorOutputSupported()) { 1848 continue; 1849 } 1850 Range<Float> zoomRatioRange = staticMeta.getZoomRatioRangeChecked(); 1851 if (zoomRatioRange.getUpper().equals(zoomRatioRange.getLower())) { 1852 continue; 1853 } 1854 1855 final float maxZoom = staticMeta.getAvailableMaxDigitalZoomChecked(); 1856 if (Math.abs(maxZoom - 1.0f) < ZOOM_ERROR_MARGIN) { 1857 return; 1858 } 1859 1860 Float zoomStep = 1861 (zoomRatioRange.getUpper() - zoomRatioRange.getLower()) / ZOOM_RATIO_STEPS; 1862 if (zoomStep < ZOOM_ERROR_MARGIN) { 1863 continue; 1864 } 1865 1866 ArrayList<Float> candidateZoomRatios = new ArrayList<>(ZOOM_RATIO_STEPS); 1867 for (int step = 0; step < (ZOOM_RATIO_STEPS - 1); step++) { 1868 candidateZoomRatios.add(step, zoomRatioRange.getLower() + step * zoomStep); 1869 } 1870 candidateZoomRatios.add(ZOOM_RATIO_STEPS - 1, zoomRatioRange.getUpper()); 1871 1872 updatePreviewSurfaceTexture(); 1873 CameraExtensionCharacteristics extensionChars = 1874 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 1875 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 1876 for (Integer extension : supportedExtensions) { 1877 Set<CaptureRequest.Key> supportedRequestKeys = 1878 extensionChars.getAvailableCaptureRequestKeys(extension); 1879 if (!supportedRequestKeys.containsAll(Arrays.asList(ZOOM_CAPTURE_REQUEST_SET))) { 1880 continue; 1881 } 1882 Set<CaptureResult.Key> supportedResultKeys = 1883 extensionChars.getAvailableCaptureResultKeys(extension); 1884 assertTrue(supportedResultKeys.containsAll( 1885 Arrays.asList(ZOOM_CAPTURE_RESULT_SET))); 1886 1887 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension, 1888 mSurfaceTexture.getClass()); 1889 Size maxSize = 1890 CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0])); 1891 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), 1892 maxSize.getHeight()); 1893 Surface texturedSurface = new Surface(mSurfaceTexture); 1894 1895 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 1896 outputConfigs.add(new OutputConfiguration(texturedSurface)); 1897 1898 BlockingExtensionSessionCallback sessionListener = 1899 new BlockingExtensionSessionCallback(mock( 1900 CameraExtensionSession.StateCallback.class)); 1901 ExtensionSessionConfiguration configuration = 1902 new ExtensionSessionConfiguration(extension, outputConfigs, 1903 new HandlerExecutor(mTestRule.getHandler()), 1904 sessionListener); 1905 1906 try { 1907 mTestRule.openDevice(id); 1908 CameraDevice camera = mTestRule.getCamera(); 1909 camera.createExtensionSession(configuration); 1910 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession( 1911 SESSION_CONFIGURE_TIMEOUT_MS); 1912 assertNotNull(extensionSession); 1913 1914 CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock = 1915 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 1916 SimpleCaptureCallback repeatingCaptureCallback = 1917 new SimpleCaptureCallback(extension, repeatingCallbackMock, 1918 extensionChars.getAvailableCaptureResultKeys(extension), 1919 mCollector); 1920 1921 CaptureRequest.Builder captureBuilder = 1922 mTestRule.getCamera().createCaptureRequest( 1923 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 1924 captureBuilder.addTarget(texturedSurface); 1925 for (Float currentZoomRatio : candidateZoomRatios) { 1926 captureBuilder.set(CaptureRequest.CONTROL_ZOOM_RATIO, currentZoomRatio); 1927 CaptureRequest request = captureBuilder.build(); 1928 1929 int seqId = extensionSession.setRepeatingRequest(request, 1930 new HandlerExecutor(mTestRule.getHandler()), 1931 repeatingCaptureCallback); 1932 assertTrue(seqId > 0); 1933 1934 verify(repeatingCallbackMock, 1935 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 1936 .onCaptureResultAvailable(eq(extensionSession), eq(request), 1937 any(TotalCaptureResult.class)); 1938 } 1939 1940 extensionSession.stopRepeating(); 1941 1942 extensionSession.close(); 1943 1944 sessionListener.getStateWaiter().waitForState( 1945 BlockingExtensionSessionCallback.SESSION_CLOSED, 1946 SESSION_CLOSE_TIMEOUT_MS); 1947 } finally { 1948 mTestRule.closeDevice(id); 1949 texturedSurface.release(); 1950 } 1951 } 1952 } 1953 } 1954 1955 public interface FlashStateListener { onFired()1956 public void onFired(); onReady()1957 public void onReady(); onCharging()1958 public void onCharging(); onPartial()1959 public void onPartial(); onUnavailable()1960 public void onUnavailable(); 1961 } 1962 1963 public interface AutoExposureStateListener { onInactive()1964 public void onInactive(); onSearching()1965 public void onSearching(); onConverged()1966 public void onConverged(); onLocked()1967 public void onLocked(); onFlashRequired()1968 public void onFlashRequired(); onPrecapture()1969 public void onPrecapture(); 1970 } 1971 1972 // Verify that camera extension sessions can support Flash related controls. The test 1973 // goal is to check that Flash controls and results are supported and can be used to 1974 // turn on torch, run the pre-capture sequence and active the main flash. 1975 @Test testFlash()1976 public void testFlash() throws Exception { 1977 final CaptureRequest.Key[] FLASH_CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_AE_MODE, 1978 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_LOCK, 1979 CaptureRequest.FLASH_MODE}; 1980 final CaptureResult.Key[] FLASH_CAPTURE_RESULT_SET = {CaptureResult.CONTROL_AE_MODE, 1981 CaptureResult.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureResult.CONTROL_AE_LOCK, 1982 CaptureResult.CONTROL_AE_STATE, CaptureResult.FLASH_MODE, 1983 CaptureResult.FLASH_STATE}; 1984 for (String id : mCameraIdsUnderTest) { 1985 StaticMetadata staticMeta = 1986 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 1987 if (!staticMeta.isColorOutputSupported()) { 1988 continue; 1989 } 1990 if (!staticMeta.hasFlash()) { 1991 continue; 1992 } 1993 1994 boolean perFrameControl = staticMeta.isPerFrameControlSupported(); 1995 updatePreviewSurfaceTexture(); 1996 CameraExtensionCharacteristics extensionChars = 1997 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 1998 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 1999 for (Integer extension : supportedExtensions) { 2000 Set<CaptureRequest.Key> supportedRequestKeys = 2001 extensionChars.getAvailableCaptureRequestKeys(extension); 2002 if (!supportedRequestKeys.containsAll(Arrays.asList(FLASH_CAPTURE_REQUEST_SET))) { 2003 continue; 2004 } 2005 Set<CaptureResult.Key> supportedResultKeys = 2006 extensionChars.getAvailableCaptureResultKeys(extension); 2007 assertTrue(supportedResultKeys.containsAll( 2008 Arrays.asList(FLASH_CAPTURE_RESULT_SET))); 2009 2010 int captureFormat = ImageFormat.JPEG; 2011 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension, 2012 captureFormat); 2013 assertFalse("No Jpeg output supported", captureSizes.isEmpty()); 2014 Size captureMaxSize = captureSizes.get(0); 2015 2016 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false, 1); 2017 ImageReader extensionImageReader = CameraTestUtils.makeImageReader( 2018 captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener, 2019 mTestRule.getHandler()); 2020 Surface imageReaderSurface = extensionImageReader.getSurface(); 2021 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 2022 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 2023 outputConfigs.add(readerOutput); 2024 2025 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension, 2026 mSurfaceTexture.getClass()); 2027 Size previewSize = repeatingSizes.get(0); 2028 2029 mSurfaceTexture.setDefaultBufferSize(previewSize.getWidth(), 2030 previewSize.getHeight()); 2031 Surface texturedSurface = new Surface(mSurfaceTexture); 2032 outputConfigs.add(new OutputConfiguration(texturedSurface)); 2033 2034 BlockingExtensionSessionCallback sessionListener = 2035 new BlockingExtensionSessionCallback(mock( 2036 CameraExtensionSession.StateCallback.class)); 2037 ExtensionSessionConfiguration configuration = 2038 new ExtensionSessionConfiguration(extension, outputConfigs, 2039 new HandlerExecutor(mTestRule.getHandler()), 2040 sessionListener); 2041 try { 2042 mTestRule.openDevice(id); 2043 CameraDevice camera = mTestRule.getCamera(); 2044 camera.createExtensionSession(configuration); 2045 CameraExtensionSession extensionSession = 2046 sessionListener.waitAndGetSession( 2047 SESSION_CONFIGURE_TIMEOUT_MS); 2048 assertNotNull(extensionSession); 2049 2050 // Test torch on and off 2051 CaptureRequest.Builder captureBuilder = 2052 mTestRule.getCamera().createCaptureRequest( 2053 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 2054 captureBuilder.addTarget(texturedSurface); 2055 captureBuilder.set(CaptureRequest.CONTROL_AE_MODE, 2056 CameraMetadata.CONTROL_AE_MODE_ON); 2057 captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false); 2058 captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_TORCH); 2059 FlashStateListener mockFlashListener = mock(FlashStateListener.class); 2060 AutoExposureStateListener mockAEStateListener = 2061 mock(AutoExposureStateListener.class); 2062 CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock = 2063 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 2064 SimpleCaptureCallback repeatingCaptureCallback = 2065 new SimpleCaptureCallback(extension, repeatingCallbackMock, 2066 extensionChars.getAvailableCaptureResultKeys(extension), 2067 mCollector, null /*afState*/, mockFlashListener, 2068 mockAEStateListener, perFrameControl); 2069 CaptureRequest repeatingRequest = captureBuilder.build(); 2070 int repeatingSequenceId = 2071 extensionSession.setRepeatingRequest(repeatingRequest, 2072 new HandlerExecutor(mTestRule.getHandler()), 2073 repeatingCaptureCallback); 2074 assertTrue(repeatingSequenceId > 0); 2075 2076 verify(repeatingCallbackMock, 2077 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 2078 .onCaptureResultAvailable(eq(extensionSession), 2079 eq(repeatingRequest), any(TotalCaptureResult.class)); 2080 verify(mockFlashListener, 2081 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired(); 2082 2083 captureBuilder.set(CaptureRequest.CONTROL_AE_MODE, 2084 CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH); 2085 captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF); 2086 repeatingRequest = captureBuilder.build(); 2087 reset(mockFlashListener); 2088 repeatingSequenceId = extensionSession.setRepeatingRequest(repeatingRequest, 2089 new HandlerExecutor(mTestRule.getHandler()), 2090 repeatingCaptureCallback); 2091 assertTrue(repeatingSequenceId > 0); 2092 2093 verify(repeatingCallbackMock, 2094 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 2095 .onCaptureResultAvailable(eq(extensionSession), 2096 eq(repeatingRequest), any(TotalCaptureResult.class)); 2097 verify(mockFlashListener, 2098 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onReady(); 2099 2100 // Test AE pre-capture sequence 2101 captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF); 2102 captureBuilder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, 2103 CameraMetadata.CONTROL_AE_PRECAPTURE_TRIGGER_START); 2104 CaptureRequest triggerRequest = captureBuilder.build(); 2105 int triggerSeqId = extensionSession.capture(triggerRequest, 2106 new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback); 2107 assertTrue(triggerSeqId > 0); 2108 2109 verify(repeatingCallbackMock, 2110 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()) 2111 .onCaptureResultAvailable(eq(extensionSession), 2112 eq(triggerRequest), any(TotalCaptureResult.class)); 2113 verify(mockAEStateListener, 2114 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onPrecapture(); 2115 verify(mockAEStateListener, 2116 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onConverged(); 2117 2118 // Test main flash 2119 captureBuilder = mTestRule.getCamera().createCaptureRequest( 2120 android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE); 2121 captureBuilder.addTarget(imageReaderSurface); 2122 captureBuilder.set(CaptureRequest.CONTROL_AE_MODE, 2123 CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH); 2124 captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, true); 2125 captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_SINGLE); 2126 CameraExtensionSession.ExtensionCaptureCallback mockCaptureCallback = 2127 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 2128 reset(mockFlashListener); 2129 SimpleCaptureCallback captureCallback = 2130 new SimpleCaptureCallback(extension, mockCaptureCallback, 2131 extensionChars.getAvailableCaptureResultKeys(extension), 2132 mCollector, null /*afState*/, mockFlashListener, 2133 mockAEStateListener, perFrameControl); 2134 2135 CaptureRequest captureRequest = captureBuilder.build(); 2136 int captureSequenceId = extensionSession.capture(captureRequest, 2137 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 2138 assertTrue(captureSequenceId > 0); 2139 2140 Image img = 2141 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS); 2142 validateImage(img, captureMaxSize.getWidth(), 2143 captureMaxSize.getHeight(), captureFormat, null); 2144 long imgTs = img.getTimestamp(); 2145 img.close(); 2146 2147 verify(mockCaptureCallback, 2148 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 2149 .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs)); 2150 verify(mockCaptureCallback, 2151 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1)) 2152 .onCaptureResultAvailable(eq(extensionSession), 2153 eq(captureRequest), any(TotalCaptureResult.class)); 2154 verify(mockFlashListener, 2155 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired(); 2156 2157 extensionSession.stopRepeating(); 2158 2159 extensionSession.close(); 2160 2161 sessionListener.getStateWaiter().waitForState( 2162 BlockingExtensionSessionCallback.SESSION_CLOSED, 2163 SESSION_CLOSE_TIMEOUT_MS); 2164 } finally { 2165 mTestRule.closeDevice(id); 2166 texturedSurface.release(); 2167 extensionImageReader.close(); 2168 } 2169 } 2170 } 2171 } 2172 2173 // Verify 'CameraExtensionSession.StillCaptureLatency' behavior 2174 @Test testSessionStillCaptureLatency()2175 public void testSessionStillCaptureLatency() throws Exception { 2176 final long CAPTURE_LATENCY_MS = 100; 2177 final long PROCESSING_LATENCY_MS = 200; 2178 final long DIFFERENT_PROCESSING_LATENCY_MS = 201; 2179 CameraExtensionSession.StillCaptureLatency stillCaptureLatency = 2180 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS, 2181 PROCESSING_LATENCY_MS); 2182 assertEquals(stillCaptureLatency.getCaptureLatency(), CAPTURE_LATENCY_MS); 2183 assertEquals(stillCaptureLatency.getProcessingLatency(), PROCESSING_LATENCY_MS); 2184 assertNotNull(stillCaptureLatency.toString()); 2185 CameraExtensionSession.StillCaptureLatency differentStillCaptureLatency = 2186 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS, 2187 DIFFERENT_PROCESSING_LATENCY_MS); 2188 assertFalse(stillCaptureLatency.equals(differentStillCaptureLatency)); 2189 assertFalse(stillCaptureLatency.hashCode() == 2190 differentStillCaptureLatency.hashCode()); 2191 } 2192 2193 @Test testIllegalArguments()2194 public void testIllegalArguments() throws Exception { 2195 for (String id : mCameraIdsUnderTest) { 2196 StaticMetadata staticMeta = 2197 new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id)); 2198 if (!staticMeta.isColorOutputSupported()) { 2199 continue; 2200 } 2201 updatePreviewSurfaceTexture(); 2202 CameraExtensionCharacteristics extensionChars = 2203 mTestRule.getCameraManager().getCameraExtensionCharacteristics(id); 2204 List<Integer> supportedExtensions = extensionChars.getSupportedExtensions(); 2205 for (Integer extension : supportedExtensions) { 2206 List<OutputConfiguration> outputConfigs = new ArrayList<>(); 2207 BlockingExtensionSessionCallback sessionListener = 2208 new BlockingExtensionSessionCallback(mock( 2209 CameraExtensionSession.StateCallback.class)); 2210 ExtensionSessionConfiguration configuration = 2211 new ExtensionSessionConfiguration(extension, outputConfigs, 2212 new HandlerExecutor(mTestRule.getHandler()), 2213 sessionListener); 2214 2215 try { 2216 mTestRule.openDevice(id); 2217 CameraDevice camera = mTestRule.getCamera(); 2218 try { 2219 camera.createExtensionSession(configuration); 2220 fail("should get IllegalArgumentException due to absent output surfaces"); 2221 } catch (IllegalArgumentException e) { 2222 // Expected, we can proceed further 2223 } 2224 2225 int captureFormat = ImageFormat.YUV_420_888; 2226 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension, 2227 captureFormat); 2228 if (captureSizes.isEmpty()) { 2229 captureFormat = ImageFormat.JPEG; 2230 captureSizes = extensionChars.getExtensionSupportedSizes(extension, 2231 captureFormat); 2232 } 2233 Size captureMaxSize = 2234 CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0])); 2235 2236 mSurfaceTexture.setDefaultBufferSize(1, 1); 2237 Surface texturedSurface = new Surface(mSurfaceTexture); 2238 outputConfigs.add(new OutputConfiguration(texturedSurface)); 2239 configuration = new ExtensionSessionConfiguration(extension, outputConfigs, 2240 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 2241 2242 try { 2243 camera.createExtensionSession(configuration); 2244 fail("should get IllegalArgumentException due to illegal repeating request" 2245 + " output surface"); 2246 } catch (IllegalArgumentException e) { 2247 // Expected, we can proceed further 2248 } finally { 2249 outputConfigs.clear(); 2250 } 2251 2252 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false, 2253 1); 2254 Size invalidCaptureSize = new Size(1, 1); 2255 ImageReader extensionImageReader = CameraTestUtils.makeImageReader( 2256 invalidCaptureSize, captureFormat, /*maxImages*/ 1, 2257 imageListener, mTestRule.getHandler()); 2258 Surface imageReaderSurface = extensionImageReader.getSurface(); 2259 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface); 2260 outputConfigs.add(readerOutput); 2261 configuration = new ExtensionSessionConfiguration(extension, outputConfigs, 2262 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 2263 2264 try{ 2265 camera.createExtensionSession(configuration); 2266 fail("should get IllegalArgumentException due to illegal multi-frame" 2267 + " request output surface"); 2268 } catch (IllegalArgumentException e) { 2269 // Expected, we can proceed further 2270 } finally { 2271 outputConfigs.clear(); 2272 extensionImageReader.close(); 2273 } 2274 2275 // Pick a supported preview/repeating size with aspect ratio close to the 2276 // multi-frame capture size 2277 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension, 2278 mSurfaceTexture.getClass()); 2279 Size maxRepeatingSize = 2280 CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0])); 2281 List<Size> previewSizes = getSupportedPreviewSizes(id, 2282 mTestRule.getCameraManager(), 2283 getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND)); 2284 List<Size> supportedPreviewSizes = 2285 previewSizes.stream().filter(repeatingSizes::contains).collect( 2286 Collectors.toList()); 2287 if (!supportedPreviewSizes.isEmpty()) { 2288 float targetAr = 2289 ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight(); 2290 for (Size s : supportedPreviewSizes) { 2291 float currentAr = ((float) s.getWidth()) / s.getHeight(); 2292 if (Math.abs(targetAr - currentAr) < 0.01) { 2293 maxRepeatingSize = s; 2294 break; 2295 } 2296 } 2297 } 2298 2299 imageListener = new SimpleImageReaderListener(false, 1); 2300 extensionImageReader = CameraTestUtils.makeImageReader(captureMaxSize, 2301 captureFormat, /*maxImages*/ 1, imageListener, mTestRule.getHandler()); 2302 imageReaderSurface = extensionImageReader.getSurface(); 2303 readerOutput = new OutputConfiguration(imageReaderSurface); 2304 outputConfigs.add(readerOutput); 2305 2306 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(), 2307 maxRepeatingSize.getHeight()); 2308 texturedSurface = new Surface(mSurfaceTexture); 2309 outputConfigs.add(new OutputConfiguration(texturedSurface)); 2310 2311 configuration = new ExtensionSessionConfiguration(extension, outputConfigs, 2312 new HandlerExecutor(mTestRule.getHandler()), sessionListener); 2313 camera.createExtensionSession(configuration); 2314 CameraExtensionSession extensionSession = 2315 sessionListener.waitAndGetSession( 2316 SESSION_CONFIGURE_TIMEOUT_MS); 2317 assertNotNull(extensionSession); 2318 2319 CaptureRequest.Builder captureBuilder = 2320 mTestRule.getCamera().createCaptureRequest( 2321 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW); 2322 captureBuilder.addTarget(imageReaderSurface); 2323 CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock = 2324 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 2325 SimpleCaptureCallback repeatingCaptureCallback = 2326 new SimpleCaptureCallback(extension, repeatingCallbackMock, 2327 extensionChars.getAvailableCaptureResultKeys(extension), 2328 mCollector); 2329 CaptureRequest repeatingRequest = captureBuilder.build(); 2330 try { 2331 extensionSession.setRepeatingRequest(repeatingRequest, 2332 new HandlerExecutor(mTestRule.getHandler()), 2333 repeatingCaptureCallback); 2334 fail("should get IllegalArgumentException due to illegal repeating request" 2335 + " output target"); 2336 } catch (IllegalArgumentException e) { 2337 // Expected, we can proceed further 2338 } 2339 2340 extensionSession.close(); 2341 2342 sessionListener.getStateWaiter().waitForState( 2343 BlockingExtensionSessionCallback.SESSION_CLOSED, 2344 SESSION_CLOSE_TIMEOUT_MS); 2345 2346 texturedSurface.release(); 2347 extensionImageReader.close(); 2348 2349 captureBuilder = mTestRule.getCamera().createCaptureRequest( 2350 CameraDevice.TEMPLATE_PREVIEW); 2351 captureBuilder.addTarget(texturedSurface); 2352 CameraExtensionSession.ExtensionCaptureCallback captureCallback = 2353 mock(CameraExtensionSession.ExtensionCaptureCallback.class); 2354 2355 CaptureRequest captureRequest = captureBuilder.build(); 2356 try { 2357 extensionSession.setRepeatingRequest(captureRequest, 2358 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 2359 fail("should get IllegalStateException due to closed session"); 2360 } catch (IllegalStateException e) { 2361 // Expected, we can proceed further 2362 } 2363 2364 try { 2365 extensionSession.stopRepeating(); 2366 fail("should get IllegalStateException due to closed session"); 2367 } catch (IllegalStateException e) { 2368 // Expected, we can proceed further 2369 } 2370 2371 try { 2372 extensionSession.capture(captureRequest, 2373 new HandlerExecutor(mTestRule.getHandler()), captureCallback); 2374 fail("should get IllegalStateException due to closed session"); 2375 } catch (IllegalStateException e) { 2376 // Expected, we can proceed further 2377 } 2378 } finally { 2379 mTestRule.closeDevice(id); 2380 } 2381 } 2382 } 2383 } 2384 } 2385