• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.cts;
18 
19 import static android.hardware.camera2.cts.ImageReaderTest.validateDynamicDepthNative;
20 import static android.hardware.camera2.cts.helpers.AssertHelpers.assertArrayContains;
21 import static org.mockito.ArgumentMatchers.anyBoolean;
22 import static org.mockito.ArgumentMatchers.any;
23 import static org.mockito.ArgumentMatchers.anyInt;
24 import static org.mockito.ArgumentMatchers.anyLong;
25 import static org.mockito.ArgumentMatchers.eq;
26 import static org.mockito.Mockito.atLeastOnce;
27 import static org.mockito.Mockito.mock;
28 import static org.mockito.Mockito.reset;
29 import static org.mockito.Mockito.timeout;
30 import static org.mockito.Mockito.times;
31 import static org.mockito.Mockito.verify;
32 
33 import com.android.compatibility.common.util.DeviceReportLog;
34 import com.android.compatibility.common.util.ResultType;
35 import com.android.compatibility.common.util.ResultUnit;
36 import com.android.compatibility.common.util.Stat;
37 import com.android.internal.camera.flags.Flags;
38 import com.android.ex.camera2.blocking.BlockingSessionCallback;
39 import com.android.ex.camera2.blocking.BlockingExtensionSessionCallback;
40 import com.android.ex.camera2.blocking.BlockingStateCallback;
41 import com.android.ex.camera2.exceptions.TimeoutRuntimeException;
42 import com.android.ex.camera2.pos.AutoFocusStateMachine;
43 
44 import android.graphics.ColorSpace;
45 import android.graphics.ImageFormat;
46 import android.graphics.PointF;
47 import android.graphics.Rect;
48 import android.graphics.SurfaceTexture;
49 import android.hardware.HardwareBuffer;
50 import android.hardware.camera2.CameraCaptureSession;
51 import android.hardware.camera2.CameraCharacteristics;
52 import android.hardware.camera2.CameraDevice;
53 import android.hardware.camera2.CameraExtensionCharacteristics;
54 import android.hardware.camera2.CameraExtensionSession;
55 import android.hardware.camera2.CameraMetadata;
56 import android.hardware.camera2.CaptureRequest;
57 import android.hardware.camera2.CaptureResult;
58 import android.hardware.camera2.TotalCaptureResult;
59 import android.hardware.camera2.cts.helpers.CameraErrorCollector;
60 import android.hardware.camera2.cts.helpers.StaticMetadata;
61 import android.hardware.camera2.cts.testcases.Camera2AndroidTestRule;
62 import android.hardware.camera2.params.ColorSpaceProfiles;
63 import android.hardware.camera2.params.DynamicRangeProfiles;
64 import android.hardware.camera2.params.ExtensionSessionConfiguration;
65 import android.hardware.camera2.params.MeteringRectangle;
66 import android.hardware.camera2.params.OutputConfiguration;
67 import android.hardware.camera2.params.SessionConfiguration;
68 import android.media.ExifInterface;
69 import android.media.Image;
70 import android.media.ImageReader;
71 import android.os.Build;
72 import android.os.SystemClock;
73 import android.platform.test.annotations.RequiresFlagsEnabled;
74 import android.util.Range;
75 import android.util.Size;
76 
77 import static android.hardware.camera2.cts.CameraTestUtils.*;
78 import static android.hardware.cts.helpers.CameraUtils.*;
79 
80 import android.util.Log;
81 import android.view.Surface;
82 import android.view.TextureView;
83 
84 import androidx.test.platform.app.InstrumentationRegistry;
85 import androidx.test.rule.ActivityTestRule;
86 
87 import org.junit.Rule;
88 import org.junit.Test;
89 import org.junit.runner.RunWith;
90 import org.junit.runners.Parameterized;
91 
92 import java.io.IOException;
93 import java.util.ArrayList;
94 import java.util.Arrays;
95 import java.util.HashSet;
96 import java.util.List;
97 import java.util.Set;
98 import java.util.stream.Collectors;
99 
100 @RunWith(Parameterized.class)
101 public class CameraExtensionSessionTest extends Camera2ParameterizedTestCase {
102     private static final String TAG = "CameraExtensionSessionTest";
103     private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE);
104     private static final long WAIT_FOR_COMMAND_TO_COMPLETE_MS = 5000;
105     private static final long REPEATING_REQUEST_TIMEOUT_MS = 5000;
106     private static final int MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS = 10000;
107     private static final float ZOOM_ERROR_MARGIN = 0.05f;
108 
109     private static final int WAIT_FOR_FOCUS_DONE_TIMEOUT_MS = 6000;
110 
111     private static final int MAX_IMAGES = 1; // Common maximum images that can be acquired during
112                                              // still capture
113 
114     private static final CaptureRequest.Key[] FOCUS_CAPTURE_REQUEST_SET = {
115             CaptureRequest.CONTROL_AF_MODE,
116             CaptureRequest.CONTROL_AF_REGIONS,
117             CaptureRequest.CONTROL_AF_TRIGGER};
118     private static final CaptureResult.Key[] FOCUS_CAPTURE_RESULT_SET = {
119             CaptureResult.CONTROL_AF_MODE,
120             CaptureResult.CONTROL_AF_REGIONS,
121             CaptureResult.CONTROL_AF_TRIGGER,
122             CaptureResult.CONTROL_AF_STATE};
123     private static final CaptureRequest.Key[] ZOOM_CAPTURE_REQUEST_SET = {
124             CaptureRequest.CONTROL_ZOOM_RATIO};
125     private static final CaptureResult.Key[] ZOOM_CAPTURE_RESULT_SET = {
126             CaptureResult.CONTROL_ZOOM_RATIO};
127 
128     private SurfaceTexture mSurfaceTexture = null;
129     private Camera2AndroidTestRule mTestRule = null;
130     private CameraErrorCollector mCollector =  null;
131 
132     private DeviceReportLog mReportLog;
133 
134     @Override
setUp()135     public void setUp() throws Exception {
136         super.setUp();
137         mTestRule = new Camera2AndroidTestRule(mContext);
138         mTestRule.before();
139         mCollector = new CameraErrorCollector();
140     }
141 
142     @Override
tearDown()143     public void tearDown() throws Exception {
144         if (mTestRule != null) {
145             mTestRule.after();
146         }
147         if (mSurfaceTexture != null) {
148             mSurfaceTexture.release();
149             mSurfaceTexture = null;
150         }
151         if (mCollector != null) {
152             try {
153                 mCollector.verify();
154             } catch (Throwable e) {
155                 throw new Exception(e.getMessage());
156             }
157         }
158         super.tearDown();
159     }
160 
161     @Rule
162     public ActivityTestRule<CameraExtensionTestActivity> mActivityRule =
163             new ActivityTestRule<>(CameraExtensionTestActivity.class);
164 
updatePreviewSurfaceTexture()165     private void updatePreviewSurfaceTexture() {
166         if (mSurfaceTexture != null) {
167             return;
168         }
169 
170         TextureView textureView = mActivityRule.getActivity().getTextureView();
171         mSurfaceTexture = getAvailableSurfaceTexture(WAIT_FOR_COMMAND_TO_COMPLETE_MS, textureView);
172         assertNotNull("Failed to acquire valid preview surface texture!", mSurfaceTexture);
173     }
174 
175     // Verify that camera extension sessions can be created and closed as expected.
176     @Test
testBasicExtensionLifecycle()177     public void testBasicExtensionLifecycle() throws Exception {
178         for (String id : getCameraIdsUnderTest()) {
179             StaticMetadata staticMeta =
180                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
181             if (!staticMeta.isColorOutputSupported()) {
182                 continue;
183             }
184             updatePreviewSurfaceTexture();
185             CameraExtensionCharacteristics extensionChars =
186                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
187             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
188             for (Integer extension : supportedExtensions) {
189                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
190                         mSurfaceTexture.getClass());
191                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
192                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
193                 OutputConfiguration outputConfig = new OutputConfiguration(
194                         new Surface(mSurfaceTexture));
195                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
196                 outputConfigs.add(outputConfig);
197 
198                 BlockingExtensionSessionCallback sessionListener =
199                         new BlockingExtensionSessionCallback(
200                                 mock(CameraExtensionSession.StateCallback.class));
201                 ExtensionSessionConfiguration configuration =
202                         new ExtensionSessionConfiguration(extension, outputConfigs,
203                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
204 
205                 try {
206                     mTestRule.openDevice(id);
207                     CameraDevice camera = mTestRule.getCamera();
208                     camera.createExtensionSession(configuration);
209                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
210                             SESSION_CONFIGURE_TIMEOUT_MS);
211 
212                     extensionSession.close();
213                     sessionListener.getStateWaiter().waitForState(
214                             BlockingExtensionSessionCallback.SESSION_CLOSED,
215                             SESSION_CLOSE_TIMEOUT_MS);
216                 } finally {
217                     mTestRule.closeDevice(id);
218                 }
219             }
220         }
221     }
222 
223     // Verify that regular camera sessions close as expected after creating a camera extension
224     // session.
225     @Test
testCloseCaptureSession()226     public void testCloseCaptureSession() throws Exception {
227         for (String id : getCameraIdsUnderTest()) {
228             StaticMetadata staticMeta =
229                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
230             if (!staticMeta.isColorOutputSupported()) {
231                 continue;
232             }
233             updatePreviewSurfaceTexture();
234             CameraExtensionCharacteristics extensionChars =
235                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
236             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
237             for (Integer extension : supportedExtensions) {
238                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
239                         mSurfaceTexture.getClass());
240                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
241                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
242                 Surface repeatingSurface = new Surface(mSurfaceTexture);
243                 OutputConfiguration textureOutput = new OutputConfiguration(repeatingSurface);
244                 List<OutputConfiguration> outputs = new ArrayList<>();
245                 outputs.add(textureOutput);
246                 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback(
247                         mock(CameraCaptureSession.StateCallback.class));
248                 SessionConfiguration regularConfiguration = new SessionConfiguration(
249                         SessionConfiguration.SESSION_REGULAR, outputs,
250                         new HandlerExecutor(mTestRule.getHandler()), regularSessionListener);
251 
252                 BlockingExtensionSessionCallback sessionListener =
253                         new BlockingExtensionSessionCallback(mock(
254                                 CameraExtensionSession.StateCallback.class));
255                 ExtensionSessionConfiguration configuration =
256                         new ExtensionSessionConfiguration(extension, outputs,
257                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
258 
259                 try {
260                     mTestRule.openDevice(id);
261                     mTestRule.getCamera().createCaptureSession(regularConfiguration);
262 
263                     CameraCaptureSession session =
264                             regularSessionListener
265                                     .waitAndGetSession(SESSION_CONFIGURE_TIMEOUT_MS);
266                     assertNotNull(session);
267 
268                     CameraDevice camera = mTestRule.getCamera();
269                     camera.createExtensionSession(configuration);
270                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
271                             SESSION_CONFIGURE_TIMEOUT_MS);
272                     assertNotNull(extensionSession);
273 
274                     regularSessionListener.getStateWaiter().waitForState(
275                             BlockingExtensionSessionCallback.SESSION_CLOSED,
276                             SESSION_CLOSE_TIMEOUT_MS);
277 
278                     extensionSession.close();
279                     sessionListener.getStateWaiter().waitForState(
280                             BlockingExtensionSessionCallback.SESSION_CLOSED,
281                             SESSION_CLOSE_TIMEOUT_MS);
282                 } finally {
283                     mTestRule.closeDevice(id);
284                 }
285             }
286         }
287     }
288 
289     // Verify that camera extension sessions close as expected when creating a regular capture
290     // session.
291     @Test
testCloseExtensionSession()292     public void testCloseExtensionSession() throws Exception {
293         for (String id : getCameraIdsUnderTest()) {
294             StaticMetadata staticMeta =
295                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
296             if (!staticMeta.isColorOutputSupported()) {
297                 continue;
298             }
299             updatePreviewSurfaceTexture();
300             CameraExtensionCharacteristics extensionChars =
301                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
302             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
303             for (Integer extension : supportedExtensions) {
304                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
305                         mSurfaceTexture.getClass());
306                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
307                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
308                 Surface surface = new Surface(mSurfaceTexture);
309                 OutputConfiguration textureOutput = new OutputConfiguration(surface);
310                 List<OutputConfiguration> outputs = new ArrayList<>();
311                 outputs.add(textureOutput);
312                 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback(
313                         mock(CameraCaptureSession.StateCallback.class));
314                 SessionConfiguration regularConfiguration = new SessionConfiguration(
315                         SessionConfiguration.SESSION_REGULAR, outputs,
316                         new HandlerExecutor(mTestRule.getHandler()), regularSessionListener);
317 
318                 BlockingExtensionSessionCallback sessionListener =
319                         new BlockingExtensionSessionCallback(mock(
320                                 CameraExtensionSession.StateCallback.class));
321                 ExtensionSessionConfiguration configuration =
322                         new ExtensionSessionConfiguration(extension, outputs,
323                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
324 
325                 try {
326                     mTestRule.openDevice(id);
327                     CameraDevice camera = mTestRule.getCamera();
328                     camera.createExtensionSession(configuration);
329                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
330                             SESSION_CONFIGURE_TIMEOUT_MS);
331                     assertNotNull(extensionSession);
332 
333                     mTestRule.getCamera().createCaptureSession(regularConfiguration);
334                     sessionListener.getStateWaiter().waitForState(
335                             BlockingExtensionSessionCallback.SESSION_CLOSED,
336                             SESSION_CLOSE_TIMEOUT_MS);
337 
338                     CameraCaptureSession session =
339                             regularSessionListener.waitAndGetSession(
340                                     SESSION_CONFIGURE_TIMEOUT_MS);
341                     session.close();
342                     regularSessionListener.getStateWaiter().waitForState(
343                             BlockingSessionCallback.SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS);
344                 } finally {
345                     mTestRule.closeDevice(id);
346                 }
347             }
348         }
349     }
350 
351     // Verify camera device query
352     @Test
testGetDevice()353     public void testGetDevice() throws Exception {
354         for (String id : getCameraIdsUnderTest()) {
355             StaticMetadata staticMeta =
356                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
357             if (!staticMeta.isColorOutputSupported()) {
358                 continue;
359             }
360             updatePreviewSurfaceTexture();
361             CameraExtensionCharacteristics extensionChars =
362                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
363             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
364             for (Integer extension : supportedExtensions) {
365                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
366                         mSurfaceTexture.getClass());
367                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
368                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
369                 OutputConfiguration privateOutput = new OutputConfiguration(
370                         new Surface(mSurfaceTexture));
371                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
372                 outputConfigs.add(privateOutput);
373 
374                 BlockingExtensionSessionCallback sessionListener =
375                         new BlockingExtensionSessionCallback(
376                                 mock(CameraExtensionSession.StateCallback.class));
377                 ExtensionSessionConfiguration configuration =
378                         new ExtensionSessionConfiguration(extension, outputConfigs,
379                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
380 
381                 try {
382                     mTestRule.openDevice(id);
383                     CameraDevice camera = mTestRule.getCamera();
384                     camera.createExtensionSession(configuration);
385                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
386                             SESSION_CONFIGURE_TIMEOUT_MS);
387 
388                     assertEquals("Unexpected/Invalid camera device", mTestRule.getCamera(),
389                             extensionSession.getDevice());
390                 } finally {
391                     mTestRule.closeDevice(id);
392                 }
393 
394                 try {
395                     sessionListener.getStateWaiter().waitForState(
396                             BlockingExtensionSessionCallback.SESSION_CLOSED,
397                             SESSION_CLOSE_TIMEOUT_MS);
398                     fail("should get TimeoutRuntimeException due to previously closed camera "
399                             + "device");
400                 } catch (TimeoutRuntimeException e) {
401                     // Expected, per API spec we should not receive any further session callbacks
402                     // besides the device state 'onClosed' callback.
403                 }
404             }
405         }
406     }
407 
408     // Test case for repeating/stopRepeating on all supported extensions and expected state/capture
409     // callbacks.
410     @Test
testRepeatingCapture()411     public void testRepeatingCapture() throws Exception {
412         for (String id : getCameraIdsUnderTest()) {
413             StaticMetadata staticMeta =
414                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
415             if (!staticMeta.isColorOutputSupported()) {
416                 continue;
417             }
418             updatePreviewSurfaceTexture();
419             CameraExtensionCharacteristics extensionChars =
420                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
421             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
422             for (Integer extension : supportedExtensions) {
423                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
424                         mSurfaceTexture.getClass());
425                 Size maxSize =
426                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
427                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
428                         maxSize.getHeight());
429                 Surface texturedSurface = new Surface(mSurfaceTexture);
430 
431                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
432                 outputConfigs.add(new OutputConfiguration(texturedSurface));
433 
434                 BlockingExtensionSessionCallback sessionListener =
435                         new BlockingExtensionSessionCallback(mock(
436                                 CameraExtensionSession.StateCallback.class));
437                 ExtensionSessionConfiguration configuration =
438                         new ExtensionSessionConfiguration(extension, outputConfigs,
439                                 new HandlerExecutor(mTestRule.getHandler()),
440                                 sessionListener);
441 
442                 boolean captureResultsSupported =
443                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
444 
445                 try {
446                     mTestRule.openDevice(id);
447                     CameraDevice camera = mTestRule.getCamera();
448                     camera.createExtensionSession(configuration);
449                     CameraExtensionSession extensionSession =
450                             sessionListener.waitAndGetSession(
451                                     SESSION_CONFIGURE_TIMEOUT_MS);
452                     assertNotNull(extensionSession);
453 
454                     CaptureRequest.Builder captureBuilder =
455                             mTestRule.getCamera().createCaptureRequest(
456                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
457                     captureBuilder.addTarget(texturedSurface);
458                     CameraExtensionSession.ExtensionCaptureCallback captureCallbackMock =
459                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
460                     SimpleCaptureCallback simpleCaptureCallback =
461                             new SimpleCaptureCallback(extension, captureCallbackMock,
462                                     extensionChars.getAvailableCaptureResultKeys(extension),
463                                     mCollector);
464                     CaptureRequest request = captureBuilder.build();
465                     int sequenceId = extensionSession.setRepeatingRequest(request,
466                             new HandlerExecutor(mTestRule.getHandler()), simpleCaptureCallback);
467 
468                     verify(captureCallbackMock,
469                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
470                             .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
471                     verify(captureCallbackMock,
472                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
473                             .onCaptureProcessStarted(extensionSession, request);
474                     if (captureResultsSupported) {
475                         verify(captureCallbackMock,
476                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
477                                 .onCaptureResultAvailable(eq(extensionSession), eq(request),
478                                         any(TotalCaptureResult.class));
479                     }
480 
481                     extensionSession.stopRepeating();
482 
483                     verify(captureCallbackMock,
484                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
485                             .onCaptureSequenceCompleted(extensionSession, sequenceId);
486 
487                     verify(captureCallbackMock, times(0))
488                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
489                                     anyInt());
490 
491                     extensionSession.close();
492 
493                     sessionListener.getStateWaiter().waitForState(
494                             BlockingExtensionSessionCallback.SESSION_CLOSED,
495                             SESSION_CLOSE_TIMEOUT_MS);
496 
497                     assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" +
498                                     " callbacks must be greater or equal than the number of calls" +
499                                     " to onCaptureStarted!",
500                             simpleCaptureCallback.getTotalFramesArrived() +
501                                     simpleCaptureCallback.getTotalFramesFailed() >=
502                                     simpleCaptureCallback.getTotalFramesStarted());
503                     assertTrue(String.format("The last repeating request surface timestamp " +
504                                     "%d must be less than or equal to the last " +
505                                     "onCaptureStarted " +
506                                     "timestamp %d", mSurfaceTexture.getTimestamp(),
507                             simpleCaptureCallback.getLastTimestamp()),
508                             mSurfaceTexture.getTimestamp() <=
509                                     simpleCaptureCallback.getLastTimestamp());
510                 } finally {
511                     mTestRule.closeDevice(id);
512                     texturedSurface.release();
513                 }
514             }
515         }
516     }
517 
518     // Test for postview of still capture on all supported extensions
519     @Test
testPostviewAndCapture()520     public void testPostviewAndCapture() throws Exception {
521         final int IMAGE_COUNT = 10;
522         final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = {
523                 ImageFormat.YUV_420_888,
524                 ImageFormat.JPEG,
525                 ImageFormat.JPEG_R
526         };
527         for (String id : getCameraIdsUnderTest()) {
528             StaticMetadata staticMeta =
529                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
530             if (!staticMeta.isColorOutputSupported()) {
531                 continue;
532             }
533             updatePreviewSurfaceTexture();
534             CameraExtensionCharacteristics extensionChars =
535                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
536             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
537             for (Integer extension : supportedExtensions) {
538                 if (!extensionChars.isPostviewAvailable(extension)) {
539                     continue;
540                 }
541 
542                 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
543                     boolean captureProgressSupported =
544                             extensionChars.isCaptureProcessProgressAvailable(extension);
545 
546                     List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
547                             captureFormat);
548                     if (extensionSizes.isEmpty()) {
549                         continue;
550                     }
551 
552                     Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
553 
554                     for (int postviewFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
555                         List<Size> postviewSizes = extensionChars.getPostviewSupportedSizes(
556                                 extension, maxSize, postviewFormat);
557                         if (postviewSizes.isEmpty()) {
558                             continue;
559                         }
560 
561                         SimpleImageReaderListener imageListener = new SimpleImageReaderListener(
562                                 false, MAX_IMAGES);
563                         ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
564                                 captureFormat, /*maxImages*/ 1, imageListener,
565                                 mTestRule.getHandler());
566                         Surface imageReaderSurface = extensionImageReader.getSurface();
567                         OutputConfiguration readerOutput =
568                                 new OutputConfiguration(imageReaderSurface);
569                         List<OutputConfiguration> outputConfigs = new ArrayList<>();
570                         outputConfigs.add(readerOutput);
571 
572                         Size postviewSize =
573                                 CameraTestUtils.getMaxSize(postviewSizes.toArray(new Size[0]));
574                         SimpleImageReaderListener imageListenerPostview =
575                                 new SimpleImageReaderListener(false, MAX_IMAGES);
576                         ImageReader postviewImageReader = CameraTestUtils.makeImageReader(
577                                     postviewSize, postviewFormat, /*maxImages*/ 1,
578                                     imageListenerPostview, mTestRule.getHandler());
579 
580                         Surface postviewImageReaderSurface = postviewImageReader.getSurface();
581                         OutputConfiguration postviewReaderOutput =
582                                 new OutputConfiguration(postviewImageReaderSurface);
583                         BlockingExtensionSessionCallback sessionListener =
584                                 new BlockingExtensionSessionCallback(mock(
585                                         CameraExtensionSession.StateCallback.class));
586                         ExtensionSessionConfiguration configuration =
587                                 new ExtensionSessionConfiguration(extension, outputConfigs,
588                                         new HandlerExecutor(mTestRule.getHandler()),
589                                         sessionListener);
590                         configuration.setPostviewOutputConfiguration(postviewReaderOutput);
591                         assertNotNull(configuration.getPostviewOutputConfiguration());
592 
593                         String streamName = "test_extension_postview_capture";
594                         mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
595                         mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
596                         mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
597                                 ResultUnit.NONE);
598                         double[] captureTimes = new double[IMAGE_COUNT];
599                         double[] postviewCaptureTimes = new double[IMAGE_COUNT];
600                         boolean captureResultsSupported =
601                                 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
602 
603                         try {
604                             mTestRule.openDevice(id);
605                             CameraDevice camera = mTestRule.getCamera();
606                             camera.createExtensionSession(configuration);
607                             CameraExtensionSession extensionSession =
608                                     sessionListener.waitAndGetSession(
609                                             SESSION_CONFIGURE_TIMEOUT_MS);
610                             assertNotNull(extensionSession);
611 
612                             CaptureRequest.Builder captureBuilder =
613                                     mTestRule.getCamera().createCaptureRequest(
614                                             CameraDevice.TEMPLATE_STILL_CAPTURE);
615                             captureBuilder.addTarget(imageReaderSurface);
616                             captureBuilder.addTarget(postviewImageReaderSurface);
617                             CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
618                                     mock(CameraExtensionSession.ExtensionCaptureCallback.class);
619                             SimpleCaptureCallback captureCallback =
620                                     new SimpleCaptureCallback(extension, captureMockCallback,
621                                             extensionChars.getAvailableCaptureResultKeys(
622                                             extension), mCollector);
623 
624                             for (int i = 0; i < IMAGE_COUNT; i++) {
625                                 int jpegOrientation = (i * 90) % 360; // degrees [0..270]
626                                 if (captureFormat == ImageFormat.JPEG
627                                         || captureFormat == ImageFormat.JPEG_R
628                                         || postviewFormat == ImageFormat.JPEG
629                                         || postviewFormat == ImageFormat.JPEG_R) {
630                                     captureBuilder.set(CaptureRequest.JPEG_ORIENTATION,
631                                             jpegOrientation);
632                                 }
633                                 CaptureRequest request = captureBuilder.build();
634                                 long startTimeMs = SystemClock.elapsedRealtime();
635                                 captureCallback.resetCaptureProgress();
636                                 int sequenceId = extensionSession.capture(request,
637                                         new HandlerExecutor(mTestRule.getHandler()),
638                                         captureCallback);
639 
640                                 Image imgPostview  =
641                                         imageListenerPostview
642                                         .getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
643                                 postviewCaptureTimes[i] =
644                                         SystemClock.elapsedRealtime() - startTimeMs;
645 
646                                 if (postviewFormat == ImageFormat.JPEG
647                                         || postviewFormat == ImageFormat.JPEG_R) {
648                                     verifyJpegOrientation(imgPostview, postviewSize,
649                                             jpegOrientation, postviewFormat);
650                                 } else {
651                                     validateImage(imgPostview, postviewSize.getWidth(),
652                                             postviewSize.getHeight(), postviewFormat, null);
653                                 }
654 
655                                 Long imgTsPostview = imgPostview.getTimestamp();
656                                 imgPostview.close();
657 
658                                 Image img =
659                                         imageListener.getImage(
660                                         MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
661                                 captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
662 
663                                 if (captureFormat == ImageFormat.JPEG
664                                         || captureFormat == ImageFormat.JPEG_R) {
665                                     verifyJpegOrientation(img, maxSize,
666                                             jpegOrientation, captureFormat);
667                                 } else {
668                                     validateImage(img, maxSize.getWidth(),
669                                             maxSize.getHeight(), captureFormat, null);
670                                 }
671 
672                                 Long imgTs = img.getTimestamp();
673                                 img.close();
674 
675                                 assertEquals("Still capture timestamp does not match its "
676                                         + "postview timestamp", imgTsPostview, imgTs);
677 
678                                 verify(captureMockCallback,
679                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
680                                         .onCaptureStarted(eq(extensionSession), eq(request),
681                                         eq(imgTs));
682                                 verify(captureMockCallback, times(1))
683                                         .onCaptureStarted(eq(extensionSession), eq(request),
684                                         anyLong());
685                                 verify(captureMockCallback,
686                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
687                                         .onCaptureProcessStarted(extensionSession, request);
688                                 verify(captureMockCallback,
689                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
690                                         .onCaptureSequenceCompleted(extensionSession, sequenceId);
691                                 if (captureResultsSupported) {
692                                     verify(captureMockCallback,
693                                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
694                                             .onCaptureResultAvailable(eq(extensionSession),
695                                             eq(request), any(TotalCaptureResult.class));
696                                 }
697                                 if (captureProgressSupported) {
698                                     verify(captureMockCallback,
699                                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
700                                             .onCaptureProcessProgressed(eq(extensionSession),
701                                             eq(request), eq(100));
702                                 }
703                             }
704 
705                             mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
706                                     ResultUnit.NONE);
707                             mReportLog.addValue("height", maxSize.getHeight(),
708                                     ResultType.NEUTRAL, ResultUnit.NONE);
709                             mReportLog.addValue("captureFormat", captureFormat, ResultType.NEUTRAL,
710                                     ResultUnit.NONE);
711                             mReportLog.addValue("postviewFormat", postviewFormat,
712                                     ResultType.NEUTRAL, ResultUnit.NONE);
713                             long avgPostviewLatency = (long) Stat.getAverage(postviewCaptureTimes);
714                             mReportLog.addValue("avg_postview_latency", avgPostviewLatency,
715                                     ResultType.LOWER_BETTER, ResultUnit.MS);
716                             long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
717                             mReportLog.addValue("avg_capture_latency", avgCaptureLatency,
718                                     ResultType.LOWER_BETTER, ResultUnit.MS);
719 
720                             verify(captureMockCallback, times(0))
721                                     .onCaptureSequenceAborted(any(CameraExtensionSession.class),
722                                             anyInt());
723                             verify(captureMockCallback, times(0))
724                                     .onCaptureFailed(any(CameraExtensionSession.class),
725                                             any(CaptureRequest.class));
726                             verify(captureMockCallback, times(0))
727                                     .onCaptureFailed(any(CameraExtensionSession.class),
728                                             any(CaptureRequest.class), anyInt());
729                             Range<Long> latencyRange =
730                                     extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
731                                             maxSize, captureFormat);
732                             if (latencyRange != null) {
733                                 String msg = String.format("Camera [%s]: The measured average "
734                                         + "capture latency of %d ms. for extension type %d  "
735                                         + "with image format: %d and size: %dx%d must be "
736                                         + "within the advertised range of [%d, %d] ms.",
737                                         id, avgCaptureLatency, extension, captureFormat,
738                                         maxSize.getWidth(), maxSize.getHeight(),
739                                         latencyRange.getLower(), latencyRange.getUpper());
740                                 assertTrue(msg, latencyRange.contains(avgCaptureLatency));
741                             }
742 
743                             extensionSession.close();
744 
745                             sessionListener.getStateWaiter().waitForState(
746                                     BlockingExtensionSessionCallback.SESSION_CLOSED,
747                                     SESSION_CLOSE_TIMEOUT_MS);
748                         } finally {
749                             mTestRule.closeDevice(id);
750                             postviewImageReader.close();
751                             extensionImageReader.close();
752                             mReportLog.submit(InstrumentationRegistry.getInstrumentation());
753                         }
754                     }
755                 }
756             }
757         }
758     }
759 
760     @Test
761     @RequiresFlagsEnabled(Flags.FLAG_CAMERA_EXTENSIONS_CHARACTERISTICS_GET)
test10bitRepeatingAndCaptureCombined()762     public void test10bitRepeatingAndCaptureCombined() throws Exception {
763         final int IMAGE_COUNT = 5;
764         for (String id : getCameraIdsUnderTest()) {
765             StaticMetadata staticMeta =
766                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
767             if (!staticMeta.isColorOutputSupported()) {
768                 continue;
769             }
770             updatePreviewSurfaceTexture();
771             CameraExtensionCharacteristics extensionChars =
772                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
773             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
774             for (Integer extension : supportedExtensions) {
775                 boolean captureProgressSupported = extensionChars.isCaptureProcessProgressAvailable(
776                         extension);
777                 int captureFormat = ImageFormat.YCBCR_P010;
778                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
779                         captureFormat);
780                 if (extensionSizes.isEmpty()) {
781                     continue;
782                 }
783 
784                 int[] capabilities = extensionChars.get(extension,
785                         CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES);
786                 assertNotNull(capabilities);
787                 assertArrayContains("Supports YCBCR_P010 format but "
788                         + "REQUEST_AVAILABLE_CAPABILITIES does not contain "
789                         + "REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT", capabilities,
790                         CameraCharacteristics
791                         .REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT);
792 
793                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
794                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
795                         MAX_IMAGES);
796                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
797                         captureFormat, /*maxImages*/ 1, imageListener,
798                         mTestRule.getHandler());
799                 Surface imageReaderSurface = extensionImageReader.getSurface();
800                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
801 
802                 DynamicRangeProfiles dynamicRangeProfiles = extensionChars
803                         .get(extension,
804                         CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES);
805                 assertNotNull(dynamicRangeProfiles);
806                 assertTrue(dynamicRangeProfiles.getSupportedProfiles()
807                         .contains(DynamicRangeProfiles.HLG10));
808 
809                 // HLG10 is supported for all 10-bit capable devices
810                 readerOutput.setDynamicRangeProfile(DynamicRangeProfiles.HLG10);
811 
812                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
813                 outputConfigs.add(readerOutput);
814 
815                 // Pick a supported preview/repeating size with aspect ratio close to the
816                 // multi-frame capture size
817                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
818                         mSurfaceTexture.getClass());
819                 Size maxRepeatingSize =
820                         CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
821                 List<Size> previewSizes = getSupportedPreviewSizes(id,
822                         mTestRule.getCameraManager(),
823                         getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
824                 List<Size> supportedPreviewSizes =
825                         previewSizes.stream().filter(repeatingSizes::contains).collect(
826                                 Collectors.toList());
827                 if (!supportedPreviewSizes.isEmpty()) {
828                     float targetAr =
829                             ((float) maxSize.getWidth()) / maxSize.getHeight();
830                     for (Size s : supportedPreviewSizes) {
831                         float currentAr = ((float) s.getWidth()) / s.getHeight();
832                         if (Math.abs(targetAr - currentAr) < 0.01) {
833                             maxRepeatingSize = s;
834                             break;
835                         }
836                     }
837                 }
838 
839                 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
840                         maxRepeatingSize.getHeight());
841                 Surface texturedSurface = new Surface(mSurfaceTexture);
842                 OutputConfiguration previewOutput = new OutputConfiguration(texturedSurface);
843                 previewOutput.setDynamicRangeProfile(DynamicRangeProfiles.HLG10);
844                 outputConfigs.add(previewOutput);
845 
846                 BlockingExtensionSessionCallback sessionListener =
847                         new BlockingExtensionSessionCallback(mock(
848                                 CameraExtensionSession.StateCallback.class));
849                 ExtensionSessionConfiguration configuration =
850                         new ExtensionSessionConfiguration(extension, outputConfigs,
851                                 new HandlerExecutor(mTestRule.getHandler()),
852                                 sessionListener);
853 
854                 ColorSpaceProfiles colorSpaceProfiles = extensionChars
855                         .get(extension,
856                         CameraCharacteristics.REQUEST_AVAILABLE_COLOR_SPACE_PROFILES);
857                 if (colorSpaceProfiles != null) {
858                     Set<ColorSpace.Named> compatibleColorSpaces =
859                             colorSpaceProfiles.getSupportedColorSpacesForDynamicRange(
860                             captureFormat, DynamicRangeProfiles.HLG10);
861                     if (!compatibleColorSpaces.isEmpty()) {
862                         configuration.setColorSpace(compatibleColorSpaces.iterator().next());
863                     }
864                 }
865 
866                 String streamName = "test_extension_10_bit_repeating_and_capture";
867                 mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
868                 mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
869                 mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
870                         ResultUnit.NONE);
871                 double[] captureTimes = new double[IMAGE_COUNT];
872                 boolean captureResultsSupported =
873                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
874 
875                 try {
876                     mTestRule.openDevice(id);
877                     CameraDevice camera = mTestRule.getCamera();
878                     camera.createExtensionSession(configuration);
879                     CameraExtensionSession extensionSession =
880                             sessionListener.waitAndGetSession(
881                                     SESSION_CONFIGURE_TIMEOUT_MS);
882                     assertNotNull(extensionSession);
883 
884                     CaptureRequest.Builder captureBuilder =
885                             mTestRule.getCamera().createCaptureRequest(
886                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
887                     captureBuilder.addTarget(texturedSurface);
888                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
889                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
890                     SimpleCaptureCallback repeatingCaptureCallback =
891                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
892                                     extensionChars.getAvailableCaptureResultKeys(extension),
893                                     mCollector);
894 
895                     CaptureRequest repeatingRequest = captureBuilder.build();
896                     int repeatingSequenceId =
897                             extensionSession.setRepeatingRequest(repeatingRequest,
898                                     new HandlerExecutor(mTestRule.getHandler()),
899                                     repeatingCaptureCallback);
900 
901                     Thread.sleep(REPEATING_REQUEST_TIMEOUT_MS);
902 
903                     verify(repeatingCallbackMock, atLeastOnce())
904                             .onCaptureStarted(eq(extensionSession), eq(repeatingRequest),
905                                     anyLong());
906                     verify(repeatingCallbackMock, atLeastOnce())
907                             .onCaptureProcessStarted(extensionSession, repeatingRequest);
908                     if (captureResultsSupported) {
909                         verify(repeatingCallbackMock,
910                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
911                                 .onCaptureResultAvailable(eq(extensionSession),
912                                         eq(repeatingRequest), any(TotalCaptureResult.class));
913                     }
914                     verify(repeatingCallbackMock, times(0)).onCaptureProcessProgressed(
915                             any(CameraExtensionSession.class), any(CaptureRequest.class), anyInt());
916 
917                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
918                             CameraDevice.TEMPLATE_STILL_CAPTURE);
919                     captureBuilder.addTarget(imageReaderSurface);
920                     CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
921                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
922                     SimpleCaptureCallback captureCallback =
923                             new SimpleCaptureCallback(extension, captureMockCallback,
924                                     extensionChars.getAvailableCaptureResultKeys(extension),
925                                     mCollector);
926 
927                     for (int i = 0; i < IMAGE_COUNT; i++) {
928                         CaptureRequest request = captureBuilder.build();
929                         long startTimeMs = SystemClock.elapsedRealtime();
930                         captureCallback.resetCaptureProgress();
931                         int sequenceId = extensionSession.capture(request,
932                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
933 
934                         Image img =
935                                 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
936                         captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
937 
938                         validateImage(img, maxSize.getWidth(),
939                                 maxSize.getHeight(), captureFormat, null);
940 
941                         long imgTs = img.getTimestamp();
942                         img.close();
943 
944                         verify(captureMockCallback,
945                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
946                                 .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs));
947                         verify(captureMockCallback, times(1))
948                                 .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
949                         verify(captureMockCallback,
950                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
951                                 .onCaptureProcessStarted(extensionSession, request);
952                         verify(captureMockCallback,
953                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
954                                 .onCaptureSequenceCompleted(extensionSession, sequenceId);
955                         if (captureResultsSupported) {
956                             verify(captureMockCallback,
957                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
958                                     .onCaptureResultAvailable(eq(extensionSession), eq(request),
959                                     any(TotalCaptureResult.class));
960                         }
961                         if (captureProgressSupported) {
962                             verify(captureMockCallback,
963                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
964                                     .onCaptureProcessProgressed(eq(extensionSession),
965                                     eq(request), eq(100));
966                         }
967                     }
968 
969                     mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
970                             ResultUnit.NONE);
971                     mReportLog.addValue("height", maxSize.getHeight(),
972                             ResultType.NEUTRAL, ResultUnit.NONE);
973                     mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL,
974                             ResultUnit.NONE);
975                     long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
976                     mReportLog.addValue("avg_latency", avgCaptureLatency,
977                             ResultType.LOWER_BETTER, ResultUnit.MS);
978 
979                     verify(captureMockCallback, times(0))
980                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
981                                     anyInt());
982                     verify(captureMockCallback, times(0))
983                             .onCaptureFailed(any(CameraExtensionSession.class),
984                                     any(CaptureRequest.class));
985                     verify(captureMockCallback, times(0))
986                             .onCaptureFailed(any(CameraExtensionSession.class),
987                                     any(CaptureRequest.class), anyInt());
988                     Range<Long> latencyRange =
989                             extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
990                                     maxSize, captureFormat);
991                     if (latencyRange != null) {
992                         String msg = String.format("Camera [%s]: The measured average "
993                                         + "capture latency of %d ms. for extension type %d  "
994                                         + "with image format: %d and size: %dx%d must be "
995                                         + "within the advertised range of [%d, %d] ms.",
996                                 id, avgCaptureLatency, extension, captureFormat,
997                                 maxSize.getWidth(), maxSize.getHeight(),
998                                 latencyRange.getLower(), latencyRange.getUpper());
999                         assertTrue(msg, latencyRange.contains(avgCaptureLatency));
1000                     }
1001 
1002                     extensionSession.stopRepeating();
1003 
1004                     verify(repeatingCallbackMock,
1005                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1006                             .onCaptureSequenceCompleted(extensionSession, repeatingSequenceId);
1007                     verify(repeatingCallbackMock, times(0))
1008                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1009                             anyInt());
1010 
1011                     extensionSession.close();
1012 
1013                     sessionListener.getStateWaiter().waitForState(
1014                             BlockingExtensionSessionCallback.SESSION_CLOSED,
1015                             SESSION_CLOSE_TIMEOUT_MS);
1016                 } finally {
1017                     mTestRule.closeDevice(id);
1018                     extensionImageReader.close();
1019                     mReportLog.submit(InstrumentationRegistry.getInstrumentation());
1020                 }
1021             }
1022         }
1023     }
1024 
1025     // Test case to ensure if night mode indicator is supported then night mode camera extension
1026     // is also available.
1027     @Test
1028     @RequiresFlagsEnabled(Flags.FLAG_NIGHT_MODE_INDICATOR)
testNightModeIndicatorSupportedWithNightModeCameraExtension()1029     public void testNightModeIndicatorSupportedWithNightModeCameraExtension() throws Exception {
1030         for (String id : getCameraIdsUnderTest()) {
1031             StaticMetadata staticMeta =
1032                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1033             if (!staticMeta.isNightModeIndicatorSupported()) {
1034                 continue;
1035             }
1036             CameraExtensionCharacteristics extensionChars =
1037                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1038             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1039             assertTrue("Night Mode Camera Extension must be available if "
1040                           + "EXTENSION_NIGHT_MODE_INDICATOR is supported",
1041                           supportedExtensions.contains(
1042                               CameraExtensionCharacteristics.EXTENSION_NIGHT));
1043         }
1044     }
1045 
1046     // Test case to ensure night mode indicator is supported for both camera extension and camera
1047     // capture sessions.
1048     @Test
1049     @RequiresFlagsEnabled(Flags.FLAG_NIGHT_MODE_INDICATOR)
testNightModeIndicatorSupportedOnCameraCaptureAndCameraExtensionSession()1050     public void testNightModeIndicatorSupportedOnCameraCaptureAndCameraExtensionSession()
1051             throws Exception {
1052         for (String id : getCameraIdsUnderTest()) {
1053             StaticMetadata staticMeta =
1054                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1055             CameraExtensionCharacteristics extensionChars =
1056                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1057             if (!extensionChars.getSupportedExtensions().contains(
1058                     CameraExtensionCharacteristics.EXTENSION_NIGHT)) {
1059                 continue;
1060             }
1061             boolean isNightModeIndicatorSupported = staticMeta.isNightModeIndicatorSupported();
1062             boolean isNightModeIndicatorCameraExtensionSupported =
1063                     extensionChars.getAvailableCaptureResultKeys(
1064                         CameraExtensionCharacteristics.EXTENSION_NIGHT).contains(
1065                         CaptureResult.EXTENSION_NIGHT_MODE_INDICATOR);
1066             // If it's not supported in Camera2 and Camera Extensions then we can ignore
1067             // However, if it's supported in either Camera2 or Camera Extensions, then it must
1068             // be supported in both.
1069             assertEquals("EXTENSION_NIGHT_MODE_INDICATOR must be supported in both camera"
1070                           + "extension and camera capture sessions.", isNightModeIndicatorSupported,
1071                           isNightModeIndicatorCameraExtensionSupported);
1072         }
1073     }
1074 
1075 
1076     // Test case for multi-frame only capture on all supported extensions and expected state
1077     // callbacks. Verify still frame output, measure the average capture latency and if possible
1078     // ensure that the value is within the reported range.
1079     @Test
testMultiFrameCapture()1080     public void testMultiFrameCapture() throws Exception {
1081         final int IMAGE_COUNT = 10;
1082         int SUPPORTED_CAPTURE_OUTPUT_FORMATS[];
1083         if (Flags.depthJpegExtensions()) {
1084             SUPPORTED_CAPTURE_OUTPUT_FORMATS = new int[] {
1085                     ImageFormat.YUV_420_888,
1086                     ImageFormat.JPEG,
1087                     ImageFormat.JPEG_R,
1088                     ImageFormat.DEPTH_JPEG
1089             };
1090         } else {
1091             SUPPORTED_CAPTURE_OUTPUT_FORMATS = new int[] {
1092                     ImageFormat.YUV_420_888,
1093                     ImageFormat.JPEG,
1094                     ImageFormat.JPEG_R
1095             };
1096         }
1097         for (String id : getCameraIdsUnderTest()) {
1098             StaticMetadata staticMeta =
1099                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1100             if (!staticMeta.isColorOutputSupported()) {
1101                 continue;
1102             }
1103             updatePreviewSurfaceTexture();
1104             CameraExtensionCharacteristics extensionChars =
1105                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1106             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1107             for (Integer extension : supportedExtensions) {
1108                 boolean captureProgressSupported = extensionChars.isCaptureProcessProgressAvailable(
1109                         extension);
1110                 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
1111                     List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
1112                             captureFormat);
1113                     if (extensionSizes.isEmpty()) {
1114                         continue;
1115                     }
1116                     Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
1117                     SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
1118                             MAX_IMAGES);
1119                     ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
1120                             captureFormat, /*maxImages*/ 1, imageListener,
1121                             mTestRule.getHandler());
1122                     Surface imageReaderSurface = extensionImageReader.getSurface();
1123                     OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1124                     List<OutputConfiguration> outputConfigs = new ArrayList<>();
1125                     outputConfigs.add(readerOutput);
1126 
1127                     BlockingExtensionSessionCallback sessionListener =
1128                             new BlockingExtensionSessionCallback(mock(
1129                                     CameraExtensionSession.StateCallback.class));
1130                     ExtensionSessionConfiguration configuration =
1131                             new ExtensionSessionConfiguration(extension, outputConfigs,
1132                                     new HandlerExecutor(mTestRule.getHandler()),
1133                                     sessionListener);
1134                     String streamName = "test_extension_capture";
1135                     mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
1136                     mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
1137                     mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
1138                             ResultUnit.NONE);
1139                     double[] captureTimes = new double[IMAGE_COUNT];
1140                     boolean captureResultsSupported =
1141                             !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
1142 
1143                     try {
1144                         mTestRule.openDevice(id);
1145                         CameraDevice camera = mTestRule.getCamera();
1146                         camera.createExtensionSession(configuration);
1147                         CameraExtensionSession extensionSession =
1148                                 sessionListener.waitAndGetSession(
1149                                         SESSION_CONFIGURE_TIMEOUT_MS);
1150                         assertNotNull(extensionSession);
1151 
1152                         CaptureRequest.Builder captureBuilder =
1153                                 mTestRule.getCamera().createCaptureRequest(
1154                                         CameraDevice.TEMPLATE_STILL_CAPTURE);
1155                         captureBuilder.addTarget(imageReaderSurface);
1156                         CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
1157                                 mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1158                         SimpleCaptureCallback captureCallback =
1159                                 new SimpleCaptureCallback(extension, captureMockCallback,
1160                                         extensionChars.getAvailableCaptureResultKeys(extension),
1161                                         mCollector);
1162 
1163                         for (int i = 0; i < IMAGE_COUNT; i++) {
1164                             int jpegOrientation = (i * 90) % 360; // degrees [0..270]
1165                             if (captureFormat == ImageFormat.JPEG
1166                                     || captureFormat == ImageFormat.JPEG_R
1167                                     || captureFormat == ImageFormat.DEPTH_JPEG) {
1168                                 captureBuilder.set(CaptureRequest.JPEG_ORIENTATION,
1169                                         jpegOrientation);
1170                             }
1171                             CaptureRequest request = captureBuilder.build();
1172                             long startTimeMs = SystemClock.elapsedRealtime();
1173                             captureCallback.resetCaptureProgress();
1174                             int sequenceId = extensionSession.capture(request,
1175                                     new HandlerExecutor(mTestRule.getHandler()), captureCallback);
1176 
1177                             Image img =
1178                                     imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
1179                             captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
1180 
1181                             if (captureFormat == ImageFormat.JPEG
1182                                     || captureFormat == ImageFormat.JPEG_R
1183                                     || captureFormat == ImageFormat.DEPTH_JPEG) {
1184                                 verifyJpegOrientation(img, maxSize,
1185                                         jpegOrientation, captureFormat);
1186                             } else {
1187                                 validateImage(img, maxSize.getWidth(),
1188                                         maxSize.getHeight(), captureFormat, null);
1189                             }
1190 
1191                             Long imgTs = img.getTimestamp();
1192                             img.close();
1193 
1194                             verify(captureMockCallback,
1195                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1196                                     .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs));
1197                             verify(captureMockCallback, times(1))
1198                                     .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
1199                             verify(captureMockCallback,
1200                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1201                                     .onCaptureProcessStarted(extensionSession, request);
1202                             verify(captureMockCallback,
1203                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1204                                     .onCaptureSequenceCompleted(extensionSession, sequenceId);
1205                             if (captureResultsSupported) {
1206                                 verify(captureMockCallback,
1207                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1208                                         .onCaptureResultAvailable(eq(extensionSession), eq(request),
1209                                                 any(TotalCaptureResult.class));
1210                             }
1211                             if (captureProgressSupported) {
1212                                 verify(captureMockCallback,
1213                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1214                                     .onCaptureProcessProgressed(eq(extensionSession),
1215                                             eq(request), eq(100));
1216                             }
1217                         }
1218 
1219                         mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
1220                                 ResultUnit.NONE);
1221                         mReportLog.addValue("height", maxSize.getHeight(),
1222                                 ResultType.NEUTRAL, ResultUnit.NONE);
1223                         mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL,
1224                                 ResultUnit.NONE);
1225                         long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
1226                         mReportLog.addValue("avg_latency", avgCaptureLatency,
1227                                 ResultType.LOWER_BETTER, ResultUnit.MS);
1228 
1229                         verify(captureMockCallback, times(0))
1230                                 .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1231                                         anyInt());
1232                         verify(captureMockCallback, times(0))
1233                                 .onCaptureFailed(any(CameraExtensionSession.class),
1234                                         any(CaptureRequest.class));
1235                         verify(captureMockCallback, times(0))
1236                                 .onCaptureFailed(any(CameraExtensionSession.class),
1237                                         any(CaptureRequest.class), anyInt());
1238                         Range<Long> latencyRange =
1239                                 extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
1240                                         maxSize, captureFormat);
1241                         if (latencyRange != null) {
1242                             String msg = String.format("Camera [%s]: The measured average "
1243                                             + "capture latency of %d ms. for extension type %d  "
1244                                             + "with image format: %d and size: %dx%d must be "
1245                                             + "within the advertised range of [%d, %d] ms.",
1246                                     id, avgCaptureLatency, extension, captureFormat,
1247                                     maxSize.getWidth(), maxSize.getHeight(),
1248                                     latencyRange.getLower(), latencyRange.getUpper());
1249                             assertTrue(msg, latencyRange.contains(avgCaptureLatency));
1250                         }
1251 
1252                         extensionSession.close();
1253 
1254                         sessionListener.getStateWaiter().waitForState(
1255                                 BlockingExtensionSessionCallback.SESSION_CLOSED,
1256                                 SESSION_CLOSE_TIMEOUT_MS);
1257                     } finally {
1258                         mTestRule.closeDevice(id);
1259                         extensionImageReader.close();
1260                         mReportLog.submit(InstrumentationRegistry.getInstrumentation());
1261                     }
1262                 }
1263             }
1264         }
1265     }
1266 
1267     // Verify concurrent extension sessions behavior
1268     @Test
testConcurrentSessions()1269     public void testConcurrentSessions() throws Exception {
1270         Set<Set<String>> concurrentCameraIdSet =
1271                 mTestRule.getCameraManager().getConcurrentCameraIds();
1272         if (concurrentCameraIdSet.isEmpty()) {
1273             return;
1274         }
1275 
1276         for (String id : getCameraIdsUnderTest()) {
1277             StaticMetadata staticMeta =
1278                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1279             if (!staticMeta.isColorOutputSupported()) {
1280                 continue;
1281             }
1282             CameraExtensionCharacteristics extensionChars =
1283                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1284             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1285             if (supportedExtensions.isEmpty()) {
1286                 continue;
1287             }
1288 
1289             Set<String> concurrentCameraIds = null;
1290             for (Set<String> entry : concurrentCameraIdSet) {
1291                 if (entry.contains(id)) {
1292                     concurrentCameraIds = entry;
1293                     break;
1294                 }
1295             }
1296             if (concurrentCameraIds == null) {
1297                 continue;
1298             }
1299 
1300             String concurrentCameraId = null;
1301             CameraExtensionCharacteristics concurrentExtensionChars = null;
1302             for (String entry : concurrentCameraIds) {
1303                 if (entry.equals(id)) {
1304                     continue;
1305                 }
1306                 if (!(new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(
1307                         entry))).isColorOutputSupported()) {
1308                     continue;
1309                 }
1310                 CameraExtensionCharacteristics chars =
1311                         mTestRule.getCameraManager().getCameraExtensionCharacteristics(entry);
1312                 if (chars.getSupportedExtensions().isEmpty()) {
1313                     continue;
1314                 }
1315                 concurrentCameraId = entry;
1316                 concurrentExtensionChars = chars;
1317                 break;
1318             }
1319             if ((concurrentCameraId == null) || (concurrentExtensionChars == null)) {
1320                 continue;
1321             }
1322 
1323             updatePreviewSurfaceTexture();
1324             int extensionId = supportedExtensions.get(0);
1325             List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extensionId,
1326                     mSurfaceTexture.getClass());
1327             Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
1328             mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
1329             OutputConfiguration outputConfig = new OutputConfiguration(
1330                     new Surface(mSurfaceTexture));
1331             List<OutputConfiguration> outputConfigs = new ArrayList<>();
1332             outputConfigs.add(outputConfig);
1333 
1334             BlockingExtensionSessionCallback sessionListener =
1335                     new BlockingExtensionSessionCallback(
1336                             mock(CameraExtensionSession.StateCallback.class));
1337             ExtensionSessionConfiguration configuration =
1338                     new ExtensionSessionConfiguration(extensionId, outputConfigs,
1339                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
1340 
1341             CameraDevice concurrentCameraDevice = null;
1342             ImageReader extensionImageReader = null;
1343             try {
1344                 mTestRule.openDevice(id);
1345                 concurrentCameraDevice = CameraTestUtils.openCamera(mTestRule.getCameraManager(),
1346                         concurrentCameraId, new BlockingStateCallback(), mTestRule.getHandler());
1347                 CameraDevice camera = mTestRule.getCamera();
1348                 camera.createExtensionSession(configuration);
1349                 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
1350                         SESSION_CONFIGURE_TIMEOUT_MS);
1351                 assertNotNull(extensionSession);
1352 
1353                 assertNotNull(concurrentCameraDevice);
1354                 int concurrentExtensionId =
1355                         concurrentExtensionChars.getSupportedExtensions().get(0);
1356                 List<Size> captureSizes = concurrentExtensionChars.getExtensionSupportedSizes(
1357                         concurrentExtensionId, mSurfaceTexture.getClass());
1358                 assertFalse("No SurfaceTexture output supported", captureSizes.isEmpty());
1359                 Size captureMaxSize =
1360                         CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
1361 
1362                 extensionImageReader = ImageReader.newInstance(
1363                         captureMaxSize.getWidth(), captureMaxSize.getHeight(), ImageFormat.PRIVATE,
1364                         /*maxImages*/ 1, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
1365                 Surface imageReaderSurface = extensionImageReader.getSurface();
1366                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1367                 outputConfigs = new ArrayList<>();
1368                 outputConfigs.add(readerOutput);
1369                 CameraExtensionSession.StateCallback mockSessionListener =
1370                         mock(CameraExtensionSession.StateCallback.class);
1371                 ExtensionSessionConfiguration concurrentConfiguration =
1372                         new ExtensionSessionConfiguration(concurrentExtensionId, outputConfigs,
1373                                 new HandlerExecutor(mTestRule.getHandler()),
1374                                 mockSessionListener);
1375                 concurrentCameraDevice.createExtensionSession(concurrentConfiguration);
1376                 // Trying to initialize multiple concurrent extension sessions is expected to fail
1377                 verify(mockSessionListener, timeout(SESSION_CONFIGURE_TIMEOUT_MS).times(1))
1378                         .onConfigureFailed(any(CameraExtensionSession.class));
1379                 verify(mockSessionListener, times(0)).onConfigured(
1380                         any(CameraExtensionSession.class));
1381 
1382                 extensionSession.close();
1383                 sessionListener.getStateWaiter().waitForState(
1384                         BlockingExtensionSessionCallback.SESSION_CLOSED,
1385                         SESSION_CLOSE_TIMEOUT_MS);
1386 
1387                 // Initialization of another extension session must now be possible
1388                 BlockingExtensionSessionCallback concurrentSessionListener =
1389                         new BlockingExtensionSessionCallback(
1390                                 mock(CameraExtensionSession.StateCallback.class));
1391                 concurrentConfiguration = new ExtensionSessionConfiguration(concurrentExtensionId,
1392                         outputConfigs, new HandlerExecutor(mTestRule.getHandler()),
1393                         concurrentSessionListener);
1394                 concurrentCameraDevice.createExtensionSession(concurrentConfiguration);
1395                 extensionSession = concurrentSessionListener.waitAndGetSession(
1396                         SESSION_CONFIGURE_TIMEOUT_MS);
1397                 assertNotNull(extensionSession);
1398                 extensionSession.close();
1399                 concurrentSessionListener.getStateWaiter().waitForState(
1400                         BlockingExtensionSessionCallback.SESSION_CLOSED,
1401                         SESSION_CLOSE_TIMEOUT_MS);
1402             } finally {
1403                 mTestRule.closeDevice(id);
1404                 if (concurrentCameraDevice != null) {
1405                     concurrentCameraDevice.close();
1406                 }
1407                 if (extensionImageReader != null) {
1408                     extensionImageReader.close();
1409                 }
1410             }
1411         }
1412     }
1413 
1414     // Test case combined repeating with multi frame capture on all supported extensions.
1415     // Verify still frame output.
1416     @Test
testRepeatingAndCaptureCombined()1417     public void testRepeatingAndCaptureCombined() throws Exception {
1418         final double LATENCY_MARGIN = .3f; // Account for system load, capture call duration etc.
1419         for (String id : getCameraIdsUnderTest()) {
1420             StaticMetadata staticMeta =
1421                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1422             if (!staticMeta.isColorOutputSupported()) {
1423                 continue;
1424             }
1425             updatePreviewSurfaceTexture();
1426             CameraExtensionCharacteristics extensionChars =
1427                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1428             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1429             for (Integer extension : supportedExtensions) {
1430 
1431                 Set<CaptureRequest.Key> supportedRequestKeys =
1432                         extensionChars.getAvailableCaptureRequestKeys(extension);
1433                 boolean supportsStrengthControl = supportedRequestKeys.contains(
1434                         CaptureRequest.EXTENSION_STRENGTH);
1435                 Set<CaptureResult.Key> supportedResultKeys =
1436                         extensionChars.getAvailableCaptureResultKeys(extension);
1437                 boolean supportsAFControlState = supportedResultKeys.contains(
1438                         CaptureResult.CONTROL_AF_STATE);
1439                 boolean supportsCAFMode = false;
1440                 if (supportedRequestKeys.contains(CaptureRequest.CONTROL_AF_MODE) &&
1441                         staticMeta.hasFocuser()) {
1442                     int[] afModes = extensionChars.get(extension,
1443                             CameraCharacteristics.CONTROL_AF_AVAILABLE_MODES);
1444                     if (afModes == null) {
1445                         afModes = staticMeta.getAfAvailableModesChecked();
1446                     }
1447                     supportsCAFMode = Arrays.stream(afModes).anyMatch(
1448                             mode -> mode == CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
1449                 }
1450                 if (supportsStrengthControl) {
1451                     assertTrue(supportedResultKeys.contains(CaptureResult.EXTENSION_STRENGTH));
1452                 }
1453 
1454                 int captureFormat = ImageFormat.JPEG;
1455                 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
1456                         captureFormat);
1457                 assertFalse("No Jpeg output supported", captureSizes.isEmpty());
1458                 Size captureMaxSize =
1459                         CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
1460 
1461                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false
1462                         , MAX_IMAGES);
1463                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
1464                         captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener,
1465                         mTestRule.getHandler());
1466                 Surface imageReaderSurface = extensionImageReader.getSurface();
1467                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1468                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
1469                 outputConfigs.add(readerOutput);
1470 
1471                 // Pick a supported preview/repeating size with aspect ratio close to the
1472                 // multi-frame capture size
1473                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
1474                         mSurfaceTexture.getClass());
1475                 Size maxRepeatingSize =
1476                         CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
1477                 List<Size> previewSizes = getSupportedPreviewSizes(id,
1478                         mTestRule.getCameraManager(),
1479                         getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
1480                 List<Size> supportedPreviewSizes =
1481                         previewSizes.stream().filter(repeatingSizes::contains).collect(
1482                                 Collectors.toList());
1483                 if (!supportedPreviewSizes.isEmpty()) {
1484                     float targetAr =
1485                             ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight();
1486                     for (Size s : supportedPreviewSizes) {
1487                         float currentAr = ((float) s.getWidth()) / s.getHeight();
1488                         if (Math.abs(targetAr - currentAr) < 0.01) {
1489                             maxRepeatingSize = s;
1490                             break;
1491                         }
1492                     }
1493                 }
1494 
1495                 boolean captureResultsSupported =
1496                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
1497 
1498                 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
1499                         maxRepeatingSize.getHeight());
1500                 Surface texturedSurface = new Surface(mSurfaceTexture);
1501                 outputConfigs.add(new OutputConfiguration(texturedSurface));
1502 
1503                 BlockingExtensionSessionCallback sessionListener =
1504                         new BlockingExtensionSessionCallback(mock(
1505                                 CameraExtensionSession.StateCallback.class));
1506                 ExtensionSessionConfiguration configuration =
1507                         new ExtensionSessionConfiguration(extension, outputConfigs,
1508                                 new HandlerExecutor(mTestRule.getHandler()),
1509                                 sessionListener);
1510                 try {
1511                     mTestRule.openDevice(id);
1512                     CameraDevice camera = mTestRule.getCamera();
1513                     camera.createExtensionSession(configuration);
1514                     CameraExtensionSession extensionSession =
1515                             sessionListener.waitAndGetSession(
1516                                     SESSION_CONFIGURE_TIMEOUT_MS);
1517                     assertNotNull(extensionSession);
1518 
1519                     CaptureRequest.Builder captureBuilder =
1520                             mTestRule.getCamera().createCaptureRequest(
1521                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
1522                     captureBuilder.addTarget(texturedSurface);
1523                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
1524                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1525 
1526                     AutoFocusStateListener mockAFListener = mock(AutoFocusStateListener.class);
1527                     AutoFocusStateMachine afState = null;
1528                     if (supportsCAFMode && supportsAFControlState) {
1529                         // Check passive AF
1530                         afState = new AutoFocusStateMachine(
1531                                 new TestAutoFocusProxy(mockAFListener));
1532                         afState.setPassiveAutoFocus(true /*picture*/, captureBuilder);
1533                     }
1534                     SimpleCaptureCallback repeatingCaptureCallback =
1535                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
1536                                     extensionChars.getAvailableCaptureResultKeys(extension),
1537                                     mCollector, afState, null /*flashState*/, null /*aeState*/,
1538                                     false);
1539 
1540                     if (supportsStrengthControl) {
1541                         captureBuilder.set(CaptureRequest.EXTENSION_STRENGTH, 100);
1542                     }
1543 
1544                     CaptureRequest repeatingRequest = captureBuilder.build();
1545                     if (supportsCAFMode && supportsAFControlState) {
1546                         captureBuilder.set(
1547                                 CaptureRequest.CONTROL_AF_MODE,
1548                                 CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE
1549                         );
1550                     }
1551                     int repeatingSequenceId =
1552                             extensionSession.setRepeatingRequest(repeatingRequest,
1553                                     new HandlerExecutor(mTestRule.getHandler()),
1554                                     repeatingCaptureCallback);
1555 
1556                     Thread.sleep(REPEATING_REQUEST_TIMEOUT_MS);
1557 
1558                     verify(repeatingCallbackMock, atLeastOnce())
1559                             .onCaptureStarted(eq(extensionSession), eq(repeatingRequest),
1560                                     anyLong());
1561                     verify(repeatingCallbackMock, atLeastOnce())
1562                             .onCaptureProcessStarted(extensionSession, repeatingRequest);
1563                     if (captureResultsSupported) {
1564                         verify(repeatingCallbackMock,
1565                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
1566                                 .onCaptureResultAvailable(eq(extensionSession),
1567                                         eq(repeatingRequest), any(TotalCaptureResult.class));
1568                     }
1569                     verify(repeatingCallbackMock, times(0)).onCaptureProcessProgressed(
1570                             any(CameraExtensionSession.class), any(CaptureRequest.class), anyInt());
1571 
1572                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
1573                             android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE);
1574                     captureBuilder.addTarget(imageReaderSurface);
1575                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
1576                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1577 
1578                     CameraExtensionSession.StillCaptureLatency stillCaptureLatency =
1579                             extensionSession.getRealtimeStillCaptureLatency();
1580                     CaptureRequest captureRequest = captureBuilder.build();
1581                     if (supportsCAFMode && supportsAFControlState) {
1582                         verify(mockAFListener,
1583                                 timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS).atLeastOnce())
1584                                 .onDone(anyBoolean());
1585                     }
1586                     long startTimeMs = SystemClock.elapsedRealtime();
1587                     int captureSequenceId = extensionSession.capture(captureRequest,
1588                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
1589 
1590                     Image img =
1591                             imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
1592                     long captureTime = SystemClock.elapsedRealtime() - startTimeMs;
1593                     validateImage(img, captureMaxSize.getWidth(),
1594                             captureMaxSize.getHeight(), captureFormat, null);
1595                     Long imgTs = img.getTimestamp();
1596                     img.close();
1597 
1598                     if (stillCaptureLatency != null) {
1599                         assertTrue("Still capture frame latency must be positive!",
1600                                 stillCaptureLatency.getCaptureLatency() > 0);
1601                         assertTrue("Processing capture latency must be non-negative!",
1602                                 stillCaptureLatency.getProcessingLatency() >= 0);
1603                         long estimatedTotalLatency = stillCaptureLatency.getCaptureLatency() +
1604                                 stillCaptureLatency.getProcessingLatency();
1605                         long estimatedTotalLatencyMin =
1606                                 (long) (estimatedTotalLatency * (1.f - LATENCY_MARGIN));
1607                         long estimatedTotalLatencyMax =
1608                                 (long) (estimatedTotalLatency * (1.f + LATENCY_MARGIN));
1609                         assertTrue(String.format("Camera %s: Measured still capture latency " +
1610                                                 "doesn't match: %d ms, expected [%d,%d]ms.", id,
1611                                         captureTime, estimatedTotalLatencyMin,
1612                                         estimatedTotalLatencyMax),
1613                                 (captureTime <= estimatedTotalLatencyMax) &&
1614                                         (captureTime >= estimatedTotalLatencyMin));
1615                     }
1616 
1617                     verify(captureCallback, times(1))
1618                             .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs));
1619                     verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1620                             .onCaptureProcessStarted(extensionSession, captureRequest);
1621                     if (captureResultsSupported) {
1622                         verify(captureCallback,
1623                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1624                                 .onCaptureResultAvailable(eq(extensionSession),
1625                                         eq(captureRequest), any(TotalCaptureResult.class));
1626                     }
1627                     verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1628                             .onCaptureSequenceCompleted(extensionSession,
1629                                     captureSequenceId);
1630                     verify(captureCallback, times(0))
1631                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1632                                     anyInt());
1633                     verify(captureCallback, times(0))
1634                             .onCaptureFailed(any(CameraExtensionSession.class),
1635                                     any(CaptureRequest.class));
1636                     verify(captureCallback, times(0))
1637                             .onCaptureFailed(any(CameraExtensionSession.class),
1638                                     any(CaptureRequest.class), anyInt());
1639 
1640                     extensionSession.stopRepeating();
1641 
1642                     verify(repeatingCallbackMock,
1643                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1644                             .onCaptureSequenceCompleted(extensionSession, repeatingSequenceId);
1645 
1646                     verify(repeatingCallbackMock, times(0))
1647                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1648                                     anyInt());
1649 
1650                     extensionSession.close();
1651 
1652                     sessionListener.getStateWaiter().waitForState(
1653                             BlockingExtensionSessionCallback.SESSION_CLOSED,
1654                             SESSION_CLOSE_TIMEOUT_MS);
1655 
1656                     assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" +
1657                                     " callbacks must be greater or equal than the number of calls" +
1658                                     " to onCaptureStarted!",
1659                             repeatingCaptureCallback.getTotalFramesArrived() +
1660                                     repeatingCaptureCallback.getTotalFramesFailed() >=
1661                             repeatingCaptureCallback.getTotalFramesStarted());
1662                     assertTrue(String.format("The last repeating request surface timestamp " +
1663                                     "%d must be less than or equal to the last " +
1664                                     "onCaptureStarted " +
1665                                     "timestamp %d", mSurfaceTexture.getTimestamp(),
1666                             repeatingCaptureCallback.getLastTimestamp()),
1667                             mSurfaceTexture.getTimestamp() <=
1668                                     repeatingCaptureCallback.getLastTimestamp());
1669 
1670                 } finally {
1671                     mTestRule.closeDevice(id);
1672                     texturedSurface.release();
1673                     extensionImageReader.close();
1674                 }
1675             }
1676         }
1677     }
1678 
verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation, int captureFormat)1679     private void verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation,
1680                 int captureFormat)
1681             throws IOException {
1682         byte[] blobBuffer = getDataFromImage(img);
1683         String blobFilename = mTestRule.getDebugFileNameBase() + "/verifyJpegKeys.jpeg";
1684         dumpFile(blobFilename, blobBuffer);
1685         ExifInterface exif = new ExifInterface(blobFilename);
1686         int exifWidth = exif.getAttributeInt(ExifInterface.TAG_IMAGE_WIDTH, /*defaultValue*/0);
1687         int exifHeight = exif.getAttributeInt(ExifInterface.TAG_IMAGE_LENGTH, /*defaultValue*/0);
1688         Size exifSize = new Size(exifWidth, exifHeight);
1689         int exifOrientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION,
1690                 /*defaultValue*/ ExifInterface.ORIENTATION_UNDEFINED);
1691         final int ORIENTATION_MIN = ExifInterface.ORIENTATION_UNDEFINED;
1692         final int ORIENTATION_MAX = ExifInterface.ORIENTATION_ROTATE_270;
1693         assertTrue(String.format("Exif orientation must be in range of [%d, %d]",
1694                 ORIENTATION_MIN, ORIENTATION_MAX),
1695                 exifOrientation >= ORIENTATION_MIN && exifOrientation <= ORIENTATION_MAX);
1696 
1697         /**
1698          * Device captured image doesn't respect the requested orientation,
1699          * which means it rotates the image buffer physically. Then we
1700          * should swap the jpegSize width/height accordingly to compare.
1701          */
1702         boolean deviceRotatedImage = exifOrientation == ExifInterface.ORIENTATION_UNDEFINED;
1703 
1704         if (deviceRotatedImage) {
1705             // Case 1.
1706             boolean needSwap = (requestedOrientation % 180 == 90);
1707             if (needSwap) {
1708                 jpegSize = new Size(jpegSize.getHeight(), jpegSize.getWidth());
1709             }
1710         } else {
1711             // Case 2.
1712             assertEquals("Exif orientation should match requested orientation",
1713                     requestedOrientation, getExifOrientationInDegree(exifOrientation));
1714         }
1715 
1716         assertEquals("Exif size should match jpeg capture size", jpegSize, exifSize);
1717 
1718         byte[] data = getDataFromImage(img);
1719         assertTrue("Invalid image data", data != null && data.length > 0);
1720 
1721         switch (captureFormat) {
1722             case ImageFormat.DEPTH_JPEG:
1723                 assertTrue("Dynamic depth validation failed!",
1724                         validateDynamicDepthNative(data));
1725             case ImageFormat.JPEG:
1726                 validateJpegData(data, jpegSize.getWidth(), jpegSize.getHeight(),
1727                         null /*filePath*/);
1728                 break;
1729             case ImageFormat.JPEG_R:
1730                 validateJpegData(data, jpegSize.getWidth(), jpegSize.getHeight(),
1731                         null /*filePath*/, null /*colorSpace*/, true /*gainMapPresent*/);
1732                 break;
1733             default:
1734                 throw new UnsupportedOperationException("Unsupported format for JPEG validation: "
1735                         + captureFormat);
1736         }
1737     }
1738 
getExifOrientationInDegree(int exifOrientation)1739     private static int getExifOrientationInDegree(int exifOrientation) {
1740         switch (exifOrientation) {
1741             case ExifInterface.ORIENTATION_NORMAL:
1742                 return 0;
1743             case ExifInterface.ORIENTATION_ROTATE_90:
1744                 return 90;
1745             case ExifInterface.ORIENTATION_ROTATE_180:
1746                 return 180;
1747             case ExifInterface.ORIENTATION_ROTATE_270:
1748                 return 270;
1749             default:
1750                 fail("It is impossible to get non 0, 90, 180, 270 degress exif" +
1751                         "info based on the request orientation range");
1752                 return -1;
1753         }
1754     }
1755 
1756     public static class SimpleCaptureCallback
1757             extends CameraExtensionSession.ExtensionCaptureCallback {
1758         private long mLastTimestamp = -1;
1759         private int mNumFramesArrived = 0;
1760         private int mNumFramesStarted = 0;
1761         private int mNumFramesFailed = 0;
1762         private int mLastProgressValue = -1;
1763         private boolean mNonIncreasingTimestamps = false;
1764         private HashSet<Long> mExpectedResultTimestamps = new HashSet<>();
1765         private StaticMetadata mStaticInfo;
1766         private final CameraExtensionSession.ExtensionCaptureCallback mProxy;
1767         private final AutoFocusStateMachine mAFStateMachine;
1768         private final FlashStateListener mFlashStateListener;
1769         private final AutoExposureStateListener mAEStateListener;
1770         private final HashSet<CaptureResult.Key> mSupportedResultKeys;
1771         private final CameraErrorCollector mCollector;
1772         private final boolean mPerFrameControl;
1773         private final int mExtensionType;
1774 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector)1775         public SimpleCaptureCallback(int extensionType,
1776                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1777                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector) {
1778             this(extensionType, proxy, supportedResultKeys, errorCollector, null /*afListener*/,
1779                     null /*flashState*/, null /*aeState*/, false /*perFrameControl*/);
1780         }
1781 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, StaticMetadata staticInfo)1782         public SimpleCaptureCallback(int extensionType,
1783                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1784                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector,
1785                 StaticMetadata staticInfo) {
1786             this(extensionType, proxy, supportedResultKeys, errorCollector, null /*afListener*/,
1787                     null /*flashState*/, null /*aeState*/, false /*perFrameControl*/);
1788             mStaticInfo = staticInfo;
1789         }
1790 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, AutoFocusStateMachine afState, FlashStateListener flashState, AutoExposureStateListener aeState, boolean perFrameControl)1791         public SimpleCaptureCallback(int extensionType,
1792                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1793                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector,
1794                 AutoFocusStateMachine afState, FlashStateListener flashState,
1795                 AutoExposureStateListener aeState, boolean perFrameControl) {
1796             mProxy = proxy;
1797             mSupportedResultKeys = new HashSet<>(supportedResultKeys);
1798             mCollector = errorCollector;
1799             mAFStateMachine = afState;
1800             mFlashStateListener = flashState;
1801             mAEStateListener = aeState;
1802             mPerFrameControl = perFrameControl;
1803             mExtensionType = extensionType;
1804         }
1805 
resetCaptureProgress()1806         public void resetCaptureProgress() {
1807             mLastProgressValue = -1;
1808         }
1809 
1810         @Override
onCaptureStarted(CameraExtensionSession session, CaptureRequest request, long timestamp)1811         public void onCaptureStarted(CameraExtensionSession session,
1812                                      CaptureRequest request, long timestamp) {
1813             mExpectedResultTimestamps.add(timestamp);
1814             if (timestamp < mLastTimestamp) {
1815                 mNonIncreasingTimestamps = true;
1816             }
1817             mLastTimestamp = timestamp;
1818             mNumFramesStarted++;
1819             if (mProxy != null) {
1820                 mProxy.onCaptureStarted(session, request, timestamp);
1821             }
1822         }
1823 
1824         @Override
onCaptureProcessStarted(CameraExtensionSession session, CaptureRequest request)1825         public void onCaptureProcessStarted(CameraExtensionSession session,
1826                                             CaptureRequest request) {
1827             mNumFramesArrived++;
1828             if (mProxy != null) {
1829                 mProxy.onCaptureProcessStarted(session, request);
1830             }
1831         }
1832 
1833         @Override
onCaptureFailed(CameraExtensionSession session, CaptureRequest request)1834         public void onCaptureFailed(CameraExtensionSession session,
1835                                     CaptureRequest request) {
1836             mNumFramesFailed++;
1837             if (mProxy != null) {
1838                 mProxy.onCaptureFailed(session, request);
1839             }
1840         }
1841 
1842         @Override
onCaptureFailed(CameraExtensionSession session, CaptureRequest request, int failure)1843         public void onCaptureFailed(CameraExtensionSession session,
1844                 CaptureRequest request, int failure) {
1845             mNumFramesFailed++;
1846             if (mProxy != null) {
1847                 mProxy.onCaptureFailed(session, request, failure);
1848             }
1849         }
1850 
1851         @Override
onCaptureSequenceAborted(CameraExtensionSession session, int sequenceId)1852         public void onCaptureSequenceAborted(CameraExtensionSession session,
1853                                              int sequenceId) {
1854             if (mProxy != null) {
1855                 mProxy.onCaptureSequenceAborted(session, sequenceId);
1856             }
1857         }
1858 
1859         @Override
onCaptureSequenceCompleted(CameraExtensionSession session, int sequenceId)1860         public void onCaptureSequenceCompleted(CameraExtensionSession session,
1861                                                int sequenceId) {
1862             if (mProxy != null) {
1863                 mProxy.onCaptureSequenceCompleted(session, sequenceId);
1864             }
1865         }
1866 
1867         @Override
onCaptureProcessProgressed(CameraExtensionSession session, CaptureRequest request, int progress)1868         public void onCaptureProcessProgressed(CameraExtensionSession session,
1869                 CaptureRequest request, int progress) {
1870             if ((progress < 0) || (progress > 100)) {
1871                 mCollector.addMessage("Capture progress invalid value: " + progress);
1872                 return;
1873             }
1874             if (mLastProgressValue >= progress) {
1875                 mCollector.addMessage("Unexpected progress value: " + progress +
1876                         " last progress value: " + mLastProgressValue);
1877                 return;
1878             }
1879             mLastProgressValue = progress;
1880             if (mProxy != null) {
1881                 mProxy.onCaptureProcessProgressed(session, request, progress);
1882             }
1883         }
1884 
1885         @Override
onCaptureResultAvailable(CameraExtensionSession session, CaptureRequest request, TotalCaptureResult result)1886         public void  onCaptureResultAvailable(CameraExtensionSession session,
1887                 CaptureRequest request, TotalCaptureResult result) {
1888             final float METERING_REGION_ERROR_PERCENT_DELTA = 0.05f;
1889             if (mSupportedResultKeys.isEmpty()) {
1890                 mCollector.addMessage("Capture results not supported, but " +
1891                         "onCaptureResultAvailable still got triggered!");
1892                 return;
1893             }
1894 
1895             List<CaptureResult.Key<?>> resultKeys = result.getKeys();
1896             for (CaptureResult.Key<?> resultKey : resultKeys) {
1897                 mCollector.expectTrue("Capture result " + resultKey + " is not among the"
1898                         + " supported result keys!", mSupportedResultKeys.contains(resultKey));
1899             }
1900 
1901             Long timeStamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
1902             assertNotNull(timeStamp);
1903             assertTrue("Capture result sensor timestamp: " + timeStamp + " must match "
1904                             + " with the timestamp passed to onCaptureStarted!",
1905                     mExpectedResultTimestamps.contains(timeStamp));
1906 
1907             Integer currentType = result.get(CaptureResult.EXTENSION_CURRENT_TYPE);
1908             if (currentType != null) {
1909                 mCollector.expectNotEquals("The reported extension type cannot be set to AUTO!",
1910                         CameraExtensionCharacteristics.EXTENSION_AUTOMATIC, currentType);
1911                 if (mExtensionType == CameraExtensionCharacteristics.EXTENSION_AUTOMATIC) {
1912                     Integer expectedValues[] = {
1913                             CameraExtensionCharacteristics.EXTENSION_BOKEH,
1914                             CameraExtensionCharacteristics.EXTENSION_HDR,
1915                             CameraExtensionCharacteristics.EXTENSION_NIGHT,
1916                             CameraExtensionCharacteristics.EXTENSION_FACE_RETOUCH};
1917                     mCollector.expectContains("Unexpected extension type result: "
1918                             + currentType, expectedValues, currentType);
1919                 } else {
1920                     mCollector.expectEquals("Unexpected extension type result: " + currentType
1921                             + " expected: " + mExtensionType, mExtensionType, currentType);
1922                 }
1923             }
1924 
1925             Integer strength = request.get(CaptureRequest.EXTENSION_STRENGTH);
1926             if (strength != null) {
1927                 Integer resultStrength = result.get(CaptureResult.EXTENSION_STRENGTH);
1928                 mCollector.expectTrue("Request extension strength: " + strength +
1929                                 " doesn't match with result: " + resultStrength,
1930                         strength.equals(resultStrength));
1931             }
1932 
1933             Integer jpegOrientation = request.get(CaptureRequest.JPEG_ORIENTATION);
1934             if (jpegOrientation != null) {
1935                 Integer resultJpegOrientation = result.get(CaptureResult.JPEG_ORIENTATION);
1936                 mCollector.expectTrue("Request Jpeg orientation: " + jpegOrientation +
1937                         " doesn't match with result: " + resultJpegOrientation,
1938                         jpegOrientation.equals(resultJpegOrientation));
1939             }
1940 
1941             Byte jpegQuality = request.get(CaptureRequest.JPEG_QUALITY);
1942             if (jpegQuality != null) {
1943                 Byte resultJpegQuality = result.get(CaptureResult.JPEG_QUALITY);
1944                 mCollector.expectTrue("Request Jpeg quality: " + jpegQuality +
1945                         " doesn't match with result: " + resultJpegQuality,
1946                         jpegQuality.equals(resultJpegQuality));
1947             }
1948 
1949             if (resultKeys.contains(CaptureResult.CONTROL_ZOOM_RATIO) && mPerFrameControl) {
1950                 Float zoomRatio = request.get(CaptureRequest.CONTROL_ZOOM_RATIO);
1951                 if (zoomRatio != null) {
1952                     Float resultZoomRatio = result.get(CaptureResult.CONTROL_ZOOM_RATIO);
1953                     mCollector.expectTrue(
1954                             String.format("Request and result zoom ratio should be similar " +
1955                                     "(requested = %f, result = %f", zoomRatio, resultZoomRatio),
1956                             Math.abs(zoomRatio - resultZoomRatio) / zoomRatio <= ZOOM_ERROR_MARGIN);
1957                 }
1958             }
1959 
1960             if (mFlashStateListener != null) {
1961                 Integer flashMode = request.get(CaptureRequest.FLASH_MODE);
1962                 if ((flashMode != null) && mPerFrameControl) {
1963                     Integer resultFlashMode = result.get(CaptureResult.FLASH_MODE);
1964                     mCollector.expectTrue("Request flash mode: " + flashMode +
1965                                     " doesn't match with result: " + resultFlashMode,
1966                             flashMode.equals(resultFlashMode));
1967                 }
1968 
1969                 Integer flashState = result.get(CaptureResult.FLASH_STATE);
1970                 if (flashState != null) {
1971                     switch (flashState) {
1972                         case CaptureResult.FLASH_STATE_UNAVAILABLE:
1973                             mFlashStateListener.onUnavailable();
1974                             break;
1975                         case CaptureResult.FLASH_STATE_FIRED:
1976                             mFlashStateListener.onFired();
1977                             break;
1978                         case CaptureResult.FLASH_STATE_CHARGING:
1979                             mFlashStateListener.onCharging();
1980                             break;
1981                         case CaptureResult.FLASH_STATE_PARTIAL:
1982                             mFlashStateListener.onPartial();
1983                             break;
1984                         case CaptureResult.FLASH_STATE_READY:
1985                             mFlashStateListener.onReady();
1986                             break;
1987                         default:
1988                             mCollector.addMessage("Unexpected flash state: " + flashState);
1989                     }
1990                 }
1991             }
1992 
1993             if (mAEStateListener != null) {
1994                 Integer aeMode = request.get(CaptureRequest.CONTROL_AE_MODE);
1995                 if ((aeMode != null) && mPerFrameControl) {
1996                     Integer resultAeMode = result.get(CaptureResult.CONTROL_AE_MODE);
1997                     mCollector.expectTrue("Request AE mode: " + aeMode +
1998                                     " doesn't match with result: " + resultAeMode,
1999                             aeMode.equals(resultAeMode));
2000                 }
2001 
2002                 Boolean aeLock = request.get(CaptureRequest.CONTROL_AE_LOCK);
2003                 if ((aeLock != null) && mPerFrameControl) {
2004                     Boolean resultAeLock = result.get(CaptureResult.CONTROL_AE_LOCK);
2005                     mCollector.expectTrue("Request AE lock: " + aeLock +
2006                                     " doesn't match with result: " + resultAeLock,
2007                             aeLock.equals(resultAeLock));
2008                 }
2009 
2010                 Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
2011                 if (aeState != null) {
2012                     switch (aeState) {
2013                         case CaptureResult.CONTROL_AE_STATE_CONVERGED:
2014                             mAEStateListener.onConverged();
2015                             break;
2016                         case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED:
2017                             mAEStateListener.onFlashRequired();
2018                             break;
2019                         case CaptureResult.CONTROL_AE_STATE_INACTIVE:
2020                             mAEStateListener.onInactive();
2021                             break;
2022                         case CaptureResult.CONTROL_AE_STATE_LOCKED:
2023                             mAEStateListener.onLocked();
2024                             break;
2025                         case CaptureResult.CONTROL_AE_STATE_PRECAPTURE:
2026                             mAEStateListener.onPrecapture();
2027                             break;
2028                         case CaptureResult.CONTROL_AE_STATE_SEARCHING:
2029                             mAEStateListener.onSearching();
2030                             break;
2031                         default:
2032                             mCollector.addMessage("Unexpected AE state: " + aeState);
2033                     }
2034                 }
2035             }
2036 
2037             if (mAFStateMachine != null) {
2038                 Integer afMode = request.get(CaptureRequest.CONTROL_AF_MODE);
2039                 if ((afMode != null) && mPerFrameControl) {
2040                     Integer resultAfMode = result.get(CaptureResult.CONTROL_AF_MODE);
2041                     mCollector.expectTrue("Request AF mode: " + afMode +
2042                                     " doesn't match with result: " + resultAfMode,
2043                             afMode.equals(resultAfMode));
2044                 }
2045 
2046                 MeteringRectangle[] afRegions = request.get(CaptureRequest.CONTROL_AF_REGIONS);
2047                 if ((afRegions != null) && mPerFrameControl) {
2048                     MeteringRectangle[] resultAfRegions = result.get(
2049                             CaptureResult.CONTROL_AF_REGIONS);
2050                     mCollector.expectMeteringRegionsAreSimilar(
2051                             "AF regions in result and request should be similar",
2052                             afRegions, resultAfRegions, METERING_REGION_ERROR_PERCENT_DELTA);
2053                 }
2054 
2055                 Integer afTrigger = request.get(CaptureRequest.CONTROL_AF_TRIGGER);
2056                 if ((afTrigger != null) && mPerFrameControl) {
2057                     Integer resultAfTrigger = result.get(CaptureResult.CONTROL_AF_TRIGGER);
2058                     mCollector.expectTrue("Request AF trigger: " + afTrigger +
2059                                     " doesn't match with result: " + resultAfTrigger,
2060                             afTrigger.equals(resultAfTrigger));
2061                 }
2062 
2063                 mAFStateMachine.onCaptureCompleted(result);
2064             }
2065 
2066             if (mProxy != null) {
2067                 mProxy.onCaptureResultAvailable(session, request, result);
2068             }
2069         }
2070 
getTotalFramesArrived()2071         public int getTotalFramesArrived() {
2072             return mNumFramesArrived;
2073         }
2074 
getTotalFramesStarted()2075         public int getTotalFramesStarted() {
2076             return mNumFramesStarted;
2077         }
2078 
getTotalFramesFailed()2079         public int getTotalFramesFailed() {
2080             return mNumFramesFailed;
2081         }
2082 
getLastTimestamp()2083         public long getLastTimestamp() throws IllegalStateException {
2084             if (mNonIncreasingTimestamps) {
2085                 throw new IllegalStateException("Non-monotonically increasing timestamps!");
2086             }
2087             return mLastTimestamp;
2088         }
2089 
2090     }
2091 
2092     public interface AutoFocusStateListener {
onDone(boolean success)2093         void onDone(boolean success);
onScan()2094         void onScan();
onInactive()2095         void onInactive();
2096     }
2097 
2098     private class TestAutoFocusProxy implements AutoFocusStateMachine.AutoFocusStateListener {
2099         private final AutoFocusStateListener mListener;
2100 
TestAutoFocusProxy(AutoFocusStateListener listener)2101         TestAutoFocusProxy(AutoFocusStateListener listener) {
2102             mListener = listener;
2103         }
2104 
2105         @Override
onAutoFocusSuccess(CaptureResult result, boolean locked)2106         public void onAutoFocusSuccess(CaptureResult result, boolean locked) {
2107             mListener.onDone(true);
2108         }
2109 
2110         @Override
onAutoFocusFail(CaptureResult result, boolean locked)2111         public void onAutoFocusFail(CaptureResult result, boolean locked) {
2112             mListener.onDone(false);
2113         }
2114 
2115         @Override
onAutoFocusScan(CaptureResult result)2116         public void onAutoFocusScan(CaptureResult result) {
2117             mListener.onScan();
2118         }
2119 
2120         @Override
onAutoFocusInactive(CaptureResult result)2121         public void onAutoFocusInactive(CaptureResult result) {
2122             mListener.onInactive();
2123         }
2124     }
2125 
2126     // Verify that camera extension sessions can support AF and AF metering controls. The test
2127     // goal is to check that AF related controls and results are supported and can be used to
2128     // lock the AF state and not to do an exhaustive check of the AF state transitions or manual AF.
2129     @Test
testAFMetering()2130     public void testAFMetering() throws Exception {
2131         final int METERING_REGION_SCALE_RATIO = 8;
2132         for (String id : getCameraIdsUnderTest()) {
2133             StaticMetadata staticMeta =
2134                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2135             if (!staticMeta.isColorOutputSupported()) {
2136                 continue;
2137             }
2138             if (!staticMeta.hasFocuser()) {
2139                 continue;
2140             }
2141 
2142             boolean perFrameControl = staticMeta.isPerFrameControlSupported();
2143             final Rect activeArraySize = staticMeta.getActiveArraySizeChecked();
2144             int regionWidth = activeArraySize.width() / METERING_REGION_SCALE_RATIO - 1;
2145             int regionHeight = activeArraySize.height() / METERING_REGION_SCALE_RATIO - 1;
2146             int centerX = activeArraySize.width() / 2;
2147             int centerY = activeArraySize.height() / 2;
2148 
2149             // Center region
2150             MeteringRectangle[] afMeteringRects = {new MeteringRectangle(
2151                     centerX - regionWidth / 2, centerY - regionHeight / 2,
2152                     regionWidth, regionHeight,
2153                     MeteringRectangle.METERING_WEIGHT_MAX)};
2154 
2155             updatePreviewSurfaceTexture();
2156             CameraExtensionCharacteristics extensionChars =
2157                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2158             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2159             for (Integer extension : supportedExtensions) {
2160                 Set<CaptureRequest.Key> supportedRequestKeys =
2161                         extensionChars.getAvailableCaptureRequestKeys(extension);
2162                 // The night extension is required to support AF controls starting with Android V
2163                 if ((Build.VERSION.DEVICE_INITIAL_SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM)
2164                     && (extension == CameraExtensionCharacteristics.EXTENSION_NIGHT)) {
2165                     assertTrue(supportedRequestKeys.containsAll(
2166                             Arrays.asList(FOCUS_CAPTURE_REQUEST_SET)));
2167                 } else if (!supportedRequestKeys.containsAll(
2168                         Arrays.asList(FOCUS_CAPTURE_REQUEST_SET))) {
2169                     continue;
2170                 }
2171                 Set<CaptureResult.Key> supportedResultKeys =
2172                         extensionChars.getAvailableCaptureResultKeys(extension);
2173                 assertTrue(supportedResultKeys.containsAll(
2174                         Arrays.asList(FOCUS_CAPTURE_RESULT_SET)));
2175 
2176                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
2177                         mSurfaceTexture.getClass());
2178                 Size maxSize =
2179                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
2180                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
2181                         maxSize.getHeight());
2182                 Surface texturedSurface = new Surface(mSurfaceTexture);
2183 
2184                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2185                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2186 
2187                 BlockingExtensionSessionCallback sessionListener =
2188                         new BlockingExtensionSessionCallback(mock(
2189                                 CameraExtensionSession.StateCallback.class));
2190                 ExtensionSessionConfiguration configuration =
2191                         new ExtensionSessionConfiguration(extension, outputConfigs,
2192                                 new HandlerExecutor(mTestRule.getHandler()),
2193                                 sessionListener);
2194 
2195                 try {
2196                     mTestRule.openDevice(id);
2197                     CameraDevice camera = mTestRule.getCamera();
2198                     camera.createExtensionSession(configuration);
2199                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
2200                             SESSION_CONFIGURE_TIMEOUT_MS);
2201                     assertNotNull(extensionSession);
2202 
2203                     // Check passive AF
2204                     AutoFocusStateListener mockAFListener =
2205                             mock(AutoFocusStateListener.class);
2206                     AutoFocusStateMachine afState = new AutoFocusStateMachine(
2207                             new TestAutoFocusProxy(mockAFListener));
2208                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2209                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2210                     SimpleCaptureCallback repeatingCaptureCallback =
2211                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2212                                     extensionChars.getAvailableCaptureResultKeys(extension),
2213                                     mCollector, afState, null /*flashState*/,
2214                                     null /*aeState*/, perFrameControl);
2215 
2216                     CaptureRequest.Builder captureBuilder =
2217                             mTestRule.getCamera().createCaptureRequest(
2218                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2219                     captureBuilder.addTarget(texturedSurface);
2220                     afState.setPassiveAutoFocus(true /*picture*/, captureBuilder);
2221                     captureBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects);
2222                     CaptureRequest request = captureBuilder.build();
2223                     int passiveSequenceId = extensionSession.setRepeatingRequest(request,
2224                             new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback);
2225                     assertTrue(passiveSequenceId > 0);
2226 
2227                     verify(repeatingCallbackMock,
2228                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2229                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
2230                                     any(TotalCaptureResult.class));
2231 
2232                     verify(mockAFListener,
2233                             timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS).atLeastOnce())
2234                             .onDone(anyBoolean());
2235 
2236                     // Check active AF
2237                     mockAFListener = mock(AutoFocusStateListener.class);
2238                     CameraExtensionSession.ExtensionCaptureCallback callbackMock = mock(
2239                             CameraExtensionSession.ExtensionCaptureCallback.class);
2240                     AutoFocusStateMachine activeAFState = new AutoFocusStateMachine(
2241                             new TestAutoFocusProxy(mockAFListener));
2242                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
2243                             new SimpleCaptureCallback(extension, callbackMock,
2244                                     extensionChars.getAvailableCaptureResultKeys(extension),
2245                                     mCollector, activeAFState, null /*flashState*/,
2246                                     null /*aeState*/, perFrameControl);
2247 
2248                     CaptureRequest.Builder triggerBuilder =
2249                             mTestRule.getCamera().createCaptureRequest(
2250                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2251                     triggerBuilder.addTarget(texturedSurface);
2252                     afState.setActiveAutoFocus(captureBuilder, triggerBuilder);
2253                     afState.unlockAutoFocus(captureBuilder, triggerBuilder);
2254                     triggerBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects);
2255                     request = captureBuilder.build();
2256                     int activeSequenceId = extensionSession.setRepeatingRequest(request,
2257                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2258                     assertTrue((activeSequenceId > 0) &&
2259                             (activeSequenceId != passiveSequenceId));
2260 
2261                     verify(callbackMock,
2262                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2263                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
2264                                     any(TotalCaptureResult.class));
2265 
2266                     CaptureRequest triggerRequest = triggerBuilder.build();
2267                     reset(mockAFListener);
2268                     int triggerSequenceId = extensionSession.capture(triggerRequest,
2269                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2270                     assertTrue((triggerSequenceId > 0) &&
2271                             (activeSequenceId != triggerSequenceId) &&
2272                             (triggerSequenceId != passiveSequenceId));
2273 
2274                     verify(callbackMock,
2275                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2276                             .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest),
2277                                     any(TotalCaptureResult.class));
2278 
2279                     afState.lockAutoFocus(captureBuilder, triggerBuilder);
2280                     triggerRequest = triggerBuilder.build();
2281                     reset(mockAFListener);
2282                     extensionSession.capture(triggerRequest,
2283                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2284 
2285                     verify(callbackMock,
2286                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2287                             .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest),
2288                                     any(TotalCaptureResult.class));
2289 
2290                     verify(mockAFListener, timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS)
2291                             .atLeast(1)).onDone(anyBoolean());
2292 
2293                     extensionSession.stopRepeating();
2294 
2295                     extensionSession.close();
2296 
2297                     sessionListener.getStateWaiter().waitForState(
2298                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2299                             SESSION_CLOSE_TIMEOUT_MS);
2300                 } finally {
2301                     mTestRule.closeDevice(id);
2302                     texturedSurface.release();
2303                 }
2304             }
2305         }
2306     }
2307 
2308     // Verify that camera extension sessions can support the zoom ratio control.
2309     @Test
testZoomRatio()2310     public void testZoomRatio() throws Exception {
2311         final int ZOOM_RATIO_STEPS = 10;
2312         for (String id : getCameraIdsUnderTest()) {
2313             StaticMetadata staticMeta =
2314                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2315             if (!staticMeta.isColorOutputSupported()) {
2316                 continue;
2317             }
2318 
2319             CameraExtensionCharacteristics extensionChars =
2320                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2321 
2322             updatePreviewSurfaceTexture();
2323             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2324             for (Integer extension : supportedExtensions) {
2325                 Range<Float> zoomRatioRange;
2326                 final float maxZoom;
2327                 zoomRatioRange = extensionChars.get(extension,
2328                     CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE);
2329                 if (zoomRatioRange == null) {
2330                     zoomRatioRange = staticMeta.getZoomRatioRangeChecked();
2331                 }
2332                 maxZoom = zoomRatioRange.getUpper();
2333 
2334                 if (zoomRatioRange.getUpper().equals(zoomRatioRange.getLower())) {
2335                     continue;
2336                 }
2337 
2338                 if (Math.abs(maxZoom - 1.0f) < ZOOM_ERROR_MARGIN) {
2339                     return;
2340                 }
2341 
2342                 Float zoomStep  =
2343                         (zoomRatioRange.getUpper() - zoomRatioRange.getLower()) / ZOOM_RATIO_STEPS;
2344                 if (zoomStep < ZOOM_ERROR_MARGIN) {
2345                     continue;
2346                 }
2347 
2348                 ArrayList<Float> candidateZoomRatios = new ArrayList<>(ZOOM_RATIO_STEPS);
2349                 for (int step = 0; step < (ZOOM_RATIO_STEPS - 1); step++) {
2350                     candidateZoomRatios.add(step, zoomRatioRange.getLower() + step * zoomStep);
2351                 }
2352                 candidateZoomRatios.add(ZOOM_RATIO_STEPS - 1, zoomRatioRange.getUpper());
2353 
2354                 Set<CaptureRequest.Key> supportedRequestKeys =
2355                         extensionChars.getAvailableCaptureRequestKeys(extension);
2356                 // The Night extension is required to support zoom controls start with Android V
2357                 if ((Build.VERSION.DEVICE_INITIAL_SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM)
2358                         && (extension == CameraExtensionCharacteristics.EXTENSION_NIGHT)) {
2359                     assertTrue(supportedRequestKeys.containsAll(
2360                             Arrays.asList(ZOOM_CAPTURE_REQUEST_SET)));
2361                 } else if (!supportedRequestKeys.containsAll(
2362                         Arrays.asList(ZOOM_CAPTURE_REQUEST_SET))) {
2363                     continue;
2364                 }
2365                 Set<CaptureResult.Key> supportedResultKeys =
2366                         extensionChars.getAvailableCaptureResultKeys(extension);
2367                 assertTrue(supportedResultKeys.containsAll(
2368                         Arrays.asList(ZOOM_CAPTURE_RESULT_SET)));
2369 
2370                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
2371                         mSurfaceTexture.getClass());
2372                 Size maxSize =
2373                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
2374                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
2375                         maxSize.getHeight());
2376                 Surface texturedSurface = new Surface(mSurfaceTexture);
2377 
2378                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2379                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2380 
2381                 int captureFormat = ImageFormat.JPEG;
2382                 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2383                         captureFormat);
2384                 assertFalse("No Jpeg output supported", captureSizes.isEmpty());
2385                 Size captureMaxSize =
2386                         CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
2387 
2388                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false
2389                         , MAX_IMAGES);
2390                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
2391                         captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener,
2392                         mTestRule.getHandler());
2393                 Surface imageReaderSurface = extensionImageReader.getSurface();
2394                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
2395                 outputConfigs.add(readerOutput);
2396 
2397                 BlockingExtensionSessionCallback sessionListener =
2398                         new BlockingExtensionSessionCallback(mock(
2399                                 CameraExtensionSession.StateCallback.class));
2400                 ExtensionSessionConfiguration configuration =
2401                         new ExtensionSessionConfiguration(extension, outputConfigs,
2402                                 new HandlerExecutor(mTestRule.getHandler()),
2403                                 sessionListener);
2404 
2405                 try {
2406                     mTestRule.openDevice(id);
2407                     CameraDevice camera = mTestRule.getCamera();
2408                     camera.createExtensionSession(configuration);
2409                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
2410                             SESSION_CONFIGURE_TIMEOUT_MS);
2411                     assertNotNull(extensionSession);
2412 
2413                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2414                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2415                     SimpleCaptureCallback repeatingCaptureCallback =
2416                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2417                                     extensionChars.getAvailableCaptureResultKeys(extension),
2418                                     mCollector);
2419 
2420                     CaptureRequest.Builder captureBuilder =
2421                             mTestRule.getCamera().createCaptureRequest(
2422                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2423                     captureBuilder.addTarget(texturedSurface);
2424                     CaptureRequest.Builder stillCaptureBuilder =
2425                             mTestRule.getCamera().createCaptureRequest(
2426                                     CameraDevice.TEMPLATE_STILL_CAPTURE);
2427                     stillCaptureBuilder.addTarget(imageReaderSurface);
2428                     for (Float currentZoomRatio : candidateZoomRatios) {
2429                         captureBuilder.set(CaptureRequest.CONTROL_ZOOM_RATIO, currentZoomRatio);
2430                         CaptureRequest request = captureBuilder.build();
2431 
2432                         int seqId = extensionSession.setRepeatingRequest(request,
2433                                 new HandlerExecutor(mTestRule.getHandler()),
2434                                 repeatingCaptureCallback);
2435                         assertTrue(seqId > 0);
2436 
2437                         verify(repeatingCallbackMock,
2438                                 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2439                                 .onCaptureResultAvailable(eq(extensionSession), eq(request),
2440                                         any(TotalCaptureResult.class));
2441 
2442                         stillCaptureBuilder.set(CaptureRequest.CONTROL_ZOOM_RATIO, currentZoomRatio);
2443                         CaptureRequest stillRequest = stillCaptureBuilder.build();
2444                         CameraExtensionSession.ExtensionCaptureCallback captureCallback =
2445                                 mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2446 
2447                         extensionSession.capture(stillRequest,
2448                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2449 
2450                         // Only ensure we have a valid image. Further image validation is already
2451                         // done by other test cases.
2452                         Image img = imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
2453                         img.close();
2454 
2455                         verify(captureCallback,
2456                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
2457                                 .onCaptureResultAvailable(eq(extensionSession),
2458                                         eq(stillRequest), any(TotalCaptureResult.class));
2459                         verify(captureCallback, times(0))
2460                                 .onCaptureFailed(any(CameraExtensionSession.class),
2461                                         any(CaptureRequest.class));
2462                         verify(captureCallback, times(0))
2463                                 .onCaptureFailed(any(CameraExtensionSession.class),
2464                                         any(CaptureRequest.class), anyInt());
2465                     }
2466 
2467                     extensionSession.stopRepeating();
2468 
2469                     extensionSession.close();
2470 
2471                     sessionListener.getStateWaiter().waitForState(
2472                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2473                             SESSION_CLOSE_TIMEOUT_MS);
2474                 } finally {
2475                     mTestRule.closeDevice(id);
2476                     texturedSurface.release();
2477                     extensionImageReader.close();
2478                 }
2479             }
2480         }
2481     }
2482 
2483     public interface FlashStateListener {
onFired()2484         public void onFired();
onReady()2485         public void onReady();
onCharging()2486         public void onCharging();
onPartial()2487         public void onPartial();
onUnavailable()2488         public void onUnavailable();
2489     }
2490 
2491     public interface AutoExposureStateListener {
onInactive()2492       public void onInactive();
onSearching()2493       public void onSearching();
onConverged()2494       public void onConverged();
onLocked()2495       public void onLocked();
onFlashRequired()2496       public void onFlashRequired();
onPrecapture()2497       public void onPrecapture();
2498     }
2499 
2500     // Verify that camera extension sessions can support Flash related controls. The test
2501     // goal is to check that Flash controls and results are supported and can be used to
2502     // turn on torch, run the pre-capture sequence and active the main flash.
2503     @Test
testFlash()2504     public void testFlash() throws Exception {
2505         final CaptureRequest.Key[] FLASH_CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_AE_MODE,
2506                 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_LOCK,
2507                 CaptureRequest.FLASH_MODE};
2508         final CaptureResult.Key[] FLASH_CAPTURE_RESULT_SET = {CaptureResult.CONTROL_AE_MODE,
2509                 CaptureResult.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureResult.CONTROL_AE_LOCK,
2510                 CaptureResult.CONTROL_AE_STATE, CaptureResult.FLASH_MODE,
2511                 CaptureResult.FLASH_STATE};
2512         for (String id : getCameraIdsUnderTest()) {
2513             StaticMetadata staticMeta =
2514                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2515             if (!staticMeta.isColorOutputSupported()) {
2516                 continue;
2517             }
2518             if (!staticMeta.hasFlash()) {
2519                 continue;
2520             }
2521 
2522             boolean perFrameControl = staticMeta.isPerFrameControlSupported();
2523             updatePreviewSurfaceTexture();
2524             CameraExtensionCharacteristics extensionChars =
2525                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2526             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2527             for (Integer extension : supportedExtensions) {
2528                 Set<CaptureRequest.Key> supportedRequestKeys =
2529                         extensionChars.getAvailableCaptureRequestKeys(extension);
2530                 if (!supportedRequestKeys.containsAll(Arrays.asList(FLASH_CAPTURE_REQUEST_SET))) {
2531                     continue;
2532                 }
2533                 Set<CaptureResult.Key> supportedResultKeys =
2534                         extensionChars.getAvailableCaptureResultKeys(extension);
2535                 assertTrue(supportedResultKeys.containsAll(
2536                         Arrays.asList(FLASH_CAPTURE_RESULT_SET)));
2537 
2538                 int captureFormat = ImageFormat.JPEG;
2539                 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2540                         captureFormat);
2541                 assertFalse("No Jpeg output supported", captureSizes.isEmpty());
2542                 Size captureMaxSize = captureSizes.get(0);
2543 
2544                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
2545                         MAX_IMAGES);
2546                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
2547                         captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener,
2548                         mTestRule.getHandler());
2549                 Surface imageReaderSurface = extensionImageReader.getSurface();
2550                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
2551                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2552                 outputConfigs.add(readerOutput);
2553 
2554                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
2555                         mSurfaceTexture.getClass());
2556                 Size previewSize = repeatingSizes.get(0);
2557 
2558                 mSurfaceTexture.setDefaultBufferSize(previewSize.getWidth(),
2559                         previewSize.getHeight());
2560                 Surface texturedSurface = new Surface(mSurfaceTexture);
2561                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2562 
2563                 BlockingExtensionSessionCallback sessionListener =
2564                         new BlockingExtensionSessionCallback(mock(
2565                                 CameraExtensionSession.StateCallback.class));
2566                 ExtensionSessionConfiguration configuration =
2567                         new ExtensionSessionConfiguration(extension, outputConfigs,
2568                                 new HandlerExecutor(mTestRule.getHandler()),
2569                                 sessionListener);
2570                 try {
2571                     mTestRule.openDevice(id);
2572                     CameraDevice camera = mTestRule.getCamera();
2573                     camera.createExtensionSession(configuration);
2574                     CameraExtensionSession extensionSession =
2575                             sessionListener.waitAndGetSession(
2576                                     SESSION_CONFIGURE_TIMEOUT_MS);
2577                     assertNotNull(extensionSession);
2578 
2579                     // Test torch on and off
2580                     CaptureRequest.Builder captureBuilder =
2581                             mTestRule.getCamera().createCaptureRequest(
2582                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2583                     captureBuilder.addTarget(texturedSurface);
2584                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2585                             CameraMetadata.CONTROL_AE_MODE_ON);
2586                     captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false);
2587                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_TORCH);
2588                     FlashStateListener mockFlashListener = mock(FlashStateListener.class);
2589                     AutoExposureStateListener mockAEStateListener =
2590                             mock(AutoExposureStateListener.class);
2591                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2592                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2593                     SimpleCaptureCallback repeatingCaptureCallback =
2594                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2595                                     extensionChars.getAvailableCaptureResultKeys(extension),
2596                                     mCollector, null /*afState*/, mockFlashListener,
2597                                     mockAEStateListener, perFrameControl);
2598                     CaptureRequest repeatingRequest = captureBuilder.build();
2599                     int repeatingSequenceId =
2600                             extensionSession.setRepeatingRequest(repeatingRequest,
2601                                     new HandlerExecutor(mTestRule.getHandler()),
2602                                     repeatingCaptureCallback);
2603                     assertTrue(repeatingSequenceId > 0);
2604 
2605                     verify(repeatingCallbackMock,
2606                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2607                             .onCaptureResultAvailable(eq(extensionSession),
2608                                     eq(repeatingRequest), any(TotalCaptureResult.class));
2609                     verify(mockFlashListener,
2610                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired();
2611 
2612                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2613                             CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH);
2614                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF);
2615                     repeatingRequest = captureBuilder.build();
2616                     reset(mockFlashListener);
2617                     repeatingSequenceId = extensionSession.setRepeatingRequest(repeatingRequest,
2618                                     new HandlerExecutor(mTestRule.getHandler()),
2619                                     repeatingCaptureCallback);
2620                     assertTrue(repeatingSequenceId > 0);
2621 
2622                     verify(repeatingCallbackMock,
2623                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2624                             .onCaptureResultAvailable(eq(extensionSession),
2625                                     eq(repeatingRequest), any(TotalCaptureResult.class));
2626                     verify(mockFlashListener,
2627                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onReady();
2628 
2629                     // Test AE pre-capture sequence
2630                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF);
2631                     captureBuilder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER,
2632                             CameraMetadata.CONTROL_AE_PRECAPTURE_TRIGGER_START);
2633                     CaptureRequest triggerRequest = captureBuilder.build();
2634                     int triggerSeqId = extensionSession.capture(triggerRequest,
2635                             new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback);
2636                     assertTrue(triggerSeqId > 0);
2637 
2638                     verify(repeatingCallbackMock,
2639                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2640                             .onCaptureResultAvailable(eq(extensionSession),
2641                                     eq(triggerRequest), any(TotalCaptureResult.class));
2642                     verify(mockAEStateListener,
2643                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onPrecapture();
2644                     verify(mockAEStateListener,
2645                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onConverged();
2646 
2647                     // Test main flash
2648                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
2649                             android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE);
2650                     captureBuilder.addTarget(imageReaderSurface);
2651                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2652                             CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH);
2653                     captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, true);
2654                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_SINGLE);
2655                     CameraExtensionSession.ExtensionCaptureCallback mockCaptureCallback =
2656                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2657                     reset(mockFlashListener);
2658                     SimpleCaptureCallback captureCallback =
2659                             new SimpleCaptureCallback(extension, mockCaptureCallback,
2660                                     extensionChars.getAvailableCaptureResultKeys(extension),
2661                                     mCollector, null /*afState*/, mockFlashListener,
2662                                     mockAEStateListener, perFrameControl);
2663 
2664                     CaptureRequest captureRequest = captureBuilder.build();
2665                     int captureSequenceId = extensionSession.capture(captureRequest,
2666                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2667                     assertTrue(captureSequenceId > 0);
2668 
2669                     Image img =
2670                             imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
2671                     validateImage(img, captureMaxSize.getWidth(),
2672                             captureMaxSize.getHeight(), captureFormat, null);
2673                     long imgTs = img.getTimestamp();
2674                     img.close();
2675 
2676                     verify(mockCaptureCallback,
2677                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
2678                             .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs));
2679                     verify(mockCaptureCallback,
2680                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
2681                             .onCaptureResultAvailable(eq(extensionSession),
2682                                     eq(captureRequest), any(TotalCaptureResult.class));
2683                     verify(mockFlashListener,
2684                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired();
2685 
2686                     extensionSession.stopRepeating();
2687 
2688                     extensionSession.close();
2689 
2690                     sessionListener.getStateWaiter().waitForState(
2691                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2692                             SESSION_CLOSE_TIMEOUT_MS);
2693                 } finally {
2694                     mTestRule.closeDevice(id);
2695                     texturedSurface.release();
2696                     extensionImageReader.close();
2697                 }
2698             }
2699         }
2700     }
2701 
2702     // Verify 'CameraExtensionSession.StillCaptureLatency' behavior
2703     @Test
testSessionStillCaptureLatency()2704     public void testSessionStillCaptureLatency() throws Exception {
2705         final long CAPTURE_LATENCY_MS = 100;
2706         final long PROCESSING_LATENCY_MS = 200;
2707         final long DIFFERENT_PROCESSING_LATENCY_MS = 201;
2708         CameraExtensionSession.StillCaptureLatency stillCaptureLatency =
2709                 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS,
2710                         PROCESSING_LATENCY_MS);
2711         assertEquals(stillCaptureLatency.getCaptureLatency(), CAPTURE_LATENCY_MS);
2712         assertEquals(stillCaptureLatency.getProcessingLatency(), PROCESSING_LATENCY_MS);
2713         assertNotNull(stillCaptureLatency.toString());
2714         CameraExtensionSession.StillCaptureLatency differentStillCaptureLatency =
2715                 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS,
2716                         DIFFERENT_PROCESSING_LATENCY_MS);
2717         assertFalse(stillCaptureLatency.equals(differentStillCaptureLatency));
2718         assertFalse(stillCaptureLatency.hashCode() ==
2719                 differentStillCaptureLatency.hashCode());
2720     }
2721 
2722     @Test
testIllegalArguments()2723     public void testIllegalArguments() throws Exception {
2724         for (String id : getCameraIdsUnderTest()) {
2725             StaticMetadata staticMeta =
2726                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2727             if (!staticMeta.isColorOutputSupported()) {
2728                 continue;
2729             }
2730             updatePreviewSurfaceTexture();
2731             CameraExtensionCharacteristics extensionChars =
2732                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2733             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2734             for (Integer extension : supportedExtensions) {
2735                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2736                 BlockingExtensionSessionCallback sessionListener =
2737                         new BlockingExtensionSessionCallback(mock(
2738                                 CameraExtensionSession.StateCallback.class));
2739                 ExtensionSessionConfiguration configuration =
2740                         new ExtensionSessionConfiguration(extension, outputConfigs,
2741                                 new HandlerExecutor(mTestRule.getHandler()),
2742                                 sessionListener);
2743 
2744                 try {
2745                     mTestRule.openDevice(id);
2746                     CameraDevice camera = mTestRule.getCamera();
2747                     try {
2748                         camera.createExtensionSession(configuration);
2749                         fail("should get IllegalArgumentException due to absent output surfaces");
2750                     } catch (IllegalArgumentException e) {
2751                         // Expected, we can proceed further
2752                     }
2753 
2754                     int captureFormat = ImageFormat.YUV_420_888;
2755                     List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2756                             captureFormat);
2757                     if (captureSizes.isEmpty()) {
2758                         captureFormat = ImageFormat.JPEG;
2759                         captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2760                                 captureFormat);
2761                     }
2762                     Size captureMaxSize =
2763                             CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
2764 
2765                     mSurfaceTexture.setDefaultBufferSize(1, 1);
2766                     Surface texturedSurface = new Surface(mSurfaceTexture);
2767                     outputConfigs.add(new OutputConfiguration(texturedSurface));
2768                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
2769                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
2770 
2771                     try {
2772                         camera.createExtensionSession(configuration);
2773                         fail("should get IllegalArgumentException due to illegal repeating request"
2774                                 + " output surface");
2775                     } catch (IllegalArgumentException e) {
2776                         // Expected, we can proceed further
2777                     } finally {
2778                         outputConfigs.clear();
2779                     }
2780 
2781                     SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
2782                             MAX_IMAGES);
2783                     Size invalidCaptureSize = new Size(1, 1);
2784                     ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
2785                             invalidCaptureSize, captureFormat, /*maxImages*/ 1,
2786                             imageListener, mTestRule.getHandler());
2787                     Surface imageReaderSurface = extensionImageReader.getSurface();
2788                     OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
2789                     outputConfigs.add(readerOutput);
2790                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
2791                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
2792 
2793                     try{
2794                         camera.createExtensionSession(configuration);
2795                         fail("should get IllegalArgumentException due to illegal multi-frame"
2796                                 + " request output surface");
2797                     } catch (IllegalArgumentException e) {
2798                         // Expected, we can proceed further
2799                     } finally {
2800                         outputConfigs.clear();
2801                         extensionImageReader.close();
2802                     }
2803 
2804                     // Pick a supported preview/repeating size with aspect ratio close to the
2805                     // multi-frame capture size
2806                     List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
2807                             mSurfaceTexture.getClass());
2808                     Size maxRepeatingSize =
2809                             CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
2810                     List<Size> previewSizes = getSupportedPreviewSizes(id,
2811                             mTestRule.getCameraManager(),
2812                             getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
2813                     List<Size> supportedPreviewSizes =
2814                             previewSizes.stream().filter(repeatingSizes::contains).collect(
2815                                     Collectors.toList());
2816                     if (!supportedPreviewSizes.isEmpty()) {
2817                         float targetAr =
2818                                 ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight();
2819                         for (Size s : supportedPreviewSizes) {
2820                             float currentAr = ((float) s.getWidth()) / s.getHeight();
2821                             if (Math.abs(targetAr - currentAr) < 0.01) {
2822                                 maxRepeatingSize = s;
2823                                 break;
2824                             }
2825                         }
2826                     }
2827 
2828                     imageListener = new SimpleImageReaderListener(false, MAX_IMAGES);
2829                     extensionImageReader = CameraTestUtils.makeImageReader(captureMaxSize,
2830                             captureFormat, /*maxImages*/ 1, imageListener, mTestRule.getHandler());
2831                     imageReaderSurface = extensionImageReader.getSurface();
2832                     readerOutput = new OutputConfiguration(imageReaderSurface);
2833                     outputConfigs.add(readerOutput);
2834 
2835                     mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
2836                             maxRepeatingSize.getHeight());
2837                     texturedSurface = new Surface(mSurfaceTexture);
2838                     outputConfigs.add(new OutputConfiguration(texturedSurface));
2839 
2840                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
2841                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
2842                     camera.createExtensionSession(configuration);
2843                     CameraExtensionSession extensionSession =
2844                             sessionListener.waitAndGetSession(
2845                                     SESSION_CONFIGURE_TIMEOUT_MS);
2846                     assertNotNull(extensionSession);
2847 
2848                     CaptureRequest.Builder captureBuilder =
2849                             mTestRule.getCamera().createCaptureRequest(
2850                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2851                     captureBuilder.addTarget(imageReaderSurface);
2852                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2853                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2854                     SimpleCaptureCallback repeatingCaptureCallback =
2855                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2856                                     extensionChars.getAvailableCaptureResultKeys(extension),
2857                                     mCollector);
2858                     CaptureRequest repeatingRequest = captureBuilder.build();
2859                     try {
2860                         extensionSession.setRepeatingRequest(repeatingRequest,
2861                                 new HandlerExecutor(mTestRule.getHandler()),
2862                                 repeatingCaptureCallback);
2863                         fail("should get IllegalArgumentException due to illegal repeating request"
2864                                 + " output target");
2865                     } catch (IllegalArgumentException e) {
2866                         // Expected, we can proceed further
2867                     }
2868 
2869                     extensionSession.close();
2870 
2871                     sessionListener.getStateWaiter().waitForState(
2872                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2873                             SESSION_CLOSE_TIMEOUT_MS);
2874 
2875                     texturedSurface.release();
2876                     extensionImageReader.close();
2877 
2878                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
2879                             CameraDevice.TEMPLATE_PREVIEW);
2880                     captureBuilder.addTarget(texturedSurface);
2881                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
2882                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2883 
2884                     CaptureRequest captureRequest = captureBuilder.build();
2885                     try {
2886                         extensionSession.setRepeatingRequest(captureRequest,
2887                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2888                         fail("should get IllegalStateException due to closed session");
2889                     } catch (IllegalStateException e) {
2890                         // Expected, we can proceed further
2891                     }
2892 
2893                     try {
2894                         extensionSession.stopRepeating();
2895                         fail("should get IllegalStateException due to closed session");
2896                     } catch (IllegalStateException e) {
2897                         // Expected, we can proceed further
2898                     }
2899 
2900                     try {
2901                         extensionSession.capture(captureRequest,
2902                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2903                         fail("should get IllegalStateException due to closed session");
2904                     } catch (IllegalStateException e) {
2905                         // Expected, we can proceed further
2906                     }
2907                 } finally {
2908                     mTestRule.closeDevice(id);
2909                 }
2910             }
2911         }
2912     }
2913 }
2914