1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.extensions.internal.sessionprocessor;
18 
19 import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR;
20 import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY;
21 
22 import android.content.Context;
23 import android.graphics.ImageFormat;
24 import android.hardware.camera2.CameraCharacteristics;
25 import android.hardware.camera2.CameraDevice;
26 import android.hardware.camera2.CaptureRequest;
27 import android.hardware.camera2.CaptureResult;
28 import android.hardware.camera2.TotalCaptureResult;
29 import android.hardware.camera2.params.SessionConfiguration;
30 import android.util.Pair;
31 import android.util.Size;
32 
33 import androidx.annotation.GuardedBy;
34 import androidx.camera.core.Logger;
35 import androidx.camera.core.impl.CameraCaptureFailure;
36 import androidx.camera.core.impl.CameraCaptureResult;
37 import androidx.camera.core.impl.Config;
38 import androidx.camera.core.impl.OutputSurface;
39 import androidx.camera.core.impl.OutputSurfaceConfiguration;
40 import androidx.camera.core.impl.RequestProcessor;
41 import androidx.camera.core.impl.SessionProcessor;
42 import androidx.camera.core.impl.TagBundle;
43 import androidx.camera.extensions.ExtensionMode;
44 import androidx.camera.extensions.impl.CaptureProcessorImpl;
45 import androidx.camera.extensions.impl.CaptureStageImpl;
46 import androidx.camera.extensions.impl.ImageCaptureExtenderImpl;
47 import androidx.camera.extensions.impl.PreviewExtenderImpl;
48 import androidx.camera.extensions.impl.PreviewImageProcessorImpl;
49 import androidx.camera.extensions.impl.RequestUpdateProcessorImpl;
50 import androidx.camera.extensions.internal.Camera2CameraCaptureResult;
51 import androidx.camera.extensions.internal.ClientVersion;
52 import androidx.camera.extensions.internal.ExtensionVersion;
53 import androidx.camera.extensions.internal.RequestOptionConfig;
54 import androidx.camera.extensions.internal.VendorExtender;
55 import androidx.camera.extensions.internal.Version;
56 import androidx.camera.extensions.internal.compat.workaround.OnEnableDisableSessionDurationCheck;
57 import androidx.core.util.Preconditions;
58 
59 import org.jspecify.annotations.NonNull;
60 import org.jspecify.annotations.Nullable;
61 
62 import java.util.ArrayList;
63 import java.util.Collections;
64 import java.util.HashMap;
65 import java.util.LinkedHashMap;
66 import java.util.List;
67 import java.util.Map;
68 import java.util.concurrent.atomic.AtomicInteger;
69 
70 /**
71  * A {@link SessionProcessor} based on OEMs' basic extender implementation.
72  */
73 public class BasicExtenderSessionProcessor extends SessionProcessorBase {
74     private static final String TAG = "BasicSessionProcessor";
75 
76     private static final int PREVIEW_PROCESS_MAX_IMAGES = 2;
77     private static final long INVALID_TIMESTAMP = -1L;
78     private final @NonNull Context mContext;
79     private final @NonNull PreviewExtenderImpl mPreviewExtenderImpl;
80     private final @NonNull ImageCaptureExtenderImpl mImageCaptureExtenderImpl;
81 
82     volatile StillCaptureProcessor mStillCaptureProcessor = null;
83     volatile PreviewProcessor mPreviewProcessor = null;
84     volatile RequestUpdateProcessorImpl mRequestUpdateProcessor = null;
85     private volatile Camera2OutputConfig mPreviewOutputConfig;
86     private volatile Camera2OutputConfig mCaptureOutputConfig;
87     private volatile @Nullable Camera2OutputConfig mAnalysisOutputConfig = null;
88     private volatile OutputSurface mPreviewOutputSurface;
89     private volatile OutputSurface mCaptureOutputSurface;
90     private volatile RequestProcessor mRequestProcessor;
91     volatile boolean mIsCapturing = false;
92     private final AtomicInteger mNextCaptureSequenceId = new AtomicInteger(0);
93     static AtomicInteger sLastOutputConfigId = new AtomicInteger(0);
94     @GuardedBy("mLock")
95     private final Map<CaptureRequest.Key<?>, Object> mParameters = new LinkedHashMap<>();
96     @GuardedBy("mLock")
97     private final Map<Integer, Long> mRequestCompletedTimestampMap = new HashMap<>();
98     private OnEnableDisableSessionDurationCheck mOnEnableDisableSessionDurationCheck =
99             new OnEnableDisableSessionDurationCheck();
100     private @Nullable OutputSurface mPostviewOutputSurface;
101     private final VendorExtender mVendorExtender;
102     private final boolean mWillReceiveOnCaptureCompleted;
103 
BasicExtenderSessionProcessor(@onNull PreviewExtenderImpl previewExtenderImpl, @NonNull ImageCaptureExtenderImpl imageCaptureExtenderImpl, @NonNull List<CaptureRequest.Key<?>> supportedRequestKeys, @NonNull VendorExtender vendorExtender, @NonNull Context context, @ExtensionMode.Mode int mode)104     public BasicExtenderSessionProcessor(@NonNull PreviewExtenderImpl previewExtenderImpl,
105             @NonNull ImageCaptureExtenderImpl imageCaptureExtenderImpl,
106             @NonNull List<CaptureRequest.Key<?>> supportedRequestKeys,
107             @NonNull VendorExtender vendorExtender,
108             @NonNull Context context,
109             @ExtensionMode.Mode int mode) {
110         super(supportedRequestKeys, mode);
111         mPreviewExtenderImpl = previewExtenderImpl;
112         mImageCaptureExtenderImpl = imageCaptureExtenderImpl;
113         mContext = context;
114         mVendorExtender = vendorExtender;
115         mWillReceiveOnCaptureCompleted = mVendorExtender.willReceiveOnCaptureCompleted();
116     }
117 
118     @Override
initSessionInternal(@onNull String cameraId, @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, @NonNull OutputSurfaceConfiguration outputSurfaceConfiguration)119     protected @NonNull Camera2SessionConfig initSessionInternal(@NonNull String cameraId,
120             @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap,
121             @NonNull OutputSurfaceConfiguration outputSurfaceConfiguration) {
122         Logger.d(TAG, "PreviewExtenderImpl.onInit");
123         mPreviewExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
124                 mContext);
125         Logger.d(TAG, "ImageCaptureExtenderImpl.onInit");
126         mImageCaptureExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
127                 mContext);
128 
129         mPreviewOutputSurface = outputSurfaceConfiguration.getPreviewOutputSurface();
130         mCaptureOutputSurface = outputSurfaceConfiguration.getImageCaptureOutputSurface();
131         mPostviewOutputSurface = outputSurfaceConfiguration.getPostviewOutputSurface();
132 
133         // Preview
134         PreviewExtenderImpl.ProcessorType processorType =
135                 mPreviewExtenderImpl.getProcessorType();
136         Logger.d(TAG, "preview processorType=" + processorType);
137         if (processorType == PROCESSOR_TYPE_IMAGE_PROCESSOR) {
138             mPreviewOutputConfig = ImageReaderOutputConfig.create(
139                     sLastOutputConfigId.getAndIncrement(),
140                     mPreviewOutputSurface.getSize(),
141                     ImageFormat.YUV_420_888,
142                     PREVIEW_PROCESS_MAX_IMAGES);
143             PreviewImageProcessorImpl previewImageProcessor =
144                     (PreviewImageProcessorImpl) mPreviewExtenderImpl.getProcessor();
145             mPreviewProcessor = new PreviewProcessor(
146                     previewImageProcessor, mPreviewOutputSurface.getSurface(),
147                     mPreviewOutputSurface.getSize());
148         } else if (processorType == PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
149             mPreviewOutputConfig = SurfaceOutputConfig.create(
150                     sLastOutputConfigId.getAndIncrement(),
151                     mPreviewOutputSurface.getSurface());
152             mRequestUpdateProcessor =
153                     (RequestUpdateProcessorImpl) mPreviewExtenderImpl.getProcessor();
154         } else {
155             mPreviewOutputConfig = SurfaceOutputConfig.create(
156                     sLastOutputConfigId.getAndIncrement(),
157                     mPreviewOutputSurface.getSurface());
158         }
159 
160         // Image Capture
161         CaptureProcessorImpl captureProcessor = mImageCaptureExtenderImpl.getCaptureProcessor();
162         Logger.d(TAG, "CaptureProcessor=" + captureProcessor);
163 
164         if (captureProcessor != null) {
165             mCaptureOutputConfig = ImageReaderOutputConfig.create(
166                     sLastOutputConfigId.getAndIncrement(),
167                     mCaptureOutputSurface.getSize(),
168                     ImageFormat.YUV_420_888,
169                     mImageCaptureExtenderImpl.getMaxCaptureStage());
170             mStillCaptureProcessor = new StillCaptureProcessor(
171                     captureProcessor, mCaptureOutputSurface.getSurface(),
172                     mCaptureOutputSurface.getSize(),
173                     mPostviewOutputSurface,
174                     /* needOverrideTimestamp */ !mWillReceiveOnCaptureCompleted);
175         } else {
176             mCaptureOutputConfig = SurfaceOutputConfig.create(
177                     sLastOutputConfigId.getAndIncrement(),
178                     mCaptureOutputSurface.getSurface());
179         }
180 
181         // Image Analysis
182         if (outputSurfaceConfiguration.getImageAnalysisOutputSurface() != null) {
183             mAnalysisOutputConfig = SurfaceOutputConfig.create(
184                     sLastOutputConfigId.getAndIncrement(),
185                     outputSurfaceConfiguration.getImageAnalysisOutputSurface()
186                             .getSurface());
187         }
188 
189         Camera2SessionConfigBuilder builder =
190                 new Camera2SessionConfigBuilder()
191                         .addOutputConfig(mPreviewOutputConfig)
192                         .addOutputConfig(mCaptureOutputConfig)
193                         .setSessionTemplateId(CameraDevice.TEMPLATE_PREVIEW);
194 
195         if (ClientVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)
196                 && ExtensionVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)) {
197             int previewSessionType = mPreviewExtenderImpl.onSessionType();
198             int captureSessionType = mImageCaptureExtenderImpl.onSessionType();
199             Preconditions.checkArgument(previewSessionType == captureSessionType,
200                     "Needs same session type in both PreviewExtenderImpl and "
201                             + "ImageCaptureExtenderImpl");
202             if (previewSessionType == -1) { // -1 means using default value
203                 previewSessionType = SessionConfiguration.SESSION_REGULAR;
204             }
205             builder.setSessionType(previewSessionType);
206         }
207 
208         if (mAnalysisOutputConfig != null) {
209             builder.addOutputConfig(mAnalysisOutputConfig);
210         }
211 
212         CaptureStageImpl captureStagePreview = mPreviewExtenderImpl.onPresetSession();
213         Logger.d(TAG, "preview onPresetSession:" + captureStagePreview);
214 
215         CaptureStageImpl captureStageCapture = mImageCaptureExtenderImpl.onPresetSession();
216         Logger.d(TAG, "capture onPresetSession:" + captureStageCapture);
217 
218         if (captureStagePreview != null && captureStagePreview.getParameters() != null) {
219             for (Pair<CaptureRequest.Key, Object> parameter :
220                     captureStagePreview.getParameters()) {
221                 builder.addSessionParameter(parameter.first, parameter.second);
222             }
223         }
224 
225         if (captureStageCapture != null && captureStageCapture.getParameters() != null) {
226             for (Pair<CaptureRequest.Key, Object> parameter :
227                     captureStageCapture.getParameters()) {
228                 builder.addSessionParameter(parameter.first, parameter.second);
229             }
230         }
231         return builder.build();
232     }
233 
234     @Override
deInitSessionInternal()235     protected void deInitSessionInternal() {
236         if (mPreviewProcessor != null) {
237             mPreviewProcessor.close();
238             mPreviewProcessor = null;
239         }
240         if (mStillCaptureProcessor != null) {
241             mStillCaptureProcessor.close();
242             mStillCaptureProcessor = null;
243         }
244 
245         // Close the processor prior to OEMs's onDeinit in case OEMs block the thread for too
246         // long and the processor is closed too late.
247         Logger.d(TAG, "preview onDeInit");
248         mPreviewExtenderImpl.onDeInit();
249         Logger.d(TAG, "capture onDeInit");
250         mImageCaptureExtenderImpl.onDeInit();
251     }
252 
253     @Override
setParameters(@onNull Config config)254     public void setParameters(@NonNull Config config) {
255         synchronized (mLock) {
256             HashMap<CaptureRequest.Key<?>, Object> map = new HashMap<>();
257 
258             RequestOptionConfig options =
259                     RequestOptionConfig.Builder.from(config).build();
260 
261             for (Config.Option<?> option : options.listOptions()) {
262                 @SuppressWarnings("unchecked")
263                 CaptureRequest.Key<Object> key = (CaptureRequest.Key<Object>) option.getToken();
264                 map.put(key, options.retrieveOption(option));
265             }
266             mParameters.clear();
267             mParameters.putAll(map);
268         }
269     }
270 
271     @Override
onCaptureSessionStart(@onNull RequestProcessor requestProcessor)272     public void onCaptureSessionStart(@NonNull RequestProcessor requestProcessor) {
273         mRequestProcessor = requestProcessor;
274 
275         List<CaptureStageImpl> captureStages = new ArrayList<>();
276         CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onEnableSession();
277         Logger.d(TAG, "preview onEnableSession: " + captureStage1);
278         if (captureStage1 != null) {
279             captureStages.add(captureStage1);
280         }
281         CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onEnableSession();
282         Logger.d(TAG, "capture onEnableSession:" + captureStage2);
283         if (captureStage2 != null) {
284             captureStages.add(captureStage2);
285         }
286         mOnEnableDisableSessionDurationCheck.onEnableSessionInvoked();
287 
288         if (!captureStages.isEmpty()) {
289             submitRequestByCaptureStages(requestProcessor, captureStages);
290         }
291 
292         if (mPreviewProcessor != null) {
293             mPreviewProcessor.resume();
294             setImageProcessor(mPreviewOutputConfig.getId(),
295                     new ImageProcessor() {
296                         @Override
297                         public void onNextImageAvailable(int outputStreamId, long timestampNs,
298                                 @NonNull ImageReference imageReference,
299                                 @Nullable String physicalCameraId) {
300                             if (mPreviewProcessor != null) {
301                                 mPreviewProcessor.notifyImage(imageReference);
302                             } else {
303                                 imageReference.decrement();
304                             }
305                         }
306                     });
307         }
308     }
309 
applyParameters(RequestBuilder builder)310     private void applyParameters(RequestBuilder builder) {
311         synchronized (mLock) {
312             for (CaptureRequest.Key<?> key : mParameters.keySet()) {
313                 Object value = mParameters.get(key);
314                 if (value != null) {
315                     builder.setParameters(key, value);
316                 }
317             }
318         }
319     }
320 
submitRequestByCaptureStages(RequestProcessor requestProcessor, List<CaptureStageImpl> captureStageList)321     private void submitRequestByCaptureStages(RequestProcessor requestProcessor,
322             List<CaptureStageImpl> captureStageList) {
323         List<RequestProcessor.Request> requestList = new ArrayList<>();
324         for (CaptureStageImpl captureStage : captureStageList) {
325             RequestBuilder builder = new RequestBuilder();
326             builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
327             if (mAnalysisOutputConfig != null) {
328                 builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
329             }
330             for (Pair<CaptureRequest.Key, Object> keyObjectPair : captureStage.getParameters()) {
331                 builder.setParameters(keyObjectPair.first, keyObjectPair.second);
332             }
333             builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
334             requestList.add(builder.build());
335         }
336         requestProcessor.submit(requestList, new RequestProcessor.Callback() {
337         });
338     }
339 
340     @Override
onCaptureSessionEnd()341     public void onCaptureSessionEnd() {
342         mOnEnableDisableSessionDurationCheck.onDisableSessionInvoked();
343         if (mPreviewProcessor != null) {
344             mPreviewProcessor.pause();
345         }
346         List<CaptureStageImpl> captureStages = new ArrayList<>();
347         CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onDisableSession();
348         Logger.d(TAG, "preview onDisableSession: " + captureStage1);
349         if (captureStage1 != null) {
350             captureStages.add(captureStage1);
351         }
352         CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onDisableSession();
353         Logger.d(TAG, "capture onDisableSession:" + captureStage2);
354         if (captureStage2 != null) {
355             captureStages.add(captureStage2);
356         }
357 
358         if (!captureStages.isEmpty()) {
359             submitRequestByCaptureStages(mRequestProcessor, captureStages);
360         }
361         mRequestProcessor = null;
362         mIsCapturing = false;
363     }
364 
getCaptureResultKeyMapFromList( List<Pair<CaptureResult.Key, Object>> list)365     Map<CaptureResult.Key, Object> getCaptureResultKeyMapFromList(
366             List<Pair<CaptureResult.Key, Object>> list) {
367         Map<CaptureResult.Key, Object> map = new HashMap<>();
368         for (Pair<CaptureResult.Key, Object> pair : list) {
369             map.put(pair.first, pair.second);
370         }
371         return map;
372     }
373 
374     @Override
startRepeating(@onNull TagBundle tagBundle, @NonNull CaptureCallback captureCallback)375     public int startRepeating(@NonNull TagBundle tagBundle,
376             @NonNull CaptureCallback captureCallback) {
377         int repeatingCaptureSequenceId = mNextCaptureSequenceId.getAndIncrement();
378         if (mRequestProcessor == null) {
379             captureCallback.onCaptureFailed(repeatingCaptureSequenceId);
380             captureCallback.onCaptureSequenceAborted(repeatingCaptureSequenceId);
381         } else {
382             if (mPreviewProcessor != null) {
383                 mPreviewProcessor.start((shutterTimestamp, result) -> {
384                     captureCallback.onCaptureCompleted(shutterTimestamp,
385                             repeatingCaptureSequenceId,
386                             new KeyValueMapCameraCaptureResult(
387                                     shutterTimestamp,
388                                     tagBundle,
389                                     getCaptureResultKeyMapFromList(result))
390                     );
391                 });
392             }
393             updateRepeating(repeatingCaptureSequenceId, captureCallback);
394         }
395 
396         return repeatingCaptureSequenceId;
397     }
398 
updateRepeating(int repeatingCaptureSequenceId, @NonNull CaptureCallback captureCallback)399     void updateRepeating(int repeatingCaptureSequenceId, @NonNull CaptureCallback captureCallback) {
400         if (mRequestProcessor == null) {
401             Logger.d(TAG, "mRequestProcessor is null, ignore repeating request");
402             return;
403         }
404         RequestBuilder builder = new RequestBuilder();
405         builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
406         if (mAnalysisOutputConfig != null) {
407             builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
408         }
409         builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
410         applyParameters(builder);
411         applyPreviewStagesParameters(builder);
412 
413         RequestProcessor.Callback callback = new RequestProcessor.Callback() {
414             @Override
415             public void onCaptureCompleted(RequestProcessor.@NonNull Request request,
416                     @NonNull CameraCaptureResult cameraCaptureResult) {
417                 CaptureResult captureResult = cameraCaptureResult.getCaptureResult();
418                 Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
419                         "Cannot get TotalCaptureResult from the cameraCaptureResult ");
420                 TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;
421 
422                 if (mPreviewProcessor != null) {
423                     mPreviewProcessor.notifyCaptureResult(totalCaptureResult);
424                 } else {
425                     if (ClientVersion.isMinimumCompatibleVersion(Version.VERSION_1_3)
426                             && ExtensionVersion
427                             .isMinimumCompatibleVersion(Version.VERSION_1_3)) {
428                         Long timestamp = totalCaptureResult.get(CaptureResult.SENSOR_TIMESTAMP);
429                         if (timestamp != null) {
430                             captureCallback.onCaptureCompleted(timestamp,
431                                     repeatingCaptureSequenceId,
432                                     new Camera2CameraCaptureResult(totalCaptureResult));
433                         }
434                     }
435                 }
436 
437                 if (mRequestUpdateProcessor != null) {
438                     CaptureStageImpl captureStage =
439                             mRequestUpdateProcessor.process(totalCaptureResult);
440 
441                     if (captureStage != null) {
442                         updateRepeating(repeatingCaptureSequenceId, captureCallback);
443                     }
444                 }
445 
446                 captureCallback.onCaptureSequenceCompleted(repeatingCaptureSequenceId);
447             }
448         };
449 
450         Logger.d(TAG, "requestProcessor setRepeating");
451         mRequestProcessor.setRepeating(builder.build(), callback);
452     }
453 
applyPreviewStagesParameters(RequestBuilder builder)454     private void applyPreviewStagesParameters(RequestBuilder builder) {
455         CaptureStageImpl captureStage = mPreviewExtenderImpl.getCaptureStage();
456         if (captureStage != null) {
457             for (Pair<CaptureRequest.Key, Object> keyObjectPair :
458                     captureStage.getParameters()) {
459                 builder.setParameters(keyObjectPair.first, keyObjectPair.second);
460             }
461         }
462     }
463 
464     @Override
stopRepeating()465     public void stopRepeating() {
466         mRequestProcessor.stopRepeating();
467     }
468 
getRequestCompletedTimestamp(int captureSequenceId)469     private long getRequestCompletedTimestamp(int captureSequenceId) {
470         synchronized (mLock) {
471             Long timestamp = mRequestCompletedTimestampMap.get(captureSequenceId);
472             if (timestamp == null) {
473                 return INVALID_TIMESTAMP;
474             }
475             mRequestCompletedTimestampMap.remove(captureSequenceId);
476             return timestamp;
477         }
478     }
479 
480     @Override
startCapture(boolean postviewEnabled, @NonNull TagBundle tagBundle, @NonNull CaptureCallback captureCallback)481     public int startCapture(boolean postviewEnabled, @NonNull TagBundle tagBundle,
482             @NonNull CaptureCallback captureCallback) {
483         Logger.d(TAG, "startCapture postviewEnabled = " + postviewEnabled
484                 + " mWillReceiveOnCaptureCompleted = " + mWillReceiveOnCaptureCompleted);
485         int captureSequenceId = mNextCaptureSequenceId.getAndIncrement();
486 
487         if (mRequestProcessor == null || mIsCapturing) {
488             Logger.d(TAG, "startCapture failed");
489             captureCallback.onCaptureFailed(captureSequenceId);
490             captureCallback.onCaptureSequenceAborted(captureSequenceId);
491             return captureSequenceId;
492         }
493         mIsCapturing = true;
494 
495         List<RequestProcessor.Request> requestList = new ArrayList<>();
496         List<CaptureStageImpl> captureStages = mImageCaptureExtenderImpl.getCaptureStages();
497         List<Integer> captureIdList = new ArrayList<>();
498 
499         for (CaptureStageImpl captureStage : captureStages) {
500             RequestBuilder builder = new RequestBuilder();
501             builder.addTargetOutputConfigIds(mCaptureOutputConfig.getId());
502             builder.setTemplateId(CameraDevice.TEMPLATE_STILL_CAPTURE);
503             builder.setCaptureStageId(captureStage.getId());
504 
505             captureIdList.add(captureStage.getId());
506 
507             applyParameters(builder);
508             applyPreviewStagesParameters(builder);
509 
510             for (Pair<CaptureRequest.Key, Object> keyObjectPair :
511                     captureStage.getParameters()) {
512                 builder.setParameters(keyObjectPair.first, keyObjectPair.second);
513             }
514             requestList.add(builder.build());
515         }
516 
517         Logger.d(TAG, "Wait for capture stage id: " + captureIdList);
518 
519         RequestProcessor.Callback callback = new RequestProcessor.Callback() {
520             boolean mIsCaptureFailed = false;
521             boolean mIsCaptureStarted = false;
522 
523             @Override
524             public void onCaptureStarted(RequestProcessor.@NonNull Request request,
525                     long frameNumber, long timestamp) {
526                 if (!mIsCaptureStarted) {
527                     mIsCaptureStarted = true;
528                     captureCallback.onCaptureStarted(captureSequenceId, timestamp);
529                 }
530             }
531 
532             @Override
533             public void onCaptureCompleted(RequestProcessor.@NonNull Request request,
534                     @NonNull CameraCaptureResult cameraCaptureResult) {
535                 CaptureResult captureResult = cameraCaptureResult.getCaptureResult();
536                 Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
537                         "Cannot get capture TotalCaptureResult from the cameraCaptureResult ");
538                 TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;
539 
540                 RequestBuilder.RequestProcessorRequest requestProcessorRequest =
541                         (RequestBuilder.RequestProcessorRequest) request;
542 
543                 if (mStillCaptureProcessor != null) {
544                     synchronized (mLock) {
545                         if (!mRequestCompletedTimestampMap.containsKey(captureSequenceId)) {
546                             mRequestCompletedTimestampMap.put(
547                                     captureSequenceId, cameraCaptureResult.getTimestamp());
548                         }
549                     }
550 
551                     mStillCaptureProcessor.notifyCaptureResult(
552                             totalCaptureResult,
553                             requestProcessorRequest.getCaptureStageId());
554                 } else {
555                     // No CaptureProcessorImpl
556                     mIsCapturing = false;
557                     if (mRequestProcessor == null) {
558                         // notify the onCaptureSequenceAborted callback if onCaptureCompleted
559                         // happens but session is closed.
560                         captureCallback.onCaptureSequenceAborted(captureSequenceId);
561                         return;
562                     }
563                     captureCallback.onCaptureProcessStarted(captureSequenceId);
564                     captureCallback.onCaptureCompleted(cameraCaptureResult.getTimestamp(),
565                             captureSequenceId, new Camera2CameraCaptureResult(
566                                     tagBundle, cameraCaptureResult.getCaptureResult()));
567                     captureCallback.onCaptureSequenceCompleted(captureSequenceId);
568                 }
569             }
570 
571             @Override
572             public void onCaptureFailed(RequestProcessor.@NonNull Request request,
573                     @NonNull CameraCaptureFailure captureFailure) {
574                 if (!mIsCaptureFailed) {
575                     mIsCaptureFailed = true;
576                     captureCallback.onCaptureFailed(captureSequenceId);
577                     captureCallback.onCaptureSequenceAborted(captureSequenceId);
578                     mIsCapturing = false;
579                 }
580             }
581 
582             @Override
583             public void onCaptureSequenceAborted(int sequenceId) {
584                 captureCallback.onCaptureSequenceAborted(captureSequenceId);
585                 mIsCapturing = false;
586             }
587         };
588 
589         Logger.d(TAG, "startCapture");
590         if (mStillCaptureProcessor != null) {
591             setImageProcessor(mCaptureOutputConfig.getId(),
592                     new ImageProcessor() {
593                         boolean mIsFirstFrame = true;
594 
595                         @Override
596                         public void onNextImageAvailable(int outputStreamId, long timestampNs,
597                                 @NonNull ImageReference imageReference,
598                                 @Nullable String physicalCameraId) {
599                             Logger.d(TAG, "onNextImageAvailable  outputStreamId=" + outputStreamId);
600                             if (mStillCaptureProcessor != null) {
601                                 mStillCaptureProcessor.notifyImage(imageReference);
602                             } else {
603                                 imageReference.decrement();
604                             }
605 
606                             if (mIsFirstFrame) {
607                                 captureCallback.onCaptureProcessStarted(captureSequenceId);
608                                 mIsFirstFrame = false;
609                             }
610                         }
611                     });
612             mStillCaptureProcessor.startCapture(postviewEnabled, captureIdList,
613                     new StillCaptureProcessor.OnCaptureResultCallback() {
614                         @Override
615                         public void onProcessCompleted() {
616                             if (!mWillReceiveOnCaptureCompleted) {
617                                 // If ProcessResultImpl.onCaptureCompleted won't be invoked,
618                                 // We finish the capture sequence using the timestamp retrieved at
619                                 // onCaptureStarted when the process() completed.
620                                 long timestamp = getRequestCompletedTimestamp(captureSequenceId);
621                                 if (timestamp == INVALID_TIMESTAMP) {
622                                     Logger.e(TAG, "Cannot get timestamp for the capture result");
623                                     captureCallback.onCaptureFailed(captureSequenceId);
624                                     captureCallback.onCaptureSequenceAborted(captureSequenceId);
625                                     mIsCapturing = false;
626                                     return;
627                                 }
628                                 captureCallback.onCaptureCompleted(timestamp,
629                                         captureSequenceId,
630                                         new KeyValueMapCameraCaptureResult(
631                                                 timestamp,
632                                                 tagBundle,
633                                                 Collections.emptyMap()));
634                                 captureCallback.onCaptureSequenceCompleted(captureSequenceId);
635                             }
636                             mIsCapturing = false;
637                         }
638 
639                         @Override
640                         public void onError(@NonNull Exception e) {
641                             captureCallback.onCaptureFailed(captureSequenceId);
642                             mIsCapturing = false;
643                         }
644 
645                         @Override
646                         public void onCaptureCompleted(long shutterTimestamp,
647                                 @NonNull List<Pair<CaptureResult.Key, Object>> result) {
648                             if (mWillReceiveOnCaptureCompleted) {
649                                 captureCallback.onCaptureCompleted(shutterTimestamp,
650                                         captureSequenceId,
651                                         new KeyValueMapCameraCaptureResult(
652                                                 shutterTimestamp, tagBundle,
653                                                 getCaptureResultKeyMapFromList(result)));
654                                 captureCallback.onCaptureSequenceCompleted(
655                                         captureSequenceId);
656                             }
657                         }
658 
659                         @Override
660                         public void onCaptureProcessProgressed(int progress) {
661                             captureCallback.onCaptureProcessProgressed(progress);
662                         }
663                     });
664         }
665 
666         mRequestProcessor.submit(requestList, callback);
667         return captureSequenceId;
668     }
669 
670     @Override
abortCapture(int captureSequenceId)671     public void abortCapture(int captureSequenceId) {
672         mRequestProcessor.abortCaptures();
673     }
674 
675     @Override
startTrigger(@onNull Config config, @NonNull TagBundle tagBundle, @NonNull CaptureCallback callback)676     public int startTrigger(@NonNull Config config, @NonNull TagBundle tagBundle,
677             @NonNull CaptureCallback callback) {
678         Logger.d(TAG, "startTrigger");
679         int captureSequenceId = mNextCaptureSequenceId.getAndIncrement();
680         RequestBuilder builder = new RequestBuilder();
681         builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
682         if (mAnalysisOutputConfig != null) {
683             builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
684         }
685         builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
686         applyParameters(builder);
687         applyPreviewStagesParameters(builder);
688 
689         RequestOptionConfig options =
690                 RequestOptionConfig.Builder.from(config).build();
691         for (Config.Option<?> option : options.listOptions()) {
692             @SuppressWarnings("unchecked")
693             CaptureRequest.Key<Object> key = (CaptureRequest.Key<Object>) option.getToken();
694             builder.setParameters(key, options.retrieveOption(option));
695         }
696 
697         mRequestProcessor.submit(builder.build(), new RequestProcessor.Callback() {
698             @Override
699             public void onCaptureCompleted(RequestProcessor.@NonNull Request request,
700                     @NonNull CameraCaptureResult captureResult) {
701                 callback.onCaptureCompleted(captureResult.getTimestamp(), captureSequenceId,
702                         new Camera2CameraCaptureResult(tagBundle,
703                                 captureResult.getCaptureResult()));
704                 callback.onCaptureSequenceCompleted(captureSequenceId);
705             }
706 
707             @Override
708             public void onCaptureFailed(RequestProcessor.@NonNull Request request,
709                     @NonNull CameraCaptureFailure captureFailure) {
710                 callback.onCaptureFailed(captureSequenceId);
711             }
712         });
713 
714         return captureSequenceId;
715     }
716 
717     @Override
getRealtimeCaptureLatency()718     public @Nullable Pair<Long, Long> getRealtimeCaptureLatency() {
719         if (ClientVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)
720                 && ExtensionVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)) {
721             return mImageCaptureExtenderImpl.getRealtimeCaptureLatency();
722         }
723         return null;
724     }
725 
726     @Override
getSupportedPostviewSize(@onNull Size captureSize)727     public @NonNull Map<Integer, List<Size>> getSupportedPostviewSize(@NonNull Size captureSize) {
728         return mVendorExtender.getSupportedPostviewResolutions(captureSize);
729     }
730 }
731