1 /*
2  * Copyright 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.extensions.internal.sessionprocessor;
18 
19 import android.hardware.camera2.CameraCharacteristics;
20 import android.hardware.camera2.CaptureRequest;
21 import android.media.Image;
22 import android.media.ImageReader;
23 import android.os.Handler;
24 import android.os.HandlerThread;
25 import android.util.Pair;
26 
27 import androidx.annotation.GuardedBy;
28 import androidx.camera.core.CameraInfo;
29 import androidx.camera.core.CameraXThreads;
30 import androidx.camera.core.Logger;
31 import androidx.camera.core.impl.AdapterCameraInfo.CameraOperation;
32 import androidx.camera.core.impl.CameraInfoInternal;
33 import androidx.camera.core.impl.DeferrableSurface;
34 import androidx.camera.core.impl.OutputSurfaceConfiguration;
35 import androidx.camera.core.impl.SessionConfig;
36 import androidx.camera.core.impl.SessionProcessor;
37 import androidx.camera.core.impl.SessionProcessorSurface;
38 import androidx.camera.core.impl.utils.executor.CameraXExecutors;
39 import androidx.camera.extensions.CameraExtensionsControl;
40 import androidx.camera.extensions.CameraExtensionsInfo;
41 import androidx.camera.extensions.ExtensionMode;
42 import androidx.camera.extensions.internal.ExtensionsUtils;
43 import androidx.camera.extensions.internal.RequestOptionConfig;
44 
45 import org.jspecify.annotations.NonNull;
46 import org.jspecify.annotations.Nullable;
47 
48 import java.util.ArrayList;
49 import java.util.HashMap;
50 import java.util.List;
51 import java.util.Map;
52 import java.util.Objects;
53 import java.util.Set;
54 
55 /**
56  * Base class for SessionProcessor implementation. It is responsible for creating image readers and
57  * maintaining the {@link ImageProcessor} associated with the image reader.
58  */
59 abstract class SessionProcessorBase implements SessionProcessor, CameraExtensionsInfo,
60         CameraExtensionsControl {
61     private static final String TAG = "SessionProcessorBase";
62     /**
63      * Unknown extension strength.
64      */
65     protected static final int EXTENSION_STRENGTH_UNKNOWN = -1;
66     @GuardedBy("mLock")
67     private final @NonNull Map<Integer, ImageReader> mImageReaderMap = new HashMap<>();
68     @GuardedBy("mLock")
69     private final Map<Integer, Camera2OutputConfig> mOutputConfigMap = new HashMap<>();
70 
71     private @Nullable HandlerThread mImageReaderHandlerThread;
72     @GuardedBy("mLock")
73     private final List<DeferrableSurface> mSurfacesList = new ArrayList<>();
74     protected final Object mLock = new Object();
75     private String mCameraId;
76 
77     private final @CameraOperation @NonNull Set<Integer> mSupportedCameraOperations;
78     private final @ExtensionMode.Mode int mMode;
79     @GuardedBy("mLock")
80     protected int mExtensionStrength = EXTENSION_STRENGTH_UNKNOWN;
81 
SessionProcessorBase(@onNull List<CaptureRequest.Key<?>> supportedParameterKeys, @ExtensionMode.Mode int mode)82     SessionProcessorBase(@NonNull List<CaptureRequest.Key<?>> supportedParameterKeys,
83             @ExtensionMode.Mode int mode) {
84         mSupportedCameraOperations = ExtensionsUtils.getSupportedCameraOperations(
85                 supportedParameterKeys);
86         mMode = mode;
87     }
88 
createOutputConfigSurface( @onNull Camera2OutputConfig outputConfig, Map<Integer, ImageReader> imageReaderMap)89     private static @NonNull SessionProcessorSurface createOutputConfigSurface(
90             @NonNull Camera2OutputConfig outputConfig, Map<Integer, ImageReader> imageReaderMap) {
91         if (outputConfig instanceof SurfaceOutputConfig) {
92             SurfaceOutputConfig surfaceOutputConfig = (SurfaceOutputConfig) outputConfig;
93             SessionProcessorSurface surface =
94                     new SessionProcessorSurface(surfaceOutputConfig.getSurface(),
95                             outputConfig.getId());
96             return surface;
97         } else if (outputConfig instanceof ImageReaderOutputConfig) {
98             ImageReaderOutputConfig imageReaderOutputConfig =
99                     (ImageReaderOutputConfig) outputConfig;
100 
101             ImageReader imageReader =
102                     ImageReader.newInstance(imageReaderOutputConfig.getSize().getWidth(),
103                             imageReaderOutputConfig.getSize().getHeight(),
104                             imageReaderOutputConfig.getImageFormat(),
105                             imageReaderOutputConfig.getMaxImages());
106             imageReaderMap.put(outputConfig.getId(), imageReader);
107             SessionProcessorSurface surface =
108                     new SessionProcessorSurface(imageReader.getSurface(),
109                             outputConfig.getId());
110             surface.getTerminationFuture().addListener(() -> {
111                 imageReader.close();
112             }, CameraXExecutors.directExecutor());
113             return surface;
114         } else if (outputConfig instanceof MultiResolutionImageReaderOutputConfig) {
115             throw new UnsupportedOperationException("MultiResolutionImageReader not supported yet");
116         }
117         throw new UnsupportedOperationException("Unsupported Camera2OutputConfig:" + outputConfig);
118     }
119 
120     @Override
initSession(@onNull CameraInfo cameraInfo, @Nullable OutputSurfaceConfiguration outputSurfaceConfiguration)121     public final @NonNull SessionConfig initSession(@NonNull CameraInfo cameraInfo,
122             @Nullable OutputSurfaceConfiguration outputSurfaceConfiguration) {
123         CameraInfoInternal cameraInfoInternal = (CameraInfoInternal) cameraInfo;
124         Map<String, CameraCharacteristics> characteristicsMap =
125                 ExtensionsUtils.getCameraCharacteristicsMap(cameraInfoInternal);
126         Camera2SessionConfig camera2SessionConfig = initSessionInternal(
127                 cameraInfoInternal.getCameraId(), characteristicsMap,
128                 Objects.requireNonNull(outputSurfaceConfiguration));
129 
130         SessionConfig.Builder sessionConfigBuilder = new SessionConfig.Builder();
131         synchronized (mLock) {
132             for (Camera2OutputConfig outputConfig : camera2SessionConfig.getOutputConfigs()) {
133                 SessionProcessorSurface sessionProcessorSurface =
134                         createOutputConfigSurface(outputConfig, mImageReaderMap);
135                 mSurfacesList.add(sessionProcessorSurface);
136                 mOutputConfigMap.put(outputConfig.getId(), outputConfig);
137 
138                 SessionConfig.OutputConfig.Builder outputConfigBuilder =
139                         SessionConfig.OutputConfig.builder(sessionProcessorSurface)
140                                 .setPhysicalCameraId(outputConfig.getPhysicalCameraId())
141                                 .setSurfaceGroupId(outputConfig.getSurfaceGroupId());
142 
143                 List<Camera2OutputConfig> sharedOutputs =
144                         outputConfig.getSurfaceSharingOutputConfigs();
145                 if (sharedOutputs != null && !sharedOutputs.isEmpty()) {
146                     List<DeferrableSurface> sharedSurfaces = new ArrayList<>();
147                     for (Camera2OutputConfig sharedOutput : sharedOutputs) {
148                         mOutputConfigMap.put(sharedOutput.getId(), sharedOutput);
149                         sharedSurfaces.add(createOutputConfigSurface(sharedOutput,
150                                 mImageReaderMap));
151                     }
152                     outputConfigBuilder.setSharedSurfaces(sharedSurfaces);
153                 }
154                 sessionConfigBuilder.addOutputConfig(outputConfigBuilder.build());
155             }
156         }
157 
158         RequestOptionConfig.Builder camera2ConfigurationBuilder = new RequestOptionConfig.Builder();
159         for (CaptureRequest.Key<?> key : camera2SessionConfig.getSessionParameters().keySet()) {
160             @SuppressWarnings("unchecked")
161             CaptureRequest.Key<Object> objKey = (CaptureRequest.Key<Object>) key;
162             Object value = camera2SessionConfig.getSessionParameters().get(objKey);
163             camera2ConfigurationBuilder.setCaptureRequestOption(objKey, value);
164         }
165         sessionConfigBuilder.setImplementationOptions(camera2ConfigurationBuilder.build());
166         sessionConfigBuilder.setTemplateType(camera2SessionConfig.getSessionTemplateId());
167         sessionConfigBuilder.setSessionType(camera2SessionConfig.getSessionType());
168 
169         mImageReaderHandlerThread = new HandlerThread(
170                 CameraXThreads.TAG + "extensions_image_reader");
171         mImageReaderHandlerThread.start();
172 
173         mCameraId = cameraInfoInternal.getCameraId();
174         Logger.d(TAG, "initSession: cameraId=" + mCameraId);
175         return sessionConfigBuilder.build();
176     }
177 
178     @Override
getSupportedCameraOperations()179     public @CameraOperation @NonNull Set<Integer> getSupportedCameraOperations() {
180         return mSupportedCameraOperations;
181     }
182 
initSessionInternal(@onNull String cameraId, @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, @NonNull OutputSurfaceConfiguration outputSurfaceConfig)183     protected abstract @NonNull Camera2SessionConfig initSessionInternal(@NonNull String cameraId,
184             @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap,
185             @NonNull OutputSurfaceConfiguration outputSurfaceConfig);
186 
187 
setImageProcessor(int outputConfigId, @NonNull ImageProcessor imageProcessor)188     protected void setImageProcessor(int outputConfigId,
189             @NonNull ImageProcessor imageProcessor) {
190         ImageReader imageReader;
191         String physicalCameraId;
192         synchronized (mLock) {
193             imageReader = mImageReaderMap.get(outputConfigId);
194             Camera2OutputConfig outputConfig = mOutputConfigMap.get(outputConfigId);
195             physicalCameraId = (outputConfig == null ? null : outputConfig.getPhysicalCameraId());
196         }
197 
198         if (imageReader != null) {
199             imageReader.setOnImageAvailableListener(reader -> {
200                 try {
201                     Image image = reader.acquireNextImage();
202                     ImageReference imageReference = new ImageRefHolder(image);
203                     imageProcessor.onNextImageAvailable(outputConfigId, image.getTimestamp(),
204                             imageReference, physicalCameraId);
205                 } catch (IllegalStateException e) {
206                     Logger.e(TAG, "Failed to acquire next image.", e);
207                 }
208             }, new Handler(mImageReaderHandlerThread.getLooper()));
209         }
210     }
211 
212     @Override
deInitSession()213     public final void deInitSession() {
214         Logger.e(TAG, "deInitSession: cameraId=" + mCameraId);
215 
216         deInitSessionInternal();
217 
218         synchronized (mLock) {
219             for (DeferrableSurface deferrableSurface : mSurfacesList) {
220                 deferrableSurface.close();
221             }
222             mSurfacesList.clear();
223             mImageReaderMap.clear();
224             mOutputConfigMap.clear();
225             mExtensionStrength = EXTENSION_STRENGTH_UNKNOWN;
226         }
227 
228         if (mImageReaderHandlerThread != null) {
229             mImageReaderHandlerThread.quitSafely();
230             mImageReaderHandlerThread = null;
231         }
232     }
233 
deInitSessionInternal()234     protected abstract void deInitSessionInternal();
235 
236     @Override
getImplementationType()237     public @NonNull Pair<Integer, Integer> getImplementationType() {
238         return Pair.create(SessionProcessor.TYPE_VENDOR_LIBRARY, mMode);
239     }
240 
241     private static class ImageRefHolder implements ImageReference {
242         private int mRefCount;
243         private final Image mImage;
244         private final Object mImageLock = new Object();
245 
246         @SuppressWarnings("WeakerAccess") /* synthetic accessor */
ImageRefHolder(@onNull Image image)247         ImageRefHolder(@NonNull Image image) {
248             mRefCount = 1;
249             mImage = image;
250         }
251 
252         @Override
increment()253         public boolean increment() {
254             synchronized (mImageLock) {
255                 if (mRefCount <= 0) {
256                     return false;
257                 }
258                 mRefCount++;
259             }
260             return true;
261         }
262 
263         @Override
decrement()264         public boolean decrement() {
265             synchronized (mImageLock) {
266                 if (mRefCount <= 0) {
267                     return false;
268                 }
269                 mRefCount--;
270                 if (mRefCount <= 0) {
271                     mImage.close();
272                 }
273             }
274             return true;
275         }
276 
277         @Override
get()278         public @Nullable Image get() {
279             return mImage;
280         }
281     }
282 }
283