1 /*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera2-FrameProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23
24 #include "common/CameraDeviceBase.h"
25 #include "api1/Camera2Client.h"
26 #include "api1/client2/FrameProcessor.h"
27
28 namespace android {
29 namespace camera2 {
30
FrameProcessor(wp<CameraDeviceBase> device,sp<Camera2Client> client)31 FrameProcessor::FrameProcessor(wp<CameraDeviceBase> device,
32 sp<Camera2Client> client) :
33 FrameProcessorBase(device),
34 mClient(client),
35 mLastFrameNumberOfFaces(0),
36 mLast3AFrameNumber(-1) {
37
38 sp<CameraDeviceBase> d = device.promote();
39 mSynthesize3ANotify = !(d->willNotify3A());
40
41 {
42 SharedParameters::Lock l(client->getParameters());
43 mUsePartialQuirk = l.mParameters.quirks.partialResults;
44
45 // Initialize starting 3A state
46 m3aState.afTriggerId = l.mParameters.afTriggerCounter;
47 m3aState.aeTriggerId = l.mParameters.precaptureTriggerCounter;
48 // Check if lens is fixed-focus
49 if (l.mParameters.focusMode == Parameters::FOCUS_MODE_FIXED) {
50 m3aState.afMode = ANDROID_CONTROL_AF_MODE_OFF;
51 }
52 }
53 }
54
~FrameProcessor()55 FrameProcessor::~FrameProcessor() {
56 }
57
processSingleFrame(CameraMetadata & frame,const sp<CameraDeviceBase> & device)58 bool FrameProcessor::processSingleFrame(CameraMetadata &frame,
59 const sp<CameraDeviceBase> &device) {
60
61 sp<Camera2Client> client = mClient.promote();
62 if (!client.get()) {
63 return false;
64 }
65
66 bool partialResult = false;
67 if (mUsePartialQuirk) {
68 camera_metadata_entry_t entry;
69 entry = frame.find(ANDROID_QUIRKS_PARTIAL_RESULT);
70 if (entry.count > 0 &&
71 entry.data.u8[0] == ANDROID_QUIRKS_PARTIAL_RESULT_PARTIAL) {
72 partialResult = true;
73 }
74 }
75
76 if (!partialResult && processFaceDetect(frame, client) != OK) {
77 return false;
78 }
79
80 if (mSynthesize3ANotify) {
81 process3aState(frame, client);
82 }
83
84 return FrameProcessorBase::processSingleFrame(frame, device);
85 }
86
processFaceDetect(const CameraMetadata & frame,const sp<Camera2Client> & client)87 status_t FrameProcessor::processFaceDetect(const CameraMetadata &frame,
88 const sp<Camera2Client> &client) {
89 status_t res = BAD_VALUE;
90 ATRACE_CALL();
91 camera_metadata_ro_entry_t entry;
92 bool enableFaceDetect;
93
94 {
95 SharedParameters::Lock l(client->getParameters());
96 enableFaceDetect = l.mParameters.enableFaceDetect;
97 }
98 entry = frame.find(ANDROID_STATISTICS_FACE_DETECT_MODE);
99
100 // TODO: This should be an error once implementations are compliant
101 if (entry.count == 0) {
102 return OK;
103 }
104
105 uint8_t faceDetectMode = entry.data.u8[0];
106
107 camera_frame_metadata metadata;
108 Vector<camera_face_t> faces;
109 metadata.number_of_faces = 0;
110
111 if (enableFaceDetect &&
112 faceDetectMode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
113
114 SharedParameters::Lock l(client->getParameters());
115 entry = frame.find(ANDROID_STATISTICS_FACE_RECTANGLES);
116 if (entry.count == 0) {
117 // No faces this frame
118 /* warning: locks SharedCameraCallbacks */
119 callbackFaceDetection(client, metadata);
120 return OK;
121 }
122 metadata.number_of_faces = entry.count / 4;
123 if (metadata.number_of_faces >
124 l.mParameters.fastInfo.maxFaces) {
125 ALOGE("%s: Camera %d: More faces than expected! (Got %d, max %d)",
126 __FUNCTION__, client->getCameraId(),
127 metadata.number_of_faces, l.mParameters.fastInfo.maxFaces);
128 return res;
129 }
130 const int32_t *faceRects = entry.data.i32;
131
132 entry = frame.find(ANDROID_STATISTICS_FACE_SCORES);
133 if (entry.count == 0) {
134 ALOGE("%s: Camera %d: Unable to read face scores",
135 __FUNCTION__, client->getCameraId());
136 return res;
137 }
138 const uint8_t *faceScores = entry.data.u8;
139
140 const int32_t *faceLandmarks = NULL;
141 const int32_t *faceIds = NULL;
142
143 if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
144 entry = frame.find(ANDROID_STATISTICS_FACE_LANDMARKS);
145 if (entry.count == 0) {
146 ALOGE("%s: Camera %d: Unable to read face landmarks",
147 __FUNCTION__, client->getCameraId());
148 return res;
149 }
150 faceLandmarks = entry.data.i32;
151
152 entry = frame.find(ANDROID_STATISTICS_FACE_IDS);
153
154 if (entry.count == 0) {
155 ALOGE("%s: Camera %d: Unable to read face IDs",
156 __FUNCTION__, client->getCameraId());
157 return res;
158 }
159 faceIds = entry.data.i32;
160 }
161
162 faces.setCapacity(metadata.number_of_faces);
163
164 size_t maxFaces = metadata.number_of_faces;
165 for (size_t i = 0; i < maxFaces; i++) {
166 if (faceScores[i] == 0) {
167 metadata.number_of_faces--;
168 continue;
169 }
170 if (faceScores[i] > 100) {
171 ALOGW("%s: Face index %d with out of range score %d",
172 __FUNCTION__, i, faceScores[i]);
173 }
174
175 camera_face_t face;
176
177 face.rect[0] = l.mParameters.arrayXToNormalized(faceRects[i*4 + 0]);
178 face.rect[1] = l.mParameters.arrayYToNormalized(faceRects[i*4 + 1]);
179 face.rect[2] = l.mParameters.arrayXToNormalized(faceRects[i*4 + 2]);
180 face.rect[3] = l.mParameters.arrayYToNormalized(faceRects[i*4 + 3]);
181
182 face.score = faceScores[i];
183 if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
184 face.id = faceIds[i];
185 face.left_eye[0] =
186 l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 0]);
187 face.left_eye[1] =
188 l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 1]);
189 face.right_eye[0] =
190 l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 2]);
191 face.right_eye[1] =
192 l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 3]);
193 face.mouth[0] =
194 l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 4]);
195 face.mouth[1] =
196 l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 5]);
197 } else {
198 face.id = 0;
199 face.left_eye[0] = face.left_eye[1] = -2000;
200 face.right_eye[0] = face.right_eye[1] = -2000;
201 face.mouth[0] = face.mouth[1] = -2000;
202 }
203 faces.push_back(face);
204 }
205
206 metadata.faces = faces.editArray();
207 }
208
209 /* warning: locks SharedCameraCallbacks */
210 callbackFaceDetection(client, metadata);
211
212 return OK;
213 }
214
process3aState(const CameraMetadata & frame,const sp<Camera2Client> & client)215 status_t FrameProcessor::process3aState(const CameraMetadata &frame,
216 const sp<Camera2Client> &client) {
217
218 ATRACE_CALL();
219 camera_metadata_ro_entry_t entry;
220 int cameraId = client->getCameraId();
221
222 entry = frame.find(ANDROID_REQUEST_FRAME_COUNT);
223 int32_t frameNumber = entry.data.i32[0];
224
225 // Don't send 3A notifications for the same frame number twice
226 if (frameNumber <= mLast3AFrameNumber) {
227 ALOGV("%s: Already sent 3A for frame number %d, skipping",
228 __FUNCTION__, frameNumber);
229 return OK;
230 }
231
232 mLast3AFrameNumber = frameNumber;
233
234 // Get 3A states from result metadata
235 bool gotAllStates = true;
236
237 AlgState new3aState;
238
239 // TODO: Also use AE mode, AE trigger ID
240
241 gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AF_MODE,
242 &new3aState.afMode, frameNumber, cameraId);
243
244 gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AWB_MODE,
245 &new3aState.awbMode, frameNumber, cameraId);
246
247 gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AE_STATE,
248 &new3aState.aeState, frameNumber, cameraId);
249
250 gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AF_STATE,
251 &new3aState.afState, frameNumber, cameraId);
252
253 gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AWB_STATE,
254 &new3aState.awbState, frameNumber, cameraId);
255
256 gotAllStates &= get3aResult<int32_t>(frame, ANDROID_CONTROL_AF_TRIGGER_ID,
257 &new3aState.afTriggerId, frameNumber, cameraId);
258
259 gotAllStates &= get3aResult<int32_t>(frame, ANDROID_CONTROL_AE_PRECAPTURE_ID,
260 &new3aState.aeTriggerId, frameNumber, cameraId);
261
262 if (!gotAllStates) return BAD_VALUE;
263
264 if (new3aState.aeState != m3aState.aeState) {
265 ALOGV("%s: Camera %d: AE state %d->%d",
266 __FUNCTION__, cameraId,
267 m3aState.aeState, new3aState.aeState);
268 client->notifyAutoExposure(new3aState.aeState, new3aState.aeTriggerId);
269 }
270
271 if (new3aState.afState != m3aState.afState ||
272 new3aState.afMode != m3aState.afMode ||
273 new3aState.afTriggerId != m3aState.afTriggerId) {
274 ALOGV("%s: Camera %d: AF state %d->%d. AF mode %d->%d. Trigger %d->%d",
275 __FUNCTION__, cameraId,
276 m3aState.afState, new3aState.afState,
277 m3aState.afMode, new3aState.afMode,
278 m3aState.afTriggerId, new3aState.afTriggerId);
279 client->notifyAutoFocus(new3aState.afState, new3aState.afTriggerId);
280 }
281 if (new3aState.awbState != m3aState.awbState ||
282 new3aState.awbMode != m3aState.awbMode) {
283 ALOGV("%s: Camera %d: AWB state %d->%d. AWB mode %d->%d",
284 __FUNCTION__, cameraId,
285 m3aState.awbState, new3aState.awbState,
286 m3aState.awbMode, new3aState.awbMode);
287 client->notifyAutoWhitebalance(new3aState.awbState,
288 new3aState.aeTriggerId);
289 }
290
291 m3aState = new3aState;
292
293 return OK;
294 }
295
296 template<typename Src, typename T>
get3aResult(const CameraMetadata & result,int32_t tag,T * value,int32_t frameNumber,int cameraId)297 bool FrameProcessor::get3aResult(const CameraMetadata& result, int32_t tag,
298 T* value, int32_t frameNumber, int cameraId) {
299 camera_metadata_ro_entry_t entry;
300 if (value == NULL) {
301 ALOGE("%s: Camera %d: Value to write to is NULL",
302 __FUNCTION__, cameraId);
303 return false;
304 }
305
306 entry = result.find(tag);
307 if (entry.count == 0) {
308 ALOGE("%s: Camera %d: No %s provided by HAL for frame %d!",
309 __FUNCTION__, cameraId,
310 get_camera_metadata_tag_name(tag), frameNumber);
311 return false;
312 } else {
313 switch(sizeof(Src)){
314 case sizeof(uint8_t):
315 *value = static_cast<T>(entry.data.u8[0]);
316 break;
317 case sizeof(int32_t):
318 *value = static_cast<T>(entry.data.i32[0]);
319 break;
320 default:
321 ALOGE("%s: Camera %d: Unsupported source",
322 __FUNCTION__, cameraId);
323 return false;
324 }
325 }
326 return true;
327 }
328
329
callbackFaceDetection(sp<Camera2Client> client,const camera_frame_metadata & metadata)330 void FrameProcessor::callbackFaceDetection(sp<Camera2Client> client,
331 const camera_frame_metadata &metadata) {
332
333 camera_frame_metadata *metadata_ptr =
334 const_cast<camera_frame_metadata*>(&metadata);
335
336 /**
337 * Filter out repeated 0-face callbacks,
338 * but not when the last frame was >0
339 */
340 if (metadata.number_of_faces != 0 ||
341 mLastFrameNumberOfFaces != metadata.number_of_faces) {
342
343 Camera2Client::SharedCameraCallbacks::Lock
344 l(client->mSharedCameraCallbacks);
345 if (l.mRemoteCallback != NULL) {
346 l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_METADATA,
347 NULL,
348 metadata_ptr);
349 }
350 }
351
352 mLastFrameNumberOfFaces = metadata.number_of_faces;
353 }
354
355 }; // namespace camera2
356 }; // namespace android
357