1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera3-RotCropMapper"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <algorithm>
22 #include <cmath>
23
24 #include "device3/RotateAndCropMapper.h"
25
26 namespace android {
27
28 namespace camera3 {
29
initRemappedKeys()30 void RotateAndCropMapper::initRemappedKeys() {
31 mRemappedKeys.insert(
32 kMeteringRegionsToCorrect.begin(),
33 kMeteringRegionsToCorrect.end());
34 mRemappedKeys.insert(
35 kResultPointsToCorrectNoClamp.begin(),
36 kResultPointsToCorrectNoClamp.end());
37
38 mRemappedKeys.insert(ANDROID_SCALER_ROTATE_AND_CROP);
39 mRemappedKeys.insert(ANDROID_SCALER_CROP_REGION);
40 }
41
isNeeded(const CameraMetadata * deviceInfo)42 bool RotateAndCropMapper::isNeeded(const CameraMetadata* deviceInfo) {
43 auto entry = deviceInfo->find(ANDROID_SCALER_AVAILABLE_ROTATE_AND_CROP_MODES);
44 for (size_t i = 0; i < entry.count; i++) {
45 if (entry.data.u8[i] == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) return true;
46 }
47 return false;
48 }
49
RotateAndCropMapper(const CameraMetadata * deviceInfo)50 RotateAndCropMapper::RotateAndCropMapper(const CameraMetadata* deviceInfo) {
51 initRemappedKeys();
52
53 auto entry = deviceInfo->find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
54 if (entry.count != 4) return;
55
56 mArrayWidth = entry.data.i32[2];
57 mArrayHeight = entry.data.i32[3];
58 mArrayAspect = static_cast<float>(mArrayWidth) / mArrayHeight;
59 mRotateAspect = 1.f/mArrayAspect;
60 }
61
62 /**
63 * Adjust capture request when rotate and crop AUTO is enabled
64 */
updateCaptureRequest(CameraMetadata * request)65 status_t RotateAndCropMapper::updateCaptureRequest(CameraMetadata *request) {
66 auto entry = request->find(ANDROID_SCALER_ROTATE_AND_CROP);
67 if (entry.count == 0) return OK;
68 uint8_t rotateMode = entry.data.u8[0];
69 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
70
71 int32_t cx = 0;
72 int32_t cy = 0;
73 int32_t cw = mArrayWidth;
74 int32_t ch = mArrayHeight;
75 entry = request->find(ANDROID_SCALER_CROP_REGION);
76 if (entry.count == 4) {
77 cx = entry.data.i32[0];
78 cy = entry.data.i32[1];
79 cw = entry.data.i32[2];
80 ch = entry.data.i32[3];
81 }
82
83 // User inputs are relative to the rotated-and-cropped view, so convert back
84 // to active array coordinates. To be more specific, the application is
85 // calculating coordinates based on the crop rectangle and the active array,
86 // even though the view the user sees is the cropped-and-rotated one. So we
87 // need to adjust the coordinates so that a point that would be on the
88 // top-left corner of the crop region is mapped to the top-left corner of
89 // the rotated-and-cropped fov within the crop region, and the same for the
90 // bottom-right corner.
91 //
92 // Since the zoom ratio control scales everything uniformly (so an app does
93 // not need to adjust anything if it wants to put a metering region on the
94 // top-left quadrant of the preview FOV, when changing zoomRatio), it does
95 // not need to be factored into this calculation at all.
96 //
97 // ->+x active array aw
98 // |+--------------------------------------------------------------------+
99 // v| |
100 // +y| a 1 cw 2 b |
101 // | +=========*HHHHHHHHHHHHHHH*===========+ |
102 // | I H rw H I |
103 // | I H H I |
104 // | I H H I |
105 //ah | ch I H rh H I crop region |
106 // | I H H I |
107 // | I H H I |
108 // | I H rotate region H I |
109 // | +=========*HHHHHHHHHHHHHHH*===========+ |
110 // | d 4 3 c |
111 // | |
112 // +--------------------------------------------------------------------+
113 //
114 // aw , ah = active array width,height
115 // cw , ch = crop region width,height
116 // rw , rh = rotated-and-cropped region width,height
117 // aw / ah = array aspect = rh / rw = 1 / rotated aspect
118 // Coordinate mappings:
119 // ROTATE_AND_CROP_90: point a -> point 2
120 // point c -> point 4 = +x -> +y, +y -> -x
121 // ROTATE_AND_CROP_180: point a -> point c
122 // point c -> point a = +x -> -x, +y -> -y
123 // ROTATE_AND_CROP_270: point a -> point 4
124 // point c -> point 2 = +x -> -y, +y -> +x
125
126 float cropAspect = static_cast<float>(cw) / ch;
127 float transformMat[4] = {0, 0,
128 0, 0};
129 float xShift = 0;
130 float yShift = 0;
131
132 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
133 transformMat[0] = -1;
134 transformMat[3] = -1;
135 xShift = cw;
136 yShift = ch;
137 } else {
138 float rw = cropAspect > mRotateAspect ?
139 ch * mRotateAspect : // pillarbox, not full width
140 cw; // letterbox or 1:1, full width
141 float rh = cropAspect >= mRotateAspect ?
142 ch : // pillarbox or 1:1, full height
143 cw / mRotateAspect; // letterbox, not full height
144 switch (rotateMode) {
145 case ANDROID_SCALER_ROTATE_AND_CROP_90:
146 transformMat[1] = -rw / ch; // +y -> -x
147 transformMat[2] = rh / cw; // +x -> +y
148 xShift = (cw + rw) / 2; // left edge of crop to right edge of rotated
149 yShift = (ch - rh) / 2; // top edge of crop to top edge of rotated
150 break;
151 case ANDROID_SCALER_ROTATE_AND_CROP_270:
152 transformMat[1] = rw / ch; // +y -> +x
153 transformMat[2] = -rh / cw; // +x -> -y
154 xShift = (cw - rw) / 2; // left edge of crop to left edge of rotated
155 yShift = (ch + rh) / 2; // top edge of crop to bottom edge of rotated
156 break;
157 default:
158 ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
159 return BAD_VALUE;
160 }
161 }
162
163 for (auto regionTag : kMeteringRegionsToCorrect) {
164 entry = request->find(regionTag);
165 for (size_t i = 0; i < entry.count; i += 5) {
166 int32_t weight = entry.data.i32[i + 4];
167 if (weight == 0) {
168 continue;
169 }
170 transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, cx, cy);
171 swapRectToMinFirst(entry.data.i32 + i);
172 }
173 }
174
175 return OK;
176 }
177
178 /**
179 * Adjust capture result when rotate and crop AUTO is enabled
180 */
updateCaptureResult(CameraMetadata * result)181 status_t RotateAndCropMapper::updateCaptureResult(CameraMetadata *result) {
182 auto entry = result->find(ANDROID_SCALER_ROTATE_AND_CROP);
183 if (entry.count == 0) return OK;
184 uint8_t rotateMode = entry.data.u8[0];
185 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
186
187 int32_t cx = 0;
188 int32_t cy = 0;
189 int32_t cw = mArrayWidth;
190 int32_t ch = mArrayHeight;
191 entry = result->find(ANDROID_SCALER_CROP_REGION);
192 if (entry.count == 4) {
193 cx = entry.data.i32[0];
194 cy = entry.data.i32[1];
195 cw = entry.data.i32[2];
196 ch = entry.data.i32[3];
197 }
198
199 // HAL inputs are relative to the full active array, so convert back to
200 // rotated-and-cropped coordinates for apps. To be more specific, the
201 // application is calculating coordinates based on the crop rectangle and
202 // the active array, even though the view the user sees is the
203 // cropped-and-rotated one. So we need to adjust the coordinates so that a
204 // point that would be on the top-left corner of the rotate-and-cropped
205 // region is mapped to the top-left corner of the crop region, and the same
206 // for the bottom-right corner.
207 //
208 // Since the zoom ratio control scales everything uniformly (so an app does
209 // not need to adjust anything if it wants to put a metering region on the
210 // top-left quadrant of the preview FOV, when changing zoomRatio), it does
211 // not need to be factored into this calculation at all.
212 //
213 // Also note that round-tripping between original request and final result
214 // fields can't be perfect, since the intermediate values have to be
215 // integers on a smaller range than the original crop region range. That
216 // means that multiple input values map to a single output value in
217 // adjusting a request, so when adjusting a result, the original answer may
218 // not be obtainable. Given that aspect ratios are rarely > 16/9, the
219 // round-trip values should generally only be off by 1 at most.
220 //
221 // ->+x active array aw
222 // |+--------------------------------------------------------------------+
223 // v| |
224 // +y| a 1 cw 2 b |
225 // | +=========*HHHHHHHHHHHHHHH*===========+ |
226 // | I H rw H I |
227 // | I H H I |
228 // | I H H I |
229 //ah | ch I H rh H I crop region |
230 // | I H H I |
231 // | I H H I |
232 // | I H rotate region H I |
233 // | +=========*HHHHHHHHHHHHHHH*===========+ |
234 // | d 4 3 c |
235 // | |
236 // +--------------------------------------------------------------------+
237 //
238 // aw , ah = active array width,height
239 // cw , ch = crop region width,height
240 // rw , rh = rotated-and-cropped region width,height
241 // aw / ah = array aspect = rh / rw = 1 / rotated aspect
242 // Coordinate mappings:
243 // ROTATE_AND_CROP_90: point 2 -> point a
244 // point 4 -> point c = +x -> -y, +y -> +x
245 // ROTATE_AND_CROP_180: point c -> point a
246 // point a -> point c = +x -> -x, +y -> -y
247 // ROTATE_AND_CROP_270: point 4 -> point a
248 // point 2 -> point c = +x -> +y, +y -> -x
249
250 float cropAspect = static_cast<float>(cw) / ch;
251 float transformMat[4] = {0, 0,
252 0, 0};
253 float xShift = 0;
254 float yShift = 0;
255 float rx = 0; // top-left corner of rotated region
256 float ry = 0;
257 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
258 transformMat[0] = -1;
259 transformMat[3] = -1;
260 xShift = cw;
261 yShift = ch;
262 rx = cx;
263 ry = cy;
264 } else {
265 float rw = cropAspect > mRotateAspect ?
266 ch * mRotateAspect : // pillarbox, not full width
267 cw; // letterbox or 1:1, full width
268 float rh = cropAspect >= mRotateAspect ?
269 ch : // pillarbox or 1:1, full height
270 cw / mRotateAspect; // letterbox, not full height
271 rx = cx + (cw - rw) / 2;
272 ry = cy + (ch - rh) / 2;
273 switch (rotateMode) {
274 case ANDROID_SCALER_ROTATE_AND_CROP_90:
275 transformMat[1] = ch / rw; // +y -> +x
276 transformMat[2] = -cw / rh; // +x -> -y
277 xShift = -(cw - rw) / 2; // left edge of rotated to left edge of cropped
278 yShift = ry - cy + ch; // top edge of rotated to bottom edge of cropped
279 break;
280 case ANDROID_SCALER_ROTATE_AND_CROP_270:
281 transformMat[1] = -ch / rw; // +y -> -x
282 transformMat[2] = cw / rh; // +x -> +y
283 xShift = (cw + rw) / 2; // left edge of rotated to left edge of cropped
284 yShift = (ch - rh) / 2; // top edge of rotated to bottom edge of cropped
285 break;
286 default:
287 ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
288 return BAD_VALUE;
289 }
290 }
291
292 for (auto regionTag : kMeteringRegionsToCorrect) {
293 entry = result->find(regionTag);
294 for (size_t i = 0; i < entry.count; i += 5) {
295 int32_t weight = entry.data.i32[i + 4];
296 if (weight == 0) {
297 continue;
298 }
299 transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, rx, ry);
300 swapRectToMinFirst(entry.data.i32 + i);
301 }
302 }
303
304 for (auto pointsTag: kResultPointsToCorrectNoClamp) {
305 entry = result->find(pointsTag);
306 transformPoints(entry.data.i32, entry.count / 2, transformMat, xShift, yShift, rx, ry);
307 if (pointsTag == ANDROID_STATISTICS_FACE_RECTANGLES) {
308 for (size_t i = 0; i < entry.count; i += 4) {
309 swapRectToMinFirst(entry.data.i32 + i);
310 }
311 }
312 }
313
314 return OK;
315 }
316
transformPoints(int32_t * pts,size_t count,float transformMat[4],float xShift,float yShift,float ox,float oy)317 void RotateAndCropMapper::transformPoints(int32_t* pts, size_t count, float transformMat[4],
318 float xShift, float yShift, float ox, float oy) {
319 for (size_t i = 0; i < count * 2; i += 2) {
320 float x0 = pts[i] - ox;
321 float y0 = pts[i + 1] - oy;
322 int32_t nx = std::round(transformMat[0] * x0 + transformMat[1] * y0 + xShift + ox);
323 int32_t ny = std::round(transformMat[2] * x0 + transformMat[3] * y0 + yShift + oy);
324
325 pts[i] = std::min(std::max(nx, 0), mArrayWidth);
326 pts[i + 1] = std::min(std::max(ny, 0), mArrayHeight);
327 }
328 }
329
swapRectToMinFirst(int32_t * rect)330 void RotateAndCropMapper::swapRectToMinFirst(int32_t* rect) {
331 if (rect[0] > rect[2]) {
332 auto tmp = rect[0];
333 rect[0] = rect[2];
334 rect[2] = tmp;
335 }
336 if (rect[1] > rect[3]) {
337 auto tmp = rect[1];
338 rect[1] = rect[3];
339 rect[3] = tmp;
340 }
341 }
342
343 } // namespace camera3
344
345 } // namespace android
346