1 /*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "SoftwareRenderer"
18 #include <utils/Log.h>
19
20 #include "../include/SoftwareRenderer.h"
21 #include <cutils/properties.h> // for property_get
22 #include <media/stagefright/foundation/ADebug.h>
23 #include <media/stagefright/foundation/AMessage.h>
24 #include <media/stagefright/foundation/ColorUtils.h>
25 #include <media/stagefright/SurfaceUtils.h>
26 #include <system/window.h>
27 #include <ui/Fence.h>
28 #include <ui/GraphicBufferMapper.h>
29 #include <ui/GraphicBuffer.h>
30 #include <ui/Rect.h>
31
32 namespace android {
33
ALIGN(int x,int y)34 static int ALIGN(int x, int y) {
35 // y must be a power of 2.
36 return (x + y - 1) & ~(y - 1);
37 }
38
SoftwareRenderer(const sp<ANativeWindow> & nativeWindow,int32_t rotation)39 SoftwareRenderer::SoftwareRenderer(
40 const sp<ANativeWindow> &nativeWindow, int32_t rotation)
41 : mColorFormat(OMX_COLOR_FormatUnused),
42 mConverter(NULL),
43 mYUVMode(None),
44 mNativeWindow(nativeWindow),
45 mWidth(0),
46 mHeight(0),
47 mStride(0),
48 mCropLeft(0),
49 mCropTop(0),
50 mCropRight(0),
51 mCropBottom(0),
52 mCropWidth(0),
53 mCropHeight(0),
54 mRotationDegrees(rotation),
55 mDataSpace(HAL_DATASPACE_UNKNOWN) {
56 memset(&mHDRStaticInfo, 0, sizeof(mHDRStaticInfo));
57 }
58
~SoftwareRenderer()59 SoftwareRenderer::~SoftwareRenderer() {
60 delete mConverter;
61 mConverter = NULL;
62 }
63
resetFormatIfChanged(const sp<AMessage> & format,size_t numOutputBuffers)64 void SoftwareRenderer::resetFormatIfChanged(
65 const sp<AMessage> &format, size_t numOutputBuffers) {
66 CHECK(format != NULL);
67
68 int32_t colorFormatNew;
69 CHECK(format->findInt32("color-format", &colorFormatNew));
70
71 int32_t widthNew, heightNew, strideNew;
72 CHECK(format->findInt32("width", &widthNew));
73 CHECK(format->findInt32("slice-height", &heightNew));
74 CHECK(format->findInt32("stride", &strideNew));
75
76 int32_t cropLeftNew, cropTopNew, cropRightNew, cropBottomNew;
77 if (!format->findRect(
78 "crop", &cropLeftNew, &cropTopNew, &cropRightNew, &cropBottomNew)) {
79 cropLeftNew = cropTopNew = 0;
80 cropRightNew = widthNew - 1;
81 cropBottomNew = heightNew - 1;
82 }
83
84 // The native window buffer format for high-bitdepth content could
85 // depend on the dataspace also.
86 android_dataspace dataSpace;
87 bool dataSpaceChangedForPlanar16 = false;
88 if (colorFormatNew == OMX_COLOR_FormatYUV420Planar16
89 && format->findInt32("android._dataspace", (int32_t *)&dataSpace)
90 && dataSpace != mDataSpace) {
91 // Do not modify mDataSpace here, it's only modified at last
92 // when we do native_window_set_buffers_data_space().
93 dataSpaceChangedForPlanar16 = true;
94 }
95
96 if (static_cast<int32_t>(mColorFormat) == colorFormatNew &&
97 mWidth == widthNew &&
98 mHeight == heightNew &&
99 mCropLeft == cropLeftNew &&
100 mCropTop == cropTopNew &&
101 mCropRight == cropRightNew &&
102 mCropBottom == cropBottomNew &&
103 !dataSpaceChangedForPlanar16) {
104 // Nothing changed, no need to reset renderer.
105 return;
106 }
107
108 mColorFormat = static_cast<OMX_COLOR_FORMATTYPE>(colorFormatNew);
109 mWidth = widthNew;
110 mHeight = heightNew;
111 mStride = strideNew;
112 mCropLeft = cropLeftNew;
113 mCropTop = cropTopNew;
114 mCropRight = cropRightNew;
115 mCropBottom = cropBottomNew;
116
117 mCropWidth = mCropRight - mCropLeft + 1;
118 mCropHeight = mCropBottom - mCropTop + 1;
119
120 // by default convert everything to RGB565
121 int halFormat = HAL_PIXEL_FORMAT_RGB_565;
122 size_t bufWidth = mCropWidth;
123 size_t bufHeight = mCropHeight;
124
125 // hardware has YUV12 and RGBA8888 support, so convert known formats
126 {
127 switch (mColorFormat) {
128 case OMX_COLOR_FormatYUV420Planar:
129 case OMX_COLOR_FormatYUV420SemiPlanar:
130 case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar:
131 {
132 halFormat = HAL_PIXEL_FORMAT_YV12;
133 bufWidth = (mCropWidth + 1) & ~1;
134 bufHeight = (mCropHeight + 1) & ~1;
135 break;
136 }
137 case OMX_COLOR_Format24bitRGB888:
138 {
139 halFormat = HAL_PIXEL_FORMAT_RGB_888;
140 bufWidth = (mCropWidth + 1) & ~1;
141 bufHeight = (mCropHeight + 1) & ~1;
142 break;
143 }
144 case OMX_COLOR_Format32bitARGB8888:
145 case OMX_COLOR_Format32BitRGBA8888:
146 {
147 halFormat = HAL_PIXEL_FORMAT_RGBA_8888;
148 bufWidth = (mCropWidth + 1) & ~1;
149 bufHeight = (mCropHeight + 1) & ~1;
150 break;
151 }
152 case OMX_COLOR_FormatYUV420Planar16:
153 {
154 if (((dataSpace & HAL_DATASPACE_STANDARD_MASK) == HAL_DATASPACE_STANDARD_BT2020)
155 && ((dataSpace & HAL_DATASPACE_TRANSFER_MASK) == HAL_DATASPACE_TRANSFER_ST2084)) {
156 // Here we would convert OMX_COLOR_FormatYUV420Planar16 into
157 // OMX_COLOR_FormatYUV444Y410, and put it inside a buffer with
158 // format HAL_PIXEL_FORMAT_RGBA_1010102. Surfaceflinger will
159 // use render engine to convert it to RGB if needed.
160 halFormat = HAL_PIXEL_FORMAT_RGBA_1010102;
161 } else {
162 halFormat = HAL_PIXEL_FORMAT_YV12;
163 }
164 bufWidth = (mCropWidth + 1) & ~1;
165 bufHeight = (mCropHeight + 1) & ~1;
166 break;
167 }
168 default:
169 {
170 break;
171 }
172 }
173 }
174
175 if (halFormat == HAL_PIXEL_FORMAT_RGB_565) {
176 mConverter = new ColorConverter(
177 mColorFormat, OMX_COLOR_Format16bitRGB565);
178 CHECK(mConverter->isValid());
179 } else if (halFormat == HAL_PIXEL_FORMAT_RGBA_1010102) {
180 mConverter = new ColorConverter(
181 mColorFormat, OMX_COLOR_FormatYUV444Y410);
182 CHECK(mConverter->isValid());
183 }
184
185 CHECK(mNativeWindow != NULL);
186 CHECK(mCropWidth > 0);
187 CHECK(mCropHeight > 0);
188 CHECK(mConverter == NULL || mConverter->isValid());
189
190 CHECK_EQ(0,
191 native_window_set_usage(
192 mNativeWindow.get(),
193 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY
194 | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_EXTERNAL_DISP));
195
196 CHECK_EQ(0,
197 native_window_set_scaling_mode(
198 mNativeWindow.get(),
199 NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW));
200
201 // Width must be multiple of 32???
202 CHECK_EQ(0, native_window_set_buffers_dimensions(
203 mNativeWindow.get(),
204 bufWidth,
205 bufHeight));
206 CHECK_EQ(0, native_window_set_buffers_format(
207 mNativeWindow.get(),
208 halFormat));
209 if (OK != native_window_set_buffer_count(
210 mNativeWindow.get(), numOutputBuffers + 4)) {
211 ALOGE("Failed to set native window buffer count to (%zu + 4)",
212 numOutputBuffers);
213 }
214
215 // NOTE: native window uses extended right-bottom coordinate
216 android_native_rect_t crop;
217 crop.left = mCropLeft;
218 crop.top = mCropTop;
219 crop.right = mCropRight + 1;
220 crop.bottom = mCropBottom + 1;
221 ALOGV("setting crop: [%d, %d, %d, %d] for size [%zu, %zu]",
222 crop.left, crop.top, crop.right, crop.bottom, bufWidth, bufHeight);
223
224 CHECK_EQ(0, native_window_set_crop(mNativeWindow.get(), &crop));
225
226 int32_t rotationDegrees;
227 if (!format->findInt32("rotation-degrees", &rotationDegrees)) {
228 rotationDegrees = mRotationDegrees;
229 }
230 uint32_t transform;
231 switch (rotationDegrees) {
232 case 0: transform = 0; break;
233 case 90: transform = HAL_TRANSFORM_ROT_90; break;
234 case 180: transform = HAL_TRANSFORM_ROT_180; break;
235 case 270: transform = HAL_TRANSFORM_ROT_270; break;
236 default: transform = 0; break;
237 }
238
239 CHECK_EQ(0, native_window_set_buffers_transform(
240 mNativeWindow.get(), transform));
241 }
242
clearTracker()243 void SoftwareRenderer::clearTracker() {
244 mRenderTracker.clear(-1 /* lastRenderTimeNs */);
245 }
246
render(const void * data,size_t,int64_t mediaTimeUs,nsecs_t renderTimeNs,size_t numOutputBuffers,const sp<AMessage> & format)247 std::list<FrameRenderTracker::Info> SoftwareRenderer::render(
248 const void *data, size_t , int64_t mediaTimeUs, nsecs_t renderTimeNs,
249 size_t numOutputBuffers, const sp<AMessage>& format) {
250 resetFormatIfChanged(format, numOutputBuffers);
251 FrameRenderTracker::Info *info = NULL;
252
253 ANativeWindowBuffer *buf;
254 int fenceFd = -1;
255 int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
256 if (err == 0 && fenceFd >= 0) {
257 info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0);
258 sp<Fence> fence = new Fence(fenceFd);
259 err = fence->waitForever("SoftwareRenderer::render");
260 }
261 if (err != 0) {
262 ALOGW("Surface::dequeueBuffer returned error %d", err);
263 // complete (drop) dequeued frame if fence wait failed; otherwise,
264 // this returns an empty list as no frames should have rendered and not yet returned.
265 return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */);
266 }
267
268 GraphicBufferMapper &mapper = GraphicBufferMapper::get();
269
270 Rect bounds(mCropWidth, mCropHeight);
271
272 void *dst;
273 CHECK_EQ(0, mapper.lock(buf->handle,
274 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY,
275 bounds, &dst));
276
277 // TODO move the other conversions also into ColorConverter, and
278 // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth)
279 if (mConverter) {
280 mConverter->convert(
281 data,
282 mWidth, mHeight, mStride,
283 mCropLeft, mCropTop, mCropRight, mCropBottom,
284 dst,
285 buf->stride, buf->height, 0,
286 0, 0, mCropWidth - 1, mCropHeight - 1);
287 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) {
288 const uint8_t *src_y = (const uint8_t *)data + mCropTop * mStride + mCropLeft;
289 const uint8_t *src_u = (const uint8_t *)data + mStride * mHeight + mCropTop * mStride / 4;
290 const uint8_t *src_v = (const uint8_t *)src_u + mStride * mHeight / 4;
291
292 uint8_t *dst_y = (uint8_t *)dst;
293 size_t dst_y_size = buf->stride * buf->height;
294 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
295 size_t dst_c_size = dst_c_stride * buf->height / 2;
296 uint8_t *dst_v = dst_y + dst_y_size;
297 uint8_t *dst_u = dst_v + dst_c_size;
298
299 dst_y += mCropTop * buf->stride + mCropLeft;
300 dst_v += (mCropTop/2) * dst_c_stride + mCropLeft/2;
301 dst_u += (mCropTop/2) * dst_c_stride + mCropLeft/2;
302
303 for (int y = 0; y < mCropHeight; ++y) {
304 memcpy(dst_y, src_y, mCropWidth);
305
306 src_y += mStride;
307 dst_y += buf->stride;
308 }
309
310 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
311 memcpy(dst_u, src_u, (mCropWidth + 1) / 2);
312 memcpy(dst_v, src_v, (mCropWidth + 1) / 2);
313
314 src_u += mStride / 2;
315 src_v += mStride / 2;
316 dst_u += dst_c_stride;
317 dst_v += dst_c_stride;
318 }
319 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar16) {
320 const uint8_t *src_y = (const uint8_t *)data + mCropTop * mStride + mCropLeft * 2;
321 const uint8_t *src_u = (const uint8_t *)data + mStride * mHeight + mCropTop * mStride / 4;
322 const uint8_t *src_v = (const uint8_t *)src_u + mStride * mHeight / 4;
323
324 uint8_t *dst_y = (uint8_t *)dst;
325 size_t dst_y_size = buf->stride * buf->height;
326 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
327 size_t dst_c_size = dst_c_stride * buf->height / 2;
328 uint8_t *dst_v = dst_y + dst_y_size;
329 uint8_t *dst_u = dst_v + dst_c_size;
330
331 dst_y += mCropTop * buf->stride + mCropLeft;
332 dst_v += (mCropTop / 2) * dst_c_stride + mCropLeft / 2;
333 dst_u += (mCropTop / 2) * dst_c_stride + mCropLeft / 2;
334
335 for (int y = 0; y < mCropHeight; ++y) {
336 for (int x = 0; x < mCropWidth; ++x) {
337 dst_y[x] = (uint8_t)(((uint16_t *)src_y)[x] >> 2);
338 }
339
340 src_y += mStride;
341 dst_y += buf->stride;
342 }
343
344 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
345 for (int x = 0; x < (mCropWidth + 1) / 2; ++x) {
346 dst_u[x] = (uint8_t)(((uint16_t *)src_u)[x] >> 2);
347 dst_v[x] = (uint8_t)(((uint16_t *)src_v)[x] >> 2);
348 }
349
350 src_u += mStride / 2;
351 src_v += mStride / 2;
352 dst_u += dst_c_stride;
353 dst_v += dst_c_stride;
354 }
355 } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar
356 || mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
357 const uint8_t *src_y = (const uint8_t *)data;
358 const uint8_t *src_uv = (const uint8_t *)data
359 + mWidth * mHeight;
360
361 src_y += mCropLeft + mCropTop * mWidth;
362 src_uv += (mCropLeft + mCropTop * mWidth) / 2;
363
364 uint8_t *dst_y = (uint8_t *)dst;
365
366 size_t dst_y_size = buf->stride * buf->height;
367 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
368 size_t dst_c_size = dst_c_stride * buf->height / 2;
369 uint8_t *dst_v = dst_y + dst_y_size;
370 uint8_t *dst_u = dst_v + dst_c_size;
371
372 dst_y += mCropTop * buf->stride + mCropLeft;
373 dst_v += (mCropTop/2) * dst_c_stride + mCropLeft/2;
374 dst_u += (mCropTop/2) * dst_c_stride + mCropLeft/2;
375
376 for (int y = 0; y < mCropHeight; ++y) {
377 memcpy(dst_y, src_y, mCropWidth);
378
379 src_y += mWidth;
380 dst_y += buf->stride;
381 }
382
383 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
384 size_t tmp = (mCropWidth + 1) / 2;
385 for (size_t x = 0; x < tmp; ++x) {
386 dst_u[x] = src_uv[2 * x];
387 dst_v[x] = src_uv[2 * x + 1];
388 }
389
390 src_uv += mWidth;
391 dst_u += dst_c_stride;
392 dst_v += dst_c_stride;
393 }
394 } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) {
395 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 3 + mCropLeft * 3;
396 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 3 + mCropLeft * 3;
397
398 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
399 memcpy(dstPtr, srcPtr, mCropWidth * 3);
400 srcPtr += mWidth * 3;
401 dstPtr += buf->stride * 3;
402 }
403 } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) {
404 uint8_t *srcPtr, *dstPtr;
405
406 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
407 srcPtr = (uint8_t*)data + mWidth * 4 * (y + mCropTop) + mCropLeft * 4;
408 dstPtr = (uint8_t*)dst + buf->stride * 4 * (y + mCropTop) + mCropLeft * 4;
409 for (size_t x = 0; x < (size_t)mCropWidth; ++x) {
410 uint8_t a = *srcPtr++;
411 for (size_t i = 0; i < 3; ++i) { // copy RGB
412 *dstPtr++ = *srcPtr++;
413 }
414 *dstPtr++ = a; // alpha last (ARGB to RGBA)
415 }
416 }
417 } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) {
418 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 4 + mCropLeft * 4;
419 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 4 + mCropLeft * 4;
420
421 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
422 memcpy(dstPtr, srcPtr, mCropWidth * 4);
423 srcPtr += mWidth * 4;
424 dstPtr += buf->stride * 4;
425 }
426 } else {
427 LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat);
428 }
429
430 skip_copying:
431 CHECK_EQ(0, mapper.unlock(buf->handle));
432
433 if (renderTimeNs >= 0) {
434 if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
435 renderTimeNs)) != 0) {
436 ALOGW("Surface::set_buffers_timestamp returned error %d", err);
437 }
438 }
439
440 // TODO: propagate color aspects to software renderer to allow better
441 // color conversion to RGB. For now, just mark dataspace for YUV rendering.
442 android_dataspace dataSpace;
443 if (format->findInt32("android._dataspace", (int32_t *)&dataSpace) && dataSpace != mDataSpace) {
444 mDataSpace = dataSpace;
445
446 if (mConverter != NULL && mConverter->isDstRGB()) {
447 // graphics only supports full range RGB. ColorConverter should have
448 // converted any YUV to full range.
449 dataSpace = (android_dataspace)
450 ((dataSpace & ~HAL_DATASPACE_RANGE_MASK) | HAL_DATASPACE_RANGE_FULL);
451 }
452
453 ALOGD("setting dataspace on output surface to #%x", dataSpace);
454 if ((err = native_window_set_buffers_data_space(mNativeWindow.get(), dataSpace))) {
455 ALOGW("failed to set dataspace on surface (%d)", err);
456 }
457 }
458 if (format->contains("hdr-static-info")) {
459 HDRStaticInfo info;
460 if (ColorUtils::getHDRStaticInfoFromFormat(format, &info)
461 && memcmp(&mHDRStaticInfo, &info, sizeof(info))) {
462 setNativeWindowHdrMetadata(mNativeWindow.get(), &info);
463 mHDRStaticInfo = info;
464 }
465 }
466
467 if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) {
468 ALOGW("Surface::queueBuffer returned error %d", err);
469 } else {
470 mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE);
471 }
472
473 buf = NULL;
474 return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */);
475 }
476
477 } // namespace android
478