1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "FormatConverter"
7
8 #include <v4l2_codec2/common/FormatConverter.h>
9
10 #include <inttypes.h>
11
12 #include <memory>
13 #include <string>
14
15 #include <C2AllocatorGralloc.h>
16 #include <C2PlatformSupport.h>
17 #include <android/hardware/graphics/common/1.0/types.h>
18 #include <inttypes.h>
19 #include <libyuv.h>
20 #include <ui/GraphicBuffer.h>
21 #include <utils/Log.h>
22
23 #include <v4l2_codec2/common/VideoTypes.h> // for HalPixelFormat
24
25 using android::hardware::graphics::common::V1_0::BufferUsage;
26
27 namespace android {
28
29 namespace {
30 // The constant expression of mapping the pixel format conversion pair (src, dst) to a unique
31 // integer.
convertMap(media::VideoPixelFormat src,media::VideoPixelFormat dst)32 constexpr int convertMap(media::VideoPixelFormat src, media::VideoPixelFormat dst) {
33 return static_cast<int>(src) *
34 (static_cast<int>(media::VideoPixelFormat::PIXEL_FORMAT_MAX) + 1) +
35 static_cast<int>(dst);
36 }
37
38 // The helper function to copy a plane pixel by pixel. It assumes bytesPerPixel is 1.
copyPlaneByPixel(const uint8_t * src,int srcStride,int srcColInc,uint8_t * dst,int dstStride,int dstColInc,int width,int height)39 void copyPlaneByPixel(const uint8_t* src, int srcStride, int srcColInc, uint8_t* dst, int dstStride,
40 int dstColInc, int width, int height) {
41 for (int row = 0; row < height; row++) {
42 const uint8_t* srcRow = src;
43 uint8_t* dstRow = dst;
44 for (int col = 0; col < width; col++) {
45 memcpy(dstRow, srcRow, 1);
46 srcRow += srcColInc;
47 dstRow += dstColInc;
48 }
49 src += srcStride;
50 dst += dstStride;
51 }
52 }
53
54 } // namespace
55
ImplDefinedToRGBXMap(sp<GraphicBuffer> buf,uint8_t * addr,int rowInc)56 ImplDefinedToRGBXMap::ImplDefinedToRGBXMap(sp<GraphicBuffer> buf, uint8_t* addr, int rowInc)
57 : mBuffer(std::move(buf)), mAddr(addr), mRowInc(rowInc) {}
58
~ImplDefinedToRGBXMap()59 ImplDefinedToRGBXMap::~ImplDefinedToRGBXMap() {
60 mBuffer->unlock();
61 }
62
63 // static
Create(const C2ConstGraphicBlock & block)64 std::unique_ptr<ImplDefinedToRGBXMap> ImplDefinedToRGBXMap::Create(
65 const C2ConstGraphicBlock& block) {
66 uint32_t width, height, format, stride, igbpSlot, generation;
67 uint64_t usage, igbpId;
68 android::_UnwrapNativeCodec2GrallocMetadata(block.handle(), &width, &height, &format, &usage,
69 &stride, &generation, &igbpId, &igbpSlot);
70
71 if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
72 ALOGE("The original format (=%u) is not IMPLEMENTATION_DEFINED", format);
73 return nullptr;
74 }
75
76 native_handle_t* grallocHandle = android::UnwrapNativeCodec2GrallocHandle(block.handle());
77 sp<GraphicBuffer> buf = new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width,
78 height, format, 1, usage, stride);
79 native_handle_delete(grallocHandle);
80
81 void* pointer = nullptr;
82 int32_t status = buf->lock(GRALLOC_USAGE_SW_READ_OFTEN, &pointer);
83 if (status != OK) {
84 ALOGE("Failed to lock buffer as IMPLEMENTATION_DEFINED format");
85 return nullptr;
86 }
87
88 uint8_t* addr = reinterpret_cast<uint8_t*>(pointer);
89 int rowInc = static_cast<int>(stride * 4); // RGBX 4-byte data per pixel
90 ALOGD("Parsed input format IMPLEMENTATION_DEFINED to RGBX_8888");
91 return std::unique_ptr<ImplDefinedToRGBXMap>(
92 new ImplDefinedToRGBXMap(std::move(buf), addr, rowInc));
93 }
94
95 // static
Create(media::VideoPixelFormat outFormat,const media::Size & visibleSize,uint32_t inputCount,const media::Size & codedSize)96 std::unique_ptr<FormatConverter> FormatConverter::Create(media::VideoPixelFormat outFormat,
97 const media::Size& visibleSize,
98 uint32_t inputCount,
99 const media::Size& codedSize) {
100 if (outFormat != media::VideoPixelFormat::PIXEL_FORMAT_I420 &&
101 outFormat != media::VideoPixelFormat::PIXEL_FORMAT_NV12) {
102 ALOGE("Unsupported output format: %d", static_cast<int32_t>(outFormat));
103 return nullptr;
104 }
105
106 std::unique_ptr<FormatConverter> converter(new FormatConverter);
107 if (converter->initialize(outFormat, visibleSize, inputCount, codedSize) != C2_OK) {
108 ALOGE("Failed to initialize FormatConverter");
109 return nullptr;
110 }
111 return converter;
112 }
113
initialize(media::VideoPixelFormat outFormat,const media::Size & visibleSize,uint32_t inputCount,const media::Size & codedSize)114 c2_status_t FormatConverter::initialize(media::VideoPixelFormat outFormat,
115 const media::Size& visibleSize, uint32_t inputCount,
116 const media::Size& codedSize) {
117 ALOGV("initialize(out_format=%s, visible_size=%dx%d, input_count=%u, coded_size=%dx%d)",
118 media::VideoPixelFormatToString(outFormat).c_str(), visibleSize.width(),
119 visibleSize.height(), inputCount, codedSize.width(), codedSize.height());
120
121 std::shared_ptr<C2BlockPool> pool;
122 c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool);
123 if (status != C2_OK) {
124 ALOGE("Failed to get basic graphic block pool (err=%d)", status);
125 return status;
126 }
127
128 HalPixelFormat halFormat;
129 if (outFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) {
130 // Android HAL format doesn't have I420, we use YV12 instead and swap U and V data while
131 // conversion to perform I420.
132 halFormat = HalPixelFormat::YV12;
133 } else {
134 halFormat = HalPixelFormat::YCBCR_420_888; // will allocate NV12 by minigbm.
135 }
136
137 uint32_t bufferCount = std::max(inputCount, kMinInputBufferCount);
138 for (uint32_t i = 0; i < bufferCount; i++) {
139 std::shared_ptr<C2GraphicBlock> block;
140 status = pool->fetchGraphicBlock(codedSize.width(), codedSize.height(),
141 static_cast<uint32_t>(halFormat),
142 {(C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE),
143 static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)},
144 &block);
145 if (status != C2_OK) {
146 ALOGE("Failed to fetch graphic block (err=%d)", status);
147 return status;
148 }
149 mGraphicBlocks.emplace_back(new BlockEntry(std::move(block)));
150 mAvailableQueue.push(mGraphicBlocks.back().get());
151 }
152
153 mOutFormat = outFormat;
154 mVisibleSize = visibleSize;
155
156 mTempPlaneU = std::unique_ptr<uint8_t[]>(
157 new uint8_t[mVisibleSize.width() * mVisibleSize.height() / 4]);
158 mTempPlaneV = std::unique_ptr<uint8_t[]>(
159 new uint8_t[mVisibleSize.width() * mVisibleSize.height() / 4]);
160
161 return C2_OK;
162 }
163
convertBlock(uint64_t frameIndex,const C2ConstGraphicBlock & inputBlock,c2_status_t * status)164 C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex,
165 const C2ConstGraphicBlock& inputBlock,
166 c2_status_t* status) {
167 if (!isReady()) {
168 ALOGV("There is no available block for conversion");
169 *status = C2_NO_MEMORY;
170 return inputBlock; // This is actually redundant and should not be used.
171 }
172
173 BlockEntry* entry = mAvailableQueue.front();
174 std::shared_ptr<C2GraphicBlock> outputBlock = entry->mBlock;
175
176 const C2GraphicView& inputView = inputBlock.map().get();
177 C2PlanarLayout inputLayout = inputView.layout();
178
179 // The above layout() cannot fill layout information and memset 0 instead if the input format is
180 // IMPLEMENTATION_DEFINED and its backed format is RGB. We fill the layout by using
181 // ImplDefinedToRGBXMap in the case.
182 std::unique_ptr<ImplDefinedToRGBXMap> idMap;
183 if (static_cast<uint32_t>(inputLayout.type) == 0u) {
184 idMap = ImplDefinedToRGBXMap::Create(inputBlock);
185 if (idMap == nullptr) {
186 ALOGE("Unable to parse RGBX_8888 from IMPLEMENTATION_DEFINED");
187 *status = C2_CORRUPTED;
188 return inputBlock; // This is actually redundant and should not be used.
189 }
190 inputLayout.type = C2PlanarLayout::TYPE_RGB;
191 }
192
193 C2GraphicView outputView = outputBlock->map().get();
194 C2PlanarLayout outputLayout = outputView.layout();
195 uint8_t* dstY = outputView.data()[C2PlanarLayout::PLANE_Y];
196 uint8_t* dstU = outputView.data()[C2PlanarLayout::PLANE_V]; // only for I420
197 uint8_t* dstV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for I420
198 uint8_t* dstUV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for NV12
199 const int dstStrideY = outputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
200 const int dstStrideU = outputLayout.planes[C2PlanarLayout::PLANE_V].rowInc; // only for I420
201 const int dstStrideV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for I420
202 const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12
203
204 media::VideoPixelFormat inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_UNKNOWN;
205 *status = C2_OK;
206 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
207 const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y];
208 const uint8_t* srcU = inputView.data()[C2PlanarLayout::PLANE_U];
209 const uint8_t* srcV = inputView.data()[C2PlanarLayout::PLANE_V];
210 const int srcStrideY = inputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
211 const int srcStrideU = inputLayout.planes[C2PlanarLayout::PLANE_U].rowInc;
212 const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc;
213 if (inputLayout.rootPlanes == 3) {
214 inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_YV12;
215 } else if (inputLayout.rootPlanes == 2) {
216 inputFormat = (srcV > srcU) ? media::VideoPixelFormat::PIXEL_FORMAT_NV12
217 : media::VideoPixelFormat::PIXEL_FORMAT_NV21;
218 }
219
220 if (inputFormat == mOutFormat) {
221 ALOGV("Zero-Copy is applied");
222 mGraphicBlocks.emplace_back(new BlockEntry(frameIndex));
223 return inputBlock;
224 }
225
226 switch (convertMap(inputFormat, mOutFormat)) {
227 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12,
228 media::VideoPixelFormat::PIXEL_FORMAT_I420):
229 libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY,
230 dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
231 mVisibleSize.height());
232 break;
233 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12,
234 media::VideoPixelFormat::PIXEL_FORMAT_NV12):
235 libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
236 dstStrideY, dstUV, dstStrideUV, mVisibleSize.width(),
237 mVisibleSize.height());
238 break;
239 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV12,
240 media::VideoPixelFormat::PIXEL_FORMAT_I420):
241 libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
242 dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
243 mVisibleSize.height());
244 break;
245 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21,
246 media::VideoPixelFormat::PIXEL_FORMAT_I420):
247 libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
248 dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
249 mVisibleSize.height());
250 break;
251 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21,
252 media::VideoPixelFormat::PIXEL_FORMAT_NV12):
253 libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width(),
254 mVisibleSize.height());
255 copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width() / 2,
256 mVisibleSize.height() / 2);
257 copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2,
258 mVisibleSize.width() / 2, mVisibleSize.height() / 2);
259 break;
260 default:
261 ALOGE("Unsupported pixel format conversion from %s to %s",
262 media::VideoPixelFormatToString(inputFormat).c_str(),
263 media::VideoPixelFormatToString(mOutFormat).c_str());
264 *status = C2_CORRUPTED;
265 return inputBlock; // This is actually redundant and should not be used.
266 }
267 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
268 // There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe
269 // BGRA_8888 is not used now?
270 inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_ABGR;
271
272 const uint8_t* srcRGB = (idMap) ? idMap->addr() : inputView.data()[C2PlanarLayout::PLANE_R];
273 const int srcStrideRGB =
274 (idMap) ? idMap->rowInc() : inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc;
275
276 switch (convertMap(inputFormat, mOutFormat)) {
277 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR,
278 media::VideoPixelFormat::PIXEL_FORMAT_I420):
279 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV,
280 dstStrideV, mVisibleSize.width(), mVisibleSize.height());
281 break;
282 case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR,
283 media::VideoPixelFormat::PIXEL_FORMAT_NV12): {
284 // There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to
285 // I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from
286 // temporary planes to dst-UV interleavedly.
287 const int tempStride = mVisibleSize.width() / 2;
288 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, mTempPlaneU.get(),
289 tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width(),
290 mVisibleSize.height());
291 libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride,
292 dstUV, dstStrideUV, mVisibleSize.width() / 2,
293 mVisibleSize.height() / 2);
294 break;
295 }
296 default:
297 ALOGE("Unsupported pixel format conversion from %s to %s",
298 media::VideoPixelFormatToString(inputFormat).c_str(),
299 media::VideoPixelFormatToString(mOutFormat).c_str());
300 *status = C2_CORRUPTED;
301 return inputBlock; // This is actually redundant and should not be used.
302 }
303 } else {
304 ALOGE("Unsupported input layout type");
305 *status = C2_CORRUPTED;
306 return inputBlock; // This is actually redundant and should not be used.
307 }
308
309 ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex,
310 media::VideoPixelFormatToString(inputFormat).c_str());
311 entry->mAssociatedFrameIndex = frameIndex;
312 mAvailableQueue.pop();
313 return outputBlock->share(C2Rect(mVisibleSize.width(), mVisibleSize.height()), C2Fence());
314 }
315
returnBlock(uint64_t frameIndex)316 c2_status_t FormatConverter::returnBlock(uint64_t frameIndex) {
317 ALOGV("returnBlock(frame_index=%" PRIu64 ")", frameIndex);
318
319 auto iter = std::find_if(mGraphicBlocks.begin(), mGraphicBlocks.end(),
320 [frameIndex](const std::unique_ptr<BlockEntry>& be) {
321 return be->mAssociatedFrameIndex == frameIndex;
322 });
323 if (iter == mGraphicBlocks.end()) {
324 ALOGE("Failed to find graphic block by converted/zero-copied frame index: %" PRIu64 "",
325 frameIndex);
326 return C2_BAD_INDEX;
327 }
328
329 if ((*iter)->mBlock) {
330 // Returned block is format converted.
331 (*iter)->mAssociatedFrameIndex = kNoFrameAssociated;
332 mAvailableQueue.push(iter->get());
333 } else {
334 // Returned block is zero-copied.
335 mGraphicBlocks.erase(iter);
336 }
337 return C2_OK;
338 }
339
340 } // namespace android
341