1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "FormatConverter"
7
8 #include <v4l2_codec2/common/FormatConverter.h>
9
10 #include <inttypes.h>
11
12 #include <memory>
13 #include <string>
14
15 #include <C2AllocatorGralloc.h>
16 #include <C2PlatformSupport.h>
17 #include <android/hardware/graphics/common/1.0/types.h>
18 #include <inttypes.h>
19 #include <libyuv.h>
20 #include <ui/GraphicBuffer.h>
21 #include <utils/Log.h>
22
23 #include <v4l2_codec2/common/VideoTypes.h> // for HalPixelFormat
24
25 using android::hardware::graphics::common::V1_0::BufferUsage;
26
27 namespace android {
28
29 namespace {
30 // The constant expression of mapping the pixel format conversion pair (src, dst) to a unique
31 // integer.
convertMap(VideoPixelFormat src,VideoPixelFormat dst)32 constexpr int convertMap(VideoPixelFormat src, VideoPixelFormat dst) {
33 return static_cast<int>(src) * (static_cast<int>(VideoPixelFormat::UNKNOWN) + 1) +
34 static_cast<int>(dst);
35 }
36
37 // The helper function to copy a plane pixel by pixel. It assumes bytesPerPixel is 1.
copyPlaneByPixel(const uint8_t * src,int srcStride,int srcColInc,uint8_t * dst,int dstStride,int dstColInc,int width,int height)38 void copyPlaneByPixel(const uint8_t* src, int srcStride, int srcColInc, uint8_t* dst, int dstStride,
39 int dstColInc, int width, int height) {
40 for (int row = 0; row < height; row++) {
41 const uint8_t* srcRow = src;
42 uint8_t* dstRow = dst;
43 for (int col = 0; col < width; col++) {
44 memcpy(dstRow, srcRow, 1);
45 srcRow += srcColInc;
46 dstRow += dstColInc;
47 }
48 src += srcStride;
49 dst += dstStride;
50 }
51 }
52
53 } // namespace
54
ImplDefinedToRGBXMap(sp<GraphicBuffer> buf,uint8_t * addr,int rowInc)55 ImplDefinedToRGBXMap::ImplDefinedToRGBXMap(sp<GraphicBuffer> buf, uint8_t* addr, int rowInc)
56 : mBuffer(std::move(buf)), mAddr(addr), mRowInc(rowInc) {}
57
~ImplDefinedToRGBXMap()58 ImplDefinedToRGBXMap::~ImplDefinedToRGBXMap() {
59 mBuffer->unlock();
60 }
61
62 // static
create(const C2ConstGraphicBlock & block)63 std::unique_ptr<ImplDefinedToRGBXMap> ImplDefinedToRGBXMap::create(
64 const C2ConstGraphicBlock& block) {
65 uint32_t width, height, format, stride, igbpSlot, generation;
66 uint64_t usage, igbpId;
67 android::_UnwrapNativeCodec2GrallocMetadata(block.handle(), &width, &height, &format, &usage,
68 &stride, &generation, &igbpId, &igbpSlot);
69
70 if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
71 ALOGE("The original format (=%u) is not IMPLEMENTATION_DEFINED", format);
72 return nullptr;
73 }
74
75 native_handle_t* grallocHandle = android::UnwrapNativeCodec2GrallocHandle(block.handle());
76 sp<GraphicBuffer> buf = new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width,
77 height, format, 1, usage, stride);
78 native_handle_delete(grallocHandle);
79
80 void* pointer = nullptr;
81 int32_t status = buf->lock(GRALLOC_USAGE_SW_READ_OFTEN, &pointer);
82 if (status != OK) {
83 ALOGE("Failed to lock buffer as IMPLEMENTATION_DEFINED format");
84 return nullptr;
85 }
86
87 uint8_t* addr = reinterpret_cast<uint8_t*>(pointer);
88 int rowInc = static_cast<int>(stride * 4); // RGBX 4-byte data per pixel
89 ALOGD("Parsed input format IMPLEMENTATION_DEFINED to RGBX_8888");
90 return std::unique_ptr<ImplDefinedToRGBXMap>(
91 new ImplDefinedToRGBXMap(std::move(buf), addr, rowInc));
92 }
93
94 // static
create(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)95 std::unique_ptr<FormatConverter> FormatConverter::create(VideoPixelFormat outFormat,
96 const ui::Size& visibleSize,
97 uint32_t inputCount,
98 const ui::Size& codedSize) {
99 if (outFormat != VideoPixelFormat::I420 && outFormat != VideoPixelFormat::NV12) {
100 ALOGE("Unsupported output format: %d", static_cast<int32_t>(outFormat));
101 return nullptr;
102 }
103
104 std::unique_ptr<FormatConverter> converter(new FormatConverter);
105 if (converter->initialize(outFormat, visibleSize, inputCount, codedSize) != C2_OK) {
106 ALOGE("Failed to initialize FormatConverter");
107 return nullptr;
108 }
109 return converter;
110 }
111
initialize(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)112 c2_status_t FormatConverter::initialize(VideoPixelFormat outFormat, const ui::Size& visibleSize,
113 uint32_t inputCount, const ui::Size& codedSize) {
114 ALOGV("initialize(out_format=%s, visible_size=%dx%d, input_count=%u, coded_size=%dx%d)",
115 videoPixelFormatToString(outFormat).c_str(), visibleSize.width, visibleSize.height,
116 inputCount, codedSize.width, codedSize.height);
117
118 mOutFormat = outFormat;
119 mVisibleSize = visibleSize;
120 mCodedSize = codedSize;
121
122 mTempPlaneU =
123 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
124 mTempPlaneV =
125 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
126
127 // Allocate graphic blocks for format conversion.
128 uint32_t requested_buffer_count = std::max(1u, inputCount);
129 c2_status_t status = allocateBuffers(requested_buffer_count);
130 if (status != C2_OK) {
131 ALOGE("Failed to allocate buffers (error: %d)", status);
132 return status;
133 }
134
135 return C2_OK;
136 }
137
allocateBuffers(uint32_t count)138 c2_status_t FormatConverter::allocateBuffers(uint32_t count) {
139 ALOGV("Allocating %u buffers (format: %s, visible size: %dx%d, coded size: %dx%d)", count,
140 videoPixelFormatToString(mOutFormat).c_str(), mVisibleSize.width, mVisibleSize.height,
141 mCodedSize.width, mCodedSize.height);
142
143 HalPixelFormat halFormat;
144 if (mOutFormat == VideoPixelFormat::I420) {
145 // Android HAL format doesn't have I420, we use YV12 instead and swap U/V while converting.
146 halFormat = HalPixelFormat::YV12;
147 } else {
148 halFormat = HalPixelFormat::YCBCR_420_888; // Will allocate NV12 in minigbm.
149 }
150
151 std::shared_ptr<C2BlockPool> pool;
152 c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool);
153 if (status != C2_OK) {
154 ALOGE("Failed to get basic graphic block pool (error: %d)", status);
155 return C2_NO_MEMORY;
156 }
157
158 for (uint32_t i = 0; i < count; i++) {
159 std::shared_ptr<C2GraphicBlock> block;
160 status = pool->fetchGraphicBlock(mCodedSize.width, mCodedSize.height,
161 static_cast<uint32_t>(halFormat),
162 {(C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE),
163 static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)},
164 &block);
165 if (status != C2_OK) {
166 ALOGE("Failed to fetch graphic block (error: %d)", status);
167 return C2_NO_MEMORY;
168 }
169 mGraphicBlocks.emplace_back(new BlockEntry(std::move(block)));
170 mAvailableQueue.push(mGraphicBlocks.back().get());
171 }
172
173 return C2_OK;
174 }
175
convertBlock(uint64_t frameIndex,const C2ConstGraphicBlock & inputBlock,C2ConstGraphicBlock * convertedBlock)176 c2_status_t FormatConverter::convertBlock(uint64_t frameIndex,
177 const C2ConstGraphicBlock& inputBlock,
178 C2ConstGraphicBlock* convertedBlock) {
179 const C2GraphicView& inputView = inputBlock.map().get();
180 C2PlanarLayout inputLayout = inputView.layout();
181
182 // Determine the input buffer pixel format.
183 VideoPixelFormat inputFormat = VideoPixelFormat::UNKNOWN;
184 std::unique_ptr<ImplDefinedToRGBXMap> idMap;
185 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
186 if (inputLayout.rootPlanes == 3) {
187 inputFormat = VideoPixelFormat::YV12;
188 } else if (inputLayout.rootPlanes == 2) {
189 const uint8_t* const* data = inputView.data();
190 inputFormat = (data[C2PlanarLayout::PLANE_V] > data[C2PlanarLayout::PLANE_U])
191 ? VideoPixelFormat::NV12
192 : VideoPixelFormat::NV21;
193 }
194 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
195 inputFormat = VideoPixelFormat::ABGR;
196 } else if (static_cast<uint32_t>(inputLayout.type) == 0u) {
197 // The above layout() cannot fill layout information and sets it to 0 instead if the input
198 // format is IMPLEMENTATION_DEFINED and its backed format is RGB. We fill the layout by
199 // using ImplDefinedToRGBXMap in this case.
200 idMap = ImplDefinedToRGBXMap::create(inputBlock);
201 if (!idMap) {
202 ALOGE("Unable to parse RGBX_8888 from IMPLEMENTATION_DEFINED");
203 return C2_CORRUPTED;
204 }
205 // There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe
206 // BGRA_8888 is not used now?
207 inputFormat = VideoPixelFormat::ABGR;
208 inputLayout.type = C2PlanarLayout::TYPE_RGB;
209 } else {
210 ALOGE("Failed to determine input pixel format: %u", inputLayout.type);
211 return C2_CORRUPTED;
212 }
213
214 if (inputFormat == mOutFormat) {
215 ALOGV("Zero-Copy is applied");
216 mGraphicBlocks.emplace_back(new BlockEntry(frameIndex));
217 *convertedBlock = inputBlock;
218 return C2_OK;
219 }
220
221 if (!isReady()) {
222 ALOGV("There is no available block for conversion");
223 return C2_NO_MEMORY;
224 }
225
226 BlockEntry* entry = mAvailableQueue.front();
227 std::shared_ptr<C2GraphicBlock> outputBlock = entry->mBlock;
228
229 C2GraphicView outputView = outputBlock->map().get();
230 C2PlanarLayout outputLayout = outputView.layout();
231 uint8_t* dstY = outputView.data()[C2PlanarLayout::PLANE_Y];
232 uint8_t* dstU = outputView.data()[C2PlanarLayout::PLANE_V]; // only for I420
233 uint8_t* dstV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for I420
234 uint8_t* dstUV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for NV12
235 const int dstStrideY = outputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
236 const int dstStrideU = outputLayout.planes[C2PlanarLayout::PLANE_V].rowInc; // only for I420
237 const int dstStrideV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for I420
238 const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12
239
240 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
241 const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y];
242 const uint8_t* srcU = inputView.data()[C2PlanarLayout::PLANE_U];
243 const uint8_t* srcV = inputView.data()[C2PlanarLayout::PLANE_V];
244 const int srcStrideY = inputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
245 const int srcStrideU = inputLayout.planes[C2PlanarLayout::PLANE_U].rowInc;
246 const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc;
247
248 switch (convertMap(inputFormat, mOutFormat)) {
249 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::I420):
250 libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY,
251 dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width,
252 mVisibleSize.height);
253 break;
254 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::NV12):
255 libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
256 dstStrideY, dstUV, dstStrideUV, mVisibleSize.width,
257 mVisibleSize.height);
258 break;
259 case convertMap(VideoPixelFormat::NV12, VideoPixelFormat::I420):
260 libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
261 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
262 mVisibleSize.height);
263 break;
264 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::I420):
265 libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
266 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
267 mVisibleSize.height);
268 break;
269 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::NV12):
270 ALOGV("%s(): Converting PIXEL_FORMAT_NV21 -> PIXEL_FORMAT_NV12", __func__);
271 libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width,
272 mVisibleSize.height);
273 copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width / 2,
274 mVisibleSize.height / 2);
275 copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2, mVisibleSize.width / 2,
276 mVisibleSize.height / 2);
277 break;
278 default:
279 ALOGE("Unsupported pixel format conversion from %s to %s",
280 videoPixelFormatToString(inputFormat).c_str(),
281 videoPixelFormatToString(mOutFormat).c_str());
282 return C2_CORRUPTED;
283 }
284 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
285 const uint8_t* srcRGB = (idMap) ? idMap->addr() : inputView.data()[C2PlanarLayout::PLANE_R];
286 const int srcStrideRGB =
287 (idMap) ? idMap->rowInc() : inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc;
288
289 switch (convertMap(inputFormat, mOutFormat)) {
290 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::I420):
291 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV,
292 dstStrideV, mVisibleSize.width, mVisibleSize.height);
293 break;
294 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::NV12): {
295 // There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to
296 // I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from
297 // temporary planes to dst-UV interleavedly.
298 const int tempStride = mVisibleSize.width / 2;
299 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, mTempPlaneU.get(),
300 tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width,
301 mVisibleSize.height);
302 libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride,
303 dstUV, dstStrideUV, mVisibleSize.width / 2,
304 mVisibleSize.height / 2);
305 break;
306 }
307 default:
308 ALOGE("Unsupported pixel format conversion from %s to %s",
309 videoPixelFormatToString(inputFormat).c_str(),
310 videoPixelFormatToString(mOutFormat).c_str());
311 return C2_CORRUPTED;
312 }
313 } else {
314 ALOGE("Unsupported input layout type");
315 return C2_CORRUPTED;
316 }
317
318 ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex,
319 videoPixelFormatToString(inputFormat).c_str());
320 entry->mAssociatedFrameIndex = frameIndex;
321 mAvailableQueue.pop();
322
323 *convertedBlock =
324 outputBlock->share(C2Rect(mVisibleSize.width, mVisibleSize.height), C2Fence());
325 return C2_OK;
326 }
327
returnBlock(uint64_t frameIndex)328 c2_status_t FormatConverter::returnBlock(uint64_t frameIndex) {
329 ALOGV("returnBlock(frame_index=%" PRIu64 ")", frameIndex);
330
331 auto iter = std::find_if(mGraphicBlocks.begin(), mGraphicBlocks.end(),
332 [frameIndex](const std::unique_ptr<BlockEntry>& be) {
333 return be->mAssociatedFrameIndex == frameIndex;
334 });
335 if (iter == mGraphicBlocks.end()) {
336 ALOGE("Failed to find graphic block by converted/zero-copied frame index: %" PRIu64 "",
337 frameIndex);
338 return C2_BAD_INDEX;
339 }
340
341 if ((*iter)->mBlock) {
342 // Returned block is format converted.
343 (*iter)->mAssociatedFrameIndex = kNoFrameAssociated;
344 mAvailableQueue.push(iter->get());
345 } else {
346 // Returned block is zero-copied.
347 mGraphicBlocks.erase(iter);
348 }
349 return C2_OK;
350 }
351
352 } // namespace android
353