1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "FormatConverter"
7
8 #include <v4l2_codec2/common/FormatConverter.h>
9
10 #include <inttypes.h>
11
12 #include <memory>
13 #include <string>
14
15 #include <C2AllocatorGralloc.h>
16 #include <C2PlatformSupport.h>
17 #include <android/hardware/graphics/common/1.0/types.h>
18 #include <inttypes.h>
19 #include <libyuv.h>
20 #include <ui/GraphicBuffer.h>
21 #include <utils/Log.h>
22
23 #include <v4l2_codec2/common/VideoTypes.h> // for HalPixelFormat
24
25 using android::hardware::graphics::common::V1_0::BufferUsage;
26
27 namespace android {
28
29 namespace {
30 // The constant expression of mapping the pixel format conversion pair (src, dst) to a unique
31 // integer.
convertMap(VideoPixelFormat src,VideoPixelFormat dst)32 constexpr int convertMap(VideoPixelFormat src, VideoPixelFormat dst) {
33 return static_cast<int>(src) * (static_cast<int>(VideoPixelFormat::UNKNOWN) + 1) +
34 static_cast<int>(dst);
35 }
36
37 // The helper function to copy a plane pixel by pixel. It assumes bytesPerPixel is 1.
copyPlaneByPixel(const uint8_t * src,int srcStride,int srcColInc,uint8_t * dst,int dstStride,int dstColInc,int width,int height)38 void copyPlaneByPixel(const uint8_t* src, int srcStride, int srcColInc, uint8_t* dst, int dstStride,
39 int dstColInc, int width, int height) {
40 for (int row = 0; row < height; row++) {
41 const uint8_t* srcRow = src;
42 uint8_t* dstRow = dst;
43 for (int col = 0; col < width; col++) {
44 memcpy(dstRow, srcRow, 1);
45 srcRow += srcColInc;
46 dstRow += dstColInc;
47 }
48 src += srcStride;
49 dst += dstStride;
50 }
51 }
52
53 } // namespace
54
ImplDefinedToRGBXMap(sp<GraphicBuffer> buf,uint8_t * addr,int rowInc)55 ImplDefinedToRGBXMap::ImplDefinedToRGBXMap(sp<GraphicBuffer> buf, uint8_t* addr, int rowInc)
56 : mBuffer(std::move(buf)), mAddr(addr), mRowInc(rowInc) {}
57
~ImplDefinedToRGBXMap()58 ImplDefinedToRGBXMap::~ImplDefinedToRGBXMap() {
59 mBuffer->unlock();
60 }
61
62 // static
Create(const C2ConstGraphicBlock & block)63 std::unique_ptr<ImplDefinedToRGBXMap> ImplDefinedToRGBXMap::Create(
64 const C2ConstGraphicBlock& block) {
65 uint32_t width, height, format, stride, igbpSlot, generation;
66 uint64_t usage, igbpId;
67 android::_UnwrapNativeCodec2GrallocMetadata(block.handle(), &width, &height, &format, &usage,
68 &stride, &generation, &igbpId, &igbpSlot);
69
70 if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
71 ALOGE("The original format (=%u) is not IMPLEMENTATION_DEFINED", format);
72 return nullptr;
73 }
74
75 native_handle_t* grallocHandle = android::UnwrapNativeCodec2GrallocHandle(block.handle());
76 sp<GraphicBuffer> buf = new GraphicBuffer(grallocHandle, GraphicBuffer::CLONE_HANDLE, width,
77 height, format, 1, usage, stride);
78 native_handle_delete(grallocHandle);
79
80 void* pointer = nullptr;
81 int32_t status = buf->lock(GRALLOC_USAGE_SW_READ_OFTEN, &pointer);
82 if (status != OK) {
83 ALOGE("Failed to lock buffer as IMPLEMENTATION_DEFINED format");
84 return nullptr;
85 }
86
87 uint8_t* addr = reinterpret_cast<uint8_t*>(pointer);
88 int rowInc = static_cast<int>(stride * 4); // RGBX 4-byte data per pixel
89 ALOGD("Parsed input format IMPLEMENTATION_DEFINED to RGBX_8888");
90 return std::unique_ptr<ImplDefinedToRGBXMap>(
91 new ImplDefinedToRGBXMap(std::move(buf), addr, rowInc));
92 }
93
94 // static
Create(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)95 std::unique_ptr<FormatConverter> FormatConverter::Create(VideoPixelFormat outFormat,
96 const ui::Size& visibleSize,
97 uint32_t inputCount,
98 const ui::Size& codedSize) {
99 if (outFormat != VideoPixelFormat::I420 && outFormat != VideoPixelFormat::NV12) {
100 ALOGE("Unsupported output format: %d", static_cast<int32_t>(outFormat));
101 return nullptr;
102 }
103
104 std::unique_ptr<FormatConverter> converter(new FormatConverter);
105 if (converter->initialize(outFormat, visibleSize, inputCount, codedSize) != C2_OK) {
106 ALOGE("Failed to initialize FormatConverter");
107 return nullptr;
108 }
109 return converter;
110 }
111
initialize(VideoPixelFormat outFormat,const ui::Size & visibleSize,uint32_t inputCount,const ui::Size & codedSize)112 c2_status_t FormatConverter::initialize(VideoPixelFormat outFormat, const ui::Size& visibleSize,
113 uint32_t inputCount, const ui::Size& codedSize) {
114 ALOGV("initialize(out_format=%s, visible_size=%dx%d, input_count=%u, coded_size=%dx%d)",
115 videoPixelFormatToString(outFormat).c_str(), visibleSize.width, visibleSize.height,
116 inputCount, codedSize.width, codedSize.height);
117
118 std::shared_ptr<C2BlockPool> pool;
119 c2_status_t status = GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool);
120 if (status != C2_OK) {
121 ALOGE("Failed to get basic graphic block pool (err=%d)", status);
122 return status;
123 }
124
125 HalPixelFormat halFormat;
126 if (outFormat == VideoPixelFormat::I420) {
127 // Android HAL format doesn't have I420, we use YV12 instead and swap U and V data while
128 // conversion to perform I420.
129 halFormat = HalPixelFormat::YV12;
130 } else {
131 halFormat = HalPixelFormat::YCBCR_420_888; // will allocate NV12 by minigbm.
132 }
133
134 uint32_t bufferCount = std::max(inputCount, kMinInputBufferCount);
135 for (uint32_t i = 0; i < bufferCount; i++) {
136 std::shared_ptr<C2GraphicBlock> block;
137 status = pool->fetchGraphicBlock(codedSize.width, codedSize.height,
138 static_cast<uint32_t>(halFormat),
139 {(C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE),
140 static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)},
141 &block);
142 if (status != C2_OK) {
143 ALOGE("Failed to fetch graphic block (err=%d)", status);
144 return status;
145 }
146 mGraphicBlocks.emplace_back(new BlockEntry(std::move(block)));
147 mAvailableQueue.push(mGraphicBlocks.back().get());
148 }
149
150 mOutFormat = outFormat;
151 mVisibleSize = visibleSize;
152
153 mTempPlaneU =
154 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
155 mTempPlaneV =
156 std::unique_ptr<uint8_t[]>(new uint8_t[mVisibleSize.width * mVisibleSize.height / 4]);
157
158 return C2_OK;
159 }
160
convertBlock(uint64_t frameIndex,const C2ConstGraphicBlock & inputBlock,c2_status_t * status)161 C2ConstGraphicBlock FormatConverter::convertBlock(uint64_t frameIndex,
162 const C2ConstGraphicBlock& inputBlock,
163 c2_status_t* status) {
164 if (!isReady()) {
165 ALOGV("There is no available block for conversion");
166 *status = C2_NO_MEMORY;
167 return inputBlock; // This is actually redundant and should not be used.
168 }
169
170 BlockEntry* entry = mAvailableQueue.front();
171 std::shared_ptr<C2GraphicBlock> outputBlock = entry->mBlock;
172
173 const C2GraphicView& inputView = inputBlock.map().get();
174 C2PlanarLayout inputLayout = inputView.layout();
175
176 // The above layout() cannot fill layout information and memset 0 instead if the input format is
177 // IMPLEMENTATION_DEFINED and its backed format is RGB. We fill the layout by using
178 // ImplDefinedToRGBXMap in the case.
179 std::unique_ptr<ImplDefinedToRGBXMap> idMap;
180 if (static_cast<uint32_t>(inputLayout.type) == 0u) {
181 idMap = ImplDefinedToRGBXMap::Create(inputBlock);
182 if (idMap == nullptr) {
183 ALOGE("Unable to parse RGBX_8888 from IMPLEMENTATION_DEFINED");
184 *status = C2_CORRUPTED;
185 return inputBlock; // This is actually redundant and should not be used.
186 }
187 inputLayout.type = C2PlanarLayout::TYPE_RGB;
188 }
189
190 C2GraphicView outputView = outputBlock->map().get();
191 C2PlanarLayout outputLayout = outputView.layout();
192 uint8_t* dstY = outputView.data()[C2PlanarLayout::PLANE_Y];
193 uint8_t* dstU = outputView.data()[C2PlanarLayout::PLANE_V]; // only for I420
194 uint8_t* dstV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for I420
195 uint8_t* dstUV = outputView.data()[C2PlanarLayout::PLANE_U]; // only for NV12
196 const int dstStrideY = outputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
197 const int dstStrideU = outputLayout.planes[C2PlanarLayout::PLANE_V].rowInc; // only for I420
198 const int dstStrideV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for I420
199 const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12
200
201 VideoPixelFormat inputFormat = VideoPixelFormat::UNKNOWN;
202 *status = C2_OK;
203 if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
204 const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y];
205 const uint8_t* srcU = inputView.data()[C2PlanarLayout::PLANE_U];
206 const uint8_t* srcV = inputView.data()[C2PlanarLayout::PLANE_V];
207 const int srcStrideY = inputLayout.planes[C2PlanarLayout::PLANE_Y].rowInc;
208 const int srcStrideU = inputLayout.planes[C2PlanarLayout::PLANE_U].rowInc;
209 const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc;
210 if (inputLayout.rootPlanes == 3) {
211 inputFormat = VideoPixelFormat::YV12;
212 } else if (inputLayout.rootPlanes == 2) {
213 inputFormat = (srcV > srcU) ? VideoPixelFormat::NV12 : VideoPixelFormat::NV21;
214 }
215
216 if (inputFormat == mOutFormat) {
217 ALOGV("Zero-Copy is applied");
218 mGraphicBlocks.emplace_back(new BlockEntry(frameIndex));
219 return inputBlock;
220 }
221
222 switch (convertMap(inputFormat, mOutFormat)) {
223 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::I420):
224 libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY,
225 dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width,
226 mVisibleSize.height);
227 break;
228 case convertMap(VideoPixelFormat::YV12, VideoPixelFormat::NV12):
229 libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
230 dstStrideY, dstUV, dstStrideUV, mVisibleSize.width,
231 mVisibleSize.height);
232 break;
233 case convertMap(VideoPixelFormat::NV12, VideoPixelFormat::I420):
234 libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
235 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
236 mVisibleSize.height);
237 break;
238 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::I420):
239 libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
240 dstStrideU, dstV, dstStrideV, mVisibleSize.width,
241 mVisibleSize.height);
242 break;
243 case convertMap(VideoPixelFormat::NV21, VideoPixelFormat::NV12):
244 ALOGV("%s(): Converting PIXEL_FORMAT_NV21 -> PIXEL_FORMAT_NV12", __func__);
245 libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width,
246 mVisibleSize.height);
247 copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2, mVisibleSize.width / 2,
248 mVisibleSize.height / 2);
249 copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2, mVisibleSize.width / 2,
250 mVisibleSize.height / 2);
251 break;
252 default:
253 ALOGE("Unsupported pixel format conversion from %s to %s",
254 videoPixelFormatToString(inputFormat).c_str(),
255 videoPixelFormatToString(mOutFormat).c_str());
256 *status = C2_CORRUPTED;
257 return inputBlock; // This is actually redundant and should not be used.
258 }
259 } else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
260 // There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe
261 // BGRA_8888 is not used now?
262 inputFormat = VideoPixelFormat::ABGR;
263
264 const uint8_t* srcRGB = (idMap) ? idMap->addr() : inputView.data()[C2PlanarLayout::PLANE_R];
265 const int srcStrideRGB =
266 (idMap) ? idMap->rowInc() : inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc;
267
268 switch (convertMap(inputFormat, mOutFormat)) {
269 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::I420):
270 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV,
271 dstStrideV, mVisibleSize.width, mVisibleSize.height);
272 break;
273 case convertMap(VideoPixelFormat::ABGR, VideoPixelFormat::NV12): {
274 // There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to
275 // I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from
276 // temporary planes to dst-UV interleavedly.
277 const int tempStride = mVisibleSize.width / 2;
278 libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, mTempPlaneU.get(),
279 tempStride, mTempPlaneV.get(), tempStride, mVisibleSize.width,
280 mVisibleSize.height);
281 libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride,
282 dstUV, dstStrideUV, mVisibleSize.width / 2,
283 mVisibleSize.height / 2);
284 break;
285 }
286 default:
287 ALOGE("Unsupported pixel format conversion from %s to %s",
288 videoPixelFormatToString(inputFormat).c_str(),
289 videoPixelFormatToString(mOutFormat).c_str());
290 *status = C2_CORRUPTED;
291 return inputBlock; // This is actually redundant and should not be used.
292 }
293 } else {
294 ALOGE("Unsupported input layout type");
295 *status = C2_CORRUPTED;
296 return inputBlock; // This is actually redundant and should not be used.
297 }
298
299 ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex,
300 videoPixelFormatToString(inputFormat).c_str());
301 entry->mAssociatedFrameIndex = frameIndex;
302 mAvailableQueue.pop();
303 return outputBlock->share(C2Rect(mVisibleSize.width, mVisibleSize.height), C2Fence());
304 }
305
returnBlock(uint64_t frameIndex)306 c2_status_t FormatConverter::returnBlock(uint64_t frameIndex) {
307 ALOGV("returnBlock(frame_index=%" PRIu64 ")", frameIndex);
308
309 auto iter = std::find_if(mGraphicBlocks.begin(), mGraphicBlocks.end(),
310 [frameIndex](const std::unique_ptr<BlockEntry>& be) {
311 return be->mAssociatedFrameIndex == frameIndex;
312 });
313 if (iter == mGraphicBlocks.end()) {
314 ALOGE("Failed to find graphic block by converted/zero-copied frame index: %" PRIu64 "",
315 frameIndex);
316 return C2_BAD_INDEX;
317 }
318
319 if ((*iter)->mBlock) {
320 // Returned block is format converted.
321 (*iter)->mAssociatedFrameIndex = kNoFrameAssociated;
322 mAvailableQueue.push(iter->get());
323 } else {
324 // Returned block is zero-copied.
325 mGraphicBlocks.erase(iter);
326 }
327 return C2_OK;
328 }
329
330 } // namespace android
331