1 // Copyright 2020 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "V4L2EncodeInterface"
7
8 #include <v4l2_codec2/components/V4L2EncodeInterface.h>
9
10 #include <inttypes.h>
11
12 #include <C2PlatformSupport.h>
13 #include <SimpleC2Interface.h>
14 #include <android/hardware/graphics/common/1.0/types.h>
15 #include <media/stagefright/MediaDefs.h>
16 #include <utils/Log.h>
17
18 #include <v4l2_device.h>
19 #include <v4l2_codec2/common/V4L2ComponentCommon.h>
20 #include <video_codecs.h>
21
22 using android::hardware::graphics::common::V1_0::BufferUsage;
23
24 namespace android {
25
26 namespace {
27
28 // Use basic linear block pool/allocator as default.
29 constexpr C2BlockPool::local_id_t kDefaultOutputBlockPool = C2BlockPool::BASIC_LINEAR;
30 // Default input and output allocators.
31 constexpr C2Allocator::id_t kDefaultInputAllocator = C2PlatformAllocatorStore::GRALLOC;
32 constexpr C2Allocator::id_t kDefaultOutputAllocator = C2PlatformAllocatorStore::BLOB;
33
34 // The default output framerate in frames per second.
35 // TODO: increase to 60 fps in the future.
36 constexpr float kDefaultFrameRate = 30.0;
37 // The default output bitrate in bits per second. Use the max bitrate of AVC Level1.0 as default.
38 constexpr uint32_t kDefaultBitrate = 64000;
39
40 // The maximal output bitrate in bits per second. It's the max bitrate of AVC Level4.1.
41 // TODO: increase this in the future for supporting higher level/resolution encoding.
42 constexpr uint32_t kMaxBitrate = 50000000;
43
44 // The frame size of 1080p video.
45 constexpr uint32_t kFrameSize1080P = 1920 * 1080;
46
videoCodecProfileToC2Profile(media::VideoCodecProfile profile)47 C2Config::profile_t videoCodecProfileToC2Profile(media::VideoCodecProfile profile) {
48 switch (profile) {
49 case media::VideoCodecProfile::H264PROFILE_BASELINE:
50 return C2Config::PROFILE_AVC_BASELINE;
51 case media::VideoCodecProfile::H264PROFILE_MAIN:
52 return C2Config::PROFILE_AVC_MAIN;
53 case media::VideoCodecProfile::H264PROFILE_EXTENDED:
54 return C2Config::PROFILE_AVC_EXTENDED;
55 case media::VideoCodecProfile::H264PROFILE_HIGH:
56 return C2Config::PROFILE_AVC_HIGH;
57 case media::VideoCodecProfile::H264PROFILE_HIGH10PROFILE:
58 return C2Config::PROFILE_AVC_HIGH_10;
59 case media::VideoCodecProfile::H264PROFILE_HIGH422PROFILE:
60 return C2Config::PROFILE_AVC_HIGH_422;
61 case media::VideoCodecProfile::H264PROFILE_HIGH444PREDICTIVEPROFILE:
62 return C2Config::PROFILE_AVC_HIGH_444_PREDICTIVE;
63 case media::VideoCodecProfile::H264PROFILE_SCALABLEBASELINE:
64 return C2Config::PROFILE_AVC_SCALABLE_BASELINE;
65 case media::VideoCodecProfile::H264PROFILE_SCALABLEHIGH:
66 return C2Config::PROFILE_AVC_SCALABLE_HIGH;
67 case media::VideoCodecProfile::H264PROFILE_STEREOHIGH:
68 return C2Config::PROFILE_AVC_STEREO_HIGH;
69 case media::VideoCodecProfile::H264PROFILE_MULTIVIEWHIGH:
70 return C2Config::PROFILE_AVC_MULTIVIEW_HIGH;
71 default:
72 ALOGE("Unrecognizable profile (value = %d)...", profile);
73 return C2Config::PROFILE_UNUSED;
74 }
75 }
76
getCodecFromComponentName(const std::string & name)77 std::optional<media::VideoCodec> getCodecFromComponentName(const std::string& name) {
78 if (name == V4L2ComponentName::kH264Encoder)
79 return media::VideoCodec::kCodecH264;
80
81 ALOGE("Unknown name: %s", name.c_str());
82 return std::nullopt;
83 }
84
85 } // namespace
86
87 // static
ProfileLevelSetter(bool mayBlock,C2P<C2StreamProfileLevelInfo::output> & info,const C2P<C2StreamPictureSizeInfo::input> & videoSize,const C2P<C2StreamFrameRateInfo::output> & frameRate,const C2P<C2StreamBitrateInfo::output> & bitrate)88 C2R V4L2EncodeInterface::ProfileLevelSetter(bool mayBlock,
89 C2P<C2StreamProfileLevelInfo::output>& info,
90 const C2P<C2StreamPictureSizeInfo::input>& videoSize,
91 const C2P<C2StreamFrameRateInfo::output>& frameRate,
92 const C2P<C2StreamBitrateInfo::output>& bitrate) {
93 (void)mayBlock;
94
95 static C2Config::level_t lowestConfigLevel = C2Config::LEVEL_UNUSED;
96
97 // Use at least PROFILE_AVC_MAIN as default for 1080p input video and up.
98 // TODO (b/114332827): Find root cause of bad quality of Baseline encoding.
99 C2Config::profile_t defaultMinProfile = C2Config::PROFILE_AVC_BASELINE;
100 if (videoSize.v.width * videoSize.v.height >= kFrameSize1080P) {
101 defaultMinProfile = C2Config::PROFILE_AVC_MAIN;
102 }
103
104 // Adopt default minimal profile instead if the requested profile is not supported, or lower
105 // than the default minimal one.
106 if (!info.F(info.v.profile).supportsAtAll(info.v.profile) ||
107 info.v.profile < defaultMinProfile) {
108 if (info.F(info.v.profile).supportsAtAll(defaultMinProfile)) {
109 ALOGV("Set profile to default (%u) instead.", defaultMinProfile);
110 info.set().profile = defaultMinProfile;
111 } else {
112 ALOGE("Unable to set either requested profile (%u) or default profile (%u).",
113 info.v.profile, defaultMinProfile);
114 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile)));
115 }
116 }
117
118 // Table A-1 in spec
119 struct LevelLimits {
120 C2Config::level_t level;
121 float maxMBPS; // max macroblock processing rate in macroblocks per second
122 uint64_t maxFS; // max frame size in macroblocks
123 uint32_t maxBR; // max video bitrate in bits per second
124 };
125 constexpr LevelLimits kLimits[] = {
126 {C2Config::LEVEL_AVC_1, 1485, 99, 64000},
127 {C2Config::LEVEL_AVC_1B, 1485, 99, 128000},
128 {C2Config::LEVEL_AVC_1_1, 3000, 396, 192000},
129 {C2Config::LEVEL_AVC_1_2, 6000, 396, 384000},
130 {C2Config::LEVEL_AVC_1_3, 11880, 396, 768000},
131 {C2Config::LEVEL_AVC_2, 11880, 396, 2000000},
132 {C2Config::LEVEL_AVC_2_1, 19800, 792, 4000000},
133 {C2Config::LEVEL_AVC_2_2, 20250, 1620, 4000000},
134 {C2Config::LEVEL_AVC_3, 40500, 1620, 10000000},
135 {C2Config::LEVEL_AVC_3_1, 108000, 3600, 14000000},
136 {C2Config::LEVEL_AVC_3_2, 216000, 5120, 20000000},
137 {C2Config::LEVEL_AVC_4, 245760, 8192, 20000000},
138 {C2Config::LEVEL_AVC_4_1, 245760, 8192, 50000000},
139 {C2Config::LEVEL_AVC_4_2, 522240, 8704, 50000000},
140 {C2Config::LEVEL_AVC_5, 589824, 22080, 135000000},
141 {C2Config::LEVEL_AVC_5_1, 983040, 36864, 240000000},
142 {C2Config::LEVEL_AVC_5_2, 2073600, 36864, 240000000},
143 };
144
145 uint64_t targetFS =
146 static_cast<uint64_t>((videoSize.v.width + 15) / 16) * ((videoSize.v.height + 15) / 16);
147 float targetMBPS = static_cast<float>(targetFS) * frameRate.v.value;
148
149 // Try the recorded lowest configed level. This level should become adaptable after input size,
150 // frame rate, and bitrate are all set.
151 if (lowestConfigLevel != C2Config::LEVEL_UNUSED && lowestConfigLevel < info.v.level) {
152 info.set().level = lowestConfigLevel;
153 }
154
155 // Check if the supplied level meets the requirements. If not, update the level with the lowest
156 // level meeting the requirements.
157
158 bool found = false;
159 bool needsUpdate = !info.F(info.v.level).supportsAtAll(info.v.level);
160 for (const LevelLimits& limit : kLimits) {
161 if (!info.F(info.v.level).supportsAtAll(limit.level)) {
162 continue;
163 }
164
165 // Table A-2 in spec
166 // The maximum bit rate for High Profile is 1.25 times that of the Base/Extended/Main
167 // Profiles, 3 times for Hi10P, and 4 times for Hi422P/Hi444PP.
168 uint32_t maxBR = limit.maxBR;
169 if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_422) {
170 maxBR *= 4;
171 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_10) {
172 maxBR *= 3;
173 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH) {
174 maxBR = maxBR * 5.0 / 4.0;
175 }
176
177 if (targetFS <= limit.maxFS && targetMBPS <= limit.maxMBPS && bitrate.v.value <= maxBR) {
178 // This is the lowest level that meets the requirements, and if
179 // we haven't seen the supplied level yet, that means we don't
180 // need the update.
181 if (needsUpdate) {
182 // Since current config update is sequential, there is an issue when we want to set
183 // lower level for small input size, frame rate, and bitrate, if we set level first,
184 // it will be adjusted to a higher one because the default input size or others are
185 // above the limit. Use lowestConfigLevel to record the level we have tried to
186 // config (but failed).
187 // TODO(johnylin): remove this solution after b/140407694 has proper fix.
188 lowestConfigLevel = info.v.level;
189
190 ALOGD("Given level %u does not cover current configuration: "
191 "adjusting to %u",
192 info.v.level, limit.level);
193 info.set().level = limit.level;
194 }
195 found = true;
196 break;
197 }
198 if (info.v.level <= limit.level) {
199 // We break out of the loop when the lowest feasible level is found. The fact that we're
200 // here means that our level doesn't meet the requirement and needs to be updated.
201 needsUpdate = true;
202 }
203 }
204 if (!found) {
205 ALOGE("Unable to find proper level with current config, requested level (%u).",
206 info.v.level);
207 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.level)));
208 }
209
210 return C2R::Ok();
211 }
212
213 // static
SizeSetter(bool mayBlock,C2P<C2StreamPictureSizeInfo::input> & videoSize)214 C2R V4L2EncodeInterface::SizeSetter(bool mayBlock, C2P<C2StreamPictureSizeInfo::input>& videoSize) {
215 (void)mayBlock;
216 // TODO: maybe apply block limit?
217 return videoSize.F(videoSize.v.width)
218 .validatePossible(videoSize.v.width)
219 .plus(videoSize.F(videoSize.v.height).validatePossible(videoSize.v.height));
220 }
221
222 // static
IntraRefreshPeriodSetter(bool mayBlock,C2P<C2StreamIntraRefreshTuning::output> & period)223 C2R V4L2EncodeInterface::IntraRefreshPeriodSetter(bool mayBlock,
224 C2P<C2StreamIntraRefreshTuning::output>& period) {
225 (void)mayBlock;
226 if (period.v.period < 1) {
227 period.set().mode = C2Config::INTRA_REFRESH_DISABLED;
228 period.set().period = 0;
229 } else {
230 // Only support arbitrary mode (cyclic in our case).
231 period.set().mode = C2Config::INTRA_REFRESH_ARBITRARY;
232 }
233 return C2R::Ok();
234 }
235
V4L2EncodeInterface(const C2String & name,std::shared_ptr<C2ReflectorHelper> helper)236 V4L2EncodeInterface::V4L2EncodeInterface(
237 const C2String& name, std::shared_ptr<C2ReflectorHelper> helper)
238 : C2InterfaceHelper(std::move(helper)) {
239 ALOGV("%s(%s)", __func__, name.c_str());
240
241 setDerivedInstance(this);
242
243 Initialize(name);
244 }
245
Initialize(const C2String & name)246 void V4L2EncodeInterface::Initialize(const C2String& name) {
247 scoped_refptr<media::V4L2Device> device = media::V4L2Device::Create();
248 if (!device) {
249 ALOGE("Failed to create V4L2 device");
250 mInitStatus = C2_CORRUPTED;
251 return;
252 }
253
254 // Use type=unsigned int here, otherwise it will cause compile error in
255 // C2F(mProfileLevel, profile).oneOf(profiles) since std::vector<C2Config::profile_t> cannot
256 // convert to std::vector<unsigned int>.
257 std::vector<unsigned int> profiles;
258 media::Size maxSize;
259 for (const auto& supportedProfile : device->GetSupportedEncodeProfiles()) {
260 C2Config::profile_t profile = videoCodecProfileToC2Profile(supportedProfile.profile);
261 if (profile == C2Config::PROFILE_UNUSED) {
262 continue; // neglect unrecognizable profile
263 }
264 ALOGV("Queried c2_profile = 0x%x : max_size = %d x %d", profile,
265 supportedProfile.max_resolution.width(), supportedProfile.max_resolution.height());
266 profiles.push_back(static_cast<unsigned int>(profile));
267 maxSize.set_width(std::max(maxSize.width(), supportedProfile.max_resolution.width()));
268 maxSize.set_height(std::max(maxSize.height(), supportedProfile.max_resolution.height()));
269 }
270
271 if (profiles.empty()) {
272 ALOGD("No supported profiles");
273 mInitStatus = C2_BAD_VALUE;
274 return;
275 }
276
277 C2Config::profile_t minProfile =
278 static_cast<C2Config::profile_t>(*std::min_element(profiles.begin(), profiles.end()));
279
280 // Special note: the order of addParameter matters if your setters are dependent on other
281 // parameters. Please make sure the dependent parameters are added prior to the
282 // one needs the setter dependency.
283
284 addParameter(DefineParam(mInputVisibleSize, C2_PARAMKEY_PICTURE_SIZE)
285 .withDefault(new C2StreamPictureSizeInfo::input(0u, 320, 240))
286 .withFields({
287 C2F(mInputVisibleSize, width).inRange(2, maxSize.width(), 2),
288 C2F(mInputVisibleSize, height).inRange(2, maxSize.height(), 2),
289 })
290 .withSetter(SizeSetter)
291 .build());
292
293 addParameter(DefineParam(mFrameRate, C2_PARAMKEY_FRAME_RATE)
294 .withDefault(new C2StreamFrameRateInfo::output(0u, kDefaultFrameRate))
295 // TODO: More restriction?
296 .withFields({C2F(mFrameRate, value).greaterThan(0.)})
297 .withSetter(Setter<decltype(*mFrameRate)>::StrictValueWithNoDeps)
298 .build());
299
300 addParameter(DefineParam(mBitrate, C2_PARAMKEY_BITRATE)
301 .withDefault(new C2StreamBitrateInfo::output(0u, kDefaultBitrate))
302 .withFields({C2F(mBitrate, value).inRange(0, kMaxBitrate)})
303 .withSetter(Setter<decltype(*mBitrate)>::StrictValueWithNoDeps)
304 .build());
305
306 std::string outputMime;
307 if (getCodecFromComponentName(name) == media::VideoCodec::kCodecH264) {
308 outputMime = MEDIA_MIMETYPE_VIDEO_AVC;
309 addParameter(
310 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
311 .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile,
312 C2Config::LEVEL_AVC_4_1))
313 .withFields(
314 {C2F(mProfileLevel, profile).oneOf(profiles),
315 C2F(mProfileLevel, level)
316 // TODO: query supported levels from adaptor.
317 .oneOf({C2Config::LEVEL_AVC_1, C2Config::LEVEL_AVC_1B,
318 C2Config::LEVEL_AVC_1_1, C2Config::LEVEL_AVC_1_2,
319 C2Config::LEVEL_AVC_1_3, C2Config::LEVEL_AVC_2,
320 C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2,
321 C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1,
322 C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4,
323 C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_5,
324 C2Config::LEVEL_AVC_5_1})})
325 .withSetter(ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
326 .build());
327 } else {
328 // TODO(johnylin): implement VP8/VP9 encoder in the future.
329 ALOGE("Unsupported component name: %s", name.c_str());
330 mInitStatus = C2_BAD_VALUE;
331 return;
332 }
333
334 addParameter(
335 DefineParam(mInputFormat, C2_PARAMKEY_INPUT_STREAM_BUFFER_TYPE)
336 .withConstValue(new C2StreamBufferTypeSetting::input(0u, C2BufferData::GRAPHIC))
337 .build());
338
339 // TODO(b/167640667) Add VIDEO_ENCODER flag once input convertor is not enabled by default.
340 // When using the format convertor (which is currently always enabled) it's not useful to add
341 // the VIDEO_ENCODER buffer flag for input buffers here. Currently zero-copy is not supported
342 // yet, so when using this flag an additional buffer will be allocated on host side and a copy
343 // will be performed between the guest and host buffer to keep them in sync. This is wasteful as
344 // the buffer is only used on guest side by the format convertor which converts and copies the
345 // buffer into another buffer.
346 //addParameter(DefineParam(mInputMemoryUsage, C2_PARAMKEY_INPUT_STREAM_USAGE)
347 // .withConstValue(new C2StreamUsageTuning::input(
348 // 0u, static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)))
349 // .build());
350
351 addParameter(
352 DefineParam(mOutputFormat, C2_PARAMKEY_OUTPUT_STREAM_BUFFER_TYPE)
353 .withConstValue(new C2StreamBufferTypeSetting::output(0u, C2BufferData::LINEAR))
354 .build());
355
356 addParameter(DefineParam(mInputMediaType, C2_PARAMKEY_INPUT_MEDIA_TYPE)
357 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::input>(
358 MEDIA_MIMETYPE_VIDEO_RAW))
359 .build());
360
361 addParameter(DefineParam(mOutputMediaType, C2_PARAMKEY_OUTPUT_MEDIA_TYPE)
362 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::output>(
363 outputMime.c_str()))
364 .build());
365
366 addParameter(DefineParam(mIntraRefreshPeriod, C2_PARAMKEY_INTRA_REFRESH)
367 .withDefault(new C2StreamIntraRefreshTuning::output(
368 0u, C2Config::INTRA_REFRESH_DISABLED, 0.))
369 .withFields({C2F(mIntraRefreshPeriod, mode)
370 .oneOf({C2Config::INTRA_REFRESH_DISABLED,
371 C2Config::INTRA_REFRESH_ARBITRARY}),
372 C2F(mIntraRefreshPeriod, period).any()})
373 .withSetter(IntraRefreshPeriodSetter)
374 .build());
375
376 addParameter(DefineParam(mRequestKeyFrame, C2_PARAMKEY_REQUEST_SYNC_FRAME)
377 .withDefault(new C2StreamRequestSyncFrameTuning::output(0u, C2_FALSE))
378 .withFields({C2F(mRequestKeyFrame, value).oneOf({C2_FALSE, C2_TRUE})})
379 .withSetter(Setter<decltype(*mRequestKeyFrame)>::NonStrictValueWithNoDeps)
380 .build());
381
382 addParameter(DefineParam(mKeyFramePeriodUs, C2_PARAMKEY_SYNC_FRAME_INTERVAL)
383 .withDefault(new C2StreamSyncFrameIntervalTuning::output(0u, 1000000))
384 .withFields({C2F(mKeyFramePeriodUs, value).any()})
385 .withSetter(Setter<decltype(*mKeyFramePeriodUs)>::StrictValueWithNoDeps)
386 .build());
387
388 C2Allocator::id_t inputAllocators[] = {kDefaultInputAllocator};
389
390 C2Allocator::id_t outputAllocators[] = {kDefaultOutputAllocator};
391
392 addParameter(
393 DefineParam(mInputAllocatorIds, C2_PARAMKEY_INPUT_ALLOCATORS)
394 .withConstValue(C2PortAllocatorsTuning::input::AllocShared(inputAllocators))
395 .build());
396
397 addParameter(
398 DefineParam(mOutputAllocatorIds, C2_PARAMKEY_OUTPUT_ALLOCATORS)
399 .withConstValue(C2PortAllocatorsTuning::output::AllocShared(outputAllocators))
400 .build());
401
402 C2BlockPool::local_id_t outputBlockPools[] = {kDefaultOutputBlockPool};
403
404 addParameter(
405 DefineParam(mOutputBlockPoolIds, C2_PARAMKEY_OUTPUT_BLOCK_POOLS)
406 .withDefault(C2PortBlockPoolsTuning::output::AllocShared(outputBlockPools))
407 .withFields({C2F(mOutputBlockPoolIds, m.values[0]).any(),
408 C2F(mOutputBlockPoolIds, m.values).inRange(0, 1)})
409 .withSetter(Setter<C2PortBlockPoolsTuning::output>::NonStrictValuesWithNoDeps)
410 .build());
411
412 mInitStatus = C2_OK;
413 }
414
getKeyFramePeriod() const415 uint32_t V4L2EncodeInterface::getKeyFramePeriod() const {
416 if (mKeyFramePeriodUs->value < 0 || mKeyFramePeriodUs->value == INT64_MAX) {
417 return 0;
418 }
419 double period = mKeyFramePeriodUs->value / 1e6 * mFrameRate->value;
420 return static_cast<uint32_t>(std::max(std::min(std::round(period), double(UINT32_MAX)), 1.));
421 }
422
423 } // namespace android
424