1 // Copyright 2023 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define LOG_TAG "V4L2EncodeInterface"
7
8 #include <v4l2_codec2/components/EncodeInterface.h>
9
10 #include <inttypes.h>
11 #include <algorithm>
12
13 #include <C2PlatformSupport.h>
14 #include <SimpleC2Interface.h>
15 #include <android/hardware/graphics/common/1.0/types.h>
16 #include <media/stagefright/MediaDefs.h>
17 #include <utils/Log.h>
18
19 #include <v4l2_codec2/common/Common.h>
20 #include <v4l2_codec2/common/VideoTypes.h>
21
22 using android::hardware::graphics::common::V1_0::BufferUsage;
23
24 namespace android {
25
26 namespace {
27
28 // Use basic linear block pool/allocator as default.
29 constexpr C2BlockPool::local_id_t kDefaultOutputBlockPool = C2BlockPool::BASIC_LINEAR;
30 // Default input and output allocators.
31 constexpr C2Allocator::id_t kDefaultInputAllocator = C2PlatformAllocatorStore::GRALLOC;
32 constexpr C2Allocator::id_t kDefaultOutputAllocator = C2PlatformAllocatorStore::BLOB;
33
34 // The default output framerate in frames per second.
35 // TODO: increase to 60 fps in the future.
36 constexpr float kDefaultFrameRate = 30.0;
37 // The default output bitrate in bits per second. Use the max bitrate of AVC Level1.0 as default.
38 constexpr uint32_t kDefaultBitrate = 64000;
39
40 // The maximal output bitrate in bits per second. It's the max bitrate of AVC Level4.1.
41 // TODO: increase this in the future for supporting higher level/resolution encoding.
42 constexpr uint32_t kMaxBitrate = 50000000;
43 } // namespace
44
45 //static
46 C2Config::level_t EncodeInterface::lowestConfigLevel = C2Config::LEVEL_UNUSED;
47
48 // static
H264ProfileLevelSetter(bool,C2P<C2StreamProfileLevelInfo::output> & info,const C2P<C2StreamPictureSizeInfo::input> & videoSize,const C2P<C2StreamFrameRateInfo::output> & frameRate,const C2P<C2StreamBitrateInfo::output> & bitrate)49 C2R EncodeInterface::H264ProfileLevelSetter(bool /*mayBlock*/,
50 C2P<C2StreamProfileLevelInfo::output>& info,
51 const C2P<C2StreamPictureSizeInfo::input>& videoSize,
52 const C2P<C2StreamFrameRateInfo::output>& frameRate,
53 const C2P<C2StreamBitrateInfo::output>& bitrate) {
54 // Adopt default minimal profile instead if the requested profile is not supported, or lower
55 // than the default minimal one.
56 constexpr C2Config::profile_t minProfile = C2Config::PROFILE_AVC_BASELINE;
57 if (!info.F(info.v.profile).supportsAtAll(info.v.profile) || info.v.profile < minProfile) {
58 if (info.F(info.v.profile).supportsAtAll(minProfile)) {
59 ALOGV("Set profile to default (%u) instead.", minProfile);
60 info.set().profile = minProfile;
61 } else {
62 ALOGE("Unable to set either requested profile (%u) or default profile (%u).",
63 info.v.profile, minProfile);
64 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile)));
65 }
66 }
67
68 // Table A-1 in spec
69 struct LevelLimits {
70 C2Config::level_t level;
71 float maxMBPS; // max macroblock processing rate in macroblocks per second
72 uint64_t maxFS; // max frame size in macroblocks
73 uint32_t maxBR; // max video bitrate in bits per second
74 };
75 constexpr LevelLimits kLimits[] = {
76 {C2Config::LEVEL_AVC_1, 1485, 99, 64000},
77 {C2Config::LEVEL_AVC_1B, 1485, 99, 128000},
78 {C2Config::LEVEL_AVC_1_1, 3000, 396, 192000},
79 {C2Config::LEVEL_AVC_1_2, 6000, 396, 384000},
80 {C2Config::LEVEL_AVC_1_3, 11880, 396, 768000},
81 {C2Config::LEVEL_AVC_2, 11880, 396, 2000000},
82 {C2Config::LEVEL_AVC_2_1, 19800, 792, 4000000},
83 {C2Config::LEVEL_AVC_2_2, 20250, 1620, 4000000},
84 {C2Config::LEVEL_AVC_3, 40500, 1620, 10000000},
85 {C2Config::LEVEL_AVC_3_1, 108000, 3600, 14000000},
86 {C2Config::LEVEL_AVC_3_2, 216000, 5120, 20000000},
87 {C2Config::LEVEL_AVC_4, 245760, 8192, 20000000},
88 {C2Config::LEVEL_AVC_4_1, 245760, 8192, 50000000},
89 {C2Config::LEVEL_AVC_4_2, 522240, 8704, 50000000},
90 {C2Config::LEVEL_AVC_5, 589824, 22080, 135000000},
91 {C2Config::LEVEL_AVC_5_1, 983040, 36864, 240000000},
92 {C2Config::LEVEL_AVC_5_2, 2073600, 36864, 240000000},
93 };
94
95 uint64_t targetFS =
96 static_cast<uint64_t>((videoSize.v.width + 15) / 16) * ((videoSize.v.height + 15) / 16);
97 float targetMBPS = static_cast<float>(targetFS) * frameRate.v.value;
98
99 // Try the recorded lowest configed level. This level should become adaptable after input size,
100 // frame rate, and bitrate are all set.
101 if (lowestConfigLevel != C2Config::LEVEL_UNUSED && lowestConfigLevel < info.v.level) {
102 info.set().level = lowestConfigLevel;
103 }
104
105 // Check if the supplied level meets the requirements. If not, update the level with the lowest
106 // level meeting the requirements.
107
108 bool found = false;
109 bool needsUpdate = !info.F(info.v.level).supportsAtAll(info.v.level);
110 for (const LevelLimits& limit : kLimits) {
111 if (!info.F(info.v.level).supportsAtAll(limit.level)) {
112 continue;
113 }
114
115 // Table A-2 in spec
116 // The maximum bit rate for High Profile is 1.25 times that of the Base/Extended/Main
117 // Profiles, 3 times for Hi10P, and 4 times for Hi422P/Hi444PP.
118 uint32_t maxBR = limit.maxBR;
119 if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_422) {
120 maxBR *= 4;
121 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH_10) {
122 maxBR *= 3;
123 } else if (info.v.profile >= C2Config::PROFILE_AVC_HIGH) {
124 maxBR = maxBR * 5.0 / 4.0;
125 }
126
127 if (targetFS <= limit.maxFS && targetMBPS <= limit.maxMBPS && bitrate.v.value <= maxBR) {
128 // This is the lowest level that meets the requirements, and if
129 // we haven't seen the supplied level yet, that means we don't
130 // need the update.
131 if (needsUpdate) {
132 // Since current config update is sequential, there is an issue when we want to set
133 // lower level for small input size, frame rate, and bitrate, if we set level first,
134 // it will be adjusted to a higher one because the default input size or others are
135 // above the limit. Use lowestConfigLevel to record the level we have tried to
136 // config (but failed).
137 // TODO(johnylin): remove this solution after b/140407694 has proper fix.
138 lowestConfigLevel = info.v.level;
139
140 ALOGD("Given level %u does not cover current configuration: "
141 "adjusting to %u",
142 info.v.level, limit.level);
143 info.set().level = limit.level;
144 }
145 found = true;
146 break;
147 }
148 if (info.v.level <= limit.level) {
149 // We break out of the loop when the lowest feasible level is found. The fact that we're
150 // here means that our level doesn't meet the requirement and needs to be updated.
151 needsUpdate = true;
152 }
153 }
154 if (!found) {
155 ALOGE("Unable to find proper level with current config, requested level (%u).",
156 info.v.level);
157 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.level)));
158 }
159
160 return C2R::Ok();
161 }
162
VP9ProfileLevelSetter(bool,C2P<C2StreamProfileLevelInfo::output> & info,const C2P<C2StreamPictureSizeInfo::input> &,const C2P<C2StreamFrameRateInfo::output> &,const C2P<C2StreamBitrateInfo::output> &)163 C2R EncodeInterface::VP9ProfileLevelSetter(bool /*mayBlock*/,
164 C2P<C2StreamProfileLevelInfo::output>& info,
165 const C2P<C2StreamPictureSizeInfo::input>& /*videoSize*/,
166 const C2P<C2StreamFrameRateInfo::output>& /*frameRate*/,
167 const C2P<C2StreamBitrateInfo::output>& /*bitrate*/) {
168 // Adopt default minimal profile instead if the requested profile is not supported, or lower
169 // than the default minimal one.
170 constexpr C2Config::profile_t defaultMinProfile = C2Config::PROFILE_VP9_0;
171 if (!info.F(info.v.profile).supportsAtAll(info.v.profile) ||
172 info.v.profile < defaultMinProfile) {
173 if (info.F(info.v.profile).supportsAtAll(defaultMinProfile)) {
174 ALOGV("Set profile to default (%u) instead.", defaultMinProfile);
175 info.set().profile = defaultMinProfile;
176 } else {
177 ALOGE("Unable to set either requested profile (%u) or default profile (%u).",
178 info.v.profile, defaultMinProfile);
179 return C2R(C2SettingResultBuilder::BadValue(info.F(info.v.profile)));
180 }
181 }
182
183 return C2R::Ok();
184 }
185
186 // static
SizeSetter(bool mayBlock,C2P<C2StreamPictureSizeInfo::input> & videoSize)187 C2R EncodeInterface::SizeSetter(bool mayBlock, C2P<C2StreamPictureSizeInfo::input>& videoSize) {
188 (void)mayBlock;
189 // TODO: maybe apply block limit?
190 return videoSize.F(videoSize.v.width)
191 .validatePossible(videoSize.v.width)
192 .plus(videoSize.F(videoSize.v.height).validatePossible(videoSize.v.height));
193 }
194
195 // static
IntraRefreshPeriodSetter(bool mayBlock,C2P<C2StreamIntraRefreshTuning::output> & period)196 C2R EncodeInterface::IntraRefreshPeriodSetter(bool mayBlock,
197 C2P<C2StreamIntraRefreshTuning::output>& period) {
198 (void)mayBlock;
199 if (period.v.period < 1) {
200 period.set().mode = C2Config::INTRA_REFRESH_DISABLED;
201 period.set().period = 0;
202 } else {
203 // Only support arbitrary mode (cyclic in our case).
204 period.set().mode = C2Config::INTRA_REFRESH_ARBITRARY;
205 }
206 return C2R::Ok();
207 }
208
EncodeInterface(const C2String & name,std::shared_ptr<C2ReflectorHelper> helper,const SupportedCapabilities & caps)209 EncodeInterface::EncodeInterface(const C2String& name, std::shared_ptr<C2ReflectorHelper> helper,
210 const SupportedCapabilities& caps)
211 : C2InterfaceHelper(std::move(helper)) {
212 ALOGV("%s(%s)", __func__, name.c_str());
213
214 setDerivedInstance(this);
215
216 Initialize(name, caps);
217 }
218
Initialize(const C2String & name,const SupportedCapabilities & caps)219 void EncodeInterface::Initialize(const C2String& name, const SupportedCapabilities& caps) {
220 // Note: unsigned int is used here, since std::vector<C2Config::profile_t> cannot convert to
221 // std::vector<unsigned int> required by the c2 framework below.
222 std::vector<unsigned int> profiles;
223 ui::Size maxSize;
224 for (const auto& supportedProfile : caps.supportedProfiles) {
225 if (!isValidProfileForCodec(caps.codec, supportedProfile.profile)) {
226 continue; // Ignore unrecognizable or unsupported profiles.
227 }
228 ALOGV("Queried c2_profile = 0x%x : max_size = %d x %d", supportedProfile.profile,
229 supportedProfile.max_resolution.width, supportedProfile.max_resolution.height);
230 profiles.push_back(static_cast<unsigned int>(supportedProfile.profile));
231 maxSize.setWidth(std::max(maxSize.width, supportedProfile.max_resolution.width));
232 maxSize.setHeight(std::max(maxSize.height, supportedProfile.max_resolution.height));
233 }
234
235 if (profiles.empty()) {
236 ALOGE("No supported profiles");
237 mInitStatus = C2_BAD_VALUE;
238 return;
239 }
240
241 // Special note: the order of addParameter matters if your setters are dependent on other
242 // parameters. Please make sure the dependent parameters are added prior to the
243 // one needs the setter dependency.
244
245 addParameter(DefineParam(mKind, C2_PARAMKEY_COMPONENT_KIND)
246 .withConstValue(new C2ComponentKindSetting(C2Component::KIND_ENCODER))
247 .build());
248
249 addParameter(DefineParam(mInputVisibleSize, C2_PARAMKEY_PICTURE_SIZE)
250 .withDefault(new C2StreamPictureSizeInfo::input(0u, 320, 240))
251 .withFields({
252 C2F(mInputVisibleSize, width).inRange(2, maxSize.width, 2),
253 C2F(mInputVisibleSize, height).inRange(2, maxSize.height, 2),
254 })
255 .withSetter(SizeSetter)
256 .build());
257
258 addParameter(DefineParam(mFrameRate, C2_PARAMKEY_FRAME_RATE)
259 .withDefault(new C2StreamFrameRateInfo::output(0u, kDefaultFrameRate))
260 // TODO: More restriction?
261 .withFields({C2F(mFrameRate, value).greaterThan(0.)})
262 .withSetter(Setter<decltype(*mFrameRate)>::StrictValueWithNoDeps)
263 .build());
264
265 addParameter(DefineParam(mBitrate, C2_PARAMKEY_BITRATE)
266 .withDefault(new C2StreamBitrateInfo::output(0u, kDefaultBitrate))
267 .withFields({C2F(mBitrate, value).inRange(0, kMaxBitrate)})
268 .withSetter(Setter<decltype(*mBitrate)>::StrictValueWithNoDeps)
269 .build());
270
271 addParameter(
272 DefineParam(mBitrateMode, C2_PARAMKEY_BITRATE_MODE)
273 .withDefault(new C2StreamBitrateModeTuning::output(0u, C2Config::BITRATE_CONST))
274 .withFields(
275 {C2F(mBitrateMode, value)
276 .oneOf({C2Config::BITRATE_CONST, C2Config::BITRATE_VARIABLE})})
277 .withSetter(Setter<decltype(*mBitrateMode)>::StrictValueWithNoDeps)
278 .build());
279
280 std::string outputMime;
281 if (caps.codec == VideoCodec::H264) {
282 outputMime = MEDIA_MIMETYPE_VIDEO_AVC;
283 C2Config::profile_t minProfile = static_cast<C2Config::profile_t>(
284 *std::min_element(profiles.begin(), profiles.end()));
285 addParameter(
286 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
287 .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile,
288 C2Config::LEVEL_AVC_4_1))
289 .withFields(
290 {C2F(mProfileLevel, profile).oneOf(profiles),
291 C2F(mProfileLevel, level)
292 // TODO: query supported levels from adaptor.
293 .oneOf({C2Config::LEVEL_AVC_1, C2Config::LEVEL_AVC_1B,
294 C2Config::LEVEL_AVC_1_1, C2Config::LEVEL_AVC_1_2,
295 C2Config::LEVEL_AVC_1_3, C2Config::LEVEL_AVC_2,
296 C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2,
297 C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1,
298 C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4,
299 C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2,
300 C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1})})
301 .withSetter(H264ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
302 .build());
303 } else if (caps.codec == VideoCodec::VP8) {
304 outputMime = MEDIA_MIMETYPE_VIDEO_VP8;
305 // VP8 doesn't have conventional profiles, we'll use profile0 if the VP8 codec is requested.
306 addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
307 .withConstValue(new C2StreamProfileLevelInfo::output(
308 0u, C2Config::PROFILE_VP8_0, C2Config::LEVEL_UNUSED))
309 .build());
310 } else if (caps.codec == VideoCodec::VP9) {
311 outputMime = MEDIA_MIMETYPE_VIDEO_VP9;
312 C2Config::profile_t minProfile = static_cast<C2Config::profile_t>(
313 *std::min_element(profiles.begin(), profiles.end()));
314 addParameter(
315 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL)
316 .withDefault(new C2StreamProfileLevelInfo::output(0u, minProfile,
317 C2Config::LEVEL_VP9_1))
318 .withFields(
319 {C2F(mProfileLevel, profile).oneOf(profiles),
320 C2F(mProfileLevel, level)
321 // TODO(dstaessens) query supported levels from adaptor.
322 .oneOf({C2Config::LEVEL_VP9_1, C2Config::LEVEL_VP9_1_1,
323 C2Config::LEVEL_VP9_2, C2Config::LEVEL_VP9_2_1,
324 C2Config::LEVEL_VP9_3, C2Config::LEVEL_VP9_3_1,
325 C2Config::LEVEL_VP9_4, C2Config::LEVEL_VP9_4_1,
326 C2Config::LEVEL_VP9_5, C2Config::LEVEL_VP9_5_1,
327 C2Config::LEVEL_VP9_5_2, C2Config::LEVEL_VP9_6,
328 C2Config::LEVEL_VP9_6_1,
329 C2Config::LEVEL_VP9_6_2})})
330 .withSetter(VP9ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
331 .build());
332 } else {
333 ALOGE("Unsupported component name: %s", name.c_str());
334 mInitStatus = C2_BAD_VALUE;
335 return;
336 }
337
338 addParameter(
339 DefineParam(mInputFormat, C2_PARAMKEY_INPUT_STREAM_BUFFER_TYPE)
340 .withConstValue(new C2StreamBufferTypeSetting::input(0u, C2BufferData::GRAPHIC))
341 .build());
342
343 addParameter(DefineParam(mInputMemoryUsage, C2_PARAMKEY_INPUT_STREAM_USAGE)
344 .withConstValue(new C2StreamUsageTuning::input(
345 0u, static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)))
346 .build());
347
348 addParameter(
349 DefineParam(mOutputFormat, C2_PARAMKEY_OUTPUT_STREAM_BUFFER_TYPE)
350 .withConstValue(new C2StreamBufferTypeSetting::output(0u, C2BufferData::LINEAR))
351 .build());
352
353 addParameter(DefineParam(mInputMediaType, C2_PARAMKEY_INPUT_MEDIA_TYPE)
354 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::input>(
355 MEDIA_MIMETYPE_VIDEO_RAW))
356 .build());
357
358 addParameter(DefineParam(mOutputMediaType, C2_PARAMKEY_OUTPUT_MEDIA_TYPE)
359 .withConstValue(AllocSharedString<C2PortMediaTypeSetting::output>(
360 outputMime.c_str()))
361 .build());
362
363 addParameter(DefineParam(mIntraRefreshPeriod, C2_PARAMKEY_INTRA_REFRESH)
364 .withDefault(new C2StreamIntraRefreshTuning::output(
365 0u, C2Config::INTRA_REFRESH_DISABLED, 0.))
366 .withFields({C2F(mIntraRefreshPeriod, mode)
367 .oneOf({C2Config::INTRA_REFRESH_DISABLED,
368 C2Config::INTRA_REFRESH_ARBITRARY}),
369 C2F(mIntraRefreshPeriod, period).any()})
370 .withSetter(IntraRefreshPeriodSetter)
371 .build());
372
373 addParameter(DefineParam(mRequestKeyFrame, C2_PARAMKEY_REQUEST_SYNC_FRAME)
374 .withDefault(new C2StreamRequestSyncFrameTuning::output(0u, C2_FALSE))
375 .withFields({C2F(mRequestKeyFrame, value).oneOf({C2_FALSE, C2_TRUE})})
376 .withSetter(Setter<decltype(*mRequestKeyFrame)>::NonStrictValueWithNoDeps)
377 .build());
378
379 addParameter(DefineParam(mKeyFramePeriodUs, C2_PARAMKEY_SYNC_FRAME_INTERVAL)
380 .withDefault(new C2StreamSyncFrameIntervalTuning::output(0u, 1000000))
381 .withFields({C2F(mKeyFramePeriodUs, value).any()})
382 .withSetter(Setter<decltype(*mKeyFramePeriodUs)>::StrictValueWithNoDeps)
383 .build());
384
385 C2Allocator::id_t inputAllocators[] = {kDefaultInputAllocator};
386
387 C2Allocator::id_t outputAllocators[] = {kDefaultOutputAllocator};
388
389 addParameter(
390 DefineParam(mInputAllocatorIds, C2_PARAMKEY_INPUT_ALLOCATORS)
391 .withConstValue(C2PortAllocatorsTuning::input::AllocShared(inputAllocators))
392 .build());
393
394 addParameter(
395 DefineParam(mOutputAllocatorIds, C2_PARAMKEY_OUTPUT_ALLOCATORS)
396 .withConstValue(C2PortAllocatorsTuning::output::AllocShared(outputAllocators))
397 .build());
398
399 C2BlockPool::local_id_t outputBlockPools[] = {kDefaultOutputBlockPool};
400
401 addParameter(
402 DefineParam(mOutputBlockPoolIds, C2_PARAMKEY_OUTPUT_BLOCK_POOLS)
403 .withDefault(C2PortBlockPoolsTuning::output::AllocShared(outputBlockPools))
404 .withFields({C2F(mOutputBlockPoolIds, m.values[0]).any(),
405 C2F(mOutputBlockPoolIds, m.values).inRange(0, 1)})
406 .withSetter(Setter<C2PortBlockPoolsTuning::output>::NonStrictValuesWithNoDeps)
407 .build());
408
409 mInitStatus = C2_OK;
410 }
411
getKeyFramePeriod() const412 uint32_t EncodeInterface::getKeyFramePeriod() const {
413 if (mKeyFramePeriodUs->value < 0 || mKeyFramePeriodUs->value == INT64_MAX) {
414 return 0;
415 }
416 double period = mKeyFramePeriodUs->value / 1e6 * mFrameRate->value;
417 return static_cast<uint32_t>(std::max(std::min(std::round(period), double(UINT32_MAX)), 1.));
418 }
419
420 } // namespace android
421