1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "VQApply"
19 #include <utils/Log.h>
20
21 #include <string>
22 #include <inttypes.h>
23
24 #include <media/NdkMediaFormat.h>
25
26 #include "VQops.h"
27 #include "CodecProperties.h"
28 #include "VideoShaper.h"
29
30 namespace android {
31 namespace mediaformatshaper {
32
33
34 // these are all NDK#31 and we run as NDK#29 (to be within the module)
35 // the __builtin_available(android 31, *) constructs didn't work for me.
36 //
37 #define AMEDIAFORMAT_VIDEO_QP_MAX "video-qp-max"
38 #define AMEDIAFORMAT_VIDEO_QP_MIN "video-qp-min"
39
40 #define AMEDIAFORMAT_VIDEO_QP_B_MAX "video-qp-b-max"
41 #define AMEDIAFORMAT_VIDEO_QP_B_MIN "video-qp-b-min"
42 #define AMEDIAFORMAT_VIDEO_QP_I_MAX "video-qp-i-max"
43 #define AMEDIAFORMAT_VIDEO_QP_I_MIN "video-qp-i-min"
44 #define AMEDIAFORMAT_VIDEO_QP_P_MAX "video-qp-p-max"
45 #define AMEDIAFORMAT_VIDEO_QP_P_MIN "video-qp-p-min"
46
47 // defined in the SDK, but not in the NDK
48 //
49 static const int BITRATE_MODE_VBR = 1;
50
51
52 //
53 // Caller retains ownership of and responsibility for inFormat
54 //
VQApply(CodecProperties * codec,vqOps_t * info,AMediaFormat * inFormat,int flags)55 int VQApply(CodecProperties *codec, vqOps_t *info, AMediaFormat* inFormat, int flags) {
56 ALOGV("codecName %s inFormat %p flags x%x", codec->getName().c_str(), inFormat, flags);
57 (void) info; // unused for now
58
59 int32_t bitRateMode = -1;
60 if (AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_BITRATE_MODE, &bitRateMode)
61 && bitRateMode != BITRATE_MODE_VBR) {
62 ALOGD("minquality: applies only to VBR encoding");
63 return 0;
64 }
65
66 // only proceed if we're in the handheld category.
67 // We embed this information within the codec record when we build up features
68 // and pass them in from MediaCodec; it's the easiest place to store it
69 //
70 // TODO: make a #define for ' _vq_eligible.device' here and in MediaCodec.cpp
71 //
72 int32_t isVQEligible = 0;
73 (void) codec->getFeatureValue("_vq_eligible.device", &isVQEligible);
74 if (!isVQEligible) {
75 ALOGD("minquality: not an eligible device class");
76 return 0;
77 }
78
79 // look at resolution to determine if we want any shaping/modification at all.
80 //
81 // we currently only shape (or ask the underlying codec to shape) for
82 // resolution range 320x240 < target <= 1920x1080
83 // NB: the < vs <=, that is deliberate.
84 //
85
86 int32_t width = 0;
87 (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_WIDTH, &width);
88 int32_t height = 0;
89 (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_HEIGHT, &height);
90 int64_t pixels = ((int64_t)width) * height;
91
92 bool eligibleSize = true;
93 if (pixels <= 320 * 240) {
94 eligibleSize = false;
95 } else if (pixels > 1920 * 1088) {
96 eligibleSize = false;
97 }
98
99 if (!eligibleSize) {
100 // we won't shape, and ask that the codec not shape
101 ALOGD("minquality: %dx%d outside of shaping range", width, height);
102 AMediaFormat_setInt32(inFormat, "android._encoding-quality-level", 0);
103 return 0;
104 }
105
106 if (codec->supportedMinimumQuality() > 0) {
107 // have the codec-provided minimum quality behavior to work at it
108 ALOGD("minquality: codec claims to implement minquality=%d",
109 codec->supportedMinimumQuality());
110
111 // tell the underlying codec to do its thing; we won't try to second guess.
112 // default to 1, aka S_HANDHELD;
113 int32_t qualityTarget = 1;
114 (void) codec->getFeatureValue("_quality.target", &qualityTarget);
115 AMediaFormat_setInt32(inFormat, "android._encoding-quality-level", qualityTarget);
116 return 0;
117 }
118
119 // let the codec know that we'll be enforcing the minimum quality standards
120 AMediaFormat_setInt32(inFormat, "android._encoding-quality-level", 0);
121
122 //
123 // consider any and all tools available
124 // -- qp
125 // -- minimum bits-per-pixel
126 //
127 int64_t bitrateChosen = 0;
128 int32_t qpChosen = INT32_MAX;
129
130 int64_t bitrateConfigured = 0;
131 int32_t bitrateConfiguredTmp = 0;
132 (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, &bitrateConfiguredTmp);
133 bitrateConfigured = bitrateConfiguredTmp;
134 bitrateChosen = bitrateConfigured;
135
136 // width, height, and pixels are calculated above
137
138 double minimumBpp = codec->getBpp(width, height);
139
140 int64_t bitrateFloor = pixels * minimumBpp;
141 int64_t bitrateCeiling = bitrateFloor * codec->getPhaseOut();
142 if (bitrateFloor > INT32_MAX) bitrateFloor = INT32_MAX;
143 if (bitrateCeiling > INT32_MAX) bitrateCeiling = INT32_MAX;
144
145 // if we are far enough above the target bpp, leave it alone
146 //
147 ALOGV("bitrate: configured %" PRId64 " floor %" PRId64, bitrateConfigured, bitrateFloor);
148 if (bitrateConfigured >= bitrateCeiling) {
149 ALOGV("high enough bitrate: configured %" PRId64 " >= ceiling %" PRId64,
150 bitrateConfigured, bitrateCeiling);
151 return 0;
152 }
153
154 // raise anything below the bitrate floor
155 if (bitrateConfigured < bitrateFloor) {
156 ALOGD("raise bitrate: configured %" PRId64 " to floor %" PRId64,
157 bitrateConfigured, bitrateFloor);
158 bitrateChosen = bitrateFloor;
159 }
160
161 bool qpPresent = hasQpMax(inFormat);
162
163 // calculate a target QP value
164 int32_t qpmax = codec->targetQpMax(width, height);
165 if (!qpPresent) {
166 // user didn't, so shaper wins
167 if (qpmax != INT32_MAX) {
168 ALOGV("choosing qp=%d", qpmax);
169 qpChosen = qpmax;
170 }
171 } else if (qpmax == INT32_MAX) {
172 // shaper didn't so user wins
173 qpChosen = INT32_MAX;
174 AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_VIDEO_QP_MAX, &qpChosen);
175 } else {
176 // both sides want it, choose most restrictive
177 int32_t value = INT32_MAX;
178 AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_VIDEO_QP_MAX, &value);
179 qpChosen = std::min(qpmax, value);
180 }
181
182 // if QP is desired but not supported, compensate with additional bits
183 if (!codec->supportsQp()) {
184 if (qpChosen != INT32_MAX) {
185 int64_t boost = bitrateChosen * codec->getMissingQpBoost();
186 ALOGD("minquality: requested QP unsupported, boost bitrate %" PRId64 " by %" PRId64,
187 bitrateChosen, boost);
188 bitrateChosen = bitrateChosen + boost;
189 qpChosen = INT32_MAX;
190 }
191 }
192
193 // limits
194 // apply our chosen values
195 //
196 if (qpChosen != INT32_MAX) {
197 ALOGD("minquality by QP: inject %s=%d", AMEDIAFORMAT_VIDEO_QP_MAX, qpChosen);
198 AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_VIDEO_QP_MAX, qpChosen);
199
200 // caller (VideoShaper) handles spreading this across the subframes
201 }
202
203 if (bitrateChosen != bitrateConfigured) {
204 if (bitrateChosen > bitrateCeiling) {
205 ALOGD("minquality: bitrate increase clamped at ceiling %" PRId64, bitrateCeiling);
206 bitrateChosen = bitrateCeiling;
207 }
208 ALOGD("minquality/target bitrate raised from %" PRId64 " to %" PRId64 " bps",
209 bitrateConfigured, bitrateChosen);
210 AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, (int32_t)bitrateChosen);
211 }
212
213 return 0;
214 }
215
216
hasQpMaxPerFrameType(AMediaFormat * format)217 bool hasQpMaxPerFrameType(AMediaFormat *format) {
218 int32_t value;
219
220 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &value)
221 || AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &value)) {
222 return true;
223 }
224 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &value)
225 || AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &value)) {
226 return true;
227 }
228 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &value)
229 || AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &value)) {
230 return true;
231 }
232 return false;
233 }
234
hasQpMaxGlobal(AMediaFormat * format)235 bool hasQpMaxGlobal(AMediaFormat *format) {
236 int32_t value;
237 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &value)
238 || AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &value)) {
239 return true;
240 }
241 return false;
242 }
243
hasQpMax(AMediaFormat * format)244 bool hasQpMax(AMediaFormat *format) {
245 if (hasQpMaxGlobal(format)) {
246 return true;
247 }
248 return hasQpMaxPerFrameType(format);
249 }
250
qpSpreadPerFrameType(AMediaFormat * format,int delta,int qplow,int qphigh,bool override)251 void qpSpreadPerFrameType(AMediaFormat *format, int delta,
252 int qplow, int qphigh, bool override) {
253
254 qpSpreadMinPerFrameType(format, qplow, override);
255 qpSpreadMaxPerFrameType(format, delta, qphigh, override);
256 // make sure that min<max for all the QP fields.
257 qpVerifyMinMaxOrdering(format);
258 }
259
qpSpreadMaxPerFrameType(AMediaFormat * format,int delta,int qphigh,bool override)260 void qpSpreadMaxPerFrameType(AMediaFormat *format, int delta, int qphigh, bool override) {
261 ALOGV("format %p delta %d hi %d override %d", format, delta, qphigh, override);
262
263 int32_t qpOffered = 0;
264 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &qpOffered)) {
265 // propagate to frame-specific keys, choosing most restrictive
266 // ensure that we don't violate min<=max rules
267 {
268 int32_t maxI = INT32_MAX;
269 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &maxI);
270 int32_t value = std::min({qpOffered, qphigh, maxI});
271 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, value);
272 }
273 {
274 int32_t maxP = INT32_MAX;
275 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &maxP);
276 int32_t value = std::min({(std::min(qpOffered, INT32_MAX-1*delta) + 1*delta),
277 qphigh, maxP});
278 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, value);
279 }
280 {
281 int32_t maxB = INT32_MAX;
282 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &maxB);
283 int32_t value = std::min({(std::min(qpOffered, INT32_MAX-2*delta) + 2*delta),
284 qphigh, maxB});
285 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, value);
286 }
287 }
288 }
289
qpSpreadMinPerFrameType(AMediaFormat * format,int qplow,bool override)290 void qpSpreadMinPerFrameType(AMediaFormat *format, int qplow, bool override) {
291 ALOGV("format %p lo %d override %d", format, qplow, override);
292
293 int32_t qpOffered = 0;
294 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &qpOffered)) {
295 int value = std::max(qplow, qpOffered);
296 // propagate to frame-specific keys, use lowest of this and existing per-frame value
297 int32_t minI = INT32_MAX;
298 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &minI);
299 int32_t setI = std::min(value, minI);
300 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, setI);
301
302 int32_t minP = INT32_MAX;
303 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &minP);
304 int32_t setP = std::min(value, minP);
305 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, setP);
306
307 int32_t minB = INT32_MAX;
308 AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &minB);
309 int32_t setB = std::min(value, minB);
310 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, setB);
311 }
312 }
313
314 // XXX whether we allow min==max, or if we'll insist that min<max
qpVerifyMinMaxOrdering(AMediaFormat * format)315 void qpVerifyMinMaxOrdering(AMediaFormat *format) {
316 // ensure that we don't violate min<=max rules
317 int32_t maxI = INT32_MAX;
318 int32_t minI = INT32_MIN;
319 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &maxI)
320 && AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &minI)
321 && minI > maxI) {
322 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, maxI);
323 }
324 int32_t maxP = INT32_MAX;
325 int32_t minP = INT32_MIN;
326 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &maxP)
327 && AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &minP)
328 && minP > maxP) {
329 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, maxP);
330 }
331 int32_t maxB = INT32_MAX;
332 int32_t minB = INT32_MIN;
333 if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &maxB)
334 && AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &minB)
335 && minB > maxB) {
336 AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, maxB);
337 }
338 }
339
340 } // namespace mediaformatshaper
341 } // namespace android
342
343