1 /*
2 * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AV1_ENCODER_RDOPT_UTILS_H_
13 #define AOM_AV1_ENCODER_RDOPT_UTILS_H_
14
15 #include "aom/aom_integer.h"
16 #include "av1/encoder/block.h"
17 #include "av1/common/cfl.h"
18 #include "av1/common/pred_common.h"
19 #include "av1/encoder/rdopt_data_defs.h"
20
21 #ifdef __cplusplus
22 extern "C" {
23 #endif
24
25 #define MAX_REF_MV_SEARCH 3
26 #define MAX_TX_RD_GATE_LEVEL 5
27 #define INTER_INTRA_RD_THRESH_SCALE 9
28 #define INTER_INTRA_RD_THRESH_SHIFT 4
29
30 typedef struct {
31 PREDICTION_MODE mode;
32 MV_REFERENCE_FRAME ref_frame[2];
33 } MODE_DEFINITION;
34
35 // This array defines the mapping from the enums in THR_MODES to the actual
36 // prediction modes and refrence frames
37 static const MODE_DEFINITION av1_mode_defs[MAX_MODES] = {
38 { NEARESTMV, { LAST_FRAME, NONE_FRAME } },
39 { NEARESTMV, { LAST2_FRAME, NONE_FRAME } },
40 { NEARESTMV, { LAST3_FRAME, NONE_FRAME } },
41 { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } },
42 { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } },
43 { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } },
44 { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } },
45
46 { NEWMV, { LAST_FRAME, NONE_FRAME } },
47 { NEWMV, { LAST2_FRAME, NONE_FRAME } },
48 { NEWMV, { LAST3_FRAME, NONE_FRAME } },
49 { NEWMV, { BWDREF_FRAME, NONE_FRAME } },
50 { NEWMV, { ALTREF2_FRAME, NONE_FRAME } },
51 { NEWMV, { ALTREF_FRAME, NONE_FRAME } },
52 { NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
53
54 { NEARMV, { LAST_FRAME, NONE_FRAME } },
55 { NEARMV, { LAST2_FRAME, NONE_FRAME } },
56 { NEARMV, { LAST3_FRAME, NONE_FRAME } },
57 { NEARMV, { BWDREF_FRAME, NONE_FRAME } },
58 { NEARMV, { ALTREF2_FRAME, NONE_FRAME } },
59 { NEARMV, { ALTREF_FRAME, NONE_FRAME } },
60 { NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
61
62 { GLOBALMV, { LAST_FRAME, NONE_FRAME } },
63 { GLOBALMV, { LAST2_FRAME, NONE_FRAME } },
64 { GLOBALMV, { LAST3_FRAME, NONE_FRAME } },
65 { GLOBALMV, { BWDREF_FRAME, NONE_FRAME } },
66 { GLOBALMV, { ALTREF2_FRAME, NONE_FRAME } },
67 { GLOBALMV, { ALTREF_FRAME, NONE_FRAME } },
68 { GLOBALMV, { GOLDEN_FRAME, NONE_FRAME } },
69
70 // TODO(zoeliu): May need to reconsider the order on the modes to check
71
72 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
73 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
74 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
75 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
76 { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
77 { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
78 { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
79 { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
80 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
81 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
82 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
83 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
84
85 { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
86 { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
87 { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
88 { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
89
90 { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
91 { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
92 { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
93 { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
94 { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
95 { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
96 { GLOBAL_GLOBALMV, { LAST_FRAME, BWDREF_FRAME } },
97
98 { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
99 { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
100 { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
101 { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
102 { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
103 { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
104 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF_FRAME } },
105
106 { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
107 { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
108 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
109 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
110 { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
111 { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
112 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF_FRAME } },
113
114 { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
115 { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
116 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
117 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
118 { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
119 { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
120 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF_FRAME } },
121
122 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
123 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
124 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
125 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
126 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
127 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
128 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF_FRAME } },
129
130 { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
131 { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
132 { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
133 { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
134 { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
135 { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
136 { GLOBAL_GLOBALMV, { LAST2_FRAME, BWDREF_FRAME } },
137
138 { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
139 { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
140 { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
141 { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
142 { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
143 { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
144 { GLOBAL_GLOBALMV, { LAST3_FRAME, BWDREF_FRAME } },
145
146 { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
147 { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
148 { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
149 { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
150 { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
151 { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
152 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, BWDREF_FRAME } },
153
154 { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
155 { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
156 { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
157 { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
158 { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
159 { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
160 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF2_FRAME } },
161
162 { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
163 { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
164 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
165 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
166 { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
167 { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
168 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF2_FRAME } },
169
170 { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
171 { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
172 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
173 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
174 { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
175 { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
176 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF2_FRAME } },
177
178 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
179 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
180 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
181 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
182 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
183 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
184 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
185
186 { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } },
187 { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } },
188 { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
189 { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } },
190 { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } },
191 { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } },
192 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST2_FRAME } },
193
194 { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } },
195 { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } },
196 { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
197 { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } },
198 { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } },
199 { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } },
200 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST3_FRAME } },
201
202 { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
203 { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
204 { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
205 { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
206 { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
207 { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
208 { GLOBAL_GLOBALMV, { LAST_FRAME, GOLDEN_FRAME } },
209
210 { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
211 { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
212 { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
213 { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
214 { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
215 { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
216 { GLOBAL_GLOBALMV, { BWDREF_FRAME, ALTREF_FRAME } },
217
218 // intra modes
219 { DC_PRED, { INTRA_FRAME, NONE_FRAME } },
220 { PAETH_PRED, { INTRA_FRAME, NONE_FRAME } },
221 { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } },
222 { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } },
223 { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } },
224 { H_PRED, { INTRA_FRAME, NONE_FRAME } },
225 { V_PRED, { INTRA_FRAME, NONE_FRAME } },
226 { D135_PRED, { INTRA_FRAME, NONE_FRAME } },
227 { D203_PRED, { INTRA_FRAME, NONE_FRAME } },
228 { D157_PRED, { INTRA_FRAME, NONE_FRAME } },
229 { D67_PRED, { INTRA_FRAME, NONE_FRAME } },
230 { D113_PRED, { INTRA_FRAME, NONE_FRAME } },
231 { D45_PRED, { INTRA_FRAME, NONE_FRAME } },
232 };
233
234 // Number of winner modes allowed for different values of the speed feature
235 // multi_winner_mode_type.
236 static const int winner_mode_count_allowed[MULTI_WINNER_MODE_LEVELS] = {
237 1, // MULTI_WINNER_MODE_OFF
238 2, // MULTI_WINNER_MODE_FAST
239 3 // MULTI_WINNER_MODE_DEFAULT
240 };
241
restore_dst_buf(MACROBLOCKD * xd,const BUFFER_SET dst,const int num_planes)242 static AOM_INLINE void restore_dst_buf(MACROBLOCKD *xd, const BUFFER_SET dst,
243 const int num_planes) {
244 for (int i = 0; i < num_planes; i++) {
245 xd->plane[i].dst.buf = dst.plane[i];
246 xd->plane[i].dst.stride = dst.stride[i];
247 }
248 }
249
swap_dst_buf(MACROBLOCKD * xd,const BUFFER_SET * dst_bufs[2],int num_planes)250 static AOM_INLINE void swap_dst_buf(MACROBLOCKD *xd,
251 const BUFFER_SET *dst_bufs[2],
252 int num_planes) {
253 const BUFFER_SET *buf0 = dst_bufs[0];
254 dst_bufs[0] = dst_bufs[1];
255 dst_bufs[1] = buf0;
256 restore_dst_buf(xd, *dst_bufs[0], num_planes);
257 }
258
259 /* clang-format on */
260 // Calculate rd threshold based on ref best rd and relevant scaling factors
get_rd_thresh_from_best_rd(int64_t ref_best_rd,int mul_factor,int div_factor)261 static AOM_INLINE int64_t get_rd_thresh_from_best_rd(int64_t ref_best_rd,
262 int mul_factor,
263 int div_factor) {
264 int64_t rd_thresh = ref_best_rd;
265 if (div_factor != 0) {
266 rd_thresh = ref_best_rd < (div_factor * (INT64_MAX / mul_factor))
267 ? ((ref_best_rd / div_factor) * mul_factor)
268 : INT64_MAX;
269 }
270 return rd_thresh;
271 }
272
273 static AOM_INLINE THR_MODES
get_prediction_mode_idx(PREDICTION_MODE this_mode,MV_REFERENCE_FRAME ref_frame,MV_REFERENCE_FRAME second_ref_frame)274 get_prediction_mode_idx(PREDICTION_MODE this_mode, MV_REFERENCE_FRAME ref_frame,
275 MV_REFERENCE_FRAME second_ref_frame) {
276 if (this_mode < INTRA_MODE_END) {
277 assert(ref_frame == INTRA_FRAME);
278 assert(second_ref_frame == NONE_FRAME);
279 return intra_to_mode_idx[this_mode - INTRA_MODE_START];
280 }
281 if (this_mode >= SINGLE_INTER_MODE_START &&
282 this_mode < SINGLE_INTER_MODE_END) {
283 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME));
284 return single_inter_to_mode_idx[this_mode - SINGLE_INTER_MODE_START]
285 [ref_frame];
286 }
287 if (this_mode >= COMP_INTER_MODE_START && this_mode < COMP_INTER_MODE_END &&
288 second_ref_frame != NONE_FRAME) {
289 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME));
290 assert((second_ref_frame > INTRA_FRAME) &&
291 (second_ref_frame <= ALTREF_FRAME));
292 return comp_inter_to_mode_idx[this_mode - COMP_INTER_MODE_START][ref_frame]
293 [second_ref_frame];
294 }
295 assert(0);
296 return THR_INVALID;
297 }
298
inter_mode_data_block_idx(BLOCK_SIZE bsize)299 static AOM_INLINE int inter_mode_data_block_idx(BLOCK_SIZE bsize) {
300 if (bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 ||
301 bsize == BLOCK_4X16 || bsize == BLOCK_16X4) {
302 return -1;
303 }
304 return 1;
305 }
306
307 // Get transform block visible dimensions cropped to the MI units.
get_txb_dimensions(const MACROBLOCKD * xd,int plane,BLOCK_SIZE plane_bsize,int blk_row,int blk_col,BLOCK_SIZE tx_bsize,int * width,int * height,int * visible_width,int * visible_height)308 static AOM_INLINE void get_txb_dimensions(const MACROBLOCKD *xd, int plane,
309 BLOCK_SIZE plane_bsize, int blk_row,
310 int blk_col, BLOCK_SIZE tx_bsize,
311 int *width, int *height,
312 int *visible_width,
313 int *visible_height) {
314 assert(tx_bsize <= plane_bsize);
315 const int txb_height = block_size_high[tx_bsize];
316 const int txb_width = block_size_wide[tx_bsize];
317 const struct macroblockd_plane *const pd = &xd->plane[plane];
318
319 // TODO(aconverse@google.com): Investigate using crop_width/height here rather
320 // than the MI size
321 if (xd->mb_to_bottom_edge >= 0) {
322 *visible_height = txb_height;
323 } else {
324 const int block_height = block_size_high[plane_bsize];
325 const int block_rows =
326 (xd->mb_to_bottom_edge >> (3 + pd->subsampling_y)) + block_height;
327 *visible_height =
328 clamp(block_rows - (blk_row << MI_SIZE_LOG2), 0, txb_height);
329 }
330 if (height) *height = txb_height;
331
332 if (xd->mb_to_right_edge >= 0) {
333 *visible_width = txb_width;
334 } else {
335 const int block_width = block_size_wide[plane_bsize];
336 const int block_cols =
337 (xd->mb_to_right_edge >> (3 + pd->subsampling_x)) + block_width;
338 *visible_width =
339 clamp(block_cols - (blk_col << MI_SIZE_LOG2), 0, txb_width);
340 }
341 if (width) *width = txb_width;
342 }
343
bsize_to_num_blk(BLOCK_SIZE bsize)344 static AOM_INLINE int bsize_to_num_blk(BLOCK_SIZE bsize) {
345 int num_blk = 1 << (num_pels_log2_lookup[bsize] - 2 * MI_SIZE_LOG2);
346 return num_blk;
347 }
348
check_txfm_eval(MACROBLOCK * const x,BLOCK_SIZE bsize,int64_t best_skip_rd,int64_t skip_rd,int level,int is_luma_only)349 static INLINE int check_txfm_eval(MACROBLOCK *const x, BLOCK_SIZE bsize,
350 int64_t best_skip_rd, int64_t skip_rd,
351 int level, int is_luma_only) {
352 int eval_txfm = 1;
353 // Derive aggressiveness factor for gating the transform search
354 // Lower value indicates more aggressiveness. Be more conservative (high
355 // value) for (i) low quantizers (ii) regions where prediction is poor
356 const int scale[MAX_TX_RD_GATE_LEVEL + 1] = { INT_MAX, 4, 3, 2, 2, 1 };
357 const int qslope = 2 * (!is_luma_only);
358 const int level_to_qindex_map[MAX_TX_RD_GATE_LEVEL + 1] = { 0, 0, 0,
359 80, 100, 140 };
360 int aggr_factor = 4;
361 assert(level <= MAX_TX_RD_GATE_LEVEL);
362 const int pred_qindex_thresh = level_to_qindex_map[level];
363 if (!is_luma_only && level <= 2) {
364 aggr_factor = 4 * AOMMAX(1, ROUND_POWER_OF_TWO((MAXQ - x->qindex) * qslope,
365 QINDEX_BITS));
366 }
367 if ((best_skip_rd >
368 (x->source_variance << (num_pels_log2_lookup[bsize] + RDDIV_BITS))) &&
369 (x->qindex >= pred_qindex_thresh))
370 aggr_factor *= scale[level];
371 // For level setting 1, be more conservative for non-luma-only case even when
372 // prediction is good.
373 else if ((level <= 1) && !is_luma_only)
374 aggr_factor = (aggr_factor >> 2) * 6;
375
376 // Be more conservative for luma only cases (called from compound type rd)
377 // since best_skip_rd is computed after and skip_rd is computed (with 8-bit
378 // prediction signals blended for WEDGE/DIFFWTD rather than 16-bit) before
379 // interpolation filter search
380 const int luma_mul[MAX_TX_RD_GATE_LEVEL + 1] = {
381 INT_MAX, 32, 29, 17, 17, 17
382 };
383 int mul_factor = is_luma_only ? luma_mul[level] : 16;
384 int64_t rd_thresh =
385 (best_skip_rd == INT64_MAX)
386 ? best_skip_rd
387 : (int64_t)(best_skip_rd * aggr_factor * mul_factor >> 6);
388 if (skip_rd > rd_thresh) eval_txfm = 0;
389 return eval_txfm;
390 }
391
select_tx_mode(const AV1_COMMON * cm,const TX_SIZE_SEARCH_METHOD tx_size_search_method)392 static TX_MODE select_tx_mode(
393 const AV1_COMMON *cm, const TX_SIZE_SEARCH_METHOD tx_size_search_method) {
394 if (cm->features.coded_lossless) return ONLY_4X4;
395 if (tx_size_search_method == USE_LARGESTALL) {
396 return TX_MODE_LARGEST;
397 } else {
398 assert(tx_size_search_method == USE_FULL_RD ||
399 tx_size_search_method == USE_FAST_RD);
400 return TX_MODE_SELECT;
401 }
402 }
403
404 // Checks the conditions to disable winner mode processing
bypass_winner_mode_processing(const MACROBLOCK * const x,const SPEED_FEATURES * sf,int use_txfm_skip,int actual_txfm_skip,PREDICTION_MODE best_mode)405 static INLINE int bypass_winner_mode_processing(const MACROBLOCK *const x,
406 const SPEED_FEATURES *sf,
407 int use_txfm_skip,
408 int actual_txfm_skip,
409 PREDICTION_MODE best_mode) {
410 const int prune_winner_mode_eval_level =
411 sf->winner_mode_sf.prune_winner_mode_eval_level;
412
413 // Disable winner mode processing for blocks with low source variance.
414 // The aggressiveness of this pruning logic reduces as qindex increases.
415 // The threshold decreases linearly from 64 as qindex varies from 0 to 255.
416 if (prune_winner_mode_eval_level == 1) {
417 const unsigned int src_var_thresh = 64 - 48 * x->qindex / (MAXQ + 1);
418 if (x->source_variance < src_var_thresh) return 1;
419 } else if (prune_winner_mode_eval_level == 2) {
420 // Skip winner mode processing of blocks for which transform turns out to be
421 // skip due to nature of eob alone except NEWMV mode.
422 if (!have_newmv_in_inter_mode(best_mode) && actual_txfm_skip) return 1;
423 } else if (prune_winner_mode_eval_level == 3) {
424 // Skip winner mode processing of blocks for which transform turns out to be
425 // skip except NEWMV mode and considered based on the quantizer.
426 // At high quantizers: Take conservative approach by considering transform
427 // skip based on eob alone.
428 // At low quantizers: Consider transform skip based on eob nature or RD cost
429 // evaluation.
430 const int is_txfm_skip =
431 x->qindex > 127 ? actual_txfm_skip : actual_txfm_skip || use_txfm_skip;
432
433 if (!have_newmv_in_inter_mode(best_mode) && is_txfm_skip) return 1;
434 } else if (prune_winner_mode_eval_level >= 4) {
435 // Do not skip winner mode evaluation at low quantizers if normal mode's
436 // transform search was too aggressive.
437 if (sf->rd_sf.perform_coeff_opt >= 5 && x->qindex <= 70) return 0;
438
439 if (use_txfm_skip || actual_txfm_skip) return 1;
440 }
441
442 return 0;
443 }
444
445 // Checks the conditions to enable winner mode processing
is_winner_mode_processing_enabled(const struct AV1_COMP * cpi,const MACROBLOCK * const x,MB_MODE_INFO * const mbmi,int actual_txfm_skip)446 static INLINE int is_winner_mode_processing_enabled(const struct AV1_COMP *cpi,
447 const MACROBLOCK *const x,
448 MB_MODE_INFO *const mbmi,
449 int actual_txfm_skip) {
450 const SPEED_FEATURES *sf = &cpi->sf;
451 const PREDICTION_MODE best_mode = mbmi->mode;
452
453 if (bypass_winner_mode_processing(x, sf, mbmi->skip_txfm, actual_txfm_skip,
454 best_mode))
455 return 0;
456
457 // TODO(any): Move block independent condition checks to frame level
458 if (is_inter_block(mbmi)) {
459 if (is_inter_mode(best_mode) &&
460 (sf->tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh != INT_MAX) &&
461 !cpi->oxcf.txfm_cfg.use_inter_dct_only)
462 return 1;
463 } else {
464 if (sf->tx_sf.tx_type_search.fast_intra_tx_type_search &&
465 !cpi->oxcf.txfm_cfg.use_intra_default_tx_only &&
466 !cpi->oxcf.txfm_cfg.use_intra_dct_only)
467 return 1;
468 }
469
470 // Check speed feature related to winner mode processing
471 if (sf->winner_mode_sf.enable_winner_mode_for_coeff_opt &&
472 cpi->optimize_seg_arr[mbmi->segment_id] != NO_TRELLIS_OPT &&
473 cpi->optimize_seg_arr[mbmi->segment_id] != FINAL_PASS_TRELLIS_OPT)
474 return 1;
475 if (sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch) return 1;
476
477 return 0;
478 }
479
set_tx_size_search_method(const AV1_COMMON * cm,const WinnerModeParams * winner_mode_params,TxfmSearchParams * txfm_params,int enable_winner_mode_for_tx_size_srch,int is_winner_mode)480 static INLINE void set_tx_size_search_method(
481 const AV1_COMMON *cm, const WinnerModeParams *winner_mode_params,
482 TxfmSearchParams *txfm_params, int enable_winner_mode_for_tx_size_srch,
483 int is_winner_mode) {
484 // Populate transform size search method/transform mode appropriately
485 txfm_params->tx_size_search_method =
486 winner_mode_params->tx_size_search_methods[DEFAULT_EVAL];
487 if (enable_winner_mode_for_tx_size_srch) {
488 if (is_winner_mode)
489 txfm_params->tx_size_search_method =
490 winner_mode_params->tx_size_search_methods[WINNER_MODE_EVAL];
491 else
492 txfm_params->tx_size_search_method =
493 winner_mode_params->tx_size_search_methods[MODE_EVAL];
494 }
495 txfm_params->tx_mode_search_type =
496 select_tx_mode(cm, txfm_params->tx_size_search_method);
497 }
498
set_tx_type_prune(const SPEED_FEATURES * sf,TxfmSearchParams * txfm_params,int winner_mode_tx_type_pruning,int is_winner_mode)499 static INLINE void set_tx_type_prune(const SPEED_FEATURES *sf,
500 TxfmSearchParams *txfm_params,
501 int winner_mode_tx_type_pruning,
502 int is_winner_mode) {
503 // Populate prune transform mode appropriately
504 txfm_params->prune_2d_txfm_mode = sf->tx_sf.tx_type_search.prune_2d_txfm_mode;
505 if (!winner_mode_tx_type_pruning) return;
506
507 const int prune_mode[4][2] = { { TX_TYPE_PRUNE_3, TX_TYPE_PRUNE_0 },
508 { TX_TYPE_PRUNE_4, TX_TYPE_PRUNE_0 },
509 { TX_TYPE_PRUNE_5, TX_TYPE_PRUNE_2 },
510 { TX_TYPE_PRUNE_5, TX_TYPE_PRUNE_3 } };
511 txfm_params->prune_2d_txfm_mode =
512 prune_mode[winner_mode_tx_type_pruning - 1][is_winner_mode];
513 }
514
set_tx_domain_dist_params(const WinnerModeParams * winner_mode_params,TxfmSearchParams * txfm_params,int enable_winner_mode_for_tx_domain_dist,int is_winner_mode)515 static INLINE void set_tx_domain_dist_params(
516 const WinnerModeParams *winner_mode_params, TxfmSearchParams *txfm_params,
517 int enable_winner_mode_for_tx_domain_dist, int is_winner_mode) {
518 if (txfm_params->use_qm_dist_metric) {
519 // QM-weighted PSNR is computed in transform space, so we need to forcibly
520 // enable the use of tx domain distortion.
521 txfm_params->use_transform_domain_distortion = 1;
522 txfm_params->tx_domain_dist_threshold = 0;
523 return;
524 }
525
526 if (!enable_winner_mode_for_tx_domain_dist) {
527 txfm_params->use_transform_domain_distortion =
528 winner_mode_params->use_transform_domain_distortion[DEFAULT_EVAL];
529 txfm_params->tx_domain_dist_threshold =
530 winner_mode_params->tx_domain_dist_threshold[DEFAULT_EVAL];
531 return;
532 }
533
534 if (is_winner_mode) {
535 txfm_params->use_transform_domain_distortion =
536 winner_mode_params->use_transform_domain_distortion[WINNER_MODE_EVAL];
537 txfm_params->tx_domain_dist_threshold =
538 winner_mode_params->tx_domain_dist_threshold[WINNER_MODE_EVAL];
539 } else {
540 txfm_params->use_transform_domain_distortion =
541 winner_mode_params->use_transform_domain_distortion[MODE_EVAL];
542 txfm_params->tx_domain_dist_threshold =
543 winner_mode_params->tx_domain_dist_threshold[MODE_EVAL];
544 }
545 }
546
547 // This function sets mode parameters for different mode evaluation stages
set_mode_eval_params(const struct AV1_COMP * cpi,MACROBLOCK * x,MODE_EVAL_TYPE mode_eval_type)548 static INLINE void set_mode_eval_params(const struct AV1_COMP *cpi,
549 MACROBLOCK *x,
550 MODE_EVAL_TYPE mode_eval_type) {
551 const AV1_COMMON *cm = &cpi->common;
552 const SPEED_FEATURES *sf = &cpi->sf;
553 const WinnerModeParams *winner_mode_params = &cpi->winner_mode_params;
554 TxfmSearchParams *txfm_params = &x->txfm_search_params;
555
556 txfm_params->use_qm_dist_metric =
557 cpi->oxcf.tune_cfg.dist_metric == AOM_DIST_METRIC_QM_PSNR;
558
559 switch (mode_eval_type) {
560 case DEFAULT_EVAL:
561 txfm_params->default_inter_tx_type_prob_thresh = INT_MAX;
562 txfm_params->use_default_intra_tx_type = 0;
563 txfm_params->skip_txfm_level =
564 winner_mode_params->skip_txfm_level[DEFAULT_EVAL];
565 txfm_params->predict_dc_level =
566 winner_mode_params->predict_dc_level[DEFAULT_EVAL];
567 // Set default transform domain distortion type
568 set_tx_domain_dist_params(winner_mode_params, txfm_params, 0, 0);
569
570 // Get default threshold for R-D optimization of coefficients
571 get_rd_opt_coeff_thresh(winner_mode_params->coeff_opt_thresholds,
572 txfm_params, 0, 0);
573
574 // Set default transform size search method
575 set_tx_size_search_method(cm, winner_mode_params, txfm_params, 0, 0);
576 // Set default transform type prune
577 set_tx_type_prune(sf, txfm_params, 0, 0);
578 break;
579 case MODE_EVAL:
580 txfm_params->use_default_intra_tx_type =
581 (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search ||
582 cpi->oxcf.txfm_cfg.use_intra_default_tx_only);
583 txfm_params->default_inter_tx_type_prob_thresh =
584 cpi->sf.tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh;
585 txfm_params->skip_txfm_level =
586 winner_mode_params->skip_txfm_level[MODE_EVAL];
587 txfm_params->predict_dc_level =
588 winner_mode_params->predict_dc_level[MODE_EVAL];
589 // Set transform domain distortion type for mode evaluation
590 set_tx_domain_dist_params(
591 winner_mode_params, txfm_params,
592 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 0);
593
594 // Get threshold for R-D optimization of coefficients during mode
595 // evaluation
596 get_rd_opt_coeff_thresh(
597 winner_mode_params->coeff_opt_thresholds, txfm_params,
598 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 0);
599
600 // Set the transform size search method for mode evaluation
601 set_tx_size_search_method(
602 cm, winner_mode_params, txfm_params,
603 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 0);
604 // Set transform type prune for mode evaluation
605 set_tx_type_prune(sf, txfm_params,
606 sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning,
607 0);
608 break;
609 case WINNER_MODE_EVAL:
610 txfm_params->default_inter_tx_type_prob_thresh = INT_MAX;
611 txfm_params->use_default_intra_tx_type = 0;
612 txfm_params->skip_txfm_level =
613 winner_mode_params->skip_txfm_level[WINNER_MODE_EVAL];
614 txfm_params->predict_dc_level =
615 winner_mode_params->predict_dc_level[WINNER_MODE_EVAL];
616
617 // Set transform domain distortion type for winner mode evaluation
618 set_tx_domain_dist_params(
619 winner_mode_params, txfm_params,
620 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 1);
621
622 // Get threshold for R-D optimization of coefficients for winner mode
623 // evaluation
624 get_rd_opt_coeff_thresh(
625 winner_mode_params->coeff_opt_thresholds, txfm_params,
626 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 1);
627
628 // Set the transform size search method for winner mode evaluation
629 set_tx_size_search_method(
630 cm, winner_mode_params, txfm_params,
631 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 1);
632 // Set default transform type prune mode for winner mode evaluation
633 set_tx_type_prune(sf, txfm_params,
634 sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning,
635 1);
636 break;
637 default: assert(0);
638 }
639
640 // Rd record collected at a specific mode evaluation stage can not be used
641 // across other evaluation stages as the transform parameters are different.
642 // Hence, reset mb rd record whenever mode evaluation stage type changes.
643 if (txfm_params->mode_eval_type != mode_eval_type)
644 reset_mb_rd_record(x->txfm_search_info.mb_rd_record);
645
646 txfm_params->mode_eval_type = mode_eval_type;
647 }
648
649 // Similar to store_cfl_required(), but for use during the RDO process,
650 // where we haven't yet determined whether this block uses CfL.
store_cfl_required_rdo(const AV1_COMMON * cm,const MACROBLOCK * x)651 static INLINE CFL_ALLOWED_TYPE store_cfl_required_rdo(const AV1_COMMON *cm,
652 const MACROBLOCK *x) {
653 const MACROBLOCKD *xd = &x->e_mbd;
654
655 if (cm->seq_params->monochrome || !xd->is_chroma_ref) return CFL_DISALLOWED;
656
657 if (!xd->is_chroma_ref) {
658 // For non-chroma-reference blocks, we should always store the luma pixels,
659 // in case the corresponding chroma-reference block uses CfL.
660 // Note that this can only happen for block sizes which are <8 on
661 // their shortest side, as otherwise they would be chroma reference
662 // blocks.
663 return CFL_ALLOWED;
664 }
665
666 // For chroma reference blocks, we should store data in the encoder iff we're
667 // allowed to try out CfL.
668 return is_cfl_allowed(xd);
669 }
670
init_sbuv_mode(MB_MODE_INFO * const mbmi)671 static AOM_INLINE void init_sbuv_mode(MB_MODE_INFO *const mbmi) {
672 mbmi->uv_mode = UV_DC_PRED;
673 mbmi->palette_mode_info.palette_size[1] = 0;
674 }
675
676 // Store best mode stats for winner mode processing
store_winner_mode_stats(const AV1_COMMON * const cm,MACROBLOCK * x,const MB_MODE_INFO * mbmi,RD_STATS * rd_cost,RD_STATS * rd_cost_y,RD_STATS * rd_cost_uv,THR_MODES mode_index,uint8_t * color_map,BLOCK_SIZE bsize,int64_t this_rd,int multi_winner_mode_type,int txfm_search_done)677 static INLINE void store_winner_mode_stats(
678 const AV1_COMMON *const cm, MACROBLOCK *x, const MB_MODE_INFO *mbmi,
679 RD_STATS *rd_cost, RD_STATS *rd_cost_y, RD_STATS *rd_cost_uv,
680 THR_MODES mode_index, uint8_t *color_map, BLOCK_SIZE bsize, int64_t this_rd,
681 int multi_winner_mode_type, int txfm_search_done) {
682 WinnerModeStats *winner_mode_stats = x->winner_mode_stats;
683 int mode_idx = 0;
684 int is_palette_mode = mbmi->palette_mode_info.palette_size[PLANE_TYPE_Y] > 0;
685 // Mode stat is not required when multiwinner mode processing is disabled
686 if (multi_winner_mode_type == MULTI_WINNER_MODE_OFF) return;
687 // Ignore mode with maximum rd
688 if (this_rd == INT64_MAX) return;
689 // TODO(any): Winner mode processing is currently not applicable for palette
690 // mode in Inter frames. Clean-up the following code, once support is added
691 if (!frame_is_intra_only(cm) && is_palette_mode) return;
692
693 int max_winner_mode_count = winner_mode_count_allowed[multi_winner_mode_type];
694 assert(x->winner_mode_count >= 0 &&
695 x->winner_mode_count <= max_winner_mode_count);
696
697 if (x->winner_mode_count) {
698 // Find the mode which has higher rd cost than this_rd
699 for (mode_idx = 0; mode_idx < x->winner_mode_count; mode_idx++)
700 if (winner_mode_stats[mode_idx].rd > this_rd) break;
701
702 if (mode_idx == max_winner_mode_count) {
703 // No mode has higher rd cost than this_rd
704 return;
705 } else if (mode_idx < max_winner_mode_count - 1) {
706 // Create a slot for current mode and move others to the next slot
707 memmove(
708 &winner_mode_stats[mode_idx + 1], &winner_mode_stats[mode_idx],
709 (max_winner_mode_count - mode_idx - 1) * sizeof(*winner_mode_stats));
710 }
711 }
712 // Add a mode stat for winner mode processing
713 winner_mode_stats[mode_idx].mbmi = *mbmi;
714 winner_mode_stats[mode_idx].rd = this_rd;
715 winner_mode_stats[mode_idx].mode_index = mode_index;
716
717 // Update rd stats required for inter frame
718 if (!frame_is_intra_only(cm) && rd_cost && rd_cost_y && rd_cost_uv) {
719 const MACROBLOCKD *xd = &x->e_mbd;
720 const int skip_ctx = av1_get_skip_txfm_context(xd);
721 const int is_intra_mode = av1_mode_defs[mode_index].mode < INTRA_MODE_END;
722 const int skip_txfm = mbmi->skip_txfm && !is_intra_mode;
723
724 winner_mode_stats[mode_idx].rd_cost = *rd_cost;
725 if (txfm_search_done) {
726 winner_mode_stats[mode_idx].rate_y =
727 rd_cost_y->rate +
728 x->mode_costs
729 .skip_txfm_cost[skip_ctx][rd_cost->skip_txfm || skip_txfm];
730 winner_mode_stats[mode_idx].rate_uv = rd_cost_uv->rate;
731 }
732 }
733
734 if (color_map) {
735 // Store color_index_map for palette mode
736 const MACROBLOCKD *const xd = &x->e_mbd;
737 int block_width, block_height;
738 av1_get_block_dimensions(bsize, AOM_PLANE_Y, xd, &block_width,
739 &block_height, NULL, NULL);
740 memcpy(winner_mode_stats[mode_idx].color_index_map, color_map,
741 block_width * block_height * sizeof(color_map[0]));
742 }
743
744 x->winner_mode_count =
745 AOMMIN(x->winner_mode_count + 1, max_winner_mode_count);
746 }
747
748 unsigned int av1_get_perpixel_variance(const AV1_COMP *cpi,
749 const MACROBLOCKD *xd,
750 const struct buf_2d *ref,
751 BLOCK_SIZE bsize, int plane,
752 int use_hbd);
753
754 unsigned int av1_get_perpixel_variance_facade(const struct AV1_COMP *cpi,
755 const MACROBLOCKD *xd,
756 const struct buf_2d *ref,
757 BLOCK_SIZE bsize, int plane);
758
is_mode_intra(PREDICTION_MODE mode)759 static INLINE int is_mode_intra(PREDICTION_MODE mode) {
760 return mode < INTRA_MODE_END;
761 }
762
763 // This function will copy usable ref_mv_stack[ref_frame][4] and
764 // weight[ref_frame][4] information from ref_mv_stack[ref_frame][8] and
765 // weight[ref_frame][8].
av1_copy_usable_ref_mv_stack_and_weight(const MACROBLOCKD * xd,MB_MODE_INFO_EXT * const mbmi_ext,MV_REFERENCE_FRAME ref_frame)766 static INLINE void av1_copy_usable_ref_mv_stack_and_weight(
767 const MACROBLOCKD *xd, MB_MODE_INFO_EXT *const mbmi_ext,
768 MV_REFERENCE_FRAME ref_frame) {
769 memcpy(mbmi_ext->weight[ref_frame], xd->weight[ref_frame],
770 USABLE_REF_MV_STACK_SIZE * sizeof(xd->weight[0][0]));
771 memcpy(mbmi_ext->ref_mv_stack[ref_frame], xd->ref_mv_stack[ref_frame],
772 USABLE_REF_MV_STACK_SIZE * sizeof(xd->ref_mv_stack[0][0]));
773 }
774
775 // Get transform rd gate level for the given transform search case.
get_txfm_rd_gate_level(const int is_masked_compound_enabled,const int txfm_rd_gate_level[TX_SEARCH_CASES],BLOCK_SIZE bsize,TX_SEARCH_CASE tx_search_case,int eval_motion_mode)776 static INLINE int get_txfm_rd_gate_level(
777 const int is_masked_compound_enabled,
778 const int txfm_rd_gate_level[TX_SEARCH_CASES], BLOCK_SIZE bsize,
779 TX_SEARCH_CASE tx_search_case, int eval_motion_mode) {
780 assert(tx_search_case < TX_SEARCH_CASES);
781 if (tx_search_case == TX_SEARCH_MOTION_MODE && !eval_motion_mode &&
782 num_pels_log2_lookup[bsize] > 8)
783 return txfm_rd_gate_level[TX_SEARCH_MOTION_MODE];
784 // Enable aggressive gating of transform search only when masked compound type
785 // is enabled.
786 else if (tx_search_case == TX_SEARCH_COMP_TYPE_MODE &&
787 is_masked_compound_enabled)
788 return txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE];
789
790 return txfm_rd_gate_level[TX_SEARCH_DEFAULT];
791 }
792
793 #ifdef __cplusplus
794 } // extern "C"
795 #endif
796
797 #endif // AOM_AV1_ENCODER_RDOPT_UTILS_H_
798