1 /*
2 * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AV1_ENCODER_RDOPT_UTILS_H_
13 #define AOM_AV1_ENCODER_RDOPT_UTILS_H_
14
15 #include "aom/aom_integer.h"
16 #include "av1/encoder/block.h"
17 #include "av1/common/cfl.h"
18 #include "av1/common/pred_common.h"
19 #include "av1/encoder/rdopt_data_defs.h"
20
21 #ifdef __cplusplus
22 extern "C" {
23 #endif
24
25 #define MAX_REF_MV_SEARCH 3
26 #define INTER_INTRA_RD_THRESH_SCALE 9
27 #define INTER_INTRA_RD_THRESH_SHIFT 4
28
29 typedef struct {
30 PREDICTION_MODE mode;
31 MV_REFERENCE_FRAME ref_frame[2];
32 } MODE_DEFINITION;
33
34 // This array defines the mapping from the enums in THR_MODES to the actual
35 // prediction modes and refrence frames
36 static const MODE_DEFINITION av1_mode_defs[MAX_MODES] = {
37 { NEARESTMV, { LAST_FRAME, NONE_FRAME } },
38 { NEARESTMV, { LAST2_FRAME, NONE_FRAME } },
39 { NEARESTMV, { LAST3_FRAME, NONE_FRAME } },
40 { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } },
41 { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } },
42 { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } },
43 { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } },
44
45 { NEWMV, { LAST_FRAME, NONE_FRAME } },
46 { NEWMV, { LAST2_FRAME, NONE_FRAME } },
47 { NEWMV, { LAST3_FRAME, NONE_FRAME } },
48 { NEWMV, { BWDREF_FRAME, NONE_FRAME } },
49 { NEWMV, { ALTREF2_FRAME, NONE_FRAME } },
50 { NEWMV, { ALTREF_FRAME, NONE_FRAME } },
51 { NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
52
53 { NEARMV, { LAST_FRAME, NONE_FRAME } },
54 { NEARMV, { LAST2_FRAME, NONE_FRAME } },
55 { NEARMV, { LAST3_FRAME, NONE_FRAME } },
56 { NEARMV, { BWDREF_FRAME, NONE_FRAME } },
57 { NEARMV, { ALTREF2_FRAME, NONE_FRAME } },
58 { NEARMV, { ALTREF_FRAME, NONE_FRAME } },
59 { NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
60
61 { GLOBALMV, { LAST_FRAME, NONE_FRAME } },
62 { GLOBALMV, { LAST2_FRAME, NONE_FRAME } },
63 { GLOBALMV, { LAST3_FRAME, NONE_FRAME } },
64 { GLOBALMV, { BWDREF_FRAME, NONE_FRAME } },
65 { GLOBALMV, { ALTREF2_FRAME, NONE_FRAME } },
66 { GLOBALMV, { ALTREF_FRAME, NONE_FRAME } },
67 { GLOBALMV, { GOLDEN_FRAME, NONE_FRAME } },
68
69 // TODO(zoeliu): May need to reconsider the order on the modes to check
70
71 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
72 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
73 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
74 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
75 { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
76 { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
77 { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
78 { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
79 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
80 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
81 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
82 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
83
84 { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
85 { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
86 { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
87 { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
88
89 { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
90 { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
91 { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
92 { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
93 { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
94 { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
95 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF_FRAME } },
96
97 { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
98 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
99 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
100 { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
101 { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
102 { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
103 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF_FRAME } },
104
105 { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
106 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
107 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
108 { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
109 { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
110 { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
111 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF_FRAME } },
112
113 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
114 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
115 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
116 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
117 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
118 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
119 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF_FRAME } },
120
121 { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
122 { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
123 { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
124 { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
125 { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
126 { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
127 { GLOBAL_GLOBALMV, { LAST_FRAME, BWDREF_FRAME } },
128
129 { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
130 { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
131 { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
132 { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
133 { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
134 { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
135 { GLOBAL_GLOBALMV, { LAST2_FRAME, BWDREF_FRAME } },
136
137 { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
138 { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
139 { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
140 { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
141 { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
142 { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
143 { GLOBAL_GLOBALMV, { LAST3_FRAME, BWDREF_FRAME } },
144
145 { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
146 { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
147 { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
148 { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
149 { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
150 { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
151 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, BWDREF_FRAME } },
152
153 { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
154 { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
155 { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
156 { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
157 { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
158 { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
159 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF2_FRAME } },
160
161 { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
162 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
163 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
164 { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
165 { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
166 { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
167 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF2_FRAME } },
168
169 { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
170 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
171 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
172 { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
173 { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
174 { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
175 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF2_FRAME } },
176
177 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
178 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
179 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
180 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
181 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
182 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
183 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
184
185 { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } },
186 { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
187 { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } },
188 { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } },
189 { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } },
190 { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } },
191 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST2_FRAME } },
192
193 { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } },
194 { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
195 { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } },
196 { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } },
197 { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } },
198 { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } },
199 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST3_FRAME } },
200
201 { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
202 { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
203 { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
204 { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
205 { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
206 { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
207 { GLOBAL_GLOBALMV, { LAST_FRAME, GOLDEN_FRAME } },
208
209 { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
210 { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
211 { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
212 { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
213 { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
214 { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
215 { GLOBAL_GLOBALMV, { BWDREF_FRAME, ALTREF_FRAME } },
216
217 // intra modes
218 { DC_PRED, { INTRA_FRAME, NONE_FRAME } },
219 { PAETH_PRED, { INTRA_FRAME, NONE_FRAME } },
220 { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } },
221 { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } },
222 { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } },
223 { H_PRED, { INTRA_FRAME, NONE_FRAME } },
224 { V_PRED, { INTRA_FRAME, NONE_FRAME } },
225 { D135_PRED, { INTRA_FRAME, NONE_FRAME } },
226 { D203_PRED, { INTRA_FRAME, NONE_FRAME } },
227 { D157_PRED, { INTRA_FRAME, NONE_FRAME } },
228 { D67_PRED, { INTRA_FRAME, NONE_FRAME } },
229 { D113_PRED, { INTRA_FRAME, NONE_FRAME } },
230 { D45_PRED, { INTRA_FRAME, NONE_FRAME } },
231 };
232
restore_dst_buf(MACROBLOCKD * xd,const BUFFER_SET dst,const int num_planes)233 static AOM_INLINE void restore_dst_buf(MACROBLOCKD *xd, const BUFFER_SET dst,
234 const int num_planes) {
235 for (int i = 0; i < num_planes; i++) {
236 xd->plane[i].dst.buf = dst.plane[i];
237 xd->plane[i].dst.stride = dst.stride[i];
238 }
239 }
240
241 /* clang-format on */
242 // Calculate rd threshold based on ref best rd and relevant scaling factors
get_rd_thresh_from_best_rd(int64_t ref_best_rd,int mul_factor,int div_factor)243 static AOM_INLINE int64_t get_rd_thresh_from_best_rd(int64_t ref_best_rd,
244 int mul_factor,
245 int div_factor) {
246 int64_t rd_thresh = ref_best_rd;
247 if (div_factor != 0) {
248 rd_thresh = ref_best_rd < (div_factor * (INT64_MAX / mul_factor))
249 ? ((ref_best_rd / div_factor) * mul_factor)
250 : INT64_MAX;
251 }
252 return rd_thresh;
253 }
254
255 static AOM_INLINE THR_MODES
get_prediction_mode_idx(PREDICTION_MODE this_mode,MV_REFERENCE_FRAME ref_frame,MV_REFERENCE_FRAME second_ref_frame)256 get_prediction_mode_idx(PREDICTION_MODE this_mode, MV_REFERENCE_FRAME ref_frame,
257 MV_REFERENCE_FRAME second_ref_frame) {
258 if (this_mode < INTRA_MODE_END) {
259 assert(ref_frame == INTRA_FRAME);
260 assert(second_ref_frame == NONE_FRAME);
261 return intra_to_mode_idx[this_mode - INTRA_MODE_START];
262 }
263 if (this_mode >= SINGLE_INTER_MODE_START &&
264 this_mode < SINGLE_INTER_MODE_END) {
265 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME));
266 return single_inter_to_mode_idx[this_mode - SINGLE_INTER_MODE_START]
267 [ref_frame];
268 }
269 if (this_mode >= COMP_INTER_MODE_START && this_mode < COMP_INTER_MODE_END) {
270 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME));
271 assert((second_ref_frame > INTRA_FRAME) &&
272 (second_ref_frame <= ALTREF_FRAME));
273 return comp_inter_to_mode_idx[this_mode - COMP_INTER_MODE_START][ref_frame]
274 [second_ref_frame];
275 }
276 assert(0);
277 return THR_INVALID;
278 }
279
inter_mode_data_block_idx(BLOCK_SIZE bsize)280 static AOM_INLINE int inter_mode_data_block_idx(BLOCK_SIZE bsize) {
281 if (bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 ||
282 bsize == BLOCK_4X16 || bsize == BLOCK_16X4) {
283 return -1;
284 }
285 return 1;
286 }
287
288 // Get transform block visible dimensions cropped to the MI units.
get_txb_dimensions(const MACROBLOCKD * xd,int plane,BLOCK_SIZE plane_bsize,int blk_row,int blk_col,BLOCK_SIZE tx_bsize,int * width,int * height,int * visible_width,int * visible_height)289 static AOM_INLINE void get_txb_dimensions(const MACROBLOCKD *xd, int plane,
290 BLOCK_SIZE plane_bsize, int blk_row,
291 int blk_col, BLOCK_SIZE tx_bsize,
292 int *width, int *height,
293 int *visible_width,
294 int *visible_height) {
295 assert(tx_bsize <= plane_bsize);
296 const int txb_height = block_size_high[tx_bsize];
297 const int txb_width = block_size_wide[tx_bsize];
298 const struct macroblockd_plane *const pd = &xd->plane[plane];
299
300 // TODO(aconverse@google.com): Investigate using crop_width/height here rather
301 // than the MI size
302 if (xd->mb_to_bottom_edge >= 0) {
303 *visible_height = txb_height;
304 } else {
305 const int block_height = block_size_high[plane_bsize];
306 const int block_rows =
307 (xd->mb_to_bottom_edge >> (3 + pd->subsampling_y)) + block_height;
308 *visible_height =
309 clamp(block_rows - (blk_row << MI_SIZE_LOG2), 0, txb_height);
310 }
311 if (height) *height = txb_height;
312
313 if (xd->mb_to_right_edge >= 0) {
314 *visible_width = txb_width;
315 } else {
316 const int block_width = block_size_wide[plane_bsize];
317 const int block_cols =
318 (xd->mb_to_right_edge >> (3 + pd->subsampling_x)) + block_width;
319 *visible_width =
320 clamp(block_cols - (blk_col << MI_SIZE_LOG2), 0, txb_width);
321 }
322 if (width) *width = txb_width;
323 }
324
bsize_to_num_blk(BLOCK_SIZE bsize)325 static AOM_INLINE int bsize_to_num_blk(BLOCK_SIZE bsize) {
326 int num_blk = 1 << (num_pels_log2_lookup[bsize] - 2 * MI_SIZE_LOG2);
327 return num_blk;
328 }
329
check_txfm_eval(MACROBLOCK * const x,BLOCK_SIZE bsize,int64_t best_skip_rd,int64_t skip_rd,int level,int is_luma_only)330 static INLINE int check_txfm_eval(MACROBLOCK *const x, BLOCK_SIZE bsize,
331 int64_t best_skip_rd, int64_t skip_rd,
332 int level, int is_luma_only) {
333 int eval_txfm = 1;
334 // Derive aggressiveness factor for gating the transform search
335 // Lower value indicates more aggressiveness. Be more conservative (high
336 // value) for (i) low quantizers (ii) regions where prediction is poor
337 const int scale[5] = { INT_MAX, 4, 3, 3, 2 };
338 const int qslope = 2 * (!is_luma_only);
339 int aggr_factor = 1;
340 if (!is_luma_only) {
341 aggr_factor = AOMMAX(
342 1, ((MAXQ - x->qindex) * qslope + QINDEX_RANGE / 2) >> QINDEX_BITS);
343 }
344 if (best_skip_rd >
345 (x->source_variance << (num_pels_log2_lookup[bsize] + RDDIV_BITS)))
346 aggr_factor *= scale[level];
347 // For level setting 1, be more conservative for luma only case even when
348 // prediction is good
349 else if ((level <= 1) && !is_luma_only)
350 aggr_factor *= 2;
351
352 // Be more conservative for luma only cases (called from compound type rd)
353 // since best_skip_rd is computed after and skip_rd is computed (with 8-bit
354 // prediction signals blended for WEDGE/DIFFWTD rather than 16-bit) before
355 // interpolation filter search
356 const int luma_mul[5] = { INT_MAX, 32, 29, 20, 17 };
357 int mul_factor = is_luma_only ? luma_mul[level] : 16;
358 int64_t rd_thresh =
359 (best_skip_rd == INT64_MAX)
360 ? best_skip_rd
361 : (int64_t)(best_skip_rd * aggr_factor * mul_factor >> 4);
362 if (skip_rd > rd_thresh) eval_txfm = 0;
363 return eval_txfm;
364 }
365
select_tx_mode(const AV1_COMMON * cm,const TX_SIZE_SEARCH_METHOD tx_size_search_method)366 static TX_MODE select_tx_mode(
367 const AV1_COMMON *cm, const TX_SIZE_SEARCH_METHOD tx_size_search_method) {
368 if (cm->features.coded_lossless) return ONLY_4X4;
369 if (tx_size_search_method == USE_LARGESTALL) {
370 return TX_MODE_LARGEST;
371 } else {
372 assert(tx_size_search_method == USE_FULL_RD ||
373 tx_size_search_method == USE_FAST_RD);
374 return TX_MODE_SELECT;
375 }
376 }
377 // Checks the conditions to enable winner mode processing
is_winner_mode_processing_enabled(const struct AV1_COMP * cpi,MB_MODE_INFO * const mbmi,const PREDICTION_MODE best_mode)378 static INLINE int is_winner_mode_processing_enabled(
379 const struct AV1_COMP *cpi, MB_MODE_INFO *const mbmi,
380 const PREDICTION_MODE best_mode) {
381 const SPEED_FEATURES *sf = &cpi->sf;
382
383 // TODO(any): Move block independent condition checks to frame level
384 if (is_inter_block(mbmi)) {
385 if (is_inter_mode(best_mode) &&
386 sf->tx_sf.tx_type_search.fast_inter_tx_type_search &&
387 !cpi->oxcf.use_inter_dct_only)
388 return 1;
389 } else {
390 if (sf->tx_sf.tx_type_search.fast_intra_tx_type_search &&
391 !cpi->oxcf.use_intra_default_tx_only && !cpi->oxcf.use_intra_dct_only)
392 return 1;
393 }
394
395 // Check speed feature related to winner mode processing
396 if (sf->winner_mode_sf.enable_winner_mode_for_coeff_opt &&
397 cpi->optimize_seg_arr[mbmi->segment_id] != NO_TRELLIS_OPT &&
398 cpi->optimize_seg_arr[mbmi->segment_id] != FINAL_PASS_TRELLIS_OPT)
399 return 1;
400 if (sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch) return 1;
401
402 return 0;
403 }
404
set_tx_size_search_method(const AV1_COMMON * cm,const WinnerModeParams * winner_mode_params,MACROBLOCK * x,int enable_winner_mode_for_tx_size_srch,int is_winner_mode)405 static INLINE void set_tx_size_search_method(
406 const AV1_COMMON *cm, const WinnerModeParams *winner_mode_params,
407 MACROBLOCK *x, int enable_winner_mode_for_tx_size_srch,
408 int is_winner_mode) {
409 // Populate transform size search method/transform mode appropriately
410 x->tx_size_search_method =
411 winner_mode_params->tx_size_search_methods[DEFAULT_EVAL];
412 if (enable_winner_mode_for_tx_size_srch) {
413 if (is_winner_mode)
414 x->tx_size_search_method =
415 winner_mode_params->tx_size_search_methods[WINNER_MODE_EVAL];
416 else
417 x->tx_size_search_method =
418 winner_mode_params->tx_size_search_methods[MODE_EVAL];
419 }
420 x->tx_mode_search_type = select_tx_mode(cm, x->tx_size_search_method);
421 }
422
set_tx_type_prune(const SPEED_FEATURES * sf,MACROBLOCK * x,int enable_winner_mode_tx_type_pruning,int is_winner_mode)423 static INLINE void set_tx_type_prune(const SPEED_FEATURES *sf, MACROBLOCK *x,
424 int enable_winner_mode_tx_type_pruning,
425 int is_winner_mode) {
426 // Populate prune transform mode appropriately
427 x->prune_mode = sf->tx_sf.tx_type_search.prune_mode;
428 if (enable_winner_mode_tx_type_pruning) {
429 if (is_winner_mode)
430 x->prune_mode = NO_PRUNE;
431 else
432 x->prune_mode = PRUNE_2D_AGGRESSIVE;
433 }
434 }
435
set_tx_domain_dist_params(const WinnerModeParams * winner_mode_params,MACROBLOCK * x,int enable_winner_mode_for_tx_domain_dist,int is_winner_mode)436 static INLINE void set_tx_domain_dist_params(
437 const WinnerModeParams *winner_mode_params, MACROBLOCK *x,
438 int enable_winner_mode_for_tx_domain_dist, int is_winner_mode) {
439 if (!enable_winner_mode_for_tx_domain_dist) {
440 x->use_transform_domain_distortion =
441 winner_mode_params->use_transform_domain_distortion[DEFAULT_EVAL];
442 x->tx_domain_dist_threshold =
443 winner_mode_params->tx_domain_dist_threshold[DEFAULT_EVAL];
444 return;
445 }
446
447 if (is_winner_mode) {
448 x->use_transform_domain_distortion =
449 winner_mode_params->use_transform_domain_distortion[WINNER_MODE_EVAL];
450 x->tx_domain_dist_threshold =
451 winner_mode_params->tx_domain_dist_threshold[WINNER_MODE_EVAL];
452 } else {
453 x->use_transform_domain_distortion =
454 winner_mode_params->use_transform_domain_distortion[MODE_EVAL];
455 x->tx_domain_dist_threshold =
456 winner_mode_params->tx_domain_dist_threshold[MODE_EVAL];
457 }
458 }
459
460 // This function sets mode parameters for different mode evaluation stages
set_mode_eval_params(const struct AV1_COMP * cpi,MACROBLOCK * x,MODE_EVAL_TYPE mode_eval_type)461 static INLINE void set_mode_eval_params(const struct AV1_COMP *cpi,
462 MACROBLOCK *x,
463 MODE_EVAL_TYPE mode_eval_type) {
464 const AV1_COMMON *cm = &cpi->common;
465 const SPEED_FEATURES *sf = &cpi->sf;
466 const WinnerModeParams *winner_mode_params = &cpi->winner_mode_params;
467
468 switch (mode_eval_type) {
469 case DEFAULT_EVAL:
470 x->use_default_inter_tx_type = 0;
471 x->use_default_intra_tx_type = 0;
472 x->predict_skip_level =
473 winner_mode_params->predict_skip_level[DEFAULT_EVAL];
474 // Set default transform domain distortion type
475 set_tx_domain_dist_params(winner_mode_params, x, 0, 0);
476
477 // Get default threshold for R-D optimization of coefficients
478 x->coeff_opt_dist_threshold = get_rd_opt_coeff_thresh(
479 winner_mode_params->coeff_opt_dist_threshold, 0, 0);
480 // Set default transform size search method
481 set_tx_size_search_method(cm, winner_mode_params, x, 0, 0);
482 // Set default transform type prune
483 set_tx_type_prune(sf, x, 0, 0);
484 break;
485 case MODE_EVAL:
486 x->use_default_intra_tx_type =
487 (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search ||
488 cpi->oxcf.use_intra_default_tx_only);
489 x->use_default_inter_tx_type =
490 cpi->sf.tx_sf.tx_type_search.fast_inter_tx_type_search;
491 x->predict_skip_level = winner_mode_params->predict_skip_level[MODE_EVAL];
492
493 // Set transform domain distortion type for mode evaluation
494 set_tx_domain_dist_params(
495 winner_mode_params, x,
496 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 0);
497
498 // Get threshold for R-D optimization of coefficients during mode
499 // evaluation
500 x->coeff_opt_dist_threshold = get_rd_opt_coeff_thresh(
501 winner_mode_params->coeff_opt_dist_threshold,
502 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 0);
503 // Set the transform size search method for mode evaluation
504 set_tx_size_search_method(
505 cm, winner_mode_params, x,
506 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 0);
507 // Set transform type prune for mode evaluation
508 set_tx_type_prune(
509 sf, x, sf->tx_sf.tx_type_search.enable_winner_mode_tx_type_pruning,
510 0);
511 break;
512 case WINNER_MODE_EVAL:
513 x->use_default_inter_tx_type = 0;
514 x->use_default_intra_tx_type = 0;
515 x->predict_skip_level =
516 winner_mode_params->predict_skip_level[WINNER_MODE_EVAL];
517
518 // Set transform domain distortion type for winner mode evaluation
519 set_tx_domain_dist_params(
520 winner_mode_params, x,
521 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 1);
522
523 // Get threshold for R-D optimization of coefficients for winner mode
524 // evaluation
525 x->coeff_opt_dist_threshold = get_rd_opt_coeff_thresh(
526 winner_mode_params->coeff_opt_dist_threshold,
527 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 1);
528 // Set the transform size search method for winner mode evaluation
529 set_tx_size_search_method(
530 cm, winner_mode_params, x,
531 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 1);
532 // Set default transform type prune mode for winner mode evaluation
533 set_tx_type_prune(
534 sf, x, sf->tx_sf.tx_type_search.enable_winner_mode_tx_type_pruning,
535 1);
536
537 // Reset hash state for winner mode processing. Winner mode and subsequent
538 // transform/mode evaluations (palette/IntraBC) cann't reuse old data as
539 // the decisions would have been sub-optimal
540 // TODO(any): Move the evaluation of palette/IntraBC modes before winner
541 // mode is processed and clean-up the code below
542 reset_hash_records(x, cpi->sf.tx_sf.use_inter_txb_hash);
543
544 break;
545 default: assert(0);
546 }
547 }
548
549 // Similar to store_cfl_required(), but for use during the RDO process,
550 // where we haven't yet determined whether this block uses CfL.
store_cfl_required_rdo(const AV1_COMMON * cm,const MACROBLOCK * x)551 static INLINE CFL_ALLOWED_TYPE store_cfl_required_rdo(const AV1_COMMON *cm,
552 const MACROBLOCK *x) {
553 const MACROBLOCKD *xd = &x->e_mbd;
554
555 if (cm->seq_params.monochrome || !xd->is_chroma_ref) return CFL_DISALLOWED;
556
557 if (!xd->is_chroma_ref) {
558 // For non-chroma-reference blocks, we should always store the luma pixels,
559 // in case the corresponding chroma-reference block uses CfL.
560 // Note that this can only happen for block sizes which are <8 on
561 // their shortest side, as otherwise they would be chroma reference
562 // blocks.
563 return CFL_ALLOWED;
564 }
565
566 // For chroma reference blocks, we should store data in the encoder iff we're
567 // allowed to try out CfL.
568 return is_cfl_allowed(xd);
569 }
570
init_sbuv_mode(MB_MODE_INFO * const mbmi)571 static AOM_INLINE void init_sbuv_mode(MB_MODE_INFO *const mbmi) {
572 mbmi->uv_mode = UV_DC_PRED;
573 mbmi->palette_mode_info.palette_size[1] = 0;
574 }
575
576 // Store best mode stats for winner mode processing
store_winner_mode_stats(const AV1_COMMON * const cm,MACROBLOCK * x,MB_MODE_INFO * mbmi,RD_STATS * rd_cost,RD_STATS * rd_cost_y,RD_STATS * rd_cost_uv,THR_MODES mode_index,uint8_t * color_map,BLOCK_SIZE bsize,int64_t this_rd,int enable_multiwinner_mode_process,int txfm_search_done)577 static INLINE void store_winner_mode_stats(
578 const AV1_COMMON *const cm, MACROBLOCK *x, MB_MODE_INFO *mbmi,
579 RD_STATS *rd_cost, RD_STATS *rd_cost_y, RD_STATS *rd_cost_uv,
580 THR_MODES mode_index, uint8_t *color_map, BLOCK_SIZE bsize, int64_t this_rd,
581 int enable_multiwinner_mode_process, int txfm_search_done) {
582 WinnerModeStats *winner_mode_stats = x->winner_mode_stats;
583 int mode_idx = 0;
584 int is_palette_mode = mbmi->palette_mode_info.palette_size[PLANE_TYPE_Y] > 0;
585 // Mode stat is not required when multiwinner mode processing is disabled
586 if (!enable_multiwinner_mode_process) return;
587 // Ignore mode with maximum rd
588 if (this_rd == INT64_MAX) return;
589 // TODO(any): Winner mode processing is currently not applicable for palette
590 // mode in Inter frames. Clean-up the following code, once support is added
591 if (!frame_is_intra_only(cm) && is_palette_mode) return;
592
593 const int max_winner_mode_count = frame_is_intra_only(cm)
594 ? MAX_WINNER_MODE_COUNT_INTRA
595 : MAX_WINNER_MODE_COUNT_INTER;
596 assert(x->winner_mode_count >= 0 &&
597 x->winner_mode_count <= max_winner_mode_count);
598
599 if (x->winner_mode_count) {
600 // Find the mode which has higher rd cost than this_rd
601 for (mode_idx = 0; mode_idx < x->winner_mode_count; mode_idx++)
602 if (winner_mode_stats[mode_idx].rd > this_rd) break;
603
604 if (mode_idx == max_winner_mode_count) {
605 // No mode has higher rd cost than this_rd
606 return;
607 } else if (mode_idx < max_winner_mode_count - 1) {
608 // Create a slot for current mode and move others to the next slot
609 memmove(
610 &winner_mode_stats[mode_idx + 1], &winner_mode_stats[mode_idx],
611 (max_winner_mode_count - mode_idx - 1) * sizeof(*winner_mode_stats));
612 }
613 }
614 // Add a mode stat for winner mode processing
615 winner_mode_stats[mode_idx].mbmi = *mbmi;
616 winner_mode_stats[mode_idx].rd = this_rd;
617 winner_mode_stats[mode_idx].mode_index = mode_index;
618
619 // Update rd stats required for inter frame
620 if (!frame_is_intra_only(cm) && rd_cost && rd_cost_y && rd_cost_uv) {
621 const MACROBLOCKD *xd = &x->e_mbd;
622 const int skip_ctx = av1_get_skip_context(xd);
623 const int is_intra_mode = av1_mode_defs[mode_index].mode < INTRA_MODE_END;
624 const int skip = mbmi->skip && !is_intra_mode;
625
626 winner_mode_stats[mode_idx].rd_cost = *rd_cost;
627 if (txfm_search_done) {
628 winner_mode_stats[mode_idx].rate_y =
629 rd_cost_y->rate + x->skip_cost[skip_ctx][rd_cost->skip || skip];
630 winner_mode_stats[mode_idx].rate_uv = rd_cost_uv->rate;
631 }
632 }
633
634 if (color_map) {
635 // Store color_index_map for palette mode
636 const MACROBLOCKD *const xd = &x->e_mbd;
637 int block_width, block_height;
638 av1_get_block_dimensions(bsize, AOM_PLANE_Y, xd, &block_width,
639 &block_height, NULL, NULL);
640 memcpy(winner_mode_stats[mode_idx].color_index_map, color_map,
641 block_width * block_height * sizeof(color_map[0]));
642 }
643
644 x->winner_mode_count =
645 AOMMIN(x->winner_mode_count + 1, max_winner_mode_count);
646 }
647
648 #ifdef __cplusplus
649 } // extern "C"
650 #endif
651
652 #endif // AOM_AV1_ENCODER_RDOPT_UTILS_H_
653