1 /*
2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
13 #define AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
14
15 #include "av1/common/pred_common.h"
16 #include "av1/encoder/encoder.h"
17
18 #ifdef __cplusplus
19 extern "C" {
20 #endif
21
22 // Set this macro as 1 to collect data about tx size selection.
23 #define COLLECT_TX_SIZE_DATA 0
24
25 #if COLLECT_TX_SIZE_DATA
26 static const char av1_tx_size_data_output_file[] = "tx_size_data.txt";
27 #endif
28
29 enum {
30 FTXS_NONE = 0,
31 FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0,
32 FTXS_DISABLE_TRELLIS_OPT = 1 << 1,
33 FTXS_USE_TRANSFORM_DOMAIN = 1 << 2
34 } UENUM1BYTE(FAST_TX_SEARCH_MODE);
35
tx_size_cost(const MACROBLOCK * const x,BLOCK_SIZE bsize,TX_SIZE tx_size)36 static AOM_INLINE int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize,
37 TX_SIZE tx_size) {
38 assert(bsize == x->e_mbd.mi[0]->bsize);
39 if (x->txfm_search_params.tx_mode_search_type != TX_MODE_SELECT ||
40 !block_signals_txsize(bsize))
41 return 0;
42
43 const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize);
44 const int depth = tx_size_to_depth(tx_size, bsize);
45 const MACROBLOCKD *const xd = &x->e_mbd;
46 const int tx_size_ctx = get_tx_size_context(xd);
47 return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth];
48 }
49
50 /*!\brief Compute the pixel domain distortion.
51 *
52 * \ingroup transform_search
53 * Compute the pixel domain distortion from diff on all visible 4x4s in the
54 * transform block.
55 *
56 * \param[in] x Pointer to structure holding the data for the
57 current encoding macroblock
58 * \param[in] plane Plane index
59 * \param[in] blk_row Block row index
60 * \param[in] blk_col Block col index
61 * \param[in] plane_bsize Current plane block size
62 * \param[in] tx_bsize Transform size
63 * \param[in] block_mse_q8 Block mse
64 * \return An int64_t value that is the block sse.
65 */
66 int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row,
67 int blk_col, const BLOCK_SIZE plane_bsize,
68 const BLOCK_SIZE tx_bsize,
69 unsigned int *block_mse_q8);
70
71 int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
72 RD_STATS *rd_stats, int64_t ref_best_rd,
73 BLOCK_SIZE bs, TX_SIZE tx_size);
74
75 /*!\brief Transform type search for luma macroblock with fixed transform size.
76 *
77 * \ingroup transform_search
78 * Search for the best transform type and return the transform coefficients RD
79 * cost of current luma macroblock with the given uniform transform size.
80 *
81 * \param[in] x Pointer to structure holding the data for the
82 current encoding macroblock
83 * \param[in] cpi Top-level encoder structure
84 * \param[in] rd_stats Pointer to struct to keep track of the RD stats
85 * \param[in] ref_best_rd Best RD cost seen for this block so far
86 * \param[in] bs Size of the current macroblock
87 * \param[in] tx_size The given transform size
88 * \param[in] ftxs_mode Transform search mode specifying desired speed
89 and quality tradeoff
90 * \param[in] skip_trellis Binary flag indicating if trellis optimization
91 should be skipped
92 * \return An int64_t value that is the best RD cost found.
93 */
94 int64_t av1_uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
95 RD_STATS *rd_stats, int64_t ref_best_rd,
96 BLOCK_SIZE bs, TX_SIZE tx_size,
97 FAST_TX_SEARCH_MODE ftxs_mode, int skip_trellis);
98
99 /*!\brief Recursive transform size and type search.
100 *
101 * \ingroup transform_search
102 * Search for best transform size and type for luma inter blocks. The transform
103 * block partitioning can be recursive resulting in non-uniform transform sizes.
104 * The best transform size and type, if found, will be saved in the MB_MODE_INFO
105 * structure, and the corresponding RD stats will be saved in rd_stats.
106 *
107 * \param[in] cpi Top-level encoder structure
108 * \param[in] x Pointer to structure holding the data for the
109 current encoding macroblock
110 * \param[in] rd_stats Pointer to struct to keep track of the RD stats
111 * \param[in] bsize Current macroblock size
112 * \param[in] ref_best_rd Best RD cost seen for this block so far
113 * \remark Nothing is returned. The selected transform size and type will
114 be saved in the MB_MODE_INFO structure
115 */
116 void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
117 RD_STATS *rd_stats, BLOCK_SIZE bsize,
118 int64_t ref_best_rd);
119
120 /*!\brief Uniform transform size and type search.
121 *
122 * \ingroup transform_search
123 * Search for the best transform size and type for current macroblock block,
124 * with the assumption that all the transform blocks have a uniform size
125 * (VP9 style). The selected transform size and type will be saved in the
126 * MB_MODE_INFO structure; the corresponding RD stats will be saved in rd_stats.
127 * This function may be used for both intra and inter predicted blocks.
128 *
129 * \param[in] cpi Top-level encoder structure
130 * \param[in] x Pointer to structure holding the data for the
131 current encoding macroblock
132 * \param[in] rd_stats Pointer to struct to keep track of the RD stats
133 * \param[in] bs Current macroblock size
134 * \param[in] ref_best_rd Best RD cost seen for this block so far
135 * \remark Nothing is returned. The selected transform size and type will
136 be saved in the MB_MODE_INFO structure
137 */
138 void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
139 RD_STATS *rd_stats, BLOCK_SIZE bs,
140 int64_t ref_best_rd);
141
142 /*!\brief Chroma block transform search.
143 *
144 * \ingroup transform_search
145 * Calculate the transform coefficient RD cost for the given chroma macroblock
146 * If the current mode is intra, then this function will compute the predictor.
147 *
148 * \param[in] cpi Top-level encoder structure
149 * \param[in] x Pointer to structure holding the data for the
150 current encoding macroblock
151 * \param[in] rd_stats Pointer to struct to keep track of the RD stats
152 * \param[in] bsize Current macroblock size
153 * \param[in] ref_best_rd Best RD cost seen for this block so far
154 * \return An integer value is returned. 0: early termination triggered,
155 no valid rd cost available; 1: rd cost values are valid.
156 */
157 int av1_txfm_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats,
158 BLOCK_SIZE bsize, int64_t ref_best_rd);
159
160 /*!\brief Transform type search with fixed transform size.
161 *
162 * \ingroup transform_search
163 * Search for the best transform type and calculate the transform coefficients
164 * RD cost of the current transform block with the specified (uniform) transform
165 * size and plane. The RD results will be saved in rd_stats.
166 *
167 * \param[in] x Pointer to structure holding the data for the
168 current encoding macroblock
169 * \param[in] cpi Top-level encoder structure
170 * \param[in] rd_stats Pointer to struct to keep track of the RD stats
171 * \param[in] ref_best_rd Best RD cost seen for this block so far
172 * \param[in] current_rd Current RD cost for this block so far
173 * \param[in] plane Plane index
174 * \param[in] plane_bsize Size of the current macroblock considering
175 sup-sampling
176 * \param[in] tx_size The given transform size
177 * \param[in] ftxs_mode Transform search mode specifying desired speed
178 and quality tradeoff
179 * \param[in] skip_trellis Binary flag indicating if trellis optimization
180 should be skipped
181 *
182 * \remark Nothing is returned. The RD results will be saved in rd_stats.
183 */
184 void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi,
185 RD_STATS *rd_stats, int64_t ref_best_rd,
186 int64_t current_rd, int plane, BLOCK_SIZE plane_bsize,
187 TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode,
188 int skip_trellis);
189
190 /*!\brief Recursive transform size and type search.
191 *
192 * \ingroup transform_search
193 * This function combines y and uv planes' transform search processes together
194 * for inter-predicted blocks (including IntraBC), when the prediction is
195 * already generated. It first does subtraction to obtain the prediction error.
196 * Then it calls
197 * av1_pick_recursive_tx_size_type_yrd/av1_pick_uniform_tx_size_type_yrd and
198 * av1_txfm_uvrd sequentially and handles possible early terminations.
199 * The RD metrics are calculated and stored in rd_stats/_y/_uv.
200 *
201 * \param[in] cpi Top-level encoder structure
202 * \param[in] x Pointer to structure holding the data for the
203 current encoding macroblock
204 * \param[in] bsize Current macroblock size
205 * \param[in] rd_stats Pointer to struct to keep track of the overal RD
206 stats
207 * \param[in] rd_stats_y Pointer to struct to keep track of the RD
208 stats for the luma plane
209 * \param[in] rd_stats_uv Pointer to struct to keep track of the RD
210 stats for the chroma planes
211 * \param[in] mode_rate Rate cost to encode the prediction mode info. of
212 the current macroblock
213 * \param[in] ref_best_rd Best RD cost seen for this block so far
214 *
215 * \return An integer value is returned indicating if a valid transform
216 candidate is found (1) or not (0).
217 */
218 int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
219 RD_STATS *rd_stats, RD_STATS *rd_stats_y,
220 RD_STATS *rd_stats_uv, int mode_rate, int64_t ref_best_rd);
221
222 #ifdef __cplusplus
223 } // extern "C"
224 #endif
225
226 #endif // AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
227