1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019, Mellanox Technologies */
3
4 #ifndef _DR_TYPES_
5 #define _DR_TYPES_
6
7 #include <linux/mlx5/vport.h>
8 #include <linux/refcount.h>
9 #include "fs_core.h"
10 #include "wq.h"
11 #include "lib/mlx5.h"
12 #include "mlx5_ifc_dr.h"
13 #include "mlx5dr.h"
14 #include "dr_dbg.h"
15
16 #define DR_RULE_MAX_STES 18
17 #define DR_ACTION_MAX_STES 5
18 #define DR_STE_SVLAN 0x1
19 #define DR_STE_CVLAN 0x2
20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
21 #define DR_NUM_OF_FLEX_PARSERS 8
22 #define DR_STE_MAX_FLEX_0_ID 3
23 #define DR_STE_MAX_FLEX_1_ID 7
24 #define DR_ACTION_CACHE_LINE_SIZE 64
25
26 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
27 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
28 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg)
29
30 struct mlx5dr_ptrn_mgr;
31 struct mlx5dr_arg_mgr;
32 struct mlx5dr_arg_obj;
33
dr_is_flex_parser_0_id(u8 parser_id)34 static inline bool dr_is_flex_parser_0_id(u8 parser_id)
35 {
36 return parser_id <= DR_STE_MAX_FLEX_0_ID;
37 }
38
dr_is_flex_parser_1_id(u8 parser_id)39 static inline bool dr_is_flex_parser_1_id(u8 parser_id)
40 {
41 return parser_id > DR_STE_MAX_FLEX_0_ID;
42 }
43
44 enum mlx5dr_icm_chunk_size {
45 DR_CHUNK_SIZE_1,
46 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */
47 DR_CHUNK_SIZE_2,
48 DR_CHUNK_SIZE_4,
49 DR_CHUNK_SIZE_8,
50 DR_CHUNK_SIZE_16,
51 DR_CHUNK_SIZE_32,
52 DR_CHUNK_SIZE_64,
53 DR_CHUNK_SIZE_128,
54 DR_CHUNK_SIZE_256,
55 DR_CHUNK_SIZE_512,
56 DR_CHUNK_SIZE_1K,
57 DR_CHUNK_SIZE_2K,
58 DR_CHUNK_SIZE_4K,
59 DR_CHUNK_SIZE_8K,
60 DR_CHUNK_SIZE_16K,
61 DR_CHUNK_SIZE_32K,
62 DR_CHUNK_SIZE_64K,
63 DR_CHUNK_SIZE_128K,
64 DR_CHUNK_SIZE_256K,
65 DR_CHUNK_SIZE_512K,
66 DR_CHUNK_SIZE_1024K,
67 DR_CHUNK_SIZE_2048K,
68 DR_CHUNK_SIZE_MAX,
69 };
70
71 enum mlx5dr_icm_type {
72 DR_ICM_TYPE_STE,
73 DR_ICM_TYPE_MODIFY_ACTION,
74 DR_ICM_TYPE_MODIFY_HDR_PTRN,
75 DR_ICM_TYPE_MAX,
76 };
77
78 static inline enum mlx5dr_icm_chunk_size
mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)79 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)
80 {
81 chunk += 2;
82 if (chunk < DR_CHUNK_SIZE_MAX)
83 return chunk;
84
85 return DR_CHUNK_SIZE_MAX;
86 }
87
88 enum {
89 DR_STE_SIZE = 64,
90 DR_STE_SIZE_CTRL = 32,
91 DR_STE_SIZE_MATCH_TAG = 32,
92 DR_STE_SIZE_TAG = 16,
93 DR_STE_SIZE_MASK = 16,
94 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK,
95 };
96
97 enum mlx5dr_ste_ctx_action_cap {
98 DR_STE_CTX_ACTION_CAP_NONE = 0,
99 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0,
100 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1,
101 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2,
102 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3,
103 };
104
105 enum {
106 DR_MODIFY_ACTION_SIZE = 8,
107 };
108
109 enum mlx5dr_matcher_criteria {
110 DR_MATCHER_CRITERIA_EMPTY = 0,
111 DR_MATCHER_CRITERIA_OUTER = 1 << 0,
112 DR_MATCHER_CRITERIA_MISC = 1 << 1,
113 DR_MATCHER_CRITERIA_INNER = 1 << 2,
114 DR_MATCHER_CRITERIA_MISC2 = 1 << 3,
115 DR_MATCHER_CRITERIA_MISC3 = 1 << 4,
116 DR_MATCHER_CRITERIA_MISC4 = 1 << 5,
117 DR_MATCHER_CRITERIA_MISC5 = 1 << 6,
118 DR_MATCHER_CRITERIA_MAX = 1 << 7,
119 };
120
121 enum mlx5dr_action_type {
122 DR_ACTION_TYP_TNL_L2_TO_L2,
123 DR_ACTION_TYP_L2_TO_TNL_L2,
124 DR_ACTION_TYP_TNL_L3_TO_L2,
125 DR_ACTION_TYP_L2_TO_TNL_L3,
126 DR_ACTION_TYP_DROP,
127 DR_ACTION_TYP_QP,
128 DR_ACTION_TYP_FT,
129 DR_ACTION_TYP_CTR,
130 DR_ACTION_TYP_TAG,
131 DR_ACTION_TYP_MODIFY_HDR,
132 DR_ACTION_TYP_VPORT,
133 DR_ACTION_TYP_POP_VLAN,
134 DR_ACTION_TYP_PUSH_VLAN,
135 DR_ACTION_TYP_INSERT_HDR,
136 DR_ACTION_TYP_REMOVE_HDR,
137 DR_ACTION_TYP_SAMPLER,
138 DR_ACTION_TYP_ASO_FLOW_METER,
139 DR_ACTION_TYP_RANGE,
140 DR_ACTION_TYP_MAX,
141 };
142
143 enum mlx5dr_ipv {
144 DR_RULE_IPV4,
145 DR_RULE_IPV6,
146 DR_RULE_IPV_MAX,
147 };
148
149 struct mlx5dr_icm_pool;
150 struct mlx5dr_icm_chunk;
151 struct mlx5dr_icm_buddy_mem;
152 struct mlx5dr_ste_htbl;
153 struct mlx5dr_match_param;
154 struct mlx5dr_cmd_caps;
155 struct mlx5dr_rule_rx_tx;
156 struct mlx5dr_matcher_rx_tx;
157 struct mlx5dr_ste_ctx;
158 struct mlx5dr_send_info_pool;
159 struct mlx5dr_icm_hot_chunk;
160
161 struct mlx5dr_ste {
162 /* refcount: indicates the num of rules that using this ste */
163 u32 refcount;
164
165 /* this ste is part of a rule, located in ste's chain */
166 u8 ste_chain_location;
167
168 /* attached to the miss_list head at each htbl entry */
169 struct list_head miss_list_node;
170
171 /* this ste is member of htbl */
172 struct mlx5dr_ste_htbl *htbl;
173
174 struct mlx5dr_ste_htbl *next_htbl;
175
176 /* The rule this STE belongs to */
177 struct mlx5dr_rule_rx_tx *rule_rx_tx;
178 };
179
180 struct mlx5dr_ste_htbl_ctrl {
181 /* total number of valid entries belonging to this hash table. This
182 * includes the non collision and collision entries
183 */
184 unsigned int num_of_valid_entries;
185
186 /* total number of collisions entries attached to this table */
187 unsigned int num_of_collisions;
188 };
189
190 struct mlx5dr_ste_htbl {
191 u16 lu_type;
192 u16 byte_mask;
193 u32 refcount;
194 struct mlx5dr_icm_chunk *chunk;
195 struct mlx5dr_ste *pointing_ste;
196 struct mlx5dr_ste_htbl_ctrl ctrl;
197 };
198
199 struct mlx5dr_ste_send_info {
200 struct mlx5dr_ste *ste;
201 struct list_head send_list;
202 u16 size;
203 u16 offset;
204 u8 data_cont[DR_STE_SIZE];
205 u8 *data;
206 };
207
208 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size,
209 u16 offset, u8 *data,
210 struct mlx5dr_ste_send_info *ste_info,
211 struct list_head *send_list,
212 bool copy_data);
213
214 struct mlx5dr_ste_build {
215 u8 inner:1;
216 u8 rx:1;
217 u8 vhca_id_valid:1;
218 struct mlx5dr_domain *dmn;
219 struct mlx5dr_cmd_caps *caps;
220 u16 lu_type;
221 u16 byte_mask;
222 u8 bit_mask[DR_STE_SIZE_MASK];
223 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec,
224 struct mlx5dr_ste_build *sb,
225 u8 *tag);
226 };
227
228 struct mlx5dr_ste_htbl *
229 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
230 enum mlx5dr_icm_chunk_size chunk_size,
231 u16 lu_type, u16 byte_mask);
232
233 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl);
234
mlx5dr_htbl_put(struct mlx5dr_ste_htbl * htbl)235 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl)
236 {
237 htbl->refcount--;
238 if (!htbl->refcount)
239 mlx5dr_ste_htbl_free(htbl);
240 }
241
mlx5dr_htbl_get(struct mlx5dr_ste_htbl * htbl)242 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl)
243 {
244 htbl->refcount++;
245 }
246
247 /* STE utils */
248 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl);
249 bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p);
250 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
251 u8 *hw_ste, u64 miss_addr);
252 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx,
253 u8 *hw_ste, u64 icm_addr, u32 ht_size);
254 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx,
255 u8 *hw_ste,
256 struct mlx5dr_ste_htbl *next_htbl);
257 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask);
258 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher,
259 u8 ste_location);
260 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste);
261 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste);
262 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste);
263
264 #define MLX5DR_MAX_VLANS 2
265 #define MLX5DR_INVALID_PATTERN_INDEX 0xffffffff
266
267 struct mlx5dr_ste_actions_attr {
268 u32 modify_index;
269 u32 modify_pat_idx;
270 u16 modify_actions;
271 u8 *single_modify_action;
272 u32 decap_index;
273 u32 decap_pat_idx;
274 u16 decap_actions;
275 u8 decap_with_vlan:1;
276 u64 final_icm_addr;
277 u32 flow_tag;
278 u32 ctr_id;
279 u16 gvmi;
280 u16 hit_gvmi;
281 struct {
282 u32 id;
283 u32 size;
284 u8 param_0;
285 u8 param_1;
286 } reformat;
287 struct {
288 int count;
289 u32 headers[MLX5DR_MAX_VLANS];
290 } vlans;
291
292 struct {
293 u32 obj_id;
294 u32 offset;
295 u8 dest_reg_id;
296 u8 init_color;
297 } aso_flow_meter;
298
299 struct {
300 u64 miss_icm_addr;
301 u32 definer_id;
302 u32 min;
303 u32 max;
304 } range;
305 };
306
307 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx,
308 struct mlx5dr_domain *dmn,
309 u8 *action_type_set,
310 u8 *last_ste,
311 struct mlx5dr_ste_actions_attr *attr,
312 u32 *added_stes);
313 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx,
314 struct mlx5dr_domain *dmn,
315 u8 *action_type_set,
316 u8 *last_ste,
317 struct mlx5dr_ste_actions_attr *attr,
318 u32 *added_stes);
319
320 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx,
321 __be64 *hw_action,
322 u8 hw_field,
323 u8 shifter,
324 u8 length,
325 u32 data);
326 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx,
327 __be64 *hw_action,
328 u8 hw_field,
329 u8 shifter,
330 u8 length,
331 u32 data);
332 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx,
333 __be64 *hw_action,
334 u8 dst_hw_field,
335 u8 dst_shifter,
336 u8 dst_len,
337 u8 src_hw_field,
338 u8 src_shifter);
339 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
340 void *data,
341 u32 data_sz,
342 u8 *hw_action,
343 u32 hw_action_sz,
344 u16 *used_hw_action_num);
345 int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action);
346 void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action);
347
348 const struct mlx5dr_ste_action_modify_field *
349 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field);
350
351 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version);
352 void mlx5dr_ste_free(struct mlx5dr_ste *ste,
353 struct mlx5dr_matcher *matcher,
354 struct mlx5dr_matcher_rx_tx *nic_matcher);
mlx5dr_ste_put(struct mlx5dr_ste * ste,struct mlx5dr_matcher * matcher,struct mlx5dr_matcher_rx_tx * nic_matcher)355 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste,
356 struct mlx5dr_matcher *matcher,
357 struct mlx5dr_matcher_rx_tx *nic_matcher)
358 {
359 ste->refcount--;
360 if (!ste->refcount)
361 mlx5dr_ste_free(ste, matcher, nic_matcher);
362 }
363
364 /* initial as 0, increased only when ste appears in a new rule */
mlx5dr_ste_get(struct mlx5dr_ste * ste)365 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste)
366 {
367 ste->refcount++;
368 }
369
mlx5dr_ste_is_not_used(struct mlx5dr_ste * ste)370 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste)
371 {
372 return !ste->refcount;
373 }
374
375 bool mlx5dr_ste_equal_tag(void *src, void *dst);
376 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher,
377 struct mlx5dr_matcher_rx_tx *nic_matcher,
378 struct mlx5dr_ste *ste,
379 u8 *cur_hw_ste,
380 enum mlx5dr_icm_chunk_size log_table_size);
381
382 /* STE build functions */
383 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
384 u8 match_criteria,
385 struct mlx5dr_match_param *mask,
386 struct mlx5dr_match_param *value);
387 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher,
388 struct mlx5dr_matcher_rx_tx *nic_matcher,
389 struct mlx5dr_match_param *value,
390 u8 *ste_arr);
391 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx,
392 struct mlx5dr_ste_build *builder,
393 struct mlx5dr_match_param *mask,
394 bool inner, bool rx);
395 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx,
396 struct mlx5dr_ste_build *sb,
397 struct mlx5dr_match_param *mask,
398 bool inner, bool rx);
399 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx,
400 struct mlx5dr_ste_build *sb,
401 struct mlx5dr_match_param *mask,
402 bool inner, bool rx);
403 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx,
404 struct mlx5dr_ste_build *sb,
405 struct mlx5dr_match_param *mask,
406 bool inner, bool rx);
407 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx,
408 struct mlx5dr_ste_build *sb,
409 struct mlx5dr_match_param *mask,
410 bool inner, bool rx);
411 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx,
412 struct mlx5dr_ste_build *sb,
413 struct mlx5dr_match_param *mask,
414 bool inner, bool rx);
415 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx,
416 struct mlx5dr_ste_build *sb,
417 struct mlx5dr_match_param *mask,
418 bool inner, bool rx);
419 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx,
420 struct mlx5dr_ste_build *sb,
421 struct mlx5dr_match_param *mask,
422 bool inner, bool rx);
423 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx,
424 struct mlx5dr_ste_build *sb,
425 struct mlx5dr_match_param *mask,
426 bool inner, bool rx);
427 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx,
428 struct mlx5dr_ste_build *sb,
429 struct mlx5dr_match_param *mask,
430 bool inner, bool rx);
431 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx,
432 struct mlx5dr_ste_build *sb,
433 struct mlx5dr_match_param *mask,
434 bool inner, bool rx);
435 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx,
436 struct mlx5dr_ste_build *sb,
437 struct mlx5dr_match_param *mask,
438 bool inner, bool rx);
439 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx,
440 struct mlx5dr_ste_build *sb,
441 struct mlx5dr_match_param *mask,
442 bool inner, bool rx);
443 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx,
444 struct mlx5dr_ste_build *sb,
445 struct mlx5dr_match_param *mask,
446 struct mlx5dr_cmd_caps *caps,
447 bool inner, bool rx);
448 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx,
449 struct mlx5dr_ste_build *sb,
450 struct mlx5dr_match_param *mask,
451 struct mlx5dr_cmd_caps *caps,
452 bool inner, bool rx);
453 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx,
454 struct mlx5dr_ste_build *sb,
455 struct mlx5dr_match_param *mask,
456 struct mlx5dr_cmd_caps *caps,
457 bool inner, bool rx);
458 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx,
459 struct mlx5dr_ste_build *sb,
460 struct mlx5dr_match_param *mask,
461 bool inner, bool rx);
462 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx,
463 struct mlx5dr_ste_build *sb,
464 struct mlx5dr_match_param *mask,
465 bool inner, bool rx);
466 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx,
467 struct mlx5dr_ste_build *sb,
468 struct mlx5dr_match_param *mask,
469 struct mlx5dr_cmd_caps *caps,
470 bool inner, bool rx);
471 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx,
472 struct mlx5dr_ste_build *sb,
473 struct mlx5dr_match_param *mask,
474 struct mlx5dr_cmd_caps *caps,
475 bool inner, bool rx);
476 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx,
477 struct mlx5dr_ste_build *sb,
478 struct mlx5dr_match_param *mask,
479 bool inner, bool rx);
480 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
481 struct mlx5dr_ste_build *sb,
482 struct mlx5dr_match_param *mask,
483 struct mlx5dr_cmd_caps *caps,
484 bool inner, bool rx);
485 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
486 struct mlx5dr_ste_build *sb,
487 struct mlx5dr_match_param *mask,
488 struct mlx5dr_cmd_caps *caps,
489 bool inner, bool rx);
490 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx,
491 struct mlx5dr_ste_build *sb,
492 struct mlx5dr_match_param *mask,
493 bool inner, bool rx);
494 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx,
495 struct mlx5dr_ste_build *sb,
496 struct mlx5dr_match_param *mask,
497 bool inner, bool rx);
498 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx,
499 struct mlx5dr_ste_build *sb,
500 struct mlx5dr_match_param *mask,
501 bool inner, bool rx);
502 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx,
503 struct mlx5dr_ste_build *sb,
504 struct mlx5dr_match_param *mask,
505 bool inner, bool rx);
506 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx,
507 struct mlx5dr_ste_build *sb,
508 struct mlx5dr_match_param *mask,
509 struct mlx5dr_domain *dmn,
510 bool inner, bool rx);
511 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
512 struct mlx5dr_ste_build *sb,
513 struct mlx5dr_match_param *mask,
514 bool inner, bool rx);
515 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
516 struct mlx5dr_ste_build *sb,
517 struct mlx5dr_match_param *mask,
518 bool inner, bool rx);
519 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx);
520
521 /* Actions utils */
522 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
523 struct mlx5dr_matcher_rx_tx *nic_matcher,
524 struct mlx5dr_action *actions[],
525 u32 num_actions,
526 u8 *ste_arr,
527 u32 *new_hw_ste_arr_sz);
528
529 struct mlx5dr_match_spec {
530 u32 smac_47_16; /* Source MAC address of incoming packet */
531 /* Incoming packet Ethertype - this is the Ethertype
532 * following the last VLAN tag of the packet
533 */
534 u32 smac_15_0:16; /* Source MAC address of incoming packet */
535 u32 ethertype:16;
536
537 u32 dmac_47_16; /* Destination MAC address of incoming packet */
538
539 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */
540 /* Priority of first VLAN tag in the incoming packet.
541 * Valid only when cvlan_tag==1 or svlan_tag==1
542 */
543 u32 first_prio:3;
544 /* CFI bit of first VLAN tag in the incoming packet.
545 * Valid only when cvlan_tag==1 or svlan_tag==1
546 */
547 u32 first_cfi:1;
548 /* VLAN ID of first VLAN tag in the incoming packet.
549 * Valid only when cvlan_tag==1 or svlan_tag==1
550 */
551 u32 first_vid:12;
552
553 u32 ip_protocol:8; /* IP protocol */
554 /* Differentiated Services Code Point derived from
555 * Traffic Class/TOS field of IPv6/v4
556 */
557 u32 ip_dscp:6;
558 /* Explicit Congestion Notification derived from
559 * Traffic Class/TOS field of IPv6/v4
560 */
561 u32 ip_ecn:2;
562 /* The first vlan in the packet is c-vlan (0x8100).
563 * cvlan_tag and svlan_tag cannot be set together
564 */
565 u32 cvlan_tag:1;
566 /* The first vlan in the packet is s-vlan (0x8a88).
567 * cvlan_tag and svlan_tag cannot be set together
568 */
569 u32 svlan_tag:1;
570 u32 frag:1; /* Packet is an IP fragment */
571 u32 ip_version:4; /* IP version */
572 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK;
573 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS
574 */
575 u32 tcp_flags:9;
576
577 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */
578 u32 tcp_sport:16;
579 /* TCP destination port.
580 * tcp and udp sport/dport are mutually exclusive
581 */
582 u32 tcp_dport:16;
583
584 u32 reserved_auto1:16;
585 u32 ipv4_ihl:4;
586 u32 reserved_auto2:4;
587 u32 ttl_hoplimit:8;
588
589 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */
590 u32 udp_sport:16;
591 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */
592 u32 udp_dport:16;
593
594 /* IPv6 source address of incoming packets
595 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
596 * This field should be qualified by an appropriate ethertype
597 */
598 u32 src_ip_127_96;
599 /* IPv6 source address of incoming packets
600 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
601 * This field should be qualified by an appropriate ethertype
602 */
603 u32 src_ip_95_64;
604 /* IPv6 source address of incoming packets
605 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
606 * This field should be qualified by an appropriate ethertype
607 */
608 u32 src_ip_63_32;
609 /* IPv6 source address of incoming packets
610 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
611 * This field should be qualified by an appropriate ethertype
612 */
613 u32 src_ip_31_0;
614 /* IPv6 destination address of incoming packets
615 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
616 * This field should be qualified by an appropriate ethertype
617 */
618 u32 dst_ip_127_96;
619 /* IPv6 destination address of incoming packets
620 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
621 * This field should be qualified by an appropriate ethertype
622 */
623 u32 dst_ip_95_64;
624 /* IPv6 destination address of incoming packets
625 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
626 * This field should be qualified by an appropriate ethertype
627 */
628 u32 dst_ip_63_32;
629 /* IPv6 destination address of incoming packets
630 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
631 * This field should be qualified by an appropriate ethertype
632 */
633 u32 dst_ip_31_0;
634 };
635
636 struct mlx5dr_match_misc {
637 /* used with GRE, checksum exist when gre_c_present == 1 */
638 u32 gre_c_present:1;
639 u32 reserved_auto1:1;
640 /* used with GRE, key exist when gre_k_present == 1 */
641 u32 gre_k_present:1;
642 /* used with GRE, sequence number exist when gre_s_present == 1 */
643 u32 gre_s_present:1;
644 u32 source_vhca_port:4;
645 u32 source_sqn:24; /* Source SQN */
646
647 u32 source_eswitch_owner_vhca_id:16;
648 /* Source port.;0xffff determines wire port */
649 u32 source_port:16;
650
651 /* Priority of second VLAN tag in the outer header of the incoming packet.
652 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
653 */
654 u32 outer_second_prio:3;
655 /* CFI bit of first VLAN tag in the outer header of the incoming packet.
656 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
657 */
658 u32 outer_second_cfi:1;
659 /* VLAN ID of first VLAN tag the outer header of the incoming packet.
660 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
661 */
662 u32 outer_second_vid:12;
663 /* Priority of second VLAN tag in the inner header of the incoming packet.
664 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
665 */
666 u32 inner_second_prio:3;
667 /* CFI bit of first VLAN tag in the inner header of the incoming packet.
668 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
669 */
670 u32 inner_second_cfi:1;
671 /* VLAN ID of first VLAN tag the inner header of the incoming packet.
672 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
673 */
674 u32 inner_second_vid:12;
675
676 u32 outer_second_cvlan_tag:1;
677 u32 inner_second_cvlan_tag:1;
678 /* The second vlan in the outer header of the packet is c-vlan (0x8100).
679 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
680 */
681 u32 outer_second_svlan_tag:1;
682 /* The second vlan in the inner header of the packet is c-vlan (0x8100).
683 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
684 */
685 u32 inner_second_svlan_tag:1;
686 /* The second vlan in the outer header of the packet is s-vlan (0x8a88).
687 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
688 */
689 u32 reserved_auto2:12;
690 /* The second vlan in the inner header of the packet is s-vlan (0x8a88).
691 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
692 */
693 u32 gre_protocol:16; /* GRE Protocol (outer) */
694
695 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */
696 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */
697
698 u32 vxlan_vni:24; /* VXLAN VNI (outer) */
699 u32 reserved_auto3:8;
700
701 u32 geneve_vni:24; /* GENEVE VNI field (outer) */
702 u32 reserved_auto4:6;
703 u32 geneve_tlv_option_0_exist:1;
704 u32 geneve_oam:1; /* GENEVE OAM field (outer) */
705
706 u32 reserved_auto5:12;
707 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */
708
709 u32 reserved_auto6:12;
710 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */
711
712 u32 reserved_auto7:10;
713 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */
714 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */
715
716 u32 reserved_auto8:8;
717 u32 bth_dst_qp:24; /* Destination QP in BTH header */
718
719 u32 reserved_auto9;
720 u32 outer_esp_spi;
721 u32 reserved_auto10[3];
722 };
723
724 struct mlx5dr_match_misc2 {
725 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */
726 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */
727 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */
728 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */
729
730 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */
731 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */
732 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */
733 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */
734
735 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */
736 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */
737 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */
738 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */
739
740 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */
741 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */
742 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */
743 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */
744
745 u32 metadata_reg_c_7; /* metadata_reg_c_7 */
746 u32 metadata_reg_c_6; /* metadata_reg_c_6 */
747 u32 metadata_reg_c_5; /* metadata_reg_c_5 */
748 u32 metadata_reg_c_4; /* metadata_reg_c_4 */
749 u32 metadata_reg_c_3; /* metadata_reg_c_3 */
750 u32 metadata_reg_c_2; /* metadata_reg_c_2 */
751 u32 metadata_reg_c_1; /* metadata_reg_c_1 */
752 u32 metadata_reg_c_0; /* metadata_reg_c_0 */
753 u32 metadata_reg_a; /* metadata_reg_a */
754 u32 reserved_auto1[3];
755 };
756
757 struct mlx5dr_match_misc3 {
758 u32 inner_tcp_seq_num;
759 u32 outer_tcp_seq_num;
760 u32 inner_tcp_ack_num;
761 u32 outer_tcp_ack_num;
762
763 u32 reserved_auto1:8;
764 u32 outer_vxlan_gpe_vni:24;
765
766 u32 outer_vxlan_gpe_next_protocol:8;
767 u32 outer_vxlan_gpe_flags:8;
768 u32 reserved_auto2:16;
769
770 u32 icmpv4_header_data;
771 u32 icmpv6_header_data;
772
773 u8 icmpv4_type;
774 u8 icmpv4_code;
775 u8 icmpv6_type;
776 u8 icmpv6_code;
777
778 u32 geneve_tlv_option_0_data;
779
780 u32 gtpu_teid;
781
782 u8 gtpu_msg_type;
783 u8 gtpu_msg_flags;
784 u32 reserved_auto3:16;
785
786 u32 gtpu_dw_2;
787 u32 gtpu_first_ext_dw_0;
788 u32 gtpu_dw_0;
789 u32 reserved_auto4;
790 };
791
792 struct mlx5dr_match_misc4 {
793 u32 prog_sample_field_value_0;
794 u32 prog_sample_field_id_0;
795 u32 prog_sample_field_value_1;
796 u32 prog_sample_field_id_1;
797 u32 prog_sample_field_value_2;
798 u32 prog_sample_field_id_2;
799 u32 prog_sample_field_value_3;
800 u32 prog_sample_field_id_3;
801 u32 reserved_auto1[8];
802 };
803
804 struct mlx5dr_match_misc5 {
805 u32 macsec_tag_0;
806 u32 macsec_tag_1;
807 u32 macsec_tag_2;
808 u32 macsec_tag_3;
809 u32 tunnel_header_0;
810 u32 tunnel_header_1;
811 u32 tunnel_header_2;
812 u32 tunnel_header_3;
813 };
814
815 struct mlx5dr_match_param {
816 struct mlx5dr_match_spec outer;
817 struct mlx5dr_match_misc misc;
818 struct mlx5dr_match_spec inner;
819 struct mlx5dr_match_misc2 misc2;
820 struct mlx5dr_match_misc3 misc3;
821 struct mlx5dr_match_misc4 misc4;
822 struct mlx5dr_match_misc5 misc5;
823 };
824
825 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \
826 (_misc3)->icmpv4_code || \
827 (_misc3)->icmpv4_header_data)
828
829 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \
830 (_spec)->src_ip_95_64 || \
831 (_spec)->src_ip_63_32 || \
832 (_spec)->src_ip_31_0)
833
834 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \
835 (_spec)->dst_ip_95_64 || \
836 (_spec)->dst_ip_63_32 || \
837 (_spec)->dst_ip_31_0)
838
839 struct mlx5dr_esw_caps {
840 u64 drop_icm_address_rx;
841 u64 drop_icm_address_tx;
842 u64 uplink_icm_address_rx;
843 u64 uplink_icm_address_tx;
844 u8 sw_owner:1;
845 u8 sw_owner_v2:1;
846 };
847
848 struct mlx5dr_cmd_vport_cap {
849 u16 vport_gvmi;
850 u16 vhca_gvmi;
851 u16 num;
852 u64 icm_address_rx;
853 u64 icm_address_tx;
854 };
855
856 struct mlx5dr_roce_cap {
857 u8 roce_en:1;
858 u8 fl_rc_qp_when_roce_disabled:1;
859 u8 fl_rc_qp_when_roce_enabled:1;
860 };
861
862 struct mlx5dr_vports {
863 struct mlx5dr_cmd_vport_cap esw_manager_caps;
864 struct mlx5dr_cmd_vport_cap uplink_caps;
865 struct xarray vports_caps_xa;
866 };
867
868 struct mlx5dr_cmd_caps {
869 u16 gvmi;
870 u64 nic_rx_drop_address;
871 u64 nic_tx_drop_address;
872 u64 nic_tx_allow_address;
873 u64 esw_rx_drop_address;
874 u64 esw_tx_drop_address;
875 u32 log_icm_size;
876 u64 hdr_modify_icm_addr;
877 u32 log_modify_pattern_icm_size;
878 u64 hdr_modify_pattern_icm_addr;
879 u32 flex_protocols;
880 u8 flex_parser_id_icmp_dw0;
881 u8 flex_parser_id_icmp_dw1;
882 u8 flex_parser_id_icmpv6_dw0;
883 u8 flex_parser_id_icmpv6_dw1;
884 u8 flex_parser_id_geneve_tlv_option_0;
885 u8 flex_parser_id_mpls_over_gre;
886 u8 flex_parser_id_mpls_over_udp;
887 u8 flex_parser_id_gtpu_dw_0;
888 u8 flex_parser_id_gtpu_teid;
889 u8 flex_parser_id_gtpu_dw_2;
890 u8 flex_parser_id_gtpu_first_ext_dw_0;
891 u8 flex_parser_ok_bits_supp;
892 u8 max_ft_level;
893 u16 roce_min_src_udp;
894 u8 sw_format_ver;
895 bool eswitch_manager;
896 bool rx_sw_owner;
897 bool tx_sw_owner;
898 bool fdb_sw_owner;
899 u8 rx_sw_owner_v2:1;
900 u8 tx_sw_owner_v2:1;
901 u8 fdb_sw_owner_v2:1;
902 struct mlx5dr_esw_caps esw_caps;
903 struct mlx5dr_vports vports;
904 bool prio_tag_required;
905 struct mlx5dr_roce_cap roce_caps;
906 u16 log_header_modify_argument_granularity;
907 u16 log_header_modify_argument_max_alloc;
908 bool support_modify_argument;
909 u8 is_ecpf:1;
910 u8 isolate_vl_tc:1;
911 };
912
913 enum mlx5dr_domain_nic_type {
914 DR_DOMAIN_NIC_TYPE_RX,
915 DR_DOMAIN_NIC_TYPE_TX,
916 };
917
918 struct mlx5dr_domain_rx_tx {
919 u64 drop_icm_addr;
920 u64 default_icm_addr;
921 enum mlx5dr_domain_nic_type type;
922 struct mutex mutex; /* protect rx/tx domain */
923 };
924
925 struct mlx5dr_domain_info {
926 bool supp_sw_steering;
927 u32 max_inline_size;
928 u32 max_send_wr;
929 u32 max_log_sw_icm_sz;
930 u32 max_log_action_icm_sz;
931 u32 max_log_modify_hdr_pattern_icm_sz;
932 struct mlx5dr_domain_rx_tx rx;
933 struct mlx5dr_domain_rx_tx tx;
934 struct mlx5dr_cmd_caps caps;
935 };
936
937 struct mlx5dr_domain {
938 struct mlx5_core_dev *mdev;
939 u32 pdn;
940 struct mlx5_uars_page *uar;
941 enum mlx5dr_domain_type type;
942 refcount_t refcount;
943 struct mlx5dr_icm_pool *ste_icm_pool;
944 struct mlx5dr_icm_pool *action_icm_pool;
945 struct mlx5dr_send_info_pool *send_info_pool_rx;
946 struct mlx5dr_send_info_pool *send_info_pool_tx;
947 struct kmem_cache *chunks_kmem_cache;
948 struct kmem_cache *htbls_kmem_cache;
949 struct mlx5dr_ptrn_mgr *ptrn_mgr;
950 struct mlx5dr_arg_mgr *arg_mgr;
951 struct mlx5dr_send_ring *send_ring;
952 struct mlx5dr_domain_info info;
953 struct xarray csum_fts_xa;
954 struct mlx5dr_ste_ctx *ste_ctx;
955 struct list_head dbg_tbl_list;
956 struct mlx5dr_dbg_dump_info dump_info;
957 struct xarray definers_xa;
958 struct xarray peer_dmn_xa;
959 /* memory management statistics */
960 u32 num_buddies[DR_ICM_TYPE_MAX];
961 };
962
963 struct mlx5dr_table_rx_tx {
964 struct mlx5dr_ste_htbl *s_anchor;
965 struct mlx5dr_domain_rx_tx *nic_dmn;
966 u64 default_icm_addr;
967 struct list_head nic_matcher_list;
968 };
969
970 struct mlx5dr_table {
971 struct mlx5dr_domain *dmn;
972 struct mlx5dr_table_rx_tx rx;
973 struct mlx5dr_table_rx_tx tx;
974 u32 level;
975 u32 table_type;
976 u32 table_id;
977 u32 flags;
978 struct list_head matcher_list;
979 struct mlx5dr_action *miss_action;
980 refcount_t refcount;
981 struct list_head dbg_node;
982 };
983
984 struct mlx5dr_matcher_rx_tx {
985 struct mlx5dr_ste_htbl *s_htbl;
986 struct mlx5dr_ste_htbl *e_anchor;
987 struct mlx5dr_ste_build *ste_builder;
988 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX]
989 [DR_RULE_IPV_MAX]
990 [DR_RULE_MAX_STES];
991 u8 num_of_builders;
992 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX];
993 u64 default_icm_addr;
994 struct mlx5dr_table_rx_tx *nic_tbl;
995 u32 prio;
996 struct list_head list_node;
997 u32 rules;
998 };
999
1000 struct mlx5dr_matcher {
1001 struct mlx5dr_table *tbl;
1002 struct mlx5dr_matcher_rx_tx rx;
1003 struct mlx5dr_matcher_rx_tx tx;
1004 struct list_head list_node; /* Used for both matchers and dbg managing */
1005 u32 prio;
1006 struct mlx5dr_match_param mask;
1007 u8 match_criteria;
1008 refcount_t refcount;
1009 struct list_head dbg_rule_list;
1010 };
1011
1012 struct mlx5dr_ste_action_modify_field {
1013 u16 hw_field;
1014 u8 start;
1015 u8 end;
1016 u8 l3_type;
1017 u8 l4_type;
1018 };
1019
1020 struct mlx5dr_ptrn_obj {
1021 struct mlx5dr_icm_chunk *chunk;
1022 u8 *data;
1023 u16 num_of_actions;
1024 u32 index;
1025 refcount_t refcount;
1026 struct list_head list;
1027 };
1028
1029 struct mlx5dr_arg_obj {
1030 u32 obj_id;
1031 u32 obj_offset;
1032 struct list_head list_node;
1033 u32 log_chunk_size;
1034 };
1035
1036 struct mlx5dr_action_rewrite {
1037 struct mlx5dr_domain *dmn;
1038 struct mlx5dr_icm_chunk *chunk;
1039 u8 *data;
1040 u16 num_of_actions;
1041 u32 index;
1042 u8 single_action_opt:1;
1043 u8 allow_rx:1;
1044 u8 allow_tx:1;
1045 u8 modify_ttl:1;
1046 struct mlx5dr_ptrn_obj *ptrn;
1047 struct mlx5dr_arg_obj *arg;
1048 };
1049
1050 struct mlx5dr_action_reformat {
1051 struct mlx5dr_domain *dmn;
1052 u32 id;
1053 u32 size;
1054 u8 param_0;
1055 u8 param_1;
1056 };
1057
1058 struct mlx5dr_action_sampler {
1059 struct mlx5dr_domain *dmn;
1060 u64 rx_icm_addr;
1061 u64 tx_icm_addr;
1062 u32 sampler_id;
1063 };
1064
1065 struct mlx5dr_action_dest_tbl {
1066 u8 is_fw_tbl:1;
1067 union {
1068 struct mlx5dr_table *tbl;
1069 struct {
1070 struct mlx5dr_domain *dmn;
1071 u32 id;
1072 u32 group_id;
1073 enum fs_flow_table_type type;
1074 u64 rx_icm_addr;
1075 u64 tx_icm_addr;
1076 struct mlx5dr_action **ref_actions;
1077 u32 num_of_ref_actions;
1078 } fw_tbl;
1079 };
1080 };
1081
1082 struct mlx5dr_action_range {
1083 struct mlx5dr_domain *dmn;
1084 struct mlx5dr_action *hit_tbl_action;
1085 struct mlx5dr_action *miss_tbl_action;
1086 u32 definer_id;
1087 u32 min;
1088 u32 max;
1089 };
1090
1091 struct mlx5dr_action_ctr {
1092 u32 ctr_id;
1093 u32 offset;
1094 };
1095
1096 struct mlx5dr_action_vport {
1097 struct mlx5dr_domain *dmn;
1098 struct mlx5dr_cmd_vport_cap *caps;
1099 };
1100
1101 struct mlx5dr_action_push_vlan {
1102 u32 vlan_hdr; /* tpid_pcp_dei_vid */
1103 };
1104
1105 struct mlx5dr_action_flow_tag {
1106 u32 flow_tag;
1107 };
1108
1109 struct mlx5dr_rule_action_member {
1110 struct mlx5dr_action *action;
1111 struct list_head list;
1112 };
1113
1114 struct mlx5dr_action_aso_flow_meter {
1115 struct mlx5dr_domain *dmn;
1116 u32 obj_id;
1117 u32 offset;
1118 u8 dest_reg_id;
1119 u8 init_color;
1120 };
1121
1122 struct mlx5dr_action {
1123 enum mlx5dr_action_type action_type;
1124 refcount_t refcount;
1125
1126 union {
1127 void *data;
1128 struct mlx5dr_action_rewrite *rewrite;
1129 struct mlx5dr_action_reformat *reformat;
1130 struct mlx5dr_action_sampler *sampler;
1131 struct mlx5dr_action_dest_tbl *dest_tbl;
1132 struct mlx5dr_action_ctr *ctr;
1133 struct mlx5dr_action_vport *vport;
1134 struct mlx5dr_action_push_vlan *push_vlan;
1135 struct mlx5dr_action_flow_tag *flow_tag;
1136 struct mlx5dr_action_aso_flow_meter *aso;
1137 struct mlx5dr_action_range *range;
1138 };
1139 };
1140
1141 enum mlx5dr_connect_type {
1142 CONNECT_HIT = 1,
1143 CONNECT_MISS = 2,
1144 };
1145
1146 struct mlx5dr_htbl_connect_info {
1147 enum mlx5dr_connect_type type;
1148 union {
1149 struct mlx5dr_ste_htbl *hit_next_htbl;
1150 u64 miss_icm_addr;
1151 };
1152 };
1153
1154 struct mlx5dr_rule_rx_tx {
1155 struct mlx5dr_matcher_rx_tx *nic_matcher;
1156 struct mlx5dr_ste *last_rule_ste;
1157 };
1158
1159 struct mlx5dr_rule {
1160 struct mlx5dr_matcher *matcher;
1161 struct mlx5dr_rule_rx_tx rx;
1162 struct mlx5dr_rule_rx_tx tx;
1163 struct list_head rule_actions_list;
1164 struct list_head dbg_node;
1165 u32 flow_source;
1166 };
1167
1168 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
1169 struct mlx5dr_ste *ste,
1170 bool force);
1171 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
1172 struct mlx5dr_ste *curr_ste,
1173 int *num_of_stes);
1174
1175 struct mlx5dr_icm_chunk {
1176 struct mlx5dr_icm_buddy_mem *buddy_mem;
1177
1178 /* indicates the index of this chunk in the whole memory,
1179 * used for deleting the chunk from the buddy
1180 */
1181 unsigned int seg;
1182 enum mlx5dr_icm_chunk_size size;
1183
1184 /* Memory optimisation */
1185 struct mlx5dr_ste *ste_arr;
1186 u8 *hw_ste_arr;
1187 struct list_head *miss_list;
1188 };
1189
mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx * nic_dmn)1190 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn)
1191 {
1192 mutex_lock(&nic_dmn->mutex);
1193 }
1194
mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx * nic_dmn)1195 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn)
1196 {
1197 mutex_unlock(&nic_dmn->mutex);
1198 }
1199
mlx5dr_domain_lock(struct mlx5dr_domain * dmn)1200 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn)
1201 {
1202 mlx5dr_domain_nic_lock(&dmn->info.rx);
1203 mlx5dr_domain_nic_lock(&dmn->info.tx);
1204 }
1205
mlx5dr_domain_unlock(struct mlx5dr_domain * dmn)1206 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn)
1207 {
1208 mlx5dr_domain_nic_unlock(&dmn->info.tx);
1209 mlx5dr_domain_nic_unlock(&dmn->info.rx);
1210 }
1211
1212 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn,
1213 struct mlx5dr_matcher_rx_tx *nic_matcher);
1214 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn,
1215 struct mlx5dr_matcher_rx_tx *nic_matcher);
1216
1217 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher,
1218 struct mlx5dr_matcher_rx_tx *nic_matcher,
1219 enum mlx5dr_ipv outer_ipv,
1220 enum mlx5dr_ipv inner_ipv);
1221
1222 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk);
1223 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk);
1224 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk);
1225 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk);
1226 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk);
1227 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste);
1228
1229 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool);
1230 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl);
1231
1232 static inline int
mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)1233 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)
1234 {
1235 if (icm_type == DR_ICM_TYPE_STE)
1236 return DR_STE_SIZE;
1237
1238 return DR_MODIFY_ACTION_SIZE;
1239 }
1240
1241 static inline u32
mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)1242 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)
1243 {
1244 return 1 << chunk_size;
1245 }
1246
1247 static inline int
mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,enum mlx5dr_icm_type icm_type)1248 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,
1249 enum mlx5dr_icm_type icm_type)
1250 {
1251 int num_of_entries;
1252 int entry_size;
1253
1254 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type);
1255 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
1256
1257 return entry_size * num_of_entries;
1258 }
1259
1260 static inline int
mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl * htbl)1261 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl)
1262 {
1263 int num_of_entries =
1264 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size);
1265
1266 /* Threshold is 50%, one is added to table of size 1 */
1267 return (num_of_entries + 1) / 2;
1268 }
1269
1270 static inline bool
mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl * htbl)1271 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl)
1272 {
1273 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask)
1274 return false;
1275
1276 return true;
1277 }
1278
1279 struct mlx5dr_cmd_vport_cap *
1280 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport);
1281
1282 struct mlx5dr_cmd_query_flow_table_details {
1283 u8 status;
1284 u8 level;
1285 u64 sw_owner_icm_root_1;
1286 u64 sw_owner_icm_root_0;
1287 };
1288
1289 struct mlx5dr_cmd_create_flow_table_attr {
1290 u32 table_type;
1291 u16 uid;
1292 u64 icm_addr_rx;
1293 u64 icm_addr_tx;
1294 u8 level;
1295 bool sw_owner;
1296 bool term_tbl;
1297 bool decap_en;
1298 bool reformat_en;
1299 };
1300
1301 /* internal API functions */
1302 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
1303 struct mlx5dr_cmd_caps *caps);
1304 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev,
1305 bool other_vport, u16 vport_number,
1306 u64 *icm_address_rx,
1307 u64 *icm_address_tx);
1308 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev,
1309 bool other_vport, u16 vport_number, u16 *gvmi);
1310 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev,
1311 struct mlx5dr_esw_caps *caps);
1312 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev,
1313 u32 sampler_id,
1314 u64 *rx_icm_addr,
1315 u64 *tx_icm_addr);
1316 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev);
1317 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev,
1318 u32 table_type,
1319 u32 table_id,
1320 u32 group_id,
1321 u32 modify_header_id,
1322 u16 vport_id);
1323 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev,
1324 u32 table_type,
1325 u32 table_id);
1326 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev,
1327 u32 table_type,
1328 u8 num_of_actions,
1329 u64 *actions,
1330 u32 *modify_header_id);
1331 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev,
1332 u32 modify_header_id);
1333 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev,
1334 u32 table_type,
1335 u32 table_id,
1336 u32 *group_id);
1337 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev,
1338 u32 table_type,
1339 u32 table_id,
1340 u32 group_id);
1341 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev,
1342 struct mlx5dr_cmd_create_flow_table_attr *attr,
1343 u64 *fdb_rx_icm_addr,
1344 u32 *table_id);
1345 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev,
1346 u32 table_id,
1347 u32 table_type);
1348 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev,
1349 enum fs_flow_table_type type,
1350 u32 table_id,
1351 struct mlx5dr_cmd_query_flow_table_details *output);
1352 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev,
1353 enum mlx5_reformat_ctx_type rt,
1354 u8 reformat_param_0,
1355 u8 reformat_param_1,
1356 size_t reformat_size,
1357 void *reformat_data,
1358 u32 *reformat_id);
1359 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev,
1360 u32 reformat_id);
1361 int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev,
1362 u16 format_id,
1363 u8 *dw_selectors,
1364 u8 *byte_selectors,
1365 u8 *match_mask,
1366 u32 *definer_id);
1367 void mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev,
1368 u32 definer_id);
1369
1370 struct mlx5dr_cmd_gid_attr {
1371 u8 gid[16];
1372 u8 mac[6];
1373 u32 roce_ver;
1374 };
1375
1376 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
1377 u16 index, struct mlx5dr_cmd_gid_attr *attr);
1378
1379 int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev,
1380 u16 log_obj_range, u32 pd,
1381 u32 *obj_id);
1382 void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev,
1383 u32 obj_id);
1384
1385 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
1386 enum mlx5dr_icm_type icm_type);
1387 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool);
1388
1389 struct mlx5dr_icm_chunk *
1390 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
1391 enum mlx5dr_icm_chunk_size chunk_size);
1392 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk);
1393
1394 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx,
1395 u8 *hw_ste_p, u32 ste_size);
1396 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn,
1397 struct mlx5dr_domain_rx_tx *nic_dmn,
1398 struct mlx5dr_ste_htbl *htbl,
1399 struct mlx5dr_htbl_connect_info *connect_info,
1400 bool update_hw_ste);
1401 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx,
1402 u16 gvmi,
1403 enum mlx5dr_domain_nic_type nic_type,
1404 struct mlx5dr_ste_htbl *htbl,
1405 u8 *formatted_ste,
1406 struct mlx5dr_htbl_connect_info *connect_info);
1407 void mlx5dr_ste_copy_param(u8 match_criteria,
1408 struct mlx5dr_match_param *set_param,
1409 struct mlx5dr_match_parameters *mask,
1410 bool clear);
1411
1412 struct mlx5dr_qp {
1413 struct mlx5_core_dev *mdev;
1414 struct mlx5_wq_qp wq;
1415 struct mlx5_uars_page *uar;
1416 struct mlx5_wq_ctrl wq_ctrl;
1417 u32 qpn;
1418 struct {
1419 unsigned int head;
1420 unsigned int pc;
1421 unsigned int cc;
1422 unsigned int size;
1423 unsigned int *wqe_head;
1424 unsigned int wqe_cnt;
1425 } sq;
1426 struct {
1427 unsigned int pc;
1428 unsigned int cc;
1429 unsigned int size;
1430 unsigned int wqe_cnt;
1431 } rq;
1432 int max_inline_data;
1433 };
1434
1435 struct mlx5dr_cq {
1436 struct mlx5_core_dev *mdev;
1437 struct mlx5_cqwq wq;
1438 struct mlx5_wq_ctrl wq_ctrl;
1439 struct mlx5_core_cq mcq;
1440 struct mlx5dr_qp *qp;
1441 };
1442
1443 struct mlx5dr_mr {
1444 struct mlx5_core_dev *mdev;
1445 u32 mkey;
1446 dma_addr_t dma_addr;
1447 void *addr;
1448 size_t size;
1449 };
1450
1451 struct mlx5dr_send_ring {
1452 struct mlx5dr_cq *cq;
1453 struct mlx5dr_qp *qp;
1454 struct mlx5dr_mr *mr;
1455 /* How much wqes are waiting for completion */
1456 u32 pending_wqe;
1457 /* Signal request per this trash hold value */
1458 u16 signal_th;
1459 /* Each post_send_size less than max_post_send_size */
1460 u32 max_post_send_size;
1461 /* manage the send queue */
1462 u32 tx_head;
1463 void *buf;
1464 u32 buf_size;
1465 u8 *sync_buff;
1466 struct mlx5dr_mr *sync_mr;
1467 spinlock_t lock; /* Protect the data path of the send ring */
1468 bool err_state; /* send_ring is not usable in err state */
1469 };
1470
1471 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn);
1472 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn,
1473 struct mlx5dr_send_ring *send_ring);
1474 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn);
1475 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn,
1476 struct mlx5dr_ste *ste,
1477 u8 *data,
1478 u16 size,
1479 u16 offset);
1480 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn,
1481 struct mlx5dr_ste_htbl *htbl,
1482 u8 *formatted_ste, u8 *mask);
1483 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn,
1484 struct mlx5dr_ste_htbl *htbl,
1485 u8 *ste_init_data,
1486 bool update_hw_ste);
1487 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
1488 struct mlx5dr_action *action);
1489 int mlx5dr_send_postsend_pattern(struct mlx5dr_domain *dmn,
1490 struct mlx5dr_icm_chunk *chunk,
1491 u16 num_of_actions,
1492 u8 *data);
1493 int mlx5dr_send_postsend_args(struct mlx5dr_domain *dmn, u64 arg_id,
1494 u16 num_of_actions, u8 *actions_data);
1495
1496 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn);
1497 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn);
1498 struct mlx5dr_ste_send_info *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn,
1499 enum mlx5dr_domain_nic_type nic_type);
1500 void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info);
1501
1502 struct mlx5dr_cmd_ft_info {
1503 u32 id;
1504 u16 vport;
1505 enum fs_flow_table_type type;
1506 };
1507
1508 struct mlx5dr_cmd_flow_destination_hw_info {
1509 enum mlx5_flow_destination_type type;
1510 union {
1511 u32 tir_num;
1512 u32 ft_num;
1513 u32 ft_id;
1514 u32 counter_id;
1515 u32 sampler_id;
1516 struct {
1517 u16 num;
1518 u16 vhca_id;
1519 u32 reformat_id;
1520 u8 flags;
1521 } vport;
1522 };
1523 };
1524
1525 struct mlx5dr_cmd_fte_info {
1526 u32 dests_size;
1527 u32 index;
1528 struct mlx5_flow_context flow_context;
1529 u32 *val;
1530 struct mlx5_flow_act action;
1531 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr;
1532 bool ignore_flow_level;
1533 };
1534
1535 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev,
1536 int opmod, int modify_mask,
1537 struct mlx5dr_cmd_ft_info *ft,
1538 u32 group_id,
1539 struct mlx5dr_cmd_fte_info *fte);
1540
1541 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps);
1542
1543 struct mlx5dr_fw_recalc_cs_ft {
1544 u64 rx_icm_addr;
1545 u32 table_id;
1546 u32 group_id;
1547 u32 modify_hdr_id;
1548 };
1549
1550 struct mlx5dr_fw_recalc_cs_ft *
1551 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num);
1552 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn,
1553 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft);
1554 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn,
1555 u16 vport_num,
1556 u64 *rx_icm_addr);
1557 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
1558 struct mlx5dr_cmd_flow_destination_hw_info *dest,
1559 int num_dest,
1560 bool reformat_req,
1561 u32 *tbl_id,
1562 u32 *group_id,
1563 bool ignore_flow_level,
1564 u32 flow_source);
1565 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id,
1566 u32 group_id);
1567
mlx5dr_is_fw_table(struct mlx5_flow_table * ft)1568 static inline bool mlx5dr_is_fw_table(struct mlx5_flow_table *ft)
1569 {
1570 return !ft->fs_dr_table.dr_table;
1571 }
1572
mlx5dr_supp_match_ranges(struct mlx5_core_dev * dev)1573 static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev)
1574 {
1575 return (MLX5_CAP_GEN(dev, steering_format_version) >=
1576 MLX5_STEERING_FORMAT_CONNECTX_6DX) &&
1577 (MLX5_CAP_GEN_64(dev, match_definer_format_supported) &
1578 (1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT));
1579 }
1580
1581 bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn);
1582 struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn);
1583 void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr);
1584 struct mlx5dr_ptrn_obj *mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr,
1585 u16 num_of_actions, u8 *data);
1586 void mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr,
1587 struct mlx5dr_ptrn_obj *pattern);
1588 struct mlx5dr_arg_mgr *mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn);
1589 void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr);
1590 struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr,
1591 u16 num_of_actions,
1592 u8 *data);
1593 void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr,
1594 struct mlx5dr_arg_obj *arg_obj);
1595 u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj);
1596
1597 #endif /* _DR_TYPES_H_ */
1598