• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2020 Mellanox Technologies.
3 
4 #include <linux/mlx5/driver.h>
5 #include <linux/mlx5/mlx5_ifc.h>
6 #include <linux/mlx5/fs.h>
7 
8 #include "lib/fs_chains.h"
9 #include "en/mapping.h"
10 #include "mlx5_core.h"
11 #include "fs_core.h"
12 #include "eswitch.h"
13 #include "en.h"
14 #include "en_tc.h"
15 
16 #define chains_lock(chains) ((chains)->lock)
17 #define chains_ht(chains) ((chains)->chains_ht)
18 #define chains_mapping(chains) ((chains)->chains_mapping)
19 #define prios_ht(chains) ((chains)->prios_ht)
20 #define ft_pool_left(chains) ((chains)->ft_left)
21 #define tc_default_ft(chains) ((chains)->tc_default_ft)
22 #define tc_end_ft(chains) ((chains)->tc_end_ft)
23 #define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
24 				  FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
25 
26 /* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
27  * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
28  * for each flow table pool. We can allocate up to 16M of each pool,
29  * and we keep track of how much we used via get_next_avail_sz_from_pool.
30  * Firmware doesn't report any of this for now.
31  * ESW_POOL is expected to be sorted from large to small and match firmware
32  * pools.
33  */
34 #define FT_SIZE (16 * 1024 * 1024)
35 static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
36 					  1 * 1024 * 1024,
37 					  64 * 1024,
38 					  128 };
39 #define FT_TBL_SZ (64 * 1024)
40 
41 struct mlx5_fs_chains {
42 	struct mlx5_core_dev *dev;
43 
44 	struct rhashtable chains_ht;
45 	struct rhashtable prios_ht;
46 	/* Protects above chains_ht and prios_ht */
47 	struct mutex lock;
48 
49 	struct mlx5_flow_table *tc_default_ft;
50 	struct mlx5_flow_table *tc_end_ft;
51 	struct mapping_ctx *chains_mapping;
52 
53 	enum mlx5_flow_namespace_type ns;
54 	u32 group_num;
55 	u32 flags;
56 
57 	int ft_left[ARRAY_SIZE(FT_POOLS)];
58 };
59 
60 struct fs_chain {
61 	struct rhash_head node;
62 
63 	u32 chain;
64 
65 	int ref;
66 	int id;
67 
68 	struct mlx5_fs_chains *chains;
69 	struct list_head prios_list;
70 	struct mlx5_flow_handle *restore_rule;
71 	struct mlx5_modify_hdr *miss_modify_hdr;
72 };
73 
74 struct prio_key {
75 	u32 chain;
76 	u32 prio;
77 	u32 level;
78 };
79 
80 struct prio {
81 	struct rhash_head node;
82 	struct list_head list;
83 
84 	struct prio_key key;
85 
86 	int ref;
87 
88 	struct fs_chain *chain;
89 	struct mlx5_flow_table *ft;
90 	struct mlx5_flow_table *next_ft;
91 	struct mlx5_flow_group *miss_group;
92 	struct mlx5_flow_handle *miss_rule;
93 };
94 
95 static const struct rhashtable_params chain_params = {
96 	.head_offset = offsetof(struct fs_chain, node),
97 	.key_offset = offsetof(struct fs_chain, chain),
98 	.key_len = sizeof_field(struct fs_chain, chain),
99 	.automatic_shrinking = true,
100 };
101 
102 static const struct rhashtable_params prio_params = {
103 	.head_offset = offsetof(struct prio, node),
104 	.key_offset = offsetof(struct prio, key),
105 	.key_len = sizeof_field(struct prio, key),
106 	.automatic_shrinking = true,
107 };
108 
mlx5_chains_prios_supported(struct mlx5_fs_chains * chains)109 bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
110 {
111 	return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
112 }
113 
mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains * chains)114 static bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
115 {
116 	return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
117 }
118 
mlx5_chains_backwards_supported(struct mlx5_fs_chains * chains)119 bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
120 {
121 	return mlx5_chains_prios_supported(chains) &&
122 	       mlx5_chains_ignore_flow_level_supported(chains);
123 }
124 
mlx5_chains_get_chain_range(struct mlx5_fs_chains * chains)125 u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
126 {
127 	if (!mlx5_chains_prios_supported(chains))
128 		return 1;
129 
130 	if (mlx5_chains_ignore_flow_level_supported(chains))
131 		return UINT_MAX - 1;
132 
133 	/* We should get here only for eswitch case */
134 	return FDB_TC_MAX_CHAIN;
135 }
136 
mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains * chains)137 u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
138 {
139 	return mlx5_chains_get_chain_range(chains) + 1;
140 }
141 
mlx5_chains_get_prio_range(struct mlx5_fs_chains * chains)142 u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
143 {
144 	if (!mlx5_chains_prios_supported(chains))
145 		return 1;
146 
147 	if (mlx5_chains_ignore_flow_level_supported(chains))
148 		return UINT_MAX;
149 
150 	/* We should get here only for eswitch case */
151 	return FDB_TC_MAX_PRIO;
152 }
153 
mlx5_chains_get_level_range(struct mlx5_fs_chains * chains)154 static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
155 {
156 	if (mlx5_chains_ignore_flow_level_supported(chains))
157 		return UINT_MAX;
158 
159 	/* Same value for FDB and NIC RX tables */
160 	return FDB_TC_LEVELS_PER_PRIO;
161 }
162 
163 void
mlx5_chains_set_end_ft(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)164 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
165 		       struct mlx5_flow_table *ft)
166 {
167 	tc_end_ft(chains) = ft;
168 }
169 
170 #define POOL_NEXT_SIZE 0
171 static int
mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains * chains,int desired_size)172 mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains,
173 				   int desired_size)
174 {
175 	int i, found_i = -1;
176 
177 	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
178 		if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) {
179 			found_i = i;
180 			if (desired_size != POOL_NEXT_SIZE)
181 				break;
182 		}
183 	}
184 
185 	if (found_i != -1) {
186 		--ft_pool_left(chains)[found_i];
187 		return FT_POOLS[found_i];
188 	}
189 
190 	return 0;
191 }
192 
193 static void
mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains * chains,int sz)194 mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz)
195 {
196 	int i;
197 
198 	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
199 		if (sz == FT_POOLS[i]) {
200 			++ft_pool_left(chains)[i];
201 			return;
202 		}
203 	}
204 
205 	WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
206 }
207 
208 static void
mlx5_chains_init_sz_pool(struct mlx5_fs_chains * chains,u32 ft_max)209 mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max)
210 {
211 	int i;
212 
213 	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
214 		ft_pool_left(chains)[i] =
215 			FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0;
216 }
217 
218 static struct mlx5_flow_table *
mlx5_chains_create_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)219 mlx5_chains_create_table(struct mlx5_fs_chains *chains,
220 			 u32 chain, u32 prio, u32 level)
221 {
222 	struct mlx5_flow_table_attr ft_attr = {};
223 	struct mlx5_flow_namespace *ns;
224 	struct mlx5_flow_table *ft;
225 	int sz;
226 
227 	if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
228 		ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
229 				  MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
230 
231 	sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
232 	     mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) :
233 	     mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE);
234 	if (!sz)
235 		return ERR_PTR(-ENOSPC);
236 	ft_attr.max_fte = sz;
237 
238 	/* We use tc_default_ft(chains) as the table's next_ft till
239 	 * ignore_flow_level is allowed on FT creation and not just for FTEs.
240 	 * Instead caller should add an explicit miss rule if needed.
241 	 */
242 	ft_attr.next_ft = tc_default_ft(chains);
243 
244 	/* The root table(chain 0, prio 1, level 0) is required to be
245 	 * connected to the previous fs_core managed prio.
246 	 * We always create it, as a managed table, in order to align with
247 	 * fs_core logic.
248 	 */
249 	if (!mlx5_chains_ignore_flow_level_supported(chains) ||
250 	    (chain == 0 && prio == 1 && level == 0)) {
251 		ft_attr.level = level;
252 		ft_attr.prio = prio - 1;
253 		ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
254 			mlx5_get_fdb_sub_ns(chains->dev, chain) :
255 			mlx5_get_flow_namespace(chains->dev, chains->ns);
256 	} else {
257 		ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
258 		ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
259 		/* Firmware doesn't allow us to create another level 0 table,
260 		 * so we create all unmanaged tables as level 1.
261 		 *
262 		 * To connect them, we use explicit miss rules with
263 		 * ignore_flow_level. Caller is responsible to create
264 		 * these rules (if needed).
265 		 */
266 		ft_attr.level = 1;
267 		ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
268 	}
269 
270 	ft_attr.autogroup.num_reserved_entries = 2;
271 	ft_attr.autogroup.max_num_groups = chains->group_num;
272 	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
273 	if (IS_ERR(ft)) {
274 		mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
275 			       (int)PTR_ERR(ft), chain, prio, level, sz);
276 		mlx5_chains_put_sz_to_pool(chains, sz);
277 		return ft;
278 	}
279 
280 	return ft;
281 }
282 
283 static void
mlx5_chains_destroy_table(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)284 mlx5_chains_destroy_table(struct mlx5_fs_chains *chains,
285 			  struct mlx5_flow_table *ft)
286 {
287 	mlx5_chains_put_sz_to_pool(chains, ft->max_fte);
288 	mlx5_destroy_flow_table(ft);
289 }
290 
291 static int
create_chain_restore(struct fs_chain * chain)292 create_chain_restore(struct fs_chain *chain)
293 {
294 	struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch;
295 	u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
296 	struct mlx5_fs_chains *chains = chain->chains;
297 	enum mlx5e_tc_attr_to_reg chain_to_reg;
298 	struct mlx5_modify_hdr *mod_hdr;
299 	u32 index;
300 	int err;
301 
302 	if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
303 	    !mlx5_chains_prios_supported(chains))
304 		return 0;
305 
306 	err = mapping_add(chains_mapping(chains), &chain->chain, &index);
307 	if (err)
308 		return err;
309 	if (index == MLX5_FS_DEFAULT_FLOW_TAG) {
310 		/* we got the special default flow tag id, so we won't know
311 		 * if we actually marked the packet with the restore rule
312 		 * we create.
313 		 *
314 		 * This case isn't possible with MLX5_FS_DEFAULT_FLOW_TAG = 0.
315 		 */
316 		err = mapping_add(chains_mapping(chains),
317 				  &chain->chain, &index);
318 		mapping_remove(chains_mapping(chains),
319 			       MLX5_FS_DEFAULT_FLOW_TAG);
320 		if (err)
321 			return err;
322 	}
323 
324 	chain->id = index;
325 
326 	if (chains->ns == MLX5_FLOW_NAMESPACE_FDB) {
327 		chain_to_reg = CHAIN_TO_REG;
328 		chain->restore_rule = esw_add_restore_rule(esw, chain->id);
329 		if (IS_ERR(chain->restore_rule)) {
330 			err = PTR_ERR(chain->restore_rule);
331 			goto err_rule;
332 		}
333 	} else if (chains->ns == MLX5_FLOW_NAMESPACE_KERNEL) {
334 		/* For NIC RX we don't need a restore rule
335 		 * since we write the metadata to reg_b
336 		 * that is passed to SW directly.
337 		 */
338 		chain_to_reg = NIC_CHAIN_TO_REG;
339 	} else {
340 		err = -EINVAL;
341 		goto err_rule;
342 	}
343 
344 	MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
345 	MLX5_SET(set_action_in, modact, field,
346 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield);
347 	MLX5_SET(set_action_in, modact, offset,
348 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8);
349 	MLX5_SET(set_action_in, modact, length,
350 		 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8);
351 	MLX5_SET(set_action_in, modact, data, chain->id);
352 	mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns,
353 					   1, modact);
354 	if (IS_ERR(mod_hdr)) {
355 		err = PTR_ERR(mod_hdr);
356 		goto err_mod_hdr;
357 	}
358 	chain->miss_modify_hdr = mod_hdr;
359 
360 	return 0;
361 
362 err_mod_hdr:
363 	if (!IS_ERR_OR_NULL(chain->restore_rule))
364 		mlx5_del_flow_rules(chain->restore_rule);
365 err_rule:
366 	/* Datapath can't find this mapping, so we can safely remove it */
367 	mapping_remove(chains_mapping(chains), chain->id);
368 	return err;
369 }
370 
destroy_chain_restore(struct fs_chain * chain)371 static void destroy_chain_restore(struct fs_chain *chain)
372 {
373 	struct mlx5_fs_chains *chains = chain->chains;
374 
375 	if (!chain->miss_modify_hdr)
376 		return;
377 
378 	if (chain->restore_rule)
379 		mlx5_del_flow_rules(chain->restore_rule);
380 
381 	mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr);
382 	mapping_remove(chains_mapping(chains), chain->id);
383 }
384 
385 static struct fs_chain *
mlx5_chains_create_chain(struct mlx5_fs_chains * chains,u32 chain)386 mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
387 {
388 	struct fs_chain *chain_s = NULL;
389 	int err;
390 
391 	chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
392 	if (!chain_s)
393 		return ERR_PTR(-ENOMEM);
394 
395 	chain_s->chains = chains;
396 	chain_s->chain = chain;
397 	INIT_LIST_HEAD(&chain_s->prios_list);
398 
399 	err = create_chain_restore(chain_s);
400 	if (err)
401 		goto err_restore;
402 
403 	err = rhashtable_insert_fast(&chains_ht(chains), &chain_s->node,
404 				     chain_params);
405 	if (err)
406 		goto err_insert;
407 
408 	return chain_s;
409 
410 err_insert:
411 	destroy_chain_restore(chain_s);
412 err_restore:
413 	kvfree(chain_s);
414 	return ERR_PTR(err);
415 }
416 
417 static void
mlx5_chains_destroy_chain(struct fs_chain * chain)418 mlx5_chains_destroy_chain(struct fs_chain *chain)
419 {
420 	struct mlx5_fs_chains *chains = chain->chains;
421 
422 	rhashtable_remove_fast(&chains_ht(chains), &chain->node,
423 			       chain_params);
424 
425 	destroy_chain_restore(chain);
426 	kvfree(chain);
427 }
428 
429 static struct fs_chain *
mlx5_chains_get_chain(struct mlx5_fs_chains * chains,u32 chain)430 mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
431 {
432 	struct fs_chain *chain_s;
433 
434 	chain_s = rhashtable_lookup_fast(&chains_ht(chains), &chain,
435 					 chain_params);
436 	if (!chain_s) {
437 		chain_s = mlx5_chains_create_chain(chains, chain);
438 		if (IS_ERR(chain_s))
439 			return chain_s;
440 	}
441 
442 	chain_s->ref++;
443 
444 	return chain_s;
445 }
446 
447 static struct mlx5_flow_handle *
mlx5_chains_add_miss_rule(struct fs_chain * chain,struct mlx5_flow_table * ft,struct mlx5_flow_table * next_ft)448 mlx5_chains_add_miss_rule(struct fs_chain *chain,
449 			  struct mlx5_flow_table *ft,
450 			  struct mlx5_flow_table *next_ft)
451 {
452 	struct mlx5_fs_chains *chains = chain->chains;
453 	struct mlx5_flow_destination dest = {};
454 	struct mlx5_flow_act act = {};
455 
456 	act.flags  = FLOW_ACT_NO_APPEND;
457 	if (mlx5_chains_ignore_flow_level_supported(chain->chains))
458 		act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
459 
460 	act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
461 	dest.type  = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
462 	dest.ft = next_ft;
463 
464 	if (next_ft == tc_end_ft(chains) &&
465 	    chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
466 	    mlx5_chains_prios_supported(chains)) {
467 		act.modify_hdr = chain->miss_modify_hdr;
468 		act.action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
469 	}
470 
471 	return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
472 }
473 
474 static int
mlx5_chains_update_prio_prevs(struct prio * prio,struct mlx5_flow_table * next_ft)475 mlx5_chains_update_prio_prevs(struct prio *prio,
476 			      struct mlx5_flow_table *next_ft)
477 {
478 	struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
479 	struct fs_chain *chain = prio->chain;
480 	struct prio *pos;
481 	int n = 0, err;
482 
483 	if (prio->key.level)
484 		return 0;
485 
486 	/* Iterate in reverse order until reaching the level 0 rule of
487 	 * the previous priority, adding all the miss rules first, so we can
488 	 * revert them if any of them fails.
489 	 */
490 	pos = prio;
491 	list_for_each_entry_continue_reverse(pos,
492 					     &chain->prios_list,
493 					     list) {
494 		miss_rules[n] = mlx5_chains_add_miss_rule(chain,
495 							  pos->ft,
496 							  next_ft);
497 		if (IS_ERR(miss_rules[n])) {
498 			err = PTR_ERR(miss_rules[n]);
499 			goto err_prev_rule;
500 		}
501 
502 		n++;
503 		if (!pos->key.level)
504 			break;
505 	}
506 
507 	/* Success, delete old miss rules, and update the pointers. */
508 	n = 0;
509 	pos = prio;
510 	list_for_each_entry_continue_reverse(pos,
511 					     &chain->prios_list,
512 					     list) {
513 		mlx5_del_flow_rules(pos->miss_rule);
514 
515 		pos->miss_rule = miss_rules[n];
516 		pos->next_ft = next_ft;
517 
518 		n++;
519 		if (!pos->key.level)
520 			break;
521 	}
522 
523 	return 0;
524 
525 err_prev_rule:
526 	while (--n >= 0)
527 		mlx5_del_flow_rules(miss_rules[n]);
528 
529 	return err;
530 }
531 
532 static void
mlx5_chains_put_chain(struct fs_chain * chain)533 mlx5_chains_put_chain(struct fs_chain *chain)
534 {
535 	if (--chain->ref == 0)
536 		mlx5_chains_destroy_chain(chain);
537 }
538 
539 static struct prio *
mlx5_chains_create_prio(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)540 mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
541 			u32 chain, u32 prio, u32 level)
542 {
543 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
544 	struct mlx5_flow_handle *miss_rule = NULL;
545 	struct mlx5_flow_group *miss_group;
546 	struct mlx5_flow_table *next_ft;
547 	struct mlx5_flow_table *ft;
548 	struct prio *prio_s = NULL;
549 	struct fs_chain *chain_s;
550 	struct list_head *pos;
551 	u32 *flow_group_in;
552 	int err;
553 
554 	chain_s = mlx5_chains_get_chain(chains, chain);
555 	if (IS_ERR(chain_s))
556 		return ERR_CAST(chain_s);
557 
558 	prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
559 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
560 	if (!prio_s || !flow_group_in) {
561 		err = -ENOMEM;
562 		goto err_alloc;
563 	}
564 
565 	/* Chain's prio list is sorted by prio and level.
566 	 * And all levels of some prio point to the next prio's level 0.
567 	 * Example list (prio, level):
568 	 * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
569 	 * In hardware, we will we have the following pointers:
570 	 * (3,0) -> (5,0) -> (7,0) -> Slow path
571 	 * (3,1) -> (5,0)
572 	 * (5,1) -> (7,0)
573 	 * (6,1) -> (7,0)
574 	 */
575 
576 	/* Default miss for each chain: */
577 	next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
578 		  tc_default_ft(chains) :
579 		  tc_end_ft(chains);
580 	list_for_each(pos, &chain_s->prios_list) {
581 		struct prio *p = list_entry(pos, struct prio, list);
582 
583 		/* exit on first pos that is larger */
584 		if (prio < p->key.prio || (prio == p->key.prio &&
585 					   level < p->key.level)) {
586 			/* Get next level 0 table */
587 			next_ft = p->key.level == 0 ? p->ft : p->next_ft;
588 			break;
589 		}
590 	}
591 
592 	ft = mlx5_chains_create_table(chains, chain, prio, level);
593 	if (IS_ERR(ft)) {
594 		err = PTR_ERR(ft);
595 		goto err_create;
596 	}
597 
598 	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
599 		 ft->max_fte - 2);
600 	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
601 		 ft->max_fte - 1);
602 	miss_group = mlx5_create_flow_group(ft, flow_group_in);
603 	if (IS_ERR(miss_group)) {
604 		err = PTR_ERR(miss_group);
605 		goto err_group;
606 	}
607 
608 	/* Add miss rule to next_ft */
609 	miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
610 	if (IS_ERR(miss_rule)) {
611 		err = PTR_ERR(miss_rule);
612 		goto err_miss_rule;
613 	}
614 
615 	prio_s->miss_group = miss_group;
616 	prio_s->miss_rule = miss_rule;
617 	prio_s->next_ft = next_ft;
618 	prio_s->chain = chain_s;
619 	prio_s->key.chain = chain;
620 	prio_s->key.prio = prio;
621 	prio_s->key.level = level;
622 	prio_s->ft = ft;
623 
624 	err = rhashtable_insert_fast(&prios_ht(chains), &prio_s->node,
625 				     prio_params);
626 	if (err)
627 		goto err_insert;
628 
629 	list_add(&prio_s->list, pos->prev);
630 
631 	/* Table is ready, connect it */
632 	err = mlx5_chains_update_prio_prevs(prio_s, ft);
633 	if (err)
634 		goto err_update;
635 
636 	kvfree(flow_group_in);
637 	return prio_s;
638 
639 err_update:
640 	list_del(&prio_s->list);
641 	rhashtable_remove_fast(&prios_ht(chains), &prio_s->node,
642 			       prio_params);
643 err_insert:
644 	mlx5_del_flow_rules(miss_rule);
645 err_miss_rule:
646 	mlx5_destroy_flow_group(miss_group);
647 err_group:
648 	mlx5_chains_destroy_table(chains, ft);
649 err_create:
650 err_alloc:
651 	kvfree(prio_s);
652 	kvfree(flow_group_in);
653 	mlx5_chains_put_chain(chain_s);
654 	return ERR_PTR(err);
655 }
656 
657 static void
mlx5_chains_destroy_prio(struct mlx5_fs_chains * chains,struct prio * prio)658 mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
659 			 struct prio *prio)
660 {
661 	struct fs_chain *chain = prio->chain;
662 
663 	WARN_ON(mlx5_chains_update_prio_prevs(prio,
664 					      prio->next_ft));
665 
666 	list_del(&prio->list);
667 	rhashtable_remove_fast(&prios_ht(chains), &prio->node,
668 			       prio_params);
669 	mlx5_del_flow_rules(prio->miss_rule);
670 	mlx5_destroy_flow_group(prio->miss_group);
671 	mlx5_chains_destroy_table(chains, prio->ft);
672 	mlx5_chains_put_chain(chain);
673 	kvfree(prio);
674 }
675 
676 struct mlx5_flow_table *
mlx5_chains_get_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)677 mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
678 		      u32 level)
679 {
680 	struct mlx5_flow_table *prev_fts;
681 	struct prio *prio_s;
682 	struct prio_key key;
683 	int l = 0;
684 
685 	if ((chain > mlx5_chains_get_chain_range(chains) &&
686 	     chain != mlx5_chains_get_nf_ft_chain(chains)) ||
687 	    prio > mlx5_chains_get_prio_range(chains) ||
688 	    level > mlx5_chains_get_level_range(chains))
689 		return ERR_PTR(-EOPNOTSUPP);
690 
691 	/* create earlier levels for correct fs_core lookup when
692 	 * connecting tables.
693 	 */
694 	for (l = 0; l < level; l++) {
695 		prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
696 		if (IS_ERR(prev_fts)) {
697 			prio_s = ERR_CAST(prev_fts);
698 			goto err_get_prevs;
699 		}
700 	}
701 
702 	key.chain = chain;
703 	key.prio = prio;
704 	key.level = level;
705 
706 	mutex_lock(&chains_lock(chains));
707 	prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
708 					prio_params);
709 	if (!prio_s) {
710 		prio_s = mlx5_chains_create_prio(chains, chain,
711 						 prio, level);
712 		if (IS_ERR(prio_s))
713 			goto err_create_prio;
714 	}
715 
716 	++prio_s->ref;
717 	mutex_unlock(&chains_lock(chains));
718 
719 	return prio_s->ft;
720 
721 err_create_prio:
722 	mutex_unlock(&chains_lock(chains));
723 err_get_prevs:
724 	while (--l >= 0)
725 		mlx5_chains_put_table(chains, chain, prio, l);
726 	return ERR_CAST(prio_s);
727 }
728 
729 void
mlx5_chains_put_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)730 mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
731 		      u32 level)
732 {
733 	struct prio *prio_s;
734 	struct prio_key key;
735 
736 	key.chain = chain;
737 	key.prio = prio;
738 	key.level = level;
739 
740 	mutex_lock(&chains_lock(chains));
741 	prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
742 					prio_params);
743 	if (!prio_s)
744 		goto err_get_prio;
745 
746 	if (--prio_s->ref == 0)
747 		mlx5_chains_destroy_prio(chains, prio_s);
748 	mutex_unlock(&chains_lock(chains));
749 
750 	while (level-- > 0)
751 		mlx5_chains_put_table(chains, chain, prio, level);
752 
753 	return;
754 
755 err_get_prio:
756 	mutex_unlock(&chains_lock(chains));
757 	WARN_ONCE(1,
758 		  "Couldn't find table: (chain: %d prio: %d level: %d)",
759 		  chain, prio, level);
760 }
761 
762 struct mlx5_flow_table *
mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains * chains)763 mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
764 {
765 	return tc_end_ft(chains);
766 }
767 
768 struct mlx5_flow_table *
mlx5_chains_create_global_table(struct mlx5_fs_chains * chains)769 mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
770 {
771 	u32 chain, prio, level;
772 	int err;
773 
774 	if (!mlx5_chains_ignore_flow_level_supported(chains)) {
775 		err = -EOPNOTSUPP;
776 
777 		mlx5_core_warn(chains->dev,
778 			       "Couldn't create global flow table, ignore_flow_level not supported.");
779 		goto err_ignore;
780 	}
781 
782 	chain = mlx5_chains_get_chain_range(chains),
783 	prio = mlx5_chains_get_prio_range(chains);
784 	level = mlx5_chains_get_level_range(chains);
785 
786 	return mlx5_chains_create_table(chains, chain, prio, level);
787 
788 err_ignore:
789 	return ERR_PTR(err);
790 }
791 
792 void
mlx5_chains_destroy_global_table(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)793 mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
794 				 struct mlx5_flow_table *ft)
795 {
796 	mlx5_chains_destroy_table(chains, ft);
797 }
798 
799 static struct mlx5_fs_chains *
mlx5_chains_init(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)800 mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
801 {
802 	struct mlx5_fs_chains *chains_priv;
803 	struct mapping_ctx *mapping;
804 	u32 max_flow_counter;
805 	int err;
806 
807 	chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
808 	if (!chains_priv)
809 		return ERR_PTR(-ENOMEM);
810 
811 	max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
812 			    MLX5_CAP_GEN(dev, max_flow_counter_15_0);
813 
814 	mlx5_core_dbg(dev,
815 		      "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
816 		      max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
817 
818 	chains_priv->dev = dev;
819 	chains_priv->flags = attr->flags;
820 	chains_priv->ns = attr->ns;
821 	chains_priv->group_num = attr->max_grp_num;
822 	tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
823 
824 	mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
825 		       mlx5_chains_get_chain_range(chains_priv),
826 		       mlx5_chains_get_prio_range(chains_priv));
827 
828 	mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz);
829 
830 	err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
831 	if (err)
832 		goto init_chains_ht_err;
833 
834 	err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
835 	if (err)
836 		goto init_prios_ht_err;
837 
838 	mapping = mapping_create(sizeof(u32), attr->max_restore_tag,
839 				 true);
840 	if (IS_ERR(mapping)) {
841 		err = PTR_ERR(mapping);
842 		goto mapping_err;
843 	}
844 	chains_mapping(chains_priv) = mapping;
845 
846 	mutex_init(&chains_lock(chains_priv));
847 
848 	return chains_priv;
849 
850 mapping_err:
851 	rhashtable_destroy(&prios_ht(chains_priv));
852 init_prios_ht_err:
853 	rhashtable_destroy(&chains_ht(chains_priv));
854 init_chains_ht_err:
855 	kfree(chains_priv);
856 	return ERR_PTR(err);
857 }
858 
859 static void
mlx5_chains_cleanup(struct mlx5_fs_chains * chains)860 mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
861 {
862 	mutex_destroy(&chains_lock(chains));
863 	mapping_destroy(chains_mapping(chains));
864 	rhashtable_destroy(&prios_ht(chains));
865 	rhashtable_destroy(&chains_ht(chains));
866 
867 	kfree(chains);
868 }
869 
870 struct mlx5_fs_chains *
mlx5_chains_create(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)871 mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
872 {
873 	struct mlx5_fs_chains *chains;
874 
875 	chains = mlx5_chains_init(dev, attr);
876 
877 	return chains;
878 }
879 
880 void
mlx5_chains_destroy(struct mlx5_fs_chains * chains)881 mlx5_chains_destroy(struct mlx5_fs_chains *chains)
882 {
883 	mlx5_chains_cleanup(chains);
884 }
885 
886 int
mlx5_chains_get_chain_mapping(struct mlx5_fs_chains * chains,u32 chain,u32 * chain_mapping)887 mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
888 			      u32 *chain_mapping)
889 {
890 	return mapping_add(chains_mapping(chains), &chain, chain_mapping);
891 }
892 
893 int
mlx5_chains_put_chain_mapping(struct mlx5_fs_chains * chains,u32 chain_mapping)894 mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
895 {
896 	return mapping_remove(chains_mapping(chains), chain_mapping);
897 }
898 
mlx5_get_chain_for_tag(struct mlx5_fs_chains * chains,u32 tag,u32 * chain)899 int mlx5_get_chain_for_tag(struct mlx5_fs_chains *chains, u32 tag,
900 			   u32 *chain)
901 {
902 	int err;
903 
904 	err = mapping_find(chains_mapping(chains), tag, chain);
905 	if (err) {
906 		mlx5_core_warn(chains->dev, "Can't find chain for tag: %d\n", tag);
907 		return -ENOENT;
908 	}
909 
910 	return 0;
911 }
912