1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2020 Mellanox Technologies.
3
4 #include <linux/mlx5/driver.h>
5 #include <linux/mlx5/mlx5_ifc.h>
6 #include <linux/mlx5/fs.h>
7
8 #include "lib/fs_chains.h"
9 #include "en/mapping.h"
10 #include "mlx5_core.h"
11 #include "fs_core.h"
12 #include "eswitch.h"
13 #include "en.h"
14 #include "en_tc.h"
15
16 #define chains_lock(chains) ((chains)->lock)
17 #define chains_ht(chains) ((chains)->chains_ht)
18 #define chains_mapping(chains) ((chains)->chains_mapping)
19 #define prios_ht(chains) ((chains)->prios_ht)
20 #define ft_pool_left(chains) ((chains)->ft_left)
21 #define tc_default_ft(chains) ((chains)->tc_default_ft)
22 #define tc_end_ft(chains) ((chains)->tc_end_ft)
23 #define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
24 FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
25
26 /* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
27 * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
28 * for each flow table pool. We can allocate up to 16M of each pool,
29 * and we keep track of how much we used via get_next_avail_sz_from_pool.
30 * Firmware doesn't report any of this for now.
31 * ESW_POOL is expected to be sorted from large to small and match firmware
32 * pools.
33 */
34 #define FT_SIZE (16 * 1024 * 1024)
35 static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
36 1 * 1024 * 1024,
37 64 * 1024,
38 128 };
39 #define FT_TBL_SZ (64 * 1024)
40
41 struct mlx5_fs_chains {
42 struct mlx5_core_dev *dev;
43
44 struct rhashtable chains_ht;
45 struct rhashtable prios_ht;
46 /* Protects above chains_ht and prios_ht */
47 struct mutex lock;
48
49 struct mlx5_flow_table *tc_default_ft;
50 struct mlx5_flow_table *tc_end_ft;
51 struct mapping_ctx *chains_mapping;
52
53 enum mlx5_flow_namespace_type ns;
54 u32 group_num;
55 u32 flags;
56
57 int ft_left[ARRAY_SIZE(FT_POOLS)];
58 };
59
60 struct fs_chain {
61 struct rhash_head node;
62
63 u32 chain;
64
65 int ref;
66 int id;
67
68 struct mlx5_fs_chains *chains;
69 struct list_head prios_list;
70 struct mlx5_flow_handle *restore_rule;
71 struct mlx5_modify_hdr *miss_modify_hdr;
72 };
73
74 struct prio_key {
75 u32 chain;
76 u32 prio;
77 u32 level;
78 };
79
80 struct prio {
81 struct rhash_head node;
82 struct list_head list;
83
84 struct prio_key key;
85
86 int ref;
87
88 struct fs_chain *chain;
89 struct mlx5_flow_table *ft;
90 struct mlx5_flow_table *next_ft;
91 struct mlx5_flow_group *miss_group;
92 struct mlx5_flow_handle *miss_rule;
93 };
94
95 static const struct rhashtable_params chain_params = {
96 .head_offset = offsetof(struct fs_chain, node),
97 .key_offset = offsetof(struct fs_chain, chain),
98 .key_len = sizeof_field(struct fs_chain, chain),
99 .automatic_shrinking = true,
100 };
101
102 static const struct rhashtable_params prio_params = {
103 .head_offset = offsetof(struct prio, node),
104 .key_offset = offsetof(struct prio, key),
105 .key_len = sizeof_field(struct prio, key),
106 .automatic_shrinking = true,
107 };
108
mlx5_chains_prios_supported(struct mlx5_fs_chains * chains)109 bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
110 {
111 return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
112 }
113
mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains * chains)114 static bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
115 {
116 return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
117 }
118
mlx5_chains_backwards_supported(struct mlx5_fs_chains * chains)119 bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
120 {
121 return mlx5_chains_prios_supported(chains) &&
122 mlx5_chains_ignore_flow_level_supported(chains);
123 }
124
mlx5_chains_get_chain_range(struct mlx5_fs_chains * chains)125 u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
126 {
127 if (!mlx5_chains_prios_supported(chains))
128 return 1;
129
130 if (mlx5_chains_ignore_flow_level_supported(chains))
131 return UINT_MAX - 1;
132
133 /* We should get here only for eswitch case */
134 return FDB_TC_MAX_CHAIN;
135 }
136
mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains * chains)137 u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
138 {
139 return mlx5_chains_get_chain_range(chains) + 1;
140 }
141
mlx5_chains_get_prio_range(struct mlx5_fs_chains * chains)142 u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
143 {
144 if (!mlx5_chains_prios_supported(chains))
145 return 1;
146
147 if (mlx5_chains_ignore_flow_level_supported(chains))
148 return UINT_MAX;
149
150 /* We should get here only for eswitch case */
151 return FDB_TC_MAX_PRIO;
152 }
153
mlx5_chains_get_level_range(struct mlx5_fs_chains * chains)154 static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
155 {
156 if (mlx5_chains_ignore_flow_level_supported(chains))
157 return UINT_MAX;
158
159 /* Same value for FDB and NIC RX tables */
160 return FDB_TC_LEVELS_PER_PRIO;
161 }
162
163 void
mlx5_chains_set_end_ft(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)164 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
165 struct mlx5_flow_table *ft)
166 {
167 tc_end_ft(chains) = ft;
168 }
169
170 #define POOL_NEXT_SIZE 0
171 static int
mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains * chains,int desired_size)172 mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains,
173 int desired_size)
174 {
175 int i, found_i = -1;
176
177 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
178 if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) {
179 found_i = i;
180 if (desired_size != POOL_NEXT_SIZE)
181 break;
182 }
183 }
184
185 if (found_i != -1) {
186 --ft_pool_left(chains)[found_i];
187 return FT_POOLS[found_i];
188 }
189
190 return 0;
191 }
192
193 static void
mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains * chains,int sz)194 mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz)
195 {
196 int i;
197
198 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
199 if (sz == FT_POOLS[i]) {
200 ++ft_pool_left(chains)[i];
201 return;
202 }
203 }
204
205 WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
206 }
207
208 static void
mlx5_chains_init_sz_pool(struct mlx5_fs_chains * chains,u32 ft_max)209 mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max)
210 {
211 int i;
212
213 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
214 ft_pool_left(chains)[i] =
215 FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0;
216 }
217
218 static struct mlx5_flow_table *
mlx5_chains_create_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)219 mlx5_chains_create_table(struct mlx5_fs_chains *chains,
220 u32 chain, u32 prio, u32 level)
221 {
222 struct mlx5_flow_table_attr ft_attr = {};
223 struct mlx5_flow_namespace *ns;
224 struct mlx5_flow_table *ft;
225 int sz;
226
227 if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
228 ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
229 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
230
231 sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
232 mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) :
233 mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE);
234 if (!sz)
235 return ERR_PTR(-ENOSPC);
236 ft_attr.max_fte = sz;
237
238 /* We use tc_default_ft(chains) as the table's next_ft till
239 * ignore_flow_level is allowed on FT creation and not just for FTEs.
240 * Instead caller should add an explicit miss rule if needed.
241 */
242 ft_attr.next_ft = tc_default_ft(chains);
243
244 /* The root table(chain 0, prio 1, level 0) is required to be
245 * connected to the previous fs_core managed prio.
246 * We always create it, as a managed table, in order to align with
247 * fs_core logic.
248 */
249 if (!mlx5_chains_ignore_flow_level_supported(chains) ||
250 (chain == 0 && prio == 1 && level == 0)) {
251 ft_attr.level = level;
252 ft_attr.prio = prio - 1;
253 ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
254 mlx5_get_fdb_sub_ns(chains->dev, chain) :
255 mlx5_get_flow_namespace(chains->dev, chains->ns);
256 } else {
257 ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
258 ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
259 /* Firmware doesn't allow us to create another level 0 table,
260 * so we create all unmanaged tables as level 1.
261 *
262 * To connect them, we use explicit miss rules with
263 * ignore_flow_level. Caller is responsible to create
264 * these rules (if needed).
265 */
266 ft_attr.level = 1;
267 ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
268 }
269
270 ft_attr.autogroup.num_reserved_entries = 2;
271 ft_attr.autogroup.max_num_groups = chains->group_num;
272 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
273 if (IS_ERR(ft)) {
274 mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
275 (int)PTR_ERR(ft), chain, prio, level, sz);
276 mlx5_chains_put_sz_to_pool(chains, sz);
277 return ft;
278 }
279
280 return ft;
281 }
282
283 static void
mlx5_chains_destroy_table(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)284 mlx5_chains_destroy_table(struct mlx5_fs_chains *chains,
285 struct mlx5_flow_table *ft)
286 {
287 mlx5_chains_put_sz_to_pool(chains, ft->max_fte);
288 mlx5_destroy_flow_table(ft);
289 }
290
291 static int
create_chain_restore(struct fs_chain * chain)292 create_chain_restore(struct fs_chain *chain)
293 {
294 struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch;
295 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
296 struct mlx5_fs_chains *chains = chain->chains;
297 enum mlx5e_tc_attr_to_reg chain_to_reg;
298 struct mlx5_modify_hdr *mod_hdr;
299 u32 index;
300 int err;
301
302 if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
303 !mlx5_chains_prios_supported(chains))
304 return 0;
305
306 err = mapping_add(chains_mapping(chains), &chain->chain, &index);
307 if (err)
308 return err;
309 if (index == MLX5_FS_DEFAULT_FLOW_TAG) {
310 /* we got the special default flow tag id, so we won't know
311 * if we actually marked the packet with the restore rule
312 * we create.
313 *
314 * This case isn't possible with MLX5_FS_DEFAULT_FLOW_TAG = 0.
315 */
316 err = mapping_add(chains_mapping(chains),
317 &chain->chain, &index);
318 mapping_remove(chains_mapping(chains),
319 MLX5_FS_DEFAULT_FLOW_TAG);
320 if (err)
321 return err;
322 }
323
324 chain->id = index;
325
326 if (chains->ns == MLX5_FLOW_NAMESPACE_FDB) {
327 chain_to_reg = CHAIN_TO_REG;
328 chain->restore_rule = esw_add_restore_rule(esw, chain->id);
329 if (IS_ERR(chain->restore_rule)) {
330 err = PTR_ERR(chain->restore_rule);
331 goto err_rule;
332 }
333 } else if (chains->ns == MLX5_FLOW_NAMESPACE_KERNEL) {
334 /* For NIC RX we don't need a restore rule
335 * since we write the metadata to reg_b
336 * that is passed to SW directly.
337 */
338 chain_to_reg = NIC_CHAIN_TO_REG;
339 } else {
340 err = -EINVAL;
341 goto err_rule;
342 }
343
344 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
345 MLX5_SET(set_action_in, modact, field,
346 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield);
347 MLX5_SET(set_action_in, modact, offset,
348 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8);
349 MLX5_SET(set_action_in, modact, length,
350 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8);
351 MLX5_SET(set_action_in, modact, data, chain->id);
352 mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns,
353 1, modact);
354 if (IS_ERR(mod_hdr)) {
355 err = PTR_ERR(mod_hdr);
356 goto err_mod_hdr;
357 }
358 chain->miss_modify_hdr = mod_hdr;
359
360 return 0;
361
362 err_mod_hdr:
363 if (!IS_ERR_OR_NULL(chain->restore_rule))
364 mlx5_del_flow_rules(chain->restore_rule);
365 err_rule:
366 /* Datapath can't find this mapping, so we can safely remove it */
367 mapping_remove(chains_mapping(chains), chain->id);
368 return err;
369 }
370
destroy_chain_restore(struct fs_chain * chain)371 static void destroy_chain_restore(struct fs_chain *chain)
372 {
373 struct mlx5_fs_chains *chains = chain->chains;
374
375 if (!chain->miss_modify_hdr)
376 return;
377
378 if (chain->restore_rule)
379 mlx5_del_flow_rules(chain->restore_rule);
380
381 mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr);
382 mapping_remove(chains_mapping(chains), chain->id);
383 }
384
385 static struct fs_chain *
mlx5_chains_create_chain(struct mlx5_fs_chains * chains,u32 chain)386 mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
387 {
388 struct fs_chain *chain_s = NULL;
389 int err;
390
391 chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
392 if (!chain_s)
393 return ERR_PTR(-ENOMEM);
394
395 chain_s->chains = chains;
396 chain_s->chain = chain;
397 INIT_LIST_HEAD(&chain_s->prios_list);
398
399 err = create_chain_restore(chain_s);
400 if (err)
401 goto err_restore;
402
403 err = rhashtable_insert_fast(&chains_ht(chains), &chain_s->node,
404 chain_params);
405 if (err)
406 goto err_insert;
407
408 return chain_s;
409
410 err_insert:
411 destroy_chain_restore(chain_s);
412 err_restore:
413 kvfree(chain_s);
414 return ERR_PTR(err);
415 }
416
417 static void
mlx5_chains_destroy_chain(struct fs_chain * chain)418 mlx5_chains_destroy_chain(struct fs_chain *chain)
419 {
420 struct mlx5_fs_chains *chains = chain->chains;
421
422 rhashtable_remove_fast(&chains_ht(chains), &chain->node,
423 chain_params);
424
425 destroy_chain_restore(chain);
426 kvfree(chain);
427 }
428
429 static struct fs_chain *
mlx5_chains_get_chain(struct mlx5_fs_chains * chains,u32 chain)430 mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
431 {
432 struct fs_chain *chain_s;
433
434 chain_s = rhashtable_lookup_fast(&chains_ht(chains), &chain,
435 chain_params);
436 if (!chain_s) {
437 chain_s = mlx5_chains_create_chain(chains, chain);
438 if (IS_ERR(chain_s))
439 return chain_s;
440 }
441
442 chain_s->ref++;
443
444 return chain_s;
445 }
446
447 static struct mlx5_flow_handle *
mlx5_chains_add_miss_rule(struct fs_chain * chain,struct mlx5_flow_table * ft,struct mlx5_flow_table * next_ft)448 mlx5_chains_add_miss_rule(struct fs_chain *chain,
449 struct mlx5_flow_table *ft,
450 struct mlx5_flow_table *next_ft)
451 {
452 struct mlx5_fs_chains *chains = chain->chains;
453 struct mlx5_flow_destination dest = {};
454 struct mlx5_flow_act act = {};
455
456 act.flags = FLOW_ACT_NO_APPEND;
457 if (mlx5_chains_ignore_flow_level_supported(chain->chains))
458 act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
459
460 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
461 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
462 dest.ft = next_ft;
463
464 if (next_ft == tc_end_ft(chains) &&
465 chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
466 mlx5_chains_prios_supported(chains)) {
467 act.modify_hdr = chain->miss_modify_hdr;
468 act.action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
469 }
470
471 return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
472 }
473
474 static int
mlx5_chains_update_prio_prevs(struct prio * prio,struct mlx5_flow_table * next_ft)475 mlx5_chains_update_prio_prevs(struct prio *prio,
476 struct mlx5_flow_table *next_ft)
477 {
478 struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
479 struct fs_chain *chain = prio->chain;
480 struct prio *pos;
481 int n = 0, err;
482
483 if (prio->key.level)
484 return 0;
485
486 /* Iterate in reverse order until reaching the level 0 rule of
487 * the previous priority, adding all the miss rules first, so we can
488 * revert them if any of them fails.
489 */
490 pos = prio;
491 list_for_each_entry_continue_reverse(pos,
492 &chain->prios_list,
493 list) {
494 miss_rules[n] = mlx5_chains_add_miss_rule(chain,
495 pos->ft,
496 next_ft);
497 if (IS_ERR(miss_rules[n])) {
498 err = PTR_ERR(miss_rules[n]);
499 goto err_prev_rule;
500 }
501
502 n++;
503 if (!pos->key.level)
504 break;
505 }
506
507 /* Success, delete old miss rules, and update the pointers. */
508 n = 0;
509 pos = prio;
510 list_for_each_entry_continue_reverse(pos,
511 &chain->prios_list,
512 list) {
513 mlx5_del_flow_rules(pos->miss_rule);
514
515 pos->miss_rule = miss_rules[n];
516 pos->next_ft = next_ft;
517
518 n++;
519 if (!pos->key.level)
520 break;
521 }
522
523 return 0;
524
525 err_prev_rule:
526 while (--n >= 0)
527 mlx5_del_flow_rules(miss_rules[n]);
528
529 return err;
530 }
531
532 static void
mlx5_chains_put_chain(struct fs_chain * chain)533 mlx5_chains_put_chain(struct fs_chain *chain)
534 {
535 if (--chain->ref == 0)
536 mlx5_chains_destroy_chain(chain);
537 }
538
539 static struct prio *
mlx5_chains_create_prio(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)540 mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
541 u32 chain, u32 prio, u32 level)
542 {
543 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
544 struct mlx5_flow_handle *miss_rule = NULL;
545 struct mlx5_flow_group *miss_group;
546 struct mlx5_flow_table *next_ft;
547 struct mlx5_flow_table *ft;
548 struct prio *prio_s = NULL;
549 struct fs_chain *chain_s;
550 struct list_head *pos;
551 u32 *flow_group_in;
552 int err;
553
554 chain_s = mlx5_chains_get_chain(chains, chain);
555 if (IS_ERR(chain_s))
556 return ERR_CAST(chain_s);
557
558 prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
559 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
560 if (!prio_s || !flow_group_in) {
561 err = -ENOMEM;
562 goto err_alloc;
563 }
564
565 /* Chain's prio list is sorted by prio and level.
566 * And all levels of some prio point to the next prio's level 0.
567 * Example list (prio, level):
568 * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
569 * In hardware, we will we have the following pointers:
570 * (3,0) -> (5,0) -> (7,0) -> Slow path
571 * (3,1) -> (5,0)
572 * (5,1) -> (7,0)
573 * (6,1) -> (7,0)
574 */
575
576 /* Default miss for each chain: */
577 next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
578 tc_default_ft(chains) :
579 tc_end_ft(chains);
580 list_for_each(pos, &chain_s->prios_list) {
581 struct prio *p = list_entry(pos, struct prio, list);
582
583 /* exit on first pos that is larger */
584 if (prio < p->key.prio || (prio == p->key.prio &&
585 level < p->key.level)) {
586 /* Get next level 0 table */
587 next_ft = p->key.level == 0 ? p->ft : p->next_ft;
588 break;
589 }
590 }
591
592 ft = mlx5_chains_create_table(chains, chain, prio, level);
593 if (IS_ERR(ft)) {
594 err = PTR_ERR(ft);
595 goto err_create;
596 }
597
598 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
599 ft->max_fte - 2);
600 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
601 ft->max_fte - 1);
602 miss_group = mlx5_create_flow_group(ft, flow_group_in);
603 if (IS_ERR(miss_group)) {
604 err = PTR_ERR(miss_group);
605 goto err_group;
606 }
607
608 /* Add miss rule to next_ft */
609 miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
610 if (IS_ERR(miss_rule)) {
611 err = PTR_ERR(miss_rule);
612 goto err_miss_rule;
613 }
614
615 prio_s->miss_group = miss_group;
616 prio_s->miss_rule = miss_rule;
617 prio_s->next_ft = next_ft;
618 prio_s->chain = chain_s;
619 prio_s->key.chain = chain;
620 prio_s->key.prio = prio;
621 prio_s->key.level = level;
622 prio_s->ft = ft;
623
624 err = rhashtable_insert_fast(&prios_ht(chains), &prio_s->node,
625 prio_params);
626 if (err)
627 goto err_insert;
628
629 list_add(&prio_s->list, pos->prev);
630
631 /* Table is ready, connect it */
632 err = mlx5_chains_update_prio_prevs(prio_s, ft);
633 if (err)
634 goto err_update;
635
636 kvfree(flow_group_in);
637 return prio_s;
638
639 err_update:
640 list_del(&prio_s->list);
641 rhashtable_remove_fast(&prios_ht(chains), &prio_s->node,
642 prio_params);
643 err_insert:
644 mlx5_del_flow_rules(miss_rule);
645 err_miss_rule:
646 mlx5_destroy_flow_group(miss_group);
647 err_group:
648 mlx5_chains_destroy_table(chains, ft);
649 err_create:
650 err_alloc:
651 kvfree(prio_s);
652 kvfree(flow_group_in);
653 mlx5_chains_put_chain(chain_s);
654 return ERR_PTR(err);
655 }
656
657 static void
mlx5_chains_destroy_prio(struct mlx5_fs_chains * chains,struct prio * prio)658 mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
659 struct prio *prio)
660 {
661 struct fs_chain *chain = prio->chain;
662
663 WARN_ON(mlx5_chains_update_prio_prevs(prio,
664 prio->next_ft));
665
666 list_del(&prio->list);
667 rhashtable_remove_fast(&prios_ht(chains), &prio->node,
668 prio_params);
669 mlx5_del_flow_rules(prio->miss_rule);
670 mlx5_destroy_flow_group(prio->miss_group);
671 mlx5_chains_destroy_table(chains, prio->ft);
672 mlx5_chains_put_chain(chain);
673 kvfree(prio);
674 }
675
676 struct mlx5_flow_table *
mlx5_chains_get_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)677 mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
678 u32 level)
679 {
680 struct mlx5_flow_table *prev_fts;
681 struct prio *prio_s;
682 struct prio_key key;
683 int l = 0;
684
685 if ((chain > mlx5_chains_get_chain_range(chains) &&
686 chain != mlx5_chains_get_nf_ft_chain(chains)) ||
687 prio > mlx5_chains_get_prio_range(chains) ||
688 level > mlx5_chains_get_level_range(chains))
689 return ERR_PTR(-EOPNOTSUPP);
690
691 /* create earlier levels for correct fs_core lookup when
692 * connecting tables.
693 */
694 for (l = 0; l < level; l++) {
695 prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
696 if (IS_ERR(prev_fts)) {
697 prio_s = ERR_CAST(prev_fts);
698 goto err_get_prevs;
699 }
700 }
701
702 key.chain = chain;
703 key.prio = prio;
704 key.level = level;
705
706 mutex_lock(&chains_lock(chains));
707 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
708 prio_params);
709 if (!prio_s) {
710 prio_s = mlx5_chains_create_prio(chains, chain,
711 prio, level);
712 if (IS_ERR(prio_s))
713 goto err_create_prio;
714 }
715
716 ++prio_s->ref;
717 mutex_unlock(&chains_lock(chains));
718
719 return prio_s->ft;
720
721 err_create_prio:
722 mutex_unlock(&chains_lock(chains));
723 err_get_prevs:
724 while (--l >= 0)
725 mlx5_chains_put_table(chains, chain, prio, l);
726 return ERR_CAST(prio_s);
727 }
728
729 void
mlx5_chains_put_table(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)730 mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
731 u32 level)
732 {
733 struct prio *prio_s;
734 struct prio_key key;
735
736 key.chain = chain;
737 key.prio = prio;
738 key.level = level;
739
740 mutex_lock(&chains_lock(chains));
741 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
742 prio_params);
743 if (!prio_s)
744 goto err_get_prio;
745
746 if (--prio_s->ref == 0)
747 mlx5_chains_destroy_prio(chains, prio_s);
748 mutex_unlock(&chains_lock(chains));
749
750 while (level-- > 0)
751 mlx5_chains_put_table(chains, chain, prio, level);
752
753 return;
754
755 err_get_prio:
756 mutex_unlock(&chains_lock(chains));
757 WARN_ONCE(1,
758 "Couldn't find table: (chain: %d prio: %d level: %d)",
759 chain, prio, level);
760 }
761
762 struct mlx5_flow_table *
mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains * chains)763 mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
764 {
765 return tc_end_ft(chains);
766 }
767
768 struct mlx5_flow_table *
mlx5_chains_create_global_table(struct mlx5_fs_chains * chains)769 mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
770 {
771 u32 chain, prio, level;
772 int err;
773
774 if (!mlx5_chains_ignore_flow_level_supported(chains)) {
775 err = -EOPNOTSUPP;
776
777 mlx5_core_warn(chains->dev,
778 "Couldn't create global flow table, ignore_flow_level not supported.");
779 goto err_ignore;
780 }
781
782 chain = mlx5_chains_get_chain_range(chains),
783 prio = mlx5_chains_get_prio_range(chains);
784 level = mlx5_chains_get_level_range(chains);
785
786 return mlx5_chains_create_table(chains, chain, prio, level);
787
788 err_ignore:
789 return ERR_PTR(err);
790 }
791
792 void
mlx5_chains_destroy_global_table(struct mlx5_fs_chains * chains,struct mlx5_flow_table * ft)793 mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
794 struct mlx5_flow_table *ft)
795 {
796 mlx5_chains_destroy_table(chains, ft);
797 }
798
799 static struct mlx5_fs_chains *
mlx5_chains_init(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)800 mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
801 {
802 struct mlx5_fs_chains *chains_priv;
803 struct mapping_ctx *mapping;
804 u32 max_flow_counter;
805 int err;
806
807 chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
808 if (!chains_priv)
809 return ERR_PTR(-ENOMEM);
810
811 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
812 MLX5_CAP_GEN(dev, max_flow_counter_15_0);
813
814 mlx5_core_dbg(dev,
815 "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
816 max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
817
818 chains_priv->dev = dev;
819 chains_priv->flags = attr->flags;
820 chains_priv->ns = attr->ns;
821 chains_priv->group_num = attr->max_grp_num;
822 tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
823
824 mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
825 mlx5_chains_get_chain_range(chains_priv),
826 mlx5_chains_get_prio_range(chains_priv));
827
828 mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz);
829
830 err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
831 if (err)
832 goto init_chains_ht_err;
833
834 err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
835 if (err)
836 goto init_prios_ht_err;
837
838 mapping = mapping_create(sizeof(u32), attr->max_restore_tag,
839 true);
840 if (IS_ERR(mapping)) {
841 err = PTR_ERR(mapping);
842 goto mapping_err;
843 }
844 chains_mapping(chains_priv) = mapping;
845
846 mutex_init(&chains_lock(chains_priv));
847
848 return chains_priv;
849
850 mapping_err:
851 rhashtable_destroy(&prios_ht(chains_priv));
852 init_prios_ht_err:
853 rhashtable_destroy(&chains_ht(chains_priv));
854 init_chains_ht_err:
855 kfree(chains_priv);
856 return ERR_PTR(err);
857 }
858
859 static void
mlx5_chains_cleanup(struct mlx5_fs_chains * chains)860 mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
861 {
862 mutex_destroy(&chains_lock(chains));
863 mapping_destroy(chains_mapping(chains));
864 rhashtable_destroy(&prios_ht(chains));
865 rhashtable_destroy(&chains_ht(chains));
866
867 kfree(chains);
868 }
869
870 struct mlx5_fs_chains *
mlx5_chains_create(struct mlx5_core_dev * dev,struct mlx5_chains_attr * attr)871 mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
872 {
873 struct mlx5_fs_chains *chains;
874
875 chains = mlx5_chains_init(dev, attr);
876
877 return chains;
878 }
879
880 void
mlx5_chains_destroy(struct mlx5_fs_chains * chains)881 mlx5_chains_destroy(struct mlx5_fs_chains *chains)
882 {
883 mlx5_chains_cleanup(chains);
884 }
885
886 int
mlx5_chains_get_chain_mapping(struct mlx5_fs_chains * chains,u32 chain,u32 * chain_mapping)887 mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
888 u32 *chain_mapping)
889 {
890 return mapping_add(chains_mapping(chains), &chain, chain_mapping);
891 }
892
893 int
mlx5_chains_put_chain_mapping(struct mlx5_fs_chains * chains,u32 chain_mapping)894 mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
895 {
896 return mapping_remove(chains_mapping(chains), chain_mapping);
897 }
898
mlx5_get_chain_for_tag(struct mlx5_fs_chains * chains,u32 tag,u32 * chain)899 int mlx5_get_chain_for_tag(struct mlx5_fs_chains *chains, u32 tag,
900 u32 *chain)
901 {
902 int err;
903
904 err = mapping_find(chains_mapping(chains), tag, chain);
905 if (err) {
906 mlx5_core_warn(chains->dev, "Can't find chain for tag: %d\n", tag);
907 return -ENOENT;
908 }
909
910 return 0;
911 }
912