1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/devcom.h"
47 #include "lib/eq.h"
48 #include "lib/fs_chains.h"
49 #include "en_tc.h"
50 #include "en/mapping.h"
51 #include "devlink.h"
52 #include "lag/lag.h"
53
54 #define mlx5_esw_for_each_rep(esw, i, rep) \
55 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
56
57 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \
58 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF)
59
60 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \
61 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \
62 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF)
63
64 /* There are two match-all miss flows, one for unicast dst mac and
65 * one for multicast.
66 */
67 #define MLX5_ESW_MISS_FLOWS (2)
68 #define UPLINK_REP_INDEX 0
69
70 #define MLX5_ESW_VPORT_TBL_SIZE 128
71 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
72
73 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
74
75 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
76 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
77 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
78 .flags = 0,
79 };
80
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)81 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
82 u16 vport_num)
83 {
84 return xa_load(&esw->offloads.vport_reps, vport_num);
85 }
86
87 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)88 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
89 struct mlx5_flow_spec *spec,
90 struct mlx5_esw_flow_attr *attr)
91 {
92 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
93 return;
94
95 if (attr->int_port) {
96 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
97
98 return;
99 }
100
101 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
102 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
103 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
104 }
105
106 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
107 * are not needed as well in the following process. So clear them all for simplicity.
108 */
109 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)110 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
111 {
112 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
113 void *misc2;
114
115 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
116 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
117
118 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
119 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
120
121 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
122 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
123 }
124 }
125
126 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)127 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
128 struct mlx5_flow_spec *spec,
129 struct mlx5_flow_attr *attr,
130 struct mlx5_eswitch *src_esw,
131 u16 vport)
132 {
133 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
134 u32 metadata;
135 void *misc2;
136 void *misc;
137
138 /* Use metadata matching because vport is not represented by single
139 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
140 */
141 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
142 if (mlx5_esw_indir_table_decap_vport(attr))
143 vport = mlx5_esw_indir_table_decap_vport(attr);
144
145 if (!attr->chain && esw_attr && esw_attr->int_port)
146 metadata =
147 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
148 else
149 metadata =
150 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
151
152 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
153 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
154
155 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
156 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
157 mlx5_eswitch_get_vport_metadata_mask());
158
159 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
160 } else {
161 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
162 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
163
164 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
165 MLX5_SET(fte_match_set_misc, misc,
166 source_eswitch_owner_vhca_id,
167 MLX5_CAP_GEN(src_esw->dev, vhca_id));
168
169 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
170 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
171 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
172 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
173 source_eswitch_owner_vhca_id);
174
175 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
176 }
177 }
178
179 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)180 esw_setup_decap_indir(struct mlx5_eswitch *esw,
181 struct mlx5_flow_attr *attr)
182 {
183 struct mlx5_flow_table *ft;
184
185 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
186 return -EOPNOTSUPP;
187
188 ft = mlx5_esw_indir_table_get(esw, attr,
189 mlx5_esw_indir_table_decap_vport(attr), true);
190 return PTR_ERR_OR_ZERO(ft);
191 }
192
193 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)194 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
195 struct mlx5_flow_attr *attr)
196 {
197 if (mlx5_esw_indir_table_decap_vport(attr))
198 mlx5_esw_indir_table_put(esw,
199 mlx5_esw_indir_table_decap_vport(attr),
200 true);
201 }
202
203 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)204 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
205 struct mlx5_flow_act *flow_act,
206 u32 sampler_id,
207 int i)
208 {
209 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
210 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
211 dest[i].sampler_id = sampler_id;
212
213 return 0;
214 }
215
216 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int i)217 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
218 struct mlx5_flow_act *flow_act,
219 struct mlx5_eswitch *esw,
220 struct mlx5_flow_attr *attr,
221 int i)
222 {
223 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
224 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
225 dest[i].ft = attr->dest_ft;
226
227 if (mlx5_esw_indir_table_decap_vport(attr))
228 return esw_setup_decap_indir(esw, attr);
229 return 0;
230 }
231
232 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)233 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
234 struct mlx5_fs_chains *chains, int i)
235 {
236 if (mlx5_chains_ignore_flow_level_supported(chains))
237 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
238 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
239 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
240 }
241
242 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)243 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
244 struct mlx5_eswitch *esw, int i)
245 {
246 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
247 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
248 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
249 dest[i].ft = esw->fdb_table.offloads.slow_fdb;
250 }
251
252 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)253 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
254 struct mlx5_flow_act *flow_act,
255 struct mlx5_fs_chains *chains,
256 u32 chain, u32 prio, u32 level,
257 int i)
258 {
259 struct mlx5_flow_table *ft;
260
261 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
262 ft = mlx5_chains_get_table(chains, chain, prio, level);
263 if (IS_ERR(ft))
264 return PTR_ERR(ft);
265
266 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
267 dest[i].ft = ft;
268 return 0;
269 }
270
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)271 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
272 int from, int to)
273 {
274 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
275 struct mlx5_fs_chains *chains = esw_chains(esw);
276 int i;
277
278 for (i = from; i < to; i++)
279 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
280 mlx5_chains_put_table(chains, 0, 1, 0);
281 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
282 esw_attr->dests[i].mdev))
283 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].rep->vport,
284 false);
285 }
286
287 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)288 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
289 {
290 int i;
291
292 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
293 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
294 return true;
295 return false;
296 }
297
298 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)299 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
300 struct mlx5_flow_act *flow_act,
301 struct mlx5_eswitch *esw,
302 struct mlx5_fs_chains *chains,
303 struct mlx5_flow_attr *attr,
304 int *i)
305 {
306 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
307 int err;
308
309 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
310 return -EOPNOTSUPP;
311
312 /* flow steering cannot handle more than one dest with the same ft
313 * in a single flow
314 */
315 if (esw_attr->out_count - esw_attr->split_count > 1)
316 return -EOPNOTSUPP;
317
318 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
319 if (err)
320 return err;
321
322 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
323 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
324 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
325 }
326 (*i)++;
327
328 return 0;
329 }
330
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)331 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
332 struct mlx5_flow_attr *attr)
333 {
334 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
335
336 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
337 }
338
339 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)340 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
341 {
342 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
343 bool result = false;
344 int i;
345
346 /* Indirect table is supported only for flows with in_port uplink
347 * and the destination is vport on the same eswitch as the uplink,
348 * return false in case at least one of destinations doesn't meet
349 * this criteria.
350 */
351 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
352 if (esw_attr->dests[i].rep &&
353 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
354 esw_attr->dests[i].mdev)) {
355 result = true;
356 } else {
357 result = false;
358 break;
359 }
360 }
361 return result;
362 }
363
364 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,bool ignore_flow_lvl,int * i)365 esw_setup_indir_table(struct mlx5_flow_destination *dest,
366 struct mlx5_flow_act *flow_act,
367 struct mlx5_eswitch *esw,
368 struct mlx5_flow_attr *attr,
369 bool ignore_flow_lvl,
370 int *i)
371 {
372 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
373 int j, err;
374
375 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
376 return -EOPNOTSUPP;
377
378 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
379 if (ignore_flow_lvl)
380 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
381 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
382
383 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
384 esw_attr->dests[j].rep->vport, false);
385 if (IS_ERR(dest[*i].ft)) {
386 err = PTR_ERR(dest[*i].ft);
387 goto err_indir_tbl_get;
388 }
389 }
390
391 if (mlx5_esw_indir_table_decap_vport(attr)) {
392 err = esw_setup_decap_indir(esw, attr);
393 if (err)
394 goto err_indir_tbl_get;
395 }
396
397 return 0;
398
399 err_indir_tbl_get:
400 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
401 return err;
402 }
403
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)404 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
405 {
406 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
407
408 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
409 esw_cleanup_decap_indir(esw, attr);
410 }
411
412 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)413 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
414 {
415 mlx5_chains_put_table(chains, chain, prio, level);
416 }
417
418 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)419 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
420 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
421 int attr_idx, int dest_idx, bool pkt_reformat)
422 {
423 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
424 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport;
425 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
426 dest[dest_idx].vport.vhca_id =
427 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
428 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
429 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
430 mlx5_lag_mpesw_is_activated(esw->dev))
431 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
432 }
433 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
434 if (pkt_reformat) {
435 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
436 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
437 }
438 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
439 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
440 }
441 }
442
443 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)444 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
445 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
446 int i)
447 {
448 int j;
449
450 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
451 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
452 return i;
453 }
454
455 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)456 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
457 {
458 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
459 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
460 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
461 }
462
463 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)464 esw_setup_dests(struct mlx5_flow_destination *dest,
465 struct mlx5_flow_act *flow_act,
466 struct mlx5_eswitch *esw,
467 struct mlx5_flow_attr *attr,
468 struct mlx5_flow_spec *spec,
469 int *i)
470 {
471 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
472 struct mlx5_fs_chains *chains = esw_chains(esw);
473 int err = 0;
474
475 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
476 esw_src_port_rewrite_supported(esw))
477 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
478
479 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE &&
480 !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) {
481 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
482 (*i)++;
483 } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
484 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
485 (*i)++;
486 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
487 esw_setup_accept_dest(dest, flow_act, chains, *i);
488 (*i)++;
489 } else if (esw_is_indir_table(esw, attr)) {
490 err = esw_setup_indir_table(dest, flow_act, esw, attr, true, i);
491 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
492 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
493 } else {
494 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
495
496 if (attr->dest_ft) {
497 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i);
498 (*i)++;
499 } else if (attr->dest_chain) {
500 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
501 1, 0, *i);
502 (*i)++;
503 }
504 }
505
506 return err;
507 }
508
509 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)510 esw_cleanup_dests(struct mlx5_eswitch *esw,
511 struct mlx5_flow_attr *attr)
512 {
513 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
514 struct mlx5_fs_chains *chains = esw_chains(esw);
515
516 if (attr->dest_ft) {
517 esw_cleanup_decap_indir(esw, attr);
518 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
519 if (attr->dest_chain)
520 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
521 else if (esw_is_indir_table(esw, attr))
522 esw_cleanup_indir_table(esw, attr);
523 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
524 esw_cleanup_chain_src_port_rewrite(esw, attr);
525 }
526 }
527
528 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)529 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
530 {
531 struct mlx5e_flow_meter_handle *meter;
532
533 meter = attr->meter_attr.meter;
534 flow_act->exe_aso.type = attr->exe_aso_type;
535 flow_act->exe_aso.object_id = meter->obj_id;
536 flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
537 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
538 /* use metadata reg 5 for packet color */
539 flow_act->exe_aso.return_reg_id = 5;
540 }
541
542 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)543 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
544 struct mlx5_flow_spec *spec,
545 struct mlx5_flow_attr *attr)
546 {
547 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
548 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
549 struct mlx5_fs_chains *chains = esw_chains(esw);
550 bool split = !!(esw_attr->split_count);
551 struct mlx5_vport_tbl_attr fwd_attr;
552 struct mlx5_flow_destination *dest;
553 struct mlx5_flow_handle *rule;
554 struct mlx5_flow_table *fdb;
555 int i = 0;
556
557 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
558 return ERR_PTR(-EOPNOTSUPP);
559
560 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
561 if (!dest)
562 return ERR_PTR(-ENOMEM);
563
564 flow_act.action = attr->action;
565 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
566 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
567 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
568 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
569 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
570 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
571 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
572 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
573 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
574 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
575 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
576 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
577 }
578 }
579
580 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
581
582 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
583 int err;
584
585 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
586 if (err) {
587 rule = ERR_PTR(err);
588 goto err_create_goto_table;
589 }
590 }
591
592 if (esw_attr->decap_pkt_reformat)
593 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
594
595 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
596 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
597 dest[i].counter_id = mlx5_fc_id(attr->counter);
598 i++;
599 }
600
601 if (attr->outer_match_level != MLX5_MATCH_NONE)
602 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
603 if (attr->inner_match_level != MLX5_MATCH_NONE)
604 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
605
606 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
607 flow_act.modify_hdr = attr->modify_hdr;
608
609 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
610 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
611 esw_setup_meter(attr, &flow_act);
612
613 if (split) {
614 fwd_attr.chain = attr->chain;
615 fwd_attr.prio = attr->prio;
616 fwd_attr.vport = esw_attr->in_rep->vport;
617 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
618
619 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
620 } else {
621 if (attr->chain || attr->prio)
622 fdb = mlx5_chains_get_table(chains, attr->chain,
623 attr->prio, 0);
624 else
625 fdb = attr->ft;
626
627 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
628 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
629 esw_attr->in_mdev->priv.eswitch,
630 esw_attr->in_rep->vport);
631 }
632 if (IS_ERR(fdb)) {
633 rule = ERR_CAST(fdb);
634 goto err_esw_get;
635 }
636
637 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
638 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
639 &flow_act, dest, i);
640 else
641 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
642 if (IS_ERR(rule))
643 goto err_add_rule;
644 else
645 atomic64_inc(&esw->offloads.num_flows);
646
647 kfree(dest);
648 return rule;
649
650 err_add_rule:
651 if (split)
652 mlx5_esw_vporttbl_put(esw, &fwd_attr);
653 else if (attr->chain || attr->prio)
654 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
655 err_esw_get:
656 esw_cleanup_dests(esw, attr);
657 err_create_goto_table:
658 kfree(dest);
659 return rule;
660 }
661
662 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)663 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
664 struct mlx5_flow_spec *spec,
665 struct mlx5_flow_attr *attr)
666 {
667 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
668 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
669 struct mlx5_fs_chains *chains = esw_chains(esw);
670 struct mlx5_vport_tbl_attr fwd_attr;
671 struct mlx5_flow_destination *dest;
672 struct mlx5_flow_table *fast_fdb;
673 struct mlx5_flow_table *fwd_fdb;
674 struct mlx5_flow_handle *rule;
675 int i, err = 0;
676
677 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
678 if (!dest)
679 return ERR_PTR(-ENOMEM);
680
681 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
682 if (IS_ERR(fast_fdb)) {
683 rule = ERR_CAST(fast_fdb);
684 goto err_get_fast;
685 }
686
687 fwd_attr.chain = attr->chain;
688 fwd_attr.prio = attr->prio;
689 fwd_attr.vport = esw_attr->in_rep->vport;
690 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
691 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
692 if (IS_ERR(fwd_fdb)) {
693 rule = ERR_CAST(fwd_fdb);
694 goto err_get_fwd;
695 }
696
697 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
698 for (i = 0; i < esw_attr->split_count; i++) {
699 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
700 /* Source port rewrite (forward to ovs internal port or statck device) isn't
701 * supported in the rule of split action.
702 */
703 err = -EOPNOTSUPP;
704 else
705 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
706
707 if (err) {
708 rule = ERR_PTR(err);
709 goto err_chain_src_rewrite;
710 }
711 }
712 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
713 dest[i].ft = fwd_fdb;
714 i++;
715
716 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
717 esw_attr->in_mdev->priv.eswitch,
718 esw_attr->in_rep->vport);
719
720 if (attr->outer_match_level != MLX5_MATCH_NONE)
721 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
722
723 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
724 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
725
726 if (IS_ERR(rule)) {
727 i = esw_attr->split_count;
728 goto err_chain_src_rewrite;
729 }
730
731 atomic64_inc(&esw->offloads.num_flows);
732
733 kfree(dest);
734 return rule;
735 err_chain_src_rewrite:
736 mlx5_esw_vporttbl_put(esw, &fwd_attr);
737 err_get_fwd:
738 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
739 err_get_fast:
740 kfree(dest);
741 return rule;
742 }
743
744 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)745 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
746 struct mlx5_flow_handle *rule,
747 struct mlx5_flow_attr *attr,
748 bool fwd_rule)
749 {
750 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
751 struct mlx5_fs_chains *chains = esw_chains(esw);
752 bool split = (esw_attr->split_count > 0);
753 struct mlx5_vport_tbl_attr fwd_attr;
754 int i;
755
756 mlx5_del_flow_rules(rule);
757
758 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
759 /* unref the term table */
760 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
761 if (esw_attr->dests[i].termtbl)
762 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
763 }
764 }
765
766 atomic64_dec(&esw->offloads.num_flows);
767
768 if (fwd_rule || split) {
769 fwd_attr.chain = attr->chain;
770 fwd_attr.prio = attr->prio;
771 fwd_attr.vport = esw_attr->in_rep->vport;
772 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
773 }
774
775 if (fwd_rule) {
776 mlx5_esw_vporttbl_put(esw, &fwd_attr);
777 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
778 } else {
779 if (split)
780 mlx5_esw_vporttbl_put(esw, &fwd_attr);
781 else if (attr->chain || attr->prio)
782 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
783 esw_cleanup_dests(esw, attr);
784 }
785 }
786
787 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)788 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
789 struct mlx5_flow_handle *rule,
790 struct mlx5_flow_attr *attr)
791 {
792 __mlx5_eswitch_del_rule(esw, rule, attr, false);
793 }
794
795 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)796 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
797 struct mlx5_flow_handle *rule,
798 struct mlx5_flow_attr *attr)
799 {
800 __mlx5_eswitch_del_rule(esw, rule, attr, true);
801 }
802
esw_set_global_vlan_pop(struct mlx5_eswitch * esw,u8 val)803 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
804 {
805 struct mlx5_eswitch_rep *rep;
806 unsigned long i;
807 int err = 0;
808
809 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
810 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) {
811 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
812 continue;
813
814 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
815 if (err)
816 goto out;
817 }
818
819 out:
820 return err;
821 }
822
823 static struct mlx5_eswitch_rep *
esw_vlan_action_get_vport(struct mlx5_esw_flow_attr * attr,bool push,bool pop)824 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
825 {
826 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
827
828 in_rep = attr->in_rep;
829 out_rep = attr->dests[0].rep;
830
831 if (push)
832 vport = in_rep;
833 else if (pop)
834 vport = out_rep;
835 else
836 vport = in_rep;
837
838 return vport;
839 }
840
esw_add_vlan_action_check(struct mlx5_esw_flow_attr * attr,bool push,bool pop,bool fwd)841 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
842 bool push, bool pop, bool fwd)
843 {
844 struct mlx5_eswitch_rep *in_rep, *out_rep;
845
846 if ((push || pop) && !fwd)
847 goto out_notsupp;
848
849 in_rep = attr->in_rep;
850 out_rep = attr->dests[0].rep;
851
852 if (push && in_rep->vport == MLX5_VPORT_UPLINK)
853 goto out_notsupp;
854
855 if (pop && out_rep->vport == MLX5_VPORT_UPLINK)
856 goto out_notsupp;
857
858 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
859 if (!push && !pop && fwd)
860 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK)
861 goto out_notsupp;
862
863 /* protects against (1) setting rules with different vlans to push and
864 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
865 */
866 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0]))
867 goto out_notsupp;
868
869 return 0;
870
871 out_notsupp:
872 return -EOPNOTSUPP;
873 }
874
mlx5_eswitch_add_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)875 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
876 struct mlx5_flow_attr *attr)
877 {
878 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
879 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
880 struct mlx5_eswitch_rep *vport = NULL;
881 bool push, pop, fwd;
882 int err = 0;
883
884 /* nop if we're on the vlan push/pop non emulation mode */
885 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
886 return 0;
887
888 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
889 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
890 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
891 !attr->dest_chain);
892
893 mutex_lock(&esw->state_lock);
894
895 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd);
896 if (err)
897 goto unlock;
898
899 attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED;
900
901 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
902
903 if (!push && !pop && fwd) {
904 /* tracks VF --> wire rules without vlan push action */
905 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
906 vport->vlan_refcount++;
907 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
908 }
909
910 goto unlock;
911 }
912
913 if (!push && !pop)
914 goto unlock;
915
916 if (!(offloads->vlan_push_pop_refcount)) {
917 /* it's the 1st vlan rule, apply global vlan pop policy */
918 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
919 if (err)
920 goto out;
921 }
922 offloads->vlan_push_pop_refcount++;
923
924 if (push) {
925 if (vport->vlan_refcount)
926 goto skip_set_push;
927
928 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0],
929 0, SET_VLAN_INSERT | SET_VLAN_STRIP);
930 if (err)
931 goto out;
932 vport->vlan = esw_attr->vlan_vid[0];
933 skip_set_push:
934 vport->vlan_refcount++;
935 }
936 out:
937 if (!err)
938 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED;
939 unlock:
940 mutex_unlock(&esw->state_lock);
941 return err;
942 }
943
mlx5_eswitch_del_vlan_action(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)944 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
945 struct mlx5_flow_attr *attr)
946 {
947 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
948 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
949 struct mlx5_eswitch_rep *vport = NULL;
950 bool push, pop, fwd;
951 int err = 0;
952
953 /* nop if we're on the vlan push/pop non emulation mode */
954 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
955 return 0;
956
957 if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED))
958 return 0;
959
960 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
961 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
962 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
963
964 mutex_lock(&esw->state_lock);
965
966 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
967
968 if (!push && !pop && fwd) {
969 /* tracks VF --> wire rules without vlan push action */
970 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK)
971 vport->vlan_refcount--;
972
973 goto out;
974 }
975
976 if (push) {
977 vport->vlan_refcount--;
978 if (vport->vlan_refcount)
979 goto skip_unset_push;
980
981 vport->vlan = 0;
982 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
983 0, 0, SET_VLAN_STRIP);
984 if (err)
985 goto out;
986 }
987
988 skip_unset_push:
989 offloads->vlan_push_pop_refcount--;
990 if (offloads->vlan_push_pop_refcount)
991 goto out;
992
993 /* no more vlan rules, stop global vlan pop policy */
994 err = esw_set_global_vlan_pop(esw, 0);
995
996 out:
997 mutex_unlock(&esw->state_lock);
998 return err;
999 }
1000
1001 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)1002 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
1003 struct mlx5_eswitch *from_esw,
1004 struct mlx5_eswitch_rep *rep,
1005 u32 sqn)
1006 {
1007 struct mlx5_flow_act flow_act = {0};
1008 struct mlx5_flow_destination dest = {};
1009 struct mlx5_flow_handle *flow_rule;
1010 struct mlx5_flow_spec *spec;
1011 void *misc;
1012
1013 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1014 if (!spec) {
1015 flow_rule = ERR_PTR(-ENOMEM);
1016 goto out;
1017 }
1018
1019 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
1020 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
1021 /* source vport is the esw manager */
1022 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport);
1023 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1024 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1025 MLX5_CAP_GEN(from_esw->dev, vhca_id));
1026
1027 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
1028 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
1029 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1030 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
1031 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1032 source_eswitch_owner_vhca_id);
1033
1034 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1035 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1036 dest.vport.num = rep->vport;
1037 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1038 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1039 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1040
1041 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1042 rep->vport == MLX5_VPORT_UPLINK)
1043 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1044
1045 flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb,
1046 spec, &flow_act, &dest, 1);
1047 if (IS_ERR(flow_rule))
1048 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1049 PTR_ERR(flow_rule));
1050 out:
1051 kvfree(spec);
1052 return flow_rule;
1053 }
1054 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1055
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1056 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1057 {
1058 mlx5_del_flow_rules(rule);
1059 }
1060
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1061 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1062 {
1063 if (rule)
1064 mlx5_del_flow_rules(rule);
1065 }
1066
1067 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1068 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1069 {
1070 struct mlx5_flow_destination dest = {};
1071 struct mlx5_flow_act flow_act = {0};
1072 struct mlx5_flow_handle *flow_rule;
1073 struct mlx5_flow_spec *spec;
1074
1075 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1076 if (!spec)
1077 return ERR_PTR(-ENOMEM);
1078
1079 MLX5_SET(fte_match_param, spec->match_criteria,
1080 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1081 MLX5_SET(fte_match_param, spec->match_criteria,
1082 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1083 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1084 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1085
1086 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1087 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1088 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1089
1090 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1091 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1092 dest.vport.num = vport_num;
1093
1094 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1095 spec, &flow_act, &dest, 1);
1096 if (IS_ERR(flow_rule))
1097 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1098 vport_num, PTR_ERR(flow_rule));
1099
1100 kvfree(spec);
1101 return flow_rule;
1102 }
1103
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1104 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1105 {
1106 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1107 MLX5_FDB_TO_VPORT_REG_C_1;
1108 }
1109
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1110 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1111 {
1112 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1113 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1114 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1115 u8 curr, wanted;
1116 int err;
1117
1118 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1119 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1120 return 0;
1121
1122 MLX5_SET(query_esw_vport_context_in, in, opcode,
1123 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1124 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1125 if (err)
1126 return err;
1127
1128 curr = MLX5_GET(query_esw_vport_context_out, out,
1129 esw_vport_context.fdb_to_vport_reg_c_id);
1130 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1131 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1132 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1133
1134 if (enable)
1135 curr |= wanted;
1136 else
1137 curr &= ~wanted;
1138
1139 MLX5_SET(modify_esw_vport_context_in, min,
1140 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1141 MLX5_SET(modify_esw_vport_context_in, min,
1142 field_select.fdb_to_vport_reg_c_id, 1);
1143
1144 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1145 if (!err) {
1146 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1147 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1148 else
1149 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1150 }
1151
1152 return err;
1153 }
1154
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1155 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1156 struct mlx5_core_dev *peer_dev,
1157 struct mlx5_flow_spec *spec,
1158 struct mlx5_flow_destination *dest)
1159 {
1160 void *misc;
1161
1162 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1163 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1164 misc_parameters_2);
1165 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1166 mlx5_eswitch_get_vport_metadata_mask());
1167
1168 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1169 } else {
1170 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1171 misc_parameters);
1172
1173 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1174 MLX5_CAP_GEN(peer_dev, vhca_id));
1175
1176 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1177
1178 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1179 misc_parameters);
1180 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1181 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1182 source_eswitch_owner_vhca_id);
1183 }
1184
1185 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1186 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1187 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1188 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1189 }
1190
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1191 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1192 struct mlx5_eswitch *peer_esw,
1193 struct mlx5_flow_spec *spec,
1194 u16 vport)
1195 {
1196 void *misc;
1197
1198 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1199 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1200 misc_parameters_2);
1201 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1202 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1203 vport));
1204 } else {
1205 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1206 misc_parameters);
1207 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1208 }
1209 }
1210
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1211 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1212 struct mlx5_core_dev *peer_dev)
1213 {
1214 struct mlx5_flow_destination dest = {};
1215 struct mlx5_flow_act flow_act = {0};
1216 struct mlx5_flow_handle **flows;
1217 /* total vports is the same for both e-switches */
1218 int nvports = esw->total_vports;
1219 struct mlx5_flow_handle *flow;
1220 struct mlx5_flow_spec *spec;
1221 struct mlx5_vport *vport;
1222 unsigned long i;
1223 void *misc;
1224 int err;
1225
1226 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1227 if (!spec)
1228 return -ENOMEM;
1229
1230 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1231
1232 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1233 if (!flows) {
1234 err = -ENOMEM;
1235 goto alloc_flows_err;
1236 }
1237
1238 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1239 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1240 misc_parameters);
1241
1242 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1243 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1244 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1245 spec, MLX5_VPORT_PF);
1246
1247 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1248 spec, &flow_act, &dest, 1);
1249 if (IS_ERR(flow)) {
1250 err = PTR_ERR(flow);
1251 goto add_pf_flow_err;
1252 }
1253 flows[vport->index] = flow;
1254 }
1255
1256 if (mlx5_ecpf_vport_exists(esw->dev)) {
1257 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1258 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1259 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1260 spec, &flow_act, &dest, 1);
1261 if (IS_ERR(flow)) {
1262 err = PTR_ERR(flow);
1263 goto add_ecpf_flow_err;
1264 }
1265 flows[vport->index] = flow;
1266 }
1267
1268 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1269 esw_set_peer_miss_rule_source_port(esw,
1270 peer_dev->priv.eswitch,
1271 spec, vport->vport);
1272
1273 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1274 spec, &flow_act, &dest, 1);
1275 if (IS_ERR(flow)) {
1276 err = PTR_ERR(flow);
1277 goto add_vf_flow_err;
1278 }
1279 flows[vport->index] = flow;
1280 }
1281
1282 esw->fdb_table.offloads.peer_miss_rules = flows;
1283
1284 kvfree(spec);
1285 return 0;
1286
1287 add_vf_flow_err:
1288 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1289 if (!flows[vport->index])
1290 continue;
1291 mlx5_del_flow_rules(flows[vport->index]);
1292 }
1293 if (mlx5_ecpf_vport_exists(esw->dev)) {
1294 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1295 mlx5_del_flow_rules(flows[vport->index]);
1296 }
1297 add_ecpf_flow_err:
1298 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1299 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1300 mlx5_del_flow_rules(flows[vport->index]);
1301 }
1302 add_pf_flow_err:
1303 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1304 kvfree(flows);
1305 alloc_flows_err:
1306 kvfree(spec);
1307 return err;
1308 }
1309
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw)1310 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw)
1311 {
1312 struct mlx5_flow_handle **flows;
1313 struct mlx5_vport *vport;
1314 unsigned long i;
1315
1316 flows = esw->fdb_table.offloads.peer_miss_rules;
1317
1318 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1319 mlx5_del_flow_rules(flows[vport->index]);
1320
1321 if (mlx5_ecpf_vport_exists(esw->dev)) {
1322 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1323 mlx5_del_flow_rules(flows[vport->index]);
1324 }
1325
1326 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1327 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1328 mlx5_del_flow_rules(flows[vport->index]);
1329 }
1330 kvfree(flows);
1331 }
1332
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1333 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1334 {
1335 struct mlx5_flow_act flow_act = {0};
1336 struct mlx5_flow_destination dest = {};
1337 struct mlx5_flow_handle *flow_rule = NULL;
1338 struct mlx5_flow_spec *spec;
1339 void *headers_c;
1340 void *headers_v;
1341 int err = 0;
1342 u8 *dmac_c;
1343 u8 *dmac_v;
1344
1345 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1346 if (!spec) {
1347 err = -ENOMEM;
1348 goto out;
1349 }
1350
1351 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1352 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1353 outer_headers);
1354 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1355 outer_headers.dmac_47_16);
1356 dmac_c[0] = 0x01;
1357
1358 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1359 dest.vport.num = esw->manager_vport;
1360 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1361
1362 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1363 spec, &flow_act, &dest, 1);
1364 if (IS_ERR(flow_rule)) {
1365 err = PTR_ERR(flow_rule);
1366 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1367 goto out;
1368 }
1369
1370 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1371
1372 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1373 outer_headers);
1374 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1375 outer_headers.dmac_47_16);
1376 dmac_v[0] = 0x01;
1377 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1378 spec, &flow_act, &dest, 1);
1379 if (IS_ERR(flow_rule)) {
1380 err = PTR_ERR(flow_rule);
1381 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1382 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1383 goto out;
1384 }
1385
1386 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1387
1388 out:
1389 kvfree(spec);
1390 return err;
1391 }
1392
1393 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1394 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1395 {
1396 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1397 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1398 struct mlx5_flow_context *flow_context;
1399 struct mlx5_flow_handle *flow_rule;
1400 struct mlx5_flow_destination dest;
1401 struct mlx5_flow_spec *spec;
1402 void *misc;
1403
1404 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1405 return ERR_PTR(-EOPNOTSUPP);
1406
1407 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1408 if (!spec)
1409 return ERR_PTR(-ENOMEM);
1410
1411 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1412 misc_parameters_2);
1413 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1414 ESW_REG_C0_USER_DATA_METADATA_MASK);
1415 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1416 misc_parameters_2);
1417 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1418 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1419 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1420 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1421 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1422
1423 flow_context = &spec->flow_context;
1424 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1425 flow_context->flow_tag = tag;
1426 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1427 dest.ft = esw->offloads.ft_offloads;
1428
1429 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1430 kvfree(spec);
1431
1432 if (IS_ERR(flow_rule))
1433 esw_warn(esw->dev,
1434 "Failed to create restore rule for tag: %d, err(%d)\n",
1435 tag, (int)PTR_ERR(flow_rule));
1436
1437 return flow_rule;
1438 }
1439
1440 #define MAX_PF_SQ 256
1441 #define MAX_SQ_NVPORTS 32
1442
esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in)1443 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1444 u32 *flow_group_in)
1445 {
1446 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1447 flow_group_in,
1448 match_criteria);
1449
1450 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1451 MLX5_SET(create_flow_group_in, flow_group_in,
1452 match_criteria_enable,
1453 MLX5_MATCH_MISC_PARAMETERS_2);
1454
1455 MLX5_SET(fte_match_param, match_criteria,
1456 misc_parameters_2.metadata_reg_c_0,
1457 mlx5_eswitch_get_vport_metadata_mask());
1458 } else {
1459 MLX5_SET(create_flow_group_in, flow_group_in,
1460 match_criteria_enable,
1461 MLX5_MATCH_MISC_PARAMETERS);
1462
1463 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1464 misc_parameters.source_port);
1465 }
1466 }
1467
1468 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1469 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1470 {
1471 struct mlx5_vport_tbl_attr attr;
1472 struct mlx5_vport *vport;
1473 unsigned long i;
1474
1475 attr.chain = 0;
1476 attr.prio = 1;
1477 mlx5_esw_for_each_vport(esw, i, vport) {
1478 attr.vport = vport->vport;
1479 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1480 mlx5_esw_vporttbl_put(esw, &attr);
1481 }
1482 }
1483
esw_vport_tbl_get(struct mlx5_eswitch * esw)1484 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1485 {
1486 struct mlx5_vport_tbl_attr attr;
1487 struct mlx5_flow_table *fdb;
1488 struct mlx5_vport *vport;
1489 unsigned long i;
1490
1491 attr.chain = 0;
1492 attr.prio = 1;
1493 mlx5_esw_for_each_vport(esw, i, vport) {
1494 attr.vport = vport->vport;
1495 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1496 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1497 if (IS_ERR(fdb))
1498 goto out;
1499 }
1500 return 0;
1501
1502 out:
1503 esw_vport_tbl_put(esw);
1504 return PTR_ERR(fdb);
1505 }
1506
1507 #define fdb_modify_header_fwd_to_table_supported(esw) \
1508 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1509 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1510 {
1511 struct mlx5_core_dev *dev = esw->dev;
1512
1513 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1514 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1515
1516 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1517 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1518 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1519 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1520 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1521 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1522 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1523 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1524 /* Disabled when ttl workaround is needed, e.g
1525 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1526 */
1527 esw_warn(dev,
1528 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1529 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1530 } else {
1531 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1532 esw_info(dev, "Supported tc chains and prios offload\n");
1533 }
1534
1535 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1536 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1537 }
1538
1539 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1540 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1541 {
1542 struct mlx5_core_dev *dev = esw->dev;
1543 struct mlx5_flow_table *nf_ft, *ft;
1544 struct mlx5_chains_attr attr = {};
1545 struct mlx5_fs_chains *chains;
1546 u32 fdb_max;
1547 int err;
1548
1549 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1550
1551 esw_init_chains_offload_flags(esw, &attr.flags);
1552 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1553 attr.max_ft_sz = fdb_max;
1554 attr.max_grp_num = esw->params.large_group_num;
1555 attr.default_ft = miss_fdb;
1556 attr.mapping = esw->offloads.reg_c0_obj_pool;
1557
1558 chains = mlx5_chains_create(dev, &attr);
1559 if (IS_ERR(chains)) {
1560 err = PTR_ERR(chains);
1561 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1562 return err;
1563 }
1564
1565 esw->fdb_table.offloads.esw_chains_priv = chains;
1566
1567 /* Create tc_end_ft which is the always created ft chain */
1568 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1569 1, 0);
1570 if (IS_ERR(nf_ft)) {
1571 err = PTR_ERR(nf_ft);
1572 goto nf_ft_err;
1573 }
1574
1575 /* Always open the root for fast path */
1576 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1577 if (IS_ERR(ft)) {
1578 err = PTR_ERR(ft);
1579 goto level_0_err;
1580 }
1581
1582 /* Open level 1 for split fdb rules now if prios isn't supported */
1583 if (!mlx5_chains_prios_supported(chains)) {
1584 err = esw_vport_tbl_get(esw);
1585 if (err)
1586 goto level_1_err;
1587 }
1588
1589 mlx5_chains_set_end_ft(chains, nf_ft);
1590
1591 return 0;
1592
1593 level_1_err:
1594 mlx5_chains_put_table(chains, 0, 1, 0);
1595 level_0_err:
1596 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1597 nf_ft_err:
1598 mlx5_chains_destroy(chains);
1599 esw->fdb_table.offloads.esw_chains_priv = NULL;
1600
1601 return err;
1602 }
1603
1604 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1605 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1606 {
1607 if (!mlx5_chains_prios_supported(chains))
1608 esw_vport_tbl_put(esw);
1609 mlx5_chains_put_table(chains, 0, 1, 0);
1610 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1611 mlx5_chains_destroy(chains);
1612 }
1613
1614 #else /* CONFIG_MLX5_CLS_ACT */
1615
1616 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1617 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1618 { return 0; }
1619
1620 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1621 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1622 {}
1623
1624 #endif
1625
1626 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1627 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1628 struct mlx5_flow_table *fdb,
1629 u32 *flow_group_in,
1630 int *ix)
1631 {
1632 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1633 struct mlx5_flow_group *g;
1634 void *match_criteria;
1635 int count, err = 0;
1636
1637 memset(flow_group_in, 0, inlen);
1638
1639 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1640 MLX5_MATCH_MISC_PARAMETERS);
1641
1642 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1643
1644 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1645 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
1646 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1647 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1648 misc_parameters.source_eswitch_owner_vhca_id);
1649 MLX5_SET(create_flow_group_in, flow_group_in,
1650 source_eswitch_owner_vhca_id_valid, 1);
1651 }
1652
1653 /* See comment at table_size calculation */
1654 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1655 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1656 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1657 *ix += count;
1658
1659 g = mlx5_create_flow_group(fdb, flow_group_in);
1660 if (IS_ERR(g)) {
1661 err = PTR_ERR(g);
1662 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1663 goto out;
1664 }
1665 esw->fdb_table.offloads.send_to_vport_grp = g;
1666
1667 out:
1668 return err;
1669 }
1670
1671 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1672 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1673 struct mlx5_flow_table *fdb,
1674 u32 *flow_group_in,
1675 int *ix)
1676 {
1677 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1678 struct mlx5_flow_group *g;
1679 void *match_criteria;
1680 int err = 0;
1681
1682 if (!esw_src_port_rewrite_supported(esw))
1683 return 0;
1684
1685 memset(flow_group_in, 0, inlen);
1686
1687 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1688 MLX5_MATCH_MISC_PARAMETERS_2);
1689
1690 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1691
1692 MLX5_SET(fte_match_param, match_criteria,
1693 misc_parameters_2.metadata_reg_c_0,
1694 mlx5_eswitch_get_vport_metadata_mask());
1695 MLX5_SET(fte_match_param, match_criteria,
1696 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1697
1698 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1699 MLX5_SET(create_flow_group_in, flow_group_in,
1700 end_flow_index, *ix + esw->total_vports - 1);
1701 *ix += esw->total_vports;
1702
1703 g = mlx5_create_flow_group(fdb, flow_group_in);
1704 if (IS_ERR(g)) {
1705 err = PTR_ERR(g);
1706 esw_warn(esw->dev,
1707 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1708 goto send_vport_meta_err;
1709 }
1710 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1711
1712 return 0;
1713
1714 send_vport_meta_err:
1715 return err;
1716 }
1717
1718 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1719 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1720 struct mlx5_flow_table *fdb,
1721 u32 *flow_group_in,
1722 int *ix)
1723 {
1724 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1725 struct mlx5_flow_group *g;
1726 void *match_criteria;
1727 int err = 0;
1728
1729 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1730 return 0;
1731
1732 memset(flow_group_in, 0, inlen);
1733
1734 esw_set_flow_group_source_port(esw, flow_group_in);
1735
1736 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1737 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1738 flow_group_in,
1739 match_criteria);
1740
1741 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1742 misc_parameters.source_eswitch_owner_vhca_id);
1743
1744 MLX5_SET(create_flow_group_in, flow_group_in,
1745 source_eswitch_owner_vhca_id_valid, 1);
1746 }
1747
1748 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1749 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1750 *ix + esw->total_vports - 1);
1751 *ix += esw->total_vports;
1752
1753 g = mlx5_create_flow_group(fdb, flow_group_in);
1754 if (IS_ERR(g)) {
1755 err = PTR_ERR(g);
1756 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1757 goto out;
1758 }
1759 esw->fdb_table.offloads.peer_miss_grp = g;
1760
1761 out:
1762 return err;
1763 }
1764
1765 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1766 esw_create_miss_group(struct mlx5_eswitch *esw,
1767 struct mlx5_flow_table *fdb,
1768 u32 *flow_group_in,
1769 int *ix)
1770 {
1771 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1772 struct mlx5_flow_group *g;
1773 void *match_criteria;
1774 int err = 0;
1775 u8 *dmac;
1776
1777 memset(flow_group_in, 0, inlen);
1778
1779 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1780 MLX5_MATCH_OUTER_HEADERS);
1781 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1782 match_criteria);
1783 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1784 outer_headers.dmac_47_16);
1785 dmac[0] = 0x01;
1786
1787 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1788 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1789 *ix + MLX5_ESW_MISS_FLOWS);
1790
1791 g = mlx5_create_flow_group(fdb, flow_group_in);
1792 if (IS_ERR(g)) {
1793 err = PTR_ERR(g);
1794 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1795 goto miss_err;
1796 }
1797 esw->fdb_table.offloads.miss_grp = g;
1798
1799 err = esw_add_fdb_miss_rule(esw);
1800 if (err)
1801 goto miss_rule_err;
1802
1803 return 0;
1804
1805 miss_rule_err:
1806 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1807 miss_err:
1808 return err;
1809 }
1810
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1811 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1812 {
1813 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1814 struct mlx5_flow_table_attr ft_attr = {};
1815 struct mlx5_core_dev *dev = esw->dev;
1816 struct mlx5_flow_namespace *root_ns;
1817 struct mlx5_flow_table *fdb = NULL;
1818 int table_size, ix = 0, err = 0;
1819 u32 flags = 0, *flow_group_in;
1820
1821 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1822
1823 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1824 if (!flow_group_in)
1825 return -ENOMEM;
1826
1827 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1828 if (!root_ns) {
1829 esw_warn(dev, "Failed to get FDB flow namespace\n");
1830 err = -EOPNOTSUPP;
1831 goto ns_err;
1832 }
1833 esw->fdb_table.offloads.ns = root_ns;
1834 err = mlx5_flow_namespace_set_mode(root_ns,
1835 esw->dev->priv.steering->mode);
1836 if (err) {
1837 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1838 goto ns_err;
1839 }
1840
1841 /* To be strictly correct:
1842 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1843 * should be:
1844 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1845 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1846 * but as the peer device might not be in switchdev mode it's not
1847 * possible. We use the fact that by default FW sets max vfs and max sfs
1848 * to the same value on both devices. If it needs to be changed in the future note
1849 * the peer miss group should also be created based on the number of
1850 * total vports of the peer (currently is also uses esw->total_vports).
1851 */
1852 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1853 esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS;
1854
1855 /* create the slow path fdb with encap set, so further table instances
1856 * can be created at run time while VFs are probed if the FW allows that.
1857 */
1858 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1859 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1860 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1861
1862 ft_attr.flags = flags;
1863 ft_attr.max_fte = table_size;
1864 ft_attr.prio = FDB_SLOW_PATH;
1865
1866 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1867 if (IS_ERR(fdb)) {
1868 err = PTR_ERR(fdb);
1869 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1870 goto slow_fdb_err;
1871 }
1872 esw->fdb_table.offloads.slow_fdb = fdb;
1873
1874 /* Create empty TC-miss managed table. This allows plugging in following
1875 * priorities without directly exposing their level 0 table to
1876 * eswitch_offloads and passing it as miss_fdb to following call to
1877 * esw_chains_create().
1878 */
1879 memset(&ft_attr, 0, sizeof(ft_attr));
1880 ft_attr.prio = FDB_TC_MISS;
1881 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1882 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1883 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1884 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1885 goto tc_miss_table_err;
1886 }
1887
1888 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1889 if (err) {
1890 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1891 goto fdb_chains_err;
1892 }
1893
1894 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1895 if (err)
1896 goto send_vport_err;
1897
1898 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1899 if (err)
1900 goto send_vport_meta_err;
1901
1902 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1903 if (err)
1904 goto peer_miss_err;
1905
1906 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
1907 if (err)
1908 goto miss_err;
1909
1910 kvfree(flow_group_in);
1911 return 0;
1912
1913 miss_err:
1914 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1915 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1916 peer_miss_err:
1917 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1918 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1919 send_vport_meta_err:
1920 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1921 send_vport_err:
1922 esw_chains_destroy(esw, esw_chains(esw));
1923 fdb_chains_err:
1924 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1925 tc_miss_table_err:
1926 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1927 slow_fdb_err:
1928 /* Holds true only as long as DMFS is the default */
1929 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1930 ns_err:
1931 kvfree(flow_group_in);
1932 return err;
1933 }
1934
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1935 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1936 {
1937 if (!esw->fdb_table.offloads.slow_fdb)
1938 return;
1939
1940 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1941 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1942 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1943 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1944 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1945 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1946 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1947 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1948 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1949
1950 esw_chains_destroy(esw, esw_chains(esw));
1951
1952 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1953 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1954 /* Holds true only as long as DMFS is the default */
1955 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1956 MLX5_FLOW_STEERING_MODE_DMFS);
1957 atomic64_set(&esw->user_count, 0);
1958 }
1959
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)1960 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1961 {
1962 int nvports;
1963
1964 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1965 if (mlx5e_tc_int_port_supported(esw))
1966 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1967
1968 return nvports;
1969 }
1970
esw_create_offloads_table(struct mlx5_eswitch * esw)1971 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1972 {
1973 struct mlx5_flow_table_attr ft_attr = {};
1974 struct mlx5_core_dev *dev = esw->dev;
1975 struct mlx5_flow_table *ft_offloads;
1976 struct mlx5_flow_namespace *ns;
1977 int err = 0;
1978
1979 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1980 if (!ns) {
1981 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1982 return -EOPNOTSUPP;
1983 }
1984
1985 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1986 MLX5_ESW_FT_OFFLOADS_DROP_RULE;
1987 ft_attr.prio = 1;
1988
1989 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
1990 if (IS_ERR(ft_offloads)) {
1991 err = PTR_ERR(ft_offloads);
1992 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1993 return err;
1994 }
1995
1996 esw->offloads.ft_offloads = ft_offloads;
1997 return 0;
1998 }
1999
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2000 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2001 {
2002 struct mlx5_esw_offload *offloads = &esw->offloads;
2003
2004 mlx5_destroy_flow_table(offloads->ft_offloads);
2005 }
2006
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2007 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2008 {
2009 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2010 struct mlx5_flow_group *g;
2011 u32 *flow_group_in;
2012 int nvports;
2013 int err = 0;
2014
2015 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2016 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2017 if (!flow_group_in)
2018 return -ENOMEM;
2019
2020 /* create vport rx group */
2021 esw_set_flow_group_source_port(esw, flow_group_in);
2022
2023 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2024 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2025
2026 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2027
2028 if (IS_ERR(g)) {
2029 err = PTR_ERR(g);
2030 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2031 goto out;
2032 }
2033
2034 esw->offloads.vport_rx_group = g;
2035 out:
2036 kvfree(flow_group_in);
2037 return err;
2038 }
2039
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2040 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2041 {
2042 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2043 }
2044
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2045 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2046 {
2047 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2048 * for the drop rule, which is placed at the end of the table.
2049 * So return the total of vport and int_port as rule index.
2050 */
2051 return esw_get_nr_ft_offloads_steering_src_ports(esw);
2052 }
2053
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2054 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2055 {
2056 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2057 struct mlx5_flow_group *g;
2058 u32 *flow_group_in;
2059 int flow_index;
2060 int err = 0;
2061
2062 flow_index = esw_create_vport_rx_drop_rule_index(esw);
2063
2064 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2065 if (!flow_group_in)
2066 return -ENOMEM;
2067
2068 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2069 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2070
2071 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2072
2073 if (IS_ERR(g)) {
2074 err = PTR_ERR(g);
2075 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2076 goto out;
2077 }
2078
2079 esw->offloads.vport_rx_drop_group = g;
2080 out:
2081 kvfree(flow_group_in);
2082 return err;
2083 }
2084
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2085 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2086 {
2087 if (esw->offloads.vport_rx_drop_group)
2088 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2089 }
2090
2091 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2092 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2093 struct mlx5_flow_destination *dest)
2094 {
2095 struct mlx5_flow_act flow_act = {0};
2096 struct mlx5_flow_handle *flow_rule;
2097 struct mlx5_flow_spec *spec;
2098 void *misc;
2099
2100 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2101 if (!spec) {
2102 flow_rule = ERR_PTR(-ENOMEM);
2103 goto out;
2104 }
2105
2106 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2107 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2108 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2109 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2110
2111 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2112 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2113 mlx5_eswitch_get_vport_metadata_mask());
2114
2115 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2116 } else {
2117 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2118 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2119
2120 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2121 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2122
2123 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2124 }
2125
2126 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2127 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2128 &flow_act, dest, 1);
2129 if (IS_ERR(flow_rule)) {
2130 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2131 goto out;
2132 }
2133
2134 out:
2135 kvfree(spec);
2136 return flow_rule;
2137 }
2138
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2139 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2140 {
2141 struct mlx5_flow_act flow_act = {};
2142 struct mlx5_flow_handle *flow_rule;
2143
2144 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2145 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2146 &flow_act, NULL, 0);
2147 if (IS_ERR(flow_rule)) {
2148 esw_warn(esw->dev,
2149 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2150 PTR_ERR(flow_rule));
2151 return PTR_ERR(flow_rule);
2152 }
2153
2154 esw->offloads.vport_rx_drop_rule = flow_rule;
2155
2156 return 0;
2157 }
2158
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2159 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2160 {
2161 if (esw->offloads.vport_rx_drop_rule)
2162 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2163 }
2164
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2165 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2166 {
2167 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2168 struct mlx5_core_dev *dev = esw->dev;
2169 struct mlx5_vport *vport;
2170 unsigned long i;
2171
2172 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2173 return -EOPNOTSUPP;
2174
2175 if (!mlx5_esw_is_fdb_created(esw))
2176 return -EOPNOTSUPP;
2177
2178 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2179 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2180 mlx5_mode = MLX5_INLINE_MODE_NONE;
2181 goto out;
2182 case MLX5_CAP_INLINE_MODE_L2:
2183 mlx5_mode = MLX5_INLINE_MODE_L2;
2184 goto out;
2185 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2186 goto query_vports;
2187 }
2188
2189 query_vports:
2190 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2191 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2192 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2193 if (prev_mlx5_mode != mlx5_mode)
2194 return -EINVAL;
2195 prev_mlx5_mode = mlx5_mode;
2196 }
2197
2198 out:
2199 *mode = mlx5_mode;
2200 return 0;
2201 }
2202
esw_destroy_restore_table(struct mlx5_eswitch * esw)2203 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2204 {
2205 struct mlx5_esw_offload *offloads = &esw->offloads;
2206
2207 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2208 return;
2209
2210 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2211 mlx5_destroy_flow_group(offloads->restore_group);
2212 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2213 }
2214
esw_create_restore_table(struct mlx5_eswitch * esw)2215 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2216 {
2217 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2218 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2219 struct mlx5_flow_table_attr ft_attr = {};
2220 struct mlx5_core_dev *dev = esw->dev;
2221 struct mlx5_flow_namespace *ns;
2222 struct mlx5_modify_hdr *mod_hdr;
2223 void *match_criteria, *misc;
2224 struct mlx5_flow_table *ft;
2225 struct mlx5_flow_group *g;
2226 u32 *flow_group_in;
2227 int err = 0;
2228
2229 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2230 return 0;
2231
2232 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2233 if (!ns) {
2234 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2235 return -EOPNOTSUPP;
2236 }
2237
2238 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2239 if (!flow_group_in) {
2240 err = -ENOMEM;
2241 goto out_free;
2242 }
2243
2244 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2245 ft = mlx5_create_flow_table(ns, &ft_attr);
2246 if (IS_ERR(ft)) {
2247 err = PTR_ERR(ft);
2248 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2249 err);
2250 goto out_free;
2251 }
2252
2253 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2254 match_criteria);
2255 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2256 misc_parameters_2);
2257
2258 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2259 ESW_REG_C0_USER_DATA_METADATA_MASK);
2260 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2261 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2262 ft_attr.max_fte - 1);
2263 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2264 MLX5_MATCH_MISC_PARAMETERS_2);
2265 g = mlx5_create_flow_group(ft, flow_group_in);
2266 if (IS_ERR(g)) {
2267 err = PTR_ERR(g);
2268 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2269 err);
2270 goto err_group;
2271 }
2272
2273 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2274 MLX5_SET(copy_action_in, modact, src_field,
2275 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2276 MLX5_SET(copy_action_in, modact, dst_field,
2277 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2278 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2279 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2280 modact);
2281 if (IS_ERR(mod_hdr)) {
2282 err = PTR_ERR(mod_hdr);
2283 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2284 err);
2285 goto err_mod_hdr;
2286 }
2287
2288 esw->offloads.ft_offloads_restore = ft;
2289 esw->offloads.restore_group = g;
2290 esw->offloads.restore_copy_hdr_id = mod_hdr;
2291
2292 kvfree(flow_group_in);
2293
2294 return 0;
2295
2296 err_mod_hdr:
2297 mlx5_destroy_flow_group(g);
2298 err_group:
2299 mlx5_destroy_flow_table(ft);
2300 out_free:
2301 kvfree(flow_group_in);
2302
2303 return err;
2304 }
2305
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2306 static int esw_offloads_start(struct mlx5_eswitch *esw,
2307 struct netlink_ext_ack *extack)
2308 {
2309 int err;
2310
2311 esw->mode = MLX5_ESWITCH_OFFLOADS;
2312 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2313 if (err) {
2314 NL_SET_ERR_MSG_MOD(extack,
2315 "Failed setting eswitch to offloads");
2316 esw->mode = MLX5_ESWITCH_LEGACY;
2317 mlx5_rescan_drivers(esw->dev);
2318 }
2319 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2320 if (mlx5_eswitch_inline_mode_get(esw,
2321 &esw->offloads.inline_mode)) {
2322 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2323 NL_SET_ERR_MSG_MOD(extack,
2324 "Inline mode is different between vports");
2325 }
2326 }
2327 return err;
2328 }
2329
mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,xa_mark_t mark)2330 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw,
2331 struct mlx5_eswitch_rep *rep,
2332 xa_mark_t mark)
2333 {
2334 bool mark_set;
2335
2336 /* Copy the mark from vport to its rep */
2337 mark_set = xa_get_mark(&esw->vports, rep->vport, mark);
2338 if (mark_set)
2339 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark);
2340 }
2341
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2342 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2343 {
2344 struct mlx5_eswitch_rep *rep;
2345 int rep_type;
2346 int err;
2347
2348 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2349 if (!rep)
2350 return -ENOMEM;
2351
2352 rep->vport = vport->vport;
2353 rep->vport_index = vport->index;
2354 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2355 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2356
2357 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2358 if (err)
2359 goto insert_err;
2360
2361 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN);
2362 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF);
2363 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF);
2364 return 0;
2365
2366 insert_err:
2367 kfree(rep);
2368 return err;
2369 }
2370
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2371 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2372 struct mlx5_eswitch_rep *rep)
2373 {
2374 xa_erase(&esw->offloads.vport_reps, rep->vport);
2375 kfree(rep);
2376 }
2377
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2378 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2379 {
2380 struct mlx5_eswitch_rep *rep;
2381 unsigned long i;
2382
2383 mlx5_esw_for_each_rep(esw, i, rep)
2384 mlx5_esw_offloads_rep_cleanup(esw, rep);
2385 xa_destroy(&esw->offloads.vport_reps);
2386 }
2387
esw_offloads_init_reps(struct mlx5_eswitch * esw)2388 int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2389 {
2390 struct mlx5_vport *vport;
2391 unsigned long i;
2392 int err;
2393
2394 xa_init(&esw->offloads.vport_reps);
2395
2396 mlx5_esw_for_each_vport(esw, i, vport) {
2397 err = mlx5_esw_offloads_rep_init(esw, vport);
2398 if (err)
2399 goto err;
2400 }
2401 return 0;
2402
2403 err:
2404 esw_offloads_cleanup_reps(esw);
2405 return err;
2406 }
2407
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2408 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2409 struct mlx5_eswitch_rep *rep, u8 rep_type)
2410 {
2411 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2412 REP_LOADED, REP_REGISTERED) == REP_LOADED)
2413 esw->offloads.rep_ops[rep_type]->unload(rep);
2414 }
2415
__unload_reps_sf_vport(struct mlx5_eswitch * esw,u8 rep_type)2416 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type)
2417 {
2418 struct mlx5_eswitch_rep *rep;
2419 unsigned long i;
2420
2421 mlx5_esw_for_each_sf_rep(esw, i, rep)
2422 __esw_offloads_unload_rep(esw, rep, rep_type);
2423 }
2424
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2425 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2426 {
2427 struct mlx5_eswitch_rep *rep;
2428 unsigned long i;
2429
2430 __unload_reps_sf_vport(esw, rep_type);
2431
2432 mlx5_esw_for_each_vf_rep(esw, i, rep)
2433 __esw_offloads_unload_rep(esw, rep, rep_type);
2434
2435 if (mlx5_ecpf_vport_exists(esw->dev)) {
2436 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF);
2437 __esw_offloads_unload_rep(esw, rep, rep_type);
2438 }
2439
2440 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
2441 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF);
2442 __esw_offloads_unload_rep(esw, rep, rep_type);
2443 }
2444
2445 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
2446 __esw_offloads_unload_rep(esw, rep, rep_type);
2447 }
2448
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2449 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2450 {
2451 struct mlx5_eswitch_rep *rep;
2452 int rep_type;
2453 int err;
2454
2455 rep = mlx5_eswitch_get_rep(esw, vport_num);
2456 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2457 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2458 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2459 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2460 if (err)
2461 goto err_reps;
2462 }
2463
2464 return 0;
2465
2466 err_reps:
2467 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2468 for (--rep_type; rep_type >= 0; rep_type--)
2469 __esw_offloads_unload_rep(esw, rep, rep_type);
2470 return err;
2471 }
2472
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2473 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2474 {
2475 struct mlx5_eswitch_rep *rep;
2476 int rep_type;
2477
2478 rep = mlx5_eswitch_get_rep(esw, vport_num);
2479 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2480 __esw_offloads_unload_rep(esw, rep, rep_type);
2481 }
2482
esw_offloads_load_rep(struct mlx5_eswitch * esw,u16 vport_num)2483 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num)
2484 {
2485 int err;
2486
2487 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2488 return 0;
2489
2490 if (vport_num != MLX5_VPORT_UPLINK) {
2491 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num);
2492 if (err)
2493 return err;
2494 }
2495
2496 err = mlx5_esw_offloads_rep_load(esw, vport_num);
2497 if (err)
2498 goto load_err;
2499 return err;
2500
2501 load_err:
2502 if (vport_num != MLX5_VPORT_UPLINK)
2503 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2504 return err;
2505 }
2506
esw_offloads_unload_rep(struct mlx5_eswitch * esw,u16 vport_num)2507 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num)
2508 {
2509 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2510 return;
2511
2512 mlx5_esw_offloads_rep_unload(esw, vport_num);
2513
2514 if (vport_num != MLX5_VPORT_UPLINK)
2515 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
2516 }
2517
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2518 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2519 struct mlx5_core_dev *slave)
2520 {
2521 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2522 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2523 struct mlx5_flow_root_namespace *root;
2524 struct mlx5_flow_namespace *ns;
2525 int err;
2526
2527 MLX5_SET(set_flow_table_root_in, in, opcode,
2528 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2529 MLX5_SET(set_flow_table_root_in, in, table_type,
2530 FS_FT_FDB);
2531
2532 if (master) {
2533 ns = mlx5_get_flow_namespace(master,
2534 MLX5_FLOW_NAMESPACE_FDB);
2535 root = find_root(&ns->node);
2536 mutex_lock(&root->chain_lock);
2537 MLX5_SET(set_flow_table_root_in, in,
2538 table_eswitch_owner_vhca_id_valid, 1);
2539 MLX5_SET(set_flow_table_root_in, in,
2540 table_eswitch_owner_vhca_id,
2541 MLX5_CAP_GEN(master, vhca_id));
2542 MLX5_SET(set_flow_table_root_in, in, table_id,
2543 root->root_ft->id);
2544 } else {
2545 ns = mlx5_get_flow_namespace(slave,
2546 MLX5_FLOW_NAMESPACE_FDB);
2547 root = find_root(&ns->node);
2548 mutex_lock(&root->chain_lock);
2549 MLX5_SET(set_flow_table_root_in, in, table_id,
2550 root->root_ft->id);
2551 }
2552
2553 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2554 mutex_unlock(&root->chain_lock);
2555
2556 return err;
2557 }
2558
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2559 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2560 struct mlx5_core_dev *slave,
2561 struct mlx5_vport *vport,
2562 struct mlx5_flow_table *acl)
2563 {
2564 struct mlx5_flow_handle *flow_rule = NULL;
2565 struct mlx5_flow_destination dest = {};
2566 struct mlx5_flow_act flow_act = {};
2567 struct mlx5_flow_spec *spec;
2568 int err = 0;
2569 void *misc;
2570
2571 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2572 if (!spec)
2573 return -ENOMEM;
2574
2575 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2576 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2577 misc_parameters);
2578 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2579 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
2580 MLX5_CAP_GEN(slave, vhca_id));
2581
2582 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2583 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2584 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2585 source_eswitch_owner_vhca_id);
2586
2587 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2588 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2589 dest.vport.num = slave->priv.eswitch->manager_vport;
2590 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2591 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2592
2593 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2594 &dest, 1);
2595 if (IS_ERR(flow_rule))
2596 err = PTR_ERR(flow_rule);
2597 else
2598 vport->egress.offloads.bounce_rule = flow_rule;
2599
2600 kvfree(spec);
2601 return err;
2602 }
2603
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2604 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2605 struct mlx5_core_dev *slave)
2606 {
2607 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2608 struct mlx5_eswitch *esw = master->priv.eswitch;
2609 struct mlx5_flow_table_attr ft_attr = {
2610 .max_fte = 1, .prio = 0, .level = 0,
2611 .flags = MLX5_FLOW_TABLE_OTHER_VPORT,
2612 };
2613 struct mlx5_flow_namespace *egress_ns;
2614 struct mlx5_flow_table *acl;
2615 struct mlx5_flow_group *g;
2616 struct mlx5_vport *vport;
2617 void *match_criteria;
2618 u32 *flow_group_in;
2619 int err;
2620
2621 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2622 if (IS_ERR(vport))
2623 return PTR_ERR(vport);
2624
2625 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2626 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2627 vport->index);
2628 if (!egress_ns)
2629 return -EINVAL;
2630
2631 if (vport->egress.acl)
2632 return -EINVAL;
2633
2634 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2635 if (!flow_group_in)
2636 return -ENOMEM;
2637
2638 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2639 if (IS_ERR(acl)) {
2640 err = PTR_ERR(acl);
2641 goto out;
2642 }
2643
2644 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2645 match_criteria);
2646 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2647 misc_parameters.source_port);
2648 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2649 misc_parameters.source_eswitch_owner_vhca_id);
2650 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2651 MLX5_MATCH_MISC_PARAMETERS);
2652
2653 MLX5_SET(create_flow_group_in, flow_group_in,
2654 source_eswitch_owner_vhca_id_valid, 1);
2655 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2656 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
2657
2658 g = mlx5_create_flow_group(acl, flow_group_in);
2659 if (IS_ERR(g)) {
2660 err = PTR_ERR(g);
2661 goto err_group;
2662 }
2663
2664 err = __esw_set_master_egress_rule(master, slave, vport, acl);
2665 if (err)
2666 goto err_rule;
2667
2668 vport->egress.acl = acl;
2669 vport->egress.offloads.bounce_grp = g;
2670
2671 kvfree(flow_group_in);
2672
2673 return 0;
2674
2675 err_rule:
2676 mlx5_destroy_flow_group(g);
2677 err_group:
2678 mlx5_destroy_flow_table(acl);
2679 out:
2680 kvfree(flow_group_in);
2681 return err;
2682 }
2683
esw_unset_master_egress_rule(struct mlx5_core_dev * dev)2684 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev)
2685 {
2686 struct mlx5_vport *vport;
2687
2688 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2689 dev->priv.eswitch->manager_vport);
2690
2691 esw_acl_egress_ofld_cleanup(vport);
2692 }
2693
mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2694 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw,
2695 struct mlx5_eswitch *slave_esw)
2696 {
2697 int err;
2698
2699 err = esw_set_slave_root_fdb(master_esw->dev,
2700 slave_esw->dev);
2701 if (err)
2702 return err;
2703
2704 err = esw_set_master_egress_rule(master_esw->dev,
2705 slave_esw->dev);
2706 if (err)
2707 goto err_acl;
2708
2709 return err;
2710
2711 err_acl:
2712 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2713
2714 return err;
2715 }
2716
mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2717 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw,
2718 struct mlx5_eswitch *slave_esw)
2719 {
2720 esw_unset_master_egress_rule(master_esw->dev);
2721 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2722 }
2723
2724 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2725 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2726
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw)2727 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw)
2728 {
2729 const struct mlx5_eswitch_rep_ops *ops;
2730 struct mlx5_eswitch_rep *rep;
2731 unsigned long i;
2732 u8 rep_type;
2733
2734 mlx5_esw_for_each_rep(esw, i, rep) {
2735 rep_type = NUM_REP_TYPES;
2736 while (rep_type--) {
2737 ops = esw->offloads.rep_ops[rep_type];
2738 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2739 ops->event)
2740 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL);
2741 }
2742 }
2743 }
2744
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw)2745 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw)
2746 {
2747 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2748 mlx5e_tc_clean_fdb_peer_flows(esw);
2749 #endif
2750 mlx5_esw_offloads_rep_event_unpair(esw);
2751 esw_del_fdb_peer_miss_rules(esw);
2752 }
2753
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2754 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2755 struct mlx5_eswitch *peer_esw)
2756 {
2757 const struct mlx5_eswitch_rep_ops *ops;
2758 struct mlx5_eswitch_rep *rep;
2759 unsigned long i;
2760 u8 rep_type;
2761 int err;
2762
2763 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2764 if (err)
2765 return err;
2766
2767 mlx5_esw_for_each_rep(esw, i, rep) {
2768 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2769 ops = esw->offloads.rep_ops[rep_type];
2770 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2771 ops->event) {
2772 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2773 if (err)
2774 goto err_out;
2775 }
2776 }
2777 }
2778
2779 return 0;
2780
2781 err_out:
2782 mlx5_esw_offloads_unpair(esw);
2783 return err;
2784 }
2785
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2786 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2787 struct mlx5_eswitch *peer_esw,
2788 bool pair)
2789 {
2790 struct mlx5_flow_root_namespace *peer_ns;
2791 struct mlx5_flow_root_namespace *ns;
2792 int err;
2793
2794 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2795 ns = esw->dev->priv.steering->fdb_root_ns;
2796
2797 if (pair) {
2798 err = mlx5_flow_namespace_set_peer(ns, peer_ns);
2799 if (err)
2800 return err;
2801
2802 err = mlx5_flow_namespace_set_peer(peer_ns, ns);
2803 if (err) {
2804 mlx5_flow_namespace_set_peer(ns, NULL);
2805 return err;
2806 }
2807 } else {
2808 mlx5_flow_namespace_set_peer(ns, NULL);
2809 mlx5_flow_namespace_set_peer(peer_ns, NULL);
2810 }
2811
2812 return 0;
2813 }
2814
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2815 static int mlx5_esw_offloads_devcom_event(int event,
2816 void *my_data,
2817 void *event_data)
2818 {
2819 struct mlx5_eswitch *esw = my_data;
2820 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2821 struct mlx5_eswitch *peer_esw = event_data;
2822 int err;
2823
2824 switch (event) {
2825 case ESW_OFFLOADS_DEVCOM_PAIR:
2826 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2827 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2828 break;
2829
2830 if (esw->paired[mlx5_get_dev_index(peer_esw->dev)])
2831 break;
2832
2833 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2834 if (err)
2835 goto err_out;
2836 err = mlx5_esw_offloads_pair(esw, peer_esw);
2837 if (err)
2838 goto err_peer;
2839
2840 err = mlx5_esw_offloads_pair(peer_esw, esw);
2841 if (err)
2842 goto err_pair;
2843
2844 esw->paired[mlx5_get_dev_index(peer_esw->dev)] = true;
2845 peer_esw->paired[mlx5_get_dev_index(esw->dev)] = true;
2846 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true);
2847 break;
2848
2849 case ESW_OFFLOADS_DEVCOM_UNPAIR:
2850 if (!esw->paired[mlx5_get_dev_index(peer_esw->dev)])
2851 break;
2852
2853 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false);
2854 esw->paired[mlx5_get_dev_index(peer_esw->dev)] = false;
2855 peer_esw->paired[mlx5_get_dev_index(esw->dev)] = false;
2856 mlx5_esw_offloads_unpair(peer_esw);
2857 mlx5_esw_offloads_unpair(esw);
2858 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2859 break;
2860 }
2861
2862 return 0;
2863
2864 err_pair:
2865 mlx5_esw_offloads_unpair(esw);
2866 err_peer:
2867 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
2868 err_out:
2869 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
2870 event, err);
2871 return err;
2872 }
2873
mlx5_esw_offloads_devcom_init(struct mlx5_eswitch * esw)2874 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw)
2875 {
2876 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2877
2878 INIT_LIST_HEAD(&esw->offloads.peer_flows);
2879 mutex_init(&esw->offloads.peer_mutex);
2880
2881 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2882 return;
2883
2884 if (!mlx5_is_lag_supported(esw->dev))
2885 return;
2886
2887 mlx5_devcom_register_component(devcom,
2888 MLX5_DEVCOM_ESW_OFFLOADS,
2889 mlx5_esw_offloads_devcom_event,
2890 esw);
2891
2892 mlx5_devcom_send_event(devcom,
2893 MLX5_DEVCOM_ESW_OFFLOADS,
2894 ESW_OFFLOADS_DEVCOM_PAIR, esw);
2895 }
2896
mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)2897 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
2898 {
2899 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2900
2901 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2902 return;
2903
2904 if (!mlx5_is_lag_supported(esw->dev))
2905 return;
2906
2907 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS,
2908 ESW_OFFLOADS_DEVCOM_UNPAIR, esw);
2909
2910 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
2911 }
2912
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)2913 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
2914 {
2915 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
2916 return false;
2917
2918 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
2919 MLX5_FDB_TO_VPORT_REG_C_0))
2920 return false;
2921
2922 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
2923 return false;
2924
2925 return true;
2926 }
2927
2928 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
2929
2930 /* Share the same metadata for uplink's. This is fine because:
2931 * (a) In shared FDB mode (LAG) both uplink's are treated the
2932 * same and tagged with the same metadata.
2933 * (b) In non shared FDB mode, packets from physical port0
2934 * cannot hit eswitch of PF1 and vice versa.
2935 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)2936 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
2937 {
2938 return MLX5_ESW_METADATA_RSVD_UPLINK;
2939 }
2940
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)2941 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
2942 {
2943 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
2944 /* Reserve 0xf for internal port offload */
2945 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
2946 u32 pf_num;
2947 int id;
2948
2949 /* Only 4 bits of pf_num */
2950 pf_num = mlx5_get_dev_index(esw->dev);
2951 if (pf_num > max_pf_num)
2952 return 0;
2953
2954 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2955 /* Use only non-zero vport_id (2-4095) for all PF's */
2956 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
2957 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
2958 vport_end_ida, GFP_KERNEL);
2959 if (id < 0)
2960 return 0;
2961 id = (pf_num << ESW_VPORT_BITS) | id;
2962 return id;
2963 }
2964
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)2965 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
2966 {
2967 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
2968
2969 /* Metadata contains only 12 bits of actual ida id */
2970 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
2971 }
2972
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2973 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
2974 struct mlx5_vport *vport)
2975 {
2976 if (vport->vport == MLX5_VPORT_UPLINK)
2977 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
2978 else
2979 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
2980
2981 vport->metadata = vport->default_metadata;
2982 return vport->metadata ? 0 : -ENOSPC;
2983 }
2984
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2985 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
2986 struct mlx5_vport *vport)
2987 {
2988 if (!vport->default_metadata)
2989 return;
2990
2991 if (vport->vport == MLX5_VPORT_UPLINK)
2992 return;
2993
2994 WARN_ON(vport->metadata != vport->default_metadata);
2995 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
2996 }
2997
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)2998 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
2999 {
3000 struct mlx5_vport *vport;
3001 unsigned long i;
3002
3003 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3004 return;
3005
3006 mlx5_esw_for_each_vport(esw, i, vport)
3007 esw_offloads_vport_metadata_cleanup(esw, vport);
3008 }
3009
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3010 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3011 {
3012 struct mlx5_vport *vport;
3013 unsigned long i;
3014 int err;
3015
3016 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3017 return 0;
3018
3019 mlx5_esw_for_each_vport(esw, i, vport) {
3020 err = esw_offloads_vport_metadata_setup(esw, vport);
3021 if (err)
3022 goto metadata_err;
3023 }
3024
3025 return 0;
3026
3027 metadata_err:
3028 esw_offloads_metadata_uninit(esw);
3029 return err;
3030 }
3031
mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch * esw,bool enable)3032 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
3033 {
3034 int err = 0;
3035
3036 down_write(&esw->mode_lock);
3037 if (mlx5_esw_is_fdb_created(esw)) {
3038 err = -EBUSY;
3039 goto done;
3040 }
3041 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
3042 err = -EOPNOTSUPP;
3043 goto done;
3044 }
3045 if (enable)
3046 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
3047 else
3048 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
3049 done:
3050 up_write(&esw->mode_lock);
3051 return err;
3052 }
3053
3054 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3055 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3056 struct mlx5_vport *vport)
3057 {
3058 int err;
3059
3060 err = esw_acl_ingress_ofld_setup(esw, vport);
3061 if (err)
3062 return err;
3063
3064 err = esw_acl_egress_ofld_setup(esw, vport);
3065 if (err)
3066 goto egress_err;
3067
3068 return 0;
3069
3070 egress_err:
3071 esw_acl_ingress_ofld_cleanup(esw, vport);
3072 return err;
3073 }
3074
3075 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3076 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3077 struct mlx5_vport *vport)
3078 {
3079 esw_acl_egress_ofld_cleanup(vport);
3080 esw_acl_ingress_ofld_cleanup(esw, vport);
3081 }
3082
esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)3083 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
3084 {
3085 struct mlx5_vport *vport;
3086
3087 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3088 if (IS_ERR(vport))
3089 return PTR_ERR(vport);
3090
3091 return esw_vport_create_offloads_acl_tables(esw, vport);
3092 }
3093
esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch * esw)3094 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
3095 {
3096 struct mlx5_vport *vport;
3097
3098 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3099 if (IS_ERR(vport))
3100 return;
3101
3102 esw_vport_destroy_offloads_acl_tables(esw, vport);
3103 }
3104
mlx5_eswitch_reload_reps(struct mlx5_eswitch * esw)3105 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
3106 {
3107 struct mlx5_eswitch_rep *rep;
3108 unsigned long i;
3109 int ret;
3110
3111 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3112 return 0;
3113
3114 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3115 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3116 return 0;
3117
3118 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3119 if (ret)
3120 return ret;
3121
3122 mlx5_esw_for_each_rep(esw, i, rep) {
3123 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3124 mlx5_esw_offloads_rep_load(esw, rep->vport);
3125 }
3126
3127 return 0;
3128 }
3129
esw_offloads_steering_init(struct mlx5_eswitch * esw)3130 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3131 {
3132 struct mlx5_esw_indir_table *indir;
3133 int err;
3134
3135 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3136 mutex_init(&esw->fdb_table.offloads.vports.lock);
3137 hash_init(esw->fdb_table.offloads.vports.table);
3138 atomic64_set(&esw->user_count, 0);
3139
3140 indir = mlx5_esw_indir_table_init();
3141 if (IS_ERR(indir)) {
3142 err = PTR_ERR(indir);
3143 goto create_indir_err;
3144 }
3145 esw->fdb_table.offloads.indir = indir;
3146
3147 err = esw_create_uplink_offloads_acl_tables(esw);
3148 if (err)
3149 goto create_acl_err;
3150
3151 err = esw_create_offloads_table(esw);
3152 if (err)
3153 goto create_offloads_err;
3154
3155 err = esw_create_restore_table(esw);
3156 if (err)
3157 goto create_restore_err;
3158
3159 err = esw_create_offloads_fdb_tables(esw);
3160 if (err)
3161 goto create_fdb_err;
3162
3163 err = esw_create_vport_rx_group(esw);
3164 if (err)
3165 goto create_fg_err;
3166
3167 err = esw_create_vport_rx_drop_group(esw);
3168 if (err)
3169 goto create_rx_drop_fg_err;
3170
3171 err = esw_create_vport_rx_drop_rule(esw);
3172 if (err)
3173 goto create_rx_drop_rule_err;
3174
3175 return 0;
3176
3177 create_rx_drop_rule_err:
3178 esw_destroy_vport_rx_drop_group(esw);
3179 create_rx_drop_fg_err:
3180 esw_destroy_vport_rx_group(esw);
3181 create_fg_err:
3182 esw_destroy_offloads_fdb_tables(esw);
3183 create_fdb_err:
3184 esw_destroy_restore_table(esw);
3185 create_restore_err:
3186 esw_destroy_offloads_table(esw);
3187 create_offloads_err:
3188 esw_destroy_uplink_offloads_acl_tables(esw);
3189 create_acl_err:
3190 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3191 create_indir_err:
3192 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3193 return err;
3194 }
3195
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3196 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3197 {
3198 esw_destroy_vport_rx_drop_rule(esw);
3199 esw_destroy_vport_rx_drop_group(esw);
3200 esw_destroy_vport_rx_group(esw);
3201 esw_destroy_offloads_fdb_tables(esw);
3202 esw_destroy_restore_table(esw);
3203 esw_destroy_offloads_table(esw);
3204 esw_destroy_uplink_offloads_acl_tables(esw);
3205 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3206 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3207 }
3208
3209 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3210 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3211 {
3212 struct devlink *devlink;
3213 bool host_pf_disabled;
3214 u16 new_num_vfs;
3215
3216 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3217 host_params_context.host_num_of_vfs);
3218 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3219 host_params_context.host_pf_disabled);
3220
3221 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3222 return;
3223
3224 devlink = priv_to_devlink(esw->dev);
3225 devl_lock(devlink);
3226 /* Number of VFs can only change from "0 to x" or "x to 0". */
3227 if (esw->esw_funcs.num_vfs > 0) {
3228 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3229 } else {
3230 int err;
3231
3232 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3233 MLX5_VPORT_UC_ADDR_CHANGE);
3234 if (err) {
3235 devl_unlock(devlink);
3236 return;
3237 }
3238 }
3239 esw->esw_funcs.num_vfs = new_num_vfs;
3240 devl_unlock(devlink);
3241 }
3242
esw_functions_changed_event_handler(struct work_struct * work)3243 static void esw_functions_changed_event_handler(struct work_struct *work)
3244 {
3245 struct mlx5_host_work *host_work;
3246 struct mlx5_eswitch *esw;
3247 const u32 *out;
3248
3249 host_work = container_of(work, struct mlx5_host_work, work);
3250 esw = host_work->esw;
3251
3252 out = mlx5_esw_query_functions(esw->dev);
3253 if (IS_ERR(out))
3254 goto out;
3255
3256 esw_vfs_changed_event_handler(esw, out);
3257 kvfree(out);
3258 out:
3259 kfree(host_work);
3260 }
3261
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3262 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3263 {
3264 struct mlx5_esw_functions *esw_funcs;
3265 struct mlx5_host_work *host_work;
3266 struct mlx5_eswitch *esw;
3267
3268 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3269 if (!host_work)
3270 return NOTIFY_DONE;
3271
3272 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3273 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3274
3275 host_work->esw = esw;
3276
3277 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3278 queue_work(esw->work_queue, &host_work->work);
3279
3280 return NOTIFY_OK;
3281 }
3282
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3283 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3284 {
3285 const u32 *query_host_out;
3286
3287 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3288 return 0;
3289
3290 query_host_out = mlx5_esw_query_functions(esw->dev);
3291 if (IS_ERR(query_host_out))
3292 return PTR_ERR(query_host_out);
3293
3294 /* Mark non local controller with non zero controller number. */
3295 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3296 host_params_context.host_number);
3297 kvfree(query_host_out);
3298 return 0;
3299 }
3300
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3301 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3302 {
3303 /* Local controller is always valid */
3304 if (controller == 0)
3305 return true;
3306
3307 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3308 return false;
3309
3310 /* External host number starts with zero in device */
3311 return (controller == esw->offloads.host_number + 1);
3312 }
3313
esw_offloads_enable(struct mlx5_eswitch * esw)3314 int esw_offloads_enable(struct mlx5_eswitch *esw)
3315 {
3316 struct mapping_ctx *reg_c0_obj_pool;
3317 struct mlx5_vport *vport;
3318 unsigned long i;
3319 u64 mapping_id;
3320 int err;
3321
3322 mutex_init(&esw->offloads.termtbl_mutex);
3323 mlx5_rdma_enable_roce(esw->dev);
3324
3325 err = mlx5_esw_host_number_init(esw);
3326 if (err)
3327 goto err_metadata;
3328
3329 err = esw_offloads_metadata_init(esw);
3330 if (err)
3331 goto err_metadata;
3332
3333 err = esw_set_passing_vport_metadata(esw, true);
3334 if (err)
3335 goto err_vport_metadata;
3336
3337 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3338
3339 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3340 sizeof(struct mlx5_mapped_obj),
3341 ESW_REG_C0_USER_DATA_METADATA_MASK,
3342 true);
3343
3344 if (IS_ERR(reg_c0_obj_pool)) {
3345 err = PTR_ERR(reg_c0_obj_pool);
3346 goto err_pool;
3347 }
3348 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3349
3350 err = esw_offloads_steering_init(esw);
3351 if (err)
3352 goto err_steering_init;
3353
3354 /* Representor will control the vport link state */
3355 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3356 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3357
3358 /* Uplink vport rep must load first. */
3359 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK);
3360 if (err)
3361 goto err_uplink;
3362
3363 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3364 if (err)
3365 goto err_vports;
3366
3367 return 0;
3368
3369 err_vports:
3370 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3371 err_uplink:
3372 esw_offloads_steering_cleanup(esw);
3373 err_steering_init:
3374 mapping_destroy(reg_c0_obj_pool);
3375 err_pool:
3376 esw_set_passing_vport_metadata(esw, false);
3377 err_vport_metadata:
3378 esw_offloads_metadata_uninit(esw);
3379 err_metadata:
3380 mlx5_rdma_disable_roce(esw->dev);
3381 mutex_destroy(&esw->offloads.termtbl_mutex);
3382 return err;
3383 }
3384
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3385 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3386 struct netlink_ext_ack *extack)
3387 {
3388 int err;
3389
3390 esw->mode = MLX5_ESWITCH_LEGACY;
3391
3392 /* If changing from switchdev to legacy mode without sriov enabled,
3393 * no need to create legacy fdb.
3394 */
3395 if (!mlx5_sriov_is_enabled(esw->dev))
3396 return 0;
3397
3398 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3399 if (err)
3400 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3401
3402 return err;
3403 }
3404
esw_offloads_disable(struct mlx5_eswitch * esw)3405 void esw_offloads_disable(struct mlx5_eswitch *esw)
3406 {
3407 mlx5_eswitch_disable_pf_vf_vports(esw);
3408 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3409 esw_set_passing_vport_metadata(esw, false);
3410 esw_offloads_steering_cleanup(esw);
3411 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3412 esw_offloads_metadata_uninit(esw);
3413 mlx5_rdma_disable_roce(esw->dev);
3414 mutex_destroy(&esw->offloads.termtbl_mutex);
3415 }
3416
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3417 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3418 {
3419 switch (mode) {
3420 case DEVLINK_ESWITCH_MODE_LEGACY:
3421 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3422 break;
3423 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3424 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3425 break;
3426 default:
3427 return -EINVAL;
3428 }
3429
3430 return 0;
3431 }
3432
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3433 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3434 {
3435 switch (mlx5_mode) {
3436 case MLX5_ESWITCH_LEGACY:
3437 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3438 break;
3439 case MLX5_ESWITCH_OFFLOADS:
3440 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3441 break;
3442 default:
3443 return -EINVAL;
3444 }
3445
3446 return 0;
3447 }
3448
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3449 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3450 {
3451 switch (mode) {
3452 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3453 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3454 break;
3455 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3456 *mlx5_mode = MLX5_INLINE_MODE_L2;
3457 break;
3458 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3459 *mlx5_mode = MLX5_INLINE_MODE_IP;
3460 break;
3461 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3462 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3463 break;
3464 default:
3465 return -EINVAL;
3466 }
3467
3468 return 0;
3469 }
3470
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3471 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3472 {
3473 switch (mlx5_mode) {
3474 case MLX5_INLINE_MODE_NONE:
3475 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3476 break;
3477 case MLX5_INLINE_MODE_L2:
3478 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3479 break;
3480 case MLX5_INLINE_MODE_IP:
3481 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3482 break;
3483 case MLX5_INLINE_MODE_TCP_UDP:
3484 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3485 break;
3486 default:
3487 return -EINVAL;
3488 }
3489
3490 return 0;
3491 }
3492
esw_offloads_devlink_ns_eq_netdev_ns(struct devlink * devlink)3493 static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink *devlink)
3494 {
3495 struct net *devl_net, *netdev_net;
3496 struct mlx5_eswitch *esw;
3497
3498 esw = mlx5_devlink_eswitch_get(devlink);
3499 netdev_net = dev_net(esw->dev->mlx5e_res.uplink_netdev);
3500 devl_net = devlink_net(devlink);
3501
3502 return net_eq(devl_net, netdev_net);
3503 }
3504
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3505 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3506 struct netlink_ext_ack *extack)
3507 {
3508 u16 cur_mlx5_mode, mlx5_mode = 0;
3509 struct mlx5_eswitch *esw;
3510 int err = 0;
3511
3512 esw = mlx5_devlink_eswitch_get(devlink);
3513 if (IS_ERR(esw))
3514 return PTR_ERR(esw);
3515
3516 if (esw_mode_from_devlink(mode, &mlx5_mode))
3517 return -EINVAL;
3518
3519 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV &&
3520 !esw_offloads_devlink_ns_eq_netdev_ns(devlink)) {
3521 NL_SET_ERR_MSG_MOD(extack,
3522 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's.");
3523 return -EPERM;
3524 }
3525
3526 mlx5_lag_disable_change(esw->dev);
3527 err = mlx5_esw_try_lock(esw);
3528 if (err < 0) {
3529 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3530 goto enable_lag;
3531 }
3532 cur_mlx5_mode = err;
3533 err = 0;
3534
3535 if (cur_mlx5_mode == mlx5_mode)
3536 goto unlock;
3537
3538 mlx5_eswitch_disable_locked(esw);
3539 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3540 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3541 NL_SET_ERR_MSG_MOD(extack,
3542 "Can't change mode while devlink traps are active");
3543 err = -EOPNOTSUPP;
3544 goto unlock;
3545 }
3546 err = esw_offloads_start(esw, extack);
3547 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3548 err = esw_offloads_stop(esw, extack);
3549 mlx5_rescan_drivers(esw->dev);
3550 } else {
3551 err = -EINVAL;
3552 }
3553
3554 unlock:
3555 mlx5_esw_unlock(esw);
3556 enable_lag:
3557 mlx5_lag_enable_change(esw->dev);
3558 return err;
3559 }
3560
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3561 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3562 {
3563 struct mlx5_eswitch *esw;
3564 int err;
3565
3566 esw = mlx5_devlink_eswitch_get(devlink);
3567 if (IS_ERR(esw))
3568 return PTR_ERR(esw);
3569
3570 down_write(&esw->mode_lock);
3571 err = esw_mode_to_devlink(esw->mode, mode);
3572 up_write(&esw->mode_lock);
3573 return err;
3574 }
3575
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3576 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3577 struct netlink_ext_ack *extack)
3578 {
3579 struct mlx5_core_dev *dev = esw->dev;
3580 struct mlx5_vport *vport;
3581 u16 err_vport_num = 0;
3582 unsigned long i;
3583 int err = 0;
3584
3585 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3586 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3587 if (err) {
3588 err_vport_num = vport->vport;
3589 NL_SET_ERR_MSG_MOD(extack,
3590 "Failed to set min inline on vport");
3591 goto revert_inline_mode;
3592 }
3593 }
3594 return 0;
3595
3596 revert_inline_mode:
3597 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3598 if (vport->vport == err_vport_num)
3599 break;
3600 mlx5_modify_nic_vport_min_inline(dev,
3601 vport->vport,
3602 esw->offloads.inline_mode);
3603 }
3604 return err;
3605 }
3606
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3607 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3608 struct netlink_ext_ack *extack)
3609 {
3610 struct mlx5_core_dev *dev = devlink_priv(devlink);
3611 struct mlx5_eswitch *esw;
3612 u8 mlx5_mode;
3613 int err;
3614
3615 esw = mlx5_devlink_eswitch_get(devlink);
3616 if (IS_ERR(esw))
3617 return PTR_ERR(esw);
3618
3619 down_write(&esw->mode_lock);
3620
3621 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3622 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3623 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3624 err = 0;
3625 goto out;
3626 }
3627
3628 fallthrough;
3629 case MLX5_CAP_INLINE_MODE_L2:
3630 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3631 err = -EOPNOTSUPP;
3632 goto out;
3633 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3634 break;
3635 }
3636
3637 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3638 NL_SET_ERR_MSG_MOD(extack,
3639 "Can't set inline mode when flows are configured");
3640 err = -EOPNOTSUPP;
3641 goto out;
3642 }
3643
3644 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3645 if (err)
3646 goto out;
3647
3648 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3649 if (err)
3650 goto out;
3651
3652 esw->offloads.inline_mode = mlx5_mode;
3653 up_write(&esw->mode_lock);
3654 return 0;
3655
3656 out:
3657 up_write(&esw->mode_lock);
3658 return err;
3659 }
3660
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3661 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3662 {
3663 struct mlx5_eswitch *esw;
3664 int err;
3665
3666 esw = mlx5_devlink_eswitch_get(devlink);
3667 if (IS_ERR(esw))
3668 return PTR_ERR(esw);
3669
3670 down_write(&esw->mode_lock);
3671 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3672 up_write(&esw->mode_lock);
3673 return err;
3674 }
3675
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3676 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3677 enum devlink_eswitch_encap_mode encap,
3678 struct netlink_ext_ack *extack)
3679 {
3680 struct mlx5_core_dev *dev = devlink_priv(devlink);
3681 struct mlx5_eswitch *esw;
3682 int err = 0;
3683
3684 esw = mlx5_devlink_eswitch_get(devlink);
3685 if (IS_ERR(esw))
3686 return PTR_ERR(esw);
3687
3688 down_write(&esw->mode_lock);
3689
3690 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3691 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3692 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3693 err = -EOPNOTSUPP;
3694 goto unlock;
3695 }
3696
3697 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3698 err = -EOPNOTSUPP;
3699 goto unlock;
3700 }
3701
3702 if (esw->mode == MLX5_ESWITCH_LEGACY) {
3703 esw->offloads.encap = encap;
3704 goto unlock;
3705 }
3706
3707 if (esw->offloads.encap == encap)
3708 goto unlock;
3709
3710 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3711 NL_SET_ERR_MSG_MOD(extack,
3712 "Can't set encapsulation when flows are configured");
3713 err = -EOPNOTSUPP;
3714 goto unlock;
3715 }
3716
3717 esw_destroy_offloads_fdb_tables(esw);
3718
3719 esw->offloads.encap = encap;
3720
3721 err = esw_create_offloads_fdb_tables(esw);
3722
3723 if (err) {
3724 NL_SET_ERR_MSG_MOD(extack,
3725 "Failed re-creating fast FDB table");
3726 esw->offloads.encap = !encap;
3727 (void)esw_create_offloads_fdb_tables(esw);
3728 }
3729
3730 unlock:
3731 up_write(&esw->mode_lock);
3732 return err;
3733 }
3734
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)3735 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
3736 enum devlink_eswitch_encap_mode *encap)
3737 {
3738 struct mlx5_eswitch *esw;
3739
3740 esw = mlx5_devlink_eswitch_get(devlink);
3741 if (IS_ERR(esw))
3742 return PTR_ERR(esw);
3743
3744 down_write(&esw->mode_lock);
3745 *encap = esw->offloads.encap;
3746 up_write(&esw->mode_lock);
3747 return 0;
3748 }
3749
3750 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)3751 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
3752 {
3753 /* Currently, only ECPF based device has representor for host PF. */
3754 if (vport_num == MLX5_VPORT_PF &&
3755 !mlx5_core_is_ecpf_esw_manager(esw->dev))
3756 return false;
3757
3758 if (vport_num == MLX5_VPORT_ECPF &&
3759 !mlx5_ecpf_vport_exists(esw->dev))
3760 return false;
3761
3762 return true;
3763 }
3764
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)3765 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
3766 const struct mlx5_eswitch_rep_ops *ops,
3767 u8 rep_type)
3768 {
3769 struct mlx5_eswitch_rep_data *rep_data;
3770 struct mlx5_eswitch_rep *rep;
3771 unsigned long i;
3772
3773 esw->offloads.rep_ops[rep_type] = ops;
3774 mlx5_esw_for_each_rep(esw, i, rep) {
3775 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
3776 rep->esw = esw;
3777 rep_data = &rep->rep_data[rep_type];
3778 atomic_set(&rep_data->state, REP_REGISTERED);
3779 }
3780 }
3781 }
3782 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
3783
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)3784 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
3785 {
3786 struct mlx5_eswitch_rep *rep;
3787 unsigned long i;
3788
3789 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
3790 __unload_reps_all_vport(esw, rep_type);
3791
3792 mlx5_esw_for_each_rep(esw, i, rep)
3793 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
3794 }
3795 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
3796
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)3797 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
3798 {
3799 struct mlx5_eswitch_rep *rep;
3800
3801 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3802 return rep->rep_data[rep_type].priv;
3803 }
3804
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)3805 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
3806 u16 vport,
3807 u8 rep_type)
3808 {
3809 struct mlx5_eswitch_rep *rep;
3810
3811 rep = mlx5_eswitch_get_rep(esw, vport);
3812
3813 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3814 esw->offloads.rep_ops[rep_type]->get_proto_dev)
3815 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
3816 return NULL;
3817 }
3818 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
3819
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)3820 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
3821 {
3822 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
3823 }
3824 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
3825
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)3826 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
3827 u16 vport)
3828 {
3829 return mlx5_eswitch_get_rep(esw, vport);
3830 }
3831 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
3832
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)3833 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
3834 {
3835 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
3836 }
3837 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
3838
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)3839 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
3840 {
3841 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
3842 }
3843 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
3844
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)3845 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
3846 u16 vport_num)
3847 {
3848 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3849
3850 if (WARN_ON_ONCE(IS_ERR(vport)))
3851 return 0;
3852
3853 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
3854 }
3855 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
3856
mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch * esw,struct devlink_port * dl_port,u16 vport_num,u32 controller,u32 sfnum)3857 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
3858 u16 vport_num, u32 controller, u32 sfnum)
3859 {
3860 int err;
3861
3862 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE);
3863 if (err)
3864 return err;
3865
3866 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum);
3867 if (err)
3868 goto devlink_err;
3869
3870 mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum);
3871 err = mlx5_esw_offloads_rep_load(esw, vport_num);
3872 if (err)
3873 goto rep_err;
3874 return 0;
3875
3876 rep_err:
3877 mlx5_esw_vport_debugfs_destroy(esw, vport_num);
3878 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3879 devlink_err:
3880 mlx5_esw_vport_disable(esw, vport_num);
3881 return err;
3882 }
3883
mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch * esw,u16 vport_num)3884 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
3885 {
3886 mlx5_esw_offloads_rep_unload(esw, vport_num);
3887 mlx5_esw_vport_debugfs_destroy(esw, vport_num);
3888 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3889 mlx5_esw_vport_disable(esw, vport_num);
3890 }
3891
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)3892 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
3893 {
3894 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
3895 void *query_ctx;
3896 void *hca_caps;
3897 int err;
3898
3899 *vhca_id = 0;
3900 if (mlx5_esw_is_manager_vport(esw, vport_num) ||
3901 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
3902 return -EPERM;
3903
3904 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
3905 if (!query_ctx)
3906 return -ENOMEM;
3907
3908 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx);
3909 if (err)
3910 goto out_free;
3911
3912 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
3913 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
3914
3915 out_free:
3916 kfree(query_ctx);
3917 return err;
3918 }
3919
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)3920 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
3921 {
3922 u16 *old_entry, *vhca_map_entry, vhca_id;
3923 int err;
3924
3925 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3926 if (err) {
3927 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3928 vport_num, err);
3929 return err;
3930 }
3931
3932 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
3933 if (!vhca_map_entry)
3934 return -ENOMEM;
3935
3936 *vhca_map_entry = vport_num;
3937 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
3938 if (xa_is_err(old_entry)) {
3939 kfree(vhca_map_entry);
3940 return xa_err(old_entry);
3941 }
3942 kfree(old_entry);
3943 return 0;
3944 }
3945
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)3946 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
3947 {
3948 u16 *vhca_map_entry, vhca_id;
3949 int err;
3950
3951 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3952 if (err)
3953 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3954 vport_num, err);
3955
3956 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
3957 kfree(vhca_map_entry);
3958 }
3959
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)3960 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
3961 {
3962 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
3963
3964 if (!res)
3965 return -ENOENT;
3966
3967 *vport_num = *res;
3968 return 0;
3969 }
3970
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)3971 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
3972 u16 vport_num)
3973 {
3974 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3975
3976 if (WARN_ON_ONCE(IS_ERR(vport)))
3977 return 0;
3978
3979 return vport->metadata;
3980 }
3981 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
3982
3983 static bool
is_port_function_supported(struct mlx5_eswitch * esw,u16 vport_num)3984 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num)
3985 {
3986 return vport_num == MLX5_VPORT_PF ||
3987 mlx5_eswitch_is_vf_vport(esw, vport_num) ||
3988 mlx5_esw_is_sf_vport(esw, vport_num);
3989 }
3990
mlx5_devlink_port_function_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)3991 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port,
3992 u8 *hw_addr, int *hw_addr_len,
3993 struct netlink_ext_ack *extack)
3994 {
3995 struct mlx5_eswitch *esw;
3996 struct mlx5_vport *vport;
3997 u16 vport_num;
3998
3999 esw = mlx5_devlink_eswitch_get(port->devlink);
4000 if (IS_ERR(esw))
4001 return PTR_ERR(esw);
4002
4003 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4004 if (!is_port_function_supported(esw, vport_num))
4005 return -EOPNOTSUPP;
4006
4007 vport = mlx5_eswitch_get_vport(esw, vport_num);
4008 if (IS_ERR(vport)) {
4009 NL_SET_ERR_MSG_MOD(extack, "Invalid port");
4010 return PTR_ERR(vport);
4011 }
4012
4013 mutex_lock(&esw->state_lock);
4014 ether_addr_copy(hw_addr, vport->info.mac);
4015 *hw_addr_len = ETH_ALEN;
4016 mutex_unlock(&esw->state_lock);
4017 return 0;
4018 }
4019
mlx5_devlink_port_function_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4020 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port,
4021 const u8 *hw_addr, int hw_addr_len,
4022 struct netlink_ext_ack *extack)
4023 {
4024 struct mlx5_eswitch *esw;
4025 u16 vport_num;
4026
4027 esw = mlx5_devlink_eswitch_get(port->devlink);
4028 if (IS_ERR(esw)) {
4029 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr");
4030 return PTR_ERR(esw);
4031 }
4032
4033 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4034 if (!is_port_function_supported(esw, vport_num)) {
4035 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr");
4036 return -EINVAL;
4037 }
4038
4039 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr);
4040 }
4041