1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2021 Marvell.
5 *
6 */
7
8 #include <linux/netdevice.h>
9 #include <linux/etherdevice.h>
10 #include <linux/inetdevice.h>
11 #include <linux/rhashtable.h>
12 #include <linux/bitfield.h>
13 #include <net/flow_dissector.h>
14 #include <net/pkt_cls.h>
15 #include <net/tc_act/tc_gact.h>
16 #include <net/tc_act/tc_mirred.h>
17 #include <net/tc_act/tc_vlan.h>
18 #include <net/ipv6.h>
19
20 #include "cn10k.h"
21 #include "otx2_common.h"
22
23 /* Egress rate limiting definitions */
24 #define MAX_BURST_EXPONENT 0x0FULL
25 #define MAX_BURST_MANTISSA 0xFFULL
26 #define MAX_BURST_SIZE 130816ULL
27 #define MAX_RATE_DIVIDER_EXPONENT 12ULL
28 #define MAX_RATE_EXPONENT 0x0FULL
29 #define MAX_RATE_MANTISSA 0xFFULL
30
31 #define CN10K_MAX_BURST_MANTISSA 0x7FFFULL
32 #define CN10K_MAX_BURST_SIZE 8453888ULL
33
34 /* Bitfields in NIX_TLX_PIR register */
35 #define TLX_RATE_MANTISSA GENMASK_ULL(8, 1)
36 #define TLX_RATE_EXPONENT GENMASK_ULL(12, 9)
37 #define TLX_RATE_DIVIDER_EXPONENT GENMASK_ULL(16, 13)
38 #define TLX_BURST_MANTISSA GENMASK_ULL(36, 29)
39 #define TLX_BURST_EXPONENT GENMASK_ULL(40, 37)
40
41 #define CN10K_TLX_BURST_MANTISSA GENMASK_ULL(43, 29)
42 #define CN10K_TLX_BURST_EXPONENT GENMASK_ULL(47, 44)
43
44 struct otx2_tc_flow_stats {
45 u64 bytes;
46 u64 pkts;
47 u64 used;
48 };
49
50 struct otx2_tc_flow {
51 struct rhash_head node;
52 unsigned long cookie;
53 unsigned int bitpos;
54 struct rcu_head rcu;
55 struct otx2_tc_flow_stats stats;
56 spinlock_t lock; /* lock for stats */
57 u16 rq;
58 u16 entry;
59 u16 leaf_profile;
60 bool is_act_police;
61 };
62
otx2_tc_alloc_ent_bitmap(struct otx2_nic * nic)63 int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic)
64 {
65 struct otx2_tc_info *tc = &nic->tc_info;
66
67 if (!nic->flow_cfg->max_flows || is_otx2_vf(nic->pcifunc))
68 return 0;
69
70 /* Max flows changed, free the existing bitmap */
71 kfree(tc->tc_entries_bitmap);
72
73 tc->tc_entries_bitmap =
74 kcalloc(BITS_TO_LONGS(nic->flow_cfg->max_flows),
75 sizeof(long), GFP_KERNEL);
76 if (!tc->tc_entries_bitmap) {
77 netdev_err(nic->netdev,
78 "Unable to alloc TC flow entries bitmap\n");
79 return -ENOMEM;
80 }
81
82 return 0;
83 }
84 EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap);
85
otx2_get_egress_burst_cfg(struct otx2_nic * nic,u32 burst,u32 * burst_exp,u32 * burst_mantissa)86 static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst,
87 u32 *burst_exp, u32 *burst_mantissa)
88 {
89 int max_burst, max_mantissa;
90 unsigned int tmp;
91
92 if (is_dev_otx2(nic->pdev)) {
93 max_burst = MAX_BURST_SIZE;
94 max_mantissa = MAX_BURST_MANTISSA;
95 } else {
96 max_burst = CN10K_MAX_BURST_SIZE;
97 max_mantissa = CN10K_MAX_BURST_MANTISSA;
98 }
99
100 /* Burst is calculated as
101 * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256
102 * Max supported burst size is 130,816 bytes.
103 */
104 burst = min_t(u32, burst, max_burst);
105 if (burst) {
106 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0;
107 tmp = burst - rounddown_pow_of_two(burst);
108 if (burst < max_mantissa)
109 *burst_mantissa = tmp * 2;
110 else
111 *burst_mantissa = tmp / (1ULL << (*burst_exp - 7));
112 } else {
113 *burst_exp = MAX_BURST_EXPONENT;
114 *burst_mantissa = max_mantissa;
115 }
116 }
117
otx2_get_egress_rate_cfg(u64 maxrate,u32 * exp,u32 * mantissa,u32 * div_exp)118 static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp,
119 u32 *mantissa, u32 *div_exp)
120 {
121 u64 tmp;
122
123 /* Rate calculation by hardware
124 *
125 * PIR_ADD = ((256 + mantissa) << exp) / 256
126 * rate = (2 * PIR_ADD) / ( 1 << div_exp)
127 * The resultant rate is in Mbps.
128 */
129
130 /* 2Mbps to 100Gbps can be expressed with div_exp = 0.
131 * Setting this to '0' will ease the calculation of
132 * exponent and mantissa.
133 */
134 *div_exp = 0;
135
136 if (maxrate) {
137 *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0;
138 tmp = maxrate - rounddown_pow_of_two(maxrate);
139 if (maxrate < MAX_RATE_MANTISSA)
140 *mantissa = tmp * 2;
141 else
142 *mantissa = tmp / (1ULL << (*exp - 7));
143 } else {
144 /* Instead of disabling rate limiting, set all values to max */
145 *exp = MAX_RATE_EXPONENT;
146 *mantissa = MAX_RATE_MANTISSA;
147 }
148 }
149
otx2_get_txschq_rate_regval(struct otx2_nic * nic,u64 maxrate,u32 burst)150 static u64 otx2_get_txschq_rate_regval(struct otx2_nic *nic,
151 u64 maxrate, u32 burst)
152 {
153 u32 burst_exp, burst_mantissa;
154 u32 exp, mantissa, div_exp;
155 u64 regval = 0;
156
157 /* Get exponent and mantissa values from the desired rate */
158 otx2_get_egress_burst_cfg(nic, burst, &burst_exp, &burst_mantissa);
159 otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp);
160
161 if (is_dev_otx2(nic->pdev)) {
162 regval = FIELD_PREP(TLX_BURST_EXPONENT, (u64)burst_exp) |
163 FIELD_PREP(TLX_BURST_MANTISSA, (u64)burst_mantissa) |
164 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) |
165 FIELD_PREP(TLX_RATE_EXPONENT, exp) |
166 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0);
167 } else {
168 regval = FIELD_PREP(CN10K_TLX_BURST_EXPONENT, (u64)burst_exp) |
169 FIELD_PREP(CN10K_TLX_BURST_MANTISSA, (u64)burst_mantissa) |
170 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) |
171 FIELD_PREP(TLX_RATE_EXPONENT, exp) |
172 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0);
173 }
174
175 return regval;
176 }
177
otx2_set_matchall_egress_rate(struct otx2_nic * nic,u32 burst,u64 maxrate)178 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic,
179 u32 burst, u64 maxrate)
180 {
181 struct otx2_hw *hw = &nic->hw;
182 struct nix_txschq_config *req;
183 int txschq, err;
184
185 /* All SQs share the same TL4, so pick the first scheduler */
186 txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0];
187
188 mutex_lock(&nic->mbox.lock);
189 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox);
190 if (!req) {
191 mutex_unlock(&nic->mbox.lock);
192 return -ENOMEM;
193 }
194
195 req->lvl = NIX_TXSCH_LVL_TL4;
196 req->num_regs = 1;
197 req->reg[0] = NIX_AF_TL4X_PIR(txschq);
198 req->regval[0] = otx2_get_txschq_rate_regval(nic, maxrate, burst);
199
200 err = otx2_sync_mbox_msg(&nic->mbox);
201 mutex_unlock(&nic->mbox.lock);
202 return err;
203 }
204
otx2_tc_validate_flow(struct otx2_nic * nic,struct flow_action * actions,struct netlink_ext_ack * extack)205 static int otx2_tc_validate_flow(struct otx2_nic *nic,
206 struct flow_action *actions,
207 struct netlink_ext_ack *extack)
208 {
209 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
210 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
211 return -EINVAL;
212 }
213
214 if (!flow_action_has_entries(actions)) {
215 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action");
216 return -EINVAL;
217 }
218
219 if (!flow_offload_has_one_action(actions)) {
220 NL_SET_ERR_MSG_MOD(extack,
221 "Egress MATCHALL offload supports only 1 policing action");
222 return -EINVAL;
223 }
224 return 0;
225 }
226
otx2_tc_egress_matchall_install(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)227 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic,
228 struct tc_cls_matchall_offload *cls)
229 {
230 struct netlink_ext_ack *extack = cls->common.extack;
231 struct flow_action *actions = &cls->rule->action;
232 struct flow_action_entry *entry;
233 u64 rate;
234 int err;
235
236 err = otx2_tc_validate_flow(nic, actions, extack);
237 if (err)
238 return err;
239
240 if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) {
241 NL_SET_ERR_MSG_MOD(extack,
242 "Only one Egress MATCHALL ratelimiter can be offloaded");
243 return -ENOMEM;
244 }
245
246 entry = &cls->rule->action.entries[0];
247 switch (entry->id) {
248 case FLOW_ACTION_POLICE:
249 if (entry->police.rate_pkt_ps) {
250 NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second");
251 return -EOPNOTSUPP;
252 }
253 /* Convert bytes per second to Mbps */
254 rate = entry->police.rate_bytes_ps * 8;
255 rate = max_t(u64, rate / 1000000, 1);
256 err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate);
257 if (err)
258 return err;
259 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
260 break;
261 default:
262 NL_SET_ERR_MSG_MOD(extack,
263 "Only police action is supported with Egress MATCHALL offload");
264 return -EOPNOTSUPP;
265 }
266
267 return 0;
268 }
269
otx2_tc_egress_matchall_delete(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)270 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic,
271 struct tc_cls_matchall_offload *cls)
272 {
273 struct netlink_ext_ack *extack = cls->common.extack;
274 int err;
275
276 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
277 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
278 return -EINVAL;
279 }
280
281 err = otx2_set_matchall_egress_rate(nic, 0, 0);
282 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
283 return err;
284 }
285
otx2_tc_act_set_police(struct otx2_nic * nic,struct otx2_tc_flow * node,struct flow_cls_offload * f,u64 rate,u32 burst,u32 mark,struct npc_install_flow_req * req,bool pps)286 static int otx2_tc_act_set_police(struct otx2_nic *nic,
287 struct otx2_tc_flow *node,
288 struct flow_cls_offload *f,
289 u64 rate, u32 burst, u32 mark,
290 struct npc_install_flow_req *req, bool pps)
291 {
292 struct netlink_ext_ack *extack = f->common.extack;
293 struct otx2_hw *hw = &nic->hw;
294 int rq_idx, rc;
295
296 rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues);
297 if (rq_idx >= hw->rx_queues) {
298 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded");
299 return -EINVAL;
300 }
301
302 mutex_lock(&nic->mbox.lock);
303
304 rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile);
305 if (rc) {
306 mutex_unlock(&nic->mbox.lock);
307 return rc;
308 }
309
310 rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, burst, rate, pps);
311 if (rc)
312 goto free_leaf;
313
314 rc = cn10k_map_unmap_rq_policer(nic, rq_idx, node->leaf_profile, true);
315 if (rc)
316 goto free_leaf;
317
318 mutex_unlock(&nic->mbox.lock);
319
320 req->match_id = mark & 0xFFFFULL;
321 req->index = rq_idx;
322 req->op = NIX_RX_ACTIONOP_UCAST;
323 set_bit(rq_idx, &nic->rq_bmap);
324 node->is_act_police = true;
325 node->rq = rq_idx;
326
327 return 0;
328
329 free_leaf:
330 if (cn10k_free_leaf_profile(nic, node->leaf_profile))
331 netdev_err(nic->netdev,
332 "Unable to free leaf bandwidth profile(%d)\n",
333 node->leaf_profile);
334 mutex_unlock(&nic->mbox.lock);
335 return rc;
336 }
337
otx2_tc_parse_actions(struct otx2_nic * nic,struct flow_action * flow_action,struct npc_install_flow_req * req,struct flow_cls_offload * f,struct otx2_tc_flow * node)338 static int otx2_tc_parse_actions(struct otx2_nic *nic,
339 struct flow_action *flow_action,
340 struct npc_install_flow_req *req,
341 struct flow_cls_offload *f,
342 struct otx2_tc_flow *node)
343 {
344 struct netlink_ext_ack *extack = f->common.extack;
345 struct flow_action_entry *act;
346 struct net_device *target;
347 struct otx2_nic *priv;
348 u32 burst, mark = 0;
349 u8 nr_police = 0;
350 bool pps = false;
351 u64 rate;
352 int i;
353
354 if (!flow_action_has_entries(flow_action)) {
355 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified");
356 return -EINVAL;
357 }
358
359 flow_action_for_each(i, act, flow_action) {
360 switch (act->id) {
361 case FLOW_ACTION_DROP:
362 req->op = NIX_RX_ACTIONOP_DROP;
363 return 0;
364 case FLOW_ACTION_ACCEPT:
365 req->op = NIX_RX_ACTION_DEFAULT;
366 return 0;
367 case FLOW_ACTION_REDIRECT_INGRESS:
368 target = act->dev;
369 priv = netdev_priv(target);
370 /* npc_install_flow_req doesn't support passing a target pcifunc */
371 if (rvu_get_pf(nic->pcifunc) != rvu_get_pf(priv->pcifunc)) {
372 NL_SET_ERR_MSG_MOD(extack,
373 "can't redirect to other pf/vf");
374 return -EOPNOTSUPP;
375 }
376 req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK;
377 req->op = NIX_RX_ACTION_DEFAULT;
378 return 0;
379 case FLOW_ACTION_VLAN_POP:
380 req->vtag0_valid = true;
381 /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */
382 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
383 break;
384 case FLOW_ACTION_POLICE:
385 /* Ingress ratelimiting is not supported on OcteonTx2 */
386 if (is_dev_otx2(nic->pdev)) {
387 NL_SET_ERR_MSG_MOD(extack,
388 "Ingress policing not supported on this platform");
389 return -EOPNOTSUPP;
390 }
391
392 if (act->police.rate_bytes_ps > 0) {
393 rate = act->police.rate_bytes_ps * 8;
394 burst = act->police.burst;
395 } else if (act->police.rate_pkt_ps > 0) {
396 /* The algorithm used to calculate rate
397 * mantissa, exponent values for a given token
398 * rate (token can be byte or packet) requires
399 * token rate to be mutiplied by 8.
400 */
401 rate = act->police.rate_pkt_ps * 8;
402 burst = act->police.burst_pkt;
403 pps = true;
404 }
405 nr_police++;
406 break;
407 case FLOW_ACTION_MARK:
408 mark = act->mark;
409 break;
410 default:
411 return -EOPNOTSUPP;
412 }
413 }
414
415 if (nr_police > 1) {
416 NL_SET_ERR_MSG_MOD(extack,
417 "rate limit police offload requires a single action");
418 return -EOPNOTSUPP;
419 }
420
421 if (nr_police)
422 return otx2_tc_act_set_police(nic, node, f, rate, burst,
423 mark, req, pps);
424
425 return 0;
426 }
427
otx2_tc_prepare_flow(struct otx2_nic * nic,struct otx2_tc_flow * node,struct flow_cls_offload * f,struct npc_install_flow_req * req)428 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node,
429 struct flow_cls_offload *f,
430 struct npc_install_flow_req *req)
431 {
432 struct netlink_ext_ack *extack = f->common.extack;
433 struct flow_msg *flow_spec = &req->packet;
434 struct flow_msg *flow_mask = &req->mask;
435 struct flow_dissector *dissector;
436 struct flow_rule *rule;
437 u8 ip_proto = 0;
438
439 rule = flow_cls_offload_flow_rule(f);
440 dissector = rule->match.dissector;
441
442 if ((dissector->used_keys &
443 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
444 BIT(FLOW_DISSECTOR_KEY_BASIC) |
445 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
446 BIT(FLOW_DISSECTOR_KEY_VLAN) |
447 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
448 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
449 BIT(FLOW_DISSECTOR_KEY_PORTS) |
450 BIT(FLOW_DISSECTOR_KEY_IP)))) {
451 netdev_info(nic->netdev, "unsupported flow used key 0x%x",
452 dissector->used_keys);
453 return -EOPNOTSUPP;
454 }
455
456 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
457 struct flow_match_basic match;
458
459 flow_rule_match_basic(rule, &match);
460
461 /* All EtherTypes can be matched, no hw limitation */
462 flow_spec->etype = match.key->n_proto;
463 flow_mask->etype = match.mask->n_proto;
464 req->features |= BIT_ULL(NPC_ETYPE);
465
466 if (match.mask->ip_proto &&
467 (match.key->ip_proto != IPPROTO_TCP &&
468 match.key->ip_proto != IPPROTO_UDP &&
469 match.key->ip_proto != IPPROTO_SCTP &&
470 match.key->ip_proto != IPPROTO_ICMP &&
471 match.key->ip_proto != IPPROTO_ICMPV6)) {
472 netdev_info(nic->netdev,
473 "ip_proto=0x%x not supported\n",
474 match.key->ip_proto);
475 return -EOPNOTSUPP;
476 }
477 if (match.mask->ip_proto)
478 ip_proto = match.key->ip_proto;
479
480 if (ip_proto == IPPROTO_UDP)
481 req->features |= BIT_ULL(NPC_IPPROTO_UDP);
482 else if (ip_proto == IPPROTO_TCP)
483 req->features |= BIT_ULL(NPC_IPPROTO_TCP);
484 else if (ip_proto == IPPROTO_SCTP)
485 req->features |= BIT_ULL(NPC_IPPROTO_SCTP);
486 else if (ip_proto == IPPROTO_ICMP)
487 req->features |= BIT_ULL(NPC_IPPROTO_ICMP);
488 else if (ip_proto == IPPROTO_ICMPV6)
489 req->features |= BIT_ULL(NPC_IPPROTO_ICMP6);
490 }
491
492 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
493 struct flow_match_eth_addrs match;
494
495 flow_rule_match_eth_addrs(rule, &match);
496 if (!is_zero_ether_addr(match.mask->src)) {
497 NL_SET_ERR_MSG_MOD(extack, "src mac match not supported");
498 return -EOPNOTSUPP;
499 }
500
501 if (!is_zero_ether_addr(match.mask->dst)) {
502 ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst);
503 ether_addr_copy(flow_mask->dmac,
504 (u8 *)&match.mask->dst);
505 req->features |= BIT_ULL(NPC_DMAC);
506 }
507 }
508
509 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
510 struct flow_match_ip match;
511
512 flow_rule_match_ip(rule, &match);
513 if ((ntohs(flow_spec->etype) != ETH_P_IP) &&
514 match.mask->tos) {
515 NL_SET_ERR_MSG_MOD(extack, "tos not supported");
516 return -EOPNOTSUPP;
517 }
518 if (match.mask->ttl) {
519 NL_SET_ERR_MSG_MOD(extack, "ttl not supported");
520 return -EOPNOTSUPP;
521 }
522 flow_spec->tos = match.key->tos;
523 flow_mask->tos = match.mask->tos;
524 req->features |= BIT_ULL(NPC_TOS);
525 }
526
527 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
528 struct flow_match_vlan match;
529 u16 vlan_tci, vlan_tci_mask;
530
531 flow_rule_match_vlan(rule, &match);
532
533 if (ntohs(match.key->vlan_tpid) != ETH_P_8021Q) {
534 netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n",
535 ntohs(match.key->vlan_tpid));
536 return -EOPNOTSUPP;
537 }
538
539 if (!match.mask->vlan_id) {
540 struct flow_action_entry *act;
541 int i;
542
543 flow_action_for_each(i, act, &rule->action) {
544 if (act->id == FLOW_ACTION_DROP) {
545 netdev_err(nic->netdev,
546 "vlan tpid 0x%x with vlan_id %d is not supported for DROP rule.\n",
547 ntohs(match.key->vlan_tpid),
548 match.key->vlan_id);
549 return -EOPNOTSUPP;
550 }
551 }
552 }
553
554 if (match.mask->vlan_id ||
555 match.mask->vlan_dei ||
556 match.mask->vlan_priority) {
557 vlan_tci = match.key->vlan_id |
558 match.key->vlan_dei << 12 |
559 match.key->vlan_priority << 13;
560
561 vlan_tci_mask = match.mask->vlan_id |
562 match.mask->vlan_dei << 12 |
563 match.mask->vlan_priority << 13;
564
565 flow_spec->vlan_tci = htons(vlan_tci);
566 flow_mask->vlan_tci = htons(vlan_tci_mask);
567 req->features |= BIT_ULL(NPC_OUTER_VID);
568 }
569 }
570
571 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
572 struct flow_match_ipv4_addrs match;
573
574 flow_rule_match_ipv4_addrs(rule, &match);
575
576 flow_spec->ip4dst = match.key->dst;
577 flow_mask->ip4dst = match.mask->dst;
578 req->features |= BIT_ULL(NPC_DIP_IPV4);
579
580 flow_spec->ip4src = match.key->src;
581 flow_mask->ip4src = match.mask->src;
582 req->features |= BIT_ULL(NPC_SIP_IPV4);
583 } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
584 struct flow_match_ipv6_addrs match;
585
586 flow_rule_match_ipv6_addrs(rule, &match);
587
588 if (ipv6_addr_loopback(&match.key->dst) ||
589 ipv6_addr_loopback(&match.key->src)) {
590 NL_SET_ERR_MSG_MOD(extack,
591 "Flow matching IPv6 loopback addr not supported");
592 return -EOPNOTSUPP;
593 }
594
595 if (!ipv6_addr_any(&match.mask->dst)) {
596 memcpy(&flow_spec->ip6dst,
597 (struct in6_addr *)&match.key->dst,
598 sizeof(flow_spec->ip6dst));
599 memcpy(&flow_mask->ip6dst,
600 (struct in6_addr *)&match.mask->dst,
601 sizeof(flow_spec->ip6dst));
602 req->features |= BIT_ULL(NPC_DIP_IPV6);
603 }
604
605 if (!ipv6_addr_any(&match.mask->src)) {
606 memcpy(&flow_spec->ip6src,
607 (struct in6_addr *)&match.key->src,
608 sizeof(flow_spec->ip6src));
609 memcpy(&flow_mask->ip6src,
610 (struct in6_addr *)&match.mask->src,
611 sizeof(flow_spec->ip6src));
612 req->features |= BIT_ULL(NPC_SIP_IPV6);
613 }
614 }
615
616 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
617 struct flow_match_ports match;
618
619 flow_rule_match_ports(rule, &match);
620
621 flow_spec->dport = match.key->dst;
622 flow_mask->dport = match.mask->dst;
623
624 if (flow_mask->dport) {
625 if (ip_proto == IPPROTO_UDP)
626 req->features |= BIT_ULL(NPC_DPORT_UDP);
627 else if (ip_proto == IPPROTO_TCP)
628 req->features |= BIT_ULL(NPC_DPORT_TCP);
629 else if (ip_proto == IPPROTO_SCTP)
630 req->features |= BIT_ULL(NPC_DPORT_SCTP);
631 }
632
633 flow_spec->sport = match.key->src;
634 flow_mask->sport = match.mask->src;
635
636 if (flow_mask->sport) {
637 if (ip_proto == IPPROTO_UDP)
638 req->features |= BIT_ULL(NPC_SPORT_UDP);
639 else if (ip_proto == IPPROTO_TCP)
640 req->features |= BIT_ULL(NPC_SPORT_TCP);
641 else if (ip_proto == IPPROTO_SCTP)
642 req->features |= BIT_ULL(NPC_SPORT_SCTP);
643 }
644 }
645
646 return otx2_tc_parse_actions(nic, &rule->action, req, f, node);
647 }
648
otx2_del_mcam_flow_entry(struct otx2_nic * nic,u16 entry)649 static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry)
650 {
651 struct npc_delete_flow_req *req;
652 int err;
653
654 mutex_lock(&nic->mbox.lock);
655 req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox);
656 if (!req) {
657 mutex_unlock(&nic->mbox.lock);
658 return -ENOMEM;
659 }
660
661 req->entry = entry;
662
663 /* Send message to AF */
664 err = otx2_sync_mbox_msg(&nic->mbox);
665 if (err) {
666 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n",
667 entry);
668 mutex_unlock(&nic->mbox.lock);
669 return -EFAULT;
670 }
671 mutex_unlock(&nic->mbox.lock);
672
673 return 0;
674 }
675
otx2_tc_del_flow(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)676 static int otx2_tc_del_flow(struct otx2_nic *nic,
677 struct flow_cls_offload *tc_flow_cmd)
678 {
679 struct otx2_flow_config *flow_cfg = nic->flow_cfg;
680 struct otx2_tc_info *tc_info = &nic->tc_info;
681 struct otx2_tc_flow *flow_node;
682 int err;
683
684 flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
685 &tc_flow_cmd->cookie,
686 tc_info->flow_ht_params);
687 if (!flow_node) {
688 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n",
689 tc_flow_cmd->cookie);
690 return -EINVAL;
691 }
692
693 if (flow_node->is_act_police) {
694 mutex_lock(&nic->mbox.lock);
695
696 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq,
697 flow_node->leaf_profile, false);
698 if (err)
699 netdev_err(nic->netdev,
700 "Unmapping RQ %d & profile %d failed\n",
701 flow_node->rq, flow_node->leaf_profile);
702
703 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile);
704 if (err)
705 netdev_err(nic->netdev,
706 "Unable to free leaf bandwidth profile(%d)\n",
707 flow_node->leaf_profile);
708
709 __clear_bit(flow_node->rq, &nic->rq_bmap);
710
711 mutex_unlock(&nic->mbox.lock);
712 }
713
714 otx2_del_mcam_flow_entry(nic, flow_node->entry);
715
716 WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table,
717 &flow_node->node,
718 nic->tc_info.flow_ht_params));
719 kfree_rcu(flow_node, rcu);
720
721 clear_bit(flow_node->bitpos, tc_info->tc_entries_bitmap);
722 flow_cfg->nr_flows--;
723
724 return 0;
725 }
726
otx2_tc_add_flow(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)727 static int otx2_tc_add_flow(struct otx2_nic *nic,
728 struct flow_cls_offload *tc_flow_cmd)
729 {
730 struct netlink_ext_ack *extack = tc_flow_cmd->common.extack;
731 struct otx2_flow_config *flow_cfg = nic->flow_cfg;
732 struct otx2_tc_info *tc_info = &nic->tc_info;
733 struct otx2_tc_flow *new_node, *old_node;
734 struct npc_install_flow_req *req, dummy;
735 int rc, err;
736
737 if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT))
738 return -ENOMEM;
739
740 if (bitmap_full(tc_info->tc_entries_bitmap, flow_cfg->max_flows)) {
741 NL_SET_ERR_MSG_MOD(extack,
742 "Free MCAM entry not available to add the flow");
743 return -ENOMEM;
744 }
745
746 /* allocate memory for the new flow and it's node */
747 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL);
748 if (!new_node)
749 return -ENOMEM;
750 spin_lock_init(&new_node->lock);
751 new_node->cookie = tc_flow_cmd->cookie;
752
753 memset(&dummy, 0, sizeof(struct npc_install_flow_req));
754
755 rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy);
756 if (rc) {
757 kfree_rcu(new_node, rcu);
758 return rc;
759 }
760
761 /* If a flow exists with the same cookie, delete it */
762 old_node = rhashtable_lookup_fast(&tc_info->flow_table,
763 &tc_flow_cmd->cookie,
764 tc_info->flow_ht_params);
765 if (old_node)
766 otx2_tc_del_flow(nic, tc_flow_cmd);
767
768 mutex_lock(&nic->mbox.lock);
769 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
770 if (!req) {
771 mutex_unlock(&nic->mbox.lock);
772 rc = -ENOMEM;
773 goto free_leaf;
774 }
775
776 memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr));
777 memcpy(req, &dummy, sizeof(struct npc_install_flow_req));
778
779 new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap,
780 flow_cfg->max_flows);
781 req->channel = nic->hw.rx_chan_base;
782 req->entry = flow_cfg->flow_ent[flow_cfg->max_flows - new_node->bitpos - 1];
783 req->intf = NIX_INTF_RX;
784 req->set_cntr = 1;
785 new_node->entry = req->entry;
786
787 /* Send message to AF */
788 rc = otx2_sync_mbox_msg(&nic->mbox);
789 if (rc) {
790 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry");
791 mutex_unlock(&nic->mbox.lock);
792 kfree_rcu(new_node, rcu);
793 goto free_leaf;
794 }
795 mutex_unlock(&nic->mbox.lock);
796
797 /* add new flow to flow-table */
798 rc = rhashtable_insert_fast(&nic->tc_info.flow_table, &new_node->node,
799 nic->tc_info.flow_ht_params);
800 if (rc) {
801 otx2_del_mcam_flow_entry(nic, req->entry);
802 kfree_rcu(new_node, rcu);
803 goto free_leaf;
804 }
805
806 set_bit(new_node->bitpos, tc_info->tc_entries_bitmap);
807 flow_cfg->nr_flows++;
808
809 return 0;
810
811 free_leaf:
812 if (new_node->is_act_police) {
813 mutex_lock(&nic->mbox.lock);
814
815 err = cn10k_map_unmap_rq_policer(nic, new_node->rq,
816 new_node->leaf_profile, false);
817 if (err)
818 netdev_err(nic->netdev,
819 "Unmapping RQ %d & profile %d failed\n",
820 new_node->rq, new_node->leaf_profile);
821 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile);
822 if (err)
823 netdev_err(nic->netdev,
824 "Unable to free leaf bandwidth profile(%d)\n",
825 new_node->leaf_profile);
826
827 __clear_bit(new_node->rq, &nic->rq_bmap);
828
829 mutex_unlock(&nic->mbox.lock);
830 }
831
832 return rc;
833 }
834
otx2_tc_get_flow_stats(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)835 static int otx2_tc_get_flow_stats(struct otx2_nic *nic,
836 struct flow_cls_offload *tc_flow_cmd)
837 {
838 struct otx2_tc_info *tc_info = &nic->tc_info;
839 struct npc_mcam_get_stats_req *req;
840 struct npc_mcam_get_stats_rsp *rsp;
841 struct otx2_tc_flow_stats *stats;
842 struct otx2_tc_flow *flow_node;
843 int err;
844
845 flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
846 &tc_flow_cmd->cookie,
847 tc_info->flow_ht_params);
848 if (!flow_node) {
849 netdev_info(nic->netdev, "tc flow not found for cookie %lx",
850 tc_flow_cmd->cookie);
851 return -EINVAL;
852 }
853
854 mutex_lock(&nic->mbox.lock);
855
856 req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox);
857 if (!req) {
858 mutex_unlock(&nic->mbox.lock);
859 return -ENOMEM;
860 }
861
862 req->entry = flow_node->entry;
863
864 err = otx2_sync_mbox_msg(&nic->mbox);
865 if (err) {
866 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n",
867 req->entry);
868 mutex_unlock(&nic->mbox.lock);
869 return -EFAULT;
870 }
871
872 rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp
873 (&nic->mbox.mbox, 0, &req->hdr);
874 if (IS_ERR(rsp)) {
875 mutex_unlock(&nic->mbox.lock);
876 return PTR_ERR(rsp);
877 }
878
879 mutex_unlock(&nic->mbox.lock);
880
881 if (!rsp->stat_ena)
882 return -EINVAL;
883
884 stats = &flow_node->stats;
885
886 spin_lock(&flow_node->lock);
887 flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0,
888 FLOW_ACTION_HW_STATS_IMMEDIATE);
889 stats->pkts = rsp->stat;
890 spin_unlock(&flow_node->lock);
891
892 return 0;
893 }
894
otx2_setup_tc_cls_flower(struct otx2_nic * nic,struct flow_cls_offload * cls_flower)895 static int otx2_setup_tc_cls_flower(struct otx2_nic *nic,
896 struct flow_cls_offload *cls_flower)
897 {
898 switch (cls_flower->command) {
899 case FLOW_CLS_REPLACE:
900 return otx2_tc_add_flow(nic, cls_flower);
901 case FLOW_CLS_DESTROY:
902 return otx2_tc_del_flow(nic, cls_flower);
903 case FLOW_CLS_STATS:
904 return otx2_tc_get_flow_stats(nic, cls_flower);
905 default:
906 return -EOPNOTSUPP;
907 }
908 }
909
otx2_tc_ingress_matchall_install(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)910 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic,
911 struct tc_cls_matchall_offload *cls)
912 {
913 struct netlink_ext_ack *extack = cls->common.extack;
914 struct flow_action *actions = &cls->rule->action;
915 struct flow_action_entry *entry;
916 u64 rate;
917 int err;
918
919 err = otx2_tc_validate_flow(nic, actions, extack);
920 if (err)
921 return err;
922
923 if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) {
924 NL_SET_ERR_MSG_MOD(extack,
925 "Only one ingress MATCHALL ratelimitter can be offloaded");
926 return -ENOMEM;
927 }
928
929 entry = &cls->rule->action.entries[0];
930 switch (entry->id) {
931 case FLOW_ACTION_POLICE:
932 /* Ingress ratelimiting is not supported on OcteonTx2 */
933 if (is_dev_otx2(nic->pdev)) {
934 NL_SET_ERR_MSG_MOD(extack,
935 "Ingress policing not supported on this platform");
936 return -EOPNOTSUPP;
937 }
938
939 err = cn10k_alloc_matchall_ipolicer(nic);
940 if (err)
941 return err;
942
943 /* Convert to bits per second */
944 rate = entry->police.rate_bytes_ps * 8;
945 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate);
946 if (err)
947 return err;
948 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
949 break;
950 default:
951 NL_SET_ERR_MSG_MOD(extack,
952 "Only police action supported with Ingress MATCHALL offload");
953 return -EOPNOTSUPP;
954 }
955
956 return 0;
957 }
958
otx2_tc_ingress_matchall_delete(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)959 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic,
960 struct tc_cls_matchall_offload *cls)
961 {
962 struct netlink_ext_ack *extack = cls->common.extack;
963 int err;
964
965 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
966 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
967 return -EINVAL;
968 }
969
970 err = cn10k_free_matchall_ipolicer(nic);
971 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
972 return err;
973 }
974
otx2_setup_tc_ingress_matchall(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls_matchall)975 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic,
976 struct tc_cls_matchall_offload *cls_matchall)
977 {
978 switch (cls_matchall->command) {
979 case TC_CLSMATCHALL_REPLACE:
980 return otx2_tc_ingress_matchall_install(nic, cls_matchall);
981 case TC_CLSMATCHALL_DESTROY:
982 return otx2_tc_ingress_matchall_delete(nic, cls_matchall);
983 case TC_CLSMATCHALL_STATS:
984 default:
985 break;
986 }
987
988 return -EOPNOTSUPP;
989 }
990
otx2_setup_tc_block_ingress_cb(enum tc_setup_type type,void * type_data,void * cb_priv)991 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type,
992 void *type_data, void *cb_priv)
993 {
994 struct otx2_nic *nic = cb_priv;
995
996 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
997 return -EOPNOTSUPP;
998
999 switch (type) {
1000 case TC_SETUP_CLSFLOWER:
1001 return otx2_setup_tc_cls_flower(nic, type_data);
1002 case TC_SETUP_CLSMATCHALL:
1003 return otx2_setup_tc_ingress_matchall(nic, type_data);
1004 default:
1005 break;
1006 }
1007
1008 return -EOPNOTSUPP;
1009 }
1010
otx2_setup_tc_egress_matchall(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls_matchall)1011 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic,
1012 struct tc_cls_matchall_offload *cls_matchall)
1013 {
1014 switch (cls_matchall->command) {
1015 case TC_CLSMATCHALL_REPLACE:
1016 return otx2_tc_egress_matchall_install(nic, cls_matchall);
1017 case TC_CLSMATCHALL_DESTROY:
1018 return otx2_tc_egress_matchall_delete(nic, cls_matchall);
1019 case TC_CLSMATCHALL_STATS:
1020 default:
1021 break;
1022 }
1023
1024 return -EOPNOTSUPP;
1025 }
1026
otx2_setup_tc_block_egress_cb(enum tc_setup_type type,void * type_data,void * cb_priv)1027 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type,
1028 void *type_data, void *cb_priv)
1029 {
1030 struct otx2_nic *nic = cb_priv;
1031
1032 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
1033 return -EOPNOTSUPP;
1034
1035 switch (type) {
1036 case TC_SETUP_CLSMATCHALL:
1037 return otx2_setup_tc_egress_matchall(nic, type_data);
1038 default:
1039 break;
1040 }
1041
1042 return -EOPNOTSUPP;
1043 }
1044
1045 static LIST_HEAD(otx2_block_cb_list);
1046
otx2_setup_tc_block(struct net_device * netdev,struct flow_block_offload * f)1047 static int otx2_setup_tc_block(struct net_device *netdev,
1048 struct flow_block_offload *f)
1049 {
1050 struct otx2_nic *nic = netdev_priv(netdev);
1051 flow_setup_cb_t *cb;
1052 bool ingress;
1053
1054 if (f->block_shared)
1055 return -EOPNOTSUPP;
1056
1057 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
1058 cb = otx2_setup_tc_block_ingress_cb;
1059 ingress = true;
1060 } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
1061 cb = otx2_setup_tc_block_egress_cb;
1062 ingress = false;
1063 } else {
1064 return -EOPNOTSUPP;
1065 }
1066
1067 return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb,
1068 nic, nic, ingress);
1069 }
1070
otx2_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)1071 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type,
1072 void *type_data)
1073 {
1074 switch (type) {
1075 case TC_SETUP_BLOCK:
1076 return otx2_setup_tc_block(netdev, type_data);
1077 default:
1078 return -EOPNOTSUPP;
1079 }
1080 }
1081
1082 static const struct rhashtable_params tc_flow_ht_params = {
1083 .head_offset = offsetof(struct otx2_tc_flow, node),
1084 .key_offset = offsetof(struct otx2_tc_flow, cookie),
1085 .key_len = sizeof(((struct otx2_tc_flow *)0)->cookie),
1086 .automatic_shrinking = true,
1087 };
1088
otx2_init_tc(struct otx2_nic * nic)1089 int otx2_init_tc(struct otx2_nic *nic)
1090 {
1091 struct otx2_tc_info *tc = &nic->tc_info;
1092 int err;
1093
1094 /* Exclude receive queue 0 being used for police action */
1095 set_bit(0, &nic->rq_bmap);
1096
1097 if (!nic->flow_cfg) {
1098 netdev_err(nic->netdev,
1099 "Can't init TC, nic->flow_cfg is not setup\n");
1100 return -EINVAL;
1101 }
1102
1103 err = otx2_tc_alloc_ent_bitmap(nic);
1104 if (err)
1105 return err;
1106
1107 tc->flow_ht_params = tc_flow_ht_params;
1108 err = rhashtable_init(&tc->flow_table, &tc->flow_ht_params);
1109 if (err) {
1110 kfree(tc->tc_entries_bitmap);
1111 tc->tc_entries_bitmap = NULL;
1112 }
1113 return err;
1114 }
1115
otx2_shutdown_tc(struct otx2_nic * nic)1116 void otx2_shutdown_tc(struct otx2_nic *nic)
1117 {
1118 struct otx2_tc_info *tc = &nic->tc_info;
1119
1120 kfree(tc->tc_entries_bitmap);
1121 rhashtable_destroy(&tc->flow_table);
1122 }
1123