• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2020 Mellanox Technologies
3 
4 #include "en/ptp.h"
5 #include "en/txrx.h"
6 #include "en/params.h"
7 #include "en/fs_tt_redirect.h"
8 
9 struct mlx5e_ptp_fs {
10 	struct mlx5_flow_handle *l2_rule;
11 	struct mlx5_flow_handle *udp_v4_rule;
12 	struct mlx5_flow_handle *udp_v6_rule;
13 	bool valid;
14 };
15 
16 struct mlx5e_ptp_params {
17 	struct mlx5e_params params;
18 	struct mlx5e_sq_param txq_sq_param;
19 	struct mlx5e_rq_param rq_param;
20 };
21 
22 struct mlx5e_skb_cb_hwtstamp {
23 	ktime_t cqe_hwtstamp;
24 	ktime_t port_hwtstamp;
25 };
26 
mlx5e_skb_cb_hwtstamp_init(struct sk_buff * skb)27 void mlx5e_skb_cb_hwtstamp_init(struct sk_buff *skb)
28 {
29 	memset(skb->cb, 0, sizeof(struct mlx5e_skb_cb_hwtstamp));
30 }
31 
mlx5e_skb_cb_get_hwts(struct sk_buff * skb)32 static struct mlx5e_skb_cb_hwtstamp *mlx5e_skb_cb_get_hwts(struct sk_buff *skb)
33 {
34 	BUILD_BUG_ON(sizeof(struct mlx5e_skb_cb_hwtstamp) > sizeof(skb->cb));
35 	return (struct mlx5e_skb_cb_hwtstamp *)skb->cb;
36 }
37 
mlx5e_skb_cb_hwtstamp_tx(struct sk_buff * skb,struct mlx5e_ptp_cq_stats * cq_stats)38 static void mlx5e_skb_cb_hwtstamp_tx(struct sk_buff *skb,
39 				     struct mlx5e_ptp_cq_stats *cq_stats)
40 {
41 	struct skb_shared_hwtstamps hwts = {};
42 	ktime_t diff;
43 
44 	diff = abs(mlx5e_skb_cb_get_hwts(skb)->port_hwtstamp -
45 		   mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp);
46 
47 	/* Maximal allowed diff is 1 / 128 second */
48 	if (diff > (NSEC_PER_SEC >> 7)) {
49 		cq_stats->abort++;
50 		cq_stats->abort_abs_diff_ns += diff;
51 		return;
52 	}
53 
54 	hwts.hwtstamp = mlx5e_skb_cb_get_hwts(skb)->port_hwtstamp;
55 	skb_tstamp_tx(skb, &hwts);
56 }
57 
mlx5e_skb_cb_hwtstamp_handler(struct sk_buff * skb,int hwtstamp_type,ktime_t hwtstamp,struct mlx5e_ptp_cq_stats * cq_stats)58 void mlx5e_skb_cb_hwtstamp_handler(struct sk_buff *skb, int hwtstamp_type,
59 				   ktime_t hwtstamp,
60 				   struct mlx5e_ptp_cq_stats *cq_stats)
61 {
62 	switch (hwtstamp_type) {
63 	case (MLX5E_SKB_CB_CQE_HWTSTAMP):
64 		mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp = hwtstamp;
65 		break;
66 	case (MLX5E_SKB_CB_PORT_HWTSTAMP):
67 		mlx5e_skb_cb_get_hwts(skb)->port_hwtstamp = hwtstamp;
68 		break;
69 	}
70 
71 	/* If both CQEs arrive, check and report the port tstamp, and clear skb cb as
72 	 * skb soon to be released.
73 	 */
74 	if (!mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp ||
75 	    !mlx5e_skb_cb_get_hwts(skb)->port_hwtstamp)
76 		return;
77 
78 	mlx5e_skb_cb_hwtstamp_tx(skb, cq_stats);
79 	memset(skb->cb, 0, sizeof(struct mlx5e_skb_cb_hwtstamp));
80 }
81 
mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq * ptpsq,struct mlx5_cqe64 * cqe,int budget)82 static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq,
83 				    struct mlx5_cqe64 *cqe,
84 				    int budget)
85 {
86 	struct sk_buff *skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo);
87 	struct mlx5e_txqsq *sq = &ptpsq->txqsq;
88 	ktime_t hwtstamp;
89 
90 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
91 		ptpsq->cq_stats->err_cqe++;
92 		goto out;
93 	}
94 
95 	hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, get_cqe_ts(cqe));
96 	mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_PORT_HWTSTAMP,
97 				      hwtstamp, ptpsq->cq_stats);
98 	ptpsq->cq_stats->cqe++;
99 
100 out:
101 	napi_consume_skb(skb, budget);
102 }
103 
mlx5e_ptp_poll_ts_cq(struct mlx5e_cq * cq,int budget)104 static bool mlx5e_ptp_poll_ts_cq(struct mlx5e_cq *cq, int budget)
105 {
106 	struct mlx5e_ptpsq *ptpsq = container_of(cq, struct mlx5e_ptpsq, ts_cq);
107 	struct mlx5_cqwq *cqwq = &cq->wq;
108 	struct mlx5_cqe64 *cqe;
109 	int work_done = 0;
110 
111 	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state)))
112 		return false;
113 
114 	cqe = mlx5_cqwq_get_cqe(cqwq);
115 	if (!cqe)
116 		return false;
117 
118 	do {
119 		mlx5_cqwq_pop(cqwq);
120 
121 		mlx5e_ptp_handle_ts_cqe(ptpsq, cqe, budget);
122 	} while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
123 
124 	mlx5_cqwq_update_db_record(cqwq);
125 
126 	/* ensure cq space is freed before enabling more cqes */
127 	wmb();
128 
129 	mlx5e_txqsq_wake(&ptpsq->txqsq);
130 
131 	return work_done == budget;
132 }
133 
mlx5e_ptp_napi_poll(struct napi_struct * napi,int budget)134 static int mlx5e_ptp_napi_poll(struct napi_struct *napi, int budget)
135 {
136 	struct mlx5e_ptp *c = container_of(napi, struct mlx5e_ptp, napi);
137 	struct mlx5e_ch_stats *ch_stats = c->stats;
138 	struct mlx5e_rq *rq = &c->rq;
139 	bool busy = false;
140 	int work_done = 0;
141 	int i;
142 
143 	rcu_read_lock();
144 
145 	ch_stats->poll++;
146 
147 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
148 		for (i = 0; i < c->num_tc; i++) {
149 			busy |= mlx5e_poll_tx_cq(&c->ptpsq[i].txqsq.cq, budget);
150 			busy |= mlx5e_ptp_poll_ts_cq(&c->ptpsq[i].ts_cq, budget);
151 		}
152 	}
153 	if (test_bit(MLX5E_PTP_STATE_RX, c->state) && likely(budget)) {
154 		work_done = mlx5e_poll_rx_cq(&rq->cq, budget);
155 		busy |= work_done == budget;
156 		busy |= INDIRECT_CALL_2(rq->post_wqes,
157 					mlx5e_post_rx_mpwqes,
158 					mlx5e_post_rx_wqes,
159 					rq);
160 	}
161 
162 	if (busy) {
163 		work_done = budget;
164 		goto out;
165 	}
166 
167 	if (unlikely(!napi_complete_done(napi, work_done)))
168 		goto out;
169 
170 	ch_stats->arm++;
171 
172 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
173 		for (i = 0; i < c->num_tc; i++) {
174 			mlx5e_cq_arm(&c->ptpsq[i].txqsq.cq);
175 			mlx5e_cq_arm(&c->ptpsq[i].ts_cq);
176 		}
177 	}
178 	if (test_bit(MLX5E_PTP_STATE_RX, c->state))
179 		mlx5e_cq_arm(&rq->cq);
180 
181 out:
182 	rcu_read_unlock();
183 
184 	return work_done;
185 }
186 
mlx5e_ptp_alloc_txqsq(struct mlx5e_ptp * c,int txq_ix,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_txqsq * sq,int tc,struct mlx5e_ptpsq * ptpsq)187 static int mlx5e_ptp_alloc_txqsq(struct mlx5e_ptp *c, int txq_ix,
188 				 struct mlx5e_params *params,
189 				 struct mlx5e_sq_param *param,
190 				 struct mlx5e_txqsq *sq, int tc,
191 				 struct mlx5e_ptpsq *ptpsq)
192 {
193 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
194 	struct mlx5_core_dev *mdev = c->mdev;
195 	struct mlx5_wq_cyc *wq = &sq->wq;
196 	int err;
197 	int node;
198 
199 	sq->pdev      = c->pdev;
200 	sq->tstamp    = c->tstamp;
201 	sq->clock     = &mdev->clock;
202 	sq->mkey_be   = c->mkey_be;
203 	sq->netdev    = c->netdev;
204 	sq->priv      = c->priv;
205 	sq->mdev      = mdev;
206 	sq->ch_ix     = MLX5E_PTP_CHANNEL_IX;
207 	sq->txq_ix    = txq_ix;
208 	sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
209 	sq->min_inline_mode = params->tx_min_inline_mode;
210 	sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
211 	sq->stats     = &c->priv->ptp_stats.sq[tc];
212 	sq->ptpsq     = ptpsq;
213 	INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
214 	if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
215 		set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
216 	sq->stop_room = param->stop_room;
217 	sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
218 
219 	node = dev_to_node(mlx5_core_dma_dev(mdev));
220 
221 	param->wq.db_numa_node = node;
222 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
223 	if (err)
224 		return err;
225 	wq->db    = &wq->db[MLX5_SND_DBR];
226 
227 	err = mlx5e_alloc_txqsq_db(sq, node);
228 	if (err)
229 		goto err_sq_wq_destroy;
230 
231 	return 0;
232 
233 err_sq_wq_destroy:
234 	mlx5_wq_destroy(&sq->wq_ctrl);
235 
236 	return err;
237 }
238 
mlx5e_ptp_destroy_sq(struct mlx5_core_dev * mdev,u32 sqn)239 static void mlx5e_ptp_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
240 {
241 	mlx5_core_destroy_sq(mdev, sqn);
242 }
243 
mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq * ptpsq,int numa)244 static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa)
245 {
246 	int wq_sz = mlx5_wq_cyc_get_size(&ptpsq->txqsq.wq);
247 
248 	ptpsq->skb_fifo.fifo = kvzalloc_node(array_size(wq_sz, sizeof(*ptpsq->skb_fifo.fifo)),
249 					     GFP_KERNEL, numa);
250 	if (!ptpsq->skb_fifo.fifo)
251 		return -ENOMEM;
252 
253 	ptpsq->skb_fifo.pc   = &ptpsq->skb_fifo_pc;
254 	ptpsq->skb_fifo.cc   = &ptpsq->skb_fifo_cc;
255 	ptpsq->skb_fifo.mask = wq_sz - 1;
256 
257 	return 0;
258 }
259 
mlx5e_ptp_drain_skb_fifo(struct mlx5e_skb_fifo * skb_fifo)260 static void mlx5e_ptp_drain_skb_fifo(struct mlx5e_skb_fifo *skb_fifo)
261 {
262 	while (*skb_fifo->pc != *skb_fifo->cc) {
263 		struct sk_buff *skb = mlx5e_skb_fifo_pop(skb_fifo);
264 
265 		dev_kfree_skb_any(skb);
266 	}
267 }
268 
mlx5e_ptp_free_traffic_db(struct mlx5e_skb_fifo * skb_fifo)269 static void mlx5e_ptp_free_traffic_db(struct mlx5e_skb_fifo *skb_fifo)
270 {
271 	mlx5e_ptp_drain_skb_fifo(skb_fifo);
272 	kvfree(skb_fifo->fifo);
273 }
274 
mlx5e_ptp_open_txqsq(struct mlx5e_ptp * c,u32 tisn,int txq_ix,struct mlx5e_ptp_params * cparams,int tc,struct mlx5e_ptpsq * ptpsq)275 static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn,
276 				int txq_ix, struct mlx5e_ptp_params *cparams,
277 				int tc, struct mlx5e_ptpsq *ptpsq)
278 {
279 	struct mlx5e_sq_param *sqp = &cparams->txq_sq_param;
280 	struct mlx5e_txqsq *txqsq = &ptpsq->txqsq;
281 	struct mlx5e_create_sq_param csp = {};
282 	int err;
283 
284 	err = mlx5e_ptp_alloc_txqsq(c, txq_ix, &cparams->params, sqp,
285 				    txqsq, tc, ptpsq);
286 	if (err)
287 		return err;
288 
289 	csp.tisn            = tisn;
290 	csp.tis_lst_sz      = 1;
291 	csp.cqn             = txqsq->cq.mcq.cqn;
292 	csp.wq_ctrl         = &txqsq->wq_ctrl;
293 	csp.min_inline_mode = txqsq->min_inline_mode;
294 	csp.ts_cqe_to_dest_cqn = ptpsq->ts_cq.mcq.cqn;
295 
296 	err = mlx5e_create_sq_rdy(c->mdev, sqp, &csp, 0, &txqsq->sqn);
297 	if (err)
298 		goto err_free_txqsq;
299 
300 	err = mlx5e_ptp_alloc_traffic_db(ptpsq,
301 					 dev_to_node(mlx5_core_dma_dev(c->mdev)));
302 	if (err)
303 		goto err_free_txqsq;
304 
305 	return 0;
306 
307 err_free_txqsq:
308 	mlx5e_free_txqsq(txqsq);
309 
310 	return err;
311 }
312 
mlx5e_ptp_close_txqsq(struct mlx5e_ptpsq * ptpsq)313 static void mlx5e_ptp_close_txqsq(struct mlx5e_ptpsq *ptpsq)
314 {
315 	struct mlx5e_txqsq *sq = &ptpsq->txqsq;
316 	struct mlx5_core_dev *mdev = sq->mdev;
317 
318 	mlx5e_ptp_free_traffic_db(&ptpsq->skb_fifo);
319 	cancel_work_sync(&sq->recover_work);
320 	mlx5e_ptp_destroy_sq(mdev, sq->sqn);
321 	mlx5e_free_txqsq_descs(sq);
322 	mlx5e_free_txqsq(sq);
323 }
324 
mlx5e_ptp_open_txqsqs(struct mlx5e_ptp * c,struct mlx5e_ptp_params * cparams)325 static int mlx5e_ptp_open_txqsqs(struct mlx5e_ptp *c,
326 				 struct mlx5e_ptp_params *cparams)
327 {
328 	struct mlx5e_params *params = &cparams->params;
329 	u8 num_tc = mlx5e_get_dcb_num_tc(params);
330 	int ix_base;
331 	int err;
332 	int tc;
333 
334 	ix_base = num_tc * params->num_channels;
335 
336 	for (tc = 0; tc < num_tc; tc++) {
337 		int txq_ix = ix_base + tc;
338 
339 		err = mlx5e_ptp_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
340 					   cparams, tc, &c->ptpsq[tc]);
341 		if (err)
342 			goto close_txqsq;
343 	}
344 
345 	return 0;
346 
347 close_txqsq:
348 	for (--tc; tc >= 0; tc--)
349 		mlx5e_ptp_close_txqsq(&c->ptpsq[tc]);
350 
351 	return err;
352 }
353 
mlx5e_ptp_close_txqsqs(struct mlx5e_ptp * c)354 static void mlx5e_ptp_close_txqsqs(struct mlx5e_ptp *c)
355 {
356 	int tc;
357 
358 	for (tc = 0; tc < c->num_tc; tc++)
359 		mlx5e_ptp_close_txqsq(&c->ptpsq[tc]);
360 }
361 
mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp * c,struct mlx5e_ptp_params * cparams)362 static int mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp *c,
363 				 struct mlx5e_ptp_params *cparams)
364 {
365 	struct mlx5e_params *params = &cparams->params;
366 	struct mlx5e_create_cq_param ccp = {};
367 	struct dim_cq_moder ptp_moder = {};
368 	struct mlx5e_cq_param *cq_param;
369 	u8 num_tc;
370 	int err;
371 	int tc;
372 
373 	num_tc = mlx5e_get_dcb_num_tc(params);
374 
375 	ccp.node     = dev_to_node(mlx5_core_dma_dev(c->mdev));
376 	ccp.ch_stats = c->stats;
377 	ccp.napi     = &c->napi;
378 	ccp.ix       = MLX5E_PTP_CHANNEL_IX;
379 
380 	cq_param = &cparams->txq_sq_param.cqp;
381 
382 	for (tc = 0; tc < num_tc; tc++) {
383 		struct mlx5e_cq *cq = &c->ptpsq[tc].txqsq.cq;
384 
385 		err = mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq);
386 		if (err)
387 			goto out_err_txqsq_cq;
388 	}
389 
390 	for (tc = 0; tc < num_tc; tc++) {
391 		struct mlx5e_cq *cq = &c->ptpsq[tc].ts_cq;
392 		struct mlx5e_ptpsq *ptpsq = &c->ptpsq[tc];
393 
394 		err = mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq);
395 		if (err)
396 			goto out_err_ts_cq;
397 
398 		ptpsq->cq_stats = &c->priv->ptp_stats.cq[tc];
399 	}
400 
401 	return 0;
402 
403 out_err_ts_cq:
404 	for (--tc; tc >= 0; tc--)
405 		mlx5e_close_cq(&c->ptpsq[tc].ts_cq);
406 	tc = num_tc;
407 out_err_txqsq_cq:
408 	for (--tc; tc >= 0; tc--)
409 		mlx5e_close_cq(&c->ptpsq[tc].txqsq.cq);
410 
411 	return err;
412 }
413 
mlx5e_ptp_open_rx_cq(struct mlx5e_ptp * c,struct mlx5e_ptp_params * cparams)414 static int mlx5e_ptp_open_rx_cq(struct mlx5e_ptp *c,
415 				struct mlx5e_ptp_params *cparams)
416 {
417 	struct mlx5e_create_cq_param ccp = {};
418 	struct dim_cq_moder ptp_moder = {};
419 	struct mlx5e_cq_param *cq_param;
420 	struct mlx5e_cq *cq = &c->rq.cq;
421 
422 	ccp.node     = dev_to_node(mlx5_core_dma_dev(c->mdev));
423 	ccp.ch_stats = c->stats;
424 	ccp.napi     = &c->napi;
425 	ccp.ix       = MLX5E_PTP_CHANNEL_IX;
426 
427 	cq_param = &cparams->rq_param.cqp;
428 
429 	return mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq);
430 }
431 
mlx5e_ptp_close_tx_cqs(struct mlx5e_ptp * c)432 static void mlx5e_ptp_close_tx_cqs(struct mlx5e_ptp *c)
433 {
434 	int tc;
435 
436 	for (tc = 0; tc < c->num_tc; tc++)
437 		mlx5e_close_cq(&c->ptpsq[tc].ts_cq);
438 
439 	for (tc = 0; tc < c->num_tc; tc++)
440 		mlx5e_close_cq(&c->ptpsq[tc].txqsq.cq);
441 }
442 
mlx5e_ptp_build_sq_param(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_sq_param * param)443 static void mlx5e_ptp_build_sq_param(struct mlx5_core_dev *mdev,
444 				     struct mlx5e_params *params,
445 				     struct mlx5e_sq_param *param)
446 {
447 	void *sqc = param->sqc;
448 	void *wq;
449 
450 	mlx5e_build_sq_param_common(mdev, param);
451 
452 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
453 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
454 	param->stop_room = mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
455 	mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
456 }
457 
mlx5e_ptp_build_rq_param(struct mlx5_core_dev * mdev,struct net_device * netdev,u16 q_counter,struct mlx5e_ptp_params * ptp_params)458 static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev,
459 				     struct net_device *netdev,
460 				     u16 q_counter,
461 				     struct mlx5e_ptp_params *ptp_params)
462 {
463 	struct mlx5e_rq_param *rq_params = &ptp_params->rq_param;
464 	struct mlx5e_params *params = &ptp_params->params;
465 
466 	params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC;
467 	mlx5e_init_rq_type_params(mdev, params);
468 	params->sw_mtu = netdev->max_mtu;
469 	mlx5e_build_rq_param(mdev, params, NULL, q_counter, rq_params);
470 }
471 
mlx5e_ptp_build_params(struct mlx5e_ptp * c,struct mlx5e_ptp_params * cparams,struct mlx5e_params * orig)472 static void mlx5e_ptp_build_params(struct mlx5e_ptp *c,
473 				   struct mlx5e_ptp_params *cparams,
474 				   struct mlx5e_params *orig)
475 {
476 	struct mlx5e_params *params = &cparams->params;
477 
478 	params->tx_min_inline_mode = orig->tx_min_inline_mode;
479 	params->num_channels = orig->num_channels;
480 	params->hard_mtu = orig->hard_mtu;
481 	params->sw_mtu = orig->sw_mtu;
482 	params->mqprio = orig->mqprio;
483 
484 	/* SQ */
485 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
486 		params->log_sq_size = orig->log_sq_size;
487 		mlx5e_ptp_build_sq_param(c->mdev, params, &cparams->txq_sq_param);
488 	}
489 	/* RQ */
490 	if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
491 		params->vlan_strip_disable = orig->vlan_strip_disable;
492 		mlx5e_ptp_build_rq_param(c->mdev, c->netdev, c->priv->q_counter, cparams);
493 	}
494 }
495 
mlx5e_init_ptp_rq(struct mlx5e_ptp * c,struct mlx5e_params * params,struct mlx5e_rq * rq)496 static int mlx5e_init_ptp_rq(struct mlx5e_ptp *c, struct mlx5e_params *params,
497 			     struct mlx5e_rq *rq)
498 {
499 	struct mlx5_core_dev *mdev = c->mdev;
500 	struct mlx5e_priv *priv = c->priv;
501 	int err;
502 
503 	rq->wq_type      = params->rq_wq_type;
504 	rq->pdev         = c->pdev;
505 	rq->netdev       = priv->netdev;
506 	rq->priv         = priv;
507 	rq->clock        = &mdev->clock;
508 	rq->tstamp       = &priv->tstamp;
509 	rq->mdev         = mdev;
510 	rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
511 	rq->stats        = &c->priv->ptp_stats.rq;
512 	rq->ix           = MLX5E_PTP_CHANNEL_IX;
513 	rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
514 	err = mlx5e_rq_set_handlers(rq, params, false);
515 	if (err)
516 		return err;
517 
518 	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, 0);
519 }
520 
mlx5e_ptp_open_rq(struct mlx5e_ptp * c,struct mlx5e_params * params,struct mlx5e_rq_param * rq_param)521 static int mlx5e_ptp_open_rq(struct mlx5e_ptp *c, struct mlx5e_params *params,
522 			     struct mlx5e_rq_param *rq_param)
523 {
524 	int node = dev_to_node(c->mdev->device);
525 	int err;
526 
527 	err = mlx5e_init_ptp_rq(c, params, &c->rq);
528 	if (err)
529 		return err;
530 
531 	return mlx5e_open_rq(params, rq_param, NULL, node, &c->rq);
532 }
533 
mlx5e_ptp_open_queues(struct mlx5e_ptp * c,struct mlx5e_ptp_params * cparams)534 static int mlx5e_ptp_open_queues(struct mlx5e_ptp *c,
535 				 struct mlx5e_ptp_params *cparams)
536 {
537 	int err;
538 
539 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
540 		err = mlx5e_ptp_open_tx_cqs(c, cparams);
541 		if (err)
542 			return err;
543 
544 		err = mlx5e_ptp_open_txqsqs(c, cparams);
545 		if (err)
546 			goto close_tx_cqs;
547 	}
548 	if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
549 		err = mlx5e_ptp_open_rx_cq(c, cparams);
550 		if (err)
551 			goto close_txqsq;
552 
553 		err = mlx5e_ptp_open_rq(c, &cparams->params, &cparams->rq_param);
554 		if (err)
555 			goto close_rx_cq;
556 	}
557 	return 0;
558 
559 close_rx_cq:
560 	if (test_bit(MLX5E_PTP_STATE_RX, c->state))
561 		mlx5e_close_cq(&c->rq.cq);
562 close_txqsq:
563 	if (test_bit(MLX5E_PTP_STATE_TX, c->state))
564 		mlx5e_ptp_close_txqsqs(c);
565 close_tx_cqs:
566 	if (test_bit(MLX5E_PTP_STATE_TX, c->state))
567 		mlx5e_ptp_close_tx_cqs(c);
568 
569 	return err;
570 }
571 
mlx5e_ptp_close_queues(struct mlx5e_ptp * c)572 static void mlx5e_ptp_close_queues(struct mlx5e_ptp *c)
573 {
574 	if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
575 		mlx5e_close_rq(&c->rq);
576 		mlx5e_close_cq(&c->rq.cq);
577 	}
578 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
579 		mlx5e_ptp_close_txqsqs(c);
580 		mlx5e_ptp_close_tx_cqs(c);
581 	}
582 }
583 
mlx5e_ptp_set_state(struct mlx5e_ptp * c,struct mlx5e_params * params)584 static int mlx5e_ptp_set_state(struct mlx5e_ptp *c, struct mlx5e_params *params)
585 {
586 	if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_TX_PORT_TS))
587 		__set_bit(MLX5E_PTP_STATE_TX, c->state);
588 
589 	if (params->ptp_rx)
590 		__set_bit(MLX5E_PTP_STATE_RX, c->state);
591 
592 	return bitmap_empty(c->state, MLX5E_PTP_STATE_NUM_STATES) ? -EINVAL : 0;
593 }
594 
mlx5e_ptp_rx_unset_fs(struct mlx5e_priv * priv)595 static void mlx5e_ptp_rx_unset_fs(struct mlx5e_priv *priv)
596 {
597 	struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs;
598 
599 	if (!ptp_fs->valid)
600 		return;
601 
602 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->l2_rule);
603 	mlx5e_fs_tt_redirect_any_destroy(priv);
604 
605 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule);
606 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule);
607 	mlx5e_fs_tt_redirect_udp_destroy(priv);
608 	ptp_fs->valid = false;
609 }
610 
mlx5e_ptp_rx_set_fs(struct mlx5e_priv * priv)611 static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv)
612 {
613 	u32 tirn = mlx5e_rx_res_get_tirn_ptp(priv->rx_res);
614 	struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs;
615 	struct mlx5_flow_handle *rule;
616 	int err;
617 
618 	if (ptp_fs->valid)
619 		return 0;
620 
621 	err = mlx5e_fs_tt_redirect_udp_create(priv);
622 	if (err)
623 		goto out_free;
624 
625 	rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV4_UDP,
626 						 tirn, PTP_EV_PORT);
627 	if (IS_ERR(rule)) {
628 		err = PTR_ERR(rule);
629 		goto out_destroy_fs_udp;
630 	}
631 	ptp_fs->udp_v4_rule = rule;
632 
633 	rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV6_UDP,
634 						 tirn, PTP_EV_PORT);
635 	if (IS_ERR(rule)) {
636 		err = PTR_ERR(rule);
637 		goto out_destroy_udp_v4_rule;
638 	}
639 	ptp_fs->udp_v6_rule = rule;
640 
641 	err = mlx5e_fs_tt_redirect_any_create(priv);
642 	if (err)
643 		goto out_destroy_udp_v6_rule;
644 
645 	rule = mlx5e_fs_tt_redirect_any_add_rule(priv, tirn, ETH_P_1588);
646 	if (IS_ERR(rule)) {
647 		err = PTR_ERR(rule);
648 		goto out_destroy_fs_any;
649 	}
650 	ptp_fs->l2_rule = rule;
651 	ptp_fs->valid = true;
652 
653 	return 0;
654 
655 out_destroy_fs_any:
656 	mlx5e_fs_tt_redirect_any_destroy(priv);
657 out_destroy_udp_v6_rule:
658 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule);
659 out_destroy_udp_v4_rule:
660 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule);
661 out_destroy_fs_udp:
662 	mlx5e_fs_tt_redirect_udp_destroy(priv);
663 out_free:
664 	return err;
665 }
666 
mlx5e_ptp_open(struct mlx5e_priv * priv,struct mlx5e_params * params,u8 lag_port,struct mlx5e_ptp ** cp)667 int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
668 		   u8 lag_port, struct mlx5e_ptp **cp)
669 {
670 	struct net_device *netdev = priv->netdev;
671 	struct mlx5_core_dev *mdev = priv->mdev;
672 	struct mlx5e_ptp_params *cparams;
673 	struct mlx5e_ptp *c;
674 	int err;
675 
676 
677 	c = kvzalloc_node(sizeof(*c), GFP_KERNEL, dev_to_node(mlx5_core_dma_dev(mdev)));
678 	cparams = kvzalloc(sizeof(*cparams), GFP_KERNEL);
679 	if (!c || !cparams) {
680 		err = -ENOMEM;
681 		goto err_free;
682 	}
683 
684 	c->priv     = priv;
685 	c->mdev     = priv->mdev;
686 	c->tstamp   = &priv->tstamp;
687 	c->pdev     = mlx5_core_dma_dev(priv->mdev);
688 	c->netdev   = priv->netdev;
689 	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey.key);
690 	c->num_tc   = mlx5e_get_dcb_num_tc(params);
691 	c->stats    = &priv->ptp_stats.ch;
692 	c->lag_port = lag_port;
693 
694 	err = mlx5e_ptp_set_state(c, params);
695 	if (err)
696 		goto err_free;
697 
698 	netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll, 64);
699 
700 	mlx5e_ptp_build_params(c, cparams, params);
701 
702 	err = mlx5e_ptp_open_queues(c, cparams);
703 	if (unlikely(err))
704 		goto err_napi_del;
705 
706 	if (test_bit(MLX5E_PTP_STATE_RX, c->state))
707 		priv->rx_ptp_opened = true;
708 
709 	*cp = c;
710 
711 	kvfree(cparams);
712 
713 	return 0;
714 
715 err_napi_del:
716 	netif_napi_del(&c->napi);
717 err_free:
718 	kvfree(cparams);
719 	kvfree(c);
720 	return err;
721 }
722 
mlx5e_ptp_close(struct mlx5e_ptp * c)723 void mlx5e_ptp_close(struct mlx5e_ptp *c)
724 {
725 	mlx5e_ptp_close_queues(c);
726 	netif_napi_del(&c->napi);
727 
728 	kvfree(c);
729 }
730 
mlx5e_ptp_activate_channel(struct mlx5e_ptp * c)731 void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c)
732 {
733 	int tc;
734 
735 	napi_enable(&c->napi);
736 
737 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
738 		for (tc = 0; tc < c->num_tc; tc++)
739 			mlx5e_activate_txqsq(&c->ptpsq[tc].txqsq);
740 	}
741 	if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
742 		mlx5e_ptp_rx_set_fs(c->priv);
743 		mlx5e_activate_rq(&c->rq);
744 	}
745 }
746 
mlx5e_ptp_deactivate_channel(struct mlx5e_ptp * c)747 void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c)
748 {
749 	int tc;
750 
751 	if (test_bit(MLX5E_PTP_STATE_RX, c->state))
752 		mlx5e_deactivate_rq(&c->rq);
753 
754 	if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
755 		for (tc = 0; tc < c->num_tc; tc++)
756 			mlx5e_deactivate_txqsq(&c->ptpsq[tc].txqsq);
757 	}
758 
759 	napi_disable(&c->napi);
760 }
761 
mlx5e_ptp_get_rqn(struct mlx5e_ptp * c,u32 * rqn)762 int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn)
763 {
764 	if (!c || !test_bit(MLX5E_PTP_STATE_RX, c->state))
765 		return -EINVAL;
766 
767 	*rqn = c->rq.rqn;
768 	return 0;
769 }
770 
mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv * priv)771 int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv)
772 {
773 	struct mlx5e_ptp_fs *ptp_fs;
774 
775 	if (!priv->profile->rx_ptp_support)
776 		return 0;
777 
778 	ptp_fs = kzalloc(sizeof(*ptp_fs), GFP_KERNEL);
779 	if (!ptp_fs)
780 		return -ENOMEM;
781 
782 	priv->fs.ptp_fs = ptp_fs;
783 	return 0;
784 }
785 
mlx5e_ptp_free_rx_fs(struct mlx5e_priv * priv)786 void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv)
787 {
788 	struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs;
789 
790 	if (!priv->profile->rx_ptp_support)
791 		return;
792 
793 	mlx5e_ptp_rx_unset_fs(priv);
794 	kfree(ptp_fs);
795 }
796 
mlx5e_ptp_rx_manage_fs(struct mlx5e_priv * priv,bool set)797 int mlx5e_ptp_rx_manage_fs(struct mlx5e_priv *priv, bool set)
798 {
799 	struct mlx5e_ptp *c = priv->channels.ptp;
800 
801 	if (!priv->profile->rx_ptp_support)
802 		return 0;
803 
804 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
805 		return 0;
806 
807 	if (set) {
808 		if (!c || !test_bit(MLX5E_PTP_STATE_RX, c->state)) {
809 			netdev_WARN_ONCE(priv->netdev, "Don't try to add PTP RX-FS rules");
810 			return -EINVAL;
811 		}
812 		return mlx5e_ptp_rx_set_fs(priv);
813 	}
814 	/* set == false */
815 	if (c && test_bit(MLX5E_PTP_STATE_RX, c->state)) {
816 		netdev_WARN_ONCE(priv->netdev, "Don't try to remove PTP RX-FS rules");
817 		return -EINVAL;
818 	}
819 	mlx5e_ptp_rx_unset_fs(priv);
820 	return 0;
821 }
822