• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2021, Mellanox Technologies inc.  All rights reserved.
4  */
5 
6 #include <linux/interrupt.h>
7 #include <linux/notifier.h>
8 #include <linux/module.h>
9 #include <linux/mlx5/driver.h>
10 #include <linux/mlx5/vport.h>
11 #include <linux/mlx5/eq.h>
12 #ifdef CONFIG_RFS_ACCEL
13 #include <linux/cpu_rmap.h>
14 #endif
15 #include "mlx5_core.h"
16 #include "lib/eq.h"
17 #include "fpga/core.h"
18 #include "eswitch.h"
19 #include "lib/clock.h"
20 #include "diag/fw_tracer.h"
21 #include "mlx5_irq.h"
22 
23 enum {
24 	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
25 };
26 
27 enum {
28 	MLX5_EQ_STATE_ARMED		= 0x9,
29 	MLX5_EQ_STATE_FIRED		= 0xa,
30 	MLX5_EQ_STATE_ALWAYS_ARMED	= 0xb,
31 };
32 
33 enum {
34 	MLX5_EQ_DOORBEL_OFFSET	= 0x40,
35 };
36 
37 /* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update
38  * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is
39  * used to set the EQ size, budget must be smaller than the EQ size.
40  */
41 enum {
42 	MLX5_EQ_POLLING_BUDGET	= 128,
43 };
44 
45 static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
46 
47 struct mlx5_eq_table {
48 	struct list_head        comp_eqs_list;
49 	struct mlx5_eq_async    pages_eq;
50 	struct mlx5_eq_async    cmd_eq;
51 	struct mlx5_eq_async    async_eq;
52 
53 	struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
54 
55 	/* Since CQ DB is stored in async_eq */
56 	struct mlx5_nb          cq_err_nb;
57 
58 	struct mutex            lock; /* sync async eqs creations */
59 	int			num_comp_eqs;
60 	struct mlx5_irq_table	*irq_table;
61 #ifdef CONFIG_RFS_ACCEL
62 	struct cpu_rmap		*rmap;
63 #endif
64 };
65 
66 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
67 			       (1ull << MLX5_EVENT_TYPE_COMM_EST)	    | \
68 			       (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)	    | \
69 			       (1ull << MLX5_EVENT_TYPE_CQ_ERROR)	    | \
70 			       (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)	    | \
71 			       (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
72 			       (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
73 			       (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
74 			       (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)	    | \
75 			       (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
76 			       (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)	    | \
77 			       (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
78 
mlx5_cmd_destroy_eq(struct mlx5_core_dev * dev,u8 eqn)79 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
80 {
81 	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
82 
83 	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
84 	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
85 	return mlx5_cmd_exec_in(dev, destroy_eq, in);
86 }
87 
88 /* caller must eventually call mlx5_cq_put on the returned cq */
mlx5_eq_cq_get(struct mlx5_eq * eq,u32 cqn)89 static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn)
90 {
91 	struct mlx5_cq_table *table = &eq->cq_table;
92 	struct mlx5_core_cq *cq = NULL;
93 
94 	rcu_read_lock();
95 	cq = radix_tree_lookup(&table->tree, cqn);
96 	if (likely(cq))
97 		mlx5_cq_hold(cq);
98 	rcu_read_unlock();
99 
100 	return cq;
101 }
102 
mlx5_eq_comp_int(struct notifier_block * nb,__always_unused unsigned long action,__always_unused void * data)103 static int mlx5_eq_comp_int(struct notifier_block *nb,
104 			    __always_unused unsigned long action,
105 			    __always_unused void *data)
106 {
107 	struct mlx5_eq_comp *eq_comp =
108 		container_of(nb, struct mlx5_eq_comp, irq_nb);
109 	struct mlx5_eq *eq = &eq_comp->core;
110 	struct mlx5_eqe *eqe;
111 	int num_eqes = 0;
112 	u32 cqn = -1;
113 
114 	eqe = next_eqe_sw(eq);
115 	if (!eqe)
116 		goto out;
117 
118 	do {
119 		struct mlx5_core_cq *cq;
120 
121 		/* Make sure we read EQ entry contents after we've
122 		 * checked the ownership bit.
123 		 */
124 		dma_rmb();
125 		/* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */
126 		cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
127 
128 		cq = mlx5_eq_cq_get(eq, cqn);
129 		if (likely(cq)) {
130 			++cq->arm_sn;
131 			cq->comp(cq, eqe);
132 			mlx5_cq_put(cq);
133 		} else {
134 			dev_dbg_ratelimited(eq->dev->device,
135 					    "Completion event for bogus CQ 0x%x\n", cqn);
136 		}
137 
138 		++eq->cons_index;
139 
140 	} while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
141 
142 out:
143 	eq_update_ci(eq, 1);
144 
145 	if (cqn != -1)
146 		tasklet_schedule(&eq_comp->tasklet_ctx.task);
147 
148 	return 0;
149 }
150 
151 /* Some architectures don't latch interrupts when they are disabled, so using
152  * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
153  * avoid losing them.  It is not recommended to use it, unless this is the last
154  * resort.
155  */
mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp * eq)156 u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq)
157 {
158 	u32 count_eqe;
159 
160 	disable_irq(eq->core.irqn);
161 	count_eqe = eq->core.cons_index;
162 	mlx5_eq_comp_int(&eq->irq_nb, 0, NULL);
163 	count_eqe = eq->core.cons_index - count_eqe;
164 	enable_irq(eq->core.irqn);
165 
166 	return count_eqe;
167 }
168 
mlx5_eq_async_int_lock(struct mlx5_eq_async * eq,bool recovery,unsigned long * flags)169 static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery,
170 				   unsigned long *flags)
171 	__acquires(&eq->lock)
172 {
173 	if (!recovery)
174 		spin_lock(&eq->lock);
175 	else
176 		spin_lock_irqsave(&eq->lock, *flags);
177 }
178 
mlx5_eq_async_int_unlock(struct mlx5_eq_async * eq,bool recovery,unsigned long * flags)179 static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery,
180 				     unsigned long *flags)
181 	__releases(&eq->lock)
182 {
183 	if (!recovery)
184 		spin_unlock(&eq->lock);
185 	else
186 		spin_unlock_irqrestore(&eq->lock, *flags);
187 }
188 
189 enum async_eq_nb_action {
190 	ASYNC_EQ_IRQ_HANDLER = 0,
191 	ASYNC_EQ_RECOVER = 1,
192 };
193 
mlx5_eq_async_int(struct notifier_block * nb,unsigned long action,void * data)194 static int mlx5_eq_async_int(struct notifier_block *nb,
195 			     unsigned long action, void *data)
196 {
197 	struct mlx5_eq_async *eq_async =
198 		container_of(nb, struct mlx5_eq_async, irq_nb);
199 	struct mlx5_eq *eq = &eq_async->core;
200 	struct mlx5_eq_table *eqt;
201 	struct mlx5_core_dev *dev;
202 	struct mlx5_eqe *eqe;
203 	unsigned long flags;
204 	int num_eqes = 0;
205 	bool recovery;
206 
207 	dev = eq->dev;
208 	eqt = dev->priv.eq_table;
209 
210 	recovery = action == ASYNC_EQ_RECOVER;
211 	mlx5_eq_async_int_lock(eq_async, recovery, &flags);
212 
213 	eqe = next_eqe_sw(eq);
214 	if (!eqe)
215 		goto out;
216 
217 	do {
218 		/*
219 		 * Make sure we read EQ entry contents after we've
220 		 * checked the ownership bit.
221 		 */
222 		dma_rmb();
223 
224 		atomic_notifier_call_chain(&eqt->nh[eqe->type], eqe->type, eqe);
225 		atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe);
226 
227 		++eq->cons_index;
228 
229 	} while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
230 
231 out:
232 	eq_update_ci(eq, 1);
233 	mlx5_eq_async_int_unlock(eq_async, recovery, &flags);
234 
235 	return unlikely(recovery) ? num_eqes : 0;
236 }
237 
mlx5_cmd_eq_recover(struct mlx5_core_dev * dev)238 void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev)
239 {
240 	struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq;
241 	int eqes;
242 
243 	eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL);
244 	if (eqes)
245 		mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes);
246 }
247 
init_eq_buf(struct mlx5_eq * eq)248 static void init_eq_buf(struct mlx5_eq *eq)
249 {
250 	struct mlx5_eqe *eqe;
251 	int i;
252 
253 	for (i = 0; i < eq_get_size(eq); i++) {
254 		eqe = get_eqe(eq, i);
255 		eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
256 	}
257 }
258 
259 static int
create_map_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct mlx5_eq_param * param)260 create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
261 	      struct mlx5_eq_param *param)
262 {
263 	u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE);
264 	struct mlx5_cq_table *cq_table = &eq->cq_table;
265 	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
266 	u8 log_eq_stride = ilog2(MLX5_EQE_SIZE);
267 	struct mlx5_priv *priv = &dev->priv;
268 	u16 vecidx = param->irq_index;
269 	__be64 *pas;
270 	void *eqc;
271 	int inlen;
272 	u32 *in;
273 	int err;
274 	int i;
275 
276 	/* Init CQ table */
277 	memset(cq_table, 0, sizeof(*cq_table));
278 	spin_lock_init(&cq_table->lock);
279 	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
280 
281 	eq->cons_index = 0;
282 
283 	err = mlx5_frag_buf_alloc_node(dev, wq_get_byte_sz(log_eq_size, log_eq_stride),
284 				       &eq->frag_buf, dev->priv.numa_node);
285 	if (err)
286 		return err;
287 
288 	mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc);
289 	init_eq_buf(eq);
290 
291 	eq->irq = mlx5_irq_request(dev, vecidx, param->affinity);
292 	if (IS_ERR(eq->irq)) {
293 		err = PTR_ERR(eq->irq);
294 		goto err_buf;
295 	}
296 
297 	vecidx = mlx5_irq_get_index(eq->irq);
298 	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
299 		MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages;
300 
301 	in = kvzalloc(inlen, GFP_KERNEL);
302 	if (!in) {
303 		err = -ENOMEM;
304 		goto err_irq;
305 	}
306 
307 	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
308 	mlx5_fill_page_frag_array(&eq->frag_buf, pas);
309 
310 	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
311 	if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
312 		MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID);
313 
314 	for (i = 0; i < 4; i++)
315 		MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
316 				 param->mask[i]);
317 
318 	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
319 	MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz);
320 	MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
321 	MLX5_SET(eqc, eqc, intr, vecidx);
322 	MLX5_SET(eqc, eqc, log_page_size,
323 		 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
324 
325 	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
326 	if (err)
327 		goto err_in;
328 
329 	eq->vecidx = vecidx;
330 	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
331 	eq->irqn = pci_irq_vector(dev->pdev, vecidx);
332 	eq->dev = dev;
333 	eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
334 
335 	err = mlx5_debug_eq_add(dev, eq);
336 	if (err)
337 		goto err_eq;
338 
339 	kvfree(in);
340 	return 0;
341 
342 err_eq:
343 	mlx5_cmd_destroy_eq(dev, eq->eqn);
344 
345 err_in:
346 	kvfree(in);
347 
348 err_irq:
349 	mlx5_irq_release(eq->irq);
350 err_buf:
351 	mlx5_frag_buf_free(dev, &eq->frag_buf);
352 	return err;
353 }
354 
355 /**
356  * mlx5_eq_enable - Enable EQ for receiving EQEs
357  * @dev : Device which owns the eq
358  * @eq  : EQ to enable
359  * @nb  : Notifier call block
360  *
361  * Must be called after EQ is created in device.
362  *
363  * @return: 0 if no error
364  */
mlx5_eq_enable(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct notifier_block * nb)365 int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
366 		   struct notifier_block *nb)
367 {
368 	int err;
369 
370 	err = mlx5_irq_attach_nb(eq->irq, nb);
371 	if (!err)
372 		eq_update_ci(eq, 1);
373 
374 	return err;
375 }
376 EXPORT_SYMBOL(mlx5_eq_enable);
377 
378 /**
379  * mlx5_eq_disable - Disable EQ for receiving EQEs
380  * @dev : Device which owns the eq
381  * @eq  : EQ to disable
382  * @nb  : Notifier call block
383  *
384  * Must be called before EQ is destroyed.
385  */
mlx5_eq_disable(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct notifier_block * nb)386 void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
387 		     struct notifier_block *nb)
388 {
389 	mlx5_irq_detach_nb(eq->irq, nb);
390 }
391 EXPORT_SYMBOL(mlx5_eq_disable);
392 
destroy_unmap_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq)393 static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
394 {
395 	int err;
396 
397 	mlx5_debug_eq_remove(dev, eq);
398 
399 	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
400 	if (err)
401 		mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
402 			       eq->eqn);
403 	mlx5_irq_release(eq->irq);
404 
405 	mlx5_frag_buf_free(dev, &eq->frag_buf);
406 	return err;
407 }
408 
mlx5_eq_add_cq(struct mlx5_eq * eq,struct mlx5_core_cq * cq)409 int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
410 {
411 	struct mlx5_cq_table *table = &eq->cq_table;
412 	int err;
413 
414 	spin_lock(&table->lock);
415 	err = radix_tree_insert(&table->tree, cq->cqn, cq);
416 	spin_unlock(&table->lock);
417 
418 	return err;
419 }
420 
mlx5_eq_del_cq(struct mlx5_eq * eq,struct mlx5_core_cq * cq)421 void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
422 {
423 	struct mlx5_cq_table *table = &eq->cq_table;
424 	struct mlx5_core_cq *tmp;
425 
426 	spin_lock(&table->lock);
427 	tmp = radix_tree_delete(&table->tree, cq->cqn);
428 	spin_unlock(&table->lock);
429 
430 	if (!tmp) {
431 		mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n",
432 			      eq->eqn, cq->cqn);
433 		return;
434 	}
435 
436 	if (tmp != cq)
437 		mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n",
438 			      eq->eqn, cq->cqn);
439 }
440 
mlx5_eq_table_init(struct mlx5_core_dev * dev)441 int mlx5_eq_table_init(struct mlx5_core_dev *dev)
442 {
443 	struct mlx5_eq_table *eq_table;
444 	int i;
445 
446 	eq_table = kvzalloc(sizeof(*eq_table), GFP_KERNEL);
447 	if (!eq_table)
448 		return -ENOMEM;
449 
450 	dev->priv.eq_table = eq_table;
451 
452 	mlx5_eq_debugfs_init(dev);
453 
454 	mutex_init(&eq_table->lock);
455 	for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)
456 		ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);
457 
458 	eq_table->irq_table = mlx5_irq_table_get(dev);
459 	return 0;
460 }
461 
mlx5_eq_table_cleanup(struct mlx5_core_dev * dev)462 void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev)
463 {
464 	mlx5_eq_debugfs_cleanup(dev);
465 	kvfree(dev->priv.eq_table);
466 }
467 
468 /* Async EQs */
469 
create_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct mlx5_eq_param * param)470 static int create_async_eq(struct mlx5_core_dev *dev,
471 			   struct mlx5_eq *eq, struct mlx5_eq_param *param)
472 {
473 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
474 	int err;
475 
476 	mutex_lock(&eq_table->lock);
477 	err = create_map_eq(dev, eq, param);
478 	mutex_unlock(&eq_table->lock);
479 	return err;
480 }
481 
destroy_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq)482 static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
483 {
484 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
485 	int err;
486 
487 	mutex_lock(&eq_table->lock);
488 	err = destroy_unmap_eq(dev, eq);
489 	mutex_unlock(&eq_table->lock);
490 	return err;
491 }
492 
cq_err_event_notifier(struct notifier_block * nb,unsigned long type,void * data)493 static int cq_err_event_notifier(struct notifier_block *nb,
494 				 unsigned long type, void *data)
495 {
496 	struct mlx5_eq_table *eqt;
497 	struct mlx5_core_cq *cq;
498 	struct mlx5_eqe *eqe;
499 	struct mlx5_eq *eq;
500 	u32 cqn;
501 
502 	/* type == MLX5_EVENT_TYPE_CQ_ERROR */
503 
504 	eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
505 	eq  = &eqt->async_eq.core;
506 	eqe = data;
507 
508 	cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
509 	mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n",
510 		       cqn, eqe->data.cq_err.syndrome);
511 
512 	cq = mlx5_eq_cq_get(eq, cqn);
513 	if (unlikely(!cq)) {
514 		mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn);
515 		return NOTIFY_OK;
516 	}
517 
518 	if (cq->event)
519 		cq->event(cq, type);
520 
521 	mlx5_cq_put(cq);
522 
523 	return NOTIFY_OK;
524 }
525 
gather_user_async_events(struct mlx5_core_dev * dev,u64 mask[4])526 static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4])
527 {
528 	__be64 *user_unaffiliated_events;
529 	__be64 *user_affiliated_events;
530 	int i;
531 
532 	user_affiliated_events =
533 		MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
534 	user_unaffiliated_events =
535 		MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
536 
537 	for (i = 0; i < 4; i++)
538 		mask[i] |= be64_to_cpu(user_affiliated_events[i] |
539 				       user_unaffiliated_events[i]);
540 }
541 
gather_async_events_mask(struct mlx5_core_dev * dev,u64 mask[4])542 static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
543 {
544 	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
545 
546 	if (MLX5_VPORT_MANAGER(dev))
547 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
548 
549 	if (MLX5_CAP_GEN(dev, general_notification_event))
550 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT);
551 
552 	if (MLX5_CAP_GEN(dev, port_module_event))
553 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
554 	else
555 		mlx5_core_dbg(dev, "port_module_event is not set\n");
556 
557 	if (MLX5_PPS_CAP(dev))
558 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT);
559 
560 	if (MLX5_CAP_GEN(dev, fpga))
561 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) |
562 				    (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR);
563 	if (MLX5_CAP_GEN_MAX(dev, dct))
564 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED);
565 
566 	if (MLX5_CAP_GEN(dev, temp_warn_event))
567 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT);
568 
569 	if (MLX5_CAP_MCAM_REG(dev, tracer_registers))
570 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER);
571 
572 	if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters))
573 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER);
574 
575 	if (mlx5_eswitch_is_funcs_handler(dev))
576 		async_event_mask |=
577 			(1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
578 
579 	if (MLX5_CAP_GEN_MAX(dev, vhca_state))
580 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);
581 
582 	mask[0] = async_event_mask;
583 
584 	if (MLX5_CAP_GEN(dev, event_cap))
585 		gather_user_async_events(dev, mask);
586 }
587 
588 static int
setup_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq_async * eq,struct mlx5_eq_param * param,const char * name)589 setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq,
590 	       struct mlx5_eq_param *param, const char *name)
591 {
592 	int err;
593 
594 	eq->irq_nb.notifier_call = mlx5_eq_async_int;
595 	spin_lock_init(&eq->lock);
596 	if (!zalloc_cpumask_var(&param->affinity, GFP_KERNEL))
597 		return -ENOMEM;
598 
599 	err = create_async_eq(dev, &eq->core, param);
600 	free_cpumask_var(param->affinity);
601 	if (err) {
602 		mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);
603 		return err;
604 	}
605 	err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
606 	if (err) {
607 		mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err);
608 		destroy_async_eq(dev, &eq->core);
609 	}
610 	return err;
611 }
612 
cleanup_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq_async * eq,const char * name)613 static void cleanup_async_eq(struct mlx5_core_dev *dev,
614 			     struct mlx5_eq_async *eq, const char *name)
615 {
616 	int err;
617 
618 	mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
619 	err = destroy_async_eq(dev, &eq->core);
620 	if (err)
621 		mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n",
622 			      name, err);
623 }
624 
create_async_eqs(struct mlx5_core_dev * dev)625 static int create_async_eqs(struct mlx5_core_dev *dev)
626 {
627 	struct mlx5_eq_table *table = dev->priv.eq_table;
628 	struct mlx5_eq_param param = {};
629 	int err;
630 
631 	MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
632 	mlx5_eq_notifier_register(dev, &table->cq_err_nb);
633 
634 	param = (struct mlx5_eq_param) {
635 		.nent = MLX5_NUM_CMD_EQE,
636 		.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD,
637 	};
638 	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ);
639 	err = setup_async_eq(dev, &table->cmd_eq, &param, "cmd");
640 	if (err)
641 		goto err1;
642 
643 	mlx5_cmd_use_events(dev);
644 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
645 
646 	param = (struct mlx5_eq_param) {
647 		.nent = MLX5_NUM_ASYNC_EQE,
648 	};
649 
650 	gather_async_events_mask(dev, param.mask);
651 	err = setup_async_eq(dev, &table->async_eq, &param, "async");
652 	if (err)
653 		goto err2;
654 
655 	param = (struct mlx5_eq_param) {
656 		.nent = /* TODO: sriov max_vf + */ 1,
657 		.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST,
658 	};
659 
660 	err = setup_async_eq(dev, &table->pages_eq, &param, "pages");
661 	if (err)
662 		goto err3;
663 
664 	return 0;
665 
666 err3:
667 	cleanup_async_eq(dev, &table->async_eq, "async");
668 err2:
669 	mlx5_cmd_use_polling(dev);
670 	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
671 err1:
672 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
673 	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
674 	return err;
675 }
676 
destroy_async_eqs(struct mlx5_core_dev * dev)677 static void destroy_async_eqs(struct mlx5_core_dev *dev)
678 {
679 	struct mlx5_eq_table *table = dev->priv.eq_table;
680 
681 	cleanup_async_eq(dev, &table->pages_eq, "pages");
682 	cleanup_async_eq(dev, &table->async_eq, "async");
683 	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ);
684 	mlx5_cmd_use_polling(dev);
685 	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
686 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
687 	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
688 }
689 
mlx5_get_async_eq(struct mlx5_core_dev * dev)690 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
691 {
692 	return &dev->priv.eq_table->async_eq.core;
693 }
694 
mlx5_eq_synchronize_async_irq(struct mlx5_core_dev * dev)695 void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
696 {
697 	synchronize_irq(dev->priv.eq_table->async_eq.core.irqn);
698 }
699 
mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev * dev)700 void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
701 {
702 	synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn);
703 }
704 
705 /* Generic EQ API for mlx5_core consumers
706  * Needed For RDMA ODP EQ for now
707  */
708 struct mlx5_eq *
mlx5_eq_create_generic(struct mlx5_core_dev * dev,struct mlx5_eq_param * param)709 mlx5_eq_create_generic(struct mlx5_core_dev *dev,
710 		       struct mlx5_eq_param *param)
711 {
712 	struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL);
713 	int err;
714 
715 	if (!cpumask_available(param->affinity))
716 		return ERR_PTR(-EINVAL);
717 
718 	if (!eq)
719 		return ERR_PTR(-ENOMEM);
720 
721 	err = create_async_eq(dev, eq, param);
722 	if (err) {
723 		kvfree(eq);
724 		eq = ERR_PTR(err);
725 	}
726 
727 	return eq;
728 }
729 EXPORT_SYMBOL(mlx5_eq_create_generic);
730 
mlx5_eq_destroy_generic(struct mlx5_core_dev * dev,struct mlx5_eq * eq)731 int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
732 {
733 	int err;
734 
735 	if (IS_ERR(eq))
736 		return -EINVAL;
737 
738 	err = destroy_async_eq(dev, eq);
739 	if (err)
740 		goto out;
741 
742 	kvfree(eq);
743 out:
744 	return err;
745 }
746 EXPORT_SYMBOL(mlx5_eq_destroy_generic);
747 
mlx5_eq_get_eqe(struct mlx5_eq * eq,u32 cc)748 struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc)
749 {
750 	u32 ci = eq->cons_index + cc;
751 	u32 nent = eq_get_size(eq);
752 	struct mlx5_eqe *eqe;
753 
754 	eqe = get_eqe(eq, ci & (nent - 1));
755 	eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe;
756 	/* Make sure we read EQ entry contents after we've
757 	 * checked the ownership bit.
758 	 */
759 	if (eqe)
760 		dma_rmb();
761 
762 	return eqe;
763 }
764 EXPORT_SYMBOL(mlx5_eq_get_eqe);
765 
mlx5_eq_update_ci(struct mlx5_eq * eq,u32 cc,bool arm)766 void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
767 {
768 	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
769 	u32 val;
770 
771 	eq->cons_index += cc;
772 	val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
773 
774 	__raw_writel((__force u32)cpu_to_be32(val), addr);
775 	/* We still want ordering, just not swabbing, so add a barrier */
776 	wmb();
777 }
778 EXPORT_SYMBOL(mlx5_eq_update_ci);
779 
destroy_comp_eqs(struct mlx5_core_dev * dev)780 static void destroy_comp_eqs(struct mlx5_core_dev *dev)
781 {
782 	struct mlx5_eq_table *table = dev->priv.eq_table;
783 	struct mlx5_eq_comp *eq, *n;
784 
785 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
786 		list_del(&eq->list);
787 		mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
788 		if (destroy_unmap_eq(dev, &eq->core))
789 			mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n",
790 				       eq->core.eqn);
791 		tasklet_disable(&eq->tasklet_ctx.task);
792 		kfree(eq);
793 	}
794 }
795 
create_comp_eqs(struct mlx5_core_dev * dev)796 static int create_comp_eqs(struct mlx5_core_dev *dev)
797 {
798 	struct mlx5_eq_table *table = dev->priv.eq_table;
799 	struct mlx5_eq_comp *eq;
800 	int ncomp_eqs;
801 	int nent;
802 	int err;
803 	int i;
804 
805 	INIT_LIST_HEAD(&table->comp_eqs_list);
806 	ncomp_eqs = table->num_comp_eqs;
807 	nent = MLX5_COMP_EQ_SIZE;
808 	for (i = 0; i < ncomp_eqs; i++) {
809 		int vecidx = i + MLX5_IRQ_VEC_COMP_BASE;
810 		struct mlx5_eq_param param = {};
811 
812 		eq = kzalloc(sizeof(*eq), GFP_KERNEL);
813 		if (!eq) {
814 			err = -ENOMEM;
815 			goto clean;
816 		}
817 
818 		INIT_LIST_HEAD(&eq->tasklet_ctx.list);
819 		INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
820 		spin_lock_init(&eq->tasklet_ctx.lock);
821 		tasklet_setup(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb);
822 
823 		eq->irq_nb.notifier_call = mlx5_eq_comp_int;
824 		param = (struct mlx5_eq_param) {
825 			.irq_index = vecidx,
826 			.nent = nent,
827 		};
828 
829 		if (!zalloc_cpumask_var(&param.affinity, GFP_KERNEL)) {
830 			err = -ENOMEM;
831 			goto clean_eq;
832 		}
833 		cpumask_set_cpu(cpumask_local_spread(i, dev->priv.numa_node),
834 				param.affinity);
835 		err = create_map_eq(dev, &eq->core, &param);
836 		free_cpumask_var(param.affinity);
837 		if (err)
838 			goto clean_eq;
839 		err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
840 		if (err) {
841 			destroy_unmap_eq(dev, &eq->core);
842 			goto clean_eq;
843 		}
844 
845 		mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
846 		/* add tail, to keep the list ordered, for mlx5_vector2eqn to work */
847 		list_add_tail(&eq->list, &table->comp_eqs_list);
848 	}
849 
850 	return 0;
851 clean_eq:
852 	kfree(eq);
853 clean:
854 	destroy_comp_eqs(dev);
855 	return err;
856 }
857 
vector2eqnirqn(struct mlx5_core_dev * dev,int vector,int * eqn,unsigned int * irqn)858 static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn,
859 			  unsigned int *irqn)
860 {
861 	struct mlx5_eq_table *table = dev->priv.eq_table;
862 	struct mlx5_eq_comp *eq, *n;
863 	int err = -ENOENT;
864 	int i = 0;
865 
866 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
867 		if (i++ == vector) {
868 			if (irqn)
869 				*irqn = eq->core.irqn;
870 			if (eqn)
871 				*eqn = eq->core.eqn;
872 			err = 0;
873 			break;
874 		}
875 	}
876 
877 	return err;
878 }
879 
mlx5_vector2eqn(struct mlx5_core_dev * dev,int vector,int * eqn)880 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn)
881 {
882 	return vector2eqnirqn(dev, vector, eqn, NULL);
883 }
884 EXPORT_SYMBOL(mlx5_vector2eqn);
885 
mlx5_vector2irqn(struct mlx5_core_dev * dev,int vector,unsigned int * irqn)886 int mlx5_vector2irqn(struct mlx5_core_dev *dev, int vector, unsigned int *irqn)
887 {
888 	return vector2eqnirqn(dev, vector, NULL, irqn);
889 }
890 
mlx5_comp_vectors_count(struct mlx5_core_dev * dev)891 unsigned int mlx5_comp_vectors_count(struct mlx5_core_dev *dev)
892 {
893 	return dev->priv.eq_table->num_comp_eqs;
894 }
895 EXPORT_SYMBOL(mlx5_comp_vectors_count);
896 
897 struct cpumask *
mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev * dev,int vector)898 mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
899 {
900 	struct mlx5_eq_table *table = dev->priv.eq_table;
901 	struct mlx5_eq_comp *eq, *n;
902 	int i = 0;
903 
904 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
905 		if (i++ == vector)
906 			break;
907 	}
908 
909 	return mlx5_irq_get_affinity_mask(eq->core.irq);
910 }
911 EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask);
912 
913 #ifdef CONFIG_RFS_ACCEL
mlx5_eq_table_get_rmap(struct mlx5_core_dev * dev)914 struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev)
915 {
916 	return dev->priv.eq_table->rmap;
917 }
918 #endif
919 
mlx5_eqn2comp_eq(struct mlx5_core_dev * dev,int eqn)920 struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
921 {
922 	struct mlx5_eq_table *table = dev->priv.eq_table;
923 	struct mlx5_eq_comp *eq;
924 
925 	list_for_each_entry(eq, &table->comp_eqs_list, list) {
926 		if (eq->core.eqn == eqn)
927 			return eq;
928 	}
929 
930 	return ERR_PTR(-ENOENT);
931 }
932 
clear_rmap(struct mlx5_core_dev * dev)933 static void clear_rmap(struct mlx5_core_dev *dev)
934 {
935 #ifdef CONFIG_RFS_ACCEL
936 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
937 
938 	free_irq_cpu_rmap(eq_table->rmap);
939 #endif
940 }
941 
set_rmap(struct mlx5_core_dev * mdev)942 static int set_rmap(struct mlx5_core_dev *mdev)
943 {
944 	int err = 0;
945 #ifdef CONFIG_RFS_ACCEL
946 	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
947 	int vecidx;
948 
949 	eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs);
950 	if (!eq_table->rmap) {
951 		err = -ENOMEM;
952 		mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
953 		goto err_out;
954 	}
955 
956 	vecidx = MLX5_IRQ_VEC_COMP_BASE;
957 	for (; vecidx < eq_table->num_comp_eqs + MLX5_IRQ_VEC_COMP_BASE;
958 	     vecidx++) {
959 		err = irq_cpu_rmap_add(eq_table->rmap,
960 				       pci_irq_vector(mdev->pdev, vecidx));
961 		if (err) {
962 			mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
963 				      err);
964 			goto err_irq_cpu_rmap_add;
965 		}
966 	}
967 	return 0;
968 
969 err_irq_cpu_rmap_add:
970 	clear_rmap(mdev);
971 err_out:
972 #endif
973 	return err;
974 }
975 
976 /* This function should only be called after mlx5_cmd_force_teardown_hca */
mlx5_core_eq_free_irqs(struct mlx5_core_dev * dev)977 void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
978 {
979 	struct mlx5_eq_table *table = dev->priv.eq_table;
980 
981 	mutex_lock(&table->lock); /* sync with create/destroy_async_eq */
982 	if (!mlx5_core_is_sf(dev))
983 		clear_rmap(dev);
984 	mlx5_irq_table_free_irqs(dev);
985 	mutex_unlock(&table->lock);
986 }
987 
988 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
989 #define MLX5_MAX_ASYNC_EQS 4
990 #else
991 #define MLX5_MAX_ASYNC_EQS 3
992 #endif
993 
mlx5_eq_table_create(struct mlx5_core_dev * dev)994 int mlx5_eq_table_create(struct mlx5_core_dev *dev)
995 {
996 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
997 	int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
998 		      MLX5_CAP_GEN(dev, max_num_eqs) :
999 		      1 << MLX5_CAP_GEN(dev, log_max_eq);
1000 	int max_eqs_sf;
1001 	int err;
1002 
1003 	eq_table->num_comp_eqs =
1004 		min_t(int,
1005 		      mlx5_irq_table_get_num_comp(eq_table->irq_table),
1006 		      num_eqs - MLX5_MAX_ASYNC_EQS);
1007 	if (mlx5_core_is_sf(dev)) {
1008 		max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF,
1009 				   mlx5_irq_table_get_sfs_vec(eq_table->irq_table));
1010 		eq_table->num_comp_eqs = min_t(int, eq_table->num_comp_eqs,
1011 					       max_eqs_sf);
1012 	}
1013 
1014 	err = create_async_eqs(dev);
1015 	if (err) {
1016 		mlx5_core_err(dev, "Failed to create async EQs\n");
1017 		goto err_async_eqs;
1018 	}
1019 
1020 	if (!mlx5_core_is_sf(dev)) {
1021 		/* rmap is a mapping between irq number and queue number.
1022 		 * each irq can be assign only to a single rmap.
1023 		 * since SFs share IRQs, rmap mapping cannot function correctly
1024 		 * for irqs that are shared for different core/netdev RX rings.
1025 		 * Hence we don't allow netdev rmap for SFs
1026 		 */
1027 		err = set_rmap(dev);
1028 		if (err)
1029 			goto err_rmap;
1030 	}
1031 
1032 	err = create_comp_eqs(dev);
1033 	if (err) {
1034 		mlx5_core_err(dev, "Failed to create completion EQs\n");
1035 		goto err_comp_eqs;
1036 	}
1037 
1038 	return 0;
1039 err_comp_eqs:
1040 	if (!mlx5_core_is_sf(dev))
1041 		clear_rmap(dev);
1042 err_rmap:
1043 	destroy_async_eqs(dev);
1044 err_async_eqs:
1045 	return err;
1046 }
1047 
mlx5_eq_table_destroy(struct mlx5_core_dev * dev)1048 void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
1049 {
1050 	if (!mlx5_core_is_sf(dev))
1051 		clear_rmap(dev);
1052 	destroy_comp_eqs(dev);
1053 	destroy_async_eqs(dev);
1054 }
1055 
mlx5_eq_notifier_register(struct mlx5_core_dev * dev,struct mlx5_nb * nb)1056 int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1057 {
1058 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1059 
1060 	return atomic_notifier_chain_register(&eqt->nh[nb->event_type], &nb->nb);
1061 }
1062 EXPORT_SYMBOL(mlx5_eq_notifier_register);
1063 
mlx5_eq_notifier_unregister(struct mlx5_core_dev * dev,struct mlx5_nb * nb)1064 int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1065 {
1066 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1067 
1068 	return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb);
1069 }
1070 EXPORT_SYMBOL(mlx5_eq_notifier_unregister);
1071