1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #ifndef MLX5_IB_H
34 #define MLX5_IB_H
35
36 #include <linux/kernel.h>
37 #include <linux/sched.h>
38 #include <rdma/ib_verbs.h>
39 #include <rdma/ib_smi.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/cq.h>
42 #include <linux/mlx5/qp.h>
43 #include <linux/mlx5/srq.h>
44 #include <linux/types.h>
45 #include <linux/mlx5/transobj.h>
46 #include <rdma/ib_user_verbs.h>
47 #include <rdma/mlx5-abi.h>
48
49 #define mlx5_ib_dbg(dev, format, arg...) \
50 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
51 __LINE__, current->pid, ##arg)
52
53 #define mlx5_ib_err(dev, format, arg...) \
54 pr_err("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
55 __LINE__, current->pid, ##arg)
56
57 #define mlx5_ib_warn(dev, format, arg...) \
58 pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
59 __LINE__, current->pid, ##arg)
60
61 #define field_avail(type, fld, sz) (offsetof(type, fld) + \
62 sizeof(((type *)0)->fld) <= (sz))
63 #define MLX5_IB_DEFAULT_UIDX 0xffffff
64 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
65
66 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
67
68 enum {
69 MLX5_IB_MMAP_CMD_SHIFT = 8,
70 MLX5_IB_MMAP_CMD_MASK = 0xff,
71 };
72
73 enum mlx5_ib_mmap_cmd {
74 MLX5_IB_MMAP_REGULAR_PAGE = 0,
75 MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES = 1,
76 MLX5_IB_MMAP_WC_PAGE = 2,
77 MLX5_IB_MMAP_NC_PAGE = 3,
78 /* 5 is chosen in order to be compatible with old versions of libmlx5 */
79 MLX5_IB_MMAP_CORE_CLOCK = 5,
80 };
81
82 enum {
83 MLX5_RES_SCAT_DATA32_CQE = 0x1,
84 MLX5_RES_SCAT_DATA64_CQE = 0x2,
85 MLX5_REQ_SCAT_DATA32_CQE = 0x11,
86 MLX5_REQ_SCAT_DATA64_CQE = 0x22,
87 };
88
89 enum mlx5_ib_latency_class {
90 MLX5_IB_LATENCY_CLASS_LOW,
91 MLX5_IB_LATENCY_CLASS_MEDIUM,
92 MLX5_IB_LATENCY_CLASS_HIGH,
93 };
94
95 enum mlx5_ib_mad_ifc_flags {
96 MLX5_MAD_IFC_IGNORE_MKEY = 1,
97 MLX5_MAD_IFC_IGNORE_BKEY = 2,
98 MLX5_MAD_IFC_NET_VIEW = 4,
99 };
100
101 enum {
102 MLX5_CROSS_CHANNEL_BFREG = 0,
103 };
104
105 enum {
106 MLX5_CQE_VERSION_V0,
107 MLX5_CQE_VERSION_V1,
108 };
109
110 enum {
111 MLX5_TM_MAX_RNDV_MSG_SIZE = 64,
112 MLX5_TM_MAX_SGE = 1,
113 };
114
115 struct mlx5_ib_vma_private_data {
116 struct list_head list;
117 struct vm_area_struct *vma;
118 /* protect vma_private_list add/del */
119 struct mutex *vma_private_list_mutex;
120 };
121
122 struct mlx5_ib_ucontext {
123 struct ib_ucontext ibucontext;
124 struct list_head db_page_list;
125
126 /* protect doorbell record alloc/free
127 */
128 struct mutex db_page_mutex;
129 struct mlx5_bfreg_info bfregi;
130 u8 cqe_version;
131 /* Transport Domain number */
132 u32 tdn;
133 struct list_head vma_private_list;
134 /* protect vma_private_list add/del */
135 struct mutex vma_private_list_mutex;
136
137 unsigned long upd_xlt_page;
138 /* protect ODP/KSM */
139 struct mutex upd_xlt_page_mutex;
140 u64 lib_caps;
141 };
142
to_mucontext(struct ib_ucontext * ibucontext)143 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
144 {
145 return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
146 }
147
148 struct mlx5_ib_pd {
149 struct ib_pd ibpd;
150 u32 pdn;
151 };
152
153 #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
154 #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
155 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
156 #error "Invalid number of bypass priorities"
157 #endif
158 #define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
159
160 #define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
161 #define MLX5_IB_NUM_SNIFFER_FTS 2
162 struct mlx5_ib_flow_prio {
163 struct mlx5_flow_table *flow_table;
164 unsigned int refcount;
165 };
166
167 struct mlx5_ib_flow_handler {
168 struct list_head list;
169 struct ib_flow ibflow;
170 struct mlx5_ib_flow_prio *prio;
171 struct mlx5_flow_handle *rule;
172 };
173
174 struct mlx5_ib_flow_db {
175 struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
176 struct mlx5_ib_flow_prio sniffer[MLX5_IB_NUM_SNIFFER_FTS];
177 struct mlx5_flow_table *lag_demux_ft;
178 /* Protect flow steering bypass flow tables
179 * when add/del flow rules.
180 * only single add/removal of flow steering rule could be done
181 * simultaneously.
182 */
183 struct mutex lock;
184 };
185
186 /* Use macros here so that don't have to duplicate
187 * enum ib_send_flags and enum ib_qp_type for low-level driver
188 */
189
190 #define MLX5_IB_SEND_UMR_ENABLE_MR (IB_SEND_RESERVED_START << 0)
191 #define MLX5_IB_SEND_UMR_DISABLE_MR (IB_SEND_RESERVED_START << 1)
192 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 2)
193 #define MLX5_IB_SEND_UMR_UPDATE_XLT (IB_SEND_RESERVED_START << 3)
194 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 4)
195 #define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS IB_SEND_RESERVED_END
196
197 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
198 /*
199 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
200 * creates the actual hardware QP.
201 */
202 #define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
203 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
204
205 #define MLX5_IB_UMR_OCTOWORD 16
206 #define MLX5_IB_UMR_XLT_ALIGNMENT 64
207
208 #define MLX5_IB_UPD_XLT_ZAP BIT(0)
209 #define MLX5_IB_UPD_XLT_ENABLE BIT(1)
210 #define MLX5_IB_UPD_XLT_ATOMIC BIT(2)
211 #define MLX5_IB_UPD_XLT_ADDR BIT(3)
212 #define MLX5_IB_UPD_XLT_PD BIT(4)
213 #define MLX5_IB_UPD_XLT_ACCESS BIT(5)
214 #define MLX5_IB_UPD_XLT_INDIRECT BIT(6)
215
216 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
217 *
218 * These flags are intended for internal use by the mlx5_ib driver, and they
219 * rely on the range reserved for that use in the ib_qp_create_flags enum.
220 */
221
222 /* Create a UD QP whose source QP number is 1 */
mlx5_ib_create_qp_sqpn_qp1(void)223 static inline enum ib_qp_create_flags mlx5_ib_create_qp_sqpn_qp1(void)
224 {
225 return IB_QP_CREATE_RESERVED_START;
226 }
227
228 struct wr_list {
229 u16 opcode;
230 u16 next;
231 };
232
233 enum mlx5_ib_rq_flags {
234 MLX5_IB_RQ_CVLAN_STRIPPING = 1 << 0,
235 };
236
237 struct mlx5_ib_wq {
238 u64 *wrid;
239 u32 *wr_data;
240 struct wr_list *w_list;
241 unsigned *wqe_head;
242 u16 unsig_count;
243
244 /* serialize post to the work queue
245 */
246 spinlock_t lock;
247 int wqe_cnt;
248 int max_post;
249 int max_gs;
250 int offset;
251 int wqe_shift;
252 unsigned head;
253 unsigned tail;
254 u16 cur_post;
255 u16 last_poll;
256 void *qend;
257 };
258
259 enum mlx5_ib_wq_flags {
260 MLX5_IB_WQ_FLAGS_DELAY_DROP = 0x1,
261 };
262
263 struct mlx5_ib_rwq {
264 struct ib_wq ibwq;
265 struct mlx5_core_qp core_qp;
266 u32 rq_num_pas;
267 u32 log_rq_stride;
268 u32 log_rq_size;
269 u32 rq_page_offset;
270 u32 log_page_size;
271 struct ib_umem *umem;
272 size_t buf_size;
273 unsigned int page_shift;
274 int create_type;
275 struct mlx5_db db;
276 u32 user_index;
277 u32 wqe_count;
278 u32 wqe_shift;
279 int wq_sig;
280 u32 create_flags; /* Use enum mlx5_ib_wq_flags */
281 };
282
283 enum {
284 MLX5_QP_USER,
285 MLX5_QP_KERNEL,
286 MLX5_QP_EMPTY
287 };
288
289 enum {
290 MLX5_WQ_USER,
291 MLX5_WQ_KERNEL
292 };
293
294 struct mlx5_ib_rwq_ind_table {
295 struct ib_rwq_ind_table ib_rwq_ind_tbl;
296 u32 rqtn;
297 };
298
299 struct mlx5_ib_ubuffer {
300 struct ib_umem *umem;
301 int buf_size;
302 u64 buf_addr;
303 };
304
305 struct mlx5_ib_qp_base {
306 struct mlx5_ib_qp *container_mibqp;
307 struct mlx5_core_qp mqp;
308 struct mlx5_ib_ubuffer ubuffer;
309 };
310
311 struct mlx5_ib_qp_trans {
312 struct mlx5_ib_qp_base base;
313 u16 xrcdn;
314 u8 alt_port;
315 u8 atomic_rd_en;
316 u8 resp_depth;
317 };
318
319 struct mlx5_ib_rss_qp {
320 u32 tirn;
321 };
322
323 struct mlx5_ib_rq {
324 struct mlx5_ib_qp_base base;
325 struct mlx5_ib_wq *rq;
326 struct mlx5_ib_ubuffer ubuffer;
327 struct mlx5_db *doorbell;
328 u32 tirn;
329 u8 state;
330 u32 flags;
331 };
332
333 struct mlx5_ib_sq {
334 struct mlx5_ib_qp_base base;
335 struct mlx5_ib_wq *sq;
336 struct mlx5_ib_ubuffer ubuffer;
337 struct mlx5_db *doorbell;
338 u32 tisn;
339 u8 state;
340 };
341
342 struct mlx5_ib_raw_packet_qp {
343 struct mlx5_ib_sq sq;
344 struct mlx5_ib_rq rq;
345 };
346
347 struct mlx5_bf {
348 int buf_size;
349 unsigned long offset;
350 struct mlx5_sq_bfreg *bfreg;
351 };
352
353 struct mlx5_ib_qp {
354 struct ib_qp ibqp;
355 union {
356 struct mlx5_ib_qp_trans trans_qp;
357 struct mlx5_ib_raw_packet_qp raw_packet_qp;
358 struct mlx5_ib_rss_qp rss_qp;
359 };
360 struct mlx5_buf buf;
361
362 struct mlx5_db db;
363 struct mlx5_ib_wq rq;
364
365 u8 sq_signal_bits;
366 u8 next_fence;
367 struct mlx5_ib_wq sq;
368
369 /* serialize qp state modifications
370 */
371 struct mutex mutex;
372 u32 flags;
373 u8 port;
374 u8 state;
375 int wq_sig;
376 int scat_cqe;
377 int max_inline_data;
378 struct mlx5_bf bf;
379 int has_rq;
380
381 /* only for user space QPs. For kernel
382 * we have it from the bf object
383 */
384 int bfregn;
385
386 int create_type;
387
388 /* Store signature errors */
389 bool signature_en;
390
391 struct list_head qps_list;
392 struct list_head cq_recv_list;
393 struct list_head cq_send_list;
394 u32 rate_limit;
395 u32 underlay_qpn;
396 };
397
398 struct mlx5_ib_cq_buf {
399 struct mlx5_buf buf;
400 struct ib_umem *umem;
401 int cqe_size;
402 int nent;
403 };
404
405 enum mlx5_ib_qp_flags {
406 MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
407 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
408 MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
409 MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
410 MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
411 MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
412 /* QP uses 1 as its source QP number */
413 MLX5_IB_QP_SQPN_QP1 = 1 << 6,
414 MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
415 MLX5_IB_QP_RSS = 1 << 8,
416 MLX5_IB_QP_CVLAN_STRIPPING = 1 << 9,
417 MLX5_IB_QP_UNDERLAY = 1 << 10,
418 };
419
420 struct mlx5_umr_wr {
421 struct ib_send_wr wr;
422 u64 virt_addr;
423 u64 offset;
424 struct ib_pd *pd;
425 unsigned int page_shift;
426 unsigned int xlt_size;
427 u64 length;
428 int access_flags;
429 u32 mkey;
430 u8 ignore_free_state:1;
431 };
432
umr_wr(struct ib_send_wr * wr)433 static inline struct mlx5_umr_wr *umr_wr(struct ib_send_wr *wr)
434 {
435 return container_of(wr, struct mlx5_umr_wr, wr);
436 }
437
438 struct mlx5_shared_mr_info {
439 int mr_id;
440 struct ib_umem *umem;
441 };
442
443 struct mlx5_ib_cq {
444 struct ib_cq ibcq;
445 struct mlx5_core_cq mcq;
446 struct mlx5_ib_cq_buf buf;
447 struct mlx5_db db;
448
449 /* serialize access to the CQ
450 */
451 spinlock_t lock;
452
453 /* protect resize cq
454 */
455 struct mutex resize_mutex;
456 struct mlx5_ib_cq_buf *resize_buf;
457 struct ib_umem *resize_umem;
458 int cqe_size;
459 struct list_head list_send_qp;
460 struct list_head list_recv_qp;
461 u32 create_flags;
462 struct list_head wc_list;
463 enum ib_cq_notify_flags notify_flags;
464 struct work_struct notify_work;
465 };
466
467 struct mlx5_ib_wc {
468 struct ib_wc wc;
469 struct list_head list;
470 };
471
472 struct mlx5_ib_srq {
473 struct ib_srq ibsrq;
474 struct mlx5_core_srq msrq;
475 struct mlx5_buf buf;
476 struct mlx5_db db;
477 u64 *wrid;
478 /* protect SRQ hanlding
479 */
480 spinlock_t lock;
481 int head;
482 int tail;
483 u16 wqe_ctr;
484 struct ib_umem *umem;
485 /* serialize arming a SRQ
486 */
487 struct mutex mutex;
488 int wq_sig;
489 };
490
491 struct mlx5_ib_xrcd {
492 struct ib_xrcd ibxrcd;
493 u32 xrcdn;
494 };
495
496 enum mlx5_ib_mtt_access_flags {
497 MLX5_IB_MTT_READ = (1 << 0),
498 MLX5_IB_MTT_WRITE = (1 << 1),
499 };
500
501 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
502
503 struct mlx5_ib_mr {
504 struct ib_mr ibmr;
505 void *descs;
506 dma_addr_t desc_map;
507 int ndescs;
508 int max_descs;
509 int desc_size;
510 int access_mode;
511 struct mlx5_core_mkey mmkey;
512 struct ib_umem *umem;
513 struct mlx5_shared_mr_info *smr_info;
514 struct list_head list;
515 int order;
516 bool allocated_from_cache;
517 int npages;
518 struct mlx5_ib_dev *dev;
519 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
520 struct mlx5_core_sig_ctx *sig;
521 int live;
522 void *descs_alloc;
523 int access_flags; /* Needed for rereg MR */
524
525 struct mlx5_ib_mr *parent;
526 atomic_t num_leaf_free;
527 wait_queue_head_t q_leaf_free;
528 };
529
530 struct mlx5_ib_mw {
531 struct ib_mw ibmw;
532 struct mlx5_core_mkey mmkey;
533 int ndescs;
534 };
535
536 struct mlx5_ib_umr_context {
537 struct ib_cqe cqe;
538 enum ib_wc_status status;
539 struct completion done;
540 };
541
542 struct umr_common {
543 struct ib_pd *pd;
544 struct ib_cq *cq;
545 struct ib_qp *qp;
546 /* control access to UMR QP
547 */
548 struct semaphore sem;
549 };
550
551 enum {
552 MLX5_FMR_INVALID,
553 MLX5_FMR_VALID,
554 MLX5_FMR_BUSY,
555 };
556
557 struct mlx5_cache_ent {
558 struct list_head head;
559 /* sync access to the cahce entry
560 */
561 spinlock_t lock;
562
563
564 struct dentry *dir;
565 char name[4];
566 u32 order;
567 u32 xlt;
568 u32 access_mode;
569 u32 page;
570
571 u32 size;
572 u32 cur;
573 u32 miss;
574 u32 limit;
575
576 struct dentry *fsize;
577 struct dentry *fcur;
578 struct dentry *fmiss;
579 struct dentry *flimit;
580
581 struct mlx5_ib_dev *dev;
582 struct work_struct work;
583 struct delayed_work dwork;
584 int pending;
585 struct completion compl;
586 };
587
588 struct mlx5_mr_cache {
589 struct workqueue_struct *wq;
590 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
591 int stopped;
592 struct dentry *root;
593 unsigned long last_add;
594 };
595
596 struct mlx5_ib_gsi_qp;
597
598 struct mlx5_ib_port_resources {
599 struct mlx5_ib_resources *devr;
600 struct mlx5_ib_gsi_qp *gsi;
601 struct work_struct pkey_change_work;
602 };
603
604 struct mlx5_ib_resources {
605 struct ib_cq *c0;
606 struct ib_xrcd *x0;
607 struct ib_xrcd *x1;
608 struct ib_pd *p0;
609 struct ib_srq *s0;
610 struct ib_srq *s1;
611 struct mlx5_ib_port_resources ports[2];
612 /* Protects changes to the port resources */
613 struct mutex mutex;
614 };
615
616 struct mlx5_ib_counters {
617 const char **names;
618 size_t *offsets;
619 u32 num_q_counters;
620 u32 num_cong_counters;
621 u16 set_id;
622 };
623
624 struct mlx5_ib_port {
625 struct mlx5_ib_counters cnts;
626 };
627
628 struct mlx5_roce {
629 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
630 * netdev pointer
631 */
632 rwlock_t netdev_lock;
633 struct net_device *netdev;
634 struct notifier_block nb;
635 atomic_t next_port;
636 enum ib_port_state last_port_state;
637 };
638
639 struct mlx5_ib_dbg_param {
640 int offset;
641 struct mlx5_ib_dev *dev;
642 struct dentry *dentry;
643 };
644
645 enum mlx5_ib_dbg_cc_types {
646 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE,
647 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI,
648 MLX5_IB_DBG_CC_RP_TIME_RESET,
649 MLX5_IB_DBG_CC_RP_BYTE_RESET,
650 MLX5_IB_DBG_CC_RP_THRESHOLD,
651 MLX5_IB_DBG_CC_RP_AI_RATE,
652 MLX5_IB_DBG_CC_RP_HAI_RATE,
653 MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,
654 MLX5_IB_DBG_CC_RP_MIN_RATE,
655 MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP,
656 MLX5_IB_DBG_CC_RP_DCE_TCP_G,
657 MLX5_IB_DBG_CC_RP_DCE_TCP_RTT,
658 MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,
659 MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,
660 MLX5_IB_DBG_CC_RP_GD,
661 MLX5_IB_DBG_CC_NP_CNP_DSCP,
662 MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
663 MLX5_IB_DBG_CC_NP_CNP_PRIO,
664 MLX5_IB_DBG_CC_MAX,
665 };
666
667 struct mlx5_ib_dbg_cc_params {
668 struct dentry *root;
669 struct mlx5_ib_dbg_param params[MLX5_IB_DBG_CC_MAX];
670 };
671
672 enum {
673 MLX5_MAX_DELAY_DROP_TIMEOUT_MS = 100,
674 };
675
676 struct mlx5_ib_dbg_delay_drop {
677 struct dentry *dir_debugfs;
678 struct dentry *rqs_cnt_debugfs;
679 struct dentry *events_cnt_debugfs;
680 struct dentry *timeout_debugfs;
681 };
682
683 struct mlx5_ib_delay_drop {
684 struct mlx5_ib_dev *dev;
685 struct work_struct delay_drop_work;
686 /* serialize setting of delay drop */
687 struct mutex lock;
688 u32 timeout;
689 bool activate;
690 atomic_t events_cnt;
691 atomic_t rqs_cnt;
692 struct mlx5_ib_dbg_delay_drop *dbg;
693 };
694
695 struct mlx5_ib_dev {
696 struct ib_device ib_dev;
697 struct mlx5_core_dev *mdev;
698 struct mlx5_roce roce;
699 int num_ports;
700 /* serialize update of capability mask
701 */
702 struct mutex cap_mask_mutex;
703 bool ib_active;
704 struct umr_common umrc;
705 /* sync used page count stats
706 */
707 struct mlx5_ib_resources devr;
708 struct mlx5_mr_cache cache;
709 struct timer_list delay_timer;
710 /* Prevents soft lock on massive reg MRs */
711 struct mutex slow_path_mutex;
712 int fill_delay;
713 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
714 struct ib_odp_caps odp_caps;
715 u64 odp_max_size;
716 /*
717 * Sleepable RCU that prevents destruction of MRs while they are still
718 * being used by a page fault handler.
719 */
720 struct srcu_struct mr_srcu;
721 u32 null_mkey;
722 #endif
723 struct mlx5_ib_flow_db flow_db;
724 /* protect resources needed as part of reset flow */
725 spinlock_t reset_flow_resource_lock;
726 struct list_head qp_list;
727 /* Array with num_ports elements */
728 struct mlx5_ib_port *port;
729 struct mlx5_sq_bfreg bfreg;
730 struct mlx5_sq_bfreg fp_bfreg;
731 struct mlx5_ib_delay_drop delay_drop;
732 struct mlx5_ib_dbg_cc_params *dbg_cc_params;
733
734 /* protect the user_td */
735 struct mutex lb_mutex;
736 u32 user_td;
737 u8 umr_fence;
738 };
739
to_mibcq(struct mlx5_core_cq * mcq)740 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
741 {
742 return container_of(mcq, struct mlx5_ib_cq, mcq);
743 }
744
to_mxrcd(struct ib_xrcd * ibxrcd)745 static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
746 {
747 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
748 }
749
to_mdev(struct ib_device * ibdev)750 static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
751 {
752 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
753 }
754
to_mcq(struct ib_cq * ibcq)755 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
756 {
757 return container_of(ibcq, struct mlx5_ib_cq, ibcq);
758 }
759
to_mibqp(struct mlx5_core_qp * mqp)760 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
761 {
762 return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
763 }
764
to_mibrwq(struct mlx5_core_qp * core_qp)765 static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
766 {
767 return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
768 }
769
to_mibmr(struct mlx5_core_mkey * mmkey)770 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
771 {
772 return container_of(mmkey, struct mlx5_ib_mr, mmkey);
773 }
774
to_mpd(struct ib_pd * ibpd)775 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
776 {
777 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
778 }
779
to_msrq(struct ib_srq * ibsrq)780 static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
781 {
782 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
783 }
784
to_mqp(struct ib_qp * ibqp)785 static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
786 {
787 return container_of(ibqp, struct mlx5_ib_qp, ibqp);
788 }
789
to_mrwq(struct ib_wq * ibwq)790 static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
791 {
792 return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
793 }
794
to_mrwq_ind_table(struct ib_rwq_ind_table * ib_rwq_ind_tbl)795 static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
796 {
797 return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
798 }
799
to_mibsrq(struct mlx5_core_srq * msrq)800 static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
801 {
802 return container_of(msrq, struct mlx5_ib_srq, msrq);
803 }
804
to_mmr(struct ib_mr * ibmr)805 static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
806 {
807 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
808 }
809
to_mmw(struct ib_mw * ibmw)810 static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
811 {
812 return container_of(ibmw, struct mlx5_ib_mw, ibmw);
813 }
814
815 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
816 struct mlx5_db *db);
817 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
818 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
819 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
820 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
821 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
822 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
823 const void *in_mad, void *response_mad);
824 struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
825 struct ib_udata *udata);
826 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
827 int mlx5_ib_destroy_ah(struct ib_ah *ah);
828 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
829 struct ib_srq_init_attr *init_attr,
830 struct ib_udata *udata);
831 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
832 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
833 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
834 int mlx5_ib_destroy_srq(struct ib_srq *srq);
835 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
836 struct ib_recv_wr **bad_wr);
837 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
838 struct ib_qp_init_attr *init_attr,
839 struct ib_udata *udata);
840 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
841 int attr_mask, struct ib_udata *udata);
842 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
843 struct ib_qp_init_attr *qp_init_attr);
844 int mlx5_ib_destroy_qp(struct ib_qp *qp);
845 int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
846 struct ib_send_wr **bad_wr);
847 int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
848 struct ib_recv_wr **bad_wr);
849 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
850 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
851 void *buffer, u32 length,
852 struct mlx5_ib_qp_base *base);
853 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
854 const struct ib_cq_init_attr *attr,
855 struct ib_ucontext *context,
856 struct ib_udata *udata);
857 int mlx5_ib_destroy_cq(struct ib_cq *cq);
858 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
859 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
860 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
861 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
862 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
863 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
864 u64 virt_addr, int access_flags,
865 struct ib_udata *udata);
866 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
867 struct ib_udata *udata);
868 int mlx5_ib_dealloc_mw(struct ib_mw *mw);
869 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
870 int page_shift, int flags);
871 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
872 int access_flags);
873 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
874 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
875 u64 length, u64 virt_addr, int access_flags,
876 struct ib_pd *pd, struct ib_udata *udata);
877 int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
878 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
879 enum ib_mr_type mr_type,
880 u32 max_num_sg);
881 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
882 unsigned int *sg_offset);
883 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
884 const struct ib_wc *in_wc, const struct ib_grh *in_grh,
885 const struct ib_mad_hdr *in, size_t in_mad_size,
886 struct ib_mad_hdr *out, size_t *out_mad_size,
887 u16 *out_mad_pkey_index);
888 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
889 struct ib_ucontext *context,
890 struct ib_udata *udata);
891 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
892 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
893 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
894 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
895 struct ib_smp *out_mad);
896 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
897 __be64 *sys_image_guid);
898 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
899 u16 *max_pkeys);
900 int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
901 u32 *vendor_id);
902 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
903 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
904 int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
905 u16 *pkey);
906 int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
907 union ib_gid *gid);
908 int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
909 struct ib_port_attr *props);
910 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
911 struct ib_port_attr *props);
912 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
913 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
914 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
915 unsigned long max_page_shift,
916 int *count, int *shift,
917 int *ncont, int *order);
918 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
919 int page_shift, size_t offset, size_t num_pages,
920 __be64 *pas, int access_flags);
921 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
922 int page_shift, __be64 *pas, int access_flags);
923 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
924 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
925 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
926 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
927
928 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
929 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
930 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
931 struct ib_mr_status *mr_status);
932 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
933 struct ib_wq_init_attr *init_attr,
934 struct ib_udata *udata);
935 int mlx5_ib_destroy_wq(struct ib_wq *wq);
936 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
937 u32 wq_attr_mask, struct ib_udata *udata);
938 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
939 struct ib_rwq_ind_table_init_attr *init_attr,
940 struct ib_udata *udata);
941 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
942
943 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
944 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
945 void mlx5_ib_pfault(struct mlx5_core_dev *mdev, void *context,
946 struct mlx5_pagefault *pfault);
947 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
948 void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
949 int __init mlx5_ib_odp_init(void);
950 void mlx5_ib_odp_cleanup(void);
951 void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
952 unsigned long end);
953 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent);
954 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
955 size_t nentries, struct mlx5_ib_mr *mr, int flags);
956 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev * dev)957 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
958 {
959 return;
960 }
961
mlx5_ib_odp_init_one(struct mlx5_ib_dev * ibdev)962 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
mlx5_ib_odp_remove_one(struct mlx5_ib_dev * ibdev)963 static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
mlx5_ib_odp_init(void)964 static inline int mlx5_ib_odp_init(void) { return 0; }
mlx5_ib_odp_cleanup(void)965 static inline void mlx5_ib_odp_cleanup(void) {}
mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent * ent)966 static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {}
mlx5_odp_populate_klm(struct mlx5_klm * pklm,size_t offset,size_t nentries,struct mlx5_ib_mr * mr,int flags)967 static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
968 size_t nentries, struct mlx5_ib_mr *mr,
969 int flags) {}
970
971 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
972
973 int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
974 u8 port, struct ifla_vf_info *info);
975 int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
976 u8 port, int state);
977 int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
978 u8 port, struct ifla_vf_stats *stats);
979 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
980 u64 guid, int type);
981
982 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
983 int index);
984 int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
985 int index, enum ib_gid_type *gid_type);
986
987 void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev);
988 int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev);
989
990 /* GSI QP helper functions */
991 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
992 struct ib_qp_init_attr *init_attr);
993 int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
994 int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
995 int attr_mask);
996 int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
997 int qp_attr_mask,
998 struct ib_qp_init_attr *qp_init_attr);
999 int mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr,
1000 struct ib_send_wr **bad_wr);
1001 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
1002 struct ib_recv_wr **bad_wr);
1003 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
1004
1005 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1006
init_query_mad(struct ib_smp * mad)1007 static inline void init_query_mad(struct ib_smp *mad)
1008 {
1009 mad->base_version = 1;
1010 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1011 mad->class_version = 1;
1012 mad->method = IB_MGMT_METHOD_GET;
1013 }
1014
convert_access(int acc)1015 static inline u8 convert_access(int acc)
1016 {
1017 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
1018 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1019 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1020 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1021 MLX5_PERM_LOCAL_READ;
1022 }
1023
is_qp1(enum ib_qp_type qp_type)1024 static inline int is_qp1(enum ib_qp_type qp_type)
1025 {
1026 return qp_type == MLX5_IB_QPT_HW_GSI;
1027 }
1028
1029 #define MLX5_MAX_UMR_SHIFT 16
1030 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1031
check_cq_create_flags(u32 flags)1032 static inline u32 check_cq_create_flags(u32 flags)
1033 {
1034 /*
1035 * It returns non-zero value for unsupported CQ
1036 * create flags, otherwise it returns zero.
1037 */
1038 return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
1039 IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
1040 }
1041
verify_assign_uidx(u8 cqe_version,u32 cmd_uidx,u32 * user_index)1042 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1043 u32 *user_index)
1044 {
1045 if (cqe_version) {
1046 if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1047 (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1048 return -EINVAL;
1049 *user_index = cmd_uidx;
1050 } else {
1051 *user_index = MLX5_IB_DEFAULT_UIDX;
1052 }
1053
1054 return 0;
1055 }
1056
get_qp_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_qp * ucmd,int inlen,u32 * user_index)1057 static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1058 struct mlx5_ib_create_qp *ucmd,
1059 int inlen,
1060 u32 *user_index)
1061 {
1062 u8 cqe_version = ucontext->cqe_version;
1063
1064 if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1065 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1066 return 0;
1067
1068 if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1069 !!cqe_version))
1070 return -EINVAL;
1071
1072 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1073 }
1074
get_srq_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_srq * ucmd,int inlen,u32 * user_index)1075 static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1076 struct mlx5_ib_create_srq *ucmd,
1077 int inlen,
1078 u32 *user_index)
1079 {
1080 u8 cqe_version = ucontext->cqe_version;
1081
1082 if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1083 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1084 return 0;
1085
1086 if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1087 !!cqe_version))
1088 return -EINVAL;
1089
1090 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1091 }
1092
get_uars_per_sys_page(struct mlx5_ib_dev * dev,bool lib_support)1093 static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1094 {
1095 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1096 MLX5_UARS_IN_PAGE : 1;
1097 }
1098
get_num_uars(struct mlx5_ib_dev * dev,struct mlx5_bfreg_info * bfregi)1099 static inline int get_num_uars(struct mlx5_ib_dev *dev,
1100 struct mlx5_bfreg_info *bfregi)
1101 {
1102 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_sys_pages;
1103 }
1104
1105 #endif /* MLX5_IB_H */
1106