1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_QUEUE_H
8 #define RXE_QUEUE_H
9
10 /* implements a simple circular buffer that can optionally be
11 * shared between user space and the kernel and can be resized
12
13 * the requested element size is rounded up to a power of 2
14 * and the number of elements in the buffer is also rounded
15 * up to a power of 2. Since the queue is empty when the
16 * producer and consumer indices match the maximum capacity
17 * of the queue is one less than the number of element slots
18 */
19
20 /* this data structure is shared between user space and kernel
21 * space for those cases where the queue is shared. It contains
22 * the producer and consumer indices. Is also contains a copy
23 * of the queue size parameters for user space to use but the
24 * kernel must use the parameters in the rxe_queue struct
25 * this MUST MATCH the corresponding librxe struct
26 * for performance reasons arrange to have producer and consumer
27 * pointers in separate cache lines
28 * the kernel should always mask the indices to avoid accessing
29 * memory outside of the data area
30 */
31 struct rxe_queue_buf {
32 __u32 log2_elem_size;
33 __u32 index_mask;
34 __u32 pad_1[30];
35 __u32 producer_index;
36 __u32 pad_2[31];
37 __u32 consumer_index;
38 __u32 pad_3[31];
39 __u8 data[];
40 };
41
42 struct rxe_queue {
43 struct rxe_dev *rxe;
44 struct rxe_queue_buf *buf;
45 struct rxe_mmap_info *ip;
46 size_t buf_size;
47 size_t elem_size;
48 unsigned int log2_elem_size;
49 unsigned int index_mask;
50 };
51
52 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
53 struct ib_udata *udata, struct rxe_queue_buf *buf,
54 size_t buf_size, struct rxe_mmap_info **ip_p);
55
56 void rxe_queue_reset(struct rxe_queue *q);
57
58 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe,
59 int *num_elem,
60 unsigned int elem_size);
61
62 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
63 unsigned int elem_size, struct ib_udata *udata,
64 struct mminfo __user *outbuf,
65 /* Protect producers while resizing queue */
66 spinlock_t *producer_lock,
67 /* Protect consumers while resizing queue */
68 spinlock_t *consumer_lock);
69
70 void rxe_queue_cleanup(struct rxe_queue *queue);
71
next_index(struct rxe_queue * q,int index)72 static inline int next_index(struct rxe_queue *q, int index)
73 {
74 return (index + 1) & q->buf->index_mask;
75 }
76
queue_empty(struct rxe_queue * q)77 static inline int queue_empty(struct rxe_queue *q)
78 {
79 return ((q->buf->producer_index - q->buf->consumer_index)
80 & q->index_mask) == 0;
81 }
82
queue_full(struct rxe_queue * q)83 static inline int queue_full(struct rxe_queue *q)
84 {
85 return ((q->buf->producer_index + 1 - q->buf->consumer_index)
86 & q->index_mask) == 0;
87 }
88
advance_producer(struct rxe_queue * q)89 static inline void advance_producer(struct rxe_queue *q)
90 {
91 q->buf->producer_index = (q->buf->producer_index + 1)
92 & q->index_mask;
93 }
94
advance_consumer(struct rxe_queue * q)95 static inline void advance_consumer(struct rxe_queue *q)
96 {
97 q->buf->consumer_index = (q->buf->consumer_index + 1)
98 & q->index_mask;
99 }
100
producer_addr(struct rxe_queue * q)101 static inline void *producer_addr(struct rxe_queue *q)
102 {
103 return q->buf->data + ((q->buf->producer_index & q->index_mask)
104 << q->log2_elem_size);
105 }
106
consumer_addr(struct rxe_queue * q)107 static inline void *consumer_addr(struct rxe_queue *q)
108 {
109 return q->buf->data + ((q->buf->consumer_index & q->index_mask)
110 << q->log2_elem_size);
111 }
112
producer_index(struct rxe_queue * q)113 static inline unsigned int producer_index(struct rxe_queue *q)
114 {
115 return q->buf->producer_index;
116 }
117
consumer_index(struct rxe_queue * q)118 static inline unsigned int consumer_index(struct rxe_queue *q)
119 {
120 return q->buf->consumer_index;
121 }
122
addr_from_index(struct rxe_queue * q,unsigned int index)123 static inline void *addr_from_index(struct rxe_queue *q, unsigned int index)
124 {
125 return q->buf->data + ((index & q->index_mask)
126 << q->buf->log2_elem_size);
127 }
128
index_from_addr(const struct rxe_queue * q,const void * addr)129 static inline unsigned int index_from_addr(const struct rxe_queue *q,
130 const void *addr)
131 {
132 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
133 & q->index_mask;
134 }
135
queue_count(const struct rxe_queue * q)136 static inline unsigned int queue_count(const struct rxe_queue *q)
137 {
138 return (q->buf->producer_index - q->buf->consumer_index)
139 & q->index_mask;
140 }
141
queue_head(struct rxe_queue * q)142 static inline void *queue_head(struct rxe_queue *q)
143 {
144 return queue_empty(q) ? NULL : consumer_addr(q);
145 }
146
147 #endif /* RXE_QUEUE_H */
148