1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Connection Data Control (CDC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12 #ifndef SMC_CDC_H
13 #define SMC_CDC_H
14
15 #include <linux/kernel.h> /* max_t */
16 #include <linux/atomic.h>
17 #include <linux/in.h>
18 #include <linux/compiler.h>
19
20 #include "smc.h"
21 #include "smc_core.h"
22 #include "smc_wr.h"
23
24 #define SMC_CDC_MSG_TYPE 0xFE
25
26 /* in network byte order */
27 union smc_cdc_cursor { /* SMC cursor */
28 struct {
29 __be16 reserved;
30 __be16 wrap;
31 __be32 count;
32 };
33 #ifdef KERNEL_HAS_ATOMIC64
34 atomic64_t acurs; /* for atomic processing */
35 #else
36 u64 acurs; /* for atomic processing */
37 #endif
38 } __aligned(8);
39
40 /* in network byte order */
41 struct smc_cdc_msg {
42 struct smc_wr_rx_hdr common; /* .type = 0xFE */
43 u8 len; /* 44 */
44 __be16 seqno;
45 __be32 token;
46 union smc_cdc_cursor prod;
47 union smc_cdc_cursor cons; /* piggy backed "ack" */
48 struct smc_cdc_producer_flags prod_flags;
49 struct smc_cdc_conn_state_flags conn_state_flags;
50 u8 reserved[18];
51 };
52
53 /* SMC-D cursor format */
54 union smcd_cdc_cursor {
55 struct {
56 u16 wrap;
57 u32 count;
58 struct smc_cdc_producer_flags prod_flags;
59 struct smc_cdc_conn_state_flags conn_state_flags;
60 } __packed;
61 #ifdef KERNEL_HAS_ATOMIC64
62 atomic64_t acurs; /* for atomic processing */
63 #else
64 u64 acurs; /* for atomic processing */
65 #endif
66 } __aligned(8);
67
68 /* CDC message for SMC-D */
69 struct smcd_cdc_msg {
70 struct smc_wr_rx_hdr common; /* Type = 0xFE */
71 u8 res1[7];
72 union smcd_cdc_cursor prod;
73 union smcd_cdc_cursor cons;
74 u8 res3[8];
75 } __aligned(8);
76
smc_cdc_rxed_any_close(struct smc_connection * conn)77 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
78 {
79 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
80 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
81 }
82
smc_cdc_rxed_any_close_or_senddone(struct smc_connection * conn)83 static inline bool smc_cdc_rxed_any_close_or_senddone(
84 struct smc_connection *conn)
85 {
86 return smc_cdc_rxed_any_close(conn) ||
87 conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
88 }
89
smc_curs_add(int size,union smc_host_cursor * curs,int value)90 static inline void smc_curs_add(int size, union smc_host_cursor *curs,
91 int value)
92 {
93 curs->count += value;
94 if (curs->count >= size) {
95 curs->wrap++;
96 curs->count -= size;
97 }
98 }
99
100 /* SMC cursors are 8 bytes long and require atomic reading and writing */
smc_curs_read(union smc_host_cursor * curs,struct smc_connection * conn)101 static inline u64 smc_curs_read(union smc_host_cursor *curs,
102 struct smc_connection *conn)
103 {
104 #ifndef KERNEL_HAS_ATOMIC64
105 unsigned long flags;
106 u64 ret;
107
108 spin_lock_irqsave(&conn->acurs_lock, flags);
109 ret = curs->acurs;
110 spin_unlock_irqrestore(&conn->acurs_lock, flags);
111 return ret;
112 #else
113 return atomic64_read(&curs->acurs);
114 #endif
115 }
116
117 /* Copy cursor src into tgt */
smc_curs_copy(union smc_host_cursor * tgt,union smc_host_cursor * src,struct smc_connection * conn)118 static inline void smc_curs_copy(union smc_host_cursor *tgt,
119 union smc_host_cursor *src,
120 struct smc_connection *conn)
121 {
122 #ifndef KERNEL_HAS_ATOMIC64
123 unsigned long flags;
124
125 spin_lock_irqsave(&conn->acurs_lock, flags);
126 tgt->acurs = src->acurs;
127 spin_unlock_irqrestore(&conn->acurs_lock, flags);
128 #else
129 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
130 #endif
131 }
132
smc_curs_copy_net(union smc_cdc_cursor * tgt,union smc_cdc_cursor * src,struct smc_connection * conn)133 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
134 union smc_cdc_cursor *src,
135 struct smc_connection *conn)
136 {
137 #ifndef KERNEL_HAS_ATOMIC64
138 unsigned long flags;
139
140 spin_lock_irqsave(&conn->acurs_lock, flags);
141 tgt->acurs = src->acurs;
142 spin_unlock_irqrestore(&conn->acurs_lock, flags);
143 #else
144 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
145 #endif
146 }
147
smcd_curs_copy(union smcd_cdc_cursor * tgt,union smcd_cdc_cursor * src,struct smc_connection * conn)148 static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
149 union smcd_cdc_cursor *src,
150 struct smc_connection *conn)
151 {
152 #ifndef KERNEL_HAS_ATOMIC64
153 unsigned long flags;
154
155 spin_lock_irqsave(&conn->acurs_lock, flags);
156 tgt->acurs = src->acurs;
157 spin_unlock_irqrestore(&conn->acurs_lock, flags);
158 #else
159 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
160 #endif
161 }
162
163 /* calculate cursor difference between old and new, where old <= new and
164 * difference cannot exceed size
165 */
smc_curs_diff(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)166 static inline int smc_curs_diff(unsigned int size,
167 union smc_host_cursor *old,
168 union smc_host_cursor *new)
169 {
170 if (old->wrap != new->wrap)
171 return max_t(int, 0,
172 ((size - old->count) + new->count));
173
174 return max_t(int, 0, (new->count - old->count));
175 }
176
177 /* calculate cursor difference between old and new - returns negative
178 * value in case old > new
179 */
smc_curs_comp(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)180 static inline int smc_curs_comp(unsigned int size,
181 union smc_host_cursor *old,
182 union smc_host_cursor *new)
183 {
184 if (old->wrap > new->wrap ||
185 (old->wrap == new->wrap && old->count > new->count))
186 return -smc_curs_diff(size, new, old);
187 return smc_curs_diff(size, old, new);
188 }
189
190 /* calculate cursor difference between old and new, where old <= new and
191 * difference may exceed size
192 */
smc_curs_diff_large(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)193 static inline int smc_curs_diff_large(unsigned int size,
194 union smc_host_cursor *old,
195 union smc_host_cursor *new)
196 {
197 if (old->wrap < new->wrap)
198 return min_t(int,
199 (size - old->count) + new->count +
200 (new->wrap - old->wrap - 1) * size,
201 size);
202
203 if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
204 return min_t(int,
205 (size - old->count) + new->count +
206 (new->wrap + 0xffff - old->wrap) * size,
207 size);
208
209 return max_t(int, 0, (new->count - old->count));
210 }
211
smc_host_cursor_to_cdc(union smc_cdc_cursor * peer,union smc_host_cursor * local,union smc_host_cursor * save,struct smc_connection * conn)212 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
213 union smc_host_cursor *local,
214 union smc_host_cursor *save,
215 struct smc_connection *conn)
216 {
217 smc_curs_copy(save, local, conn);
218 peer->count = htonl(save->count);
219 peer->wrap = htons(save->wrap);
220 /* peer->reserved = htons(0); must be ensured by caller */
221 }
222
smc_host_msg_to_cdc(struct smc_cdc_msg * peer,struct smc_connection * conn,union smc_host_cursor * save)223 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
224 struct smc_connection *conn,
225 union smc_host_cursor *save)
226 {
227 struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
228
229 peer->common.type = local->common.type;
230 peer->len = local->len;
231 peer->seqno = htons(local->seqno);
232 peer->token = htonl(local->token);
233 smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
234 smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
235 peer->prod_flags = local->prod_flags;
236 peer->conn_state_flags = local->conn_state_flags;
237 }
238
smc_cdc_cursor_to_host(union smc_host_cursor * local,union smc_cdc_cursor * peer,struct smc_connection * conn)239 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
240 union smc_cdc_cursor *peer,
241 struct smc_connection *conn)
242 {
243 union smc_host_cursor temp, old;
244 union smc_cdc_cursor net;
245
246 smc_curs_copy(&old, local, conn);
247 smc_curs_copy_net(&net, peer, conn);
248 temp.count = ntohl(net.count);
249 temp.wrap = ntohs(net.wrap);
250 if ((old.wrap > temp.wrap) && temp.wrap)
251 return;
252 if ((old.wrap == temp.wrap) &&
253 (old.count > temp.count))
254 return;
255 smc_curs_copy(local, &temp, conn);
256 }
257
smcr_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)258 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
259 struct smc_cdc_msg *peer,
260 struct smc_connection *conn)
261 {
262 local->common.type = peer->common.type;
263 local->len = peer->len;
264 local->seqno = ntohs(peer->seqno);
265 local->token = ntohl(peer->token);
266 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
267 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
268 local->prod_flags = peer->prod_flags;
269 local->conn_state_flags = peer->conn_state_flags;
270 }
271
smcd_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smcd_cdc_msg * peer,struct smc_connection * conn)272 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
273 struct smcd_cdc_msg *peer,
274 struct smc_connection *conn)
275 {
276 union smc_host_cursor temp;
277
278 temp.wrap = peer->prod.wrap;
279 temp.count = peer->prod.count;
280 smc_curs_copy(&local->prod, &temp, conn);
281
282 temp.wrap = peer->cons.wrap;
283 temp.count = peer->cons.count;
284 smc_curs_copy(&local->cons, &temp, conn);
285 local->prod_flags = peer->cons.prod_flags;
286 local->conn_state_flags = peer->cons.conn_state_flags;
287 }
288
smc_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)289 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
290 struct smc_cdc_msg *peer,
291 struct smc_connection *conn)
292 {
293 if (conn->lgr->is_smcd)
294 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
295 else
296 smcr_cdc_msg_to_host(local, peer, conn);
297 }
298
299 struct smc_cdc_tx_pend {
300 struct smc_connection *conn; /* socket connection */
301 union smc_host_cursor cursor; /* tx sndbuf cursor sent */
302 union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
303 u16 ctrl_seq; /* conn. tx sequence # */
304 };
305
306 int smc_cdc_get_free_slot(struct smc_connection *conn,
307 struct smc_wr_buf **wr_buf,
308 struct smc_rdma_wr **wr_rdma_buf,
309 struct smc_cdc_tx_pend **pend);
310 void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
311 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
312 struct smc_cdc_tx_pend *pend);
313 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
314 int smcd_cdc_msg_send(struct smc_connection *conn);
315 int smc_cdc_init(void) __init;
316 void smcd_cdc_rx_init(struct smc_connection *conn);
317
318 #endif /* SMC_CDC_H */
319