1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Connection Data Control (CDC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12 #ifndef SMC_CDC_H
13 #define SMC_CDC_H
14
15 #include <linux/kernel.h> /* max_t */
16 #include <linux/atomic.h>
17 #include <linux/in.h>
18 #include <linux/compiler.h>
19
20 #include "smc.h"
21 #include "smc_core.h"
22 #include "smc_wr.h"
23
24 #define SMC_CDC_MSG_TYPE 0xFE
25
26 /* in network byte order */
27 union smc_cdc_cursor { /* SMC cursor */
28 struct {
29 __be16 reserved;
30 __be16 wrap;
31 __be32 count;
32 };
33 #ifdef KERNEL_HAS_ATOMIC64
34 atomic64_t acurs; /* for atomic processing */
35 #else
36 u64 acurs; /* for atomic processing */
37 #endif
38 } __aligned(8);
39
40 /* in network byte order */
41 struct smc_cdc_msg {
42 struct smc_wr_rx_hdr common; /* .type = 0xFE */
43 u8 len; /* 44 */
44 __be16 seqno;
45 __be32 token;
46 union smc_cdc_cursor prod;
47 union smc_cdc_cursor cons; /* piggy backed "ack" */
48 struct smc_cdc_producer_flags prod_flags;
49 struct smc_cdc_conn_state_flags conn_state_flags;
50 u8 reserved[18];
51 };
52
53 /* SMC-D cursor format */
54 union smcd_cdc_cursor {
55 struct {
56 u16 wrap;
57 u32 count;
58 struct smc_cdc_producer_flags prod_flags;
59 struct smc_cdc_conn_state_flags conn_state_flags;
60 } __packed;
61 #ifdef KERNEL_HAS_ATOMIC64
62 atomic64_t acurs; /* for atomic processing */
63 #else
64 u64 acurs; /* for atomic processing */
65 #endif
66 } __aligned(8);
67
68 /* CDC message for SMC-D */
69 struct smcd_cdc_msg {
70 struct smc_wr_rx_hdr common; /* Type = 0xFE */
71 u8 res1[7];
72 union smcd_cdc_cursor prod;
73 union smcd_cdc_cursor cons;
74 u8 res3[8];
75 } __aligned(8);
76
smc_cdc_rxed_any_close(struct smc_connection * conn)77 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
78 {
79 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
80 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
81 }
82
smc_cdc_rxed_any_close_or_senddone(struct smc_connection * conn)83 static inline bool smc_cdc_rxed_any_close_or_senddone(
84 struct smc_connection *conn)
85 {
86 return smc_cdc_rxed_any_close(conn) ||
87 conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
88 }
89
smc_curs_add(int size,union smc_host_cursor * curs,int value)90 static inline void smc_curs_add(int size, union smc_host_cursor *curs,
91 int value)
92 {
93 curs->count += value;
94 if (curs->count >= size) {
95 curs->wrap++;
96 curs->count -= size;
97 }
98 }
99
100 /* Copy cursor src into tgt */
smc_curs_copy(union smc_host_cursor * tgt,union smc_host_cursor * src,struct smc_connection * conn)101 static inline void smc_curs_copy(union smc_host_cursor *tgt,
102 union smc_host_cursor *src,
103 struct smc_connection *conn)
104 {
105 #ifndef KERNEL_HAS_ATOMIC64
106 unsigned long flags;
107
108 spin_lock_irqsave(&conn->acurs_lock, flags);
109 tgt->acurs = src->acurs;
110 spin_unlock_irqrestore(&conn->acurs_lock, flags);
111 #else
112 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
113 #endif
114 }
115
smc_curs_copy_net(union smc_cdc_cursor * tgt,union smc_cdc_cursor * src,struct smc_connection * conn)116 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
117 union smc_cdc_cursor *src,
118 struct smc_connection *conn)
119 {
120 #ifndef KERNEL_HAS_ATOMIC64
121 unsigned long flags;
122
123 spin_lock_irqsave(&conn->acurs_lock, flags);
124 tgt->acurs = src->acurs;
125 spin_unlock_irqrestore(&conn->acurs_lock, flags);
126 #else
127 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
128 #endif
129 }
130
smcd_curs_copy(union smcd_cdc_cursor * tgt,union smcd_cdc_cursor * src,struct smc_connection * conn)131 static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
132 union smcd_cdc_cursor *src,
133 struct smc_connection *conn)
134 {
135 #ifndef KERNEL_HAS_ATOMIC64
136 unsigned long flags;
137
138 spin_lock_irqsave(&conn->acurs_lock, flags);
139 tgt->acurs = src->acurs;
140 spin_unlock_irqrestore(&conn->acurs_lock, flags);
141 #else
142 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
143 #endif
144 }
145
146 /* calculate cursor difference between old and new, where old <= new and
147 * difference cannot exceed size
148 */
smc_curs_diff(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)149 static inline int smc_curs_diff(unsigned int size,
150 union smc_host_cursor *old,
151 union smc_host_cursor *new)
152 {
153 if (old->wrap != new->wrap)
154 return max_t(int, 0,
155 ((size - old->count) + new->count));
156
157 return max_t(int, 0, (new->count - old->count));
158 }
159
160 /* calculate cursor difference between old and new - returns negative
161 * value in case old > new
162 */
smc_curs_comp(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)163 static inline int smc_curs_comp(unsigned int size,
164 union smc_host_cursor *old,
165 union smc_host_cursor *new)
166 {
167 if (old->wrap > new->wrap ||
168 (old->wrap == new->wrap && old->count > new->count))
169 return -smc_curs_diff(size, new, old);
170 return smc_curs_diff(size, old, new);
171 }
172
173 /* calculate cursor difference between old and new, where old <= new and
174 * difference may exceed size
175 */
smc_curs_diff_large(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)176 static inline int smc_curs_diff_large(unsigned int size,
177 union smc_host_cursor *old,
178 union smc_host_cursor *new)
179 {
180 if (old->wrap < new->wrap)
181 return min_t(int,
182 (size - old->count) + new->count +
183 (new->wrap - old->wrap - 1) * size,
184 size);
185
186 if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
187 return min_t(int,
188 (size - old->count) + new->count +
189 (new->wrap + 0xffff - old->wrap) * size,
190 size);
191
192 return max_t(int, 0, (new->count - old->count));
193 }
194
smc_host_cursor_to_cdc(union smc_cdc_cursor * peer,union smc_host_cursor * local,union smc_host_cursor * save,struct smc_connection * conn)195 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
196 union smc_host_cursor *local,
197 union smc_host_cursor *save,
198 struct smc_connection *conn)
199 {
200 smc_curs_copy(save, local, conn);
201 peer->count = htonl(save->count);
202 peer->wrap = htons(save->wrap);
203 /* peer->reserved = htons(0); must be ensured by caller */
204 }
205
smc_host_msg_to_cdc(struct smc_cdc_msg * peer,struct smc_connection * conn,union smc_host_cursor * save)206 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
207 struct smc_connection *conn,
208 union smc_host_cursor *save)
209 {
210 struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
211
212 peer->common.type = local->common.type;
213 peer->len = local->len;
214 peer->seqno = htons(local->seqno);
215 peer->token = htonl(local->token);
216 smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
217 smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
218 peer->prod_flags = local->prod_flags;
219 peer->conn_state_flags = local->conn_state_flags;
220 }
221
smc_cdc_cursor_to_host(union smc_host_cursor * local,union smc_cdc_cursor * peer,struct smc_connection * conn)222 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
223 union smc_cdc_cursor *peer,
224 struct smc_connection *conn)
225 {
226 union smc_host_cursor temp, old;
227 union smc_cdc_cursor net;
228
229 smc_curs_copy(&old, local, conn);
230 smc_curs_copy_net(&net, peer, conn);
231 temp.count = ntohl(net.count);
232 temp.wrap = ntohs(net.wrap);
233 if ((old.wrap > temp.wrap) && temp.wrap)
234 return;
235 if ((old.wrap == temp.wrap) &&
236 (old.count > temp.count))
237 return;
238 smc_curs_copy(local, &temp, conn);
239 }
240
smcr_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)241 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
242 struct smc_cdc_msg *peer,
243 struct smc_connection *conn)
244 {
245 local->common.type = peer->common.type;
246 local->len = peer->len;
247 local->seqno = ntohs(peer->seqno);
248 local->token = ntohl(peer->token);
249 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
250 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
251 local->prod_flags = peer->prod_flags;
252 local->conn_state_flags = peer->conn_state_flags;
253 }
254
smcd_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smcd_cdc_msg * peer,struct smc_connection * conn)255 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
256 struct smcd_cdc_msg *peer,
257 struct smc_connection *conn)
258 {
259 union smc_host_cursor temp;
260
261 temp.wrap = peer->prod.wrap;
262 temp.count = peer->prod.count;
263 smc_curs_copy(&local->prod, &temp, conn);
264
265 temp.wrap = peer->cons.wrap;
266 temp.count = peer->cons.count;
267 smc_curs_copy(&local->cons, &temp, conn);
268 local->prod_flags = peer->cons.prod_flags;
269 local->conn_state_flags = peer->cons.conn_state_flags;
270 }
271
smc_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)272 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
273 struct smc_cdc_msg *peer,
274 struct smc_connection *conn)
275 {
276 if (conn->lgr->is_smcd)
277 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
278 else
279 smcr_cdc_msg_to_host(local, peer, conn);
280 }
281
282 struct smc_cdc_tx_pend {
283 struct smc_connection *conn; /* socket connection */
284 union smc_host_cursor cursor; /* tx sndbuf cursor sent */
285 union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
286 u16 ctrl_seq; /* conn. tx sequence # */
287 };
288
289 int smc_cdc_get_free_slot(struct smc_connection *conn,
290 struct smc_link *link,
291 struct smc_wr_buf **wr_buf,
292 struct smc_rdma_wr **wr_rdma_buf,
293 struct smc_cdc_tx_pend **pend);
294 void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
295 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
296 struct smc_cdc_tx_pend *pend);
297 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
298 int smcd_cdc_msg_send(struct smc_connection *conn);
299 int smcr_cdc_msg_send_validation(struct smc_connection *conn,
300 struct smc_cdc_tx_pend *pend,
301 struct smc_wr_buf *wr_buf);
302 int smc_cdc_init(void) __init;
303 void smcd_cdc_rx_init(struct smc_connection *conn);
304
305 #endif /* SMC_CDC_H */
306