1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Connection Data Control (CDC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12 #ifndef SMC_CDC_H
13 #define SMC_CDC_H
14
15 #include <linux/kernel.h> /* max_t */
16 #include <linux/atomic.h>
17 #include <linux/in.h>
18 #include <linux/compiler.h>
19
20 #include "smc.h"
21 #include "smc_core.h"
22 #include "smc_wr.h"
23
24 #define SMC_CDC_MSG_TYPE 0xFE
25
26 /* in network byte order */
27 union smc_cdc_cursor { /* SMC cursor */
28 struct {
29 __be16 reserved;
30 __be16 wrap;
31 __be32 count;
32 };
33 #ifdef KERNEL_HAS_ATOMIC64
34 atomic64_t acurs; /* for atomic processing */
35 #else
36 u64 acurs; /* for atomic processing */
37 #endif
38 } __aligned(8);
39
40 /* in network byte order */
41 struct smc_cdc_msg {
42 struct smc_wr_rx_hdr common; /* .type = 0xFE */
43 u8 len; /* 44 */
44 __be16 seqno;
45 __be32 token;
46 union smc_cdc_cursor prod;
47 union smc_cdc_cursor cons; /* piggy backed "ack" */
48 struct smc_cdc_producer_flags prod_flags;
49 struct smc_cdc_conn_state_flags conn_state_flags;
50 u8 reserved[18];
51 } __packed; /* format defined in RFC7609 */
52
53 /* CDC message for SMC-D */
54 struct smcd_cdc_msg {
55 struct smc_wr_rx_hdr common; /* Type = 0xFE */
56 u8 res1[7];
57 u16 prod_wrap;
58 u32 prod_count;
59 u8 res2[2];
60 u16 cons_wrap;
61 u32 cons_count;
62 struct smc_cdc_producer_flags prod_flags;
63 struct smc_cdc_conn_state_flags conn_state_flags;
64 u8 res3[8];
65 } __packed;
66
smc_cdc_rxed_any_close(struct smc_connection * conn)67 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
68 {
69 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
70 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
71 }
72
smc_cdc_rxed_any_close_or_senddone(struct smc_connection * conn)73 static inline bool smc_cdc_rxed_any_close_or_senddone(
74 struct smc_connection *conn)
75 {
76 return smc_cdc_rxed_any_close(conn) ||
77 conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
78 }
79
smc_curs_add(int size,union smc_host_cursor * curs,int value)80 static inline void smc_curs_add(int size, union smc_host_cursor *curs,
81 int value)
82 {
83 curs->count += value;
84 if (curs->count >= size) {
85 curs->wrap++;
86 curs->count -= size;
87 }
88 }
89
90 /* SMC cursors are 8 bytes long and require atomic reading and writing */
smc_curs_read(union smc_host_cursor * curs,struct smc_connection * conn)91 static inline u64 smc_curs_read(union smc_host_cursor *curs,
92 struct smc_connection *conn)
93 {
94 #ifndef KERNEL_HAS_ATOMIC64
95 unsigned long flags;
96 u64 ret;
97
98 spin_lock_irqsave(&conn->acurs_lock, flags);
99 ret = curs->acurs;
100 spin_unlock_irqrestore(&conn->acurs_lock, flags);
101 return ret;
102 #else
103 return atomic64_read(&curs->acurs);
104 #endif
105 }
106
107 /* Copy cursor src into tgt */
smc_curs_copy(union smc_host_cursor * tgt,union smc_host_cursor * src,struct smc_connection * conn)108 static inline void smc_curs_copy(union smc_host_cursor *tgt,
109 union smc_host_cursor *src,
110 struct smc_connection *conn)
111 {
112 #ifndef KERNEL_HAS_ATOMIC64
113 unsigned long flags;
114
115 spin_lock_irqsave(&conn->acurs_lock, flags);
116 tgt->acurs = src->acurs;
117 spin_unlock_irqrestore(&conn->acurs_lock, flags);
118 #else
119 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
120 #endif
121 }
122
smc_curs_copy_net(union smc_cdc_cursor * tgt,union smc_cdc_cursor * src,struct smc_connection * conn)123 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
124 union smc_cdc_cursor *src,
125 struct smc_connection *conn)
126 {
127 #ifndef KERNEL_HAS_ATOMIC64
128 unsigned long flags;
129
130 spin_lock_irqsave(&conn->acurs_lock, flags);
131 tgt->acurs = src->acurs;
132 spin_unlock_irqrestore(&conn->acurs_lock, flags);
133 #else
134 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
135 #endif
136 }
137
138 /* calculate cursor difference between old and new, where old <= new and
139 * difference cannot exceed size
140 */
smc_curs_diff(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)141 static inline int smc_curs_diff(unsigned int size,
142 union smc_host_cursor *old,
143 union smc_host_cursor *new)
144 {
145 if (old->wrap != new->wrap)
146 return max_t(int, 0,
147 ((size - old->count) + new->count));
148
149 return max_t(int, 0, (new->count - old->count));
150 }
151
152 /* calculate cursor difference between old and new - returns negative
153 * value in case old > new
154 */
smc_curs_comp(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)155 static inline int smc_curs_comp(unsigned int size,
156 union smc_host_cursor *old,
157 union smc_host_cursor *new)
158 {
159 if (old->wrap > new->wrap ||
160 (old->wrap == new->wrap && old->count > new->count))
161 return -smc_curs_diff(size, new, old);
162 return smc_curs_diff(size, old, new);
163 }
164
165 /* calculate cursor difference between old and new, where old <= new and
166 * difference may exceed size
167 */
smc_curs_diff_large(unsigned int size,union smc_host_cursor * old,union smc_host_cursor * new)168 static inline int smc_curs_diff_large(unsigned int size,
169 union smc_host_cursor *old,
170 union smc_host_cursor *new)
171 {
172 if (old->wrap < new->wrap)
173 return min_t(int,
174 (size - old->count) + new->count +
175 (new->wrap - old->wrap - 1) * size,
176 size);
177
178 if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
179 return min_t(int,
180 (size - old->count) + new->count +
181 (new->wrap + 0xffff - old->wrap) * size,
182 size);
183
184 return max_t(int, 0, (new->count - old->count));
185 }
186
smc_host_cursor_to_cdc(union smc_cdc_cursor * peer,union smc_host_cursor * local,union smc_host_cursor * save,struct smc_connection * conn)187 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
188 union smc_host_cursor *local,
189 union smc_host_cursor *save,
190 struct smc_connection *conn)
191 {
192 smc_curs_copy(save, local, conn);
193 peer->count = htonl(save->count);
194 peer->wrap = htons(save->wrap);
195 /* peer->reserved = htons(0); must be ensured by caller */
196 }
197
smc_host_msg_to_cdc(struct smc_cdc_msg * peer,struct smc_connection * conn,union smc_host_cursor * save)198 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
199 struct smc_connection *conn,
200 union smc_host_cursor *save)
201 {
202 struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
203
204 peer->common.type = local->common.type;
205 peer->len = local->len;
206 peer->seqno = htons(local->seqno);
207 peer->token = htonl(local->token);
208 smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
209 smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
210 peer->prod_flags = local->prod_flags;
211 peer->conn_state_flags = local->conn_state_flags;
212 }
213
smc_cdc_cursor_to_host(union smc_host_cursor * local,union smc_cdc_cursor * peer,struct smc_connection * conn)214 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
215 union smc_cdc_cursor *peer,
216 struct smc_connection *conn)
217 {
218 union smc_host_cursor temp, old;
219 union smc_cdc_cursor net;
220
221 smc_curs_copy(&old, local, conn);
222 smc_curs_copy_net(&net, peer, conn);
223 temp.count = ntohl(net.count);
224 temp.wrap = ntohs(net.wrap);
225 if ((old.wrap > temp.wrap) && temp.wrap)
226 return;
227 if ((old.wrap == temp.wrap) &&
228 (old.count > temp.count))
229 return;
230 smc_curs_copy(local, &temp, conn);
231 }
232
smcr_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)233 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
234 struct smc_cdc_msg *peer,
235 struct smc_connection *conn)
236 {
237 local->common.type = peer->common.type;
238 local->len = peer->len;
239 local->seqno = ntohs(peer->seqno);
240 local->token = ntohl(peer->token);
241 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
242 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
243 local->prod_flags = peer->prod_flags;
244 local->conn_state_flags = peer->conn_state_flags;
245 }
246
smcd_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smcd_cdc_msg * peer)247 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
248 struct smcd_cdc_msg *peer)
249 {
250 local->prod.wrap = peer->prod_wrap;
251 local->prod.count = peer->prod_count;
252 local->cons.wrap = peer->cons_wrap;
253 local->cons.count = peer->cons_count;
254 local->prod_flags = peer->prod_flags;
255 local->conn_state_flags = peer->conn_state_flags;
256 }
257
smc_cdc_msg_to_host(struct smc_host_cdc_msg * local,struct smc_cdc_msg * peer,struct smc_connection * conn)258 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
259 struct smc_cdc_msg *peer,
260 struct smc_connection *conn)
261 {
262 if (conn->lgr->is_smcd)
263 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer);
264 else
265 smcr_cdc_msg_to_host(local, peer, conn);
266 }
267
268 struct smc_cdc_tx_pend;
269
270 int smc_cdc_get_free_slot(struct smc_connection *conn,
271 struct smc_wr_buf **wr_buf,
272 struct smc_cdc_tx_pend **pend);
273 void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
274 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
275 struct smc_cdc_tx_pend *pend);
276 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
277 int smcd_cdc_msg_send(struct smc_connection *conn);
278 int smc_cdc_init(void) __init;
279 void smcd_cdc_rx_init(struct smc_connection *conn);
280
281 #endif /* SMC_CDC_H */
282