1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (c) 2018 Chelsio Communications, Inc.
4 */
5
6 #ifndef __CHTLS_H__
7 #define __CHTLS_H__
8
9 #include <crypto/aes.h>
10 #include <crypto/algapi.h>
11 #include <crypto/hash.h>
12 #include <crypto/sha.h>
13 #include <crypto/authenc.h>
14 #include <crypto/ctr.h>
15 #include <crypto/gf128mul.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/null.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/aead.h>
20 #include <crypto/scatterwalk.h>
21 #include <crypto/internal/hash.h>
22 #include <linux/tls.h>
23 #include <net/tls.h>
24 #include <net/tls_toe.h>
25
26 #include "t4fw_api.h"
27 #include "t4_msg.h"
28 #include "cxgb4.h"
29 #include "cxgb4_uld.h"
30 #include "l2t.h"
31 #include "chcr_algo.h"
32 #include "chcr_core.h"
33 #include "chcr_crypto.h"
34
35 #define CHTLS_DRV_VERSION "1.0.0.0-ko"
36
37 #define TLS_KEYCTX_RXFLIT_CNT_S 24
38 #define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
39
40 #define TLS_KEYCTX_RXPROT_VER_S 20
41 #define TLS_KEYCTX_RXPROT_VER_M 0xf
42 #define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
43
44 #define TLS_KEYCTX_RXCIPH_MODE_S 16
45 #define TLS_KEYCTX_RXCIPH_MODE_M 0xf
46 #define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
47
48 #define TLS_KEYCTX_RXAUTH_MODE_S 12
49 #define TLS_KEYCTX_RXAUTH_MODE_M 0xf
50 #define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
51
52 #define TLS_KEYCTX_RXCIAU_CTRL_S 11
53 #define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
54
55 #define TLS_KEYCTX_RX_SEQCTR_S 9
56 #define TLS_KEYCTX_RX_SEQCTR_M 0x3
57 #define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
58
59 #define TLS_KEYCTX_RX_VALID_S 8
60 #define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
61
62 #define TLS_KEYCTX_RXCK_SIZE_S 3
63 #define TLS_KEYCTX_RXCK_SIZE_M 0x7
64 #define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
65
66 #define TLS_KEYCTX_RXMK_SIZE_S 0
67 #define TLS_KEYCTX_RXMK_SIZE_M 0x7
68 #define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
69
70 #define KEYCTX_TX_WR_IV_S 55
71 #define KEYCTX_TX_WR_IV_M 0x1ffULL
72 #define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
73 #define KEYCTX_TX_WR_IV_G(x) \
74 (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
75
76 #define KEYCTX_TX_WR_AAD_S 47
77 #define KEYCTX_TX_WR_AAD_M 0xffULL
78 #define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
79 #define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
80 KEYCTX_TX_WR_AAD_M)
81
82 #define KEYCTX_TX_WR_AADST_S 39
83 #define KEYCTX_TX_WR_AADST_M 0xffULL
84 #define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
85 #define KEYCTX_TX_WR_AADST_G(x) \
86 (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
87
88 #define KEYCTX_TX_WR_CIPHER_S 30
89 #define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
90 #define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
91 #define KEYCTX_TX_WR_CIPHER_G(x) \
92 (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
93
94 #define KEYCTX_TX_WR_CIPHERST_S 23
95 #define KEYCTX_TX_WR_CIPHERST_M 0x7f
96 #define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
97 #define KEYCTX_TX_WR_CIPHERST_G(x) \
98 (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
99
100 #define KEYCTX_TX_WR_AUTH_S 14
101 #define KEYCTX_TX_WR_AUTH_M 0x1ff
102 #define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
103 #define KEYCTX_TX_WR_AUTH_G(x) \
104 (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
105
106 #define KEYCTX_TX_WR_AUTHST_S 7
107 #define KEYCTX_TX_WR_AUTHST_M 0x7f
108 #define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
109 #define KEYCTX_TX_WR_AUTHST_G(x) \
110 (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
111
112 #define KEYCTX_TX_WR_AUTHIN_S 0
113 #define KEYCTX_TX_WR_AUTHIN_M 0x7f
114 #define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
115 #define KEYCTX_TX_WR_AUTHIN_G(x) \
116 (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
117
118 struct sge_opaque_hdr {
119 void *dev;
120 dma_addr_t addr[MAX_SKB_FRAGS + 1];
121 };
122
123 #define MAX_IVS_PAGE 256
124 #define TLS_KEY_CONTEXT_SZ 64
125 #define CIPHER_BLOCK_SIZE 16
126 #define GCM_TAG_SIZE 16
127 #define KEY_ON_MEM_SZ 16
128 #define AEAD_EXPLICIT_DATA_SIZE 8
129 #define TLS_HEADER_LENGTH 5
130 #define SCMD_CIPH_MODE_AES_GCM 2
131 /* Any MFS size should work and come from openssl */
132 #define TLS_MFS 16384
133
134 #define RSS_HDR sizeof(struct rss_header)
135 #define TLS_WR_CPL_LEN \
136 (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
137
138 enum {
139 CHTLS_KEY_CONTEXT_DSGL,
140 CHTLS_KEY_CONTEXT_IMM,
141 CHTLS_KEY_CONTEXT_DDR,
142 };
143
144 enum {
145 CHTLS_LISTEN_START,
146 CHTLS_LISTEN_STOP,
147 };
148
149 /* Flags for return value of CPL message handlers */
150 enum {
151 CPL_RET_BUF_DONE = 1, /* buffer processing done */
152 CPL_RET_BAD_MSG = 2, /* bad CPL message */
153 CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
154 };
155
156 #define LISTEN_INFO_HASH_SIZE 32
157 #define RSPQ_HASH_BITS 5
158 struct listen_info {
159 struct listen_info *next; /* Link to next entry */
160 struct sock *sk; /* The listening socket */
161 unsigned int stid; /* The server TID */
162 };
163
164 enum {
165 T4_LISTEN_START_PENDING,
166 T4_LISTEN_STARTED
167 };
168
169 enum csk_flags {
170 CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
171 CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
172 CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
173 CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
174 CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
175 CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
176 CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
177 CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
178 CSK_TX_FAILOVER, /* Tx traffic failing over */
179 CSK_UPDATE_RCV_WND, /* Need to update rcv window */
180 CSK_RST_ABORTED, /* outgoing RST was aborted */
181 CSK_TLS_HANDSHK, /* TLS Handshake */
182 CSK_CONN_INLINE, /* Connection on HW */
183 };
184
185 enum chtls_cdev_state {
186 CHTLS_CDEV_STATE_UP = 1
187 };
188
189 struct listen_ctx {
190 struct sock *lsk;
191 struct chtls_dev *cdev;
192 struct sk_buff_head synq;
193 u32 state;
194 };
195
196 struct key_map {
197 unsigned long *addr;
198 unsigned int start;
199 unsigned int available;
200 unsigned int size;
201 spinlock_t lock; /* lock for key id request from map */
202 } __packed;
203
204 struct tls_scmd {
205 u32 seqno_numivs;
206 u32 ivgen_hdrlen;
207 };
208
209 struct chtls_dev {
210 struct tls_toe_device tlsdev;
211 struct list_head list;
212 struct cxgb4_lld_info *lldi;
213 struct pci_dev *pdev;
214 struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
215 spinlock_t listen_lock; /* lock for listen list */
216 struct net_device **ports;
217 struct tid_info *tids;
218 unsigned int pfvf;
219 const unsigned short *mtus;
220
221 struct idr hwtid_idr;
222 struct idr stid_idr;
223
224 spinlock_t idr_lock ____cacheline_aligned_in_smp;
225
226 struct net_device *egr_dev[NCHAN * 2];
227 struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
228 struct sk_buff *askb;
229
230 struct sk_buff_head deferq;
231 struct work_struct deferq_task;
232
233 struct list_head list_node;
234 struct list_head rcu_node;
235 struct list_head na_node;
236 unsigned int send_page_order;
237 int max_host_sndbuf;
238 struct key_map kmap;
239 unsigned int cdev_state;
240 };
241
242 struct chtls_listen {
243 struct chtls_dev *cdev;
244 struct sock *sk;
245 };
246
247 struct chtls_hws {
248 struct sk_buff_head sk_recv_queue;
249 u8 txqid;
250 u8 ofld;
251 u16 type;
252 u16 rstate;
253 u16 keyrpl;
254 u16 pldlen;
255 u16 rcvpld;
256 u16 compute;
257 u16 expansion;
258 u16 keylen;
259 u16 pdus;
260 u16 adjustlen;
261 u16 ivsize;
262 u16 txleft;
263 u32 mfs;
264 s32 txkey;
265 s32 rxkey;
266 u32 fcplenmax;
267 u32 copied_seq;
268 u64 tx_seq_no;
269 struct tls_scmd scmd;
270 union {
271 struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
272 struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
273 } crypto_info;
274 };
275
276 struct chtls_sock {
277 struct sock *sk;
278 struct chtls_dev *cdev;
279 struct l2t_entry *l2t_entry; /* pointer to the L2T entry */
280 struct net_device *egress_dev; /* TX_CHAN for act open retry */
281
282 struct sk_buff_head txq;
283 struct sk_buff *wr_skb_head;
284 struct sk_buff *wr_skb_tail;
285 struct sk_buff *ctrl_skb_cache;
286 struct sk_buff *txdata_skb_cache; /* abort path messages */
287 struct kref kref;
288 unsigned long flags;
289 u32 opt2;
290 u32 wr_credits;
291 u32 wr_unacked;
292 u32 wr_max_credits;
293 u32 wr_nondata;
294 u32 hwtid; /* TCP Control Block ID */
295 u32 txq_idx;
296 u32 rss_qid;
297 u32 tid;
298 u32 idr;
299 u32 mss;
300 u32 ulp_mode;
301 u32 tx_chan;
302 u32 rx_chan;
303 u32 sndbuf;
304 u32 txplen_max;
305 u32 mtu_idx; /* MTU table index */
306 u32 smac_idx;
307 u8 port_id;
308 u8 tos;
309 u16 resv2;
310 u32 delack_mode;
311 u32 delack_seq;
312 u32 snd_win;
313 u32 rcv_win;
314
315 void *passive_reap_next; /* placeholder for passive */
316 struct chtls_hws tlshws;
317 struct synq {
318 struct sk_buff *next;
319 struct sk_buff *prev;
320 } synq;
321 struct listen_ctx *listen_ctx;
322 };
323
324 struct tls_hdr {
325 u8 type;
326 u16 version;
327 u16 length;
328 } __packed;
329
330 struct tlsrx_cmp_hdr {
331 u8 type;
332 u16 version;
333 u16 length;
334
335 u64 tls_seq;
336 u16 reserved1;
337 u8 res_to_mac_error;
338 } __packed;
339
340 /* res_to_mac_error fields */
341 #define TLSRX_HDR_PKT_INT_ERROR_S 4
342 #define TLSRX_HDR_PKT_INT_ERROR_M 0x1
343 #define TLSRX_HDR_PKT_INT_ERROR_V(x) \
344 ((x) << TLSRX_HDR_PKT_INT_ERROR_S)
345 #define TLSRX_HDR_PKT_INT_ERROR_G(x) \
346 (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
347 #define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
348
349 #define TLSRX_HDR_PKT_SPP_ERROR_S 3
350 #define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
351 #define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
352 #define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
353 (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
354 #define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
355
356 #define TLSRX_HDR_PKT_CCDX_ERROR_S 2
357 #define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
358 #define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
359 #define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
360 (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
361 #define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
362
363 #define TLSRX_HDR_PKT_PAD_ERROR_S 1
364 #define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
365 #define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
366 #define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
367 (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
368 #define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
369
370 #define TLSRX_HDR_PKT_MAC_ERROR_S 0
371 #define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
372 #define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
373 #define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
374 (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
375 #define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
376
377 #define TLSRX_HDR_PKT_ERROR_M 0x1F
378 #define CONTENT_TYPE_ERROR 0x7F
379
380 struct ulp_mem_rw {
381 __be32 cmd;
382 __be32 len16; /* command length */
383 __be32 dlen; /* data length in 32-byte units */
384 __be32 lock_addr;
385 };
386
387 struct tls_key_wr {
388 __be32 op_to_compl;
389 __be32 flowid_len16;
390 __be32 ftid;
391 u8 reneg_to_write_rx;
392 u8 protocol;
393 __be16 mfs;
394 };
395
396 struct tls_key_req {
397 struct tls_key_wr wr;
398 struct ulp_mem_rw req;
399 struct ulptx_idata sc_imm;
400 };
401
402 /*
403 * This lives in skb->cb and is used to chain WRs in a linked list.
404 */
405 struct wr_skb_cb {
406 struct l2t_skb_cb l2t; /* reserve space for l2t CB */
407 struct sk_buff *next_wr; /* next write request */
408 };
409
410 /* Per-skb backlog handler. Run when a socket's backlog is processed. */
411 struct blog_skb_cb {
412 void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
413 struct chtls_dev *cdev;
414 };
415
416 /*
417 * Similar to tcp_skb_cb but with ULP elements added to support TLS,
418 * etc.
419 */
420 struct ulp_skb_cb {
421 struct wr_skb_cb wr; /* reserve space for write request */
422 u16 flags; /* TCP-like flags */
423 u8 psh;
424 u8 ulp_mode; /* ULP mode/submode of sk_buff */
425 u32 seq; /* TCP sequence number */
426 union { /* ULP-specific fields */
427 struct {
428 u8 type;
429 u8 ofld;
430 u8 iv;
431 } tls;
432 } ulp;
433 };
434
435 #define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
436 #define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
437
438 /*
439 * Flags for ulp_skb_cb.flags.
440 */
441 enum {
442 ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
443 ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
444 ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
445 ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
446 ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
447 ULPCB_FLAG_URG = 1 << 5, /* urgent data */
448 ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
449 ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
450 };
451
452 /* The ULP mode/submode of an skbuff */
453 #define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
454 #define TCP_PAGE(sk) (sk->sk_frag.page)
455 #define TCP_OFF(sk) (sk->sk_frag.offset)
456
to_chtls_dev(struct tls_toe_device * tlsdev)457 static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
458 {
459 return container_of(tlsdev, struct chtls_dev, tlsdev);
460 }
461
csk_set_flag(struct chtls_sock * csk,enum csk_flags flag)462 static inline void csk_set_flag(struct chtls_sock *csk,
463 enum csk_flags flag)
464 {
465 __set_bit(flag, &csk->flags);
466 }
467
csk_reset_flag(struct chtls_sock * csk,enum csk_flags flag)468 static inline void csk_reset_flag(struct chtls_sock *csk,
469 enum csk_flags flag)
470 {
471 __clear_bit(flag, &csk->flags);
472 }
473
csk_conn_inline(const struct chtls_sock * csk)474 static inline bool csk_conn_inline(const struct chtls_sock *csk)
475 {
476 return test_bit(CSK_CONN_INLINE, &csk->flags);
477 }
478
csk_flag(const struct sock * sk,enum csk_flags flag)479 static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
480 {
481 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
482
483 if (!csk_conn_inline(csk))
484 return 0;
485 return test_bit(flag, &csk->flags);
486 }
487
csk_flag_nochk(const struct chtls_sock * csk,enum csk_flags flag)488 static inline int csk_flag_nochk(const struct chtls_sock *csk,
489 enum csk_flags flag)
490 {
491 return test_bit(flag, &csk->flags);
492 }
493
cplhdr(struct sk_buff * skb)494 static inline void *cplhdr(struct sk_buff *skb)
495 {
496 return skb->data;
497 }
498
is_neg_adv(unsigned int status)499 static inline int is_neg_adv(unsigned int status)
500 {
501 return status == CPL_ERR_RTX_NEG_ADVICE ||
502 status == CPL_ERR_KEEPALV_NEG_ADVICE ||
503 status == CPL_ERR_PERSIST_NEG_ADVICE;
504 }
505
process_cpl_msg(void (* fn)(struct sock *,struct sk_buff *),struct sock * sk,struct sk_buff * skb)506 static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
507 struct sock *sk,
508 struct sk_buff *skb)
509 {
510 skb_reset_mac_header(skb);
511 skb_reset_network_header(skb);
512 skb_reset_transport_header(skb);
513
514 bh_lock_sock(sk);
515 if (unlikely(sock_owned_by_user(sk))) {
516 BLOG_SKB_CB(skb)->backlog_rcv = fn;
517 __sk_add_backlog(sk, skb);
518 } else {
519 fn(sk, skb);
520 }
521 bh_unlock_sock(sk);
522 }
523
chtls_sock_free(struct kref * ref)524 static inline void chtls_sock_free(struct kref *ref)
525 {
526 struct chtls_sock *csk = container_of(ref, struct chtls_sock,
527 kref);
528 kfree(csk);
529 }
530
__chtls_sock_put(const char * fn,struct chtls_sock * csk)531 static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
532 {
533 kref_put(&csk->kref, chtls_sock_free);
534 }
535
__chtls_sock_get(const char * fn,struct chtls_sock * csk)536 static inline void __chtls_sock_get(const char *fn,
537 struct chtls_sock *csk)
538 {
539 kref_get(&csk->kref);
540 }
541
send_or_defer(struct sock * sk,struct tcp_sock * tp,struct sk_buff * skb,int through_l2t)542 static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
543 struct sk_buff *skb, int through_l2t)
544 {
545 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
546
547 if (through_l2t) {
548 /* send through L2T */
549 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
550 } else {
551 /* send directly */
552 cxgb4_ofld_send(csk->egress_dev, skb);
553 }
554 }
555
556 typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
557 extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
558 void chtls_install_cpl_ops(struct sock *sk);
559 int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
560 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
561 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
562 void chtls_close(struct sock *sk, long timeout);
563 int chtls_disconnect(struct sock *sk, int flags);
564 void chtls_shutdown(struct sock *sk, int how);
565 void chtls_destroy_sock(struct sock *sk);
566 int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
567 int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
568 size_t len, int nonblock, int flags, int *addr_len);
569 int chtls_sendpage(struct sock *sk, struct page *page,
570 int offset, size_t size, int flags);
571 int send_tx_flowc_wr(struct sock *sk, int compl,
572 u32 snd_nxt, u32 rcv_nxt);
573 void chtls_tcp_push(struct sock *sk, int flags);
574 int chtls_push_frames(struct chtls_sock *csk, int comp);
575 int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
576 void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
577 u64 mask, u64 val, u8 cookie,
578 int through_l2t);
579 int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
580 void chtls_set_quiesce_ctrl(struct sock *sk, int val);
581 void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
582 unsigned int keyid_to_addr(int start_addr, int keyid);
583 void free_tls_keyid(struct sock *sk);
584 #endif
585