1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (c) 2018 Chelsio Communications, Inc.
4 */
5
6 #ifndef __CHTLS_H__
7 #define __CHTLS_H__
8
9 #include <crypto/aes.h>
10 #include <crypto/algapi.h>
11 #include <crypto/hash.h>
12 #include <crypto/sha.h>
13 #include <crypto/authenc.h>
14 #include <crypto/ctr.h>
15 #include <crypto/gf128mul.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/null.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/aead.h>
20 #include <crypto/scatterwalk.h>
21 #include <crypto/internal/hash.h>
22 #include <linux/tls.h>
23 #include <net/tls.h>
24
25 #include "t4fw_api.h"
26 #include "t4_msg.h"
27 #include "cxgb4.h"
28 #include "cxgb4_uld.h"
29 #include "l2t.h"
30 #include "chcr_algo.h"
31 #include "chcr_core.h"
32 #include "chcr_crypto.h"
33
34 #define MAX_IVS_PAGE 256
35 #define TLS_KEY_CONTEXT_SZ 64
36 #define CIPHER_BLOCK_SIZE 16
37 #define GCM_TAG_SIZE 16
38 #define KEY_ON_MEM_SZ 16
39 #define AEAD_EXPLICIT_DATA_SIZE 8
40 #define TLS_HEADER_LENGTH 5
41 #define SCMD_CIPH_MODE_AES_GCM 2
42 /* Any MFS size should work and come from openssl */
43 #define TLS_MFS 16384
44
45 #define RSS_HDR sizeof(struct rss_header)
46 #define TLS_WR_CPL_LEN \
47 (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
48
49 enum {
50 CHTLS_KEY_CONTEXT_DSGL,
51 CHTLS_KEY_CONTEXT_IMM,
52 CHTLS_KEY_CONTEXT_DDR,
53 };
54
55 enum {
56 CHTLS_LISTEN_START,
57 CHTLS_LISTEN_STOP,
58 };
59
60 /* Flags for return value of CPL message handlers */
61 enum {
62 CPL_RET_BUF_DONE = 1, /* buffer processing done */
63 CPL_RET_BAD_MSG = 2, /* bad CPL message */
64 CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
65 };
66
67 #define LISTEN_INFO_HASH_SIZE 32
68 #define RSPQ_HASH_BITS 5
69 struct listen_info {
70 struct listen_info *next; /* Link to next entry */
71 struct sock *sk; /* The listening socket */
72 unsigned int stid; /* The server TID */
73 };
74
75 enum {
76 T4_LISTEN_START_PENDING,
77 T4_LISTEN_STARTED
78 };
79
80 enum csk_flags {
81 CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
82 CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
83 CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
84 CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
85 CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
86 CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
87 CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
88 CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
89 CSK_TX_FAILOVER, /* Tx traffic failing over */
90 CSK_UPDATE_RCV_WND, /* Need to update rcv window */
91 CSK_RST_ABORTED, /* outgoing RST was aborted */
92 CSK_TLS_HANDSHK, /* TLS Handshake */
93 CSK_CONN_INLINE, /* Connection on HW */
94 };
95
96 enum chtls_cdev_state {
97 CHTLS_CDEV_STATE_UP = 1
98 };
99
100 struct listen_ctx {
101 struct sock *lsk;
102 struct chtls_dev *cdev;
103 struct sk_buff_head synq;
104 u32 state;
105 };
106
107 struct key_map {
108 unsigned long *addr;
109 unsigned int start;
110 unsigned int available;
111 unsigned int size;
112 spinlock_t lock; /* lock for key id request from map */
113 } __packed;
114
115 struct tls_scmd {
116 u32 seqno_numivs;
117 u32 ivgen_hdrlen;
118 };
119
120 struct chtls_dev {
121 struct tls_device tlsdev;
122 struct list_head list;
123 struct cxgb4_lld_info *lldi;
124 struct pci_dev *pdev;
125 struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
126 spinlock_t listen_lock; /* lock for listen list */
127 struct net_device **ports;
128 struct tid_info *tids;
129 unsigned int pfvf;
130 const unsigned short *mtus;
131
132 struct idr hwtid_idr;
133 struct idr stid_idr;
134
135 spinlock_t idr_lock ____cacheline_aligned_in_smp;
136
137 struct net_device *egr_dev[NCHAN * 2];
138 struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
139 struct sk_buff *askb;
140
141 struct sk_buff_head deferq;
142 struct work_struct deferq_task;
143
144 struct list_head list_node;
145 struct list_head rcu_node;
146 struct list_head na_node;
147 unsigned int send_page_order;
148 int max_host_sndbuf;
149 struct key_map kmap;
150 unsigned int cdev_state;
151 };
152
153 struct chtls_listen {
154 struct chtls_dev *cdev;
155 struct sock *sk;
156 };
157
158 struct chtls_hws {
159 struct sk_buff_head sk_recv_queue;
160 u8 txqid;
161 u8 ofld;
162 u16 type;
163 u16 rstate;
164 u16 keyrpl;
165 u16 pldlen;
166 u16 rcvpld;
167 u16 compute;
168 u16 expansion;
169 u16 keylen;
170 u16 pdus;
171 u16 adjustlen;
172 u16 ivsize;
173 u16 txleft;
174 u32 mfs;
175 s32 txkey;
176 s32 rxkey;
177 u32 fcplenmax;
178 u32 copied_seq;
179 u64 tx_seq_no;
180 struct tls_scmd scmd;
181 struct tls12_crypto_info_aes_gcm_128 crypto_info;
182 };
183
184 struct chtls_sock {
185 struct sock *sk;
186 struct chtls_dev *cdev;
187 struct l2t_entry *l2t_entry; /* pointer to the L2T entry */
188 struct net_device *egress_dev; /* TX_CHAN for act open retry */
189
190 struct sk_buff_head txq;
191 struct sk_buff *wr_skb_head;
192 struct sk_buff *wr_skb_tail;
193 struct sk_buff *ctrl_skb_cache;
194 struct sk_buff *txdata_skb_cache; /* abort path messages */
195 struct kref kref;
196 unsigned long flags;
197 u32 opt2;
198 u32 wr_credits;
199 u32 wr_unacked;
200 u32 wr_max_credits;
201 u32 wr_nondata;
202 u32 hwtid; /* TCP Control Block ID */
203 u32 txq_idx;
204 u32 rss_qid;
205 u32 tid;
206 u32 idr;
207 u32 mss;
208 u32 ulp_mode;
209 u32 tx_chan;
210 u32 rx_chan;
211 u32 sndbuf;
212 u32 txplen_max;
213 u32 mtu_idx; /* MTU table index */
214 u32 smac_idx;
215 u8 port_id;
216 u8 tos;
217 u16 resv2;
218 u32 delack_mode;
219 u32 delack_seq;
220 u32 snd_win;
221 u32 rcv_win;
222
223 void *passive_reap_next; /* placeholder for passive */
224 struct chtls_hws tlshws;
225 struct synq {
226 struct sk_buff *next;
227 struct sk_buff *prev;
228 } synq;
229 struct listen_ctx *listen_ctx;
230 };
231
232 struct tls_hdr {
233 u8 type;
234 u16 version;
235 u16 length;
236 } __packed;
237
238 struct tlsrx_cmp_hdr {
239 u8 type;
240 u16 version;
241 u16 length;
242
243 u64 tls_seq;
244 u16 reserved1;
245 u8 res_to_mac_error;
246 } __packed;
247
248 /* res_to_mac_error fields */
249 #define TLSRX_HDR_PKT_INT_ERROR_S 4
250 #define TLSRX_HDR_PKT_INT_ERROR_M 0x1
251 #define TLSRX_HDR_PKT_INT_ERROR_V(x) \
252 ((x) << TLSRX_HDR_PKT_INT_ERROR_S)
253 #define TLSRX_HDR_PKT_INT_ERROR_G(x) \
254 (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
255 #define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
256
257 #define TLSRX_HDR_PKT_SPP_ERROR_S 3
258 #define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
259 #define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
260 #define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
261 (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
262 #define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
263
264 #define TLSRX_HDR_PKT_CCDX_ERROR_S 2
265 #define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
266 #define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
267 #define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
268 (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
269 #define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
270
271 #define TLSRX_HDR_PKT_PAD_ERROR_S 1
272 #define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
273 #define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
274 #define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
275 (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
276 #define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
277
278 #define TLSRX_HDR_PKT_MAC_ERROR_S 0
279 #define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
280 #define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
281 #define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
282 (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
283 #define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
284
285 #define TLSRX_HDR_PKT_ERROR_M 0x1F
286 #define CONTENT_TYPE_ERROR 0x7F
287
288 struct ulp_mem_rw {
289 __be32 cmd;
290 __be32 len16; /* command length */
291 __be32 dlen; /* data length in 32-byte units */
292 __be32 lock_addr;
293 };
294
295 struct tls_key_wr {
296 __be32 op_to_compl;
297 __be32 flowid_len16;
298 __be32 ftid;
299 u8 reneg_to_write_rx;
300 u8 protocol;
301 __be16 mfs;
302 };
303
304 struct tls_key_req {
305 struct tls_key_wr wr;
306 struct ulp_mem_rw req;
307 struct ulptx_idata sc_imm;
308 };
309
310 /*
311 * This lives in skb->cb and is used to chain WRs in a linked list.
312 */
313 struct wr_skb_cb {
314 struct l2t_skb_cb l2t; /* reserve space for l2t CB */
315 struct sk_buff *next_wr; /* next write request */
316 };
317
318 /* Per-skb backlog handler. Run when a socket's backlog is processed. */
319 struct blog_skb_cb {
320 void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
321 struct chtls_dev *cdev;
322 };
323
324 /*
325 * Similar to tcp_skb_cb but with ULP elements added to support TLS,
326 * etc.
327 */
328 struct ulp_skb_cb {
329 struct wr_skb_cb wr; /* reserve space for write request */
330 u16 flags; /* TCP-like flags */
331 u8 psh;
332 u8 ulp_mode; /* ULP mode/submode of sk_buff */
333 u32 seq; /* TCP sequence number */
334 union { /* ULP-specific fields */
335 struct {
336 u8 type;
337 u8 ofld;
338 u8 iv;
339 } tls;
340 } ulp;
341 };
342
343 #define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
344 #define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
345
346 /*
347 * Flags for ulp_skb_cb.flags.
348 */
349 enum {
350 ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
351 ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
352 ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
353 ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
354 ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
355 ULPCB_FLAG_URG = 1 << 5, /* urgent data */
356 ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
357 ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
358 };
359
360 /* The ULP mode/submode of an skbuff */
361 #define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
362 #define TCP_PAGE(sk) (sk->sk_frag.page)
363 #define TCP_OFF(sk) (sk->sk_frag.offset)
364
to_chtls_dev(struct tls_device * tlsdev)365 static inline struct chtls_dev *to_chtls_dev(struct tls_device *tlsdev)
366 {
367 return container_of(tlsdev, struct chtls_dev, tlsdev);
368 }
369
csk_set_flag(struct chtls_sock * csk,enum csk_flags flag)370 static inline void csk_set_flag(struct chtls_sock *csk,
371 enum csk_flags flag)
372 {
373 __set_bit(flag, &csk->flags);
374 }
375
csk_reset_flag(struct chtls_sock * csk,enum csk_flags flag)376 static inline void csk_reset_flag(struct chtls_sock *csk,
377 enum csk_flags flag)
378 {
379 __clear_bit(flag, &csk->flags);
380 }
381
csk_conn_inline(const struct chtls_sock * csk)382 static inline bool csk_conn_inline(const struct chtls_sock *csk)
383 {
384 return test_bit(CSK_CONN_INLINE, &csk->flags);
385 }
386
csk_flag(const struct sock * sk,enum csk_flags flag)387 static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
388 {
389 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
390
391 if (!csk_conn_inline(csk))
392 return 0;
393 return test_bit(flag, &csk->flags);
394 }
395
csk_flag_nochk(const struct chtls_sock * csk,enum csk_flags flag)396 static inline int csk_flag_nochk(const struct chtls_sock *csk,
397 enum csk_flags flag)
398 {
399 return test_bit(flag, &csk->flags);
400 }
401
cplhdr(struct sk_buff * skb)402 static inline void *cplhdr(struct sk_buff *skb)
403 {
404 return skb->data;
405 }
406
is_neg_adv(unsigned int status)407 static inline int is_neg_adv(unsigned int status)
408 {
409 return status == CPL_ERR_RTX_NEG_ADVICE ||
410 status == CPL_ERR_KEEPALV_NEG_ADVICE ||
411 status == CPL_ERR_PERSIST_NEG_ADVICE;
412 }
413
process_cpl_msg(void (* fn)(struct sock *,struct sk_buff *),struct sock * sk,struct sk_buff * skb)414 static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
415 struct sock *sk,
416 struct sk_buff *skb)
417 {
418 skb_reset_mac_header(skb);
419 skb_reset_network_header(skb);
420 skb_reset_transport_header(skb);
421
422 bh_lock_sock(sk);
423 if (unlikely(sock_owned_by_user(sk))) {
424 BLOG_SKB_CB(skb)->backlog_rcv = fn;
425 __sk_add_backlog(sk, skb);
426 } else {
427 fn(sk, skb);
428 }
429 bh_unlock_sock(sk);
430 }
431
chtls_sock_free(struct kref * ref)432 static inline void chtls_sock_free(struct kref *ref)
433 {
434 struct chtls_sock *csk = container_of(ref, struct chtls_sock,
435 kref);
436 kfree(csk);
437 }
438
__chtls_sock_put(const char * fn,struct chtls_sock * csk)439 static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
440 {
441 kref_put(&csk->kref, chtls_sock_free);
442 }
443
__chtls_sock_get(const char * fn,struct chtls_sock * csk)444 static inline void __chtls_sock_get(const char *fn,
445 struct chtls_sock *csk)
446 {
447 kref_get(&csk->kref);
448 }
449
send_or_defer(struct sock * sk,struct tcp_sock * tp,struct sk_buff * skb,int through_l2t)450 static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
451 struct sk_buff *skb, int through_l2t)
452 {
453 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
454
455 if (through_l2t) {
456 /* send through L2T */
457 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
458 } else {
459 /* send directly */
460 cxgb4_ofld_send(csk->egress_dev, skb);
461 }
462 }
463
464 typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
465 extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
466 void chtls_install_cpl_ops(struct sock *sk);
467 int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
468 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
469 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
470 void chtls_close(struct sock *sk, long timeout);
471 int chtls_disconnect(struct sock *sk, int flags);
472 void chtls_shutdown(struct sock *sk, int how);
473 void chtls_destroy_sock(struct sock *sk);
474 int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
475 int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
476 size_t len, int nonblock, int flags, int *addr_len);
477 int chtls_sendpage(struct sock *sk, struct page *page,
478 int offset, size_t size, int flags);
479 int send_tx_flowc_wr(struct sock *sk, int compl,
480 u32 snd_nxt, u32 rcv_nxt);
481 void chtls_tcp_push(struct sock *sk, int flags);
482 int chtls_push_frames(struct chtls_sock *csk, int comp);
483 int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
484 int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode);
485 void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
486 unsigned int keyid_to_addr(int start_addr, int keyid);
487 void free_tls_keyid(struct sock *sk);
488 #endif
489