1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/socket.h>
3 #include <linux/in.h>
4 #include <linux/in6.h>
5 #include <rdma/ib_verbs.h>
6 #include <rdma/rdma_cm.h>
7 #include <rdma/rw.h>
8 #include <scsi/iser.h>
9
10
11 #define DRV_NAME "isert"
12 #define PFX DRV_NAME ": "
13
14 #define isert_dbg(fmt, arg...) \
15 do { \
16 if (unlikely(isert_debug_level > 2)) \
17 printk(KERN_DEBUG PFX "%s: " fmt,\
18 __func__ , ## arg); \
19 } while (0)
20
21 #define isert_warn(fmt, arg...) \
22 do { \
23 if (unlikely(isert_debug_level > 0)) \
24 pr_warn(PFX "%s: " fmt, \
25 __func__ , ## arg); \
26 } while (0)
27
28 #define isert_info(fmt, arg...) \
29 do { \
30 if (unlikely(isert_debug_level > 1)) \
31 pr_info(PFX "%s: " fmt, \
32 __func__ , ## arg); \
33 } while (0)
34
35 #define isert_err(fmt, arg...) \
36 pr_err(PFX "%s: " fmt, __func__ , ## arg)
37
38 /* Constant PDU lengths calculations */
39 #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \
40 sizeof(struct iscsi_hdr))
41 #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
42
43 /* QP settings */
44 /* Maximal bounds on received asynchronous PDUs */
45 #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */
46
47 #define ISERT_MAX_RX_MISC_PDUS 6 /*
48 * NOOP_OUT(2), TEXT(1),
49 * SCSI_TMFUNC(2), LOGOUT(1)
50 */
51
52 #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
53
54 #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX)
55
56 #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2)
57
58 #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX + \
59 ISERT_MAX_TX_MISC_PDUS + \
60 ISERT_MAX_RX_MISC_PDUS)
61
62 /*
63 * RX size is default of 8k plus headers, but data needs to align to
64 * 512 boundary, so use 1024 to have the extra space for alignment.
65 */
66 #define ISER_RX_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 1024)
67
68 #define ISCSI_ISER_SG_TABLESIZE 256
69
70 enum isert_desc_type {
71 ISCSI_TX_CONTROL,
72 ISCSI_TX_DATAIN
73 };
74
75 enum iser_conn_state {
76 ISER_CONN_INIT,
77 ISER_CONN_UP,
78 ISER_CONN_BOUND,
79 ISER_CONN_FULL_FEATURE,
80 ISER_CONN_TERMINATING,
81 ISER_CONN_DOWN,
82 };
83
84 struct iser_rx_desc {
85 char buf[ISER_RX_SIZE];
86 u64 dma_addr;
87 struct ib_sge rx_sg;
88 struct ib_cqe rx_cqe;
89 bool in_use;
90 };
91
cqe_to_rx_desc(struct ib_cqe * cqe)92 static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe)
93 {
94 return container_of(cqe, struct iser_rx_desc, rx_cqe);
95 }
96
isert_get_iser_hdr(struct iser_rx_desc * desc)97 static void *isert_get_iser_hdr(struct iser_rx_desc *desc)
98 {
99 return PTR_ALIGN(desc->buf + ISER_HEADERS_LEN, 512) - ISER_HEADERS_LEN;
100 }
101
isert_get_hdr_offset(struct iser_rx_desc * desc)102 static size_t isert_get_hdr_offset(struct iser_rx_desc *desc)
103 {
104 return isert_get_iser_hdr(desc) - (void *)desc->buf;
105 }
106
isert_get_iscsi_hdr(struct iser_rx_desc * desc)107 static void *isert_get_iscsi_hdr(struct iser_rx_desc *desc)
108 {
109 return isert_get_iser_hdr(desc) + sizeof(struct iser_ctrl);
110 }
111
isert_get_data(struct iser_rx_desc * desc)112 static void *isert_get_data(struct iser_rx_desc *desc)
113 {
114 void *data = isert_get_iser_hdr(desc) + ISER_HEADERS_LEN;
115
116 WARN_ON((uintptr_t)data & 511);
117 return data;
118 }
119
120 struct iser_tx_desc {
121 struct iser_ctrl iser_header;
122 struct iscsi_hdr iscsi_header;
123 enum isert_desc_type type;
124 u64 dma_addr;
125 struct ib_sge tx_sg[2];
126 struct ib_cqe tx_cqe;
127 int num_sge;
128 struct ib_send_wr send_wr;
129 } __packed;
130
cqe_to_tx_desc(struct ib_cqe * cqe)131 static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe)
132 {
133 return container_of(cqe, struct iser_tx_desc, tx_cqe);
134 }
135
136 struct isert_cmd {
137 uint32_t read_stag;
138 uint32_t write_stag;
139 uint64_t read_va;
140 uint64_t write_va;
141 uint32_t inv_rkey;
142 u64 pdu_buf_dma;
143 u32 pdu_buf_len;
144 struct isert_conn *conn;
145 struct iscsi_cmd *iscsi_cmd;
146 struct iser_tx_desc tx_desc;
147 struct iser_rx_desc *rx_desc;
148 struct rdma_rw_ctx rw;
149 struct work_struct comp_work;
150 struct scatterlist sg;
151 bool ctx_init_done;
152 };
153
tx_desc_to_cmd(struct iser_tx_desc * desc)154 static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)
155 {
156 return container_of(desc, struct isert_cmd, tx_desc);
157 }
158
159 struct isert_device;
160
161 struct isert_conn {
162 enum iser_conn_state state;
163 u32 responder_resources;
164 u32 initiator_depth;
165 bool pi_support;
166 struct iser_rx_desc *login_desc;
167 char *login_rsp_buf;
168 int login_req_len;
169 u64 login_rsp_dma;
170 struct iser_rx_desc *rx_descs;
171 struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS];
172 struct iscsi_conn *conn;
173 struct list_head node;
174 struct completion login_comp;
175 struct completion login_req_comp;
176 struct iser_tx_desc login_tx_desc;
177 struct rdma_cm_id *cm_id;
178 struct ib_qp *qp;
179 struct isert_device *device;
180 struct mutex mutex;
181 struct kref kref;
182 struct work_struct release_work;
183 bool logout_posted;
184 bool snd_w_inv;
185 wait_queue_head_t rem_wait;
186 bool dev_removed;
187 };
188
189 #define ISERT_MAX_CQ 64
190
191 /**
192 * struct isert_comp - iSER completion context
193 *
194 * @device: pointer to device handle
195 * @cq: completion queue
196 * @active_qps: Number of active QPs attached
197 * to completion context
198 */
199 struct isert_comp {
200 struct isert_device *device;
201 struct ib_cq *cq;
202 int active_qps;
203 };
204
205 struct isert_device {
206 bool pi_capable;
207 int refcount;
208 struct ib_device *ib_device;
209 struct ib_pd *pd;
210 struct isert_comp *comps;
211 int comps_used;
212 struct list_head dev_node;
213 };
214
215 struct isert_np {
216 struct iscsi_np *np;
217 struct semaphore sem;
218 struct rdma_cm_id *cm_id;
219 struct mutex mutex;
220 struct list_head accepted;
221 struct list_head pending;
222 };
223