1 #include <linux/socket.h> 2 #include <linux/in.h> 3 #include <linux/in6.h> 4 #include <rdma/ib_verbs.h> 5 #include <rdma/rdma_cm.h> 6 7 #define ISERT_RDMA_LISTEN_BACKLOG 10 8 #define ISCSI_ISER_SG_TABLESIZE 256 9 #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 10 11 enum isert_desc_type { 12 ISCSI_TX_CONTROL, 13 ISCSI_TX_DATAIN 14 }; 15 16 enum iser_ib_op_code { 17 ISER_IB_RECV, 18 ISER_IB_SEND, 19 ISER_IB_RDMA_WRITE, 20 ISER_IB_RDMA_READ, 21 }; 22 23 enum iser_conn_state { 24 ISER_CONN_INIT, 25 ISER_CONN_UP, 26 ISER_CONN_FULL_FEATURE, 27 ISER_CONN_TERMINATING, 28 ISER_CONN_DOWN, 29 }; 30 31 struct iser_rx_desc { 32 struct iser_hdr iser_header; 33 struct iscsi_hdr iscsi_header; 34 char data[ISER_RECV_DATA_SEG_LEN]; 35 u64 dma_addr; 36 struct ib_sge rx_sg; 37 char pad[ISER_RX_PAD_SIZE]; 38 } __packed; 39 40 struct iser_tx_desc { 41 struct iser_hdr iser_header; 42 struct iscsi_hdr iscsi_header; 43 enum isert_desc_type type; 44 u64 dma_addr; 45 struct ib_sge tx_sg[2]; 46 int num_sge; 47 struct isert_cmd *isert_cmd; 48 struct llist_node *comp_llnode_batch; 49 struct llist_node comp_llnode; 50 bool llnode_active; 51 struct ib_send_wr send_wr; 52 } __packed; 53 54 enum isert_indicator { 55 ISERT_PROTECTED = 1 << 0, 56 ISERT_DATA_KEY_VALID = 1 << 1, 57 ISERT_PROT_KEY_VALID = 1 << 2, 58 ISERT_SIG_KEY_VALID = 1 << 3, 59 }; 60 61 struct pi_context { 62 struct ib_mr *prot_mr; 63 struct ib_fast_reg_page_list *prot_frpl; 64 struct ib_mr *sig_mr; 65 }; 66 67 struct fast_reg_descriptor { 68 struct list_head list; 69 struct ib_mr *data_mr; 70 struct ib_fast_reg_page_list *data_frpl; 71 u8 ind; 72 struct pi_context *pi_ctx; 73 }; 74 75 struct isert_data_buf { 76 struct scatterlist *sg; 77 int nents; 78 u32 sg_off; 79 u32 len; /* cur_rdma_length */ 80 u32 offset; 81 unsigned int dma_nents; 82 enum dma_data_direction dma_dir; 83 }; 84 85 enum { 86 DATA = 0, 87 PROT = 1, 88 SIG = 2, 89 }; 90 91 struct isert_rdma_wr { 92 struct list_head wr_list; 93 struct isert_cmd *isert_cmd; 94 enum iser_ib_op_code iser_ib_op; 95 struct ib_sge *ib_sge; 96 struct ib_sge s_ib_sge; 97 int send_wr_num; 98 struct ib_send_wr *send_wr; 99 struct ib_send_wr s_send_wr; 100 struct ib_sge ib_sg[3]; 101 struct isert_data_buf data; 102 struct isert_data_buf prot; 103 struct fast_reg_descriptor *fr_desc; 104 }; 105 106 struct isert_cmd { 107 uint32_t read_stag; 108 uint32_t write_stag; 109 uint64_t read_va; 110 uint64_t write_va; 111 u64 pdu_buf_dma; 112 u32 pdu_buf_len; 113 u32 read_va_off; 114 u32 write_va_off; 115 u32 rdma_wr_num; 116 struct isert_conn *conn; 117 struct iscsi_cmd *iscsi_cmd; 118 struct iser_tx_desc tx_desc; 119 struct isert_rdma_wr rdma_wr; 120 struct work_struct comp_work; 121 }; 122 123 struct isert_device; 124 125 struct isert_conn { 126 enum iser_conn_state state; 127 int post_recv_buf_count; 128 atomic_t post_send_buf_count; 129 u32 responder_resources; 130 u32 initiator_depth; 131 bool pi_support; 132 u32 max_sge; 133 char *login_buf; 134 char *login_req_buf; 135 char *login_rsp_buf; 136 u64 login_req_dma; 137 int login_req_len; 138 u64 login_rsp_dma; 139 unsigned int conn_rx_desc_head; 140 struct iser_rx_desc *conn_rx_descs; 141 struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX]; 142 struct iscsi_conn *conn; 143 struct list_head conn_accept_node; 144 struct completion conn_login_comp; 145 struct completion login_req_comp; 146 struct iser_tx_desc conn_login_tx_desc; 147 struct rdma_cm_id *conn_cm_id; 148 struct ib_pd *conn_pd; 149 struct ib_mr *conn_mr; 150 struct ib_qp *conn_qp; 151 struct isert_device *conn_device; 152 struct mutex conn_mutex; 153 struct completion conn_wait; 154 struct completion conn_wait_comp_err; 155 struct kref conn_kref; 156 struct list_head conn_fr_pool; 157 int conn_fr_pool_size; 158 /* lock to protect fastreg pool */ 159 spinlock_t conn_lock; 160 struct work_struct release_work; 161 #define ISERT_COMP_BATCH_COUNT 8 162 int conn_comp_batch; 163 struct llist_head conn_comp_llist; 164 }; 165 166 #define ISERT_MAX_CQ 64 167 168 struct isert_cq_desc { 169 struct isert_device *device; 170 int cq_index; 171 struct work_struct cq_rx_work; 172 struct work_struct cq_tx_work; 173 }; 174 175 struct isert_device { 176 int use_fastreg; 177 bool pi_capable; 178 int cqs_used; 179 int refcount; 180 int cq_active_qps[ISERT_MAX_CQ]; 181 struct ib_device *ib_device; 182 struct ib_cq *dev_rx_cq[ISERT_MAX_CQ]; 183 struct ib_cq *dev_tx_cq[ISERT_MAX_CQ]; 184 struct isert_cq_desc *cq_desc; 185 struct list_head dev_node; 186 struct ib_device_attr dev_attr; 187 int (*reg_rdma_mem)(struct iscsi_conn *conn, 188 struct iscsi_cmd *cmd, 189 struct isert_rdma_wr *wr); 190 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, 191 struct isert_conn *isert_conn); 192 }; 193 194 struct isert_np { 195 struct iscsi_np *np; 196 struct semaphore np_sem; 197 struct rdma_cm_id *np_cm_id; 198 struct mutex np_accept_mutex; 199 struct list_head np_accept_list; 200 struct completion np_login_comp; 201 }; 202