1 /*
2 * cxgb3i_pdu.c: Chelsio S3xx iSCSI driver.
3 *
4 * Copyright (c) 2008 Chelsio Communications, Inc.
5 * Copyright (c) 2008 Mike Christie
6 * Copyright (c) 2008 Red Hat, Inc. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation.
11 *
12 * Written by: Karen Xie (kxie@chelsio.com)
13 */
14
15 #include <linux/skbuff.h>
16 #include <linux/crypto.h>
17 #include <scsi/scsi_cmnd.h>
18 #include <scsi/scsi_host.h>
19
20 #include "cxgb3i.h"
21 #include "cxgb3i_pdu.h"
22
23 #ifdef __DEBUG_CXGB3I_RX__
24 #define cxgb3i_rx_debug cxgb3i_log_debug
25 #else
26 #define cxgb3i_rx_debug(fmt...)
27 #endif
28
29 #ifdef __DEBUG_CXGB3I_TX__
30 #define cxgb3i_tx_debug cxgb3i_log_debug
31 #else
32 #define cxgb3i_tx_debug(fmt...)
33 #endif
34
35 /* always allocate rooms for AHS */
36 #define SKB_TX_PDU_HEADER_LEN \
37 (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE)
38 static unsigned int skb_extra_headroom;
39 static struct page *pad_page;
40
41 /*
42 * pdu receive, interact with libiscsi_tcp
43 */
read_pdu_skb(struct iscsi_conn * conn,struct sk_buff * skb,unsigned int offset,int offloaded)44 static inline int read_pdu_skb(struct iscsi_conn *conn, struct sk_buff *skb,
45 unsigned int offset, int offloaded)
46 {
47 int status = 0;
48 int bytes_read;
49
50 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
51 switch (status) {
52 case ISCSI_TCP_CONN_ERR:
53 return -EIO;
54 case ISCSI_TCP_SUSPENDED:
55 /* no transfer - just have caller flush queue */
56 return bytes_read;
57 case ISCSI_TCP_SKB_DONE:
58 /*
59 * pdus should always fit in the skb and we should get
60 * segment done notifcation.
61 */
62 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
63 return -EFAULT;
64 case ISCSI_TCP_SEGMENT_DONE:
65 return bytes_read;
66 default:
67 iscsi_conn_printk(KERN_ERR, conn, "Invalid iscsi_tcp_recv_skb "
68 "status %d\n", status);
69 return -EINVAL;
70 }
71 }
72
cxgb3i_conn_read_pdu_skb(struct iscsi_conn * conn,struct sk_buff * skb)73 static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
74 struct sk_buff *skb)
75 {
76 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
77 bool offloaded = 0;
78 unsigned int offset;
79 int rc;
80
81 cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
82 conn, skb, skb->len, skb_ulp_mode(skb));
83
84 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
85 iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
86 return -EIO;
87 }
88
89 if (conn->hdrdgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) {
90 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
91 return -EIO;
92 }
93
94 if (conn->datadgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) {
95 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
96 return -EIO;
97 }
98
99 /* iscsi hdr */
100 rc = read_pdu_skb(conn, skb, 0, 0);
101 if (rc <= 0)
102 return rc;
103
104 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
105 return 0;
106
107 offset = rc;
108 if (conn->hdrdgst_en)
109 offset += ISCSI_DIGEST_SIZE;
110
111 /* iscsi data */
112 if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
113 cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
114 "itt 0x%x.\n",
115 skb,
116 tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
117 tcp_conn->in.datalen,
118 ntohl(tcp_conn->in.hdr->itt));
119 offloaded = 1;
120 } else {
121 cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, NOT ddp'ed, "
122 "itt 0x%x.\n",
123 skb,
124 tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
125 tcp_conn->in.datalen,
126 ntohl(tcp_conn->in.hdr->itt));
127 offset += sizeof(struct cpl_iscsi_hdr_norss);
128 }
129
130 rc = read_pdu_skb(conn, skb, offset, offloaded);
131 if (rc < 0)
132 return rc;
133 else
134 return 0;
135 }
136
137 /*
138 * pdu transmit, interact with libiscsi_tcp
139 */
tx_skb_setmode(struct sk_buff * skb,int hcrc,int dcrc)140 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
141 {
142 u8 submode = 0;
143
144 if (hcrc)
145 submode |= 1;
146 if (dcrc)
147 submode |= 2;
148 skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
149 }
150
cxgb3i_conn_cleanup_task(struct iscsi_task * task)151 void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
152 {
153 struct cxgb3i_task_data *tdata = task->dd_data +
154 sizeof(struct iscsi_tcp_task);
155
156 /* never reached the xmit task callout */
157 if (tdata->skb)
158 __kfree_skb(tdata->skb);
159 memset(tdata, 0, sizeof(struct cxgb3i_task_data));
160
161 /* MNC - Do we need a check in case this is called but
162 * cxgb3i_conn_alloc_pdu has never been called on the task */
163 cxgb3i_release_itt(task, task->hdr_itt);
164 iscsi_tcp_cleanup_task(task);
165 }
166
sgl_seek_offset(struct scatterlist * sgl,unsigned int sgcnt,unsigned int offset,unsigned int * off,struct scatterlist ** sgp)167 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
168 unsigned int offset, unsigned int *off,
169 struct scatterlist **sgp)
170 {
171 int i;
172 struct scatterlist *sg;
173
174 for_each_sg(sgl, sg, sgcnt, i) {
175 if (offset < sg->length) {
176 *off = offset;
177 *sgp = sg;
178 return 0;
179 }
180 offset -= sg->length;
181 }
182 return -EFAULT;
183 }
184
sgl_read_to_frags(struct scatterlist * sg,unsigned int sgoffset,unsigned int dlen,skb_frag_t * frags,int frag_max)185 static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
186 unsigned int dlen, skb_frag_t *frags,
187 int frag_max)
188 {
189 unsigned int datalen = dlen;
190 unsigned int sglen = sg->length - sgoffset;
191 struct page *page = sg_page(sg);
192 int i;
193
194 i = 0;
195 do {
196 unsigned int copy;
197
198 if (!sglen) {
199 sg = sg_next(sg);
200 if (!sg) {
201 cxgb3i_log_error("%s, sg NULL, len %u/%u.\n",
202 __func__, datalen, dlen);
203 return -EINVAL;
204 }
205 sgoffset = 0;
206 sglen = sg->length;
207 page = sg_page(sg);
208
209 }
210 copy = min(datalen, sglen);
211 if (i && page == frags[i - 1].page &&
212 sgoffset + sg->offset ==
213 frags[i - 1].page_offset + frags[i - 1].size) {
214 frags[i - 1].size += copy;
215 } else {
216 if (i >= frag_max) {
217 cxgb3i_log_error("%s, too many pages %u, "
218 "dlen %u.\n", __func__,
219 frag_max, dlen);
220 return -EINVAL;
221 }
222
223 frags[i].page = page;
224 frags[i].page_offset = sg->offset + sgoffset;
225 frags[i].size = copy;
226 i++;
227 }
228 datalen -= copy;
229 sgoffset += copy;
230 sglen -= copy;
231 } while (datalen);
232
233 return i;
234 }
235
cxgb3i_conn_alloc_pdu(struct iscsi_task * task,u8 opcode)236 int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
237 {
238 struct iscsi_conn *conn = task->conn;
239 struct iscsi_tcp_task *tcp_task = task->dd_data;
240 struct cxgb3i_task_data *tdata = task->dd_data + sizeof(*tcp_task);
241 struct scsi_cmnd *sc = task->sc;
242 int headroom = SKB_TX_PDU_HEADER_LEN;
243
244 tcp_task->dd_data = tdata;
245 task->hdr = NULL;
246
247 /* write command, need to send data pdus */
248 if (skb_extra_headroom && (opcode == ISCSI_OP_SCSI_DATA_OUT ||
249 (opcode == ISCSI_OP_SCSI_CMD &&
250 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE))))
251 headroom += min(skb_extra_headroom, conn->max_xmit_dlength);
252
253 tdata->skb = alloc_skb(TX_HEADER_LEN + headroom, GFP_ATOMIC);
254 if (!tdata->skb)
255 return -ENOMEM;
256 skb_reserve(tdata->skb, TX_HEADER_LEN);
257
258 cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n",
259 task, opcode, tdata->skb);
260
261 task->hdr = (struct iscsi_hdr *)tdata->skb->data;
262 task->hdr_max = SKB_TX_PDU_HEADER_LEN;
263
264 /* data_out uses scsi_cmd's itt */
265 if (opcode != ISCSI_OP_SCSI_DATA_OUT)
266 cxgb3i_reserve_itt(task, &task->hdr->itt);
267
268 return 0;
269 }
270
cxgb3i_conn_init_pdu(struct iscsi_task * task,unsigned int offset,unsigned int count)271 int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
272 unsigned int count)
273 {
274 struct iscsi_conn *conn = task->conn;
275 struct iscsi_tcp_task *tcp_task = task->dd_data;
276 struct cxgb3i_task_data *tdata = tcp_task->dd_data;
277 struct sk_buff *skb = tdata->skb;
278 unsigned int datalen = count;
279 int i, padlen = iscsi_padding(count);
280 struct page *pg;
281
282 cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
283 task, task->sc, offset, count, skb);
284
285 skb_put(skb, task->hdr_len);
286 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
287 if (!count)
288 return 0;
289
290 if (task->sc) {
291 struct scsi_data_buffer *sdb = scsi_out(task->sc);
292 struct scatterlist *sg = NULL;
293 int err;
294
295 tdata->offset = offset;
296 tdata->count = count;
297 err = sgl_seek_offset(sdb->table.sgl, sdb->table.nents,
298 tdata->offset, &tdata->sgoffset, &sg);
299 if (err < 0) {
300 cxgb3i_log_warn("tpdu, sgl %u, bad offset %u/%u.\n",
301 sdb->table.nents, tdata->offset,
302 sdb->length);
303 return err;
304 }
305 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
306 tdata->frags, MAX_PDU_FRAGS);
307 if (err < 0) {
308 cxgb3i_log_warn("tpdu, sgl %u, bad offset %u + %u.\n",
309 sdb->table.nents, tdata->offset,
310 tdata->count);
311 return err;
312 }
313 tdata->nr_frags = err;
314
315 if (tdata->nr_frags > MAX_SKB_FRAGS ||
316 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
317 char *dst = skb->data + task->hdr_len;
318 skb_frag_t *frag = tdata->frags;
319
320 /* data fits in the skb's headroom */
321 for (i = 0; i < tdata->nr_frags; i++, frag++) {
322 char *src = kmap_atomic(frag->page,
323 KM_SOFTIRQ0);
324
325 memcpy(dst, src+frag->page_offset, frag->size);
326 dst += frag->size;
327 kunmap_atomic(src, KM_SOFTIRQ0);
328 }
329 if (padlen) {
330 memset(dst, 0, padlen);
331 padlen = 0;
332 }
333 skb_put(skb, count + padlen);
334 } else {
335 /* data fit into frag_list */
336 for (i = 0; i < tdata->nr_frags; i++)
337 get_page(tdata->frags[i].page);
338
339 memcpy(skb_shinfo(skb)->frags, tdata->frags,
340 sizeof(skb_frag_t) * tdata->nr_frags);
341 skb_shinfo(skb)->nr_frags = tdata->nr_frags;
342 skb->len += count;
343 skb->data_len += count;
344 skb->truesize += count;
345 }
346
347 } else {
348 pg = virt_to_page(task->data);
349
350 get_page(pg);
351 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
352 count);
353 skb->len += count;
354 skb->data_len += count;
355 skb->truesize += count;
356 }
357
358 if (padlen) {
359 i = skb_shinfo(skb)->nr_frags;
360 get_page(pad_page);
361 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, pad_page, 0,
362 padlen);
363
364 skb->data_len += padlen;
365 skb->truesize += padlen;
366 skb->len += padlen;
367 }
368
369 return 0;
370 }
371
cxgb3i_conn_xmit_pdu(struct iscsi_task * task)372 int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
373 {
374 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
375 struct cxgb3i_conn *cconn = tcp_conn->dd_data;
376 struct iscsi_tcp_task *tcp_task = task->dd_data;
377 struct cxgb3i_task_data *tdata = tcp_task->dd_data;
378 struct sk_buff *skb = tdata->skb;
379 unsigned int datalen;
380 int err;
381
382 if (!skb)
383 return 0;
384
385 datalen = skb->data_len;
386 tdata->skb = NULL;
387 err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb);
388 if (err > 0) {
389 int pdulen = err;
390
391 cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
392 task, skb, skb->len, skb->data_len, err);
393
394 if (task->conn->hdrdgst_en)
395 pdulen += ISCSI_DIGEST_SIZE;
396 if (datalen && task->conn->datadgst_en)
397 pdulen += ISCSI_DIGEST_SIZE;
398
399 task->conn->txdata_octets += pdulen;
400 return 0;
401 }
402
403 if (err < 0 && err != -EAGAIN) {
404 kfree_skb(skb);
405 cxgb3i_tx_debug("itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
406 task->itt, skb, skb->len, skb->data_len, err);
407 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
408 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
409 return err;
410 }
411 /* reset skb to send when we are called again */
412 tdata->skb = skb;
413 return -EAGAIN;
414 }
415
cxgb3i_pdu_init(void)416 int cxgb3i_pdu_init(void)
417 {
418 if (SKB_TX_HEADROOM > (512 * MAX_SKB_FRAGS))
419 skb_extra_headroom = SKB_TX_HEADROOM;
420 pad_page = alloc_page(GFP_KERNEL);
421 if (!pad_page)
422 return -ENOMEM;
423 memset(page_address(pad_page), 0, PAGE_SIZE);
424 return 0;
425 }
426
cxgb3i_pdu_cleanup(void)427 void cxgb3i_pdu_cleanup(void)
428 {
429 if (pad_page) {
430 __free_page(pad_page);
431 pad_page = NULL;
432 }
433 }
434
cxgb3i_conn_pdu_ready(struct s3_conn * c3cn)435 void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
436 {
437 struct sk_buff *skb;
438 unsigned int read = 0;
439 struct iscsi_conn *conn = c3cn->user_data;
440 int err = 0;
441
442 cxgb3i_rx_debug("cn 0x%p.\n", c3cn);
443
444 read_lock(&c3cn->callback_lock);
445 if (unlikely(!conn || conn->suspend_rx)) {
446 cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
447 conn, conn ? conn->id : 0xFF,
448 conn ? conn->suspend_rx : 0xFF);
449 read_unlock(&c3cn->callback_lock);
450 return;
451 }
452 skb = skb_peek(&c3cn->receive_queue);
453 while (!err && skb) {
454 __skb_unlink(skb, &c3cn->receive_queue);
455 read += skb_rx_pdulen(skb);
456 cxgb3i_rx_debug("conn 0x%p, cn 0x%p, rx skb 0x%p, pdulen %u.\n",
457 conn, c3cn, skb, skb_rx_pdulen(skb));
458 err = cxgb3i_conn_read_pdu_skb(conn, skb);
459 __kfree_skb(skb);
460 skb = skb_peek(&c3cn->receive_queue);
461 }
462 read_unlock(&c3cn->callback_lock);
463 if (c3cn) {
464 c3cn->copied_seq += read;
465 cxgb3i_c3cn_rx_credits(c3cn, read);
466 }
467 conn->rxdata_octets += read;
468
469 if (err) {
470 cxgb3i_log_info("conn 0x%p rx failed err %d.\n", conn, err);
471 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
472 }
473 }
474
cxgb3i_conn_tx_open(struct s3_conn * c3cn)475 void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
476 {
477 struct iscsi_conn *conn = c3cn->user_data;
478
479 cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
480 if (conn) {
481 cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
482 scsi_queue_work(conn->session->host, &conn->xmitwork);
483 }
484 }
485
cxgb3i_conn_closing(struct s3_conn * c3cn)486 void cxgb3i_conn_closing(struct s3_conn *c3cn)
487 {
488 struct iscsi_conn *conn;
489
490 read_lock(&c3cn->callback_lock);
491 conn = c3cn->user_data;
492 if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
493 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
494 read_unlock(&c3cn->callback_lock);
495 }
496