• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Intel I/OAT DMA Linux driver
3  * Copyright(c) 2004 - 2015 Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * The full GNU General Public License is included in this distribution in
15  * the file called "COPYING".
16  *
17  */
18 #include <linux/module.h>
19 #include <linux/pci.h>
20 #include <linux/gfp.h>
21 #include <linux/dmaengine.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/prefetch.h>
24 #include "../dmaengine.h"
25 #include "registers.h"
26 #include "hw.h"
27 #include "dma.h"
28 
29 #define MAX_SCF	1024
30 
31 /* provide a lookup table for setting the source address in the base or
32  * extended descriptor of an xor or pq descriptor
33  */
34 static const u8 xor_idx_to_desc = 0xe0;
35 static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
36 static const u8 pq_idx_to_desc = 0xf8;
37 static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1,
38 				       2, 2, 2, 2, 2, 2, 2 };
39 static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
40 static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
41 					0, 1, 2, 3, 4, 5, 6 };
42 
xor_set_src(struct ioat_raw_descriptor * descs[2],dma_addr_t addr,u32 offset,int idx)43 static void xor_set_src(struct ioat_raw_descriptor *descs[2],
44 			dma_addr_t addr, u32 offset, int idx)
45 {
46 	struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
47 
48 	raw->field[xor_idx_to_field[idx]] = addr + offset;
49 }
50 
pq_get_src(struct ioat_raw_descriptor * descs[2],int idx)51 static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
52 {
53 	struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
54 
55 	return raw->field[pq_idx_to_field[idx]];
56 }
57 
pq16_get_src(struct ioat_raw_descriptor * desc[3],int idx)58 static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx)
59 {
60 	struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
61 
62 	return raw->field[pq16_idx_to_field[idx]];
63 }
64 
pq_set_src(struct ioat_raw_descriptor * descs[2],dma_addr_t addr,u32 offset,u8 coef,int idx)65 static void pq_set_src(struct ioat_raw_descriptor *descs[2],
66 		       dma_addr_t addr, u32 offset, u8 coef, int idx)
67 {
68 	struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
69 	struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
70 
71 	raw->field[pq_idx_to_field[idx]] = addr + offset;
72 	pq->coef[idx] = coef;
73 }
74 
pq16_set_src(struct ioat_raw_descriptor * desc[3],dma_addr_t addr,u32 offset,u8 coef,unsigned idx)75 static void pq16_set_src(struct ioat_raw_descriptor *desc[3],
76 			dma_addr_t addr, u32 offset, u8 coef, unsigned idx)
77 {
78 	struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0];
79 	struct ioat_pq16a_descriptor *pq16 =
80 		(struct ioat_pq16a_descriptor *)desc[1];
81 	struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
82 
83 	raw->field[pq16_idx_to_field[idx]] = addr + offset;
84 
85 	if (idx < 8)
86 		pq->coef[idx] = coef;
87 	else
88 		pq16->coef[idx - 8] = coef;
89 }
90 
91 static struct ioat_sed_ent *
ioat3_alloc_sed(struct ioatdma_device * ioat_dma,unsigned int hw_pool)92 ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool)
93 {
94 	struct ioat_sed_ent *sed;
95 	gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
96 
97 	sed = kmem_cache_alloc(ioat_sed_cache, flags);
98 	if (!sed)
99 		return NULL;
100 
101 	sed->hw_pool = hw_pool;
102 	sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool],
103 				 flags, &sed->dma);
104 	if (!sed->hw) {
105 		kmem_cache_free(ioat_sed_cache, sed);
106 		return NULL;
107 	}
108 
109 	return sed;
110 }
111 
112 struct dma_async_tx_descriptor *
ioat_dma_prep_memcpy_lock(struct dma_chan * c,dma_addr_t dma_dest,dma_addr_t dma_src,size_t len,unsigned long flags)113 ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
114 			   dma_addr_t dma_src, size_t len, unsigned long flags)
115 {
116 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
117 	struct ioat_dma_descriptor *hw;
118 	struct ioat_ring_ent *desc;
119 	dma_addr_t dst = dma_dest;
120 	dma_addr_t src = dma_src;
121 	size_t total_len = len;
122 	int num_descs, idx, i;
123 
124 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
125 		return NULL;
126 
127 	num_descs = ioat_xferlen_to_descs(ioat_chan, len);
128 	if (likely(num_descs) &&
129 	    ioat_check_space_lock(ioat_chan, num_descs) == 0)
130 		idx = ioat_chan->head;
131 	else
132 		return NULL;
133 	i = 0;
134 	do {
135 		size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
136 
137 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
138 		hw = desc->hw;
139 
140 		hw->size = copy;
141 		hw->ctl = 0;
142 		hw->src_addr = src;
143 		hw->dst_addr = dst;
144 
145 		len -= copy;
146 		dst += copy;
147 		src += copy;
148 		dump_desc_dbg(ioat_chan, desc);
149 	} while (++i < num_descs);
150 
151 	desc->txd.flags = flags;
152 	desc->len = total_len;
153 	hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
154 	hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
155 	hw->ctl_f.compl_write = 1;
156 	dump_desc_dbg(ioat_chan, desc);
157 	/* we leave the channel locked to ensure in order submission */
158 
159 	return &desc->txd;
160 }
161 
162 
163 static struct dma_async_tx_descriptor *
__ioat_prep_xor_lock(struct dma_chan * c,enum sum_check_flags * result,dma_addr_t dest,dma_addr_t * src,unsigned int src_cnt,size_t len,unsigned long flags)164 __ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
165 		      dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
166 		      size_t len, unsigned long flags)
167 {
168 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
169 	struct ioat_ring_ent *compl_desc;
170 	struct ioat_ring_ent *desc;
171 	struct ioat_ring_ent *ext;
172 	size_t total_len = len;
173 	struct ioat_xor_descriptor *xor;
174 	struct ioat_xor_ext_descriptor *xor_ex = NULL;
175 	struct ioat_dma_descriptor *hw;
176 	int num_descs, with_ext, idx, i;
177 	u32 offset = 0;
178 	u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
179 
180 	BUG_ON(src_cnt < 2);
181 
182 	num_descs = ioat_xferlen_to_descs(ioat_chan, len);
183 	/* we need 2x the number of descriptors to cover greater than 5
184 	 * sources
185 	 */
186 	if (src_cnt > 5) {
187 		with_ext = 1;
188 		num_descs *= 2;
189 	} else
190 		with_ext = 0;
191 
192 	/* completion writes from the raid engine may pass completion
193 	 * writes from the legacy engine, so we need one extra null
194 	 * (legacy) descriptor to ensure all completion writes arrive in
195 	 * order.
196 	 */
197 	if (likely(num_descs) &&
198 	    ioat_check_space_lock(ioat_chan, num_descs+1) == 0)
199 		idx = ioat_chan->head;
200 	else
201 		return NULL;
202 	i = 0;
203 	do {
204 		struct ioat_raw_descriptor *descs[2];
205 		size_t xfer_size = min_t(size_t,
206 					 len, 1 << ioat_chan->xfercap_log);
207 		int s;
208 
209 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
210 		xor = desc->xor;
211 
212 		/* save a branch by unconditionally retrieving the
213 		 * extended descriptor xor_set_src() knows to not write
214 		 * to it in the single descriptor case
215 		 */
216 		ext = ioat_get_ring_ent(ioat_chan, idx + i + 1);
217 		xor_ex = ext->xor_ex;
218 
219 		descs[0] = (struct ioat_raw_descriptor *) xor;
220 		descs[1] = (struct ioat_raw_descriptor *) xor_ex;
221 		for (s = 0; s < src_cnt; s++)
222 			xor_set_src(descs, src[s], offset, s);
223 		xor->size = xfer_size;
224 		xor->dst_addr = dest + offset;
225 		xor->ctl = 0;
226 		xor->ctl_f.op = op;
227 		xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
228 
229 		len -= xfer_size;
230 		offset += xfer_size;
231 		dump_desc_dbg(ioat_chan, desc);
232 	} while ((i += 1 + with_ext) < num_descs);
233 
234 	/* last xor descriptor carries the unmap parameters and fence bit */
235 	desc->txd.flags = flags;
236 	desc->len = total_len;
237 	if (result)
238 		desc->result = result;
239 	xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
240 
241 	/* completion descriptor carries interrupt bit */
242 	compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
243 	compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
244 	hw = compl_desc->hw;
245 	hw->ctl = 0;
246 	hw->ctl_f.null = 1;
247 	hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
248 	hw->ctl_f.compl_write = 1;
249 	hw->size = NULL_DESC_BUFFER_SIZE;
250 	dump_desc_dbg(ioat_chan, compl_desc);
251 
252 	/* we leave the channel locked to ensure in order submission */
253 	return &compl_desc->txd;
254 }
255 
256 struct dma_async_tx_descriptor *
ioat_prep_xor(struct dma_chan * chan,dma_addr_t dest,dma_addr_t * src,unsigned int src_cnt,size_t len,unsigned long flags)257 ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
258 	       unsigned int src_cnt, size_t len, unsigned long flags)
259 {
260 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
261 
262 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
263 		return NULL;
264 
265 	return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
266 }
267 
268 struct dma_async_tx_descriptor *
ioat_prep_xor_val(struct dma_chan * chan,dma_addr_t * src,unsigned int src_cnt,size_t len,enum sum_check_flags * result,unsigned long flags)269 ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
270 		    unsigned int src_cnt, size_t len,
271 		    enum sum_check_flags *result, unsigned long flags)
272 {
273 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
274 
275 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
276 		return NULL;
277 
278 	/* the cleanup routine only sets bits on validate failure, it
279 	 * does not clear bits on validate success... so clear it here
280 	 */
281 	*result = 0;
282 
283 	return __ioat_prep_xor_lock(chan, result, src[0], &src[1],
284 				     src_cnt - 1, len, flags);
285 }
286 
287 static void
dump_pq_desc_dbg(struct ioatdma_chan * ioat_chan,struct ioat_ring_ent * desc,struct ioat_ring_ent * ext)288 dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc,
289 		 struct ioat_ring_ent *ext)
290 {
291 	struct device *dev = to_dev(ioat_chan);
292 	struct ioat_pq_descriptor *pq = desc->pq;
293 	struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
294 	struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
295 	int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
296 	int i;
297 
298 	dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
299 		" sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
300 		" src_cnt: %d)\n",
301 		desc_id(desc), (unsigned long long) desc->txd.phys,
302 		(unsigned long long) (pq_ex ? pq_ex->next : pq->next),
303 		desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op,
304 		pq->ctl_f.int_en, pq->ctl_f.compl_write,
305 		pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
306 		pq->ctl_f.src_cnt);
307 	for (i = 0; i < src_cnt; i++)
308 		dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
309 			(unsigned long long) pq_get_src(descs, i), pq->coef[i]);
310 	dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
311 	dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
312 	dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);
313 }
314 
dump_pq16_desc_dbg(struct ioatdma_chan * ioat_chan,struct ioat_ring_ent * desc)315 static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan,
316 			       struct ioat_ring_ent *desc)
317 {
318 	struct device *dev = to_dev(ioat_chan);
319 	struct ioat_pq_descriptor *pq = desc->pq;
320 	struct ioat_raw_descriptor *descs[] = { (void *)pq,
321 						(void *)pq,
322 						(void *)pq };
323 	int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
324 	int i;
325 
326 	if (desc->sed) {
327 		descs[1] = (void *)desc->sed->hw;
328 		descs[2] = (void *)desc->sed->hw + 64;
329 	}
330 
331 	dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
332 		" sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
333 		" src_cnt: %d)\n",
334 		desc_id(desc), (unsigned long long) desc->txd.phys,
335 		(unsigned long long) pq->next,
336 		desc->txd.flags, pq->size, pq->ctl,
337 		pq->ctl_f.op, pq->ctl_f.int_en,
338 		pq->ctl_f.compl_write,
339 		pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
340 		pq->ctl_f.src_cnt);
341 	for (i = 0; i < src_cnt; i++) {
342 		dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
343 			(unsigned long long) pq16_get_src(descs, i),
344 			pq->coef[i]);
345 	}
346 	dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
347 	dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
348 }
349 
350 static struct dma_async_tx_descriptor *
__ioat_prep_pq_lock(struct dma_chan * c,enum sum_check_flags * result,const dma_addr_t * dst,const dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)351 __ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
352 		     const dma_addr_t *dst, const dma_addr_t *src,
353 		     unsigned int src_cnt, const unsigned char *scf,
354 		     size_t len, unsigned long flags)
355 {
356 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
357 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
358 	struct ioat_ring_ent *compl_desc;
359 	struct ioat_ring_ent *desc;
360 	struct ioat_ring_ent *ext;
361 	size_t total_len = len;
362 	struct ioat_pq_descriptor *pq;
363 	struct ioat_pq_ext_descriptor *pq_ex = NULL;
364 	struct ioat_dma_descriptor *hw;
365 	u32 offset = 0;
366 	u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
367 	int i, s, idx, with_ext, num_descs;
368 	int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0;
369 
370 	dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
371 	/* the engine requires at least two sources (we provide
372 	 * at least 1 implied source in the DMA_PREP_CONTINUE case)
373 	 */
374 	BUG_ON(src_cnt + dmaf_continue(flags) < 2);
375 
376 	num_descs = ioat_xferlen_to_descs(ioat_chan, len);
377 	/* we need 2x the number of descriptors to cover greater than 3
378 	 * sources (we need 1 extra source in the q-only continuation
379 	 * case and 3 extra sources in the p+q continuation case.
380 	 */
381 	if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
382 	    (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
383 		with_ext = 1;
384 		num_descs *= 2;
385 	} else
386 		with_ext = 0;
387 
388 	/* completion writes from the raid engine may pass completion
389 	 * writes from the legacy engine, so we need one extra null
390 	 * (legacy) descriptor to ensure all completion writes arrive in
391 	 * order.
392 	 */
393 	if (likely(num_descs) &&
394 	    ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0)
395 		idx = ioat_chan->head;
396 	else
397 		return NULL;
398 	i = 0;
399 	do {
400 		struct ioat_raw_descriptor *descs[2];
401 		size_t xfer_size = min_t(size_t, len,
402 					 1 << ioat_chan->xfercap_log);
403 
404 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
405 		pq = desc->pq;
406 
407 		/* save a branch by unconditionally retrieving the
408 		 * extended descriptor pq_set_src() knows to not write
409 		 * to it in the single descriptor case
410 		 */
411 		ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext);
412 		pq_ex = ext->pq_ex;
413 
414 		descs[0] = (struct ioat_raw_descriptor *) pq;
415 		descs[1] = (struct ioat_raw_descriptor *) pq_ex;
416 
417 		for (s = 0; s < src_cnt; s++)
418 			pq_set_src(descs, src[s], offset, scf[s], s);
419 
420 		/* see the comment for dma_maxpq in include/linux/dmaengine.h */
421 		if (dmaf_p_disabled_continue(flags))
422 			pq_set_src(descs, dst[1], offset, 1, s++);
423 		else if (dmaf_continue(flags)) {
424 			pq_set_src(descs, dst[0], offset, 0, s++);
425 			pq_set_src(descs, dst[1], offset, 1, s++);
426 			pq_set_src(descs, dst[1], offset, 0, s++);
427 		}
428 		pq->size = xfer_size;
429 		pq->p_addr = dst[0] + offset;
430 		pq->q_addr = dst[1] + offset;
431 		pq->ctl = 0;
432 		pq->ctl_f.op = op;
433 		/* we turn on descriptor write back error status */
434 		if (ioat_dma->cap & IOAT_CAP_DWBES)
435 			pq->ctl_f.wb_en = result ? 1 : 0;
436 		pq->ctl_f.src_cnt = src_cnt_to_hw(s);
437 		pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
438 		pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
439 
440 		len -= xfer_size;
441 		offset += xfer_size;
442 	} while ((i += 1 + with_ext) < num_descs);
443 
444 	/* last pq descriptor carries the unmap parameters and fence bit */
445 	desc->txd.flags = flags;
446 	desc->len = total_len;
447 	if (result)
448 		desc->result = result;
449 	pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
450 	dump_pq_desc_dbg(ioat_chan, desc, ext);
451 
452 	if (!cb32) {
453 		pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
454 		pq->ctl_f.compl_write = 1;
455 		compl_desc = desc;
456 	} else {
457 		/* completion descriptor carries interrupt bit */
458 		compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
459 		compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
460 		hw = compl_desc->hw;
461 		hw->ctl = 0;
462 		hw->ctl_f.null = 1;
463 		hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
464 		hw->ctl_f.compl_write = 1;
465 		hw->size = NULL_DESC_BUFFER_SIZE;
466 		dump_desc_dbg(ioat_chan, compl_desc);
467 	}
468 
469 
470 	/* we leave the channel locked to ensure in order submission */
471 	return &compl_desc->txd;
472 }
473 
474 static struct dma_async_tx_descriptor *
__ioat_prep_pq16_lock(struct dma_chan * c,enum sum_check_flags * result,const dma_addr_t * dst,const dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)475 __ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
476 		       const dma_addr_t *dst, const dma_addr_t *src,
477 		       unsigned int src_cnt, const unsigned char *scf,
478 		       size_t len, unsigned long flags)
479 {
480 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
481 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
482 	struct ioat_ring_ent *desc;
483 	size_t total_len = len;
484 	struct ioat_pq_descriptor *pq;
485 	u32 offset = 0;
486 	u8 op;
487 	int i, s, idx, num_descs;
488 
489 	/* this function is only called with 9-16 sources */
490 	op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
491 
492 	dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
493 
494 	num_descs = ioat_xferlen_to_descs(ioat_chan, len);
495 
496 	/*
497 	 * 16 source pq is only available on cb3.3 and has no completion
498 	 * write hw bug.
499 	 */
500 	if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0)
501 		idx = ioat_chan->head;
502 	else
503 		return NULL;
504 
505 	i = 0;
506 
507 	do {
508 		struct ioat_raw_descriptor *descs[4];
509 		size_t xfer_size = min_t(size_t, len,
510 					 1 << ioat_chan->xfercap_log);
511 
512 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
513 		pq = desc->pq;
514 
515 		descs[0] = (struct ioat_raw_descriptor *) pq;
516 
517 		desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3);
518 		if (!desc->sed) {
519 			dev_err(to_dev(ioat_chan),
520 				"%s: no free sed entries\n", __func__);
521 			return NULL;
522 		}
523 
524 		pq->sed_addr = desc->sed->dma;
525 		desc->sed->parent = desc;
526 
527 		descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;
528 		descs[2] = (void *)descs[1] + 64;
529 
530 		for (s = 0; s < src_cnt; s++)
531 			pq16_set_src(descs, src[s], offset, scf[s], s);
532 
533 		/* see the comment for dma_maxpq in include/linux/dmaengine.h */
534 		if (dmaf_p_disabled_continue(flags))
535 			pq16_set_src(descs, dst[1], offset, 1, s++);
536 		else if (dmaf_continue(flags)) {
537 			pq16_set_src(descs, dst[0], offset, 0, s++);
538 			pq16_set_src(descs, dst[1], offset, 1, s++);
539 			pq16_set_src(descs, dst[1], offset, 0, s++);
540 		}
541 
542 		pq->size = xfer_size;
543 		pq->p_addr = dst[0] + offset;
544 		pq->q_addr = dst[1] + offset;
545 		pq->ctl = 0;
546 		pq->ctl_f.op = op;
547 		pq->ctl_f.src_cnt = src16_cnt_to_hw(s);
548 		/* we turn on descriptor write back error status */
549 		if (ioat_dma->cap & IOAT_CAP_DWBES)
550 			pq->ctl_f.wb_en = result ? 1 : 0;
551 		pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
552 		pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
553 
554 		len -= xfer_size;
555 		offset += xfer_size;
556 	} while (++i < num_descs);
557 
558 	/* last pq descriptor carries the unmap parameters and fence bit */
559 	desc->txd.flags = flags;
560 	desc->len = total_len;
561 	if (result)
562 		desc->result = result;
563 	pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
564 
565 	/* with cb3.3 we should be able to do completion w/o a null desc */
566 	pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
567 	pq->ctl_f.compl_write = 1;
568 
569 	dump_pq16_desc_dbg(ioat_chan, desc);
570 
571 	/* we leave the channel locked to ensure in order submission */
572 	return &desc->txd;
573 }
574 
src_cnt_flags(unsigned int src_cnt,unsigned long flags)575 static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
576 {
577 	if (dmaf_p_disabled_continue(flags))
578 		return src_cnt + 1;
579 	else if (dmaf_continue(flags))
580 		return src_cnt + 3;
581 	else
582 		return src_cnt;
583 }
584 
585 struct dma_async_tx_descriptor *
ioat_prep_pq(struct dma_chan * chan,dma_addr_t * dst,dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)586 ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
587 	      unsigned int src_cnt, const unsigned char *scf, size_t len,
588 	      unsigned long flags)
589 {
590 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
591 
592 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
593 		return NULL;
594 
595 	/* specify valid address for disabled result */
596 	if (flags & DMA_PREP_PQ_DISABLE_P)
597 		dst[0] = dst[1];
598 	if (flags & DMA_PREP_PQ_DISABLE_Q)
599 		dst[1] = dst[0];
600 
601 	/* handle the single source multiply case from the raid6
602 	 * recovery path
603 	 */
604 	if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
605 		dma_addr_t single_source[2];
606 		unsigned char single_source_coef[2];
607 
608 		BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
609 		single_source[0] = src[0];
610 		single_source[1] = src[0];
611 		single_source_coef[0] = scf[0];
612 		single_source_coef[1] = 0;
613 
614 		return src_cnt_flags(src_cnt, flags) > 8 ?
615 			__ioat_prep_pq16_lock(chan, NULL, dst, single_source,
616 					       2, single_source_coef, len,
617 					       flags) :
618 			__ioat_prep_pq_lock(chan, NULL, dst, single_source, 2,
619 					     single_source_coef, len, flags);
620 
621 	} else {
622 		return src_cnt_flags(src_cnt, flags) > 8 ?
623 			__ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
624 					       scf, len, flags) :
625 			__ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt,
626 					     scf, len, flags);
627 	}
628 }
629 
630 struct dma_async_tx_descriptor *
ioat_prep_pq_val(struct dma_chan * chan,dma_addr_t * pq,dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,enum sum_check_flags * pqres,unsigned long flags)631 ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
632 		  unsigned int src_cnt, const unsigned char *scf, size_t len,
633 		  enum sum_check_flags *pqres, unsigned long flags)
634 {
635 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
636 
637 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
638 		return NULL;
639 
640 	/* specify valid address for disabled result */
641 	if (flags & DMA_PREP_PQ_DISABLE_P)
642 		pq[0] = pq[1];
643 	if (flags & DMA_PREP_PQ_DISABLE_Q)
644 		pq[1] = pq[0];
645 
646 	/* the cleanup routine only sets bits on validate failure, it
647 	 * does not clear bits on validate success... so clear it here
648 	 */
649 	*pqres = 0;
650 
651 	return src_cnt_flags(src_cnt, flags) > 8 ?
652 		__ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
653 				       flags) :
654 		__ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
655 				     flags);
656 }
657 
658 struct dma_async_tx_descriptor *
ioat_prep_pqxor(struct dma_chan * chan,dma_addr_t dst,dma_addr_t * src,unsigned int src_cnt,size_t len,unsigned long flags)659 ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
660 		 unsigned int src_cnt, size_t len, unsigned long flags)
661 {
662 	unsigned char scf[MAX_SCF];
663 	dma_addr_t pq[2];
664 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
665 
666 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
667 		return NULL;
668 
669 	if (src_cnt > MAX_SCF)
670 		return NULL;
671 
672 	memset(scf, 0, src_cnt);
673 	pq[0] = dst;
674 	flags |= DMA_PREP_PQ_DISABLE_Q;
675 	pq[1] = dst; /* specify valid address for disabled result */
676 
677 	return src_cnt_flags(src_cnt, flags) > 8 ?
678 		__ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
679 				       flags) :
680 		__ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
681 				     flags);
682 }
683 
684 struct dma_async_tx_descriptor *
ioat_prep_pqxor_val(struct dma_chan * chan,dma_addr_t * src,unsigned int src_cnt,size_t len,enum sum_check_flags * result,unsigned long flags)685 ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
686 		     unsigned int src_cnt, size_t len,
687 		     enum sum_check_flags *result, unsigned long flags)
688 {
689 	unsigned char scf[MAX_SCF];
690 	dma_addr_t pq[2];
691 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
692 
693 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
694 		return NULL;
695 
696 	if (src_cnt > MAX_SCF)
697 		return NULL;
698 
699 	/* the cleanup routine only sets bits on validate failure, it
700 	 * does not clear bits on validate success... so clear it here
701 	 */
702 	*result = 0;
703 
704 	memset(scf, 0, src_cnt);
705 	pq[0] = src[0];
706 	flags |= DMA_PREP_PQ_DISABLE_Q;
707 	pq[1] = pq[0]; /* specify valid address for disabled result */
708 
709 	return src_cnt_flags(src_cnt, flags) > 8 ?
710 		__ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
711 				       scf, len, flags) :
712 		__ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
713 				     scf, len, flags);
714 }
715 
716 struct dma_async_tx_descriptor *
ioat_prep_interrupt_lock(struct dma_chan * c,unsigned long flags)717 ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
718 {
719 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
720 	struct ioat_ring_ent *desc;
721 	struct ioat_dma_descriptor *hw;
722 
723 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
724 		return NULL;
725 
726 	if (ioat_check_space_lock(ioat_chan, 1) == 0)
727 		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
728 	else
729 		return NULL;
730 
731 	hw = desc->hw;
732 	hw->ctl = 0;
733 	hw->ctl_f.null = 1;
734 	hw->ctl_f.int_en = 1;
735 	hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
736 	hw->ctl_f.compl_write = 1;
737 	hw->size = NULL_DESC_BUFFER_SIZE;
738 	hw->src_addr = 0;
739 	hw->dst_addr = 0;
740 
741 	desc->txd.flags = flags;
742 	desc->len = 1;
743 
744 	dump_desc_dbg(ioat_chan, desc);
745 
746 	/* we leave the channel locked to ensure in order submission */
747 	return &desc->txd;
748 }
749 
750