• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**
2  * IBM Accelerator Family 'GenWQE'
3  *
4  * (C) Copyright IBM Corp. 2013
5  *
6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8  * Author: Michael Jung <mijung@gmx.net>
9  * Author: Michael Ruettger <michael@ibmra.de>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License (version 2 only)
13  * as published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  */
20 
21 /*
22  * Miscelanous functionality used in the other GenWQE driver parts.
23  */
24 
25 #include <linux/kernel.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/sched.h>
28 #include <linux/vmalloc.h>
29 #include <linux/page-flags.h>
30 #include <linux/scatterlist.h>
31 #include <linux/hugetlb.h>
32 #include <linux/iommu.h>
33 #include <linux/delay.h>
34 #include <linux/pci.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/ctype.h>
37 #include <linux/module.h>
38 #include <linux/platform_device.h>
39 #include <linux/delay.h>
40 #include <asm/pgtable.h>
41 
42 #include "genwqe_driver.h"
43 #include "card_base.h"
44 #include "card_ddcb.h"
45 
46 /**
47  * __genwqe_writeq() - Write 64-bit register
48  * @cd:	        genwqe device descriptor
49  * @byte_offs:  byte offset within BAR
50  * @val:        64-bit value
51  *
52  * Return: 0 if success; < 0 if error
53  */
__genwqe_writeq(struct genwqe_dev * cd,u64 byte_offs,u64 val)54 int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val)
55 {
56 	struct pci_dev *pci_dev = cd->pci_dev;
57 
58 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
59 		return -EIO;
60 
61 	if (cd->mmio == NULL)
62 		return -EIO;
63 
64 	if (pci_channel_offline(pci_dev))
65 		return -EIO;
66 
67 	__raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs);
68 	return 0;
69 }
70 
71 /**
72  * __genwqe_readq() - Read 64-bit register
73  * @cd:         genwqe device descriptor
74  * @byte_offs:  offset within BAR
75  *
76  * Return: value from register
77  */
__genwqe_readq(struct genwqe_dev * cd,u64 byte_offs)78 u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs)
79 {
80 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
81 		return 0xffffffffffffffffull;
82 
83 	if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) &&
84 	    (byte_offs == IO_SLC_CFGREG_GFIR))
85 		return 0x000000000000ffffull;
86 
87 	if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) &&
88 	    (byte_offs == IO_SLC_CFGREG_GFIR))
89 		return 0x00000000ffff0000ull;
90 
91 	if (cd->mmio == NULL)
92 		return 0xffffffffffffffffull;
93 
94 	return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs));
95 }
96 
97 /**
98  * __genwqe_writel() - Write 32-bit register
99  * @cd:	        genwqe device descriptor
100  * @byte_offs:  byte offset within BAR
101  * @val:        32-bit value
102  *
103  * Return: 0 if success; < 0 if error
104  */
__genwqe_writel(struct genwqe_dev * cd,u64 byte_offs,u32 val)105 int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val)
106 {
107 	struct pci_dev *pci_dev = cd->pci_dev;
108 
109 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
110 		return -EIO;
111 
112 	if (cd->mmio == NULL)
113 		return -EIO;
114 
115 	if (pci_channel_offline(pci_dev))
116 		return -EIO;
117 
118 	__raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs);
119 	return 0;
120 }
121 
122 /**
123  * __genwqe_readl() - Read 32-bit register
124  * @cd:         genwqe device descriptor
125  * @byte_offs:  offset within BAR
126  *
127  * Return: Value from register
128  */
__genwqe_readl(struct genwqe_dev * cd,u64 byte_offs)129 u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs)
130 {
131 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
132 		return 0xffffffff;
133 
134 	if (cd->mmio == NULL)
135 		return 0xffffffff;
136 
137 	return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs));
138 }
139 
140 /**
141  * genwqe_read_app_id() - Extract app_id
142  *
143  * app_unitcfg need to be filled with valid data first
144  */
genwqe_read_app_id(struct genwqe_dev * cd,char * app_name,int len)145 int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len)
146 {
147 	int i, j;
148 	u32 app_id = (u32)cd->app_unitcfg;
149 
150 	memset(app_name, 0, len);
151 	for (i = 0, j = 0; j < min(len, 4); j++) {
152 		char ch = (char)((app_id >> (24 - j*8)) & 0xff);
153 
154 		if (ch == ' ')
155 			continue;
156 		app_name[i++] = isprint(ch) ? ch : 'X';
157 	}
158 	return i;
159 }
160 
161 /**
162  * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations
163  *
164  * Existing kernel functions seem to use a different polynom,
165  * therefore we could not use them here.
166  *
167  * Genwqe's Polynomial = 0x20044009
168  */
169 #define CRC32_POLYNOMIAL	0x20044009
170 static u32 crc32_tab[256];	/* crc32 lookup table */
171 
genwqe_init_crc32(void)172 void genwqe_init_crc32(void)
173 {
174 	int i, j;
175 	u32 crc;
176 
177 	for (i = 0;  i < 256;  i++) {
178 		crc = i << 24;
179 		for (j = 0;  j < 8;  j++) {
180 			if (crc & 0x80000000)
181 				crc = (crc << 1) ^ CRC32_POLYNOMIAL;
182 			else
183 				crc = (crc << 1);
184 		}
185 		crc32_tab[i] = crc;
186 	}
187 }
188 
189 /**
190  * genwqe_crc32() - Generate 32-bit crc as required for DDCBs
191  * @buff:       pointer to data buffer
192  * @len:        length of data for calculation
193  * @init:       initial crc (0xffffffff at start)
194  *
195  * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009)
196 
197  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should
198  * result in a crc32 of 0xf33cb7d3.
199  *
200  * The existing kernel crc functions did not cover this polynom yet.
201  *
202  * Return: crc32 checksum.
203  */
genwqe_crc32(u8 * buff,size_t len,u32 init)204 u32 genwqe_crc32(u8 *buff, size_t len, u32 init)
205 {
206 	int i;
207 	u32 crc;
208 
209 	crc = init;
210 	while (len--) {
211 		i = ((crc >> 24) ^ *buff++) & 0xFF;
212 		crc = (crc << 8) ^ crc32_tab[i];
213 	}
214 	return crc;
215 }
216 
__genwqe_alloc_consistent(struct genwqe_dev * cd,size_t size,dma_addr_t * dma_handle)217 void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
218 			       dma_addr_t *dma_handle)
219 {
220 	if (get_order(size) >= MAX_ORDER)
221 		return NULL;
222 
223 	return dma_alloc_coherent(&cd->pci_dev->dev, size, dma_handle,
224 				  GFP_KERNEL);
225 }
226 
__genwqe_free_consistent(struct genwqe_dev * cd,size_t size,void * vaddr,dma_addr_t dma_handle)227 void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
228 			     void *vaddr, dma_addr_t dma_handle)
229 {
230 	if (vaddr == NULL)
231 		return;
232 
233 	dma_free_coherent(&cd->pci_dev->dev, size, vaddr, dma_handle);
234 }
235 
genwqe_unmap_pages(struct genwqe_dev * cd,dma_addr_t * dma_list,int num_pages)236 static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list,
237 			      int num_pages)
238 {
239 	int i;
240 	struct pci_dev *pci_dev = cd->pci_dev;
241 
242 	for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) {
243 		pci_unmap_page(pci_dev, dma_list[i],
244 			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
245 		dma_list[i] = 0x0;
246 	}
247 }
248 
genwqe_map_pages(struct genwqe_dev * cd,struct page ** page_list,int num_pages,dma_addr_t * dma_list)249 static int genwqe_map_pages(struct genwqe_dev *cd,
250 			   struct page **page_list, int num_pages,
251 			   dma_addr_t *dma_list)
252 {
253 	int i;
254 	struct pci_dev *pci_dev = cd->pci_dev;
255 
256 	/* establish DMA mapping for requested pages */
257 	for (i = 0; i < num_pages; i++) {
258 		dma_addr_t daddr;
259 
260 		dma_list[i] = 0x0;
261 		daddr = pci_map_page(pci_dev, page_list[i],
262 				     0,	 /* map_offs */
263 				     PAGE_SIZE,
264 				     PCI_DMA_BIDIRECTIONAL);  /* FIXME rd/rw */
265 
266 		if (pci_dma_mapping_error(pci_dev, daddr)) {
267 			dev_err(&pci_dev->dev,
268 				"[%s] err: no dma addr daddr=%016llx!\n",
269 				__func__, (long long)daddr);
270 			goto err;
271 		}
272 
273 		dma_list[i] = daddr;
274 	}
275 	return 0;
276 
277  err:
278 	genwqe_unmap_pages(cd, dma_list, num_pages);
279 	return -EIO;
280 }
281 
genwqe_sgl_size(int num_pages)282 static int genwqe_sgl_size(int num_pages)
283 {
284 	int len, num_tlb = num_pages / 7;
285 
286 	len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1);
287 	return roundup(len, PAGE_SIZE);
288 }
289 
290 /**
291  * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages
292  *
293  * Allocates memory for sgl and overlapping pages. Pages which might
294  * overlap other user-space memory blocks are being cached for DMAs,
295  * such that we do not run into syncronization issues. Data is copied
296  * from user-space into the cached pages.
297  */
genwqe_alloc_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,void __user * user_addr,size_t user_size)298 int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
299 			  void __user *user_addr, size_t user_size)
300 {
301 	int ret = -ENOMEM;
302 	struct pci_dev *pci_dev = cd->pci_dev;
303 
304 	sgl->fpage_offs = offset_in_page((unsigned long)user_addr);
305 	sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size);
306 	sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE);
307 	sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE;
308 
309 	dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n",
310 		__func__, user_addr, user_size, sgl->nr_pages,
311 		sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size);
312 
313 	sgl->user_addr = user_addr;
314 	sgl->user_size = user_size;
315 	sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages);
316 
317 	if (get_order(sgl->sgl_size) > MAX_ORDER) {
318 		dev_err(&pci_dev->dev,
319 			"[%s] err: too much memory requested!\n", __func__);
320 		return ret;
321 	}
322 
323 	sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size,
324 					     &sgl->sgl_dma_addr);
325 	if (sgl->sgl == NULL) {
326 		dev_err(&pci_dev->dev,
327 			"[%s] err: no memory available!\n", __func__);
328 		return ret;
329 	}
330 
331 	/* Only use buffering on incomplete pages */
332 	if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) {
333 		sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
334 						       &sgl->fpage_dma_addr);
335 		if (sgl->fpage == NULL)
336 			goto err_out;
337 
338 		/* Sync with user memory */
339 		if (copy_from_user(sgl->fpage + sgl->fpage_offs,
340 				   user_addr, sgl->fpage_size)) {
341 			ret = -EFAULT;
342 			goto err_out;
343 		}
344 	}
345 	if (sgl->lpage_size != 0) {
346 		sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
347 						       &sgl->lpage_dma_addr);
348 		if (sgl->lpage == NULL)
349 			goto err_out1;
350 
351 		/* Sync with user memory */
352 		if (copy_from_user(sgl->lpage, user_addr + user_size -
353 				   sgl->lpage_size, sgl->lpage_size)) {
354 			ret = -EFAULT;
355 			goto err_out2;
356 		}
357 	}
358 	return 0;
359 
360  err_out2:
361 	__genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
362 				 sgl->lpage_dma_addr);
363 	sgl->lpage = NULL;
364 	sgl->lpage_dma_addr = 0;
365  err_out1:
366 	__genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
367 				 sgl->fpage_dma_addr);
368 	sgl->fpage = NULL;
369 	sgl->fpage_dma_addr = 0;
370  err_out:
371 	__genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
372 				 sgl->sgl_dma_addr);
373 	sgl->sgl = NULL;
374 	sgl->sgl_dma_addr = 0;
375 	sgl->sgl_size = 0;
376 
377 	return ret;
378 }
379 
genwqe_setup_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,dma_addr_t * dma_list)380 int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
381 		     dma_addr_t *dma_list)
382 {
383 	int i = 0, j = 0, p;
384 	unsigned long dma_offs, map_offs;
385 	dma_addr_t prev_daddr = 0;
386 	struct sg_entry *s, *last_s = NULL;
387 	size_t size = sgl->user_size;
388 
389 	dma_offs = 128;		/* next block if needed/dma_offset */
390 	map_offs = sgl->fpage_offs; /* offset in first page */
391 
392 	s = &sgl->sgl[0];	/* first set of 8 entries */
393 	p = 0;			/* page */
394 	while (p < sgl->nr_pages) {
395 		dma_addr_t daddr;
396 		unsigned int size_to_map;
397 
398 		/* always write the chaining entry, cleanup is done later */
399 		j = 0;
400 		s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs);
401 		s[j].len	 = cpu_to_be32(128);
402 		s[j].flags	 = cpu_to_be32(SG_CHAINED);
403 		j++;
404 
405 		while (j < 8) {
406 			/* DMA mapping for requested page, offs, size */
407 			size_to_map = min(size, PAGE_SIZE - map_offs);
408 
409 			if ((p == 0) && (sgl->fpage != NULL)) {
410 				daddr = sgl->fpage_dma_addr + map_offs;
411 
412 			} else if ((p == sgl->nr_pages - 1) &&
413 				   (sgl->lpage != NULL)) {
414 				daddr = sgl->lpage_dma_addr;
415 			} else {
416 				daddr = dma_list[p] + map_offs;
417 			}
418 
419 			size -= size_to_map;
420 			map_offs = 0;
421 
422 			if (prev_daddr == daddr) {
423 				u32 prev_len = be32_to_cpu(last_s->len);
424 
425 				/* pr_info("daddr combining: "
426 					"%016llx/%08x -> %016llx\n",
427 					prev_daddr, prev_len, daddr); */
428 
429 				last_s->len = cpu_to_be32(prev_len +
430 							  size_to_map);
431 
432 				p++; /* process next page */
433 				if (p == sgl->nr_pages)
434 					goto fixup;  /* nothing to do */
435 
436 				prev_daddr = daddr + size_to_map;
437 				continue;
438 			}
439 
440 			/* start new entry */
441 			s[j].target_addr = cpu_to_be64(daddr);
442 			s[j].len	 = cpu_to_be32(size_to_map);
443 			s[j].flags	 = cpu_to_be32(SG_DATA);
444 			prev_daddr = daddr + size_to_map;
445 			last_s = &s[j];
446 			j++;
447 
448 			p++;	/* process next page */
449 			if (p == sgl->nr_pages)
450 				goto fixup;  /* nothing to do */
451 		}
452 		dma_offs += 128;
453 		s += 8;		/* continue 8 elements further */
454 	}
455  fixup:
456 	if (j == 1) {		/* combining happend on last entry! */
457 		s -= 8;		/* full shift needed on previous sgl block */
458 		j =  7;		/* shift all elements */
459 	}
460 
461 	for (i = 0; i < j; i++)	/* move elements 1 up */
462 		s[i] = s[i + 1];
463 
464 	s[i].target_addr = cpu_to_be64(0);
465 	s[i].len	 = cpu_to_be32(0);
466 	s[i].flags	 = cpu_to_be32(SG_END_LIST);
467 	return 0;
468 }
469 
470 /**
471  * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages
472  *
473  * After the DMA transfer has been completed we free the memory for
474  * the sgl and the cached pages. Data is being transfered from cached
475  * pages into user-space buffers.
476  */
genwqe_free_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl)477 int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
478 {
479 	int rc = 0;
480 	struct pci_dev *pci_dev = cd->pci_dev;
481 
482 	if (sgl->fpage) {
483 		if (copy_to_user(sgl->user_addr, sgl->fpage + sgl->fpage_offs,
484 				 sgl->fpage_size)) {
485 			dev_err(&pci_dev->dev, "[%s] err: copying fpage!\n",
486 				__func__);
487 			rc = -EFAULT;
488 		}
489 		__genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
490 					 sgl->fpage_dma_addr);
491 		sgl->fpage = NULL;
492 		sgl->fpage_dma_addr = 0;
493 	}
494 	if (sgl->lpage) {
495 		if (copy_to_user(sgl->user_addr + sgl->user_size -
496 				 sgl->lpage_size, sgl->lpage,
497 				 sgl->lpage_size)) {
498 			dev_err(&pci_dev->dev, "[%s] err: copying lpage!\n",
499 				__func__);
500 			rc = -EFAULT;
501 		}
502 		__genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
503 					 sgl->lpage_dma_addr);
504 		sgl->lpage = NULL;
505 		sgl->lpage_dma_addr = 0;
506 	}
507 	__genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
508 				 sgl->sgl_dma_addr);
509 
510 	sgl->sgl = NULL;
511 	sgl->sgl_dma_addr = 0x0;
512 	sgl->sgl_size = 0;
513 	return rc;
514 }
515 
516 /**
517  * free_user_pages() - Give pinned pages back
518  *
519  * Documentation of get_user_pages is in mm/memory.c:
520  *
521  * If the page is written to, set_page_dirty (or set_page_dirty_lock,
522  * as appropriate) must be called after the page is finished with, and
523  * before put_page is called.
524  *
525  * FIXME Could be of use to others and might belong in the generic
526  * code, if others agree. E.g.
527  *    ll_free_user_pages in drivers/staging/lustre/lustre/llite/rw26.c
528  *    ceph_put_page_vector in net/ceph/pagevec.c
529  *    maybe more?
530  */
free_user_pages(struct page ** page_list,unsigned int nr_pages,int dirty)531 static int free_user_pages(struct page **page_list, unsigned int nr_pages,
532 			   int dirty)
533 {
534 	unsigned int i;
535 
536 	for (i = 0; i < nr_pages; i++) {
537 		if (page_list[i] != NULL) {
538 			if (dirty)
539 				set_page_dirty_lock(page_list[i]);
540 			put_page(page_list[i]);
541 		}
542 	}
543 	return 0;
544 }
545 
546 /**
547  * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
548  * @cd:         pointer to genwqe device
549  * @m:          mapping params
550  * @uaddr:      user virtual address
551  * @size:       size of memory to be mapped
552  *
553  * We need to think about how we could speed this up. Of course it is
554  * not a good idea to do this over and over again, like we are
555  * currently doing it. Nevertheless, I am curious where on the path
556  * the performance is spend. Most probably within the memory
557  * allocation functions, but maybe also in the DMA mapping code.
558  *
559  * Restrictions: The maximum size of the possible mapping currently depends
560  *               on the amount of memory we can get using kzalloc() for the
561  *               page_list and pci_alloc_consistent for the sg_list.
562  *               The sg_list is currently itself not scattered, which could
563  *               be fixed with some effort. The page_list must be split into
564  *               PAGE_SIZE chunks too. All that will make the complicated
565  *               code more complicated.
566  *
567  * Return: 0 if success
568  */
genwqe_user_vmap(struct genwqe_dev * cd,struct dma_mapping * m,void * uaddr,unsigned long size,struct ddcb_requ * req)569 int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
570 		     unsigned long size, struct ddcb_requ *req)
571 {
572 	int rc = -EINVAL;
573 	unsigned long data, offs;
574 	struct pci_dev *pci_dev = cd->pci_dev;
575 
576 	if ((uaddr == NULL) || (size == 0)) {
577 		m->size = 0;	/* mark unused and not added */
578 		return -EINVAL;
579 	}
580 	m->u_vaddr = uaddr;
581 	m->size    = size;
582 
583 	/* determine space needed for page_list. */
584 	data = (unsigned long)uaddr;
585 	offs = offset_in_page(data);
586 	if (size > ULONG_MAX - PAGE_SIZE - offs) {
587 		m->size = 0;	/* mark unused and not added */
588 		return -EINVAL;
589 	}
590 	m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE);
591 
592 	m->page_list = kcalloc(m->nr_pages,
593 			       sizeof(struct page *) + sizeof(dma_addr_t),
594 			       GFP_KERNEL);
595 	if (!m->page_list) {
596 		dev_err(&pci_dev->dev, "err: alloc page_list failed\n");
597 		m->nr_pages = 0;
598 		m->u_vaddr = NULL;
599 		m->size = 0;	/* mark unused and not added */
600 		return -ENOMEM;
601 	}
602 	m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
603 
604 	/* pin user pages in memory */
605 	rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
606 				 m->nr_pages,
607 				 1,		/* write by caller */
608 				 m->page_list);	/* ptrs to pages */
609 	if (rc < 0)
610 		goto fail_get_user_pages;
611 
612 	/* assumption: get_user_pages can be killed by signals. */
613 	if (rc < m->nr_pages) {
614 		free_user_pages(m->page_list, rc, 0);
615 		rc = -EFAULT;
616 		goto fail_get_user_pages;
617 	}
618 
619 	rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
620 	if (rc != 0)
621 		goto fail_free_user_pages;
622 
623 	return 0;
624 
625  fail_free_user_pages:
626 	free_user_pages(m->page_list, m->nr_pages, 0);
627 
628  fail_get_user_pages:
629 	kfree(m->page_list);
630 	m->page_list = NULL;
631 	m->dma_list = NULL;
632 	m->nr_pages = 0;
633 	m->u_vaddr = NULL;
634 	m->size = 0;		/* mark unused and not added */
635 	return rc;
636 }
637 
638 /**
639  * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel
640  *                        memory
641  * @cd:         pointer to genwqe device
642  * @m:          mapping params
643  */
genwqe_user_vunmap(struct genwqe_dev * cd,struct dma_mapping * m,struct ddcb_requ * req)644 int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m,
645 		       struct ddcb_requ *req)
646 {
647 	struct pci_dev *pci_dev = cd->pci_dev;
648 
649 	if (!dma_mapping_used(m)) {
650 		dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n",
651 			__func__, m);
652 		return -EINVAL;
653 	}
654 
655 	if (m->dma_list)
656 		genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
657 
658 	if (m->page_list) {
659 		free_user_pages(m->page_list, m->nr_pages, 1);
660 
661 		kfree(m->page_list);
662 		m->page_list = NULL;
663 		m->dma_list = NULL;
664 		m->nr_pages = 0;
665 	}
666 
667 	m->u_vaddr = NULL;
668 	m->size = 0;		/* mark as unused and not added */
669 	return 0;
670 }
671 
672 /**
673  * genwqe_card_type() - Get chip type SLU Configuration Register
674  * @cd:         pointer to the genwqe device descriptor
675  * Return: 0: Altera Stratix-IV 230
676  *         1: Altera Stratix-IV 530
677  *         2: Altera Stratix-V A4
678  *         3: Altera Stratix-V A7
679  */
genwqe_card_type(struct genwqe_dev * cd)680 u8 genwqe_card_type(struct genwqe_dev *cd)
681 {
682 	u64 card_type = cd->slu_unitcfg;
683 
684 	return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20);
685 }
686 
687 /**
688  * genwqe_card_reset() - Reset the card
689  * @cd:         pointer to the genwqe device descriptor
690  */
genwqe_card_reset(struct genwqe_dev * cd)691 int genwqe_card_reset(struct genwqe_dev *cd)
692 {
693 	u64 softrst;
694 	struct pci_dev *pci_dev = cd->pci_dev;
695 
696 	if (!genwqe_is_privileged(cd))
697 		return -ENODEV;
698 
699 	/* new SL */
700 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull);
701 	msleep(1000);
702 	__genwqe_readq(cd, IO_HSU_FIR_CLR);
703 	__genwqe_readq(cd, IO_APP_FIR_CLR);
704 	__genwqe_readq(cd, IO_SLU_FIR_CLR);
705 
706 	/*
707 	 * Read-modify-write to preserve the stealth bits
708 	 *
709 	 * For SL >= 039, Stealth WE bit allows removing
710 	 * the read-modify-wrote.
711 	 * r-m-w may require a mask 0x3C to avoid hitting hard
712 	 * reset again for error reset (should be 0, chicken).
713 	 */
714 	softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull;
715 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull);
716 
717 	/* give ERRORRESET some time to finish */
718 	msleep(50);
719 
720 	if (genwqe_need_err_masking(cd)) {
721 		dev_info(&pci_dev->dev,
722 			 "[%s] masking errors for old bitstreams\n", __func__);
723 		__genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
724 	}
725 	return 0;
726 }
727 
genwqe_read_softreset(struct genwqe_dev * cd)728 int genwqe_read_softreset(struct genwqe_dev *cd)
729 {
730 	u64 bitstream;
731 
732 	if (!genwqe_is_privileged(cd))
733 		return -ENODEV;
734 
735 	bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1;
736 	cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull;
737 	return 0;
738 }
739 
740 /**
741  * genwqe_set_interrupt_capability() - Configure MSI capability structure
742  * @cd:         pointer to the device
743  * Return: 0 if no error
744  */
genwqe_set_interrupt_capability(struct genwqe_dev * cd,int count)745 int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
746 {
747 	int rc;
748 	struct pci_dev *pci_dev = cd->pci_dev;
749 
750 	rc = pci_enable_msi_range(pci_dev, 1, count);
751 	if (rc < 0)
752 		return rc;
753 
754 	cd->flags |= GENWQE_FLAG_MSI_ENABLED;
755 	return 0;
756 }
757 
758 /**
759  * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability()
760  * @cd:         pointer to the device
761  */
genwqe_reset_interrupt_capability(struct genwqe_dev * cd)762 void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
763 {
764 	struct pci_dev *pci_dev = cd->pci_dev;
765 
766 	if (cd->flags & GENWQE_FLAG_MSI_ENABLED) {
767 		pci_disable_msi(pci_dev);
768 		cd->flags &= ~GENWQE_FLAG_MSI_ENABLED;
769 	}
770 }
771 
772 /**
773  * set_reg_idx() - Fill array with data. Ignore illegal offsets.
774  * @cd:         card device
775  * @r:          debug register array
776  * @i:          index to desired entry
777  * @m:          maximum possible entries
778  * @addr:       addr which is read
779  * @index:      index in debug array
780  * @val:        read value
781  */
set_reg_idx(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u32 idx,u64 val)782 static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r,
783 		       unsigned int *i, unsigned int m, u32 addr, u32 idx,
784 		       u64 val)
785 {
786 	if (WARN_ON_ONCE(*i >= m))
787 		return -EFAULT;
788 
789 	r[*i].addr = addr;
790 	r[*i].idx = idx;
791 	r[*i].val = val;
792 	++*i;
793 	return 0;
794 }
795 
set_reg(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u64 val)796 static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r,
797 		   unsigned int *i, unsigned int m, u32 addr, u64 val)
798 {
799 	return set_reg_idx(cd, r, i, m, addr, 0, val);
800 }
801 
genwqe_read_ffdc_regs(struct genwqe_dev * cd,struct genwqe_reg * regs,unsigned int max_regs,int all)802 int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs,
803 			 unsigned int max_regs, int all)
804 {
805 	unsigned int i, j, idx = 0;
806 	u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr;
807 	u64 gfir, sluid, appid, ufir, ufec, sfir, sfec;
808 
809 	/* Global FIR */
810 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
811 	set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir);
812 
813 	/* UnitCfg for SLU */
814 	sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */
815 	set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid);
816 
817 	/* UnitCfg for APP */
818 	appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */
819 	set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid);
820 
821 	/* Check all chip Units */
822 	for (i = 0; i < GENWQE_MAX_UNITS; i++) {
823 
824 		/* Unit FIR */
825 		ufir_addr = (i << 24) | 0x008;
826 		ufir = __genwqe_readq(cd, ufir_addr);
827 		set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir);
828 
829 		/* Unit FEC */
830 		ufec_addr = (i << 24) | 0x018;
831 		ufec = __genwqe_readq(cd, ufec_addr);
832 		set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec);
833 
834 		for (j = 0; j < 64; j++) {
835 			/* wherever there is a primary 1, read the 2ndary */
836 			if (!all && (!(ufir & (1ull << j))))
837 				continue;
838 
839 			sfir_addr = (i << 24) | (0x100 + 8 * j);
840 			sfir = __genwqe_readq(cd, sfir_addr);
841 			set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir);
842 
843 			sfec_addr = (i << 24) | (0x300 + 8 * j);
844 			sfec = __genwqe_readq(cd, sfec_addr);
845 			set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec);
846 		}
847 	}
848 
849 	/* fill with invalid data until end */
850 	for (i = idx; i < max_regs; i++) {
851 		regs[i].addr = 0xffffffff;
852 		regs[i].val = 0xffffffffffffffffull;
853 	}
854 	return idx;
855 }
856 
857 /**
858  * genwqe_ffdc_buff_size() - Calculates the number of dump registers
859  */
genwqe_ffdc_buff_size(struct genwqe_dev * cd,int uid)860 int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid)
861 {
862 	int entries = 0, ring, traps, traces, trace_entries;
863 	u32 eevptr_addr, l_addr, d_len, d_type;
864 	u64 eevptr, val, addr;
865 
866 	eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
867 	eevptr = __genwqe_readq(cd, eevptr_addr);
868 
869 	if ((eevptr != 0x0) && (eevptr != -1ull)) {
870 		l_addr = GENWQE_UID_OFFS(uid) | eevptr;
871 
872 		while (1) {
873 			val = __genwqe_readq(cd, l_addr);
874 
875 			if ((val == 0x0) || (val == -1ull))
876 				break;
877 
878 			/* 38:24 */
879 			d_len  = (val & 0x0000007fff000000ull) >> 24;
880 
881 			/* 39 */
882 			d_type = (val & 0x0000008000000000ull) >> 36;
883 
884 			if (d_type) {	/* repeat */
885 				entries += d_len;
886 			} else {	/* size in bytes! */
887 				entries += d_len >> 3;
888 			}
889 
890 			l_addr += 8;
891 		}
892 	}
893 
894 	for (ring = 0; ring < 8; ring++) {
895 		addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
896 		val = __genwqe_readq(cd, addr);
897 
898 		if ((val == 0x0ull) || (val == -1ull))
899 			continue;
900 
901 		traps = (val >> 24) & 0xff;
902 		traces = (val >> 16) & 0xff;
903 		trace_entries = val & 0xffff;
904 
905 		entries += traps + (traces * trace_entries);
906 	}
907 	return entries;
908 }
909 
910 /**
911  * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure
912  */
genwqe_ffdc_buff_read(struct genwqe_dev * cd,int uid,struct genwqe_reg * regs,unsigned int max_regs)913 int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid,
914 			  struct genwqe_reg *regs, unsigned int max_regs)
915 {
916 	int i, traps, traces, trace, trace_entries, trace_entry, ring;
917 	unsigned int idx = 0;
918 	u32 eevptr_addr, l_addr, d_addr, d_len, d_type;
919 	u64 eevptr, e, val, addr;
920 
921 	eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
922 	eevptr = __genwqe_readq(cd, eevptr_addr);
923 
924 	if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) {
925 		l_addr = GENWQE_UID_OFFS(uid) | eevptr;
926 		while (1) {
927 			e = __genwqe_readq(cd, l_addr);
928 			if ((e == 0x0) || (e == 0xffffffffffffffffull))
929 				break;
930 
931 			d_addr = (e & 0x0000000000ffffffull);	    /* 23:0 */
932 			d_len  = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */
933 			d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */
934 			d_addr |= GENWQE_UID_OFFS(uid);
935 
936 			if (d_type) {
937 				for (i = 0; i < (int)d_len; i++) {
938 					val = __genwqe_readq(cd, d_addr);
939 					set_reg_idx(cd, regs, &idx, max_regs,
940 						    d_addr, i, val);
941 				}
942 			} else {
943 				d_len >>= 3; /* Size in bytes! */
944 				for (i = 0; i < (int)d_len; i++, d_addr += 8) {
945 					val = __genwqe_readq(cd, d_addr);
946 					set_reg_idx(cd, regs, &idx, max_regs,
947 						    d_addr, 0, val);
948 				}
949 			}
950 			l_addr += 8;
951 		}
952 	}
953 
954 	/*
955 	 * To save time, there are only 6 traces poplulated on Uid=2,
956 	 * Ring=1. each with iters=512.
957 	 */
958 	for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds,
959 					      2...7 are ASI rings */
960 		addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
961 		val = __genwqe_readq(cd, addr);
962 
963 		if ((val == 0x0ull) || (val == -1ull))
964 			continue;
965 
966 		traps = (val >> 24) & 0xff;	/* Number of Traps	*/
967 		traces = (val >> 16) & 0xff;	/* Number of Traces	*/
968 		trace_entries = val & 0xffff;	/* Entries per trace	*/
969 
970 		/* Note: This is a combined loop that dumps both the traps */
971 		/* (for the trace == 0 case) as well as the traces 1 to    */
972 		/* 'traces'.						   */
973 		for (trace = 0; trace <= traces; trace++) {
974 			u32 diag_sel =
975 				GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace);
976 
977 			addr = (GENWQE_UID_OFFS(uid) |
978 				IO_EXTENDED_DIAG_SELECTOR);
979 			__genwqe_writeq(cd, addr, diag_sel);
980 
981 			for (trace_entry = 0;
982 			     trace_entry < (trace ? trace_entries : traps);
983 			     trace_entry++) {
984 				addr = (GENWQE_UID_OFFS(uid) |
985 					IO_EXTENDED_DIAG_READ_MBX);
986 				val = __genwqe_readq(cd, addr);
987 				set_reg_idx(cd, regs, &idx, max_regs, addr,
988 					    (diag_sel<<16) | trace_entry, val);
989 			}
990 		}
991 	}
992 	return 0;
993 }
994 
995 /**
996  * genwqe_write_vreg() - Write register in virtual window
997  *
998  * Note, these registers are only accessible to the PF through the
999  * VF-window. It is not intended for the VF to access.
1000  */
genwqe_write_vreg(struct genwqe_dev * cd,u32 reg,u64 val,int func)1001 int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func)
1002 {
1003 	__genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1004 	__genwqe_writeq(cd, reg, val);
1005 	return 0;
1006 }
1007 
1008 /**
1009  * genwqe_read_vreg() - Read register in virtual window
1010  *
1011  * Note, these registers are only accessible to the PF through the
1012  * VF-window. It is not intended for the VF to access.
1013  */
genwqe_read_vreg(struct genwqe_dev * cd,u32 reg,int func)1014 u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func)
1015 {
1016 	__genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1017 	return __genwqe_readq(cd, reg);
1018 }
1019 
1020 /**
1021  * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card
1022  *
1023  * Note: From a design perspective it turned out to be a bad idea to
1024  * use codes here to specifiy the frequency/speed values. An old
1025  * driver cannot understand new codes and is therefore always a
1026  * problem. Better is to measure out the value or put the
1027  * speed/frequency directly into a register which is always a valid
1028  * value for old as well as for new software.
1029  *
1030  * Return: Card clock in MHz
1031  */
genwqe_base_clock_frequency(struct genwqe_dev * cd)1032 int genwqe_base_clock_frequency(struct genwqe_dev *cd)
1033 {
1034 	u16 speed;		/*         MHz  MHz  MHz  MHz */
1035 	static const int speed_grade[] = { 250, 200, 166, 175 };
1036 
1037 	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
1038 	if (speed >= ARRAY_SIZE(speed_grade))
1039 		return 0;	/* illegal value */
1040 
1041 	return speed_grade[speed];
1042 }
1043 
1044 /**
1045  * genwqe_stop_traps() - Stop traps
1046  *
1047  * Before reading out the analysis data, we need to stop the traps.
1048  */
genwqe_stop_traps(struct genwqe_dev * cd)1049 void genwqe_stop_traps(struct genwqe_dev *cd)
1050 {
1051 	__genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull);
1052 }
1053 
1054 /**
1055  * genwqe_start_traps() - Start traps
1056  *
1057  * After having read the data, we can/must enable the traps again.
1058  */
genwqe_start_traps(struct genwqe_dev * cd)1059 void genwqe_start_traps(struct genwqe_dev *cd)
1060 {
1061 	__genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull);
1062 
1063 	if (genwqe_need_err_masking(cd))
1064 		__genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
1065 }
1066