1 /*
2 * Driver for IBM Power 842 compression accelerator
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 *
18 * Copyright (C) IBM Corporation, 2012
19 *
20 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
21 * Seth Jennings <sjenning@linux.vnet.ibm.com>
22 */
23
24 #include <asm/vio.h>
25
26 #include "nx-842.h"
27 #include "nx_csbcpb.h" /* struct nx_csbcpb */
28
29 MODULE_LICENSE("GPL");
30 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
31 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
32 MODULE_ALIAS_CRYPTO("842");
33 MODULE_ALIAS_CRYPTO("842-nx");
34
35 static struct nx842_constraints nx842_pseries_constraints = {
36 .alignment = DDE_BUFFER_ALIGN,
37 .multiple = DDE_BUFFER_LAST_MULT,
38 .minimum = DDE_BUFFER_LAST_MULT,
39 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
40 };
41
check_constraints(unsigned long buf,unsigned int * len,bool in)42 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
43 {
44 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
45 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
46 in ? "input" : "output", buf,
47 nx842_pseries_constraints.alignment);
48 return -EINVAL;
49 }
50 if (*len % nx842_pseries_constraints.multiple) {
51 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
52 in ? "input" : "output", *len,
53 nx842_pseries_constraints.multiple);
54 if (in)
55 return -EINVAL;
56 *len = round_down(*len, nx842_pseries_constraints.multiple);
57 }
58 if (*len < nx842_pseries_constraints.minimum) {
59 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
60 in ? "input" : "output", *len,
61 nx842_pseries_constraints.minimum);
62 return -EINVAL;
63 }
64 if (*len > nx842_pseries_constraints.maximum) {
65 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
66 in ? "input" : "output", *len,
67 nx842_pseries_constraints.maximum);
68 if (in)
69 return -EINVAL;
70 *len = nx842_pseries_constraints.maximum;
71 }
72 return 0;
73 }
74
75 /* I assume we need to align the CSB? */
76 #define WORKMEM_ALIGN (256)
77
78 struct nx842_workmem {
79 /* scatterlist */
80 char slin[4096];
81 char slout[4096];
82 /* coprocessor status/parameter block */
83 struct nx_csbcpb csbcpb;
84
85 char padding[WORKMEM_ALIGN];
86 } __aligned(WORKMEM_ALIGN);
87
88 /* Macros for fields within nx_csbcpb */
89 /* Check the valid bit within the csbcpb valid field */
90 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
91
92 /* CE macros operate on the completion_extension field bits in the csbcpb.
93 * CE0 0=full completion, 1=partial completion
94 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
95 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
96 #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
97 #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
98 #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
99
100 /* The NX unit accepts data only on 4K page boundaries */
101 #define NX842_HW_PAGE_SIZE (4096)
102 #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
103
104 struct ibm_nx842_counters {
105 atomic64_t comp_complete;
106 atomic64_t comp_failed;
107 atomic64_t decomp_complete;
108 atomic64_t decomp_failed;
109 atomic64_t swdecomp;
110 atomic64_t comp_times[32];
111 atomic64_t decomp_times[32];
112 };
113
114 static struct nx842_devdata {
115 struct vio_dev *vdev;
116 struct device *dev;
117 struct ibm_nx842_counters *counters;
118 unsigned int max_sg_len;
119 unsigned int max_sync_size;
120 unsigned int max_sync_sg;
121 } __rcu *devdata;
122 static DEFINE_SPINLOCK(devdata_mutex);
123
124 #define NX842_COUNTER_INC(_x) \
125 static inline void nx842_inc_##_x( \
126 const struct nx842_devdata *dev) { \
127 if (dev) \
128 atomic64_inc(&dev->counters->_x); \
129 }
130 NX842_COUNTER_INC(comp_complete);
131 NX842_COUNTER_INC(comp_failed);
132 NX842_COUNTER_INC(decomp_complete);
133 NX842_COUNTER_INC(decomp_failed);
134 NX842_COUNTER_INC(swdecomp);
135
136 #define NX842_HIST_SLOTS 16
137
ibm_nx842_incr_hist(atomic64_t * times,unsigned int time)138 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
139 {
140 int bucket = fls(time);
141
142 if (bucket)
143 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
144
145 atomic64_inc(×[bucket]);
146 }
147
148 /* NX unit operation flags */
149 #define NX842_OP_COMPRESS 0x0
150 #define NX842_OP_CRC 0x1
151 #define NX842_OP_DECOMPRESS 0x2
152 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
153 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
154 #define NX842_OP_ASYNC (1<<23)
155 #define NX842_OP_NOTIFY (1<<22)
156 #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
157
nx842_get_desired_dma(struct vio_dev * viodev)158 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
159 {
160 /* No use of DMA mappings within the driver. */
161 return 0;
162 }
163
164 struct nx842_slentry {
165 __be64 ptr; /* Real address (use __pa()) */
166 __be64 len;
167 };
168
169 /* pHyp scatterlist entry */
170 struct nx842_scatterlist {
171 int entry_nr; /* number of slentries */
172 struct nx842_slentry *entries; /* ptr to array of slentries */
173 };
174
175 /* Does not include sizeof(entry_nr) in the size */
nx842_get_scatterlist_size(struct nx842_scatterlist * sl)176 static inline unsigned long nx842_get_scatterlist_size(
177 struct nx842_scatterlist *sl)
178 {
179 return sl->entry_nr * sizeof(struct nx842_slentry);
180 }
181
nx842_build_scatterlist(unsigned long buf,int len,struct nx842_scatterlist * sl)182 static int nx842_build_scatterlist(unsigned long buf, int len,
183 struct nx842_scatterlist *sl)
184 {
185 unsigned long entrylen;
186 struct nx842_slentry *entry;
187
188 sl->entry_nr = 0;
189
190 entry = sl->entries;
191 while (len) {
192 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
193 entrylen = min_t(int, len,
194 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
195 entry->len = cpu_to_be64(entrylen);
196
197 len -= entrylen;
198 buf += entrylen;
199
200 sl->entry_nr++;
201 entry++;
202 }
203
204 return 0;
205 }
206
nx842_validate_result(struct device * dev,struct cop_status_block * csb)207 static int nx842_validate_result(struct device *dev,
208 struct cop_status_block *csb)
209 {
210 /* The csb must be valid after returning from vio_h_cop_sync */
211 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
212 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
213 __func__);
214 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
215 csb->valid,
216 csb->crb_seq_number,
217 csb->completion_code,
218 csb->completion_extension);
219 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
220 be32_to_cpu(csb->processed_byte_count),
221 (unsigned long)be64_to_cpu(csb->address));
222 return -EIO;
223 }
224
225 /* Check return values from the hardware in the CSB */
226 switch (csb->completion_code) {
227 case 0: /* Completed without error */
228 break;
229 case 64: /* Compression ok, but output larger than input */
230 dev_dbg(dev, "%s: output size larger than input size\n",
231 __func__);
232 break;
233 case 13: /* Output buffer too small */
234 dev_dbg(dev, "%s: Out of space in output buffer\n",
235 __func__);
236 return -ENOSPC;
237 case 65: /* Calculated CRC doesn't match the passed value */
238 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
239 __func__);
240 return -EINVAL;
241 case 66: /* Input data contains an illegal template field */
242 case 67: /* Template indicates data past the end of the input stream */
243 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
244 __func__, csb->completion_code);
245 return -EINVAL;
246 default:
247 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
248 __func__, csb->completion_code);
249 return -EIO;
250 }
251
252 /* Hardware sanity check */
253 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
254 dev_err(dev, "%s: No error returned by hardware, but "
255 "data returned is unusable, contact support.\n"
256 "(Additional info: csbcbp->processed bytes "
257 "does not specify processed bytes for the "
258 "target buffer.)\n", __func__);
259 return -EIO;
260 }
261
262 return 0;
263 }
264
265 /**
266 * nx842_pseries_compress - Compress data using the 842 algorithm
267 *
268 * Compression provide by the NX842 coprocessor on IBM Power systems.
269 * The input buffer is compressed and the result is stored in the
270 * provided output buffer.
271 *
272 * Upon return from this function @outlen contains the length of the
273 * compressed data. If there is an error then @outlen will be 0 and an
274 * error will be specified by the return code from this function.
275 *
276 * @in: Pointer to input buffer
277 * @inlen: Length of input buffer
278 * @out: Pointer to output buffer
279 * @outlen: Length of output buffer
280 * @wrkmem: ptr to buffer for working memory, size determined by
281 * nx842_pseries_driver.workmem_size
282 *
283 * Returns:
284 * 0 Success, output of length @outlen stored in the buffer at @out
285 * -ENOMEM Unable to allocate internal buffers
286 * -ENOSPC Output buffer is to small
287 * -EIO Internal error
288 * -ENODEV Hardware unavailable
289 */
nx842_pseries_compress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)290 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
291 unsigned char *out, unsigned int *outlen,
292 void *wmem)
293 {
294 struct nx842_devdata *local_devdata;
295 struct device *dev = NULL;
296 struct nx842_workmem *workmem;
297 struct nx842_scatterlist slin, slout;
298 struct nx_csbcpb *csbcpb;
299 int ret = 0, max_sync_size;
300 unsigned long inbuf, outbuf;
301 struct vio_pfo_op op = {
302 .done = NULL,
303 .handle = 0,
304 .timeout = 0,
305 };
306 unsigned long start = get_tb();
307
308 inbuf = (unsigned long)in;
309 if (check_constraints(inbuf, &inlen, true))
310 return -EINVAL;
311
312 outbuf = (unsigned long)out;
313 if (check_constraints(outbuf, outlen, false))
314 return -EINVAL;
315
316 rcu_read_lock();
317 local_devdata = rcu_dereference(devdata);
318 if (!local_devdata || !local_devdata->dev) {
319 rcu_read_unlock();
320 return -ENODEV;
321 }
322 max_sync_size = local_devdata->max_sync_size;
323 dev = local_devdata->dev;
324
325 /* Init scatterlist */
326 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
327 slin.entries = (struct nx842_slentry *)workmem->slin;
328 slout.entries = (struct nx842_slentry *)workmem->slout;
329
330 /* Init operation */
331 op.flags = NX842_OP_COMPRESS_CRC;
332 csbcpb = &workmem->csbcpb;
333 memset(csbcpb, 0, sizeof(*csbcpb));
334 op.csbcpb = nx842_get_pa(csbcpb);
335
336 if ((inbuf & NX842_HW_PAGE_MASK) ==
337 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
338 /* Create direct DDE */
339 op.in = nx842_get_pa((void *)inbuf);
340 op.inlen = inlen;
341 } else {
342 /* Create indirect DDE (scatterlist) */
343 nx842_build_scatterlist(inbuf, inlen, &slin);
344 op.in = nx842_get_pa(slin.entries);
345 op.inlen = -nx842_get_scatterlist_size(&slin);
346 }
347
348 if ((outbuf & NX842_HW_PAGE_MASK) ==
349 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
350 /* Create direct DDE */
351 op.out = nx842_get_pa((void *)outbuf);
352 op.outlen = *outlen;
353 } else {
354 /* Create indirect DDE (scatterlist) */
355 nx842_build_scatterlist(outbuf, *outlen, &slout);
356 op.out = nx842_get_pa(slout.entries);
357 op.outlen = -nx842_get_scatterlist_size(&slout);
358 }
359
360 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
361 __func__, (unsigned long)op.in, (long)op.inlen,
362 (unsigned long)op.out, (long)op.outlen);
363
364 /* Send request to pHyp */
365 ret = vio_h_cop_sync(local_devdata->vdev, &op);
366
367 /* Check for pHyp error */
368 if (ret) {
369 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
370 __func__, ret, op.hcall_err);
371 ret = -EIO;
372 goto unlock;
373 }
374
375 /* Check for hardware error */
376 ret = nx842_validate_result(dev, &csbcpb->csb);
377 if (ret)
378 goto unlock;
379
380 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
381 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
382
383 unlock:
384 if (ret)
385 nx842_inc_comp_failed(local_devdata);
386 else {
387 nx842_inc_comp_complete(local_devdata);
388 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
389 (get_tb() - start) / tb_ticks_per_usec);
390 }
391 rcu_read_unlock();
392 return ret;
393 }
394
395 /**
396 * nx842_pseries_decompress - Decompress data using the 842 algorithm
397 *
398 * Decompression provide by the NX842 coprocessor on IBM Power systems.
399 * The input buffer is decompressed and the result is stored in the
400 * provided output buffer. The size allocated to the output buffer is
401 * provided by the caller of this function in @outlen. Upon return from
402 * this function @outlen contains the length of the decompressed data.
403 * If there is an error then @outlen will be 0 and an error will be
404 * specified by the return code from this function.
405 *
406 * @in: Pointer to input buffer
407 * @inlen: Length of input buffer
408 * @out: Pointer to output buffer
409 * @outlen: Length of output buffer
410 * @wrkmem: ptr to buffer for working memory, size determined by
411 * nx842_pseries_driver.workmem_size
412 *
413 * Returns:
414 * 0 Success, output of length @outlen stored in the buffer at @out
415 * -ENODEV Hardware decompression device is unavailable
416 * -ENOMEM Unable to allocate internal buffers
417 * -ENOSPC Output buffer is to small
418 * -EINVAL Bad input data encountered when attempting decompress
419 * -EIO Internal error
420 */
nx842_pseries_decompress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)421 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
422 unsigned char *out, unsigned int *outlen,
423 void *wmem)
424 {
425 struct nx842_devdata *local_devdata;
426 struct device *dev = NULL;
427 struct nx842_workmem *workmem;
428 struct nx842_scatterlist slin, slout;
429 struct nx_csbcpb *csbcpb;
430 int ret = 0, max_sync_size;
431 unsigned long inbuf, outbuf;
432 struct vio_pfo_op op = {
433 .done = NULL,
434 .handle = 0,
435 .timeout = 0,
436 };
437 unsigned long start = get_tb();
438
439 /* Ensure page alignment and size */
440 inbuf = (unsigned long)in;
441 if (check_constraints(inbuf, &inlen, true))
442 return -EINVAL;
443
444 outbuf = (unsigned long)out;
445 if (check_constraints(outbuf, outlen, false))
446 return -EINVAL;
447
448 rcu_read_lock();
449 local_devdata = rcu_dereference(devdata);
450 if (!local_devdata || !local_devdata->dev) {
451 rcu_read_unlock();
452 return -ENODEV;
453 }
454 max_sync_size = local_devdata->max_sync_size;
455 dev = local_devdata->dev;
456
457 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
458
459 /* Init scatterlist */
460 slin.entries = (struct nx842_slentry *)workmem->slin;
461 slout.entries = (struct nx842_slentry *)workmem->slout;
462
463 /* Init operation */
464 op.flags = NX842_OP_DECOMPRESS_CRC;
465 csbcpb = &workmem->csbcpb;
466 memset(csbcpb, 0, sizeof(*csbcpb));
467 op.csbcpb = nx842_get_pa(csbcpb);
468
469 if ((inbuf & NX842_HW_PAGE_MASK) ==
470 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
471 /* Create direct DDE */
472 op.in = nx842_get_pa((void *)inbuf);
473 op.inlen = inlen;
474 } else {
475 /* Create indirect DDE (scatterlist) */
476 nx842_build_scatterlist(inbuf, inlen, &slin);
477 op.in = nx842_get_pa(slin.entries);
478 op.inlen = -nx842_get_scatterlist_size(&slin);
479 }
480
481 if ((outbuf & NX842_HW_PAGE_MASK) ==
482 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
483 /* Create direct DDE */
484 op.out = nx842_get_pa((void *)outbuf);
485 op.outlen = *outlen;
486 } else {
487 /* Create indirect DDE (scatterlist) */
488 nx842_build_scatterlist(outbuf, *outlen, &slout);
489 op.out = nx842_get_pa(slout.entries);
490 op.outlen = -nx842_get_scatterlist_size(&slout);
491 }
492
493 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
494 __func__, (unsigned long)op.in, (long)op.inlen,
495 (unsigned long)op.out, (long)op.outlen);
496
497 /* Send request to pHyp */
498 ret = vio_h_cop_sync(local_devdata->vdev, &op);
499
500 /* Check for pHyp error */
501 if (ret) {
502 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
503 __func__, ret, op.hcall_err);
504 goto unlock;
505 }
506
507 /* Check for hardware error */
508 ret = nx842_validate_result(dev, &csbcpb->csb);
509 if (ret)
510 goto unlock;
511
512 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
513
514 unlock:
515 if (ret)
516 /* decompress fail */
517 nx842_inc_decomp_failed(local_devdata);
518 else {
519 nx842_inc_decomp_complete(local_devdata);
520 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
521 (get_tb() - start) / tb_ticks_per_usec);
522 }
523
524 rcu_read_unlock();
525 return ret;
526 }
527
528 /**
529 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
530 *
531 * @devdata - struct nx842_devdata to update
532 *
533 * Returns:
534 * 0 on success
535 * -ENOENT if @devdata ptr is NULL
536 */
nx842_OF_set_defaults(struct nx842_devdata * devdata)537 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
538 {
539 if (devdata) {
540 devdata->max_sync_size = 0;
541 devdata->max_sync_sg = 0;
542 devdata->max_sg_len = 0;
543 return 0;
544 } else
545 return -ENOENT;
546 }
547
548 /**
549 * nx842_OF_upd_status -- Check the device info from OF status prop
550 *
551 * The status property indicates if the accelerator is enabled. If the
552 * device is in the OF tree it indicates that the hardware is present.
553 * The status field indicates if the device is enabled when the status
554 * is 'okay'. Otherwise the device driver will be disabled.
555 *
556 * @devdata: struct nx842_devdata to use for dev_info
557 * @prop: struct property point containing the maxsyncop for the update
558 *
559 * Returns:
560 * 0 - Device is available
561 * -ENODEV - Device is not available
562 */
nx842_OF_upd_status(struct nx842_devdata * devdata,struct property * prop)563 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
564 struct property *prop)
565 {
566 const char *status = (const char *)prop->value;
567
568 if (!strncmp(status, "okay", (size_t)prop->length))
569 return 0;
570 if (!strncmp(status, "disabled", (size_t)prop->length))
571 return -ENODEV;
572 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
573
574 return -EINVAL;
575 }
576
577 /**
578 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
579 *
580 * Definition of the 'ibm,max-sg-len' OF property:
581 * This field indicates the maximum byte length of a scatter list
582 * for the platform facility. It is a single cell encoded as with encode-int.
583 *
584 * Example:
585 * # od -x ibm,max-sg-len
586 * 0000000 0000 0ff0
587 *
588 * In this example, the maximum byte length of a scatter list is
589 * 0x0ff0 (4,080).
590 *
591 * @devdata - struct nx842_devdata to update
592 * @prop - struct property point containing the maxsyncop for the update
593 *
594 * Returns:
595 * 0 on success
596 * -EINVAL on failure
597 */
nx842_OF_upd_maxsglen(struct nx842_devdata * devdata,struct property * prop)598 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
599 struct property *prop) {
600 int ret = 0;
601 const unsigned int maxsglen = of_read_number(prop->value, 1);
602
603 if (prop->length != sizeof(maxsglen)) {
604 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
605 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
606 prop->length, sizeof(maxsglen));
607 ret = -EINVAL;
608 } else {
609 devdata->max_sg_len = min_t(unsigned int,
610 maxsglen, NX842_HW_PAGE_SIZE);
611 }
612
613 return ret;
614 }
615
616 /**
617 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
618 *
619 * Definition of the 'ibm,max-sync-cop' OF property:
620 * Two series of cells. The first series of cells represents the maximums
621 * that can be synchronously compressed. The second series of cells
622 * represents the maximums that can be synchronously decompressed.
623 * 1. The first cell in each series contains the count of the number of
624 * data length, scatter list elements pairs that follow – each being
625 * of the form
626 * a. One cell data byte length
627 * b. One cell total number of scatter list elements
628 *
629 * Example:
630 * # od -x ibm,max-sync-cop
631 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
632 * 0000020 0000 1000 0000 01fe
633 *
634 * In this example, compression supports 0x1000 (4,096) data byte length
635 * and 0x1fe (510) total scatter list elements. Decompression supports
636 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
637 * elements.
638 *
639 * @devdata - struct nx842_devdata to update
640 * @prop - struct property point containing the maxsyncop for the update
641 *
642 * Returns:
643 * 0 on success
644 * -EINVAL on failure
645 */
nx842_OF_upd_maxsyncop(struct nx842_devdata * devdata,struct property * prop)646 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
647 struct property *prop) {
648 int ret = 0;
649 unsigned int comp_data_limit, decomp_data_limit;
650 unsigned int comp_sg_limit, decomp_sg_limit;
651 const struct maxsynccop_t {
652 __be32 comp_elements;
653 __be32 comp_data_limit;
654 __be32 comp_sg_limit;
655 __be32 decomp_elements;
656 __be32 decomp_data_limit;
657 __be32 decomp_sg_limit;
658 } *maxsynccop;
659
660 if (prop->length != sizeof(*maxsynccop)) {
661 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
662 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
663 sizeof(*maxsynccop));
664 ret = -EINVAL;
665 goto out;
666 }
667
668 maxsynccop = (const struct maxsynccop_t *)prop->value;
669 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
670 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
671 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
672 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
673
674 /* Use one limit rather than separate limits for compression and
675 * decompression. Set a maximum for this so as not to exceed the
676 * size that the header can support and round the value down to
677 * the hardware page size (4K) */
678 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
679
680 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
681 65536);
682
683 if (devdata->max_sync_size < 4096) {
684 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
685 "less than the driver minimum, unable to use "
686 "the hardware device\n",
687 __func__, devdata->max_sync_size);
688 ret = -EINVAL;
689 goto out;
690 }
691
692 nx842_pseries_constraints.maximum = devdata->max_sync_size;
693
694 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
695 if (devdata->max_sync_sg < 1) {
696 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
697 "less than the driver minimum, unable to use "
698 "the hardware device\n",
699 __func__, devdata->max_sync_sg);
700 ret = -EINVAL;
701 goto out;
702 }
703
704 out:
705 return ret;
706 }
707
708 /**
709 *
710 * nx842_OF_upd -- Handle OF properties updates for the device.
711 *
712 * Set all properties from the OF tree. Optionally, a new property
713 * can be provided by the @new_prop pointer to overwrite an existing value.
714 * The device will remain disabled until all values are valid, this function
715 * will return an error for updates unless all values are valid.
716 *
717 * @new_prop: If not NULL, this property is being updated. If NULL, update
718 * all properties from the current values in the OF tree.
719 *
720 * Returns:
721 * 0 - Success
722 * -ENOMEM - Could not allocate memory for new devdata structure
723 * -EINVAL - property value not found, new_prop is not a recognized
724 * property for the device or property value is not valid.
725 * -ENODEV - Device is not available
726 */
nx842_OF_upd(struct property * new_prop)727 static int nx842_OF_upd(struct property *new_prop)
728 {
729 struct nx842_devdata *old_devdata = NULL;
730 struct nx842_devdata *new_devdata = NULL;
731 struct device_node *of_node = NULL;
732 struct property *status = NULL;
733 struct property *maxsglen = NULL;
734 struct property *maxsyncop = NULL;
735 int ret = 0;
736 unsigned long flags;
737
738 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
739 if (!new_devdata)
740 return -ENOMEM;
741
742 spin_lock_irqsave(&devdata_mutex, flags);
743 old_devdata = rcu_dereference_check(devdata,
744 lockdep_is_held(&devdata_mutex));
745 if (old_devdata)
746 of_node = old_devdata->dev->of_node;
747
748 if (!old_devdata || !of_node) {
749 pr_err("%s: device is not available\n", __func__);
750 spin_unlock_irqrestore(&devdata_mutex, flags);
751 kfree(new_devdata);
752 return -ENODEV;
753 }
754
755 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
756 new_devdata->counters = old_devdata->counters;
757
758 /* Set ptrs for existing properties */
759 status = of_find_property(of_node, "status", NULL);
760 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
761 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
762 if (!status || !maxsglen || !maxsyncop) {
763 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
764 ret = -EINVAL;
765 goto error_out;
766 }
767
768 /*
769 * If this is a property update, there are only certain properties that
770 * we care about. Bail if it isn't in the below list
771 */
772 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
773 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
774 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
775 goto out;
776
777 /* Perform property updates */
778 ret = nx842_OF_upd_status(new_devdata, status);
779 if (ret)
780 goto error_out;
781
782 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
783 if (ret)
784 goto error_out;
785
786 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
787 if (ret)
788 goto error_out;
789
790 out:
791 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
792 __func__, new_devdata->max_sync_size,
793 old_devdata->max_sync_size);
794 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
795 __func__, new_devdata->max_sync_sg,
796 old_devdata->max_sync_sg);
797 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
798 __func__, new_devdata->max_sg_len,
799 old_devdata->max_sg_len);
800
801 rcu_assign_pointer(devdata, new_devdata);
802 spin_unlock_irqrestore(&devdata_mutex, flags);
803 synchronize_rcu();
804 dev_set_drvdata(new_devdata->dev, new_devdata);
805 kfree(old_devdata);
806 return 0;
807
808 error_out:
809 if (new_devdata) {
810 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
811 nx842_OF_set_defaults(new_devdata);
812 rcu_assign_pointer(devdata, new_devdata);
813 spin_unlock_irqrestore(&devdata_mutex, flags);
814 synchronize_rcu();
815 dev_set_drvdata(new_devdata->dev, new_devdata);
816 kfree(old_devdata);
817 } else {
818 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
819 spin_unlock_irqrestore(&devdata_mutex, flags);
820 }
821
822 if (!ret)
823 ret = -EINVAL;
824 return ret;
825 }
826
827 /**
828 * nx842_OF_notifier - Process updates to OF properties for the device
829 *
830 * @np: notifier block
831 * @action: notifier action
832 * @update: struct pSeries_reconfig_prop_update pointer if action is
833 * PSERIES_UPDATE_PROPERTY
834 *
835 * Returns:
836 * NOTIFY_OK on success
837 * NOTIFY_BAD encoded with error number on failure, use
838 * notifier_to_errno() to decode this value
839 */
nx842_OF_notifier(struct notifier_block * np,unsigned long action,void * data)840 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
841 void *data)
842 {
843 struct of_reconfig_data *upd = data;
844 struct nx842_devdata *local_devdata;
845 struct device_node *node = NULL;
846
847 rcu_read_lock();
848 local_devdata = rcu_dereference(devdata);
849 if (local_devdata)
850 node = local_devdata->dev->of_node;
851
852 if (local_devdata &&
853 action == OF_RECONFIG_UPDATE_PROPERTY &&
854 !strcmp(upd->dn->name, node->name)) {
855 rcu_read_unlock();
856 nx842_OF_upd(upd->prop);
857 } else
858 rcu_read_unlock();
859
860 return NOTIFY_OK;
861 }
862
863 static struct notifier_block nx842_of_nb = {
864 .notifier_call = nx842_OF_notifier,
865 };
866
867 #define nx842_counter_read(_name) \
868 static ssize_t nx842_##_name##_show(struct device *dev, \
869 struct device_attribute *attr, \
870 char *buf) { \
871 struct nx842_devdata *local_devdata; \
872 int p = 0; \
873 rcu_read_lock(); \
874 local_devdata = rcu_dereference(devdata); \
875 if (local_devdata) \
876 p = snprintf(buf, PAGE_SIZE, "%ld\n", \
877 atomic64_read(&local_devdata->counters->_name)); \
878 rcu_read_unlock(); \
879 return p; \
880 }
881
882 #define NX842DEV_COUNTER_ATTR_RO(_name) \
883 nx842_counter_read(_name); \
884 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
885 0444, \
886 nx842_##_name##_show,\
887 NULL);
888
889 NX842DEV_COUNTER_ATTR_RO(comp_complete);
890 NX842DEV_COUNTER_ATTR_RO(comp_failed);
891 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
892 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
893 NX842DEV_COUNTER_ATTR_RO(swdecomp);
894
895 static ssize_t nx842_timehist_show(struct device *,
896 struct device_attribute *, char *);
897
898 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
899 nx842_timehist_show, NULL);
900 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
901 0444, nx842_timehist_show, NULL);
902
nx842_timehist_show(struct device * dev,struct device_attribute * attr,char * buf)903 static ssize_t nx842_timehist_show(struct device *dev,
904 struct device_attribute *attr, char *buf) {
905 char *p = buf;
906 struct nx842_devdata *local_devdata;
907 atomic64_t *times;
908 int bytes_remain = PAGE_SIZE;
909 int bytes;
910 int i;
911
912 rcu_read_lock();
913 local_devdata = rcu_dereference(devdata);
914 if (!local_devdata) {
915 rcu_read_unlock();
916 return 0;
917 }
918
919 if (attr == &dev_attr_comp_times)
920 times = local_devdata->counters->comp_times;
921 else if (attr == &dev_attr_decomp_times)
922 times = local_devdata->counters->decomp_times;
923 else {
924 rcu_read_unlock();
925 return 0;
926 }
927
928 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
929 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
930 i ? (2<<(i-1)) : 0, (2<<i)-1,
931 atomic64_read(×[i]));
932 bytes_remain -= bytes;
933 p += bytes;
934 }
935 /* The last bucket holds everything over
936 * 2<<(NX842_HIST_SLOTS - 2) us */
937 bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
938 2<<(NX842_HIST_SLOTS - 2),
939 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
940 p += bytes;
941
942 rcu_read_unlock();
943 return p - buf;
944 }
945
946 static struct attribute *nx842_sysfs_entries[] = {
947 &dev_attr_comp_complete.attr,
948 &dev_attr_comp_failed.attr,
949 &dev_attr_decomp_complete.attr,
950 &dev_attr_decomp_failed.attr,
951 &dev_attr_swdecomp.attr,
952 &dev_attr_comp_times.attr,
953 &dev_attr_decomp_times.attr,
954 NULL,
955 };
956
957 static struct attribute_group nx842_attribute_group = {
958 .name = NULL, /* put in device directory */
959 .attrs = nx842_sysfs_entries,
960 };
961
962 static struct nx842_driver nx842_pseries_driver = {
963 .name = KBUILD_MODNAME,
964 .owner = THIS_MODULE,
965 .workmem_size = sizeof(struct nx842_workmem),
966 .constraints = &nx842_pseries_constraints,
967 .compress = nx842_pseries_compress,
968 .decompress = nx842_pseries_decompress,
969 };
970
nx842_pseries_crypto_init(struct crypto_tfm * tfm)971 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
972 {
973 return nx842_crypto_init(tfm, &nx842_pseries_driver);
974 }
975
976 static struct crypto_alg nx842_pseries_alg = {
977 .cra_name = "842",
978 .cra_driver_name = "842-nx",
979 .cra_priority = 300,
980 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
981 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
982 .cra_module = THIS_MODULE,
983 .cra_init = nx842_pseries_crypto_init,
984 .cra_exit = nx842_crypto_exit,
985 .cra_u = { .compress = {
986 .coa_compress = nx842_crypto_compress,
987 .coa_decompress = nx842_crypto_decompress } }
988 };
989
nx842_probe(struct vio_dev * viodev,const struct vio_device_id * id)990 static int nx842_probe(struct vio_dev *viodev,
991 const struct vio_device_id *id)
992 {
993 struct nx842_devdata *old_devdata, *new_devdata = NULL;
994 unsigned long flags;
995 int ret = 0;
996
997 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
998 if (!new_devdata)
999 return -ENOMEM;
1000
1001 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1002 GFP_NOFS);
1003 if (!new_devdata->counters) {
1004 kfree(new_devdata);
1005 return -ENOMEM;
1006 }
1007
1008 spin_lock_irqsave(&devdata_mutex, flags);
1009 old_devdata = rcu_dereference_check(devdata,
1010 lockdep_is_held(&devdata_mutex));
1011
1012 if (old_devdata && old_devdata->vdev != NULL) {
1013 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1014 ret = -1;
1015 goto error_unlock;
1016 }
1017
1018 dev_set_drvdata(&viodev->dev, NULL);
1019
1020 new_devdata->vdev = viodev;
1021 new_devdata->dev = &viodev->dev;
1022 nx842_OF_set_defaults(new_devdata);
1023
1024 rcu_assign_pointer(devdata, new_devdata);
1025 spin_unlock_irqrestore(&devdata_mutex, flags);
1026 synchronize_rcu();
1027 kfree(old_devdata);
1028
1029 of_reconfig_notifier_register(&nx842_of_nb);
1030
1031 ret = nx842_OF_upd(NULL);
1032 if (ret)
1033 goto error;
1034
1035 ret = crypto_register_alg(&nx842_pseries_alg);
1036 if (ret) {
1037 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1038 goto error;
1039 }
1040
1041 rcu_read_lock();
1042 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1043 rcu_read_unlock();
1044
1045 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1046 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1047 ret = -1;
1048 goto error;
1049 }
1050
1051 return 0;
1052
1053 error_unlock:
1054 spin_unlock_irqrestore(&devdata_mutex, flags);
1055 if (new_devdata)
1056 kfree(new_devdata->counters);
1057 kfree(new_devdata);
1058 error:
1059 return ret;
1060 }
1061
nx842_remove(struct vio_dev * viodev)1062 static int nx842_remove(struct vio_dev *viodev)
1063 {
1064 struct nx842_devdata *old_devdata;
1065 unsigned long flags;
1066
1067 pr_info("Removing IBM Power 842 compression device\n");
1068 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1069
1070 crypto_unregister_alg(&nx842_pseries_alg);
1071
1072 spin_lock_irqsave(&devdata_mutex, flags);
1073 old_devdata = rcu_dereference_check(devdata,
1074 lockdep_is_held(&devdata_mutex));
1075 of_reconfig_notifier_unregister(&nx842_of_nb);
1076 RCU_INIT_POINTER(devdata, NULL);
1077 spin_unlock_irqrestore(&devdata_mutex, flags);
1078 synchronize_rcu();
1079 dev_set_drvdata(&viodev->dev, NULL);
1080 if (old_devdata)
1081 kfree(old_devdata->counters);
1082 kfree(old_devdata);
1083
1084 return 0;
1085 }
1086
1087 static struct vio_device_id nx842_vio_driver_ids[] = {
1088 {"ibm,compression-v1", "ibm,compression"},
1089 {"", ""},
1090 };
1091 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1092
1093 static struct vio_driver nx842_vio_driver = {
1094 .name = KBUILD_MODNAME,
1095 .probe = nx842_probe,
1096 .remove = nx842_remove,
1097 .get_desired_dma = nx842_get_desired_dma,
1098 .id_table = nx842_vio_driver_ids,
1099 };
1100
nx842_pseries_init(void)1101 static int __init nx842_pseries_init(void)
1102 {
1103 struct nx842_devdata *new_devdata;
1104 int ret;
1105
1106 if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1107 return -ENODEV;
1108
1109 RCU_INIT_POINTER(devdata, NULL);
1110 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1111 if (!new_devdata) {
1112 pr_err("Could not allocate memory for device data\n");
1113 return -ENOMEM;
1114 }
1115 RCU_INIT_POINTER(devdata, new_devdata);
1116
1117 ret = vio_register_driver(&nx842_vio_driver);
1118 if (ret) {
1119 pr_err("Could not register VIO driver %d\n", ret);
1120
1121 kfree(new_devdata);
1122 return ret;
1123 }
1124
1125 return 0;
1126 }
1127
1128 module_init(nx842_pseries_init);
1129
nx842_pseries_exit(void)1130 static void __exit nx842_pseries_exit(void)
1131 {
1132 struct nx842_devdata *old_devdata;
1133 unsigned long flags;
1134
1135 crypto_unregister_alg(&nx842_pseries_alg);
1136
1137 spin_lock_irqsave(&devdata_mutex, flags);
1138 old_devdata = rcu_dereference_check(devdata,
1139 lockdep_is_held(&devdata_mutex));
1140 RCU_INIT_POINTER(devdata, NULL);
1141 spin_unlock_irqrestore(&devdata_mutex, flags);
1142 synchronize_rcu();
1143 if (old_devdata && old_devdata->dev)
1144 dev_set_drvdata(old_devdata->dev, NULL);
1145 kfree(old_devdata);
1146 vio_unregister_driver(&nx842_vio_driver);
1147 }
1148
1149 module_exit(nx842_pseries_exit);
1150
1151