1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Driver for IBM Power 842 compression accelerator
4 *
5 * Copyright (C) IBM Corporation, 2012
6 *
7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
8 * Seth Jennings <sjenning@linux.vnet.ibm.com>
9 */
10
11 #include <asm/vio.h>
12
13 #include "nx-842.h"
14 #include "nx_csbcpb.h" /* struct nx_csbcpb */
15
16 MODULE_LICENSE("GPL");
17 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
18 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
19 MODULE_ALIAS_CRYPTO("842");
20 MODULE_ALIAS_CRYPTO("842-nx");
21
22 static struct nx842_constraints nx842_pseries_constraints = {
23 .alignment = DDE_BUFFER_ALIGN,
24 .multiple = DDE_BUFFER_LAST_MULT,
25 .minimum = DDE_BUFFER_LAST_MULT,
26 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
27 };
28
check_constraints(unsigned long buf,unsigned int * len,bool in)29 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
30 {
31 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
32 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
33 in ? "input" : "output", buf,
34 nx842_pseries_constraints.alignment);
35 return -EINVAL;
36 }
37 if (*len % nx842_pseries_constraints.multiple) {
38 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
39 in ? "input" : "output", *len,
40 nx842_pseries_constraints.multiple);
41 if (in)
42 return -EINVAL;
43 *len = round_down(*len, nx842_pseries_constraints.multiple);
44 }
45 if (*len < nx842_pseries_constraints.minimum) {
46 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
47 in ? "input" : "output", *len,
48 nx842_pseries_constraints.minimum);
49 return -EINVAL;
50 }
51 if (*len > nx842_pseries_constraints.maximum) {
52 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
53 in ? "input" : "output", *len,
54 nx842_pseries_constraints.maximum);
55 if (in)
56 return -EINVAL;
57 *len = nx842_pseries_constraints.maximum;
58 }
59 return 0;
60 }
61
62 /* I assume we need to align the CSB? */
63 #define WORKMEM_ALIGN (256)
64
65 struct nx842_workmem {
66 /* scatterlist */
67 char slin[4096];
68 char slout[4096];
69 /* coprocessor status/parameter block */
70 struct nx_csbcpb csbcpb;
71
72 char padding[WORKMEM_ALIGN];
73 } __aligned(WORKMEM_ALIGN);
74
75 /* Macros for fields within nx_csbcpb */
76 /* Check the valid bit within the csbcpb valid field */
77 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
78
79 /* CE macros operate on the completion_extension field bits in the csbcpb.
80 * CE0 0=full completion, 1=partial completion
81 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
82 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
83 #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
84 #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
85 #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
86
87 /* The NX unit accepts data only on 4K page boundaries */
88 #define NX842_HW_PAGE_SIZE (4096)
89 #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
90
91 struct ibm_nx842_counters {
92 atomic64_t comp_complete;
93 atomic64_t comp_failed;
94 atomic64_t decomp_complete;
95 atomic64_t decomp_failed;
96 atomic64_t swdecomp;
97 atomic64_t comp_times[32];
98 atomic64_t decomp_times[32];
99 };
100
101 static struct nx842_devdata {
102 struct vio_dev *vdev;
103 struct device *dev;
104 struct ibm_nx842_counters *counters;
105 unsigned int max_sg_len;
106 unsigned int max_sync_size;
107 unsigned int max_sync_sg;
108 } __rcu *devdata;
109 static DEFINE_SPINLOCK(devdata_mutex);
110
111 #define NX842_COUNTER_INC(_x) \
112 static inline void nx842_inc_##_x( \
113 const struct nx842_devdata *dev) { \
114 if (dev) \
115 atomic64_inc(&dev->counters->_x); \
116 }
117 NX842_COUNTER_INC(comp_complete);
118 NX842_COUNTER_INC(comp_failed);
119 NX842_COUNTER_INC(decomp_complete);
120 NX842_COUNTER_INC(decomp_failed);
121 NX842_COUNTER_INC(swdecomp);
122
123 #define NX842_HIST_SLOTS 16
124
ibm_nx842_incr_hist(atomic64_t * times,unsigned int time)125 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
126 {
127 int bucket = fls(time);
128
129 if (bucket)
130 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
131
132 atomic64_inc(×[bucket]);
133 }
134
135 /* NX unit operation flags */
136 #define NX842_OP_COMPRESS 0x0
137 #define NX842_OP_CRC 0x1
138 #define NX842_OP_DECOMPRESS 0x2
139 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
140 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
141 #define NX842_OP_ASYNC (1<<23)
142 #define NX842_OP_NOTIFY (1<<22)
143 #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
144
nx842_get_desired_dma(struct vio_dev * viodev)145 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
146 {
147 /* No use of DMA mappings within the driver. */
148 return 0;
149 }
150
151 struct nx842_slentry {
152 __be64 ptr; /* Real address (use __pa()) */
153 __be64 len;
154 };
155
156 /* pHyp scatterlist entry */
157 struct nx842_scatterlist {
158 int entry_nr; /* number of slentries */
159 struct nx842_slentry *entries; /* ptr to array of slentries */
160 };
161
162 /* Does not include sizeof(entry_nr) in the size */
nx842_get_scatterlist_size(struct nx842_scatterlist * sl)163 static inline unsigned long nx842_get_scatterlist_size(
164 struct nx842_scatterlist *sl)
165 {
166 return sl->entry_nr * sizeof(struct nx842_slentry);
167 }
168
nx842_build_scatterlist(unsigned long buf,int len,struct nx842_scatterlist * sl)169 static int nx842_build_scatterlist(unsigned long buf, int len,
170 struct nx842_scatterlist *sl)
171 {
172 unsigned long entrylen;
173 struct nx842_slentry *entry;
174
175 sl->entry_nr = 0;
176
177 entry = sl->entries;
178 while (len) {
179 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
180 entrylen = min_t(int, len,
181 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
182 entry->len = cpu_to_be64(entrylen);
183
184 len -= entrylen;
185 buf += entrylen;
186
187 sl->entry_nr++;
188 entry++;
189 }
190
191 return 0;
192 }
193
nx842_validate_result(struct device * dev,struct cop_status_block * csb)194 static int nx842_validate_result(struct device *dev,
195 struct cop_status_block *csb)
196 {
197 /* The csb must be valid after returning from vio_h_cop_sync */
198 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
199 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
200 __func__);
201 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
202 csb->valid,
203 csb->crb_seq_number,
204 csb->completion_code,
205 csb->completion_extension);
206 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
207 be32_to_cpu(csb->processed_byte_count),
208 (unsigned long)be64_to_cpu(csb->address));
209 return -EIO;
210 }
211
212 /* Check return values from the hardware in the CSB */
213 switch (csb->completion_code) {
214 case 0: /* Completed without error */
215 break;
216 case 64: /* Compression ok, but output larger than input */
217 dev_dbg(dev, "%s: output size larger than input size\n",
218 __func__);
219 break;
220 case 13: /* Output buffer too small */
221 dev_dbg(dev, "%s: Out of space in output buffer\n",
222 __func__);
223 return -ENOSPC;
224 case 65: /* Calculated CRC doesn't match the passed value */
225 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
226 __func__);
227 return -EINVAL;
228 case 66: /* Input data contains an illegal template field */
229 case 67: /* Template indicates data past the end of the input stream */
230 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
231 __func__, csb->completion_code);
232 return -EINVAL;
233 default:
234 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
235 __func__, csb->completion_code);
236 return -EIO;
237 }
238
239 /* Hardware sanity check */
240 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
241 dev_err(dev, "%s: No error returned by hardware, but "
242 "data returned is unusable, contact support.\n"
243 "(Additional info: csbcbp->processed bytes "
244 "does not specify processed bytes for the "
245 "target buffer.)\n", __func__);
246 return -EIO;
247 }
248
249 return 0;
250 }
251
252 /**
253 * nx842_pseries_compress - Compress data using the 842 algorithm
254 *
255 * Compression provide by the NX842 coprocessor on IBM Power systems.
256 * The input buffer is compressed and the result is stored in the
257 * provided output buffer.
258 *
259 * Upon return from this function @outlen contains the length of the
260 * compressed data. If there is an error then @outlen will be 0 and an
261 * error will be specified by the return code from this function.
262 *
263 * @in: Pointer to input buffer
264 * @inlen: Length of input buffer
265 * @out: Pointer to output buffer
266 * @outlen: Length of output buffer
267 * @wrkmem: ptr to buffer for working memory, size determined by
268 * nx842_pseries_driver.workmem_size
269 *
270 * Returns:
271 * 0 Success, output of length @outlen stored in the buffer at @out
272 * -ENOMEM Unable to allocate internal buffers
273 * -ENOSPC Output buffer is to small
274 * -EIO Internal error
275 * -ENODEV Hardware unavailable
276 */
nx842_pseries_compress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)277 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
278 unsigned char *out, unsigned int *outlen,
279 void *wmem)
280 {
281 struct nx842_devdata *local_devdata;
282 struct device *dev = NULL;
283 struct nx842_workmem *workmem;
284 struct nx842_scatterlist slin, slout;
285 struct nx_csbcpb *csbcpb;
286 int ret = 0;
287 unsigned long inbuf, outbuf;
288 struct vio_pfo_op op = {
289 .done = NULL,
290 .handle = 0,
291 .timeout = 0,
292 };
293 unsigned long start = get_tb();
294
295 inbuf = (unsigned long)in;
296 if (check_constraints(inbuf, &inlen, true))
297 return -EINVAL;
298
299 outbuf = (unsigned long)out;
300 if (check_constraints(outbuf, outlen, false))
301 return -EINVAL;
302
303 rcu_read_lock();
304 local_devdata = rcu_dereference(devdata);
305 if (!local_devdata || !local_devdata->dev) {
306 rcu_read_unlock();
307 return -ENODEV;
308 }
309 dev = local_devdata->dev;
310
311 /* Init scatterlist */
312 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
313 slin.entries = (struct nx842_slentry *)workmem->slin;
314 slout.entries = (struct nx842_slentry *)workmem->slout;
315
316 /* Init operation */
317 op.flags = NX842_OP_COMPRESS_CRC;
318 csbcpb = &workmem->csbcpb;
319 memset(csbcpb, 0, sizeof(*csbcpb));
320 op.csbcpb = nx842_get_pa(csbcpb);
321
322 if ((inbuf & NX842_HW_PAGE_MASK) ==
323 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
324 /* Create direct DDE */
325 op.in = nx842_get_pa((void *)inbuf);
326 op.inlen = inlen;
327 } else {
328 /* Create indirect DDE (scatterlist) */
329 nx842_build_scatterlist(inbuf, inlen, &slin);
330 op.in = nx842_get_pa(slin.entries);
331 op.inlen = -nx842_get_scatterlist_size(&slin);
332 }
333
334 if ((outbuf & NX842_HW_PAGE_MASK) ==
335 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
336 /* Create direct DDE */
337 op.out = nx842_get_pa((void *)outbuf);
338 op.outlen = *outlen;
339 } else {
340 /* Create indirect DDE (scatterlist) */
341 nx842_build_scatterlist(outbuf, *outlen, &slout);
342 op.out = nx842_get_pa(slout.entries);
343 op.outlen = -nx842_get_scatterlist_size(&slout);
344 }
345
346 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
347 __func__, (unsigned long)op.in, (long)op.inlen,
348 (unsigned long)op.out, (long)op.outlen);
349
350 /* Send request to pHyp */
351 ret = vio_h_cop_sync(local_devdata->vdev, &op);
352
353 /* Check for pHyp error */
354 if (ret) {
355 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
356 __func__, ret, op.hcall_err);
357 ret = -EIO;
358 goto unlock;
359 }
360
361 /* Check for hardware error */
362 ret = nx842_validate_result(dev, &csbcpb->csb);
363 if (ret)
364 goto unlock;
365
366 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
367 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
368
369 unlock:
370 if (ret)
371 nx842_inc_comp_failed(local_devdata);
372 else {
373 nx842_inc_comp_complete(local_devdata);
374 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
375 (get_tb() - start) / tb_ticks_per_usec);
376 }
377 rcu_read_unlock();
378 return ret;
379 }
380
381 /**
382 * nx842_pseries_decompress - Decompress data using the 842 algorithm
383 *
384 * Decompression provide by the NX842 coprocessor on IBM Power systems.
385 * The input buffer is decompressed and the result is stored in the
386 * provided output buffer. The size allocated to the output buffer is
387 * provided by the caller of this function in @outlen. Upon return from
388 * this function @outlen contains the length of the decompressed data.
389 * If there is an error then @outlen will be 0 and an error will be
390 * specified by the return code from this function.
391 *
392 * @in: Pointer to input buffer
393 * @inlen: Length of input buffer
394 * @out: Pointer to output buffer
395 * @outlen: Length of output buffer
396 * @wrkmem: ptr to buffer for working memory, size determined by
397 * nx842_pseries_driver.workmem_size
398 *
399 * Returns:
400 * 0 Success, output of length @outlen stored in the buffer at @out
401 * -ENODEV Hardware decompression device is unavailable
402 * -ENOMEM Unable to allocate internal buffers
403 * -ENOSPC Output buffer is to small
404 * -EINVAL Bad input data encountered when attempting decompress
405 * -EIO Internal error
406 */
nx842_pseries_decompress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)407 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
408 unsigned char *out, unsigned int *outlen,
409 void *wmem)
410 {
411 struct nx842_devdata *local_devdata;
412 struct device *dev = NULL;
413 struct nx842_workmem *workmem;
414 struct nx842_scatterlist slin, slout;
415 struct nx_csbcpb *csbcpb;
416 int ret = 0;
417 unsigned long inbuf, outbuf;
418 struct vio_pfo_op op = {
419 .done = NULL,
420 .handle = 0,
421 .timeout = 0,
422 };
423 unsigned long start = get_tb();
424
425 /* Ensure page alignment and size */
426 inbuf = (unsigned long)in;
427 if (check_constraints(inbuf, &inlen, true))
428 return -EINVAL;
429
430 outbuf = (unsigned long)out;
431 if (check_constraints(outbuf, outlen, false))
432 return -EINVAL;
433
434 rcu_read_lock();
435 local_devdata = rcu_dereference(devdata);
436 if (!local_devdata || !local_devdata->dev) {
437 rcu_read_unlock();
438 return -ENODEV;
439 }
440 dev = local_devdata->dev;
441
442 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
443
444 /* Init scatterlist */
445 slin.entries = (struct nx842_slentry *)workmem->slin;
446 slout.entries = (struct nx842_slentry *)workmem->slout;
447
448 /* Init operation */
449 op.flags = NX842_OP_DECOMPRESS_CRC;
450 csbcpb = &workmem->csbcpb;
451 memset(csbcpb, 0, sizeof(*csbcpb));
452 op.csbcpb = nx842_get_pa(csbcpb);
453
454 if ((inbuf & NX842_HW_PAGE_MASK) ==
455 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
456 /* Create direct DDE */
457 op.in = nx842_get_pa((void *)inbuf);
458 op.inlen = inlen;
459 } else {
460 /* Create indirect DDE (scatterlist) */
461 nx842_build_scatterlist(inbuf, inlen, &slin);
462 op.in = nx842_get_pa(slin.entries);
463 op.inlen = -nx842_get_scatterlist_size(&slin);
464 }
465
466 if ((outbuf & NX842_HW_PAGE_MASK) ==
467 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
468 /* Create direct DDE */
469 op.out = nx842_get_pa((void *)outbuf);
470 op.outlen = *outlen;
471 } else {
472 /* Create indirect DDE (scatterlist) */
473 nx842_build_scatterlist(outbuf, *outlen, &slout);
474 op.out = nx842_get_pa(slout.entries);
475 op.outlen = -nx842_get_scatterlist_size(&slout);
476 }
477
478 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
479 __func__, (unsigned long)op.in, (long)op.inlen,
480 (unsigned long)op.out, (long)op.outlen);
481
482 /* Send request to pHyp */
483 ret = vio_h_cop_sync(local_devdata->vdev, &op);
484
485 /* Check for pHyp error */
486 if (ret) {
487 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
488 __func__, ret, op.hcall_err);
489 goto unlock;
490 }
491
492 /* Check for hardware error */
493 ret = nx842_validate_result(dev, &csbcpb->csb);
494 if (ret)
495 goto unlock;
496
497 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
498
499 unlock:
500 if (ret)
501 /* decompress fail */
502 nx842_inc_decomp_failed(local_devdata);
503 else {
504 nx842_inc_decomp_complete(local_devdata);
505 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
506 (get_tb() - start) / tb_ticks_per_usec);
507 }
508
509 rcu_read_unlock();
510 return ret;
511 }
512
513 /**
514 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
515 *
516 * @devdata - struct nx842_devdata to update
517 *
518 * Returns:
519 * 0 on success
520 * -ENOENT if @devdata ptr is NULL
521 */
nx842_OF_set_defaults(struct nx842_devdata * devdata)522 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
523 {
524 if (devdata) {
525 devdata->max_sync_size = 0;
526 devdata->max_sync_sg = 0;
527 devdata->max_sg_len = 0;
528 return 0;
529 } else
530 return -ENOENT;
531 }
532
533 /**
534 * nx842_OF_upd_status -- Check the device info from OF status prop
535 *
536 * The status property indicates if the accelerator is enabled. If the
537 * device is in the OF tree it indicates that the hardware is present.
538 * The status field indicates if the device is enabled when the status
539 * is 'okay'. Otherwise the device driver will be disabled.
540 *
541 * @devdata: struct nx842_devdata to use for dev_info
542 * @prop: struct property point containing the maxsyncop for the update
543 *
544 * Returns:
545 * 0 - Device is available
546 * -ENODEV - Device is not available
547 */
nx842_OF_upd_status(struct nx842_devdata * devdata,struct property * prop)548 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
549 struct property *prop)
550 {
551 const char *status = (const char *)prop->value;
552
553 if (!strncmp(status, "okay", (size_t)prop->length))
554 return 0;
555 if (!strncmp(status, "disabled", (size_t)prop->length))
556 return -ENODEV;
557 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
558
559 return -EINVAL;
560 }
561
562 /**
563 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
564 *
565 * Definition of the 'ibm,max-sg-len' OF property:
566 * This field indicates the maximum byte length of a scatter list
567 * for the platform facility. It is a single cell encoded as with encode-int.
568 *
569 * Example:
570 * # od -x ibm,max-sg-len
571 * 0000000 0000 0ff0
572 *
573 * In this example, the maximum byte length of a scatter list is
574 * 0x0ff0 (4,080).
575 *
576 * @devdata - struct nx842_devdata to update
577 * @prop - struct property point containing the maxsyncop for the update
578 *
579 * Returns:
580 * 0 on success
581 * -EINVAL on failure
582 */
nx842_OF_upd_maxsglen(struct nx842_devdata * devdata,struct property * prop)583 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
584 struct property *prop) {
585 int ret = 0;
586 const unsigned int maxsglen = of_read_number(prop->value, 1);
587
588 if (prop->length != sizeof(maxsglen)) {
589 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
590 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
591 prop->length, sizeof(maxsglen));
592 ret = -EINVAL;
593 } else {
594 devdata->max_sg_len = min_t(unsigned int,
595 maxsglen, NX842_HW_PAGE_SIZE);
596 }
597
598 return ret;
599 }
600
601 /**
602 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
603 *
604 * Definition of the 'ibm,max-sync-cop' OF property:
605 * Two series of cells. The first series of cells represents the maximums
606 * that can be synchronously compressed. The second series of cells
607 * represents the maximums that can be synchronously decompressed.
608 * 1. The first cell in each series contains the count of the number of
609 * data length, scatter list elements pairs that follow – each being
610 * of the form
611 * a. One cell data byte length
612 * b. One cell total number of scatter list elements
613 *
614 * Example:
615 * # od -x ibm,max-sync-cop
616 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
617 * 0000020 0000 1000 0000 01fe
618 *
619 * In this example, compression supports 0x1000 (4,096) data byte length
620 * and 0x1fe (510) total scatter list elements. Decompression supports
621 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
622 * elements.
623 *
624 * @devdata - struct nx842_devdata to update
625 * @prop - struct property point containing the maxsyncop for the update
626 *
627 * Returns:
628 * 0 on success
629 * -EINVAL on failure
630 */
nx842_OF_upd_maxsyncop(struct nx842_devdata * devdata,struct property * prop)631 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
632 struct property *prop) {
633 int ret = 0;
634 unsigned int comp_data_limit, decomp_data_limit;
635 unsigned int comp_sg_limit, decomp_sg_limit;
636 const struct maxsynccop_t {
637 __be32 comp_elements;
638 __be32 comp_data_limit;
639 __be32 comp_sg_limit;
640 __be32 decomp_elements;
641 __be32 decomp_data_limit;
642 __be32 decomp_sg_limit;
643 } *maxsynccop;
644
645 if (prop->length != sizeof(*maxsynccop)) {
646 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
647 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
648 sizeof(*maxsynccop));
649 ret = -EINVAL;
650 goto out;
651 }
652
653 maxsynccop = (const struct maxsynccop_t *)prop->value;
654 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
655 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
656 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
657 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
658
659 /* Use one limit rather than separate limits for compression and
660 * decompression. Set a maximum for this so as not to exceed the
661 * size that the header can support and round the value down to
662 * the hardware page size (4K) */
663 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
664
665 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
666 65536);
667
668 if (devdata->max_sync_size < 4096) {
669 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
670 "less than the driver minimum, unable to use "
671 "the hardware device\n",
672 __func__, devdata->max_sync_size);
673 ret = -EINVAL;
674 goto out;
675 }
676
677 nx842_pseries_constraints.maximum = devdata->max_sync_size;
678
679 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
680 if (devdata->max_sync_sg < 1) {
681 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
682 "less than the driver minimum, unable to use "
683 "the hardware device\n",
684 __func__, devdata->max_sync_sg);
685 ret = -EINVAL;
686 goto out;
687 }
688
689 out:
690 return ret;
691 }
692
693 /**
694 *
695 * nx842_OF_upd -- Handle OF properties updates for the device.
696 *
697 * Set all properties from the OF tree. Optionally, a new property
698 * can be provided by the @new_prop pointer to overwrite an existing value.
699 * The device will remain disabled until all values are valid, this function
700 * will return an error for updates unless all values are valid.
701 *
702 * @new_prop: If not NULL, this property is being updated. If NULL, update
703 * all properties from the current values in the OF tree.
704 *
705 * Returns:
706 * 0 - Success
707 * -ENOMEM - Could not allocate memory for new devdata structure
708 * -EINVAL - property value not found, new_prop is not a recognized
709 * property for the device or property value is not valid.
710 * -ENODEV - Device is not available
711 */
nx842_OF_upd(struct property * new_prop)712 static int nx842_OF_upd(struct property *new_prop)
713 {
714 struct nx842_devdata *old_devdata = NULL;
715 struct nx842_devdata *new_devdata = NULL;
716 struct device_node *of_node = NULL;
717 struct property *status = NULL;
718 struct property *maxsglen = NULL;
719 struct property *maxsyncop = NULL;
720 int ret = 0;
721 unsigned long flags;
722
723 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
724 if (!new_devdata)
725 return -ENOMEM;
726
727 spin_lock_irqsave(&devdata_mutex, flags);
728 old_devdata = rcu_dereference_check(devdata,
729 lockdep_is_held(&devdata_mutex));
730 if (old_devdata)
731 of_node = old_devdata->dev->of_node;
732
733 if (!old_devdata || !of_node) {
734 pr_err("%s: device is not available\n", __func__);
735 spin_unlock_irqrestore(&devdata_mutex, flags);
736 kfree(new_devdata);
737 return -ENODEV;
738 }
739
740 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
741 new_devdata->counters = old_devdata->counters;
742
743 /* Set ptrs for existing properties */
744 status = of_find_property(of_node, "status", NULL);
745 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
746 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
747 if (!status || !maxsglen || !maxsyncop) {
748 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
749 ret = -EINVAL;
750 goto error_out;
751 }
752
753 /*
754 * If this is a property update, there are only certain properties that
755 * we care about. Bail if it isn't in the below list
756 */
757 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
758 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
759 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
760 goto out;
761
762 /* Perform property updates */
763 ret = nx842_OF_upd_status(new_devdata, status);
764 if (ret)
765 goto error_out;
766
767 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
768 if (ret)
769 goto error_out;
770
771 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
772 if (ret)
773 goto error_out;
774
775 out:
776 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
777 __func__, new_devdata->max_sync_size,
778 old_devdata->max_sync_size);
779 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
780 __func__, new_devdata->max_sync_sg,
781 old_devdata->max_sync_sg);
782 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
783 __func__, new_devdata->max_sg_len,
784 old_devdata->max_sg_len);
785
786 rcu_assign_pointer(devdata, new_devdata);
787 spin_unlock_irqrestore(&devdata_mutex, flags);
788 synchronize_rcu();
789 dev_set_drvdata(new_devdata->dev, new_devdata);
790 kfree(old_devdata);
791 return 0;
792
793 error_out:
794 if (new_devdata) {
795 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
796 nx842_OF_set_defaults(new_devdata);
797 rcu_assign_pointer(devdata, new_devdata);
798 spin_unlock_irqrestore(&devdata_mutex, flags);
799 synchronize_rcu();
800 dev_set_drvdata(new_devdata->dev, new_devdata);
801 kfree(old_devdata);
802 } else {
803 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
804 spin_unlock_irqrestore(&devdata_mutex, flags);
805 }
806
807 if (!ret)
808 ret = -EINVAL;
809 return ret;
810 }
811
812 /**
813 * nx842_OF_notifier - Process updates to OF properties for the device
814 *
815 * @np: notifier block
816 * @action: notifier action
817 * @update: struct pSeries_reconfig_prop_update pointer if action is
818 * PSERIES_UPDATE_PROPERTY
819 *
820 * Returns:
821 * NOTIFY_OK on success
822 * NOTIFY_BAD encoded with error number on failure, use
823 * notifier_to_errno() to decode this value
824 */
nx842_OF_notifier(struct notifier_block * np,unsigned long action,void * data)825 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
826 void *data)
827 {
828 struct of_reconfig_data *upd = data;
829 struct nx842_devdata *local_devdata;
830 struct device_node *node = NULL;
831
832 rcu_read_lock();
833 local_devdata = rcu_dereference(devdata);
834 if (local_devdata)
835 node = local_devdata->dev->of_node;
836
837 if (local_devdata &&
838 action == OF_RECONFIG_UPDATE_PROPERTY &&
839 !strcmp(upd->dn->name, node->name)) {
840 rcu_read_unlock();
841 nx842_OF_upd(upd->prop);
842 } else
843 rcu_read_unlock();
844
845 return NOTIFY_OK;
846 }
847
848 static struct notifier_block nx842_of_nb = {
849 .notifier_call = nx842_OF_notifier,
850 };
851
852 #define nx842_counter_read(_name) \
853 static ssize_t nx842_##_name##_show(struct device *dev, \
854 struct device_attribute *attr, \
855 char *buf) { \
856 struct nx842_devdata *local_devdata; \
857 int p = 0; \
858 rcu_read_lock(); \
859 local_devdata = rcu_dereference(devdata); \
860 if (local_devdata) \
861 p = snprintf(buf, PAGE_SIZE, "%lld\n", \
862 atomic64_read(&local_devdata->counters->_name)); \
863 rcu_read_unlock(); \
864 return p; \
865 }
866
867 #define NX842DEV_COUNTER_ATTR_RO(_name) \
868 nx842_counter_read(_name); \
869 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
870 0444, \
871 nx842_##_name##_show,\
872 NULL);
873
874 NX842DEV_COUNTER_ATTR_RO(comp_complete);
875 NX842DEV_COUNTER_ATTR_RO(comp_failed);
876 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
877 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
878 NX842DEV_COUNTER_ATTR_RO(swdecomp);
879
880 static ssize_t nx842_timehist_show(struct device *,
881 struct device_attribute *, char *);
882
883 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
884 nx842_timehist_show, NULL);
885 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
886 0444, nx842_timehist_show, NULL);
887
nx842_timehist_show(struct device * dev,struct device_attribute * attr,char * buf)888 static ssize_t nx842_timehist_show(struct device *dev,
889 struct device_attribute *attr, char *buf) {
890 char *p = buf;
891 struct nx842_devdata *local_devdata;
892 atomic64_t *times;
893 int bytes_remain = PAGE_SIZE;
894 int bytes;
895 int i;
896
897 rcu_read_lock();
898 local_devdata = rcu_dereference(devdata);
899 if (!local_devdata) {
900 rcu_read_unlock();
901 return 0;
902 }
903
904 if (attr == &dev_attr_comp_times)
905 times = local_devdata->counters->comp_times;
906 else if (attr == &dev_attr_decomp_times)
907 times = local_devdata->counters->decomp_times;
908 else {
909 rcu_read_unlock();
910 return 0;
911 }
912
913 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
914 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
915 i ? (2<<(i-1)) : 0, (2<<i)-1,
916 atomic64_read(×[i]));
917 bytes_remain -= bytes;
918 p += bytes;
919 }
920 /* The last bucket holds everything over
921 * 2<<(NX842_HIST_SLOTS - 2) us */
922 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
923 2<<(NX842_HIST_SLOTS - 2),
924 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
925 p += bytes;
926
927 rcu_read_unlock();
928 return p - buf;
929 }
930
931 static struct attribute *nx842_sysfs_entries[] = {
932 &dev_attr_comp_complete.attr,
933 &dev_attr_comp_failed.attr,
934 &dev_attr_decomp_complete.attr,
935 &dev_attr_decomp_failed.attr,
936 &dev_attr_swdecomp.attr,
937 &dev_attr_comp_times.attr,
938 &dev_attr_decomp_times.attr,
939 NULL,
940 };
941
942 static struct attribute_group nx842_attribute_group = {
943 .name = NULL, /* put in device directory */
944 .attrs = nx842_sysfs_entries,
945 };
946
947 static struct nx842_driver nx842_pseries_driver = {
948 .name = KBUILD_MODNAME,
949 .owner = THIS_MODULE,
950 .workmem_size = sizeof(struct nx842_workmem),
951 .constraints = &nx842_pseries_constraints,
952 .compress = nx842_pseries_compress,
953 .decompress = nx842_pseries_decompress,
954 };
955
nx842_pseries_crypto_init(struct crypto_tfm * tfm)956 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
957 {
958 return nx842_crypto_init(tfm, &nx842_pseries_driver);
959 }
960
961 static struct crypto_alg nx842_pseries_alg = {
962 .cra_name = "842",
963 .cra_driver_name = "842-nx",
964 .cra_priority = 300,
965 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
966 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
967 .cra_module = THIS_MODULE,
968 .cra_init = nx842_pseries_crypto_init,
969 .cra_exit = nx842_crypto_exit,
970 .cra_u = { .compress = {
971 .coa_compress = nx842_crypto_compress,
972 .coa_decompress = nx842_crypto_decompress } }
973 };
974
nx842_probe(struct vio_dev * viodev,const struct vio_device_id * id)975 static int nx842_probe(struct vio_dev *viodev,
976 const struct vio_device_id *id)
977 {
978 struct nx842_devdata *old_devdata, *new_devdata = NULL;
979 unsigned long flags;
980 int ret = 0;
981
982 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
983 if (!new_devdata)
984 return -ENOMEM;
985
986 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
987 GFP_NOFS);
988 if (!new_devdata->counters) {
989 kfree(new_devdata);
990 return -ENOMEM;
991 }
992
993 spin_lock_irqsave(&devdata_mutex, flags);
994 old_devdata = rcu_dereference_check(devdata,
995 lockdep_is_held(&devdata_mutex));
996
997 if (old_devdata && old_devdata->vdev != NULL) {
998 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
999 ret = -1;
1000 goto error_unlock;
1001 }
1002
1003 dev_set_drvdata(&viodev->dev, NULL);
1004
1005 new_devdata->vdev = viodev;
1006 new_devdata->dev = &viodev->dev;
1007 nx842_OF_set_defaults(new_devdata);
1008
1009 rcu_assign_pointer(devdata, new_devdata);
1010 spin_unlock_irqrestore(&devdata_mutex, flags);
1011 synchronize_rcu();
1012 kfree(old_devdata);
1013
1014 of_reconfig_notifier_register(&nx842_of_nb);
1015
1016 ret = nx842_OF_upd(NULL);
1017 if (ret)
1018 goto error;
1019
1020 ret = crypto_register_alg(&nx842_pseries_alg);
1021 if (ret) {
1022 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1023 goto error;
1024 }
1025
1026 rcu_read_lock();
1027 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1028 rcu_read_unlock();
1029
1030 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1031 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1032 ret = -1;
1033 goto error;
1034 }
1035
1036 return 0;
1037
1038 error_unlock:
1039 spin_unlock_irqrestore(&devdata_mutex, flags);
1040 if (new_devdata)
1041 kfree(new_devdata->counters);
1042 kfree(new_devdata);
1043 error:
1044 return ret;
1045 }
1046
nx842_remove(struct vio_dev * viodev)1047 static int nx842_remove(struct vio_dev *viodev)
1048 {
1049 struct nx842_devdata *old_devdata;
1050 unsigned long flags;
1051
1052 pr_info("Removing IBM Power 842 compression device\n");
1053 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1054
1055 crypto_unregister_alg(&nx842_pseries_alg);
1056
1057 spin_lock_irqsave(&devdata_mutex, flags);
1058 old_devdata = rcu_dereference_check(devdata,
1059 lockdep_is_held(&devdata_mutex));
1060 of_reconfig_notifier_unregister(&nx842_of_nb);
1061 RCU_INIT_POINTER(devdata, NULL);
1062 spin_unlock_irqrestore(&devdata_mutex, flags);
1063 synchronize_rcu();
1064 dev_set_drvdata(&viodev->dev, NULL);
1065 if (old_devdata)
1066 kfree(old_devdata->counters);
1067 kfree(old_devdata);
1068
1069 return 0;
1070 }
1071
1072 static const struct vio_device_id nx842_vio_driver_ids[] = {
1073 {"ibm,compression-v1", "ibm,compression"},
1074 {"", ""},
1075 };
1076 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1077
1078 static struct vio_driver nx842_vio_driver = {
1079 .name = KBUILD_MODNAME,
1080 .probe = nx842_probe,
1081 .remove = nx842_remove,
1082 .get_desired_dma = nx842_get_desired_dma,
1083 .id_table = nx842_vio_driver_ids,
1084 };
1085
nx842_pseries_init(void)1086 static int __init nx842_pseries_init(void)
1087 {
1088 struct nx842_devdata *new_devdata;
1089 int ret;
1090
1091 if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1092 return -ENODEV;
1093
1094 RCU_INIT_POINTER(devdata, NULL);
1095 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1096 if (!new_devdata)
1097 return -ENOMEM;
1098
1099 RCU_INIT_POINTER(devdata, new_devdata);
1100
1101 ret = vio_register_driver(&nx842_vio_driver);
1102 if (ret) {
1103 pr_err("Could not register VIO driver %d\n", ret);
1104
1105 kfree(new_devdata);
1106 return ret;
1107 }
1108
1109 return 0;
1110 }
1111
1112 module_init(nx842_pseries_init);
1113
nx842_pseries_exit(void)1114 static void __exit nx842_pseries_exit(void)
1115 {
1116 struct nx842_devdata *old_devdata;
1117 unsigned long flags;
1118
1119 crypto_unregister_alg(&nx842_pseries_alg);
1120
1121 spin_lock_irqsave(&devdata_mutex, flags);
1122 old_devdata = rcu_dereference_check(devdata,
1123 lockdep_is_held(&devdata_mutex));
1124 RCU_INIT_POINTER(devdata, NULL);
1125 spin_unlock_irqrestore(&devdata_mutex, flags);
1126 synchronize_rcu();
1127 if (old_devdata && old_devdata->dev)
1128 dev_set_drvdata(old_devdata->dev, NULL);
1129 kfree(old_devdata);
1130 vio_unregister_driver(&nx842_vio_driver);
1131 }
1132
1133 module_exit(nx842_pseries_exit);
1134
1135