1 /*
2 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
3 *
4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <linux/cpufeature.h>
12 #include <linux/crc32.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/string.h>
17
18 #include <crypto/internal/hash.h>
19
20 #include <asm/hwcap.h>
21 #include <asm/neon.h>
22 #include <asm/simd.h>
23 #include <asm/unaligned.h>
24
25 #define PMULL_MIN_LEN 64L /* minimum size of buffer
26 * for crc32_pmull_le_16 */
27 #define SCALE_F 16L /* size of NEON register */
28
29 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
30 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
31
32 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
33 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
34
35 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
36 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
37
crc32_cra_init(struct crypto_tfm * tfm)38 static int crc32_cra_init(struct crypto_tfm *tfm)
39 {
40 u32 *key = crypto_tfm_ctx(tfm);
41
42 *key = 0;
43 return 0;
44 }
45
crc32c_cra_init(struct crypto_tfm * tfm)46 static int crc32c_cra_init(struct crypto_tfm *tfm)
47 {
48 u32 *key = crypto_tfm_ctx(tfm);
49
50 *key = ~0;
51 return 0;
52 }
53
crc32_setkey(struct crypto_shash * hash,const u8 * key,unsigned int keylen)54 static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
55 unsigned int keylen)
56 {
57 u32 *mctx = crypto_shash_ctx(hash);
58
59 if (keylen != sizeof(u32)) {
60 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
61 return -EINVAL;
62 }
63 *mctx = le32_to_cpup((__le32 *)key);
64 return 0;
65 }
66
crc32_init(struct shash_desc * desc)67 static int crc32_init(struct shash_desc *desc)
68 {
69 u32 *mctx = crypto_shash_ctx(desc->tfm);
70 u32 *crc = shash_desc_ctx(desc);
71
72 *crc = *mctx;
73 return 0;
74 }
75
crc32_update(struct shash_desc * desc,const u8 * data,unsigned int length)76 static int crc32_update(struct shash_desc *desc, const u8 *data,
77 unsigned int length)
78 {
79 u32 *crc = shash_desc_ctx(desc);
80
81 *crc = crc32_armv8_le(*crc, data, length);
82 return 0;
83 }
84
crc32c_update(struct shash_desc * desc,const u8 * data,unsigned int length)85 static int crc32c_update(struct shash_desc *desc, const u8 *data,
86 unsigned int length)
87 {
88 u32 *crc = shash_desc_ctx(desc);
89
90 *crc = crc32c_armv8_le(*crc, data, length);
91 return 0;
92 }
93
crc32_final(struct shash_desc * desc,u8 * out)94 static int crc32_final(struct shash_desc *desc, u8 *out)
95 {
96 u32 *crc = shash_desc_ctx(desc);
97
98 put_unaligned_le32(*crc, out);
99 return 0;
100 }
101
crc32c_final(struct shash_desc * desc,u8 * out)102 static int crc32c_final(struct shash_desc *desc, u8 *out)
103 {
104 u32 *crc = shash_desc_ctx(desc);
105
106 put_unaligned_le32(~*crc, out);
107 return 0;
108 }
109
crc32_pmull_update(struct shash_desc * desc,const u8 * data,unsigned int length)110 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
111 unsigned int length)
112 {
113 u32 *crc = shash_desc_ctx(desc);
114 unsigned int l;
115
116 if (may_use_simd()) {
117 if ((u32)data % SCALE_F) {
118 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
119
120 *crc = fallback_crc32(*crc, data, l);
121
122 data += l;
123 length -= l;
124 }
125
126 if (length >= PMULL_MIN_LEN) {
127 l = round_down(length, SCALE_F);
128
129 kernel_neon_begin();
130 *crc = crc32_pmull_le(data, l, *crc);
131 kernel_neon_end();
132
133 data += l;
134 length -= l;
135 }
136 }
137
138 if (length > 0)
139 *crc = fallback_crc32(*crc, data, length);
140
141 return 0;
142 }
143
crc32c_pmull_update(struct shash_desc * desc,const u8 * data,unsigned int length)144 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
145 unsigned int length)
146 {
147 u32 *crc = shash_desc_ctx(desc);
148 unsigned int l;
149
150 if (may_use_simd()) {
151 if ((u32)data % SCALE_F) {
152 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
153
154 *crc = fallback_crc32c(*crc, data, l);
155
156 data += l;
157 length -= l;
158 }
159
160 if (length >= PMULL_MIN_LEN) {
161 l = round_down(length, SCALE_F);
162
163 kernel_neon_begin();
164 *crc = crc32c_pmull_le(data, l, *crc);
165 kernel_neon_end();
166
167 data += l;
168 length -= l;
169 }
170 }
171
172 if (length > 0)
173 *crc = fallback_crc32c(*crc, data, length);
174
175 return 0;
176 }
177
178 static struct shash_alg crc32_pmull_algs[] = { {
179 .setkey = crc32_setkey,
180 .init = crc32_init,
181 .update = crc32_update,
182 .final = crc32_final,
183 .descsize = sizeof(u32),
184 .digestsize = sizeof(u32),
185
186 .base.cra_ctxsize = sizeof(u32),
187 .base.cra_init = crc32_cra_init,
188 .base.cra_name = "crc32",
189 .base.cra_driver_name = "crc32-arm-ce",
190 .base.cra_priority = 200,
191 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
192 .base.cra_blocksize = 1,
193 .base.cra_module = THIS_MODULE,
194 }, {
195 .setkey = crc32_setkey,
196 .init = crc32_init,
197 .update = crc32c_update,
198 .final = crc32c_final,
199 .descsize = sizeof(u32),
200 .digestsize = sizeof(u32),
201
202 .base.cra_ctxsize = sizeof(u32),
203 .base.cra_init = crc32c_cra_init,
204 .base.cra_name = "crc32c",
205 .base.cra_driver_name = "crc32c-arm-ce",
206 .base.cra_priority = 200,
207 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
208 .base.cra_blocksize = 1,
209 .base.cra_module = THIS_MODULE,
210 } };
211
crc32_pmull_mod_init(void)212 static int __init crc32_pmull_mod_init(void)
213 {
214 if (elf_hwcap2 & HWCAP2_PMULL) {
215 crc32_pmull_algs[0].update = crc32_pmull_update;
216 crc32_pmull_algs[1].update = crc32c_pmull_update;
217
218 if (elf_hwcap2 & HWCAP2_CRC32) {
219 fallback_crc32 = crc32_armv8_le;
220 fallback_crc32c = crc32c_armv8_le;
221 } else {
222 fallback_crc32 = crc32_le;
223 fallback_crc32c = __crc32c_le;
224 }
225 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
226 return -ENODEV;
227 }
228
229 return crypto_register_shashes(crc32_pmull_algs,
230 ARRAY_SIZE(crc32_pmull_algs));
231 }
232
crc32_pmull_mod_exit(void)233 static void __exit crc32_pmull_mod_exit(void)
234 {
235 crypto_unregister_shashes(crc32_pmull_algs,
236 ARRAY_SIZE(crc32_pmull_algs));
237 }
238
239 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
240 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
241 };
242 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
243
244 module_init(crc32_pmull_mod_init);
245 module_exit(crc32_pmull_mod_exit);
246
247 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
248 MODULE_LICENSE("GPL v2");
249 MODULE_ALIAS_CRYPTO("crc32");
250 MODULE_ALIAS_CRYPTO("crc32c");
251