1 /* ====================================================================
2 * Copyright (c) 2008 The OpenSSL Project. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * 3. All advertising materials mentioning features or use of this
17 * software must display the following acknowledgment:
18 * "This product includes software developed by the OpenSSL Project
19 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
20 *
21 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
22 * endorse or promote products derived from this software without
23 * prior written permission. For written permission, please contact
24 * openssl-core@openssl.org.
25 *
26 * 5. Products derived from this software may not be called "OpenSSL"
27 * nor may "OpenSSL" appear in their names without prior written
28 * permission of the OpenSSL Project.
29 *
30 * 6. Redistributions of any form whatsoever must retain the following
31 * acknowledgment:
32 * "This product includes software developed by the OpenSSL Project
33 * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
36 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
38 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
39 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
42 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
44 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
46 * OF THE POSSIBILITY OF SUCH DAMAGE.
47 * ==================================================================== */
48
49 #include <openssl/base.h>
50
51 #include <assert.h>
52 #include <string.h>
53
54 #include <openssl/mem.h>
55
56 #include "internal.h"
57 #include "../../internal.h"
58
59
60 // kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four
61 // bits of a |size_t|.
62 static const size_t kSizeTWithoutLower4Bits = (size_t) -16;
63
64
65 #define GCM_MUL(ctx, Xi) gcm_gmult_nohw((ctx)->Xi.u, (ctx)->gcm_key.Htable)
66 #define GHASH(ctx, in, len) \
67 gcm_ghash_nohw((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
68 // GHASH_CHUNK is "stride parameter" missioned to mitigate cache
69 // trashing effect. In other words idea is to hash data while it's
70 // still in L1 cache after encryption pass...
71 #define GHASH_CHUNK (3 * 1024)
72
73 #if defined(GHASH_ASM_X86_64) || defined(GHASH_ASM_X86)
gcm_reduce_1bit(u128 * V)74 static inline void gcm_reduce_1bit(u128 *V) {
75 if (sizeof(crypto_word_t) == 8) {
76 uint64_t T = UINT64_C(0xe100000000000000) & (0 - (V->hi & 1));
77 V->hi = (V->lo << 63) | (V->hi >> 1);
78 V->lo = (V->lo >> 1) ^ T;
79 } else {
80 uint32_t T = 0xe1000000U & (0 - (uint32_t)(V->hi & 1));
81 V->hi = (V->lo << 63) | (V->hi >> 1);
82 V->lo = (V->lo >> 1) ^ ((uint64_t)T << 32);
83 }
84 }
85
gcm_init_ssse3(u128 Htable[16],const uint64_t H[2])86 void gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) {
87 Htable[0].hi = 0;
88 Htable[0].lo = 0;
89 u128 V;
90 V.hi = H[1];
91 V.lo = H[0];
92
93 Htable[8] = V;
94 gcm_reduce_1bit(&V);
95 Htable[4] = V;
96 gcm_reduce_1bit(&V);
97 Htable[2] = V;
98 gcm_reduce_1bit(&V);
99 Htable[1] = V;
100 Htable[3].hi = V.hi ^ Htable[2].hi, Htable[3].lo = V.lo ^ Htable[2].lo;
101 V = Htable[4];
102 Htable[5].hi = V.hi ^ Htable[1].hi, Htable[5].lo = V.lo ^ Htable[1].lo;
103 Htable[6].hi = V.hi ^ Htable[2].hi, Htable[6].lo = V.lo ^ Htable[2].lo;
104 Htable[7].hi = V.hi ^ Htable[3].hi, Htable[7].lo = V.lo ^ Htable[3].lo;
105 V = Htable[8];
106 Htable[9].hi = V.hi ^ Htable[1].hi, Htable[9].lo = V.lo ^ Htable[1].lo;
107 Htable[10].hi = V.hi ^ Htable[2].hi, Htable[10].lo = V.lo ^ Htable[2].lo;
108 Htable[11].hi = V.hi ^ Htable[3].hi, Htable[11].lo = V.lo ^ Htable[3].lo;
109 Htable[12].hi = V.hi ^ Htable[4].hi, Htable[12].lo = V.lo ^ Htable[4].lo;
110 Htable[13].hi = V.hi ^ Htable[5].hi, Htable[13].lo = V.lo ^ Htable[5].lo;
111 Htable[14].hi = V.hi ^ Htable[6].hi, Htable[14].lo = V.lo ^ Htable[6].lo;
112 Htable[15].hi = V.hi ^ Htable[7].hi, Htable[15].lo = V.lo ^ Htable[7].lo;
113
114 // Treat |Htable| as a 16x16 byte table and transpose it. Thus, Htable[i]
115 // contains the i'th byte of j*H for all j.
116 uint8_t *Hbytes = (uint8_t *)Htable;
117 for (int i = 0; i < 16; i++) {
118 for (int j = 0; j < i; j++) {
119 uint8_t tmp = Hbytes[16*i + j];
120 Hbytes[16*i + j] = Hbytes[16*j + i];
121 Hbytes[16*j + i] = tmp;
122 }
123 }
124 }
125 #endif // GHASH_ASM_X86_64 || GHASH_ASM_X86
126
127 #ifdef GCM_FUNCREF
128 #undef GCM_MUL
129 #define GCM_MUL(ctx, Xi) (*gcm_gmult_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable)
130 #undef GHASH
131 #define GHASH(ctx, in, len) \
132 (*gcm_ghash_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
133 #endif // GCM_FUNCREF
134
135 #if defined(HW_GCM) && defined(OPENSSL_X86_64)
hw_gcm_encrypt(const uint8_t * in,uint8_t * out,size_t len,const AES_KEY * key,uint8_t ivec[16],uint64_t * Xi)136 static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
137 const AES_KEY *key, uint8_t ivec[16],
138 uint64_t *Xi) {
139 return aesni_gcm_encrypt(in, out, len, key, ivec, Xi);
140 }
141
hw_gcm_decrypt(const uint8_t * in,uint8_t * out,size_t len,const AES_KEY * key,uint8_t ivec[16],uint64_t * Xi)142 static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
143 const AES_KEY *key, uint8_t ivec[16],
144 uint64_t *Xi) {
145 return aesni_gcm_decrypt(in, out, len, key, ivec, Xi);
146 }
147 #endif // HW_GCM && X86_64
148
149 #if defined(HW_GCM) && defined(OPENSSL_AARCH64)
150
hw_gcm_encrypt(const uint8_t * in,uint8_t * out,size_t len,const AES_KEY * key,uint8_t ivec[16],uint64_t * Xi)151 static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
152 const AES_KEY *key, uint8_t ivec[16],
153 uint64_t *Xi) {
154 const size_t len_blocks = len & kSizeTWithoutLower4Bits;
155 if (!len_blocks) {
156 return 0;
157 }
158 aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key);
159 return len_blocks;
160 }
161
hw_gcm_decrypt(const uint8_t * in,uint8_t * out,size_t len,const AES_KEY * key,uint8_t ivec[16],uint64_t * Xi)162 static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
163 const AES_KEY *key, uint8_t ivec[16],
164 uint64_t *Xi) {
165 const size_t len_blocks = len & kSizeTWithoutLower4Bits;
166 if (!len_blocks) {
167 return 0;
168 }
169 aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key);
170 return len_blocks;
171 }
172
173 #endif // HW_GCM && AARCH64
174
CRYPTO_ghash_init(gmult_func * out_mult,ghash_func * out_hash,u128 * out_key,u128 out_table[16],int * out_is_avx,const uint8_t gcm_key[16])175 void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
176 u128 *out_key, u128 out_table[16], int *out_is_avx,
177 const uint8_t gcm_key[16]) {
178 *out_is_avx = 0;
179
180 // H is stored in host byte order.
181 uint64_t H[2] = {CRYPTO_load_u64_be(gcm_key),
182 CRYPTO_load_u64_be(gcm_key + 8)};
183 out_key->hi = H[0];
184 out_key->lo = H[1];
185
186 #if defined(GHASH_ASM_X86_64)
187 if (crypto_gcm_clmul_enabled()) {
188 if (CRYPTO_is_AVX_capable() && CRYPTO_is_MOVBE_capable()) {
189 gcm_init_avx(out_table, H);
190 *out_mult = gcm_gmult_avx;
191 *out_hash = gcm_ghash_avx;
192 *out_is_avx = 1;
193 return;
194 }
195 gcm_init_clmul(out_table, H);
196 *out_mult = gcm_gmult_clmul;
197 *out_hash = gcm_ghash_clmul;
198 return;
199 }
200 if (CRYPTO_is_SSSE3_capable()) {
201 gcm_init_ssse3(out_table, H);
202 *out_mult = gcm_gmult_ssse3;
203 *out_hash = gcm_ghash_ssse3;
204 return;
205 }
206 #elif defined(GHASH_ASM_X86)
207 if (crypto_gcm_clmul_enabled()) {
208 gcm_init_clmul(out_table, H);
209 *out_mult = gcm_gmult_clmul;
210 *out_hash = gcm_ghash_clmul;
211 return;
212 }
213 if (CRYPTO_is_SSSE3_capable()) {
214 gcm_init_ssse3(out_table, H);
215 *out_mult = gcm_gmult_ssse3;
216 *out_hash = gcm_ghash_ssse3;
217 return;
218 }
219 #elif defined(GHASH_ASM_ARM)
220 if (gcm_pmull_capable()) {
221 gcm_init_v8(out_table, H);
222 *out_mult = gcm_gmult_v8;
223 *out_hash = gcm_ghash_v8;
224 return;
225 }
226
227 if (gcm_neon_capable()) {
228 gcm_init_neon(out_table, H);
229 *out_mult = gcm_gmult_neon;
230 *out_hash = gcm_ghash_neon;
231 return;
232 }
233 #endif
234
235 gcm_init_nohw(out_table, H);
236 *out_mult = gcm_gmult_nohw;
237 *out_hash = gcm_ghash_nohw;
238 }
239
CRYPTO_gcm128_init_key(GCM128_KEY * gcm_key,const AES_KEY * aes_key,block128_f block,int block_is_hwaes)240 void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key, const AES_KEY *aes_key,
241 block128_f block, int block_is_hwaes) {
242 OPENSSL_memset(gcm_key, 0, sizeof(*gcm_key));
243 gcm_key->block = block;
244
245 uint8_t ghash_key[16];
246 OPENSSL_memset(ghash_key, 0, sizeof(ghash_key));
247 (*block)(ghash_key, ghash_key, aes_key);
248
249 int is_avx;
250 CRYPTO_ghash_init(&gcm_key->gmult, &gcm_key->ghash, &gcm_key->H,
251 gcm_key->Htable, &is_avx, ghash_key);
252
253 #if defined(OPENSSL_AARCH64) && !defined(OPENSSL_NO_ASM)
254 gcm_key->use_hw_gcm_crypt = (gcm_pmull_capable() && block_is_hwaes) ? 1 :
255 0;
256 #else
257 gcm_key->use_hw_gcm_crypt = (is_avx && block_is_hwaes) ? 1 : 0;
258 #endif
259 }
260
CRYPTO_gcm128_setiv(GCM128_CONTEXT * ctx,const AES_KEY * key,const uint8_t * iv,size_t len)261 void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
262 const uint8_t *iv, size_t len) {
263 #ifdef GCM_FUNCREF
264 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
265 ctx->gcm_key.gmult;
266 #endif
267
268 ctx->Yi.u[0] = 0;
269 ctx->Yi.u[1] = 0;
270 ctx->Xi.u[0] = 0;
271 ctx->Xi.u[1] = 0;
272 ctx->len.u[0] = 0; // AAD length
273 ctx->len.u[1] = 0; // message length
274 ctx->ares = 0;
275 ctx->mres = 0;
276
277 uint32_t ctr;
278 if (len == 12) {
279 OPENSSL_memcpy(ctx->Yi.c, iv, 12);
280 ctx->Yi.c[15] = 1;
281 ctr = 1;
282 } else {
283 uint64_t len0 = len;
284
285 while (len >= 16) {
286 for (size_t i = 0; i < 16; ++i) {
287 ctx->Yi.c[i] ^= iv[i];
288 }
289 GCM_MUL(ctx, Yi);
290 iv += 16;
291 len -= 16;
292 }
293 if (len) {
294 for (size_t i = 0; i < len; ++i) {
295 ctx->Yi.c[i] ^= iv[i];
296 }
297 GCM_MUL(ctx, Yi);
298 }
299 len0 <<= 3;
300 ctx->Yi.u[1] ^= CRYPTO_bswap8(len0);
301
302 GCM_MUL(ctx, Yi);
303 ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
304 }
305
306 (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EK0.c, key);
307 ++ctr;
308 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
309 }
310
CRYPTO_gcm128_aad(GCM128_CONTEXT * ctx,const uint8_t * aad,size_t len)311 int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) {
312 #ifdef GCM_FUNCREF
313 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
314 ctx->gcm_key.gmult;
315 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
316 size_t len) = ctx->gcm_key.ghash;
317 #endif
318
319 if (ctx->len.u[1]) {
320 return 0;
321 }
322
323 uint64_t alen = ctx->len.u[0] + len;
324 if (alen > (UINT64_C(1) << 61) || (sizeof(len) == 8 && alen < len)) {
325 return 0;
326 }
327 ctx->len.u[0] = alen;
328
329 unsigned n = ctx->ares;
330 if (n) {
331 while (n && len) {
332 ctx->Xi.c[n] ^= *(aad++);
333 --len;
334 n = (n + 1) % 16;
335 }
336 if (n == 0) {
337 GCM_MUL(ctx, Xi);
338 } else {
339 ctx->ares = n;
340 return 1;
341 }
342 }
343
344 // Process a whole number of blocks.
345 size_t len_blocks = len & kSizeTWithoutLower4Bits;
346 if (len_blocks != 0) {
347 GHASH(ctx, aad, len_blocks);
348 aad += len_blocks;
349 len -= len_blocks;
350 }
351
352 // Process the remainder.
353 if (len != 0) {
354 n = (unsigned int)len;
355 for (size_t i = 0; i < len; ++i) {
356 ctx->Xi.c[i] ^= aad[i];
357 }
358 }
359
360 ctx->ares = n;
361 return 1;
362 }
363
CRYPTO_gcm128_encrypt(GCM128_CONTEXT * ctx,const AES_KEY * key,const uint8_t * in,uint8_t * out,size_t len)364 int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
365 const uint8_t *in, uint8_t *out, size_t len) {
366 block128_f block = ctx->gcm_key.block;
367 #ifdef GCM_FUNCREF
368 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
369 ctx->gcm_key.gmult;
370 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
371 size_t len) = ctx->gcm_key.ghash;
372 #endif
373
374 uint64_t mlen = ctx->len.u[1] + len;
375 if (mlen > ((UINT64_C(1) << 36) - 32) ||
376 (sizeof(len) == 8 && mlen < len)) {
377 return 0;
378 }
379 ctx->len.u[1] = mlen;
380
381 if (ctx->ares) {
382 // First call to encrypt finalizes GHASH(AAD)
383 GCM_MUL(ctx, Xi);
384 ctx->ares = 0;
385 }
386
387 unsigned n = ctx->mres;
388 if (n) {
389 while (n && len) {
390 ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
391 --len;
392 n = (n + 1) % 16;
393 }
394 if (n == 0) {
395 GCM_MUL(ctx, Xi);
396 } else {
397 ctx->mres = n;
398 return 1;
399 }
400 }
401
402 uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
403 while (len >= GHASH_CHUNK) {
404 size_t j = GHASH_CHUNK;
405
406 while (j) {
407 (*block)(ctx->Yi.c, ctx->EKi.c, key);
408 ++ctr;
409 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
410 for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
411 CRYPTO_store_word_le(out + i,
412 CRYPTO_load_word_le(in + i) ^
413 ctx->EKi.t[i / sizeof(crypto_word_t)]);
414 }
415 out += 16;
416 in += 16;
417 j -= 16;
418 }
419 GHASH(ctx, out - GHASH_CHUNK, GHASH_CHUNK);
420 len -= GHASH_CHUNK;
421 }
422 size_t len_blocks = len & kSizeTWithoutLower4Bits;
423 if (len_blocks != 0) {
424 while (len >= 16) {
425 (*block)(ctx->Yi.c, ctx->EKi.c, key);
426 ++ctr;
427 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
428 for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
429 CRYPTO_store_word_le(out + i,
430 CRYPTO_load_word_le(in + i) ^
431 ctx->EKi.t[i / sizeof(crypto_word_t)]);
432 }
433 out += 16;
434 in += 16;
435 len -= 16;
436 }
437 GHASH(ctx, out - len_blocks, len_blocks);
438 }
439 if (len) {
440 (*block)(ctx->Yi.c, ctx->EKi.c, key);
441 ++ctr;
442 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
443 while (len--) {
444 ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
445 ++n;
446 }
447 }
448
449 ctx->mres = n;
450 return 1;
451 }
452
CRYPTO_gcm128_decrypt(GCM128_CONTEXT * ctx,const AES_KEY * key,const unsigned char * in,unsigned char * out,size_t len)453 int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
454 const unsigned char *in, unsigned char *out,
455 size_t len) {
456 block128_f block = ctx->gcm_key.block;
457 #ifdef GCM_FUNCREF
458 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
459 ctx->gcm_key.gmult;
460 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
461 size_t len) = ctx->gcm_key.ghash;
462 #endif
463
464 uint64_t mlen = ctx->len.u[1] + len;
465 if (mlen > ((UINT64_C(1) << 36) - 32) ||
466 (sizeof(len) == 8 && mlen < len)) {
467 return 0;
468 }
469 ctx->len.u[1] = mlen;
470
471 if (ctx->ares) {
472 // First call to decrypt finalizes GHASH(AAD)
473 GCM_MUL(ctx, Xi);
474 ctx->ares = 0;
475 }
476
477 unsigned n = ctx->mres;
478 if (n) {
479 while (n && len) {
480 uint8_t c = *(in++);
481 *(out++) = c ^ ctx->EKi.c[n];
482 ctx->Xi.c[n] ^= c;
483 --len;
484 n = (n + 1) % 16;
485 }
486 if (n == 0) {
487 GCM_MUL(ctx, Xi);
488 } else {
489 ctx->mres = n;
490 return 1;
491 }
492 }
493
494 uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
495 while (len >= GHASH_CHUNK) {
496 size_t j = GHASH_CHUNK;
497
498 GHASH(ctx, in, GHASH_CHUNK);
499 while (j) {
500 (*block)(ctx->Yi.c, ctx->EKi.c, key);
501 ++ctr;
502 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
503 for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
504 CRYPTO_store_word_le(out + i,
505 CRYPTO_load_word_le(in + i) ^
506 ctx->EKi.t[i / sizeof(crypto_word_t)]);
507 }
508 out += 16;
509 in += 16;
510 j -= 16;
511 }
512 len -= GHASH_CHUNK;
513 }
514 size_t len_blocks = len & kSizeTWithoutLower4Bits;
515 if (len_blocks != 0) {
516 GHASH(ctx, in, len_blocks);
517 while (len >= 16) {
518 (*block)(ctx->Yi.c, ctx->EKi.c, key);
519 ++ctr;
520 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
521 for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
522 CRYPTO_store_word_le(out + i,
523 CRYPTO_load_word_le(in + i) ^
524 ctx->EKi.t[i / sizeof(crypto_word_t)]);
525 }
526 out += 16;
527 in += 16;
528 len -= 16;
529 }
530 }
531 if (len) {
532 (*block)(ctx->Yi.c, ctx->EKi.c, key);
533 ++ctr;
534 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
535 while (len--) {
536 uint8_t c = in[n];
537 ctx->Xi.c[n] ^= c;
538 out[n] = c ^ ctx->EKi.c[n];
539 ++n;
540 }
541 }
542
543 ctx->mres = n;
544 return 1;
545 }
546
CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT * ctx,const AES_KEY * key,const uint8_t * in,uint8_t * out,size_t len,ctr128_f stream)547 int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
548 const uint8_t *in, uint8_t *out, size_t len,
549 ctr128_f stream) {
550 #ifdef GCM_FUNCREF
551 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
552 ctx->gcm_key.gmult;
553 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
554 size_t len) = ctx->gcm_key.ghash;
555 #endif
556
557 uint64_t mlen = ctx->len.u[1] + len;
558 if (mlen > ((UINT64_C(1) << 36) - 32) ||
559 (sizeof(len) == 8 && mlen < len)) {
560 return 0;
561 }
562 ctx->len.u[1] = mlen;
563
564 if (ctx->ares) {
565 // First call to encrypt finalizes GHASH(AAD)
566 GCM_MUL(ctx, Xi);
567 ctx->ares = 0;
568 }
569
570 unsigned n = ctx->mres;
571 if (n) {
572 while (n && len) {
573 ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
574 --len;
575 n = (n + 1) % 16;
576 }
577 if (n == 0) {
578 GCM_MUL(ctx, Xi);
579 } else {
580 ctx->mres = n;
581 return 1;
582 }
583 }
584
585 #if defined(HW_GCM)
586 // Check |len| to work around a C language bug. See https://crbug.com/1019588.
587 if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) {
588 // |hw_gcm_encrypt| may not process all the input given to it. It may
589 // not process *any* of its input if it is deemed too small.
590 size_t bulk = hw_gcm_encrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
591 in += bulk;
592 out += bulk;
593 len -= bulk;
594 }
595 #endif
596
597 uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
598 while (len >= GHASH_CHUNK) {
599 (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
600 ctr += GHASH_CHUNK / 16;
601 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
602 GHASH(ctx, out, GHASH_CHUNK);
603 out += GHASH_CHUNK;
604 in += GHASH_CHUNK;
605 len -= GHASH_CHUNK;
606 }
607 size_t len_blocks = len & kSizeTWithoutLower4Bits;
608 if (len_blocks != 0) {
609 size_t j = len_blocks / 16;
610
611 (*stream)(in, out, j, key, ctx->Yi.c);
612 ctr += (unsigned int)j;
613 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
614 in += len_blocks;
615 len -= len_blocks;
616 GHASH(ctx, out, len_blocks);
617 out += len_blocks;
618 }
619 if (len) {
620 (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
621 ++ctr;
622 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
623 while (len--) {
624 ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
625 ++n;
626 }
627 }
628
629 ctx->mres = n;
630 return 1;
631 }
632
CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT * ctx,const AES_KEY * key,const uint8_t * in,uint8_t * out,size_t len,ctr128_f stream)633 int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
634 const uint8_t *in, uint8_t *out, size_t len,
635 ctr128_f stream) {
636 #ifdef GCM_FUNCREF
637 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
638 ctx->gcm_key.gmult;
639 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
640 size_t len) = ctx->gcm_key.ghash;
641 #endif
642
643 uint64_t mlen = ctx->len.u[1] + len;
644 if (mlen > ((UINT64_C(1) << 36) - 32) ||
645 (sizeof(len) == 8 && mlen < len)) {
646 return 0;
647 }
648 ctx->len.u[1] = mlen;
649
650 if (ctx->ares) {
651 // First call to decrypt finalizes GHASH(AAD)
652 GCM_MUL(ctx, Xi);
653 ctx->ares = 0;
654 }
655
656 unsigned n = ctx->mres;
657 if (n) {
658 while (n && len) {
659 uint8_t c = *(in++);
660 *(out++) = c ^ ctx->EKi.c[n];
661 ctx->Xi.c[n] ^= c;
662 --len;
663 n = (n + 1) % 16;
664 }
665 if (n == 0) {
666 GCM_MUL(ctx, Xi);
667 } else {
668 ctx->mres = n;
669 return 1;
670 }
671 }
672
673 #if defined(HW_GCM)
674 // Check |len| to work around a C language bug. See https://crbug.com/1019588.
675 if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) {
676 // |hw_gcm_decrypt| may not process all the input given to it. It may
677 // not process *any* of its input if it is deemed too small.
678 size_t bulk = hw_gcm_decrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
679 in += bulk;
680 out += bulk;
681 len -= bulk;
682 }
683 #endif
684
685 uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
686 while (len >= GHASH_CHUNK) {
687 GHASH(ctx, in, GHASH_CHUNK);
688 (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
689 ctr += GHASH_CHUNK / 16;
690 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
691 out += GHASH_CHUNK;
692 in += GHASH_CHUNK;
693 len -= GHASH_CHUNK;
694 }
695 size_t len_blocks = len & kSizeTWithoutLower4Bits;
696 if (len_blocks != 0) {
697 size_t j = len_blocks / 16;
698
699 GHASH(ctx, in, len_blocks);
700 (*stream)(in, out, j, key, ctx->Yi.c);
701 ctr += (unsigned int)j;
702 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
703 out += len_blocks;
704 in += len_blocks;
705 len -= len_blocks;
706 }
707 if (len) {
708 (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
709 ++ctr;
710 ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
711 while (len--) {
712 uint8_t c = in[n];
713 ctx->Xi.c[n] ^= c;
714 out[n] = c ^ ctx->EKi.c[n];
715 ++n;
716 }
717 }
718
719 ctx->mres = n;
720 return 1;
721 }
722
CRYPTO_gcm128_finish(GCM128_CONTEXT * ctx,const uint8_t * tag,size_t len)723 int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
724 #ifdef GCM_FUNCREF
725 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
726 ctx->gcm_key.gmult;
727 #endif
728
729 if (ctx->mres || ctx->ares) {
730 GCM_MUL(ctx, Xi);
731 }
732
733 ctx->Xi.u[0] ^= CRYPTO_bswap8(ctx->len.u[0] << 3);
734 ctx->Xi.u[1] ^= CRYPTO_bswap8(ctx->len.u[1] << 3);
735 GCM_MUL(ctx, Xi);
736
737 ctx->Xi.u[0] ^= ctx->EK0.u[0];
738 ctx->Xi.u[1] ^= ctx->EK0.u[1];
739
740 if (tag && len <= sizeof(ctx->Xi)) {
741 return CRYPTO_memcmp(ctx->Xi.c, tag, len) == 0;
742 } else {
743 return 0;
744 }
745 }
746
CRYPTO_gcm128_tag(GCM128_CONTEXT * ctx,unsigned char * tag,size_t len)747 void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len) {
748 CRYPTO_gcm128_finish(ctx, NULL, 0);
749 OPENSSL_memcpy(tag, ctx->Xi.c,
750 len <= sizeof(ctx->Xi.c) ? len : sizeof(ctx->Xi.c));
751 }
752
753 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
crypto_gcm_clmul_enabled(void)754 int crypto_gcm_clmul_enabled(void) {
755 #if defined(GHASH_ASM_X86) || defined(GHASH_ASM_X86_64)
756 return CRYPTO_is_FXSR_capable() && CRYPTO_is_PCLMUL_capable();
757 #else
758 return 0;
759 #endif
760 }
761 #endif
762