1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 *
5 * This is based in part on Andrew Moon's poly1305-donna, which is in the
6 * public domain.
7 */
8
9 #include <linux/kernel.h>
10 #include <asm/unaligned.h>
11 #include <crypto/internal/poly1305.h>
12
13 typedef __uint128_t u128;
14
poly1305_core_setkey(struct poly1305_core_key * key,const u8 raw_key[POLY1305_BLOCK_SIZE])15 void poly1305_core_setkey(struct poly1305_core_key *key,
16 const u8 raw_key[POLY1305_BLOCK_SIZE])
17 {
18 u64 t0, t1;
19
20 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
21 t0 = get_unaligned_le64(&raw_key[0]);
22 t1 = get_unaligned_le64(&raw_key[8]);
23
24 key->key.r64[0] = t0 & 0xffc0fffffffULL;
25 key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
26 key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
27
28 /* s = 20*r */
29 key->precomputed_s.r64[0] = key->key.r64[1] * 20;
30 key->precomputed_s.r64[1] = key->key.r64[2] * 20;
31 }
32 EXPORT_SYMBOL(poly1305_core_setkey);
33
poly1305_core_blocks(struct poly1305_state * state,const struct poly1305_core_key * key,const void * src,unsigned int nblocks,u32 hibit)34 void poly1305_core_blocks(struct poly1305_state *state,
35 const struct poly1305_core_key *key, const void *src,
36 unsigned int nblocks, u32 hibit)
37 {
38 const u8 *input = src;
39 u64 hibit64;
40 u64 r0, r1, r2;
41 u64 s1, s2;
42 u64 h0, h1, h2;
43 u64 c;
44 u128 d0, d1, d2, d;
45
46 if (!nblocks)
47 return;
48
49 hibit64 = ((u64)hibit) << 40;
50
51 r0 = key->key.r64[0];
52 r1 = key->key.r64[1];
53 r2 = key->key.r64[2];
54
55 h0 = state->h64[0];
56 h1 = state->h64[1];
57 h2 = state->h64[2];
58
59 s1 = key->precomputed_s.r64[0];
60 s2 = key->precomputed_s.r64[1];
61
62 do {
63 u64 t0, t1;
64
65 /* h += m[i] */
66 t0 = get_unaligned_le64(&input[0]);
67 t1 = get_unaligned_le64(&input[8]);
68
69 h0 += t0 & 0xfffffffffffULL;
70 h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
71 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
72
73 /* h *= r */
74 d0 = (u128)h0 * r0;
75 d = (u128)h1 * s2;
76 d0 += d;
77 d = (u128)h2 * s1;
78 d0 += d;
79 d1 = (u128)h0 * r1;
80 d = (u128)h1 * r0;
81 d1 += d;
82 d = (u128)h2 * s2;
83 d1 += d;
84 d2 = (u128)h0 * r2;
85 d = (u128)h1 * r1;
86 d2 += d;
87 d = (u128)h2 * r0;
88 d2 += d;
89
90 /* (partial) h %= p */
91 c = (u64)(d0 >> 44);
92 h0 = (u64)d0 & 0xfffffffffffULL;
93 d1 += c;
94 c = (u64)(d1 >> 44);
95 h1 = (u64)d1 & 0xfffffffffffULL;
96 d2 += c;
97 c = (u64)(d2 >> 42);
98 h2 = (u64)d2 & 0x3ffffffffffULL;
99 h0 += c * 5;
100 c = h0 >> 44;
101 h0 = h0 & 0xfffffffffffULL;
102 h1 += c;
103
104 input += POLY1305_BLOCK_SIZE;
105 } while (--nblocks);
106
107 state->h64[0] = h0;
108 state->h64[1] = h1;
109 state->h64[2] = h2;
110 }
111 EXPORT_SYMBOL(poly1305_core_blocks);
112
poly1305_core_emit(const struct poly1305_state * state,const u32 nonce[4],void * dst)113 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
114 void *dst)
115 {
116 u8 *mac = dst;
117 u64 h0, h1, h2, c;
118 u64 g0, g1, g2;
119 u64 t0, t1;
120
121 /* fully carry h */
122 h0 = state->h64[0];
123 h1 = state->h64[1];
124 h2 = state->h64[2];
125
126 c = h1 >> 44;
127 h1 &= 0xfffffffffffULL;
128 h2 += c;
129 c = h2 >> 42;
130 h2 &= 0x3ffffffffffULL;
131 h0 += c * 5;
132 c = h0 >> 44;
133 h0 &= 0xfffffffffffULL;
134 h1 += c;
135 c = h1 >> 44;
136 h1 &= 0xfffffffffffULL;
137 h2 += c;
138 c = h2 >> 42;
139 h2 &= 0x3ffffffffffULL;
140 h0 += c * 5;
141 c = h0 >> 44;
142 h0 &= 0xfffffffffffULL;
143 h1 += c;
144
145 /* compute h + -p */
146 g0 = h0 + 5;
147 c = g0 >> 44;
148 g0 &= 0xfffffffffffULL;
149 g1 = h1 + c;
150 c = g1 >> 44;
151 g1 &= 0xfffffffffffULL;
152 g2 = h2 + c - (1ULL << 42);
153
154 /* select h if h < p, or h + -p if h >= p */
155 c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
156 g0 &= c;
157 g1 &= c;
158 g2 &= c;
159 c = ~c;
160 h0 = (h0 & c) | g0;
161 h1 = (h1 & c) | g1;
162 h2 = (h2 & c) | g2;
163
164 if (likely(nonce)) {
165 /* h = (h + nonce) */
166 t0 = ((u64)nonce[1] << 32) | nonce[0];
167 t1 = ((u64)nonce[3] << 32) | nonce[2];
168
169 h0 += t0 & 0xfffffffffffULL;
170 c = h0 >> 44;
171 h0 &= 0xfffffffffffULL;
172 h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
173 c = h1 >> 44;
174 h1 &= 0xfffffffffffULL;
175 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
176 h2 &= 0x3ffffffffffULL;
177 }
178
179 /* mac = h % (2^128) */
180 h0 = h0 | (h1 << 44);
181 h1 = (h1 >> 20) | (h2 << 24);
182
183 put_unaligned_le64(h0, &mac[0]);
184 put_unaligned_le64(h1, &mac[8]);
185 }
186 EXPORT_SYMBOL(poly1305_core_emit);
187