18 asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
20 asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
22 asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
24 asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
26 asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
28 asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
31 asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
33 asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
35 asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
48 static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,  in chacha_dosimd()  argument
54 			chacha_8block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
59 			state[12] += 8;  in chacha_dosimd()
62 			chacha_8block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
64 			state[12] += chacha_advance(bytes, 8);  in chacha_dosimd()
68 			chacha_4block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
70 			state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
74 			chacha_2block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
76 			state[12] += chacha_advance(bytes, 2);  in chacha_dosimd()
83 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
87 			state[12] += 8;  in chacha_dosimd()
90 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
91 			state[12] += chacha_advance(bytes, 8);  in chacha_dosimd()
95 			chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
96 			state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
100 			chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
101 			state[12] += chacha_advance(bytes, 2);  in chacha_dosimd()
107 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
111 		state[12] += 4;  in chacha_dosimd()
114 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
115 		state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
119 		chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
120 		state[12]++;  in chacha_dosimd()
124 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)  in hchacha_block_arch()  argument
127 		hchacha_block_generic(state, stream, nrounds);  in hchacha_block_arch()
130 		hchacha_block_ssse3(state, stream, nrounds);  in hchacha_block_arch()
136 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)  in chacha_init_arch()  argument
138 	chacha_init_generic(state, key, iv);  in chacha_init_arch()
142 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,  in chacha_crypt_arch()  argument
147 		return chacha_crypt_generic(state, dst, src, bytes, nrounds);  in chacha_crypt_arch()
153 		chacha_dosimd(state, dst, src, todo, nrounds);  in chacha_crypt_arch()
166 	u32 state[CHACHA_STATE_WORDS] __aligned(8);  in chacha_simd_stream_xor()  local
172 	chacha_init_generic(state, ctx->key, iv);  in chacha_simd_stream_xor()
182 			chacha_crypt_generic(state, walk.dst.virt.addr,  in chacha_simd_stream_xor()
187 			chacha_dosimd(state, walk.dst.virt.addr,  in chacha_simd_stream_xor()
210 	u32 state[CHACHA_STATE_WORDS] __aligned(8);  in xchacha_simd()  local
214 	chacha_init_generic(state, ctx->key, req->iv);  in xchacha_simd()
218 		hchacha_block_ssse3(state, subctx.key, ctx->nrounds);  in xchacha_simd()
221 		hchacha_block_generic(state, subctx.key, ctx->nrounds);  in xchacha_simd()