19 asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
21 asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
23 asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
25 asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
27 asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
29 asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
33 asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
35 asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
37 asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
49 static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,  in chacha_dosimd()  argument
56 			chacha_8block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
61 			state[12] += 8;  in chacha_dosimd()
64 			chacha_8block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
66 			state[12] += chacha_advance(bytes, 8);  in chacha_dosimd()
70 			chacha_4block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
72 			state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
76 			chacha_2block_xor_avx512vl(state, dst, src, bytes,  in chacha_dosimd()
78 			state[12] += chacha_advance(bytes, 2);  in chacha_dosimd()
85 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
89 			state[12] += 8;  in chacha_dosimd()
92 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
93 			state[12] += chacha_advance(bytes, 8);  in chacha_dosimd()
97 			chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
98 			state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
102 			chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);  in chacha_dosimd()
103 			state[12] += chacha_advance(bytes, 2);  in chacha_dosimd()
109 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
113 		state[12] += 4;  in chacha_dosimd()
116 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
117 		state[12] += chacha_advance(bytes, 4);  in chacha_dosimd()
121 		chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);  in chacha_dosimd()
122 		state[12]++;  in chacha_dosimd()
129 	u32 *state, state_buf[16 + 2] __aligned(8);  in chacha_simd_stream_xor()  local
134 	state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);  in chacha_simd_stream_xor()
136 	crypto_chacha_init(state, ctx, iv);  in chacha_simd_stream_xor()
146 		chacha_dosimd(state, walk->dst.virt.addr, walk->src.virt.addr,  in chacha_simd_stream_xor()
188 	u32 *state, state_buf[16 + 2] __aligned(8);  in xchacha_simd()  local
200 	state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);  in xchacha_simd()
201 	crypto_chacha_init(state, ctx, req->iv);  in xchacha_simd()
205 	hchacha_block_ssse3(state, subctx.key, ctx->nrounds);  in xchacha_simd()