40 asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
42 asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
46 asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
48 asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
51 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,  in twofish_enc_blk_3way()  argument
54 	__twofish_enc_blk_3way(ctx, dst, src, false);  in twofish_enc_blk_3way()
57 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,  in twofish_enc_blk_xor_3way()  argument
60 	__twofish_enc_blk_3way(ctx, dst, src, true);  in twofish_enc_blk_xor_3way()
76 		u8 *wdst = walk->dst.virt.addr;  in ecb_crypt()
108 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in ecb_encrypt()  argument
113 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ecb_encrypt()
117 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in ecb_decrypt()  argument
122 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ecb_decrypt()
133 	u128 *dst = (u128 *)walk->dst.virt.addr;  in __cbc_encrypt()  local
137 		u128_xor(dst, src, iv);  in __cbc_encrypt()
138 		twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);  in __cbc_encrypt()
139 		iv = dst;  in __cbc_encrypt()
142 		dst += 1;  in __cbc_encrypt()
150 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in cbc_encrypt()  argument
156 	blkcipher_walk_init(&walk, dst, src, nbytes);  in cbc_encrypt()
174 	u128 *dst = (u128 *)walk->dst.virt.addr;  in __cbc_decrypt()  local
180 	dst += nbytes / bsize - 1;  in __cbc_decrypt()
189 			dst -= 3 - 1;  in __cbc_decrypt()
194 			twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);  in __cbc_decrypt()
196 			u128_xor(dst + 1, dst + 1, ivs + 0);  in __cbc_decrypt()
197 			u128_xor(dst + 2, dst + 2, ivs + 1);  in __cbc_decrypt()
203 			u128_xor(dst, dst, src - 1);  in __cbc_decrypt()
205 			dst -= 1;  in __cbc_decrypt()
214 		twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);  in __cbc_decrypt()
220 		u128_xor(dst, dst, src - 1);  in __cbc_decrypt()
222 		dst -= 1;  in __cbc_decrypt()
226 	u128_xor(dst, dst, (u128 *)walk->iv);  in __cbc_decrypt()
232 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in cbc_decrypt()  argument
238 	blkcipher_walk_init(&walk, dst, src, nbytes);  in cbc_decrypt()
249 static inline void u128_to_be128(be128 *dst, const u128 *src)  in u128_to_be128()  argument
251 	dst->a = cpu_to_be64(src->a);  in u128_to_be128()
252 	dst->b = cpu_to_be64(src->b);  in u128_to_be128()
255 static inline void be128_to_u128(u128 *dst, const be128 *src)  in be128_to_u128()  argument
257 	dst->a = be64_to_cpu(src->a);  in be128_to_u128()
258 	dst->b = be64_to_cpu(src->b);  in be128_to_u128()
275 	u8 *dst = walk->dst.virt.addr;  in ctr_crypt_final()  local
280 	memcpy(dst, keystream, nbytes);  in ctr_crypt_final()
292 	u128 *dst = (u128 *)walk->dst.virt.addr;  in __ctr_crypt()  local
301 			if (dst != src) {  in __ctr_crypt()
302 				dst[0] = src[0];  in __ctr_crypt()
303 				dst[1] = src[1];  in __ctr_crypt()
304 				dst[2] = src[2];  in __ctr_crypt()
315 			twofish_enc_blk_xor_3way(ctx, (u8 *)dst,  in __ctr_crypt()
319 			dst += 3;  in __ctr_crypt()
329 		if (dst != src)  in __ctr_crypt()
330 			*dst = *src;  in __ctr_crypt()
336 		u128_xor(dst, dst, (u128 *)ctrblocks);  in __ctr_crypt()
339 		dst += 1;  in __ctr_crypt()
348 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in ctr_crypt()  argument
354 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ctr_crypt()
419 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in lrw_encrypt()  argument
433 	return lrw_crypt(desc, dst, src, nbytes, &req);  in lrw_encrypt()
436 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in lrw_decrypt()  argument
450 	return lrw_crypt(desc, dst, src, nbytes, &req);  in lrw_decrypt()
490 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in xts_encrypt()  argument
505 	return xts_crypt(desc, dst, src, nbytes, &req);  in xts_encrypt()
508 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,  in xts_decrypt()  argument
523 	return xts_crypt(desc, dst, src, nbytes, &req);  in xts_decrypt()