82 static int xts_xor_tweak(struct skcipher_request *req, bool second_pass,  in xts_xor_tweak()  argument
85 	struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_xor_tweak()
86 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);  in xts_xor_tweak()
87 	const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);  in xts_xor_tweak()
94 		req = &rctx->subreq;  in xts_xor_tweak()
96 		skcipher_request_set_tfm(req, tfm);  in xts_xor_tweak()
98 	err = skcipher_walk_virt(&w, req, false);  in xts_xor_tweak()
133 static int xts_xor_tweak_pre(struct skcipher_request *req, bool enc)  in xts_xor_tweak_pre()  argument
135 	return xts_xor_tweak(req, false, enc);  in xts_xor_tweak_pre()
138 static int xts_xor_tweak_post(struct skcipher_request *req, bool enc)  in xts_xor_tweak_post()  argument
140 	return xts_xor_tweak(req, true, enc);  in xts_xor_tweak_post()
145 	struct skcipher_request *req = areq->data;  in xts_cts_done()  local
149 		struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_cts_done()
156 	skcipher_request_complete(req, err);  in xts_cts_done()
159 static int xts_cts_final(struct skcipher_request *req,  in xts_cts_final()  argument
160 			 int (*crypt)(struct skcipher_request *req))  in xts_cts_final()  argument
163 		crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));  in xts_cts_final()
164 	int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1);  in xts_cts_final()
165 	struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_cts_final()
167 	int tail = req->cryptlen % XTS_BLOCK_SIZE;  in xts_cts_final()
171 	rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst,  in xts_cts_final()
176 	scatterwalk_map_and_copy(b, req->src, offset, tail, 0);  in xts_cts_final()
183 	skcipher_request_set_callback(subreq, req->base.flags, xts_cts_done,  in xts_cts_final()
184 				      req);  in xts_cts_final()
201 	struct skcipher_request *req = areq->data;  in xts_encrypt_done()  local
204 		struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_encrypt_done()
207 		err = xts_xor_tweak_post(req, true);  in xts_encrypt_done()
209 		if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {  in xts_encrypt_done()
210 			err = xts_cts_final(req, crypto_skcipher_encrypt);  in xts_encrypt_done()
216 	skcipher_request_complete(req, err);  in xts_encrypt_done()
221 	struct skcipher_request *req = areq->data;  in xts_decrypt_done()  local
224 		struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_decrypt_done()
227 		err = xts_xor_tweak_post(req, false);  in xts_decrypt_done()
229 		if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {  in xts_decrypt_done()
230 			err = xts_cts_final(req, crypto_skcipher_decrypt);  in xts_decrypt_done()
236 	skcipher_request_complete(req, err);  in xts_decrypt_done()
239 static int xts_init_crypt(struct skcipher_request *req,  in xts_init_crypt()  argument
243 		crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));  in xts_init_crypt()
244 	struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_init_crypt()
247 	if (req->cryptlen < XTS_BLOCK_SIZE)  in xts_init_crypt()
251 	skcipher_request_set_callback(subreq, req->base.flags, compl, req);  in xts_init_crypt()
252 	skcipher_request_set_crypt(subreq, req->dst, req->dst,  in xts_init_crypt()
253 				   req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL);  in xts_init_crypt()
256 	crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);  in xts_init_crypt()
261 static int xts_encrypt(struct skcipher_request *req)  in xts_encrypt()  argument
263 	struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_encrypt()
267 	err = xts_init_crypt(req, xts_encrypt_done) ?:  in xts_encrypt()
268 	      xts_xor_tweak_pre(req, true) ?:  in xts_encrypt()
270 	      xts_xor_tweak_post(req, true);  in xts_encrypt()
272 	if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))  in xts_encrypt()
275 	return xts_cts_final(req, crypto_skcipher_encrypt);  in xts_encrypt()
278 static int xts_decrypt(struct skcipher_request *req)  in xts_decrypt()  argument
280 	struct xts_request_ctx *rctx = skcipher_request_ctx(req);  in xts_decrypt()
284 	err = xts_init_crypt(req, xts_decrypt_done) ?:  in xts_decrypt()
285 	      xts_xor_tweak_pre(req, false) ?:  in xts_decrypt()
287 	      xts_xor_tweak_post(req, false);  in xts_decrypt()
289 	if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))  in xts_decrypt()
292 	return xts_cts_final(req, crypto_skcipher_decrypt);  in xts_decrypt()