87 static int xor_tweak(struct skcipher_request *req, bool second_pass, bool enc)  in xor_tweak()  argument
89 	struct rctx *rctx = skcipher_request_ctx(req);  in xor_tweak()
90 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);  in xor_tweak()
91 	const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);  in xor_tweak()
98 		req = &rctx->subreq;  in xor_tweak()
100 		skcipher_request_set_tfm(req, tfm);  in xor_tweak()
102 	err = skcipher_walk_virt(&w, req, false);  in xor_tweak()
137 static int xor_tweak_pre(struct skcipher_request *req, bool enc)  in xor_tweak_pre()  argument
139 	return xor_tweak(req, false, enc);  in xor_tweak_pre()
142 static int xor_tweak_post(struct skcipher_request *req, bool enc)  in xor_tweak_post()  argument
144 	return xor_tweak(req, true, enc);  in xor_tweak_post()
149 	struct skcipher_request *req = areq->data;  in cts_done()  local
153 		struct rctx *rctx = skcipher_request_ctx(req);  in cts_done()
160 	skcipher_request_complete(req, err);  in cts_done()
163 static int cts_final(struct skcipher_request *req,  in cts_final()  argument
164 		     int (*crypt)(struct skcipher_request *req))  in cts_final()  argument
166 	struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));  in cts_final()
167 	int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1);  in cts_final()
168 	struct rctx *rctx = skcipher_request_ctx(req);  in cts_final()
170 	int tail = req->cryptlen % XTS_BLOCK_SIZE;  in cts_final()
174 	rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst,  in cts_final()
179 	scatterwalk_map_and_copy(b, req->src, offset, tail, 0);  in cts_final()
186 	skcipher_request_set_callback(subreq, req->base.flags, cts_done, req);  in cts_final()
203 	struct skcipher_request *req = areq->data;  in encrypt_done()  local
206 		struct rctx *rctx = skcipher_request_ctx(req);  in encrypt_done()
209 		err = xor_tweak_post(req, true);  in encrypt_done()
211 		if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {  in encrypt_done()
212 			err = cts_final(req, crypto_skcipher_encrypt);  in encrypt_done()
218 	skcipher_request_complete(req, err);  in encrypt_done()
223 	struct skcipher_request *req = areq->data;  in decrypt_done()  local
226 		struct rctx *rctx = skcipher_request_ctx(req);  in decrypt_done()
229 		err = xor_tweak_post(req, false);  in decrypt_done()
231 		if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {  in decrypt_done()
232 			err = cts_final(req, crypto_skcipher_decrypt);  in decrypt_done()
238 	skcipher_request_complete(req, err);  in decrypt_done()
241 static int init_crypt(struct skcipher_request *req, crypto_completion_t compl)  in init_crypt()  argument
243 	struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));  in init_crypt()
244 	struct rctx *rctx = skcipher_request_ctx(req);  in init_crypt()
247 	if (req->cryptlen < XTS_BLOCK_SIZE)  in init_crypt()
251 	skcipher_request_set_callback(subreq, req->base.flags, compl, req);  in init_crypt()
252 	skcipher_request_set_crypt(subreq, req->dst, req->dst,  in init_crypt()
253 				   req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL);  in init_crypt()
256 	crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);  in init_crypt()
261 static int encrypt(struct skcipher_request *req)  in encrypt()  argument
263 	struct rctx *rctx = skcipher_request_ctx(req);  in encrypt()
267 	err = init_crypt(req, encrypt_done) ?:  in encrypt()
268 	      xor_tweak_pre(req, true) ?:  in encrypt()
270 	      xor_tweak_post(req, true);  in encrypt()
272 	if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))  in encrypt()
275 	return cts_final(req, crypto_skcipher_encrypt);  in encrypt()
278 static int decrypt(struct skcipher_request *req)  in decrypt()  argument
280 	struct rctx *rctx = skcipher_request_ctx(req);  in decrypt()
284 	err = init_crypt(req, decrypt_done) ?:  in decrypt()
285 	      xor_tweak_pre(req, false) ?:  in decrypt()
287 	      xor_tweak_post(req, false);  in decrypt()
289 	if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))  in decrypt()
292 	return cts_final(req, crypto_skcipher_decrypt);  in decrypt()