229 static void print_req_error(struct request *req, blk_status_t status,  in print_req_error()  argument
241 		req->rq_disk ? req->rq_disk->disk_name : "?",  in print_req_error()
242 		blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),  in print_req_error()
243 		req->cmd_flags & ~REQ_OP_MASK,  in print_req_error()
244 		req->nr_phys_segments,  in print_req_error()
245 		IOPRIO_PRIO_CLASS(req->ioprio));  in print_req_error()
642 	struct request *req;  in blk_get_request()  local
647 	req = blk_mq_alloc_request(q, op, flags);  in blk_get_request()
648 	if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)  in blk_get_request()
649 		q->mq_ops->initialize_rq_fn(req);  in blk_get_request()
651 	return req;  in blk_get_request()
655 void blk_put_request(struct request *req)  in blk_put_request()  argument
657 	blk_mq_free_request(req);  in blk_put_request()
1286 static void blk_account_io_completion(struct request *req, unsigned int bytes)  in blk_account_io_completion()  argument
1288 	if (req->part && blk_do_io_stat(req)) {  in blk_account_io_completion()
1289 		const int sgrp = op_stat_group(req_op(req));  in blk_account_io_completion()
1293 		part = req->part;  in blk_account_io_completion()
1299 void blk_account_io_done(struct request *req, u64 now)  in blk_account_io_done()  argument
1306 	if (req->part && blk_do_io_stat(req) &&  in blk_account_io_done()
1307 	    !(req->rq_flags & RQF_FLUSH_SEQ)) {  in blk_account_io_done()
1308 		const int sgrp = op_stat_group(req_op(req));  in blk_account_io_done()
1312 		part = req->part;  in blk_account_io_done()
1316 		part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);  in blk_account_io_done()
1443 bool blk_update_request(struct request *req, blk_status_t error,  in blk_update_request()  argument
1448 	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);  in blk_update_request()
1450 	if (!req->bio)  in blk_update_request()
1454 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&  in blk_update_request()
1456 		req->q->integrity.profile->complete_fn(req, nr_bytes);  in blk_update_request()
1463 	if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))  in blk_update_request()
1464 		__blk_crypto_rq_put_keyslot(req);  in blk_update_request()
1466 	if (unlikely(error && !blk_rq_is_passthrough(req) &&  in blk_update_request()
1467 		     !(req->rq_flags & RQF_QUIET)))  in blk_update_request()
1468 		print_req_error(req, error, __func__);  in blk_update_request()
1470 	blk_account_io_completion(req, nr_bytes);  in blk_update_request()
1473 	while (req->bio) {  in blk_update_request()
1474 		struct bio *bio = req->bio;  in blk_update_request()
1478 			req->bio = bio->bi_next;  in blk_update_request()
1482 		req_bio_endio(req, bio, bio_bytes, error);  in blk_update_request()
1494 	if (!req->bio) {  in blk_update_request()
1500 		req->__data_len = 0;  in blk_update_request()
1504 	req->__data_len -= total_bytes;  in blk_update_request()
1507 	if (!blk_rq_is_passthrough(req))  in blk_update_request()
1508 		req->__sector += total_bytes >> 9;  in blk_update_request()
1511 	if (req->rq_flags & RQF_MIXED_MERGE) {  in blk_update_request()
1512 		req->cmd_flags &= ~REQ_FAILFAST_MASK;  in blk_update_request()
1513 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;  in blk_update_request()
1516 	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {  in blk_update_request()
1521 		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {  in blk_update_request()
1522 			blk_dump_rq_flags(req, "request botched");  in blk_update_request()
1523 			req->__data_len = blk_rq_cur_bytes(req);  in blk_update_request()
1527 		req->nr_phys_segments = blk_recalc_rq_segments(req);  in blk_update_request()