55 static inline bool req_gap_back_merge(struct request *req, struct bio *bio)  in req_gap_back_merge()  argument
57 	return bio_will_gap(req->q, req, req->biotail, bio);  in req_gap_back_merge()
60 static inline bool req_gap_front_merge(struct request *req, struct bio *bio)  in req_gap_front_merge()  argument
62 	return bio_will_gap(req->q, NULL, bio, req->bio);  in req_gap_front_merge()
557 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,  in ll_new_hw_segment()  argument
560 	if (!blk_cgroup_mergeable(req, bio))  in ll_new_hw_segment()
563 	if (blk_integrity_merge_bio(req->q, req, bio) == false)  in ll_new_hw_segment()
567 	if (req_op(req) == REQ_OP_DISCARD)  in ll_new_hw_segment()
570 	if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))  in ll_new_hw_segment()
577 	req->nr_phys_segments += nr_phys_segs;  in ll_new_hw_segment()
581 	req_set_nomerge(req->q, req);  in ll_new_hw_segment()
585 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)  in ll_back_merge_fn()  argument
587 	if (req_gap_back_merge(req, bio))  in ll_back_merge_fn()
589 	if (blk_integrity_rq(req) &&  in ll_back_merge_fn()
590 	    integrity_req_gap_back_merge(req, bio))  in ll_back_merge_fn()
592 	if (!bio_crypt_ctx_back_mergeable(req, bio))  in ll_back_merge_fn()
594 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_back_merge_fn()
595 	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {  in ll_back_merge_fn()
596 		req_set_nomerge(req->q, req);  in ll_back_merge_fn()
600 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_back_merge_fn()
603 static int ll_front_merge_fn(struct request *req, struct bio *bio,  in ll_front_merge_fn()  argument
606 	if (req_gap_front_merge(req, bio))  in ll_front_merge_fn()
608 	if (blk_integrity_rq(req) &&  in ll_front_merge_fn()
609 	    integrity_req_gap_front_merge(req, bio))  in ll_front_merge_fn()
611 	if (!bio_crypt_ctx_front_mergeable(req, bio))  in ll_front_merge_fn()
613 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_front_merge_fn()
614 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {  in ll_front_merge_fn()
615 		req_set_nomerge(req->q, req);  in ll_front_merge_fn()
619 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_front_merge_fn()
622 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,  in req_attempt_discard_merge()  argument
625 	unsigned short segments = blk_rq_nr_discard_segments(req);  in req_attempt_discard_merge()
629 	if (blk_rq_sectors(req) + bio_sectors(next->bio) >  in req_attempt_discard_merge()
630 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in req_attempt_discard_merge()
633 	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);  in req_attempt_discard_merge()
636 	req_set_nomerge(q, req);  in req_attempt_discard_merge()
640 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,  in ll_merge_requests_fn()  argument
645 	if (req_gap_back_merge(req, next->bio))  in ll_merge_requests_fn()
651 	if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >  in ll_merge_requests_fn()
652 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in ll_merge_requests_fn()
655 	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;  in ll_merge_requests_fn()
656 	if (total_phys_segments > blk_rq_get_max_segments(req))  in ll_merge_requests_fn()
659 	if (!blk_cgroup_mergeable(req, next->bio))  in ll_merge_requests_fn()
662 	if (blk_integrity_merge_rq(q, req, next) == false)  in ll_merge_requests_fn()
665 	if (!bio_crypt_ctx_merge_rq(req, next))  in ll_merge_requests_fn()
669 	req->nr_phys_segments = total_phys_segments;  in ll_merge_requests_fn()
703 static void blk_account_io_merge_request(struct request *req)  in blk_account_io_merge_request()  argument
705 	if (blk_do_io_stat(req)) {  in blk_account_io_merge_request()
707 		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);  in blk_account_io_merge_request()
710 		hd_struct_put(req->part);  in blk_account_io_merge_request()
714 static enum elv_merge blk_try_req_merge(struct request *req,  in blk_try_req_merge()  argument
717 	if (blk_discard_mergable(req))  in blk_try_req_merge()
719 	else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))  in blk_try_req_merge()
730 				     struct request *req, struct request *next)  in attempt_merge()  argument
732 	if (!rq_mergeable(req) || !rq_mergeable(next))  in attempt_merge()
735 	if (req_op(req) != req_op(next))  in attempt_merge()
738 	if (rq_data_dir(req) != rq_data_dir(next)  in attempt_merge()
739 	    || req->rq_disk != next->rq_disk)  in attempt_merge()
742 	if (req_op(req) == REQ_OP_WRITE_SAME &&  in attempt_merge()
743 	    !blk_write_same_mergeable(req->bio, next->bio))  in attempt_merge()
750 	if (req->write_hint != next->write_hint)  in attempt_merge()
753 	if (req->ioprio != next->ioprio)  in attempt_merge()
764 	switch (blk_try_req_merge(req, next)) {  in attempt_merge()
766 		if (!req_attempt_discard_merge(q, req, next))  in attempt_merge()
770 		if (!ll_merge_requests_fn(q, req, next))  in attempt_merge()
783 	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||  in attempt_merge()
784 	    (req->cmd_flags & REQ_FAILFAST_MASK) !=  in attempt_merge()
786 		blk_rq_set_mixed_merge(req);  in attempt_merge()
795 	if (next->start_time_ns < req->start_time_ns)  in attempt_merge()
796 		req->start_time_ns = next->start_time_ns;  in attempt_merge()
798 	req->biotail->bi_next = next->bio;  in attempt_merge()
799 	req->biotail = next->biotail;  in attempt_merge()
801 	req->__data_len += blk_rq_bytes(next);  in attempt_merge()
803 	if (!blk_discard_mergable(req))  in attempt_merge()
804 		elv_merge_requests(q, req, next);  in attempt_merge()
916 static void blk_account_io_merge_bio(struct request *req)  in blk_account_io_merge_bio()  argument
918 	if (!blk_do_io_stat(req))  in blk_account_io_merge_bio()
922 	part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);  in blk_account_io_merge_bio()
932 static enum bio_merge_status bio_attempt_back_merge(struct request *req,  in bio_attempt_back_merge()  argument
937 	if (!ll_back_merge_fn(req, bio, nr_segs))  in bio_attempt_back_merge()
940 	trace_block_bio_backmerge(req->q, req, bio);  in bio_attempt_back_merge()
941 	rq_qos_merge(req->q, req, bio);  in bio_attempt_back_merge()
943 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_back_merge()
944 		blk_rq_set_mixed_merge(req);  in bio_attempt_back_merge()
946 	req->biotail->bi_next = bio;  in bio_attempt_back_merge()
947 	req->biotail = bio;  in bio_attempt_back_merge()
948 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_back_merge()
952 	blk_account_io_merge_bio(req);  in bio_attempt_back_merge()
956 static enum bio_merge_status bio_attempt_front_merge(struct request *req,  in bio_attempt_front_merge()  argument
961 	if (!ll_front_merge_fn(req, bio, nr_segs))  in bio_attempt_front_merge()
964 	trace_block_bio_frontmerge(req->q, req, bio);  in bio_attempt_front_merge()
965 	rq_qos_merge(req->q, req, bio);  in bio_attempt_front_merge()
967 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_front_merge()
968 		blk_rq_set_mixed_merge(req);  in bio_attempt_front_merge()
970 	bio->bi_next = req->bio;  in bio_attempt_front_merge()
971 	req->bio = bio;  in bio_attempt_front_merge()
973 	req->__sector = bio->bi_iter.bi_sector;  in bio_attempt_front_merge()
974 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_front_merge()
976 	bio_crypt_do_front_merge(req, bio);  in bio_attempt_front_merge()
978 	blk_account_io_merge_bio(req);  in bio_attempt_front_merge()
983 		struct request *req, struct bio *bio)  in bio_attempt_discard_merge()  argument
985 	unsigned short segments = blk_rq_nr_discard_segments(req);  in bio_attempt_discard_merge()
989 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in bio_attempt_discard_merge()
990 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in bio_attempt_discard_merge()
993 	rq_qos_merge(q, req, bio);  in bio_attempt_discard_merge()
995 	req->biotail->bi_next = bio;  in bio_attempt_discard_merge()
996 	req->biotail = bio;  in bio_attempt_discard_merge()
997 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_discard_merge()
998 	req->nr_phys_segments = segments + 1;  in bio_attempt_discard_merge()
1000 	blk_account_io_merge_bio(req);  in bio_attempt_discard_merge()
1003 	req_set_nomerge(q, req);  in bio_attempt_discard_merge()