54 #define rq_hash_key(rq)		(blk_rq_pos(rq) + blk_rq_sectors(rq))  argument
60 static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)  in elv_iosched_allow_bio_merge()  argument
62 	struct request_queue *q = rq->q;  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
74 bool elv_bio_merge_ok(struct request *rq, struct bio *bio)  in elv_bio_merge_ok()  argument
76 	if (!blk_rq_merge_ok(rq, bio))  in elv_bio_merge_ok()
79 	if (!elv_iosched_allow_bio_merge(rq, bio))  in elv_bio_merge_ok()
200 static inline void __elv_rqhash_del(struct request *rq)  in __elv_rqhash_del()  argument
202 	hash_del(&rq->hash);  in __elv_rqhash_del()
203 	rq->rq_flags &= ~RQF_HASHED;  in __elv_rqhash_del()
206 void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
208 	if (ELV_ON_HASH(rq))  in elv_rqhash_del()
209 		__elv_rqhash_del(rq);  in elv_rqhash_del()
213 void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
217 	BUG_ON(ELV_ON_HASH(rq));  in elv_rqhash_add()
218 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));  in elv_rqhash_add()
219 	rq->rq_flags |= RQF_HASHED;  in elv_rqhash_add()
223 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
225 	__elv_rqhash_del(rq);  in elv_rqhash_reposition()
226 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
233 	struct request *rq;  in elv_rqhash_find()  local
235 	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {  in elv_rqhash_find()
236 		BUG_ON(!ELV_ON_HASH(rq));  in elv_rqhash_find()
238 		if (unlikely(!rq_mergeable(rq))) {  in elv_rqhash_find()
239 			__elv_rqhash_del(rq);  in elv_rqhash_find()
243 		if (rq_hash_key(rq) == offset)  in elv_rqhash_find()
244 			return rq;  in elv_rqhash_find()
254 void elv_rb_add(struct rb_root *root, struct request *rq)  in elv_rb_add()  argument
264 		if (blk_rq_pos(rq) < blk_rq_pos(__rq))  in elv_rb_add()
266 		else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))  in elv_rb_add()
270 	rb_link_node(&rq->rb_node, parent, p);  in elv_rb_add()
271 	rb_insert_color(&rq->rb_node, root);  in elv_rb_add()
275 void elv_rb_del(struct rb_root *root, struct request *rq)  in elv_rb_del()  argument
277 	BUG_ON(RB_EMPTY_NODE(&rq->rb_node));  in elv_rb_del()
278 	rb_erase(&rq->rb_node, root);  in elv_rb_del()
279 	RB_CLEAR_NODE(&rq->rb_node);  in elv_rb_del()
286 	struct request *rq;  in elv_rb_find()  local
289 		rq = rb_entry(n, struct request, rb_node);  in elv_rb_find()
291 		if (sector < blk_rq_pos(rq))  in elv_rb_find()
293 		else if (sector > blk_rq_pos(rq))  in elv_rb_find()
296 			return rq;  in elv_rb_find()
358 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)  in elv_attempt_insert_merge()  argument
369 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))  in elv_attempt_insert_merge()
380 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
381 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
386 		rq = __rq;  in elv_attempt_insert_merge()
392 void elv_merged_request(struct request_queue *q, struct request *rq,  in elv_merged_request()  argument
398 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
401 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
403 	q->last_merge = rq;  in elv_merged_request()
406 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
412 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
414 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
415 	q->last_merge = rq;  in elv_merge_requests()
418 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
423 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
428 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
433 		return e->type->ops.former_request(q, rq);  in elv_former_request()
810 				      struct request *rq)  in elv_rb_former_request()  argument
812 	struct rb_node *rbprev = rb_prev(&rq->rb_node);  in elv_rb_former_request()
822 				      struct request *rq)  in elv_rb_latter_request()  argument
824 	struct rb_node *rbnext = rb_next(&rq->rb_node);  in elv_rb_latter_request()