Lines Matching refs:rq
50 #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) argument
56 static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) in elv_iosched_allow_merge() argument
58 struct request_queue *q = rq->q; in elv_iosched_allow_merge()
62 return e->type->ops.elevator_allow_merge_fn(q, rq, bio); in elv_iosched_allow_merge()
70 bool elv_rq_merge_ok(struct request *rq, struct bio *bio) in elv_rq_merge_ok() argument
72 if (!blk_rq_merge_ok(rq, bio)) in elv_rq_merge_ok()
75 if (!elv_iosched_allow_merge(rq, bio)) in elv_rq_merge_ok()
249 static inline void __elv_rqhash_del(struct request *rq) in __elv_rqhash_del() argument
251 hash_del(&rq->hash); in __elv_rqhash_del()
254 static void elv_rqhash_del(struct request_queue *q, struct request *rq) in elv_rqhash_del() argument
256 if (ELV_ON_HASH(rq)) in elv_rqhash_del()
257 __elv_rqhash_del(rq); in elv_rqhash_del()
260 static void elv_rqhash_add(struct request_queue *q, struct request *rq) in elv_rqhash_add() argument
264 BUG_ON(ELV_ON_HASH(rq)); in elv_rqhash_add()
265 hash_add(e->hash, &rq->hash, rq_hash_key(rq)); in elv_rqhash_add()
268 static void elv_rqhash_reposition(struct request_queue *q, struct request *rq) in elv_rqhash_reposition() argument
270 __elv_rqhash_del(rq); in elv_rqhash_reposition()
271 elv_rqhash_add(q, rq); in elv_rqhash_reposition()
278 struct request *rq; in elv_rqhash_find() local
280 hash_for_each_possible_safe(e->hash, rq, next, hash, offset) { in elv_rqhash_find()
281 BUG_ON(!ELV_ON_HASH(rq)); in elv_rqhash_find()
283 if (unlikely(!rq_mergeable(rq))) { in elv_rqhash_find()
284 __elv_rqhash_del(rq); in elv_rqhash_find()
288 if (rq_hash_key(rq) == offset) in elv_rqhash_find()
289 return rq; in elv_rqhash_find()
299 void elv_rb_add(struct rb_root *root, struct request *rq) in elv_rb_add() argument
309 if (blk_rq_pos(rq) < blk_rq_pos(__rq)) in elv_rb_add()
311 else if (blk_rq_pos(rq) >= blk_rq_pos(__rq)) in elv_rb_add()
315 rb_link_node(&rq->rb_node, parent, p); in elv_rb_add()
316 rb_insert_color(&rq->rb_node, root); in elv_rb_add()
320 void elv_rb_del(struct rb_root *root, struct request *rq) in elv_rb_del() argument
322 BUG_ON(RB_EMPTY_NODE(&rq->rb_node)); in elv_rb_del()
323 rb_erase(&rq->rb_node, root); in elv_rb_del()
324 RB_CLEAR_NODE(&rq->rb_node); in elv_rb_del()
331 struct request *rq; in elv_rb_find() local
334 rq = rb_entry(n, struct request, rb_node); in elv_rb_find()
336 if (sector < blk_rq_pos(rq)) in elv_rb_find()
338 else if (sector > blk_rq_pos(rq)) in elv_rb_find()
341 return rq; in elv_rb_find()
353 void elv_dispatch_sort(struct request_queue *q, struct request *rq) in elv_dispatch_sort() argument
359 if (q->last_merge == rq) in elv_dispatch_sort()
362 elv_rqhash_del(q, rq); in elv_dispatch_sort()
371 if ((rq->cmd_flags & REQ_DISCARD) != in elv_dispatch_sort()
374 if (rq_data_dir(rq) != rq_data_dir(pos)) in elv_dispatch_sort()
378 if (blk_rq_pos(rq) >= boundary) { in elv_dispatch_sort()
385 if (blk_rq_pos(rq) >= blk_rq_pos(pos)) in elv_dispatch_sort()
389 list_add(&rq->queuelist, entry); in elv_dispatch_sort()
398 void elv_dispatch_add_tail(struct request_queue *q, struct request *rq) in elv_dispatch_add_tail() argument
400 if (q->last_merge == rq) in elv_dispatch_add_tail()
403 elv_rqhash_del(q, rq); in elv_dispatch_add_tail()
407 q->end_sector = rq_end_sector(rq); in elv_dispatch_add_tail()
408 q->boundary_rq = rq; in elv_dispatch_add_tail()
409 list_add_tail(&rq->queuelist, &q->queue_head); in elv_dispatch_add_tail()
465 struct request *rq) in elv_attempt_insert_merge() argument
476 if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) in elv_attempt_insert_merge()
487 __rq = elv_rqhash_find(q, blk_rq_pos(rq)); in elv_attempt_insert_merge()
488 if (!__rq || !blk_attempt_req_merge(q, __rq, rq)) in elv_attempt_insert_merge()
493 rq = __rq; in elv_attempt_insert_merge()
499 void elv_merged_request(struct request_queue *q, struct request *rq, int type) in elv_merged_request() argument
504 e->type->ops.elevator_merged_fn(q, rq, type); in elv_merged_request()
507 elv_rqhash_reposition(q, rq); in elv_merged_request()
509 q->last_merge = rq; in elv_merged_request()
512 void elv_merge_requests(struct request_queue *q, struct request *rq, in elv_merge_requests() argument
519 e->type->ops.elevator_merge_req_fn(q, rq, next); in elv_merge_requests()
521 elv_rqhash_reposition(q, rq); in elv_merge_requests()
528 q->last_merge = rq; in elv_merge_requests()
531 void elv_bio_merged(struct request_queue *q, struct request *rq, in elv_bio_merged() argument
537 e->type->ops.elevator_bio_merged_fn(q, rq, bio); in elv_bio_merged()
541 static void blk_pm_requeue_request(struct request *rq) in blk_pm_requeue_request() argument
543 if (rq->q->dev && !(rq->cmd_flags & REQ_PM)) in blk_pm_requeue_request()
544 rq->q->nr_pending--; in blk_pm_requeue_request()
547 static void blk_pm_add_request(struct request_queue *q, struct request *rq) in blk_pm_add_request() argument
549 if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 && in blk_pm_add_request()
554 static inline void blk_pm_requeue_request(struct request *rq) {} in blk_pm_requeue_request() argument
556 struct request *rq) in blk_pm_add_request() argument
561 void elv_requeue_request(struct request_queue *q, struct request *rq) in elv_requeue_request() argument
567 if (blk_account_rq(rq)) { in elv_requeue_request()
568 q->in_flight[rq_is_sync(rq)]--; in elv_requeue_request()
569 if (rq->cmd_flags & REQ_SORTED) in elv_requeue_request()
570 elv_deactivate_rq(q, rq); in elv_requeue_request()
573 rq->cmd_flags &= ~REQ_STARTED; in elv_requeue_request()
575 blk_pm_requeue_request(rq); in elv_requeue_request()
577 __elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE); in elv_requeue_request()
595 void __elv_add_request(struct request_queue *q, struct request *rq, int where) in __elv_add_request() argument
597 trace_block_rq_insert(q, rq); in __elv_add_request()
599 blk_pm_add_request(q, rq); in __elv_add_request()
601 rq->q = q; in __elv_add_request()
603 if (rq->cmd_flags & REQ_SOFTBARRIER) { in __elv_add_request()
605 if (rq->cmd_type == REQ_TYPE_FS) { in __elv_add_request()
606 q->end_sector = rq_end_sector(rq); in __elv_add_request()
607 q->boundary_rq = rq; in __elv_add_request()
609 } else if (!(rq->cmd_flags & REQ_ELVPRIV) && in __elv_add_request()
617 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
618 list_add(&rq->queuelist, &q->queue_head); in __elv_add_request()
622 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
624 list_add_tail(&rq->queuelist, &q->queue_head); in __elv_add_request()
644 if (elv_attempt_insert_merge(q, rq)) in __elv_add_request()
647 BUG_ON(rq->cmd_type != REQ_TYPE_FS); in __elv_add_request()
648 rq->cmd_flags |= REQ_SORTED; in __elv_add_request()
650 if (rq_mergeable(rq)) { in __elv_add_request()
651 elv_rqhash_add(q, rq); in __elv_add_request()
653 q->last_merge = rq; in __elv_add_request()
661 q->elevator->type->ops.elevator_add_req_fn(q, rq); in __elv_add_request()
665 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
666 blk_insert_flush(rq); in __elv_add_request()
676 void elv_add_request(struct request_queue *q, struct request *rq, int where) in elv_add_request() argument
681 __elv_add_request(q, rq, where); in elv_add_request()
686 struct request *elv_latter_request(struct request_queue *q, struct request *rq) in elv_latter_request() argument
691 return e->type->ops.elevator_latter_req_fn(q, rq); in elv_latter_request()
695 struct request *elv_former_request(struct request_queue *q, struct request *rq) in elv_former_request() argument
700 return e->type->ops.elevator_former_req_fn(q, rq); in elv_former_request()
704 int elv_set_request(struct request_queue *q, struct request *rq, in elv_set_request() argument
710 return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask); in elv_set_request()
714 void elv_put_request(struct request_queue *q, struct request *rq) in elv_put_request() argument
719 e->type->ops.elevator_put_req_fn(rq); in elv_put_request()
734 struct request *rq; in elv_abort_queue() local
739 rq = list_entry_rq(q->queue_head.next); in elv_abort_queue()
740 rq->cmd_flags |= REQ_QUIET; in elv_abort_queue()
741 trace_block_rq_abort(q, rq); in elv_abort_queue()
746 blk_start_request(rq); in elv_abort_queue()
747 __blk_end_request_all(rq, -EIO); in elv_abort_queue()
752 void elv_completed_request(struct request_queue *q, struct request *rq) in elv_completed_request() argument
759 if (blk_account_rq(rq)) { in elv_completed_request()
760 q->in_flight[rq_is_sync(rq)]--; in elv_completed_request()
761 if ((rq->cmd_flags & REQ_SORTED) && in elv_completed_request()
763 e->type->ops.elevator_completed_req_fn(q, rq); in elv_completed_request()
1043 struct request *rq) in elv_rb_former_request() argument
1045 struct rb_node *rbprev = rb_prev(&rq->rb_node); in elv_rb_former_request()
1055 struct request *rq) in elv_rb_latter_request() argument
1057 struct rb_node *rbnext = rb_next(&rq->rb_node); in elv_rb_latter_request()