Lines Matching refs:rq
50 #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) argument
56 static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) in elv_iosched_allow_merge() argument
58 struct request_queue *q = rq->q; in elv_iosched_allow_merge()
62 return e->type->ops.elevator_allow_merge_fn(q, rq, bio); in elv_iosched_allow_merge()
70 bool elv_rq_merge_ok(struct request *rq, struct bio *bio) in elv_rq_merge_ok() argument
72 if (!blk_rq_merge_ok(rq, bio)) in elv_rq_merge_ok()
75 if (!elv_iosched_allow_merge(rq, bio)) in elv_rq_merge_ok()
245 static inline void __elv_rqhash_del(struct request *rq) in __elv_rqhash_del() argument
247 hash_del(&rq->hash); in __elv_rqhash_del()
248 rq->cmd_flags &= ~REQ_HASHED; in __elv_rqhash_del()
251 static void elv_rqhash_del(struct request_queue *q, struct request *rq) in elv_rqhash_del() argument
253 if (ELV_ON_HASH(rq)) in elv_rqhash_del()
254 __elv_rqhash_del(rq); in elv_rqhash_del()
257 static void elv_rqhash_add(struct request_queue *q, struct request *rq) in elv_rqhash_add() argument
261 BUG_ON(ELV_ON_HASH(rq)); in elv_rqhash_add()
262 hash_add(e->hash, &rq->hash, rq_hash_key(rq)); in elv_rqhash_add()
263 rq->cmd_flags |= REQ_HASHED; in elv_rqhash_add()
266 static void elv_rqhash_reposition(struct request_queue *q, struct request *rq) in elv_rqhash_reposition() argument
268 __elv_rqhash_del(rq); in elv_rqhash_reposition()
269 elv_rqhash_add(q, rq); in elv_rqhash_reposition()
276 struct request *rq; in elv_rqhash_find() local
278 hash_for_each_possible_safe(e->hash, rq, next, hash, offset) { in elv_rqhash_find()
279 BUG_ON(!ELV_ON_HASH(rq)); in elv_rqhash_find()
281 if (unlikely(!rq_mergeable(rq))) { in elv_rqhash_find()
282 __elv_rqhash_del(rq); in elv_rqhash_find()
286 if (rq_hash_key(rq) == offset) in elv_rqhash_find()
287 return rq; in elv_rqhash_find()
297 void elv_rb_add(struct rb_root *root, struct request *rq) in elv_rb_add() argument
307 if (blk_rq_pos(rq) < blk_rq_pos(__rq)) in elv_rb_add()
309 else if (blk_rq_pos(rq) >= blk_rq_pos(__rq)) in elv_rb_add()
313 rb_link_node(&rq->rb_node, parent, p); in elv_rb_add()
314 rb_insert_color(&rq->rb_node, root); in elv_rb_add()
318 void elv_rb_del(struct rb_root *root, struct request *rq) in elv_rb_del() argument
320 BUG_ON(RB_EMPTY_NODE(&rq->rb_node)); in elv_rb_del()
321 rb_erase(&rq->rb_node, root); in elv_rb_del()
322 RB_CLEAR_NODE(&rq->rb_node); in elv_rb_del()
329 struct request *rq; in elv_rb_find() local
332 rq = rb_entry(n, struct request, rb_node); in elv_rb_find()
334 if (sector < blk_rq_pos(rq)) in elv_rb_find()
336 else if (sector > blk_rq_pos(rq)) in elv_rb_find()
339 return rq; in elv_rb_find()
351 void elv_dispatch_sort(struct request_queue *q, struct request *rq) in elv_dispatch_sort() argument
357 if (q->last_merge == rq) in elv_dispatch_sort()
360 elv_rqhash_del(q, rq); in elv_dispatch_sort()
369 if ((rq->cmd_flags & REQ_DISCARD) != in elv_dispatch_sort()
372 if (rq_data_dir(rq) != rq_data_dir(pos)) in elv_dispatch_sort()
376 if (blk_rq_pos(rq) >= boundary) { in elv_dispatch_sort()
383 if (blk_rq_pos(rq) >= blk_rq_pos(pos)) in elv_dispatch_sort()
387 list_add(&rq->queuelist, entry); in elv_dispatch_sort()
396 void elv_dispatch_add_tail(struct request_queue *q, struct request *rq) in elv_dispatch_add_tail() argument
398 if (q->last_merge == rq) in elv_dispatch_add_tail()
401 elv_rqhash_del(q, rq); in elv_dispatch_add_tail()
405 q->end_sector = rq_end_sector(rq); in elv_dispatch_add_tail()
406 q->boundary_rq = rq; in elv_dispatch_add_tail()
407 list_add_tail(&rq->queuelist, &q->queue_head); in elv_dispatch_add_tail()
463 struct request *rq) in elv_attempt_insert_merge() argument
474 if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) in elv_attempt_insert_merge()
485 __rq = elv_rqhash_find(q, blk_rq_pos(rq)); in elv_attempt_insert_merge()
486 if (!__rq || !blk_attempt_req_merge(q, __rq, rq)) in elv_attempt_insert_merge()
491 rq = __rq; in elv_attempt_insert_merge()
497 void elv_merged_request(struct request_queue *q, struct request *rq, int type) in elv_merged_request() argument
502 e->type->ops.elevator_merged_fn(q, rq, type); in elv_merged_request()
505 elv_rqhash_reposition(q, rq); in elv_merged_request()
507 q->last_merge = rq; in elv_merged_request()
510 void elv_merge_requests(struct request_queue *q, struct request *rq, in elv_merge_requests() argument
517 e->type->ops.elevator_merge_req_fn(q, rq, next); in elv_merge_requests()
519 elv_rqhash_reposition(q, rq); in elv_merge_requests()
526 q->last_merge = rq; in elv_merge_requests()
529 void elv_bio_merged(struct request_queue *q, struct request *rq, in elv_bio_merged() argument
535 e->type->ops.elevator_bio_merged_fn(q, rq, bio); in elv_bio_merged()
539 static void blk_pm_requeue_request(struct request *rq) in blk_pm_requeue_request() argument
541 if (rq->q->dev && !(rq->cmd_flags & REQ_PM)) in blk_pm_requeue_request()
542 rq->q->nr_pending--; in blk_pm_requeue_request()
545 static void blk_pm_add_request(struct request_queue *q, struct request *rq) in blk_pm_add_request() argument
547 if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 && in blk_pm_add_request()
552 static inline void blk_pm_requeue_request(struct request *rq) {} in blk_pm_requeue_request() argument
554 struct request *rq) in blk_pm_add_request() argument
559 void elv_requeue_request(struct request_queue *q, struct request *rq) in elv_requeue_request() argument
565 if (blk_account_rq(rq)) { in elv_requeue_request()
566 q->in_flight[rq_is_sync(rq)]--; in elv_requeue_request()
567 if (rq->cmd_flags & REQ_SORTED) in elv_requeue_request()
568 elv_deactivate_rq(q, rq); in elv_requeue_request()
571 rq->cmd_flags &= ~REQ_STARTED; in elv_requeue_request()
573 blk_pm_requeue_request(rq); in elv_requeue_request()
575 __elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE); in elv_requeue_request()
593 void __elv_add_request(struct request_queue *q, struct request *rq, int where) in __elv_add_request() argument
595 trace_block_rq_insert(q, rq); in __elv_add_request()
597 blk_pm_add_request(q, rq); in __elv_add_request()
599 rq->q = q; in __elv_add_request()
601 if (rq->cmd_flags & REQ_SOFTBARRIER) { in __elv_add_request()
603 if (rq->cmd_type == REQ_TYPE_FS) { in __elv_add_request()
604 q->end_sector = rq_end_sector(rq); in __elv_add_request()
605 q->boundary_rq = rq; in __elv_add_request()
607 } else if (!(rq->cmd_flags & REQ_ELVPRIV) && in __elv_add_request()
615 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
616 list_add(&rq->queuelist, &q->queue_head); in __elv_add_request()
620 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
622 list_add_tail(&rq->queuelist, &q->queue_head); in __elv_add_request()
642 if (elv_attempt_insert_merge(q, rq)) in __elv_add_request()
645 BUG_ON(rq->cmd_type != REQ_TYPE_FS); in __elv_add_request()
646 rq->cmd_flags |= REQ_SORTED; in __elv_add_request()
648 if (rq_mergeable(rq)) { in __elv_add_request()
649 elv_rqhash_add(q, rq); in __elv_add_request()
651 q->last_merge = rq; in __elv_add_request()
659 q->elevator->type->ops.elevator_add_req_fn(q, rq); in __elv_add_request()
663 rq->cmd_flags |= REQ_SOFTBARRIER; in __elv_add_request()
664 blk_insert_flush(rq); in __elv_add_request()
674 void elv_add_request(struct request_queue *q, struct request *rq, int where) in elv_add_request() argument
679 __elv_add_request(q, rq, where); in elv_add_request()
684 struct request *elv_latter_request(struct request_queue *q, struct request *rq) in elv_latter_request() argument
689 return e->type->ops.elevator_latter_req_fn(q, rq); in elv_latter_request()
693 struct request *elv_former_request(struct request_queue *q, struct request *rq) in elv_former_request() argument
698 return e->type->ops.elevator_former_req_fn(q, rq); in elv_former_request()
702 int elv_set_request(struct request_queue *q, struct request *rq, in elv_set_request() argument
708 return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask); in elv_set_request()
712 void elv_put_request(struct request_queue *q, struct request *rq) in elv_put_request() argument
717 e->type->ops.elevator_put_req_fn(rq); in elv_put_request()
730 void elv_completed_request(struct request_queue *q, struct request *rq) in elv_completed_request() argument
737 if (blk_account_rq(rq)) { in elv_completed_request()
738 q->in_flight[rq_is_sync(rq)]--; in elv_completed_request()
739 if ((rq->cmd_flags & REQ_SORTED) && in elv_completed_request()
741 e->type->ops.elevator_completed_req_fn(q, rq); in elv_completed_request()
1028 struct request *rq) in elv_rb_former_request() argument
1030 struct rb_node *rbprev = rb_prev(&rq->rb_node); in elv_rb_former_request()
1040 struct request *rq) in elv_rb_latter_request() argument
1042 struct rb_node *rbnext = rb_next(&rq->rb_node); in elv_rb_latter_request()