Lines Matching refs:bfqq
136 void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \
138 __set_bit(BFQQF_##name, &(bfqq)->flags); \
140 void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \
142 __clear_bit(BFQQF_##name, &(bfqq)->flags); \
144 int bfq_bfqq_##name(const struct bfq_queue *bfqq) \
146 return test_bit(BFQQF_##name, &(bfqq)->flags); \
237 #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) argument
245 #define BFQQ_TOTALLY_SEEKY(bfqq) (bfqq->seek_history == -1) argument
371 return bic->bfqq[is_sync]; in bic_to_bfqq()
374 void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync) in bic_set_bfqq() argument
376 struct bfq_queue *old_bfqq = bic->bfqq[is_sync]; in bic_set_bfqq()
382 bic->bfqq[is_sync] = bfqq; in bic_set_bfqq()
438 #define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) argument
564 struct bfq_queue *bfqq = NULL; in bfq_rq_pos_tree_lookup() local
572 bfqq = rb_entry(parent, struct bfq_queue, pos_node); in bfq_rq_pos_tree_lookup()
578 if (sector > blk_rq_pos(bfqq->next_rq)) in bfq_rq_pos_tree_lookup()
580 else if (sector < blk_rq_pos(bfqq->next_rq)) in bfq_rq_pos_tree_lookup()
585 bfqq = NULL; in bfq_rq_pos_tree_lookup()
594 bfqq ? bfqq->pid : 0); in bfq_rq_pos_tree_lookup()
596 return bfqq; in bfq_rq_pos_tree_lookup()
599 static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) in bfq_too_late_for_merging() argument
601 return bfqq->service_from_backlogged > 0 && in bfq_too_late_for_merging()
602 time_is_before_jiffies(bfqq->first_IO_time + in bfq_too_late_for_merging()
615 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_pos_tree_add_move() argument
620 if (bfqq->pos_root) { in bfq_pos_tree_add_move()
621 rb_erase(&bfqq->pos_node, bfqq->pos_root); in bfq_pos_tree_add_move()
622 bfqq->pos_root = NULL; in bfq_pos_tree_add_move()
626 if (bfqq == &bfqd->oom_bfqq) in bfq_pos_tree_add_move()
634 if (bfq_too_late_for_merging(bfqq)) in bfq_pos_tree_add_move()
637 if (bfq_class_idle(bfqq)) in bfq_pos_tree_add_move()
639 if (!bfqq->next_rq) in bfq_pos_tree_add_move()
642 bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; in bfq_pos_tree_add_move()
643 __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, in bfq_pos_tree_add_move()
644 blk_rq_pos(bfqq->next_rq), &parent, &p); in bfq_pos_tree_add_move()
646 rb_link_node(&bfqq->pos_node, parent, p); in bfq_pos_tree_add_move()
647 rb_insert_color(&bfqq->pos_node, bfqq->pos_root); in bfq_pos_tree_add_move()
649 bfqq->pos_root = NULL; in bfq_pos_tree_add_move()
686 struct bfq_queue *bfqq) in bfq_asymmetric_scenario() argument
688 bool smallest_weight = bfqq && in bfq_asymmetric_scenario()
689 bfqq->weight_counter && in bfq_asymmetric_scenario()
690 bfqq->weight_counter == in bfq_asymmetric_scenario()
730 void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_weights_tree_add() argument
733 struct bfq_entity *entity = &bfqq->entity; in bfq_weights_tree_add()
749 if (bfqq->weight_counter) in bfq_weights_tree_add()
759 bfqq->weight_counter = __counter; in bfq_weights_tree_add()
770 bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), in bfq_weights_tree_add()
785 if (unlikely(!bfqq->weight_counter)) in bfq_weights_tree_add()
788 bfqq->weight_counter->weight = entity->weight; in bfq_weights_tree_add()
789 rb_link_node(&bfqq->weight_counter->weights_node, parent, new); in bfq_weights_tree_add()
790 rb_insert_color_cached(&bfqq->weight_counter->weights_node, root, in bfq_weights_tree_add()
794 bfqq->weight_counter->num_active++; in bfq_weights_tree_add()
795 bfqq->ref++; in bfq_weights_tree_add()
805 struct bfq_queue *bfqq, in __bfq_weights_tree_remove() argument
808 if (!bfqq->weight_counter) in __bfq_weights_tree_remove()
811 bfqq->weight_counter->num_active--; in __bfq_weights_tree_remove()
812 if (bfqq->weight_counter->num_active > 0) in __bfq_weights_tree_remove()
815 rb_erase_cached(&bfqq->weight_counter->weights_node, root); in __bfq_weights_tree_remove()
816 kfree(bfqq->weight_counter); in __bfq_weights_tree_remove()
819 bfqq->weight_counter = NULL; in __bfq_weights_tree_remove()
820 bfq_put_queue(bfqq); in __bfq_weights_tree_remove()
828 struct bfq_queue *bfqq) in bfq_weights_tree_remove() argument
830 struct bfq_entity *entity = bfqq->entity.parent; in bfq_weights_tree_remove()
872 __bfq_weights_tree_remove(bfqd, bfqq, in bfq_weights_tree_remove()
879 static struct request *bfq_check_fifo(struct bfq_queue *bfqq, in bfq_check_fifo() argument
884 if (bfq_bfqq_fifo_expire(bfqq)) in bfq_check_fifo()
887 bfq_mark_bfqq_fifo_expire(bfqq); in bfq_check_fifo()
889 rq = rq_entry_fifo(bfqq->fifo.next); in bfq_check_fifo()
894 bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); in bfq_check_fifo()
899 struct bfq_queue *bfqq, in bfq_find_next_rq() argument
907 next = bfq_check_fifo(bfqq, last); in bfq_find_next_rq()
917 rbnext = rb_first(&bfqq->sort_list); in bfq_find_next_rq()
927 struct bfq_queue *bfqq) in bfq_serv_to_charge() argument
929 if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 || in bfq_serv_to_charge()
930 bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) in bfq_serv_to_charge()
948 struct bfq_queue *bfqq) in bfq_updated_next_req() argument
950 struct bfq_entity *entity = &bfqq->entity; in bfq_updated_next_req()
951 struct request *next_rq = bfqq->next_rq; in bfq_updated_next_req()
957 if (bfqq == bfqd->in_service_queue) in bfq_updated_next_req()
965 max_t(unsigned long, bfqq->max_budget, in bfq_updated_next_req()
966 bfq_serv_to_charge(next_rq, bfqq)), in bfq_updated_next_req()
970 bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", in bfq_updated_next_req()
972 bfq_requeue_bfqq(bfqd, bfqq, false); in bfq_updated_next_req()
1010 static void switch_back_to_interactive_wr(struct bfq_queue *bfqq, in switch_back_to_interactive_wr() argument
1013 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in switch_back_to_interactive_wr()
1014 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in switch_back_to_interactive_wr()
1015 bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt; in switch_back_to_interactive_wr()
1019 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, in bfq_bfqq_resume_state() argument
1022 unsigned int old_wr_coeff = bfqq->wr_coeff; in bfq_bfqq_resume_state()
1023 bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); in bfq_bfqq_resume_state()
1026 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_bfqq_resume_state()
1028 bfq_clear_bfqq_has_short_ttime(bfqq); in bfq_bfqq_resume_state()
1031 bfq_mark_bfqq_IO_bound(bfqq); in bfq_bfqq_resume_state()
1033 bfq_clear_bfqq_IO_bound(bfqq); in bfq_bfqq_resume_state()
1035 bfqq->entity.new_weight = bic->saved_weight; in bfq_bfqq_resume_state()
1036 bfqq->ttime = bic->saved_ttime; in bfq_bfqq_resume_state()
1037 bfqq->wr_coeff = bic->saved_wr_coeff; in bfq_bfqq_resume_state()
1038 bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; in bfq_bfqq_resume_state()
1039 bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; in bfq_bfqq_resume_state()
1040 bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time; in bfq_bfqq_resume_state()
1042 if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || in bfq_bfqq_resume_state()
1043 time_is_before_jiffies(bfqq->last_wr_start_finish + in bfq_bfqq_resume_state()
1044 bfqq->wr_cur_max_time))) { in bfq_bfqq_resume_state()
1045 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_bfqq_resume_state()
1046 !bfq_bfqq_in_large_burst(bfqq) && in bfq_bfqq_resume_state()
1047 time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt + in bfq_bfqq_resume_state()
1049 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_bfqq_resume_state()
1051 bfqq->wr_coeff = 1; in bfq_bfqq_resume_state()
1052 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_bfqq_resume_state()
1058 bfqq->entity.prio_changed = 1; in bfq_bfqq_resume_state()
1063 if (old_wr_coeff == 1 && bfqq->wr_coeff > 1) in bfq_bfqq_resume_state()
1065 else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1) in bfq_bfqq_resume_state()
1069 static int bfqq_process_refs(struct bfq_queue *bfqq) in bfqq_process_refs() argument
1071 return bfqq->ref - bfqq->allocated - bfqq->entity.on_st_or_in_serv - in bfqq_process_refs()
1072 (bfqq->weight_counter != NULL); in bfqq_process_refs()
1076 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_reset_burst_list() argument
1090 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_reset_burst_list()
1095 bfqd->burst_parent_entity = bfqq->entity.parent; in bfq_reset_burst_list()
1099 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_add_to_burst() argument
1121 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_add_to_burst()
1139 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_add_to_burst()
1251 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_handle_burst() argument
1258 if (!hlist_unhashed(&bfqq->burst_list_node) || in bfq_handle_burst()
1259 bfq_bfqq_in_large_burst(bfqq) || in bfq_handle_burst()
1260 time_is_after_eq_jiffies(bfqq->split_time + in bfq_handle_burst()
1283 bfqq->entity.parent != bfqd->burst_parent_entity) { in bfq_handle_burst()
1285 bfq_reset_burst_list(bfqd, bfqq); in bfq_handle_burst()
1295 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_handle_burst()
1304 bfq_add_to_burst(bfqd, bfqq); in bfq_handle_burst()
1317 static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) in bfq_bfqq_budget_left() argument
1319 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_budget_left()
1454 struct bfq_queue *bfqq, in bfq_bfqq_update_budg_for_activation() argument
1457 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_update_budg_for_activation()
1466 if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time && in bfq_bfqq_update_budg_for_activation()
1467 bfq_bfqq_budget_left(bfqq) > 0) { in bfq_bfqq_update_budg_for_activation()
1486 bfq_bfqq_budget_left(bfqq), in bfq_bfqq_update_budg_for_activation()
1487 bfqq->max_budget); in bfq_bfqq_update_budg_for_activation()
1507 entity->budget = max_t(unsigned long, bfqq->max_budget, in bfq_bfqq_update_budg_for_activation()
1508 bfq_serv_to_charge(bfqq->next_rq, bfqq)); in bfq_bfqq_update_budg_for_activation()
1509 bfq_clear_bfqq_non_blocking_wait_rq(bfqq); in bfq_bfqq_update_budg_for_activation()
1523 struct bfq_queue *bfqq, in bfq_update_bfqq_wr_on_rq_arrival() argument
1533 bfqq->service_from_wr = 0; in bfq_update_bfqq_wr_on_rq_arrival()
1534 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1535 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1548 bfqq->wr_start_at_switch_to_srt = in bfq_update_bfqq_wr_on_rq_arrival()
1550 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1552 bfqq->wr_cur_max_time = in bfq_update_bfqq_wr_on_rq_arrival()
1565 bfqq->entity.budget = min_t(unsigned long, in bfq_update_bfqq_wr_on_rq_arrival()
1566 bfqq->entity.budget, in bfq_update_bfqq_wr_on_rq_arrival()
1570 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1571 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1573 bfqq->wr_coeff = 1; in bfq_update_bfqq_wr_on_rq_arrival()
1604 if (bfqq->wr_cur_max_time != in bfq_update_bfqq_wr_on_rq_arrival()
1606 bfqq->wr_start_at_switch_to_srt = in bfq_update_bfqq_wr_on_rq_arrival()
1607 bfqq->last_wr_start_finish; in bfq_update_bfqq_wr_on_rq_arrival()
1609 bfqq->wr_cur_max_time = in bfq_update_bfqq_wr_on_rq_arrival()
1611 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1614 bfqq->last_wr_start_finish = jiffies; in bfq_update_bfqq_wr_on_rq_arrival()
1620 struct bfq_queue *bfqq) in bfq_bfqq_idle_for_long_time() argument
1622 return bfqq->dispatched == 0 && in bfq_bfqq_idle_for_long_time()
1624 bfqq->budget_timeout + in bfq_bfqq_idle_for_long_time()
1633 static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, in bfq_bfqq_higher_class_or_weight() argument
1638 if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class) in bfq_bfqq_higher_class_or_weight()
1641 if (in_serv_bfqq->entity.parent == bfqq->entity.parent) { in bfq_bfqq_higher_class_or_weight()
1642 bfqq_weight = bfqq->entity.weight; in bfq_bfqq_higher_class_or_weight()
1645 if (bfqq->entity.parent) in bfq_bfqq_higher_class_or_weight()
1646 bfqq_weight = bfqq->entity.parent->weight; in bfq_bfqq_higher_class_or_weight()
1648 bfqq_weight = bfqq->entity.weight; in bfq_bfqq_higher_class_or_weight()
1659 struct bfq_queue *bfqq, in bfq_bfqq_handle_idle_busy_switch() argument
1666 idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), in bfq_bfqq_handle_idle_busy_switch()
1673 bfqq->ttime.last_end_request + in bfq_bfqq_handle_idle_busy_switch()
1684 in_burst = bfq_bfqq_in_large_burst(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1686 !BFQQ_TOTALLY_SEEKY(bfqq) && in bfq_bfqq_handle_idle_busy_switch()
1688 time_is_before_jiffies(bfqq->soft_rt_next_start) && in bfq_bfqq_handle_idle_busy_switch()
1689 bfqq->dispatched == 0; in bfq_bfqq_handle_idle_busy_switch()
1692 (bfqq->wr_coeff > 1 || in bfq_bfqq_handle_idle_busy_switch()
1693 (bfq_bfqq_sync(bfqq) && in bfq_bfqq_handle_idle_busy_switch()
1694 bfqq->bic && (*interactive || soft_rt))); in bfq_bfqq_handle_idle_busy_switch()
1701 bfq_bfqq_update_budg_for_activation(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1717 if (likely(!bfq_bfqq_just_created(bfqq)) && in bfq_bfqq_handle_idle_busy_switch()
1720 bfqq->budget_timeout + in bfq_bfqq_handle_idle_busy_switch()
1722 hlist_del_init(&bfqq->burst_list_node); in bfq_bfqq_handle_idle_busy_switch()
1723 bfq_clear_bfqq_in_large_burst(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1726 bfq_clear_bfqq_just_created(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1729 if (!bfq_bfqq_IO_bound(bfqq)) { in bfq_bfqq_handle_idle_busy_switch()
1731 bfqq->requests_within_timer++; in bfq_bfqq_handle_idle_busy_switch()
1732 if (bfqq->requests_within_timer >= in bfq_bfqq_handle_idle_busy_switch()
1734 bfq_mark_bfqq_IO_bound(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1736 bfqq->requests_within_timer = 0; in bfq_bfqq_handle_idle_busy_switch()
1740 if (unlikely(time_is_after_jiffies(bfqq->split_time))) in bfq_bfqq_handle_idle_busy_switch()
1742 bfqq->split_time = in bfq_bfqq_handle_idle_busy_switch()
1745 if (time_is_before_jiffies(bfqq->split_time + in bfq_bfqq_handle_idle_busy_switch()
1747 bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1754 if (old_wr_coeff != bfqq->wr_coeff) in bfq_bfqq_handle_idle_busy_switch()
1755 bfqq->entity.prio_changed = 1; in bfq_bfqq_handle_idle_busy_switch()
1759 bfqq->last_idle_bklogged = jiffies; in bfq_bfqq_handle_idle_busy_switch()
1760 bfqq->service_from_backlogged = 0; in bfq_bfqq_handle_idle_busy_switch()
1761 bfq_clear_bfqq_softrt_update(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1763 bfq_add_bfqq_busy(bfqd, bfqq); in bfq_bfqq_handle_idle_busy_switch()
1799 bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || in bfq_bfqq_handle_idle_busy_switch()
1800 bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) && in bfq_bfqq_handle_idle_busy_switch()
1807 struct bfq_queue *bfqq) in bfq_reset_inject_limit() argument
1810 bfqq->last_serv_time_ns = 0; in bfq_reset_inject_limit()
1864 if (bfq_bfqq_has_short_ttime(bfqq)) in bfq_reset_inject_limit()
1865 bfqq->inject_limit = 0; in bfq_reset_inject_limit()
1867 bfqq->inject_limit = 1; in bfq_reset_inject_limit()
1869 bfqq->decrease_time_jif = jiffies; in bfq_reset_inject_limit()
1874 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_add_request() local
1875 struct bfq_data *bfqd = bfqq->bfqd; in bfq_add_request()
1877 unsigned int old_wr_coeff = bfqq->wr_coeff; in bfq_add_request()
1880 bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); in bfq_add_request()
1881 bfqq->queued[rq_is_sync(rq)]++; in bfq_add_request()
1884 if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) { in bfq_add_request()
1940 !bfq_bfqq_has_short_ttime(bfqq) && in bfq_add_request()
1943 if (bfqd->last_completed_rq_bfqq != bfqq && in bfq_add_request()
1945 bfqq->waker_bfqq) { in bfq_add_request()
1952 bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; in bfq_add_request()
1974 if (!hlist_unhashed(&bfqq->woken_list_node)) in bfq_add_request()
1975 hlist_del_init(&bfqq->woken_list_node); in bfq_add_request()
1976 hlist_add_head(&bfqq->woken_list_node, in bfq_add_request()
1979 bfq_clear_bfqq_has_waker(bfqq); in bfq_add_request()
1981 bfqq->waker_bfqq && in bfq_add_request()
1982 !bfq_bfqq_has_waker(bfqq)) { in bfq_add_request()
1987 bfq_mark_bfqq_has_waker(bfqq); in bfq_add_request()
1997 if (time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_add_request()
1999 bfq_reset_inject_limit(bfqd, bfqq); in bfq_add_request()
2026 if (bfqq == bfqd->in_service_queue && in bfq_add_request()
2028 (bfqq->last_serv_time_ns > 0 && in bfq_add_request()
2030 time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_add_request()
2058 elv_rb_add(&bfqq->sort_list, rq); in bfq_add_request()
2063 prev = bfqq->next_rq; in bfq_add_request()
2064 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); in bfq_add_request()
2065 bfqq->next_rq = next_rq; in bfq_add_request()
2071 if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) in bfq_add_request()
2072 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_add_request()
2074 if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ in bfq_add_request()
2075 bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, in bfq_add_request()
2080 bfqq->last_wr_start_finish + in bfq_add_request()
2082 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_add_request()
2083 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_add_request()
2086 bfqq->entity.prio_changed = 1; in bfq_add_request()
2088 if (prev != bfqq->next_rq) in bfq_add_request()
2089 bfq_updated_next_req(bfqd, bfqq); in bfq_add_request()
2119 (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) in bfq_add_request()
2120 bfqq->last_wr_start_finish = jiffies; in bfq_add_request()
2127 struct bfq_queue *bfqq = bfqd->bio_bfqq; in bfq_find_rq_fmerge() local
2130 if (bfqq) in bfq_find_rq_fmerge()
2131 return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); in bfq_find_rq_fmerge()
2163 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_remove_request() local
2164 struct bfq_data *bfqd = bfqq->bfqd; in bfq_remove_request()
2167 if (bfqq->next_rq == rq) { in bfq_remove_request()
2168 bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); in bfq_remove_request()
2169 bfq_updated_next_req(bfqd, bfqq); in bfq_remove_request()
2174 bfqq->queued[sync]--; in bfq_remove_request()
2176 elv_rb_del(&bfqq->sort_list, rq); in bfq_remove_request()
2182 if (RB_EMPTY_ROOT(&bfqq->sort_list)) { in bfq_remove_request()
2183 bfqq->next_rq = NULL; in bfq_remove_request()
2185 if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { in bfq_remove_request()
2186 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_remove_request()
2200 bfqq->entity.budget = bfqq->entity.service = 0; in bfq_remove_request()
2206 if (bfqq->pos_root) { in bfq_remove_request()
2207 rb_erase(&bfqq->pos_node, bfqq->pos_root); in bfq_remove_request()
2208 bfqq->pos_root = NULL; in bfq_remove_request()
2213 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_remove_request()
2217 bfqq->meta_pending--; in bfq_remove_request()
2286 struct bfq_queue *bfqq = RQ_BFQQ(req); in bfq_request_merged() local
2290 if (!bfqq) in bfq_request_merged()
2293 bfqd = bfqq->bfqd; in bfq_request_merged()
2296 elv_rb_del(&bfqq->sort_list, req); in bfq_request_merged()
2297 elv_rb_add(&bfqq->sort_list, req); in bfq_request_merged()
2300 prev = bfqq->next_rq; in bfq_request_merged()
2301 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, in bfq_request_merged()
2303 bfqq->next_rq = next_rq; in bfq_request_merged()
2309 if (prev != bfqq->next_rq) { in bfq_request_merged()
2310 bfq_updated_next_req(bfqd, bfqq); in bfq_request_merged()
2316 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_request_merged()
2338 struct bfq_queue *bfqq = RQ_BFQQ(rq), in bfq_requests_merged() local
2341 if (!bfqq) in bfq_requests_merged()
2353 if (bfqq == next_bfqq && in bfq_requests_merged()
2361 if (bfqq->next_rq == next) in bfq_requests_merged()
2362 bfqq->next_rq = rq; in bfq_requests_merged()
2364 bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags); in bfq_requests_merged()
2368 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) in bfq_bfqq_end_wr() argument
2370 if (bfq_bfqq_busy(bfqq)) in bfq_bfqq_end_wr()
2371 bfqq->bfqd->wr_busy_queues--; in bfq_bfqq_end_wr()
2372 bfqq->wr_coeff = 1; in bfq_bfqq_end_wr()
2373 bfqq->wr_cur_max_time = 0; in bfq_bfqq_end_wr()
2374 bfqq->last_wr_start_finish = jiffies; in bfq_bfqq_end_wr()
2379 bfqq->entity.prio_changed = 1; in bfq_bfqq_end_wr()
2397 struct bfq_queue *bfqq; in bfq_end_wr() local
2401 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_end_wr()
2402 bfq_bfqq_end_wr(bfqq); in bfq_end_wr()
2403 list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) in bfq_end_wr()
2404 bfq_bfqq_end_wr(bfqq); in bfq_end_wr()
2426 struct bfq_queue *bfqq, in bfqq_find_close()
2429 struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; in bfqq_find_close()
2471 struct bfq_queue *bfqq; in bfq_find_close_cooperator() local
2480 bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); in bfq_find_close_cooperator()
2481 if (!bfqq || bfqq == cur_bfqq) in bfq_find_close_cooperator()
2484 return bfqq; in bfq_find_close_cooperator()
2488 bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) in bfq_setup_merge() argument
2504 if (__bfqq == bfqq) in bfq_setup_merge()
2509 process_refs = bfqq_process_refs(bfqq); in bfq_setup_merge()
2523 if (new_bfqq->entity.parent != bfqq->entity.parent) in bfq_setup_merge()
2526 bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", in bfq_setup_merge()
2549 bfqq->new_bfqq = new_bfqq; in bfq_setup_merge()
2563 static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, in bfq_may_be_close_cooperator() argument
2569 if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || in bfq_may_be_close_cooperator()
2570 (bfqq->ioprio_class != new_bfqq->ioprio_class)) in bfq_may_be_close_cooperator()
2578 if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) in bfq_may_be_close_cooperator()
2586 if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) in bfq_may_be_close_cooperator()
2613 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_cooperator() argument
2619 if (bfqq->new_bfqq) in bfq_setup_cooperator()
2620 return bfqq->new_bfqq; in bfq_setup_cooperator()
2673 if (bfq_too_late_for_merging(bfqq)) in bfq_setup_cooperator()
2676 if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) in bfq_setup_cooperator()
2685 if (in_service_bfqq && in_service_bfqq != bfqq && in bfq_setup_cooperator()
2689 bfqq->entity.parent == in_service_bfqq->entity.parent && in bfq_setup_cooperator()
2690 bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { in bfq_setup_cooperator()
2691 new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); in bfq_setup_cooperator()
2700 new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, in bfq_setup_cooperator()
2704 bfq_may_be_close_cooperator(bfqq, new_bfqq)) in bfq_setup_cooperator()
2705 return bfq_setup_merge(bfqq, new_bfqq); in bfq_setup_cooperator()
2710 static void bfq_bfqq_save_state(struct bfq_queue *bfqq) in bfq_bfqq_save_state() argument
2712 struct bfq_io_cq *bic = bfqq->bic; in bfq_bfqq_save_state()
2722 bic->saved_weight = bfqq->entity.orig_weight; in bfq_bfqq_save_state()
2723 bic->saved_ttime = bfqq->ttime; in bfq_bfqq_save_state()
2724 bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq); in bfq_bfqq_save_state()
2725 bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); in bfq_bfqq_save_state()
2726 bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); in bfq_bfqq_save_state()
2727 bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); in bfq_bfqq_save_state()
2728 if (unlikely(bfq_bfqq_just_created(bfqq) && in bfq_bfqq_save_state()
2729 !bfq_bfqq_in_large_burst(bfqq) && in bfq_bfqq_save_state()
2730 bfqq->bfqd->low_latency)) { in bfq_bfqq_save_state()
2740 bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; in bfq_bfqq_save_state()
2742 bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); in bfq_bfqq_save_state()
2745 bic->saved_wr_coeff = bfqq->wr_coeff; in bfq_bfqq_save_state()
2747 bfqq->wr_start_at_switch_to_srt; in bfq_bfqq_save_state()
2748 bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; in bfq_bfqq_save_state()
2749 bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time; in bfq_bfqq_save_state()
2753 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_release_process_ref() argument
2766 if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_release_process_ref()
2767 bfqq != bfqd->in_service_queue) in bfq_release_process_ref()
2768 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_release_process_ref()
2770 bfq_put_queue(bfqq); in bfq_release_process_ref()
2775 struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) in bfq_merge_bfqqs() argument
2777 bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", in bfq_merge_bfqqs()
2780 bfq_bfqq_save_state(bfqq); in bfq_merge_bfqqs()
2782 if (bfq_bfqq_IO_bound(bfqq)) in bfq_merge_bfqqs()
2784 bfq_clear_bfqq_IO_bound(bfqq); in bfq_merge_bfqqs()
2795 if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { in bfq_merge_bfqqs()
2796 new_bfqq->wr_coeff = bfqq->wr_coeff; in bfq_merge_bfqqs()
2797 new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; in bfq_merge_bfqqs()
2798 new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; in bfq_merge_bfqqs()
2800 bfqq->wr_start_at_switch_to_srt; in bfq_merge_bfqqs()
2806 if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ in bfq_merge_bfqqs()
2807 bfqq->wr_coeff = 1; in bfq_merge_bfqqs()
2808 bfqq->entity.prio_changed = 1; in bfq_merge_bfqqs()
2809 if (bfq_bfqq_busy(bfqq)) in bfq_merge_bfqqs()
2842 bfqq->bic = NULL; in bfq_merge_bfqqs()
2843 bfq_release_process_ref(bfqd, bfqq); in bfq_merge_bfqqs()
2851 struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; in bfq_allow_bio_merge() local
2863 if (!bfqq) in bfq_allow_bio_merge()
2870 new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); in bfq_allow_bio_merge()
2879 bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, in bfq_allow_bio_merge()
2886 bfqq = new_bfqq; in bfq_allow_bio_merge()
2894 bfqd->bio_bfqq = bfqq; in bfq_allow_bio_merge()
2897 return bfqq == RQ_BFQQ(rq); in bfq_allow_bio_merge()
2907 struct bfq_queue *bfqq) in bfq_set_budget_timeout() argument
2911 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) in bfq_set_budget_timeout()
2914 timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; in bfq_set_budget_timeout()
2918 bfqq->budget_timeout = jiffies + in bfq_set_budget_timeout()
2923 struct bfq_queue *bfqq) in __bfq_set_in_service_queue() argument
2925 if (bfqq) { in __bfq_set_in_service_queue()
2926 bfq_clear_bfqq_fifo_expire(bfqq); in __bfq_set_in_service_queue()
2930 if (time_is_before_jiffies(bfqq->last_wr_start_finish) && in __bfq_set_in_service_queue()
2931 bfqq->wr_coeff > 1 && in __bfq_set_in_service_queue()
2932 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in __bfq_set_in_service_queue()
2933 time_is_before_jiffies(bfqq->budget_timeout)) { in __bfq_set_in_service_queue()
2958 if (time_after(bfqq->budget_timeout, in __bfq_set_in_service_queue()
2959 bfqq->last_wr_start_finish)) in __bfq_set_in_service_queue()
2960 bfqq->last_wr_start_finish += in __bfq_set_in_service_queue()
2961 jiffies - bfqq->budget_timeout; in __bfq_set_in_service_queue()
2963 bfqq->last_wr_start_finish = jiffies; in __bfq_set_in_service_queue()
2966 bfq_set_budget_timeout(bfqd, bfqq); in __bfq_set_in_service_queue()
2967 bfq_log_bfqq(bfqd, bfqq, in __bfq_set_in_service_queue()
2969 bfqq->entity.budget); in __bfq_set_in_service_queue()
2972 bfqd->in_service_queue = bfqq; in __bfq_set_in_service_queue()
2981 struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); in bfq_set_in_service_queue() local
2983 __bfq_set_in_service_queue(bfqd, bfqq); in bfq_set_in_service_queue()
2984 return bfqq; in bfq_set_in_service_queue()
2989 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_arm_slice_timer() local
2992 bfq_mark_bfqq_wait_request(bfqq); in bfq_arm_slice_timer()
3010 if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && in bfq_arm_slice_timer()
3011 !bfq_asymmetric_scenario(bfqd, bfqq)) in bfq_arm_slice_timer()
3013 else if (bfqq->wr_coeff > 1) in bfq_arm_slice_timer()
3021 bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); in bfq_arm_slice_timer()
3274 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_dispatch_remove() local
3288 bfqq->dispatched++; in bfq_dispatch_remove()
3481 struct bfq_queue *bfqq) in idling_needed_for_service_guarantees() argument
3484 if (unlikely(!bfqq_process_refs(bfqq))) in idling_needed_for_service_guarantees()
3487 return (bfqq->wr_coeff > 1 && in idling_needed_for_service_guarantees()
3491 bfqq->dispatched + 4)) || in idling_needed_for_service_guarantees()
3492 bfq_asymmetric_scenario(bfqd, bfqq); in idling_needed_for_service_guarantees()
3495 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, in __bfq_bfqq_expire() argument
3504 if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) in __bfq_bfqq_expire()
3505 bfq_mark_bfqq_split_coop(bfqq); in __bfq_bfqq_expire()
3520 if (RB_EMPTY_ROOT(&bfqq->sort_list) && in __bfq_bfqq_expire()
3522 idling_needed_for_service_guarantees(bfqd, bfqq))) { in __bfq_bfqq_expire()
3523 if (bfqq->dispatched == 0) in __bfq_bfqq_expire()
3530 bfqq->budget_timeout = jiffies; in __bfq_bfqq_expire()
3532 bfq_del_bfqq_busy(bfqd, bfqq, true); in __bfq_bfqq_expire()
3534 bfq_requeue_bfqq(bfqd, bfqq, true); in __bfq_bfqq_expire()
3540 !RB_EMPTY_ROOT(&bfqq->sort_list))) in __bfq_bfqq_expire()
3541 bfq_pos_tree_add_move(bfqd, bfqq); in __bfq_bfqq_expire()
3564 struct bfq_queue *bfqq, in __bfq_bfqq_recalc_budget() argument
3572 if (bfqq->wr_coeff == 1) in __bfq_bfqq_recalc_budget()
3573 budget = bfqq->max_budget; in __bfq_bfqq_recalc_budget()
3582 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", in __bfq_bfqq_recalc_budget()
3583 bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); in __bfq_bfqq_recalc_budget()
3584 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", in __bfq_bfqq_recalc_budget()
3586 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", in __bfq_bfqq_recalc_budget()
3587 bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); in __bfq_bfqq_recalc_budget()
3589 if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { in __bfq_bfqq_recalc_budget()
3620 if (bfqq->dispatched > 0) /* still outstanding reqs */ in __bfq_bfqq_recalc_budget()
3683 budget = max_t(int, bfqq->entity.service, min_budget); in __bfq_bfqq_recalc_budget()
3688 } else if (!bfq_bfqq_sync(bfqq)) { in __bfq_bfqq_recalc_budget()
3698 bfqq->max_budget = budget; in __bfq_bfqq_recalc_budget()
3702 bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3714 next_rq = bfqq->next_rq; in __bfq_bfqq_recalc_budget()
3716 bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, in __bfq_bfqq_recalc_budget()
3717 bfq_serv_to_charge(next_rq, bfqq)); in __bfq_bfqq_recalc_budget()
3719 bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", in __bfq_bfqq_recalc_budget()
3721 bfqq->entity.budget); in __bfq_bfqq_recalc_budget()
3755 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_bfqq_is_slow() argument
3761 bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */ in bfq_bfqq_is_slow()
3763 if (!bfq_bfqq_sync(bfqq)) in bfq_bfqq_is_slow()
3804 slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; in bfq_bfqq_is_slow()
3807 bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); in bfq_bfqq_is_slow()
3906 struct bfq_queue *bfqq) in bfq_bfqq_softrt_next_start() argument
3908 return max3(bfqq->soft_rt_next_start, in bfq_bfqq_softrt_next_start()
3909 bfqq->last_idle_bklogged + in bfq_bfqq_softrt_next_start()
3910 HZ * bfqq->service_from_backlogged / in bfq_bfqq_softrt_next_start()
3912 jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); in bfq_bfqq_softrt_next_start()
3942 struct bfq_queue *bfqq, in bfq_bfqq_expire() argument
3948 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_expire()
3953 slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); in bfq_bfqq_expire()
3970 if (bfqq->wr_coeff == 1 && in bfq_bfqq_expire()
3973 bfq_bfqq_budget_left(bfqq) >= entity->budget / 3))) in bfq_bfqq_expire()
3974 bfq_bfqq_charge_time(bfqd, bfqq, delta); in bfq_bfqq_expire()
3978 bfq_clear_bfqq_IO_bound(bfqq); in bfq_bfqq_expire()
3980 if (bfqd->low_latency && bfqq->wr_coeff == 1) in bfq_bfqq_expire()
3981 bfqq->last_wr_start_finish = jiffies; in bfq_bfqq_expire()
3984 RB_EMPTY_ROOT(&bfqq->sort_list)) { in bfq_bfqq_expire()
4011 if (bfqq->dispatched == 0 && in bfq_bfqq_expire()
4012 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_bfqq_expire()
4013 bfqq->soft_rt_next_start = in bfq_bfqq_expire()
4014 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_bfqq_expire()
4015 else if (bfqq->dispatched > 0) { in bfq_bfqq_expire()
4020 bfq_mark_bfqq_softrt_update(bfqq); in bfq_bfqq_expire()
4024 bfq_log_bfqq(bfqd, bfqq, in bfq_bfqq_expire()
4026 slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq)); in bfq_bfqq_expire()
4040 __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); in bfq_bfqq_expire()
4041 if (__bfq_bfqq_expire(bfqd, bfqq, reason)) in bfq_bfqq_expire()
4046 if (!bfq_bfqq_busy(bfqq) && in bfq_bfqq_expire()
4049 bfq_mark_bfqq_non_blocking_wait_rq(bfqq); in bfq_bfqq_expire()
4085 static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) in bfq_bfqq_budget_timeout() argument
4087 return time_is_before_eq_jiffies(bfqq->budget_timeout); in bfq_bfqq_budget_timeout()
4098 static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) in bfq_may_expire_for_budg_timeout() argument
4100 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_may_expire_for_budg_timeout()
4102 bfq_bfqq_wait_request(bfqq), in bfq_may_expire_for_budg_timeout()
4103 bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, in bfq_may_expire_for_budg_timeout()
4104 bfq_bfqq_budget_timeout(bfqq)); in bfq_may_expire_for_budg_timeout()
4106 return (!bfq_bfqq_wait_request(bfqq) || in bfq_may_expire_for_budg_timeout()
4107 bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) in bfq_may_expire_for_budg_timeout()
4109 bfq_bfqq_budget_timeout(bfqq); in bfq_may_expire_for_budg_timeout()
4113 struct bfq_queue *bfqq) in idling_boosts_thr_without_issues() argument
4121 if (unlikely(!bfqq_process_refs(bfqq))) in idling_boosts_thr_without_issues()
4124 bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) && in idling_boosts_thr_without_issues()
4125 bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq); in idling_boosts_thr_without_issues()
4212 static bool bfq_better_to_idle(struct bfq_queue *bfqq) in bfq_better_to_idle() argument
4214 struct bfq_data *bfqd = bfqq->bfqd; in bfq_better_to_idle()
4218 if (unlikely(!bfqq_process_refs(bfqq))) in bfq_better_to_idle()
4232 if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) || in bfq_better_to_idle()
4233 bfq_class_idle(bfqq)) in bfq_better_to_idle()
4237 idling_boosts_thr_without_issues(bfqd, bfqq); in bfq_better_to_idle()
4240 idling_needed_for_service_guarantees(bfqd, bfqq); in bfq_better_to_idle()
4263 static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) in bfq_bfqq_must_idle() argument
4265 return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq); in bfq_bfqq_must_idle()
4278 struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; in bfq_choose_bfqq_for_injection() local
4326 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_choose_bfqq_for_injection()
4327 if (!RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_choose_bfqq_for_injection()
4328 (in_serv_always_inject || bfqq->wr_coeff > 1) && in bfq_choose_bfqq_for_injection()
4329 bfq_serv_to_charge(bfqq->next_rq, bfqq) <= in bfq_choose_bfqq_for_injection()
4330 bfq_bfqq_budget_left(bfqq)) { in bfq_choose_bfqq_for_injection()
4349 blk_rq_sectors(bfqq->next_rq) >= in bfq_choose_bfqq_for_injection()
4357 return bfqq; in bfq_choose_bfqq_for_injection()
4370 struct bfq_queue *bfqq; in bfq_select_queue() local
4374 bfqq = bfqd->in_service_queue; in bfq_select_queue()
4375 if (!bfqq) in bfq_select_queue()
4378 bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); in bfq_select_queue()
4387 if (bfq_may_expire_for_budg_timeout(bfqq) && in bfq_select_queue()
4388 !bfq_bfqq_must_idle(bfqq)) in bfq_select_queue()
4398 next_rq = bfqq->next_rq; in bfq_select_queue()
4404 if (bfq_serv_to_charge(next_rq, bfqq) > in bfq_select_queue()
4405 bfq_bfqq_budget_left(bfqq)) { in bfq_select_queue()
4420 if (bfq_bfqq_wait_request(bfqq)) { in bfq_select_queue()
4434 bfq_clear_bfqq_wait_request(bfqq); in bfq_select_queue()
4449 if (bfq_bfqq_wait_request(bfqq) || in bfq_select_queue()
4450 (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) { in bfq_select_queue()
4452 bfqq->bic && bfqq->bic->bfqq[0] && in bfq_select_queue()
4453 bfq_bfqq_busy(bfqq->bic->bfqq[0]) && in bfq_select_queue()
4454 bfqq->bic->bfqq[0]->next_rq ? in bfq_select_queue()
4455 bfqq->bic->bfqq[0] : NULL; in bfq_select_queue()
4530 icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic && in bfq_select_queue()
4533 bfqq = bfqq->bic->bfqq[0]; in bfq_select_queue()
4534 else if (bfq_bfqq_has_waker(bfqq) && in bfq_select_queue()
4535 bfq_bfqq_busy(bfqq->waker_bfqq) && in bfq_select_queue()
4536 bfqq->waker_bfqq->next_rq && in bfq_select_queue()
4537 bfq_serv_to_charge(bfqq->waker_bfqq->next_rq, in bfq_select_queue()
4538 bfqq->waker_bfqq) <= in bfq_select_queue()
4539 bfq_bfqq_budget_left(bfqq->waker_bfqq) in bfq_select_queue()
4541 bfqq = bfqq->waker_bfqq; in bfq_select_queue()
4542 else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_select_queue()
4543 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || in bfq_select_queue()
4544 !bfq_bfqq_has_short_ttime(bfqq))) in bfq_select_queue()
4545 bfqq = bfq_choose_bfqq_for_injection(bfqd); in bfq_select_queue()
4547 bfqq = NULL; in bfq_select_queue()
4554 bfq_bfqq_expire(bfqd, bfqq, false, reason); in bfq_select_queue()
4556 bfqq = bfq_set_in_service_queue(bfqd); in bfq_select_queue()
4557 if (bfqq) { in bfq_select_queue()
4558 bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); in bfq_select_queue()
4562 if (bfqq) in bfq_select_queue()
4563 bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); in bfq_select_queue()
4567 return bfqq; in bfq_select_queue()
4570 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_update_wr_data() argument
4572 struct bfq_entity *entity = &bfqq->entity; in bfq_update_wr_data()
4574 if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ in bfq_update_wr_data()
4575 bfq_log_bfqq(bfqd, bfqq, in bfq_update_wr_data()
4577 jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), in bfq_update_wr_data()
4578 jiffies_to_msecs(bfqq->wr_cur_max_time), in bfq_update_wr_data()
4579 bfqq->wr_coeff, in bfq_update_wr_data()
4580 bfqq->entity.weight, bfqq->entity.orig_weight); in bfq_update_wr_data()
4583 bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); in bfq_update_wr_data()
4590 if (bfq_bfqq_in_large_burst(bfqq)) in bfq_update_wr_data()
4591 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
4592 else if (time_is_before_jiffies(bfqq->last_wr_start_finish + in bfq_update_wr_data()
4593 bfqq->wr_cur_max_time)) { in bfq_update_wr_data()
4594 if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || in bfq_update_wr_data()
4595 time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + in bfq_update_wr_data()
4597 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
4599 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_wr_data()
4600 bfqq->entity.prio_changed = 1; in bfq_update_wr_data()
4603 if (bfqq->wr_coeff > 1 && in bfq_update_wr_data()
4604 bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && in bfq_update_wr_data()
4605 bfqq->service_from_wr > max_service_from_wr) { in bfq_update_wr_data()
4607 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
4618 if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) in bfq_update_wr_data()
4627 struct bfq_queue *bfqq) in bfq_dispatch_rq_from_bfqq() argument
4629 struct request *rq = bfqq->next_rq; in bfq_dispatch_rq_from_bfqq()
4632 service_to_charge = bfq_serv_to_charge(rq, bfqq); in bfq_dispatch_rq_from_bfqq()
4634 bfq_bfqq_served(bfqq, service_to_charge); in bfq_dispatch_rq_from_bfqq()
4636 if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { in bfq_dispatch_rq_from_bfqq()
4643 if (bfqq != bfqd->in_service_queue) in bfq_dispatch_rq_from_bfqq()
4657 bfq_update_wr_data(bfqd, bfqq); in bfq_dispatch_rq_from_bfqq()
4664 if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq))) in bfq_dispatch_rq_from_bfqq()
4667 bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); in bfq_dispatch_rq_from_bfqq()
4689 struct bfq_queue *bfqq = NULL; in __bfq_dispatch_request() local
4696 bfqq = RQ_BFQQ(rq); in __bfq_dispatch_request()
4698 if (bfqq) { in __bfq_dispatch_request()
4705 bfqq->dispatched++; in __bfq_dispatch_request()
4757 bfqq = bfq_select_queue(bfqd); in __bfq_dispatch_request()
4758 if (!bfqq) in __bfq_dispatch_request()
4761 rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); in __bfq_dispatch_request()
4779 struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL; in bfq_update_dispatch_stats() local
4781 if (!idle_timer_disabled && !bfqq) in bfq_update_dispatch_stats()
4809 if (bfqq) { in bfq_update_dispatch_stats()
4810 struct bfq_group *bfqg = bfqq_group(bfqq); in bfq_update_dispatch_stats()
4858 void bfq_put_queue(struct bfq_queue *bfqq) in bfq_put_queue() argument
4862 struct bfq_group *bfqg = bfqq_group(bfqq); in bfq_put_queue()
4864 if (bfqq->bfqd) in bfq_put_queue()
4865 bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", in bfq_put_queue()
4866 bfqq, bfqq->ref); in bfq_put_queue()
4868 bfqq->ref--; in bfq_put_queue()
4869 if (bfqq->ref) in bfq_put_queue()
4872 if (!hlist_unhashed(&bfqq->burst_list_node)) { in bfq_put_queue()
4873 hlist_del_init(&bfqq->burst_list_node); in bfq_put_queue()
4900 if (bfqq->bic && bfqq->bfqd->burst_size > 0) in bfq_put_queue()
4901 bfqq->bfqd->burst_size--; in bfq_put_queue()
4920 if (!hlist_unhashed(&bfqq->woken_list_node)) in bfq_put_queue()
4921 hlist_del_init(&bfqq->woken_list_node); in bfq_put_queue()
4924 hlist_for_each_entry_safe(item, n, &bfqq->woken_list, in bfq_put_queue()
4931 if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) in bfq_put_queue()
4932 bfqq->bfqd->last_completed_rq_bfqq = NULL; in bfq_put_queue()
4934 kmem_cache_free(bfq_pool, bfqq); in bfq_put_queue()
4938 void bfq_put_cooperator(struct bfq_queue *bfqq) in bfq_put_cooperator() argument
4947 __bfqq = bfqq->new_bfqq; in bfq_put_cooperator()
4949 if (__bfqq == bfqq) in bfq_put_cooperator()
4957 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_exit_bfqq() argument
4959 if (bfqq == bfqd->in_service_queue) { in bfq_exit_bfqq()
4960 __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT); in bfq_exit_bfqq()
4964 bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); in bfq_exit_bfqq()
4966 bfq_put_cooperator(bfqq); in bfq_exit_bfqq()
4968 bfq_release_process_ref(bfqd, bfqq); in bfq_exit_bfqq()
4973 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync); in bfq_exit_icq_bfqq() local
4976 if (bfqq) in bfq_exit_icq_bfqq()
4977 bfqd = bfqq->bfqd; /* NULL if scheduler already exited */ in bfq_exit_icq_bfqq()
4979 if (bfqq && bfqd) { in bfq_exit_icq_bfqq()
4984 bfq_exit_bfqq(bfqd, bfqq); in bfq_exit_icq_bfqq()
5002 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) in bfq_set_next_ioprio_data() argument
5006 struct bfq_data *bfqd = bfqq->bfqd; in bfq_set_next_ioprio_data()
5015 bdi_dev_name(bfqq->bfqd->queue->backing_dev_info), in bfq_set_next_ioprio_data()
5022 bfqq->new_ioprio = task_nice_ioprio(tsk); in bfq_set_next_ioprio_data()
5023 bfqq->new_ioprio_class = task_nice_ioclass(tsk); in bfq_set_next_ioprio_data()
5026 bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); in bfq_set_next_ioprio_data()
5027 bfqq->new_ioprio_class = IOPRIO_CLASS_RT; in bfq_set_next_ioprio_data()
5030 bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); in bfq_set_next_ioprio_data()
5031 bfqq->new_ioprio_class = IOPRIO_CLASS_BE; in bfq_set_next_ioprio_data()
5034 bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE; in bfq_set_next_ioprio_data()
5035 bfqq->new_ioprio = 7; in bfq_set_next_ioprio_data()
5039 if (bfqq->new_ioprio >= IOPRIO_BE_NR) { in bfq_set_next_ioprio_data()
5041 bfqq->new_ioprio); in bfq_set_next_ioprio_data()
5042 bfqq->new_ioprio = IOPRIO_BE_NR - 1; in bfq_set_next_ioprio_data()
5045 bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); in bfq_set_next_ioprio_data()
5046 bfqq->entity.prio_changed = 1; in bfq_set_next_ioprio_data()
5056 struct bfq_queue *bfqq; in bfq_check_ioprio_change() local
5068 bfqq = bic_to_bfqq(bic, false); in bfq_check_ioprio_change()
5069 if (bfqq) { in bfq_check_ioprio_change()
5070 struct bfq_queue *old_bfqq = bfqq; in bfq_check_ioprio_change()
5072 bfqq = bfq_get_queue(bfqd, bio, false, bic); in bfq_check_ioprio_change()
5073 bic_set_bfqq(bic, bfqq, false); in bfq_check_ioprio_change()
5077 bfqq = bic_to_bfqq(bic, true); in bfq_check_ioprio_change()
5078 if (bfqq) in bfq_check_ioprio_change()
5079 bfq_set_next_ioprio_data(bfqq, bic); in bfq_check_ioprio_change()
5082 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_init_bfqq() argument
5085 RB_CLEAR_NODE(&bfqq->entity.rb_node); in bfq_init_bfqq()
5086 INIT_LIST_HEAD(&bfqq->fifo); in bfq_init_bfqq()
5087 INIT_HLIST_NODE(&bfqq->burst_list_node); in bfq_init_bfqq()
5088 INIT_HLIST_NODE(&bfqq->woken_list_node); in bfq_init_bfqq()
5089 INIT_HLIST_HEAD(&bfqq->woken_list); in bfq_init_bfqq()
5091 bfqq->ref = 0; in bfq_init_bfqq()
5092 bfqq->bfqd = bfqd; in bfq_init_bfqq()
5095 bfq_set_next_ioprio_data(bfqq, bic); in bfq_init_bfqq()
5103 if (!bfq_class_idle(bfqq)) in bfq_init_bfqq()
5105 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_init_bfqq()
5106 bfq_mark_bfqq_sync(bfqq); in bfq_init_bfqq()
5107 bfq_mark_bfqq_just_created(bfqq); in bfq_init_bfqq()
5109 bfq_clear_bfqq_sync(bfqq); in bfq_init_bfqq()
5112 bfqq->ttime.last_end_request = ktime_get_ns() + 1; in bfq_init_bfqq()
5114 bfq_mark_bfqq_IO_bound(bfqq); in bfq_init_bfqq()
5116 bfqq->pid = pid; in bfq_init_bfqq()
5119 bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; in bfq_init_bfqq()
5120 bfqq->budget_timeout = bfq_smallest_from_now(); in bfq_init_bfqq()
5122 bfqq->wr_coeff = 1; in bfq_init_bfqq()
5123 bfqq->last_wr_start_finish = jiffies; in bfq_init_bfqq()
5124 bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); in bfq_init_bfqq()
5125 bfqq->split_time = bfq_smallest_from_now(); in bfq_init_bfqq()
5136 bfqq->soft_rt_next_start = jiffies; in bfq_init_bfqq()
5139 bfqq->seek_history = 1; in bfq_init_bfqq()
5168 struct bfq_queue *bfqq; in bfq_get_queue() local
5175 bfqq = *async_bfqq; in bfq_get_queue()
5176 if (bfqq) in bfq_get_queue()
5180 bfqq = kmem_cache_alloc_node(bfq_pool, in bfq_get_queue()
5184 if (bfqq) { in bfq_get_queue()
5185 bfq_init_bfqq(bfqd, bfqq, bic, current->pid, in bfq_get_queue()
5187 bfq_init_entity(&bfqq->entity, bfqg); in bfq_get_queue()
5188 bfq_log_bfqq(bfqd, bfqq, "allocated"); in bfq_get_queue()
5190 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5191 bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); in bfq_get_queue()
5200 bfqq->ref++; /* in bfq_get_queue()
5207 bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", in bfq_get_queue()
5208 bfqq, bfqq->ref); in bfq_get_queue()
5209 *async_bfqq = bfqq; in bfq_get_queue()
5213 bfqq->ref++; /* get a process reference to this queue */ in bfq_get_queue()
5214 bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); in bfq_get_queue()
5215 return bfqq; in bfq_get_queue()
5219 struct bfq_queue *bfqq) in bfq_update_io_thinktime() argument
5221 struct bfq_ttime *ttime = &bfqq->ttime; in bfq_update_io_thinktime()
5222 u64 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request; in bfq_update_io_thinktime()
5226 ttime->ttime_samples = (7*bfqq->ttime.ttime_samples + 256) / 8; in bfq_update_io_thinktime()
5233 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_update_io_seektime() argument
5236 bfqq->seek_history <<= 1; in bfq_update_io_seektime()
5237 bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); in bfq_update_io_seektime()
5239 if (bfqq->wr_coeff > 1 && in bfq_update_io_seektime()
5240 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_update_io_seektime()
5241 BFQQ_TOTALLY_SEEKY(bfqq)) in bfq_update_io_seektime()
5242 bfq_bfqq_end_wr(bfqq); in bfq_update_io_seektime()
5246 struct bfq_queue *bfqq, in bfq_update_has_short_ttime() argument
5256 if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) || in bfq_update_has_short_ttime()
5261 if (time_is_after_eq_jiffies(bfqq->split_time + in bfq_update_has_short_ttime()
5270 (bfq_sample_valid(bfqq->ttime.ttime_samples) && in bfq_update_has_short_ttime()
5271 bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle)) in bfq_update_has_short_ttime()
5274 state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
5277 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
5279 bfq_clear_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
5365 if (state_changed && bfqq->last_serv_time_ns == 0 && in bfq_update_has_short_ttime()
5366 (time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_update_has_short_ttime()
5369 bfq_reset_inject_limit(bfqd, bfqq); in bfq_update_has_short_ttime()
5376 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_rq_enqueued() argument
5380 bfqq->meta_pending++; in bfq_rq_enqueued()
5382 bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); in bfq_rq_enqueued()
5384 if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { in bfq_rq_enqueued()
5385 bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 && in bfq_rq_enqueued()
5387 bool budget_timeout = bfq_bfqq_budget_timeout(bfqq); in bfq_rq_enqueued()
5405 if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_rq_enqueued()
5416 bfq_clear_bfqq_wait_request(bfqq); in bfq_rq_enqueued()
5427 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_rq_enqueued()
5435 struct bfq_queue *bfqq = RQ_BFQQ(rq), in __bfq_insert_request() local
5436 *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); in __bfq_insert_request()
5445 bfqq->allocated--; in __bfq_insert_request()
5455 if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) in __bfq_insert_request()
5457 bfqq, new_bfqq); in __bfq_insert_request()
5459 bfq_clear_bfqq_just_created(bfqq); in __bfq_insert_request()
5464 bfq_put_queue(bfqq); in __bfq_insert_request()
5466 bfqq = new_bfqq; in __bfq_insert_request()
5469 bfq_update_io_thinktime(bfqd, bfqq); in __bfq_insert_request()
5470 bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq)); in __bfq_insert_request()
5471 bfq_update_io_seektime(bfqd, bfqq, rq); in __bfq_insert_request()
5473 waiting = bfqq && bfq_bfqq_wait_request(bfqq); in __bfq_insert_request()
5475 idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq); in __bfq_insert_request()
5478 list_add_tail(&rq->queuelist, &bfqq->fifo); in __bfq_insert_request()
5480 bfq_rq_enqueued(bfqd, bfqq, rq); in __bfq_insert_request()
5487 struct bfq_queue *bfqq, in bfq_update_insert_stats() argument
5491 if (!bfqq) in bfq_update_insert_stats()
5505 bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags); in bfq_update_insert_stats()
5507 bfqg_stats_update_idle_time(bfqq_group(bfqq)); in bfq_update_insert_stats()
5512 struct bfq_queue *bfqq, in bfq_update_insert_stats() argument
5524 struct bfq_queue *bfqq; in bfq_insert_request() local
5533 bfqq = bfq_init_rq(rq); in bfq_insert_request()
5541 if (!bfqq || at_head || blk_rq_is_passthrough(rq)) { in bfq_insert_request()
5553 bfqq = RQ_BFQQ(rq); in bfq_insert_request()
5571 bfq_update_insert_stats(q, bfqq, idle_timer_disabled, in bfq_insert_request()
5589 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_update_hw_tag() local
5611 if (bfqq && bfq_bfqq_has_short_ttime(bfqq) && in bfq_update_hw_tag()
5612 bfqq->dispatched + bfqq->queued[0] + bfqq->queued[1] < in bfq_update_hw_tag()
5628 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) in bfq_completed_request() argument
5636 bfqq->dispatched--; in bfq_completed_request()
5638 if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { in bfq_completed_request()
5645 bfqq->budget_timeout = jiffies; in bfq_completed_request()
5647 bfq_weights_tree_remove(bfqd, bfqq); in bfq_completed_request()
5652 bfqq->ttime.last_end_request = now_ns; in bfq_completed_request()
5681 bfqd->last_completed_rq_bfqq = bfqq; in bfq_completed_request()
5694 if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && in bfq_completed_request()
5695 RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_completed_request()
5696 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_completed_request()
5697 bfqq->soft_rt_next_start = in bfq_completed_request()
5698 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_completed_request()
5704 if (bfqd->in_service_queue == bfqq) { in bfq_completed_request()
5705 if (bfq_bfqq_must_idle(bfqq)) { in bfq_completed_request()
5706 if (bfqq->dispatched == 0) in bfq_completed_request()
5732 } else if (bfq_may_expire_for_budg_timeout(bfqq)) in bfq_completed_request()
5733 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
5735 else if (RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_completed_request()
5736 (bfqq->dispatched == 0 || in bfq_completed_request()
5737 !bfq_better_to_idle(bfqq))) in bfq_completed_request()
5738 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
5746 static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq) in bfq_finish_requeue_request_body() argument
5748 bfqq->allocated--; in bfq_finish_requeue_request_body()
5750 bfq_put_queue(bfqq); in bfq_finish_requeue_request_body()
5858 struct bfq_queue *bfqq) in bfq_update_inject_limit() argument
5861 unsigned int old_limit = bfqq->inject_limit; in bfq_update_inject_limit()
5863 if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) { in bfq_update_inject_limit()
5864 u64 threshold = (bfqq->last_serv_time_ns * 3)>>1; in bfq_update_inject_limit()
5867 bfqq->inject_limit--; in bfq_update_inject_limit()
5868 bfqq->decrease_time_jif = jiffies; in bfq_update_inject_limit()
5871 bfqq->inject_limit++; in bfq_update_inject_limit()
5886 if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) || in bfq_update_inject_limit()
5887 tot_time_ns < bfqq->last_serv_time_ns) { in bfq_update_inject_limit()
5888 if (bfqq->last_serv_time_ns == 0) { in bfq_update_inject_limit()
5893 bfqq->inject_limit = max_t(unsigned int, 1, old_limit); in bfq_update_inject_limit()
5895 bfqq->last_serv_time_ns = tot_time_ns; in bfq_update_inject_limit()
5906 bfqq->last_serv_time_ns = tot_time_ns; in bfq_update_inject_limit()
5922 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_finish_requeue_request() local
5930 if (!rq->elv.icq || !bfqq) in bfq_finish_requeue_request()
5933 bfqd = bfqq->bfqd; in bfq_finish_requeue_request()
5936 bfqg_stats_update_completion(bfqq_group(bfqq), in bfq_finish_requeue_request()
5947 bfq_update_inject_limit(bfqd, bfqq); in bfq_finish_requeue_request()
5949 bfq_completed_request(bfqq, bfqd); in bfq_finish_requeue_request()
5950 bfq_finish_requeue_request_body(bfqq); in bfq_finish_requeue_request()
5969 bfqg_stats_update_io_remove(bfqq_group(bfqq), in bfq_finish_requeue_request()
5972 bfq_finish_requeue_request_body(bfqq); in bfq_finish_requeue_request()
6003 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) in bfq_split_bfqq() argument
6005 bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); in bfq_split_bfqq()
6007 if (bfqq_process_refs(bfqq) == 1) { in bfq_split_bfqq()
6008 bfqq->pid = current->pid; in bfq_split_bfqq()
6009 bfq_clear_bfqq_coop(bfqq); in bfq_split_bfqq()
6010 bfq_clear_bfqq_split_coop(bfqq); in bfq_split_bfqq()
6011 return bfqq; in bfq_split_bfqq()
6016 bfq_put_cooperator(bfqq); in bfq_split_bfqq()
6018 bfq_release_process_ref(bfqq->bfqd, bfqq); in bfq_split_bfqq()
6028 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync); in bfq_get_bfqq_handle_split() local
6030 if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) in bfq_get_bfqq_handle_split()
6031 return bfqq; in bfq_get_bfqq_handle_split()
6036 if (bfqq) in bfq_get_bfqq_handle_split()
6037 bfq_put_queue(bfqq); in bfq_get_bfqq_handle_split()
6038 bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); in bfq_get_bfqq_handle_split()
6040 bic_set_bfqq(bic, bfqq, is_sync); in bfq_get_bfqq_handle_split()
6044 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_get_bfqq_handle_split()
6046 bfq_clear_bfqq_in_large_burst(bfqq); in bfq_get_bfqq_handle_split()
6076 hlist_add_head(&bfqq->burst_list_node, in bfq_get_bfqq_handle_split()
6079 bfqq->split_time = jiffies; in bfq_get_bfqq_handle_split()
6082 return bfqq; in bfq_get_bfqq_handle_split()
6131 struct bfq_queue *bfqq; in bfq_init_rq() local
6154 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, in bfq_init_rq()
6159 if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { in bfq_init_rq()
6160 bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); in bfq_init_rq()
6163 if (bfq_bfqq_in_large_burst(bfqq)) in bfq_init_rq()
6166 bfqq = bfq_split_bfqq(bic, bfqq); in bfq_init_rq()
6169 if (!bfqq) in bfq_init_rq()
6170 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, in bfq_init_rq()
6178 bfqq->allocated++; in bfq_init_rq()
6179 bfqq->ref++; in bfq_init_rq()
6180 bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", in bfq_init_rq()
6181 rq, bfqq, bfqq->ref); in bfq_init_rq()
6184 rq->elv.priv[1] = bfqq; in bfq_init_rq()
6192 if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { in bfq_init_rq()
6193 bfqq->bic = bic; in bfq_init_rq()
6200 bfq_bfqq_resume_state(bfqq, bfqd, bic, in bfq_init_rq()
6225 if (unlikely(bfq_bfqq_just_created(bfqq) && in bfq_init_rq()
6228 bfq_handle_burst(bfqd, bfqq); in bfq_init_rq()
6230 return bfqq; in bfq_init_rq()
6234 bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_idle_slice_timer_body() argument
6248 if (bfqq != bfqd->in_service_queue) { in bfq_idle_slice_timer_body()
6253 bfq_clear_bfqq_wait_request(bfqq); in bfq_idle_slice_timer_body()
6255 if (bfq_bfqq_budget_timeout(bfqq)) in bfq_idle_slice_timer_body()
6262 else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) in bfq_idle_slice_timer_body()
6273 bfq_bfqq_expire(bfqd, bfqq, true, reason); in bfq_idle_slice_timer_body()
6288 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_idle_slice_timer() local
6298 if (bfqq) in bfq_idle_slice_timer()
6299 bfq_idle_slice_timer_body(bfqd, bfqq); in bfq_idle_slice_timer()
6307 struct bfq_queue *bfqq = *bfqq_ptr; in __bfq_put_async_bfqq() local
6309 bfq_log(bfqd, "put_async_bfqq: %p", bfqq); in __bfq_put_async_bfqq()
6310 if (bfqq) { in __bfq_put_async_bfqq()
6311 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); in __bfq_put_async_bfqq()
6313 bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", in __bfq_put_async_bfqq()
6314 bfqq, bfqq->ref); in __bfq_put_async_bfqq()
6315 bfq_put_queue(bfqq); in __bfq_put_async_bfqq()
6403 struct bfq_queue *bfqq, *n; in bfq_exit_queue() local
6408 list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) in bfq_exit_queue()
6409 bfq_deactivate_bfqq(bfqd, bfqq, false, false); in bfq_exit_queue()