Lines Matching refs:entity
30 static unsigned int bfq_class_idx(struct bfq_entity *entity) in bfq_class_idx() argument
32 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_class_idx()
150 struct bfq_entity *group_entity = bfqq->entity.parent; in bfq_bfqq_to_bfqg()
153 group_entity = &bfqq->bfqd->root_group->entity; in bfq_bfqq_to_bfqg()
155 return container_of(group_entity, struct bfq_group, entity); in bfq_bfqq_to_bfqg()
207 static bool bfq_no_longer_next_in_service(struct bfq_entity *entity) in bfq_no_longer_next_in_service() argument
211 if (bfq_entity_to_bfqq(entity)) in bfq_no_longer_next_in_service()
214 bfqg = container_of(entity, struct bfq_group, entity); in bfq_no_longer_next_in_service()
246 static bool bfq_no_longer_next_in_service(struct bfq_entity *entity) in bfq_no_longer_next_in_service() argument
262 struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) in bfq_entity_to_bfqq() argument
266 if (!entity->my_sched_data) in bfq_entity_to_bfqq()
267 bfqq = container_of(entity, struct bfq_queue, entity); in bfq_entity_to_bfqq()
288 static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) in bfq_calc_finish() argument
290 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_calc_finish()
292 entity->finish = entity->start + in bfq_calc_finish()
293 bfq_delta(service, entity->weight); in bfq_calc_finish()
298 service, entity->weight); in bfq_calc_finish()
301 entity->start, entity->finish, in bfq_calc_finish()
302 bfq_delta(service, entity->weight)); in bfq_calc_finish()
317 struct bfq_entity *entity = NULL; in bfq_entity_of() local
320 entity = rb_entry(node, struct bfq_entity, rb_node); in bfq_entity_of()
322 return entity; in bfq_entity_of()
330 static void bfq_extract(struct rb_root *root, struct bfq_entity *entity) in bfq_extract() argument
332 entity->tree = NULL; in bfq_extract()
333 rb_erase(&entity->rb_node, root); in bfq_extract()
342 struct bfq_entity *entity) in bfq_idle_extract() argument
344 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_idle_extract()
347 if (entity == st->first_idle) { in bfq_idle_extract()
348 next = rb_next(&entity->rb_node); in bfq_idle_extract()
352 if (entity == st->last_idle) { in bfq_idle_extract()
353 next = rb_prev(&entity->rb_node); in bfq_idle_extract()
357 bfq_extract(&st->idle, entity); in bfq_idle_extract()
371 static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) in bfq_insert() argument
381 if (bfq_gt(entry->finish, entity->finish)) in bfq_insert()
387 rb_link_node(&entity->rb_node, parent, node); in bfq_insert()
388 rb_insert_color(&entity->rb_node, root); in bfq_insert()
390 entity->tree = root; in bfq_insert()
403 static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node) in bfq_update_min() argument
409 if (bfq_gt(entity->min_start, child->min_start)) in bfq_update_min()
410 entity->min_start = child->min_start; in bfq_update_min()
424 struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node); in bfq_update_active_node() local
426 entity->min_start = entity->start; in bfq_update_active_node()
427 bfq_update_min(entity, node->rb_right); in bfq_update_active_node()
428 bfq_update_min(entity, node->rb_left); in bfq_update_active_node()
473 struct bfq_entity *entity) in bfq_active_insert() argument
475 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_active_insert()
476 struct rb_node *node = &entity->rb_node; in bfq_active_insert()
483 bfq_insert(&st->active, entity); in bfq_active_insert()
493 sd = entity->sched_data; in bfq_active_insert()
528 static void bfq_get_entity(struct bfq_entity *entity) in bfq_get_entity() argument
530 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_get_entity()
575 struct bfq_entity *entity) in bfq_active_extract() argument
577 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_active_extract()
585 node = bfq_find_deepest(&entity->rb_node); in bfq_active_extract()
586 bfq_extract(&st->active, entity); in bfq_active_extract()
592 sd = entity->sched_data; in bfq_active_extract()
610 struct bfq_entity *entity) in bfq_idle_insert() argument
612 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_idle_insert()
616 if (!first_idle || bfq_gt(first_idle->finish, entity->finish)) in bfq_idle_insert()
617 st->first_idle = entity; in bfq_idle_insert()
618 if (!last_idle || bfq_gt(entity->finish, last_idle->finish)) in bfq_idle_insert()
619 st->last_idle = entity; in bfq_idle_insert()
621 bfq_insert(&st->idle, entity); in bfq_idle_insert()
643 struct bfq_entity *entity, in bfq_forget_entity() argument
646 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_forget_entity()
648 entity->on_st_or_in_serv = false; in bfq_forget_entity()
649 st->wsum -= entity->weight; in bfq_forget_entity()
659 void bfq_put_idle_entity(struct bfq_service_tree *st, struct bfq_entity *entity) in bfq_put_idle_entity() argument
661 bfq_idle_extract(st, entity); in bfq_put_idle_entity()
662 bfq_forget_entity(st, entity, in bfq_put_idle_entity()
663 entity == entity->sched_data->in_service_entity); in bfq_put_idle_entity()
691 struct bfq_service_tree *bfq_entity_service_tree(struct bfq_entity *entity) in bfq_entity_service_tree() argument
693 struct bfq_sched_data *sched_data = entity->sched_data; in bfq_entity_service_tree()
694 unsigned int idx = bfq_class_idx(entity); in bfq_entity_service_tree()
719 struct bfq_entity *entity, in __bfq_entity_update_weight_prio() argument
724 if (entity->prio_changed) { in __bfq_entity_update_weight_prio()
725 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in __bfq_entity_update_weight_prio()
738 sd = entity->my_sched_data; in __bfq_entity_update_weight_prio()
746 old_st->wsum -= entity->weight; in __bfq_entity_update_weight_prio()
748 if (entity->new_weight != entity->orig_weight) { in __bfq_entity_update_weight_prio()
749 if (entity->new_weight < BFQ_MIN_WEIGHT || in __bfq_entity_update_weight_prio()
750 entity->new_weight > BFQ_MAX_WEIGHT) { in __bfq_entity_update_weight_prio()
752 entity->new_weight); in __bfq_entity_update_weight_prio()
753 if (entity->new_weight < BFQ_MIN_WEIGHT) in __bfq_entity_update_weight_prio()
754 entity->new_weight = BFQ_MIN_WEIGHT; in __bfq_entity_update_weight_prio()
756 entity->new_weight = BFQ_MAX_WEIGHT; in __bfq_entity_update_weight_prio()
758 entity->orig_weight = entity->new_weight; in __bfq_entity_update_weight_prio()
761 bfq_weight_to_ioprio(entity->orig_weight); in __bfq_entity_update_weight_prio()
772 entity->prio_changed = 0; in __bfq_entity_update_weight_prio()
781 new_st = bfq_entity_service_tree(entity); in __bfq_entity_update_weight_prio()
783 prev_weight = entity->weight; in __bfq_entity_update_weight_prio()
784 new_weight = entity->orig_weight * in __bfq_entity_update_weight_prio()
795 entity->weight = new_weight; in __bfq_entity_update_weight_prio()
805 new_st->wsum += entity->weight; in __bfq_entity_update_weight_prio()
808 entity->start = new_st->vtime; in __bfq_entity_update_weight_prio()
826 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_served() local
836 for_each_entity(entity) { in bfq_bfqq_served()
837 st = bfq_entity_service_tree(entity); in bfq_bfqq_served()
839 entity->service += served; in bfq_bfqq_served()
877 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_charge_time() local
882 int tot_serv_to_charge = max(serv_to_charge_for_time, entity->service); in bfq_bfqq_charge_time()
885 if (tot_serv_to_charge > entity->budget) in bfq_bfqq_charge_time()
886 entity->budget = tot_serv_to_charge; in bfq_bfqq_charge_time()
889 max_t(int, 0, tot_serv_to_charge - entity->service)); in bfq_bfqq_charge_time()
892 static void bfq_update_fin_time_enqueue(struct bfq_entity *entity, in bfq_update_fin_time_enqueue() argument
896 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); in bfq_update_fin_time_enqueue()
903 st = __bfq_entity_update_weight_prio(st, entity, true); in bfq_update_fin_time_enqueue()
904 bfq_calc_finish(entity, entity->budget); in bfq_update_fin_time_enqueue()
939 if (backshifted && bfq_gt(st->vtime, entity->finish)) { in bfq_update_fin_time_enqueue()
940 unsigned long delta = st->vtime - entity->finish; in bfq_update_fin_time_enqueue()
945 entity->start += delta; in bfq_update_fin_time_enqueue()
946 entity->finish += delta; in bfq_update_fin_time_enqueue()
949 bfq_active_insert(st, entity); in bfq_update_fin_time_enqueue()
964 static void __bfq_activate_entity(struct bfq_entity *entity, in __bfq_activate_entity() argument
967 struct bfq_service_tree *st = bfq_entity_service_tree(entity); in __bfq_activate_entity()
972 if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) { in __bfq_activate_entity()
974 min_vstart = entity->finish; in __bfq_activate_entity()
978 if (entity->tree == &st->idle) { in __bfq_activate_entity()
983 bfq_idle_extract(st, entity); in __bfq_activate_entity()
984 entity->start = bfq_gt(min_vstart, entity->finish) ? in __bfq_activate_entity()
985 min_vstart : entity->finish; in __bfq_activate_entity()
992 entity->start = min_vstart; in __bfq_activate_entity()
993 st->wsum += entity->weight; in __bfq_activate_entity()
1000 bfq_get_entity(entity); in __bfq_activate_entity()
1002 entity->on_st_or_in_serv = true; in __bfq_activate_entity()
1006 if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */ in __bfq_activate_entity()
1008 container_of(entity, struct bfq_group, entity); in __bfq_activate_entity()
1011 if (!entity->in_groups_with_pending_reqs) { in __bfq_activate_entity()
1012 entity->in_groups_with_pending_reqs = true; in __bfq_activate_entity()
1018 bfq_update_fin_time_enqueue(entity, st, backshifted); in __bfq_activate_entity()
1036 static void __bfq_requeue_entity(struct bfq_entity *entity) in __bfq_requeue_entity() argument
1038 struct bfq_sched_data *sd = entity->sched_data; in __bfq_requeue_entity()
1039 struct bfq_service_tree *st = bfq_entity_service_tree(entity); in __bfq_requeue_entity()
1041 if (entity == sd->in_service_entity) { in __bfq_requeue_entity()
1064 bfq_calc_finish(entity, entity->service); in __bfq_requeue_entity()
1065 entity->start = entity->finish; in __bfq_requeue_entity()
1079 if (entity->tree) in __bfq_requeue_entity()
1080 bfq_active_extract(st, entity); in __bfq_requeue_entity()
1096 bfq_active_extract(st, entity); in __bfq_requeue_entity()
1099 bfq_update_fin_time_enqueue(entity, st, false); in __bfq_requeue_entity()
1102 static void __bfq_activate_requeue_entity(struct bfq_entity *entity, in __bfq_activate_requeue_entity() argument
1106 struct bfq_service_tree *st = bfq_entity_service_tree(entity); in __bfq_activate_requeue_entity()
1108 if (sd->in_service_entity == entity || entity->tree == &st->active) in __bfq_activate_requeue_entity()
1113 __bfq_requeue_entity(entity); in __bfq_activate_requeue_entity()
1119 __bfq_activate_entity(entity, non_blocking_wait_rq); in __bfq_activate_requeue_entity()
1136 static void bfq_activate_requeue_entity(struct bfq_entity *entity, in bfq_activate_requeue_entity() argument
1142 for_each_entity(entity) { in bfq_activate_requeue_entity()
1143 sd = entity->sched_data; in bfq_activate_requeue_entity()
1144 __bfq_activate_requeue_entity(entity, sd, non_blocking_wait_rq); in bfq_activate_requeue_entity()
1146 if (!bfq_update_next_in_service(sd, entity, expiration) && in bfq_activate_requeue_entity()
1162 bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree) in __bfq_deactivate_entity() argument
1164 struct bfq_sched_data *sd = entity->sched_data; in __bfq_deactivate_entity()
1168 if (!entity->on_st_or_in_serv) /* in __bfq_deactivate_entity()
1180 st = bfq_entity_service_tree(entity); in __bfq_deactivate_entity()
1181 is_in_service = entity == sd->in_service_entity; in __bfq_deactivate_entity()
1183 bfq_calc_finish(entity, entity->service); in __bfq_deactivate_entity()
1193 entity->service = 0; in __bfq_deactivate_entity()
1195 if (entity->tree == &st->active) in __bfq_deactivate_entity()
1196 bfq_active_extract(st, entity); in __bfq_deactivate_entity()
1197 else if (!is_in_service && entity->tree == &st->idle) in __bfq_deactivate_entity()
1198 bfq_idle_extract(st, entity); in __bfq_deactivate_entity()
1200 if (!ins_into_idle_tree || !bfq_gt(entity->finish, st->vtime)) in __bfq_deactivate_entity()
1201 bfq_forget_entity(st, entity, is_in_service); in __bfq_deactivate_entity()
1203 bfq_idle_insert(st, entity); in __bfq_deactivate_entity()
1215 static void bfq_deactivate_entity(struct bfq_entity *entity, in bfq_deactivate_entity() argument
1222 for_each_entity_safe(entity, parent) { in bfq_deactivate_entity()
1223 sd = entity->sched_data; in bfq_deactivate_entity()
1225 if (!__bfq_deactivate_entity(entity, ins_into_idle_tree)) { in bfq_deactivate_entity()
1236 if (sd->next_in_service == entity) in bfq_deactivate_entity()
1290 entity = parent; in bfq_deactivate_entity()
1291 for_each_entity(entity) { in bfq_deactivate_entity()
1298 __bfq_requeue_entity(entity); in bfq_deactivate_entity()
1300 sd = entity->sched_data; in bfq_deactivate_entity()
1301 if (!bfq_update_next_in_service(sd, entity, expiration) && in bfq_deactivate_entity()
1403 struct bfq_entity *entity; in __bfq_lookup_next_entity() local
1428 entity = bfq_first_active_entity(st, new_vtime); in __bfq_lookup_next_entity()
1430 return entity; in __bfq_lookup_next_entity()
1447 struct bfq_entity *entity = NULL; in bfq_lookup_next_entity() local
1485 entity = __bfq_lookup_next_entity(st + class_idx, in bfq_lookup_next_entity()
1489 if (entity) in bfq_lookup_next_entity()
1493 if (!entity) in bfq_lookup_next_entity()
1496 return entity; in bfq_lookup_next_entity()
1511 struct bfq_entity *entity = NULL; in bfq_get_next_queue() local
1524 for (; sd ; sd = entity->my_sched_data) { in bfq_get_next_queue()
1550 entity = sd->next_in_service; in bfq_get_next_queue()
1551 sd->in_service_entity = entity; in bfq_get_next_queue()
1561 if (bfq_no_longer_next_in_service(entity)) in bfq_get_next_queue()
1562 bfq_active_extract(bfq_entity_service_tree(entity), in bfq_get_next_queue()
1563 entity); in bfq_get_next_queue()
1585 bfqq = bfq_entity_to_bfqq(entity); in bfq_get_next_queue()
1591 for_each_entity(entity) { in bfq_get_next_queue()
1592 struct bfq_sched_data *sd = entity->sched_data; in bfq_get_next_queue()
1605 struct bfq_entity *in_serv_entity = &in_serv_bfqq->entity; in __bfq_bfqd_reset_in_service()
1606 struct bfq_entity *entity = in_serv_entity; in __bfq_bfqd_reset_in_service() local
1618 for_each_entity(entity) in __bfq_bfqd_reset_in_service()
1619 entity->sched_data->in_service_entity = NULL; in __bfq_bfqd_reset_in_service()
1645 struct bfq_entity *entity = &bfqq->entity; in bfq_deactivate_bfqq() local
1647 bfq_deactivate_entity(entity, ins_into_idle_tree, expiration); in bfq_deactivate_bfqq()
1652 struct bfq_entity *entity = &bfqq->entity; in bfq_activate_bfqq() local
1654 bfq_activate_requeue_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq), in bfq_activate_bfqq()
1662 struct bfq_entity *entity = &bfqq->entity; in bfq_requeue_bfqq() local
1664 bfq_activate_requeue_entity(entity, false, in bfq_requeue_bfqq()