Lines Matching refs:rt_se
171 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) argument
173 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
176 WARN_ON_ONCE(!rt_entity_is_task(rt_se)); in rt_task_of()
178 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
186 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
188 return rt_se->rt_rq; in rt_rq_of_se()
191 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
193 struct rt_rq *rt_rq = rt_se->rt_rq; in rq_of_rt_se()
200 if (tg->rt_se) in unregister_rt_sched_group()
212 if (tg->rt_se) in free_rt_sched_group()
213 kfree(tg->rt_se[i]); in free_rt_sched_group()
217 kfree(tg->rt_se); in free_rt_sched_group()
221 struct sched_rt_entity *rt_se, int cpu, in init_tg_rt_entry() argument
232 tg->rt_se[cpu] = rt_se; in init_tg_rt_entry()
234 if (!rt_se) in init_tg_rt_entry()
238 rt_se->rt_rq = &rq->rt; in init_tg_rt_entry()
240 rt_se->rt_rq = parent->my_q; in init_tg_rt_entry()
242 rt_se->my_q = rt_rq; in init_tg_rt_entry()
243 rt_se->parent = parent; in init_tg_rt_entry()
244 INIT_LIST_HEAD(&rt_se->run_list); in init_tg_rt_entry()
250 struct sched_rt_entity *rt_se; in alloc_rt_sched_group() local
256 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL); in alloc_rt_sched_group()
257 if (!tg->rt_se) in alloc_rt_sched_group()
269 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), in alloc_rt_sched_group()
271 if (!rt_se) in alloc_rt_sched_group()
276 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); in alloc_rt_sched_group()
289 #define rt_entity_is_task(rt_se) (1) argument
291 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
293 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
301 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
303 struct task_struct *p = rt_task_of(rt_se); in rq_of_rt_se()
308 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
310 struct rq *rq = rq_of_rt_se(rt_se); in rt_rq_of_se()
380 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
384 if (!rt_entity_is_task(rt_se)) in inc_rt_migration()
387 p = rt_task_of(rt_se); in inc_rt_migration()
397 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
401 if (!rt_entity_is_task(rt_se)) in dec_rt_migration()
404 p = rt_task_of(rt_se); in dec_rt_migration()
474 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_migration() argument
479 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_migration() argument
491 static inline int on_rt_rq(struct sched_rt_entity *rt_se) in on_rt_rq() argument
493 return rt_se->on_rq; in on_rt_rq()
570 #define for_each_sched_rt_entity(rt_se) \ argument
571 for (; rt_se; rt_se = rt_se->parent)
573 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
575 return rt_se->my_q; in group_rt_rq()
578 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
579 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
585 struct sched_rt_entity *rt_se; in sched_rt_rq_enqueue() local
589 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_enqueue()
592 if (!rt_se) in sched_rt_rq_enqueue()
594 else if (!on_rt_rq(rt_se)) in sched_rt_rq_enqueue()
595 enqueue_rt_entity(rt_se, 0); in sched_rt_rq_enqueue()
604 struct sched_rt_entity *rt_se; in sched_rt_rq_dequeue() local
607 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_dequeue()
609 if (!rt_se) { in sched_rt_rq_dequeue()
614 else if (on_rt_rq(rt_se)) in sched_rt_rq_dequeue()
615 dequeue_rt_entity(rt_se, 0); in sched_rt_rq_dequeue()
623 static int rt_se_boosted(struct sched_rt_entity *rt_se) in rt_se_boosted() argument
625 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_boosted()
631 p = rt_task_of(rt_se); in rt_se_boosted()
675 #define for_each_sched_rt_entity(rt_se) \ argument
676 for (; rt_se; rt_se = NULL)
678 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
992 static inline int rt_se_prio(struct sched_rt_entity *rt_se) in rt_se_prio() argument
995 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_prio()
1001 return rt_task_of(rt_se)->prio; in rt_se_prio()
1061 struct sched_rt_entity *rt_se = &curr->rt; in update_curr_rt() local
1085 for_each_sched_rt_entity(rt_se) { in update_curr_rt()
1086 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in update_curr_rt()
1233 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1235 if (rt_se_boosted(rt_se)) in inc_rt_group()
1243 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_group() argument
1245 if (rt_se_boosted(rt_se)) in dec_rt_group()
1254 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1260 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} in dec_rt_group() argument
1265 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) in rt_se_nr_running() argument
1267 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_nr_running()
1276 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se) in rt_se_rr_nr_running() argument
1278 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_rr_nr_running()
1284 tsk = rt_task_of(rt_se); in rt_se_rr_nr_running()
1290 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_tasks() argument
1292 int prio = rt_se_prio(rt_se); in inc_rt_tasks()
1295 rt_rq->rt_nr_running += rt_se_nr_running(rt_se); in inc_rt_tasks()
1296 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); in inc_rt_tasks()
1299 inc_rt_migration(rt_se, rt_rq); in inc_rt_tasks()
1300 inc_rt_group(rt_se, rt_rq); in inc_rt_tasks()
1304 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_tasks() argument
1306 WARN_ON(!rt_prio(rt_se_prio(rt_se))); in dec_rt_tasks()
1308 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); in dec_rt_tasks()
1309 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); in dec_rt_tasks()
1311 dec_rt_prio(rt_rq, rt_se_prio(rt_se)); in dec_rt_tasks()
1312 dec_rt_migration(rt_se, rt_rq); in dec_rt_tasks()
1313 dec_rt_group(rt_se, rt_rq); in dec_rt_tasks()
1329 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array) in __delist_rt_entity() argument
1331 list_del_init(&rt_se->run_list); in __delist_rt_entity()
1333 if (list_empty(array->queue + rt_se_prio(rt_se))) in __delist_rt_entity()
1334 __clear_bit(rt_se_prio(rt_se), array->bitmap); in __delist_rt_entity()
1336 rt_se->on_list = 0; in __delist_rt_entity()
1340 __schedstats_from_rt_se(struct sched_rt_entity *rt_se) in __schedstats_from_rt_se() argument
1344 if (!rt_entity_is_task(rt_se)) in __schedstats_from_rt_se()
1348 return &rt_task_of(rt_se)->stats; in __schedstats_from_rt_se()
1352 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_start_rt() argument
1360 if (rt_entity_is_task(rt_se)) in update_stats_wait_start_rt()
1361 p = rt_task_of(rt_se); in update_stats_wait_start_rt()
1363 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_start_rt()
1371 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_enqueue_sleeper_rt() argument
1379 if (rt_entity_is_task(rt_se)) in update_stats_enqueue_sleeper_rt()
1380 p = rt_task_of(rt_se); in update_stats_enqueue_sleeper_rt()
1382 stats = __schedstats_from_rt_se(rt_se); in update_stats_enqueue_sleeper_rt()
1390 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_enqueue_rt() argument
1397 update_stats_enqueue_sleeper_rt(rt_rq, rt_se); in update_stats_enqueue_rt()
1401 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_end_rt() argument
1409 if (rt_entity_is_task(rt_se)) in update_stats_wait_end_rt()
1410 p = rt_task_of(rt_se); in update_stats_wait_end_rt()
1412 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_end_rt()
1420 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_dequeue_rt() argument
1428 if (rt_entity_is_task(rt_se)) in update_stats_dequeue_rt()
1429 p = rt_task_of(rt_se); in update_stats_dequeue_rt()
1445 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __enqueue_rt_entity() argument
1447 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __enqueue_rt_entity()
1449 struct rt_rq *group_rq = group_rt_rq(rt_se); in __enqueue_rt_entity()
1450 struct list_head *queue = array->queue + rt_se_prio(rt_se); in __enqueue_rt_entity()
1459 if (rt_se->on_list) in __enqueue_rt_entity()
1460 __delist_rt_entity(rt_se, array); in __enqueue_rt_entity()
1465 WARN_ON_ONCE(rt_se->on_list); in __enqueue_rt_entity()
1467 list_add(&rt_se->run_list, queue); in __enqueue_rt_entity()
1469 list_add_tail(&rt_se->run_list, queue); in __enqueue_rt_entity()
1471 __set_bit(rt_se_prio(rt_se), array->bitmap); in __enqueue_rt_entity()
1472 rt_se->on_list = 1; in __enqueue_rt_entity()
1474 rt_se->on_rq = 1; in __enqueue_rt_entity()
1476 inc_rt_tasks(rt_se, rt_rq); in __enqueue_rt_entity()
1479 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __dequeue_rt_entity() argument
1481 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __dequeue_rt_entity()
1485 WARN_ON_ONCE(!rt_se->on_list); in __dequeue_rt_entity()
1486 __delist_rt_entity(rt_se, array); in __dequeue_rt_entity()
1488 rt_se->on_rq = 0; in __dequeue_rt_entity()
1490 dec_rt_tasks(rt_se, rt_rq); in __dequeue_rt_entity()
1497 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_stack() argument
1502 for_each_sched_rt_entity(rt_se) { in dequeue_rt_stack()
1503 rt_se->back = back; in dequeue_rt_stack()
1504 back = rt_se; in dequeue_rt_stack()
1509 for (rt_se = back; rt_se; rt_se = rt_se->back) { in dequeue_rt_stack()
1510 if (on_rt_rq(rt_se)) in dequeue_rt_stack()
1511 __dequeue_rt_entity(rt_se, flags); in dequeue_rt_stack()
1517 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in enqueue_rt_entity() argument
1519 struct rq *rq = rq_of_rt_se(rt_se); in enqueue_rt_entity()
1521 update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags); in enqueue_rt_entity()
1523 dequeue_rt_stack(rt_se, flags); in enqueue_rt_entity()
1524 for_each_sched_rt_entity(rt_se) in enqueue_rt_entity()
1525 __enqueue_rt_entity(rt_se, flags); in enqueue_rt_entity()
1529 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_entity() argument
1531 struct rq *rq = rq_of_rt_se(rt_se); in dequeue_rt_entity()
1533 update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags); in dequeue_rt_entity()
1535 dequeue_rt_stack(rt_se, flags); in dequeue_rt_entity()
1537 for_each_sched_rt_entity(rt_se) { in dequeue_rt_entity()
1538 struct rt_rq *rt_rq = group_rt_rq(rt_se); in dequeue_rt_entity()
1541 __enqueue_rt_entity(rt_se, flags); in dequeue_rt_entity()
1573 struct sched_rt_entity *rt_se = &p->rt; in enqueue_task_rt() local
1577 rt_se->timeout = 0; in enqueue_task_rt()
1580 update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se); in enqueue_task_rt()
1582 enqueue_rt_entity(rt_se, flags); in enqueue_task_rt()
1591 struct sched_rt_entity *rt_se = &p->rt; in dequeue_task_rt() local
1594 dequeue_rt_entity(rt_se, flags); in dequeue_task_rt()
1604 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) in requeue_rt_entity() argument
1606 if (on_rt_rq(rt_se)) { in requeue_rt_entity()
1608 struct list_head *queue = array->queue + rt_se_prio(rt_se); in requeue_rt_entity()
1611 list_move(&rt_se->run_list, queue); in requeue_rt_entity()
1613 list_move_tail(&rt_se->run_list, queue); in requeue_rt_entity()
1619 struct sched_rt_entity *rt_se = &p->rt; in requeue_task_rt() local
1622 for_each_sched_rt_entity(rt_se) { in requeue_task_rt()
1623 rt_rq = rt_rq_of_se(rt_se); in requeue_task_rt()
1624 requeue_rt_entity(rt_rq, rt_se, head); in requeue_task_rt()
1836 struct sched_rt_entity *rt_se = &p->rt; in set_next_task_rt() local
1841 update_stats_wait_end_rt(rt_rq, rt_se); in set_next_task_rt()
1881 struct sched_rt_entity *rt_se; in _pick_next_task_rt() local
1885 rt_se = pick_next_rt_entity(rt_rq); in _pick_next_task_rt()
1886 if (unlikely(!rt_se)) in _pick_next_task_rt()
1888 rt_rq = group_rt_rq(rt_se); in _pick_next_task_rt()
1891 return rt_task_of(rt_se); in _pick_next_task_rt()
1918 struct sched_rt_entity *rt_se = &p->rt; in put_prev_task_rt() local
1922 update_stats_wait_start_rt(rt_rq, rt_se); in put_prev_task_rt()
2751 struct sched_rt_entity *rt_se = &p->rt; in task_tick_rt() local
2775 for_each_sched_rt_entity(rt_se) { in task_tick_rt()
2776 if (rt_se->run_list.prev != rt_se->run_list.next) { in task_tick_rt()