Lines Matching refs:group
180 static void group_init(struct psi_group *group) in group_init() argument
184 group->enabled = true; in group_init()
186 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq); in group_init()
187 group->avg_last_update = sched_clock(); in group_init()
188 group->avg_next_update = group->avg_last_update + psi_period; in group_init()
189 INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); in group_init()
190 mutex_init(&group->avgs_lock); in group_init()
192 atomic_set(&group->poll_scheduled, 0); in group_init()
193 mutex_init(&group->trigger_lock); in group_init()
194 INIT_LIST_HEAD(&group->triggers); in group_init()
195 group->poll_min_period = U32_MAX; in group_init()
196 group->polling_next_update = ULLONG_MAX; in group_init()
197 init_waitqueue_head(&group->poll_wait); in group_init()
198 timer_setup(&group->poll_timer, poll_timer_fn, 0); in group_init()
199 rcu_assign_pointer(group->poll_task, NULL); in group_init()
241 static void get_recent_times(struct psi_group *group, int cpu, in get_recent_times() argument
245 struct psi_group_cpu *groupc = per_cpu_ptr(group->pcpu, cpu); in get_recent_times()
306 static void collect_percpu_times(struct psi_group *group, in collect_percpu_times() argument
329 get_recent_times(group, cpu, aggregator, times, in collect_percpu_times()
354 group->total[aggregator][s] += in collect_percpu_times()
361 static u64 update_averages(struct psi_group *group, u64 now) in update_averages() argument
369 expires = group->avg_next_update; in update_averages()
381 period = now - (group->avg_last_update + (missed_periods * psi_period)); in update_averages()
382 group->avg_last_update = now; in update_averages()
387 sample = group->total[PSI_AVGS][s] - group->avg_total[s]; in update_averages()
407 group->avg_total[s] += sample; in update_averages()
408 calc_avgs(group->avg[s], missed_periods, sample, period); in update_averages()
417 struct psi_group *group; in psi_avgs_work() local
423 group = container_of(dwork, struct psi_group, avgs_work); in psi_avgs_work()
425 mutex_lock(&group->avgs_lock); in psi_avgs_work()
429 collect_percpu_times(group, PSI_AVGS, &changed_states); in psi_avgs_work()
438 if (now >= group->avg_next_update) in psi_avgs_work()
439 group->avg_next_update = update_averages(group, now); in psi_avgs_work()
443 group->avg_next_update - now) + 1); in psi_avgs_work()
446 mutex_unlock(&group->avgs_lock); in psi_avgs_work()
495 static void init_triggers(struct psi_group *group, u64 now) in init_triggers() argument
499 list_for_each_entry(t, &group->triggers, node) in init_triggers()
501 group->total[PSI_POLL][t->state], 0); in init_triggers()
502 memcpy(group->polling_total, group->total[PSI_POLL], in init_triggers()
503 sizeof(group->polling_total)); in init_triggers()
504 group->polling_next_update = now + group->poll_min_period; in init_triggers()
507 static u64 update_triggers(struct psi_group *group, u64 now) in update_triggers() argument
511 u64 *total = group->total[PSI_POLL]; in update_triggers()
517 list_for_each_entry(t, &group->triggers, node) { in update_triggers()
521 new_stall = group->polling_total[t->state] != total[t->state]; in update_triggers()
563 memcpy(group->polling_total, total, in update_triggers()
564 sizeof(group->polling_total)); in update_triggers()
566 return now + group->poll_min_period; in update_triggers()
570 static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay, in psi_schedule_poll_work() argument
579 if (atomic_xchg(&group->poll_scheduled, 1) && !force) in psi_schedule_poll_work()
584 task = rcu_dereference(group->poll_task); in psi_schedule_poll_work()
590 mod_timer(&group->poll_timer, jiffies + delay); in psi_schedule_poll_work()
592 atomic_set(&group->poll_scheduled, 0); in psi_schedule_poll_work()
597 static void psi_poll_work(struct psi_group *group) in psi_poll_work() argument
603 mutex_lock(&group->trigger_lock); in psi_poll_work()
607 if (now > group->polling_until) { in psi_poll_work()
617 atomic_set(&group->poll_scheduled, 0); in psi_poll_work()
644 collect_percpu_times(group, PSI_POLL, &changed_states); in psi_poll_work()
646 if (changed_states & group->poll_states) { in psi_poll_work()
648 if (now > group->polling_until) in psi_poll_work()
649 init_triggers(group, now); in psi_poll_work()
656 group->polling_until = now + in psi_poll_work()
657 group->poll_min_period * UPDATES_PER_WINDOW; in psi_poll_work()
660 if (now > group->polling_until) { in psi_poll_work()
661 group->polling_next_update = ULLONG_MAX; in psi_poll_work()
665 if (now >= group->polling_next_update) in psi_poll_work()
666 group->polling_next_update = update_triggers(group, now); in psi_poll_work()
668 psi_schedule_poll_work(group, in psi_poll_work()
669 nsecs_to_jiffies(group->polling_next_update - now) + 1, in psi_poll_work()
673 mutex_unlock(&group->trigger_lock); in psi_poll_work()
678 struct psi_group *group = (struct psi_group *)data; in psi_poll_worker() local
683 wait_event_interruptible(group->poll_wait, in psi_poll_worker()
684 atomic_cmpxchg(&group->poll_wakeup, 1, 0) || in psi_poll_worker()
689 psi_poll_work(group); in psi_poll_worker()
696 struct psi_group *group = from_timer(group, t, poll_timer); in poll_timer_fn() local
698 atomic_set(&group->poll_wakeup, 1); in poll_timer_fn()
699 wake_up_interruptible(&group->poll_wait); in poll_timer_fn()
731 static void psi_group_change(struct psi_group *group, int cpu, in psi_group_change() argument
740 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_group_change()
791 if (!group->enabled) { in psi_group_change()
830 if (state_mask & group->poll_states) in psi_group_change()
831 psi_schedule_poll_work(group, 1, false); in psi_group_change()
833 if (wake_clock && !delayed_work_pending(&group->avgs_work)) in psi_group_change()
834 schedule_delayed_work(&group->avgs_work, PSI_FREQ); in psi_group_change()
864 struct psi_group *group; in psi_task_change() local
874 group = task_psi_group(task); in psi_task_change()
876 psi_group_change(group, cpu, clear, set, now, true); in psi_task_change()
877 } while ((group = group->parent)); in psi_task_change()
883 struct psi_group *group, *common = NULL; in psi_task_switch() local
894 group = task_psi_group(next); in psi_task_switch()
896 if (per_cpu_ptr(group->pcpu, cpu)->state_mask & in psi_task_switch()
898 common = group; in psi_task_switch()
902 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true); in psi_task_switch()
903 } while ((group = group->parent)); in psi_task_switch()
936 group = task_psi_group(prev); in psi_task_switch()
938 if (group == common) in psi_task_switch()
940 psi_group_change(group, cpu, clear, set, now, wake_clock); in psi_task_switch()
941 } while ((group = group->parent)); in psi_task_switch()
951 for (; group; group = group->parent) in psi_task_switch()
952 psi_group_change(group, cpu, clear, set, now, wake_clock); in psi_task_switch()
961 struct psi_group *group; in psi_account_irqtime() local
970 group = task_psi_group(task); in psi_account_irqtime()
972 if (!group->enabled) in psi_account_irqtime()
975 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_account_irqtime()
984 if (group->poll_states & (1 << PSI_IRQ_FULL)) in psi_account_irqtime()
985 psi_schedule_poll_work(group, 1, false); in psi_account_irqtime()
986 } while ((group = group->parent)); in psi_account_irqtime()
1151 void psi_cgroup_restart(struct psi_group *group) in psi_cgroup_restart() argument
1170 if (!group->enabled) in psi_cgroup_restart()
1180 psi_group_change(group, cpu, 0, 0, now, true); in psi_cgroup_restart()
1186 int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) in psi_show() argument
1196 mutex_lock(&group->avgs_lock); in psi_show()
1198 collect_percpu_times(group, PSI_AVGS, NULL); in psi_show()
1199 if (now >= group->avg_next_update) in psi_show()
1200 group->avg_next_update = update_averages(group, now); in psi_show()
1201 mutex_unlock(&group->avgs_lock); in psi_show()
1213 if (!(group == &psi_system && res == PSI_CPU && full)) { in psi_show()
1215 avg[w] = group->avg[res * 2 + full][w]; in psi_show()
1216 total = div_u64(group->total[PSI_AVGS][res * 2 + full], in psi_show()
1231 struct psi_trigger *psi_trigger_create(struct psi_group *group, in psi_trigger_create() argument
1268 t->group = group; in psi_trigger_create()
1273 group->total[PSI_POLL][t->state], 0); in psi_trigger_create()
1280 mutex_lock(&group->trigger_lock); in psi_trigger_create()
1282 if (!rcu_access_pointer(group->poll_task)) { in psi_trigger_create()
1285 task = kthread_create(psi_poll_worker, group, "psimon"); in psi_trigger_create()
1288 mutex_unlock(&group->trigger_lock); in psi_trigger_create()
1291 atomic_set(&group->poll_wakeup, 0); in psi_trigger_create()
1293 rcu_assign_pointer(group->poll_task, task); in psi_trigger_create()
1296 list_add(&t->node, &group->triggers); in psi_trigger_create()
1297 group->poll_min_period = min(group->poll_min_period, in psi_trigger_create()
1299 group->nr_triggers[t->state]++; in psi_trigger_create()
1300 group->poll_states |= (1 << t->state); in psi_trigger_create()
1302 mutex_unlock(&group->trigger_lock); in psi_trigger_create()
1309 struct psi_group *group; in psi_trigger_destroy() local
1319 group = t->group; in psi_trigger_destroy()
1327 mutex_lock(&group->trigger_lock); in psi_trigger_destroy()
1334 group->nr_triggers[t->state]--; in psi_trigger_destroy()
1335 if (!group->nr_triggers[t->state]) in psi_trigger_destroy()
1336 group->poll_states &= ~(1 << t->state); in psi_trigger_destroy()
1338 list_for_each_entry(tmp, &group->triggers, node) in psi_trigger_destroy()
1341 group->poll_min_period = period; in psi_trigger_destroy()
1343 if (group->poll_states == 0) { in psi_trigger_destroy()
1344 group->polling_until = 0; in psi_trigger_destroy()
1346 group->poll_task, in psi_trigger_destroy()
1347 lockdep_is_held(&group->trigger_lock)); in psi_trigger_destroy()
1348 rcu_assign_pointer(group->poll_task, NULL); in psi_trigger_destroy()
1349 del_timer(&group->poll_timer); in psi_trigger_destroy()
1353 mutex_unlock(&group->trigger_lock); in psi_trigger_destroy()
1371 atomic_set(&group->poll_scheduled, 0); in psi_trigger_destroy()