245 	struct psi_group_cpu *groupc = per_cpu_ptr(group->pcpu, cpu);  in get_recent_times()  local
255 		seq = read_seqcount_begin(&groupc->seq);  in get_recent_times()
257 		memcpy(times, groupc->times, sizeof(groupc->times));  in get_recent_times()
258 		state_mask = groupc->state_mask;  in get_recent_times()
259 		state_start = groupc->state_start;  in get_recent_times()
260 	} while (read_seqcount_retry(&groupc->seq, seq));  in get_recent_times()
277 		delta = times[s] - groupc->times_prev[aggregator][s];  in get_recent_times()
278 		groupc->times_prev[aggregator][s] = times[s];  in get_recent_times()
702 static void record_times(struct psi_group_cpu *groupc, u64 now)  in record_times()  argument
706 	delta = now - groupc->state_start;  in record_times()
707 	groupc->state_start = now;  in record_times()
709 	if (groupc->state_mask & (1 << PSI_IO_SOME)) {  in record_times()
710 		groupc->times[PSI_IO_SOME] += delta;  in record_times()
711 		if (groupc->state_mask & (1 << PSI_IO_FULL))  in record_times()
712 			groupc->times[PSI_IO_FULL] += delta;  in record_times()
715 	if (groupc->state_mask & (1 << PSI_MEM_SOME)) {  in record_times()
716 		groupc->times[PSI_MEM_SOME] += delta;  in record_times()
717 		if (groupc->state_mask & (1 << PSI_MEM_FULL))  in record_times()
718 			groupc->times[PSI_MEM_FULL] += delta;  in record_times()
721 	if (groupc->state_mask & (1 << PSI_CPU_SOME)) {  in record_times()
722 		groupc->times[PSI_CPU_SOME] += delta;  in record_times()
723 		if (groupc->state_mask & (1 << PSI_CPU_FULL))  in record_times()
724 			groupc->times[PSI_CPU_FULL] += delta;  in record_times()
727 	if (groupc->state_mask & (1 << PSI_NONIDLE))  in record_times()
728 		groupc->times[PSI_NONIDLE] += delta;  in record_times()
735 	struct psi_group_cpu *groupc;  in psi_group_change()  local
740 	groupc = per_cpu_ptr(group->pcpu, cpu);  in psi_group_change()
751 	write_seqcount_begin(&groupc->seq);  in psi_group_change()
766 		state_mask = groupc->state_mask & PSI_ONCPU;  in psi_group_change()
776 		if (groupc->tasks[t]) {  in psi_group_change()
777 			groupc->tasks[t]--;  in psi_group_change()
780 					cpu, t, groupc->tasks[0],  in psi_group_change()
781 					groupc->tasks[1], groupc->tasks[2],  in psi_group_change()
782 					groupc->tasks[3], clear, set);  in psi_group_change()
789 			groupc->tasks[t]++;  in psi_group_change()
799 		if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE)))  in psi_group_change()
800 			record_times(groupc, now);  in psi_group_change()
802 		groupc->state_mask = state_mask;  in psi_group_change()
804 		write_seqcount_end(&groupc->seq);  in psi_group_change()
809 		if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU))  in psi_group_change()
824 	record_times(groupc, now);  in psi_group_change()
826 	groupc->state_mask = state_mask;  in psi_group_change()
828 	write_seqcount_end(&groupc->seq);  in psi_group_change()
962 	struct psi_group_cpu *groupc;  in psi_account_irqtime()  local
975 		groupc = per_cpu_ptr(group->pcpu, cpu);  in psi_account_irqtime()
977 		write_seqcount_begin(&groupc->seq);  in psi_account_irqtime()
979 		record_times(groupc, now);  in psi_account_irqtime()
980 		groupc->times[PSI_IRQ_FULL] += delta;  in psi_account_irqtime()
982 		write_seqcount_end(&groupc->seq);  in psi_account_irqtime()