300 	struct cfqg_stats stats;	/* stats for this cfqg */  member
466 static inline void cfqg_stats_mark_##name(struct cfqg_stats *stats)	\
468 	stats->flags |= (1 << CFQG_stats_##name);			\
470 static inline void cfqg_stats_clear_##name(struct cfqg_stats *stats)	\
472 	stats->flags &= ~(1 << CFQG_stats_##name);			\
474 static inline int cfqg_stats_##name(struct cfqg_stats *stats)		\
476 	return (stats->flags & (1 << CFQG_stats_##name)) != 0;		\
485 static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)  in CFQG_FLAG_FNS()
489 	if (!cfqg_stats_waiting(stats))  in CFQG_FLAG_FNS()
493 	if (time_after64(now, stats->start_group_wait_time))  in CFQG_FLAG_FNS()
494 		blkg_stat_add(&stats->group_wait_time,  in CFQG_FLAG_FNS()
495 			      now - stats->start_group_wait_time);  in CFQG_FLAG_FNS()
496 	cfqg_stats_clear_waiting(stats);  in CFQG_FLAG_FNS()
503 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_group_wait_time()  local
505 	if (cfqg_stats_waiting(stats))  in cfqg_stats_set_start_group_wait_time()
509 	stats->start_group_wait_time = sched_clock();  in cfqg_stats_set_start_group_wait_time()
510 	cfqg_stats_mark_waiting(stats);  in cfqg_stats_set_start_group_wait_time()
514 static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)  in cfqg_stats_end_empty_time()  argument
518 	if (!cfqg_stats_empty(stats))  in cfqg_stats_end_empty_time()
522 	if (time_after64(now, stats->start_empty_time))  in cfqg_stats_end_empty_time()
523 		blkg_stat_add(&stats->empty_time,  in cfqg_stats_end_empty_time()
524 			      now - stats->start_empty_time);  in cfqg_stats_end_empty_time()
525 	cfqg_stats_clear_empty(stats);  in cfqg_stats_end_empty_time()
530 	blkg_stat_add(&cfqg->stats.dequeue, 1);  in cfqg_stats_update_dequeue()
535 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_empty_time()  local
537 	if (blkg_rwstat_total(&stats->queued))  in cfqg_stats_set_start_empty_time()
545 	if (cfqg_stats_empty(stats))  in cfqg_stats_set_start_empty_time()
548 	stats->start_empty_time = sched_clock();  in cfqg_stats_set_start_empty_time()
549 	cfqg_stats_mark_empty(stats);  in cfqg_stats_set_start_empty_time()
554 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_idle_time()  local
556 	if (cfqg_stats_idling(stats)) {  in cfqg_stats_update_idle_time()
559 		if (time_after64(now, stats->start_idle_time))  in cfqg_stats_update_idle_time()
560 			blkg_stat_add(&stats->idle_time,  in cfqg_stats_update_idle_time()
561 				      now - stats->start_idle_time);  in cfqg_stats_update_idle_time()
562 		cfqg_stats_clear_idling(stats);  in cfqg_stats_update_idle_time()
568 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_idle_time()  local
570 	BUG_ON(cfqg_stats_idling(stats));  in cfqg_stats_set_start_idle_time()
572 	stats->start_idle_time = sched_clock();  in cfqg_stats_set_start_idle_time()
573 	cfqg_stats_mark_idling(stats);  in cfqg_stats_set_start_idle_time()
578 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_avg_queue_size()  local
580 	blkg_stat_add(&stats->avg_queue_size_sum,  in cfqg_stats_update_avg_queue_size()
581 		      blkg_rwstat_total(&stats->queued));  in cfqg_stats_update_avg_queue_size()
582 	blkg_stat_add(&stats->avg_queue_size_samples, 1);  in cfqg_stats_update_avg_queue_size()
583 	cfqg_stats_update_group_wait_time(stats);  in cfqg_stats_update_avg_queue_size()
589 static inline void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }  in cfqg_stats_end_empty_time()  argument
665 	blkg_rwstat_add(&cfqg->stats.queued, rw, 1);  in cfqg_stats_update_io_add()
666 	cfqg_stats_end_empty_time(&cfqg->stats);  in cfqg_stats_update_io_add()
673 	blkg_stat_add(&cfqg->stats.time, time);  in cfqg_stats_update_timeslice_used()
675 	blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);  in cfqg_stats_update_timeslice_used()
681 	blkg_rwstat_add(&cfqg->stats.queued, rw, -1);  in cfqg_stats_update_io_remove()
686 	blkg_rwstat_add(&cfqg->stats.merged, rw, 1);  in cfqg_stats_update_io_merged()
692 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_completion()  local
696 		blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);  in cfqg_stats_update_completion()
698 		blkg_rwstat_add(&stats->wait_time, rw,  in cfqg_stats_update_completion()
703 static void cfqg_stats_reset(struct cfqg_stats *stats)  in cfqg_stats_reset()  argument
706 	blkg_rwstat_reset(&stats->merged);  in cfqg_stats_reset()
707 	blkg_rwstat_reset(&stats->service_time);  in cfqg_stats_reset()
708 	blkg_rwstat_reset(&stats->wait_time);  in cfqg_stats_reset()
709 	blkg_stat_reset(&stats->time);  in cfqg_stats_reset()
711 	blkg_stat_reset(&stats->unaccounted_time);  in cfqg_stats_reset()
712 	blkg_stat_reset(&stats->avg_queue_size_sum);  in cfqg_stats_reset()
713 	blkg_stat_reset(&stats->avg_queue_size_samples);  in cfqg_stats_reset()
714 	blkg_stat_reset(&stats->dequeue);  in cfqg_stats_reset()
715 	blkg_stat_reset(&stats->group_wait_time);  in cfqg_stats_reset()
716 	blkg_stat_reset(&stats->idle_time);  in cfqg_stats_reset()
717 	blkg_stat_reset(&stats->empty_time);  in cfqg_stats_reset()
754 	cfqg_stats_add_aux(&parent->stats, &cfqg->stats);  in cfqg_stats_xfer_dead()
755 	cfqg_stats_reset(&cfqg->stats);  in cfqg_stats_xfer_dead()
1528 static void cfqg_stats_exit(struct cfqg_stats *stats)  in cfqg_stats_exit()  argument
1530 	blkg_rwstat_exit(&stats->merged);  in cfqg_stats_exit()
1531 	blkg_rwstat_exit(&stats->service_time);  in cfqg_stats_exit()
1532 	blkg_rwstat_exit(&stats->wait_time);  in cfqg_stats_exit()
1533 	blkg_rwstat_exit(&stats->queued);  in cfqg_stats_exit()
1534 	blkg_stat_exit(&stats->time);  in cfqg_stats_exit()
1536 	blkg_stat_exit(&stats->unaccounted_time);  in cfqg_stats_exit()
1537 	blkg_stat_exit(&stats->avg_queue_size_sum);  in cfqg_stats_exit()
1538 	blkg_stat_exit(&stats->avg_queue_size_samples);  in cfqg_stats_exit()
1539 	blkg_stat_exit(&stats->dequeue);  in cfqg_stats_exit()
1540 	blkg_stat_exit(&stats->group_wait_time);  in cfqg_stats_exit()
1541 	blkg_stat_exit(&stats->idle_time);  in cfqg_stats_exit()
1542 	blkg_stat_exit(&stats->empty_time);  in cfqg_stats_exit()
1546 static int cfqg_stats_init(struct cfqg_stats *stats, gfp_t gfp)  in cfqg_stats_init()  argument
1548 	if (blkg_rwstat_init(&stats->merged, gfp) ||  in cfqg_stats_init()
1549 	    blkg_rwstat_init(&stats->service_time, gfp) ||  in cfqg_stats_init()
1550 	    blkg_rwstat_init(&stats->wait_time, gfp) ||  in cfqg_stats_init()
1551 	    blkg_rwstat_init(&stats->queued, gfp) ||  in cfqg_stats_init()
1552 	    blkg_stat_init(&stats->time, gfp))  in cfqg_stats_init()
1556 	if (blkg_stat_init(&stats->unaccounted_time, gfp) ||  in cfqg_stats_init()
1557 	    blkg_stat_init(&stats->avg_queue_size_sum, gfp) ||  in cfqg_stats_init()
1558 	    blkg_stat_init(&stats->avg_queue_size_samples, gfp) ||  in cfqg_stats_init()
1559 	    blkg_stat_init(&stats->dequeue, gfp) ||  in cfqg_stats_init()
1560 	    blkg_stat_init(&stats->group_wait_time, gfp) ||  in cfqg_stats_init()
1561 	    blkg_stat_init(&stats->idle_time, gfp) ||  in cfqg_stats_init()
1562 	    blkg_stat_init(&stats->empty_time, gfp))  in cfqg_stats_init()
1567 	cfqg_stats_exit(stats);  in cfqg_stats_init()
1621 	if (cfqg_stats_init(&cfqg->stats, gfp)) {  in cfq_pd_alloc()
1666 	cfqg_stats_exit(&cfqg->stats);  in cfq_pd_free()
1674 	cfqg_stats_reset(&cfqg->stats);  in cfq_pd_reset_stats()
1963 	u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);  in cfqg_prfill_avg_queue_size()
1967 		v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);  in cfqg_prfill_avg_queue_size()
2027 		.private = offsetof(struct cfq_group, stats.time),
2046 		.private = offsetof(struct cfq_group, stats.service_time),
2051 		.private = offsetof(struct cfq_group, stats.wait_time),
2056 		.private = offsetof(struct cfq_group, stats.merged),
2061 		.private = offsetof(struct cfq_group, stats.queued),
2068 		.private = offsetof(struct cfq_group, stats.time),
2087 		.private = offsetof(struct cfq_group, stats.service_time),
2092 		.private = offsetof(struct cfq_group, stats.wait_time),
2097 		.private = offsetof(struct cfq_group, stats.merged),
2102 		.private = offsetof(struct cfq_group, stats.queued),
2112 		.private = offsetof(struct cfq_group, stats.group_wait_time),
2117 		.private = offsetof(struct cfq_group, stats.idle_time),
2122 		.private = offsetof(struct cfq_group, stats.empty_time),
2127 		.private = offsetof(struct cfq_group, stats.dequeue),
2132 		.private = offsetof(struct cfq_group, stats.unaccounted_time),