1 #include "blk-rq-qos.h"
27 	return atomic_inc_below(&rq_wait->inflight, limit);  in rq_wait_inc_below()
34 	for (rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_cleanup()
35 		if (rqos->ops->cleanup)  in rq_qos_cleanup()
36 			rqos->ops->cleanup(rqos, bio);  in rq_qos_cleanup()
44 	for (rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_done()
45 		if (rqos->ops->done)  in rq_qos_done()
46 			rqos->ops->done(rqos, rq);  in rq_qos_done()
54 	for(rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_issue()
55 		if (rqos->ops->issue)  in rq_qos_issue()
56 			rqos->ops->issue(rqos, rq);  in rq_qos_issue()
64 	for(rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_requeue()
65 		if (rqos->ops->requeue)  in rq_qos_requeue()
66 			rqos->ops->requeue(rqos, rq);  in rq_qos_requeue()
75 	for(rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_throttle()
76 		if (rqos->ops->throttle)  in rq_qos_throttle()
77 			rqos->ops->throttle(rqos, bio, lock);  in rq_qos_throttle()
85 	for(rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_track()
86 		if (rqos->ops->track)  in rq_qos_track()
87 			rqos->ops->track(rqos, rq, bio);  in rq_qos_track()
95 	for(rqos = q->rq_qos; rqos; rqos = rqos->next) {  in rq_qos_done_bio()
96 		if (rqos->ops->done_bio)  in rq_qos_done_bio()
97 			rqos->ops->done_bio(rqos, bio);  in rq_qos_done_bio()
102  * Return true, if we can't increase the depth further by scaling
114 	 * scaling down, then keep a setting of 1/1/1.  in rq_depth_calc_max_depth()
116 	if (rqd->queue_depth == 1) {  in rq_depth_calc_max_depth()
117 		if (rqd->scale_step > 0)  in rq_depth_calc_max_depth()
118 			rqd->max_depth = 1;  in rq_depth_calc_max_depth()
120 			rqd->max_depth = 2;  in rq_depth_calc_max_depth()
131 		depth = min_t(unsigned int, rqd->default_depth,  in rq_depth_calc_max_depth()
132 			      rqd->queue_depth);  in rq_depth_calc_max_depth()
133 		if (rqd->scale_step > 0)  in rq_depth_calc_max_depth()
134 			depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));  in rq_depth_calc_max_depth()
135 		else if (rqd->scale_step < 0) {  in rq_depth_calc_max_depth()
136 			unsigned int maxd = 3 * rqd->queue_depth / 4;  in rq_depth_calc_max_depth()
138 			depth = 1 + ((depth - 1) << -rqd->scale_step);  in rq_depth_calc_max_depth()
145 		rqd->max_depth = depth;  in rq_depth_calc_max_depth()
151 /* Returns true on success and false if scaling up wasn't possible */
157 	if (rqd->scaled_max)  in rq_depth_scale_up()
160 	rqd->scale_step--;  in rq_depth_scale_up()
162 	rqd->scaled_max = rq_depth_calc_max_depth(rqd);  in rq_depth_scale_up()
167  * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
169  * scaling down wasn't possible.
174 	 * Stop scaling down when we've hit the limit. This also prevents  in rq_depth_scale_down()
175 	 * ->scale_step from going to crazy values, if the device can't  in rq_depth_scale_down()
178 	if (rqd->max_depth == 1)  in rq_depth_scale_down()
181 	if (rqd->scale_step < 0 && hard_throttle)  in rq_depth_scale_down()
182 		rqd->scale_step = 0;  in rq_depth_scale_down()
184 		rqd->scale_step++;  in rq_depth_scale_down()
186 	rqd->scaled_max = false;  in rq_depth_scale_down()
193 	while (q->rq_qos) {  in rq_qos_exit()
194 		struct rq_qos *rqos = q->rq_qos;  in rq_qos_exit()
195 		q->rq_qos = rqos->next;  in rq_qos_exit()
196 		rqos->ops->exit(rqos);  in rq_qos_exit()