61 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)  in __blk_mq_tag_busy()  argument
63 	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&  in __blk_mq_tag_busy()
64 	    !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in __blk_mq_tag_busy()
65 		atomic_inc(&hctx->tags->active_queues);  in __blk_mq_tag_busy()
104 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)  in __blk_mq_tag_idle()  argument
106 	struct blk_mq_tags *tags = hctx->tags;  in __blk_mq_tag_idle()
108 	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in __blk_mq_tag_idle()
120 static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,  in hctx_may_queue()  argument
125 	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))  in hctx_may_queue()
127 	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in hctx_may_queue()
136 	users = atomic_read(&hctx->tags->active_queues);  in hctx_may_queue()
144 	return atomic_read(&hctx->nr_active) < depth;  in hctx_may_queue()
191 static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,  in __bt_get()  argument
197 	if (!hctx_may_queue(hctx, bt))  in __bt_get()
244 					 struct blk_mq_hw_ctx *hctx)  in bt_wait_ptr()  argument
249 	if (!hctx)  in bt_wait_ptr()
252 	wait_index = atomic_read(&hctx->wait_index);  in bt_wait_ptr()
254 	bt_index_atomic_inc(&hctx->wait_index);  in bt_wait_ptr()
260 		struct blk_mq_hw_ctx *hctx,  in bt_get()  argument
267 	tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
274 	bs = bt_wait_ptr(bt, hctx);  in bt_get()
278 		tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
288 		if (hctx)  in bt_get()
289 			blk_mq_run_hw_queue(hctx, false);  in bt_get()
295 		tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
304 		data->hctx = data->q->mq_ops->map_queue(data->q,  in bt_get()
307 			bt = &data->hctx->tags->breserved_tags;  in bt_get()
310 			hctx = data->hctx;  in bt_get()
311 			bt = &hctx->tags->bitmap_tags;  in bt_get()
314 		bs = bt_wait_ptr(bt, hctx);  in bt_get()
325 	tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,  in __blk_mq_get_tag()
326 			&data->ctx->last_tag, data->hctx->tags);  in __blk_mq_get_tag()
328 		return tag + data->hctx->tags->nr_reserved_tags;  in __blk_mq_get_tag()
337 	if (unlikely(!data->hctx->tags->nr_reserved_tags)) {  in __blk_mq_get_reserved_tag()
342 	tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero,  in __blk_mq_get_reserved_tag()
343 		data->hctx->tags);  in __blk_mq_get_reserved_tag()
405 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag,  in blk_mq_put_tag()  argument
408 	struct blk_mq_tags *tags = hctx->tags;  in blk_mq_put_tag()
423 static void bt_for_each(struct blk_mq_hw_ctx *hctx,  in bt_for_each()  argument
436 			rq = hctx->tags->rqs[off + bit];  in bt_for_each()
437 			if (rq->q == hctx->queue)  in bt_for_each()
438 				fn(hctx, rq, data, reserved);  in bt_for_each()
481 	struct blk_mq_hw_ctx *hctx;  in blk_mq_queue_tag_busy_iter()  local
490 	queue_for_each_hw_ctx(q, hctx, i) {  in blk_mq_queue_tag_busy_iter()
491 		struct blk_mq_tags *tags = hctx->tags;  in blk_mq_queue_tag_busy_iter()
497 		if (!blk_mq_hw_queue_mapped(hctx))  in blk_mq_queue_tag_busy_iter()
501 			bt_for_each(hctx, &tags->breserved_tags, 0, fn, priv, true);  in blk_mq_queue_tag_busy_iter()
502 		bt_for_each(hctx, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv,  in blk_mq_queue_tag_busy_iter()
694 	struct blk_mq_hw_ctx *hctx;  in blk_mq_unique_tag()  local
698 		hctx = q->mq_ops->map_queue(q, rq->mq_ctx->cpu);  in blk_mq_unique_tag()
699 		hwq = hctx->queue_num;  in blk_mq_unique_tag()