461 static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_init_hctx()  argument
463 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_init_hctx()
467 	khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
471 	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,  in kyber_init_hctx()
473 				       GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
477 	for (i = 0; i < hctx->nr_ctx; i++)  in kyber_init_hctx()
481 		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,  in kyber_init_hctx()
482 				      ilog2(8), GFP_KERNEL, hctx->numa_node)) {  in kyber_init_hctx()
496 		khd->domain_wait[i].wait.private = hctx;  in kyber_init_hctx()
504 	hctx->sched_data = khd;  in kyber_init_hctx()
505 	sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags,  in kyber_init_hctx()
517 static void kyber_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_exit_hctx()  argument
519 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_exit_hctx()
525 	kfree(hctx->sched_data);  in kyber_exit_hctx()
569 	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx);  in kyber_bio_merge()  local
570 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_bio_merge()
571 	struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]];  in kyber_bio_merge()
577 	merged = blk_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);  in kyber_bio_merge()
588 static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,  in kyber_insert_requests()  argument
591 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_insert_requests()
596 		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw[hctx->type]];  in kyber_insert_requests()
605 				rq->mq_ctx->index_hw[hctx->type]);  in kyber_insert_requests()
695 	struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);  in kyber_domain_wake()  local
699 	blk_mq_run_hw_queue(hctx, true);  in kyber_domain_wake()
705 				  struct blk_mq_hw_ctx *hctx)  in kyber_get_domain_token()  argument
753 			  struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_cur_domain()  argument
771 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
782 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
800 static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_request()  argument
802 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_dispatch_request()
803 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_dispatch_request()
814 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
835 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
846 static bool kyber_has_work(struct blk_mq_hw_ctx *hctx)  in kyber_has_work()  argument
848 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_has_work()
910 	struct blk_mq_hw_ctx *hctx = m->private;			\
911 	struct kyber_hctx_data *khd = hctx->sched_data;			\
920 	struct blk_mq_hw_ctx *hctx = m->private;			\
921 	struct kyber_hctx_data *khd = hctx->sched_data;			\
929 	struct blk_mq_hw_ctx *hctx = m->private;			\
930 	struct kyber_hctx_data *khd = hctx->sched_data;			\
944 	struct blk_mq_hw_ctx *hctx = data;				\
945 	struct kyber_hctx_data *khd = hctx->sched_data;			\
968 	struct blk_mq_hw_ctx *hctx = data;  in kyber_cur_domain_show()  local
969 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_cur_domain_show()
977 	struct blk_mq_hw_ctx *hctx = data;  in kyber_batching_show()  local
978 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_batching_show()