Home
last modified time | relevance | path

Searched refs:hctx (Results 1 – 25 of 62) sorted by relevance

123

/kernel/linux/linux-5.10/block/
Dblk-mq-sched.c51 void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx) in blk_mq_sched_mark_restart_hctx() argument
53 if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) in blk_mq_sched_mark_restart_hctx()
56 set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); in blk_mq_sched_mark_restart_hctx()
60 void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) in blk_mq_sched_restart() argument
62 if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) in blk_mq_sched_restart()
64 clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); in blk_mq_sched_restart()
75 blk_mq_run_hw_queue(hctx, true); in blk_mq_sched_restart()
89 struct blk_mq_hw_ctx *hctx = in blk_mq_dispatch_hctx_list() local
96 if (rq->mq_hctx != hctx) { in blk_mq_dispatch_hctx_list()
105 return blk_mq_dispatch_rq_list(hctx, &hctx_list, count); in blk_mq_dispatch_hctx_list()
[all …]
Dblk-mq.c70 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) in blk_mq_hctx_has_pending() argument
72 return !list_empty_careful(&hctx->dispatch) || in blk_mq_hctx_has_pending()
73 sbitmap_any_bit_set(&hctx->ctx_map) || in blk_mq_hctx_has_pending()
74 blk_mq_sched_has_work(hctx); in blk_mq_hctx_has_pending()
80 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, in blk_mq_hctx_mark_pending() argument
83 const int bit = ctx->index_hw[hctx->type]; in blk_mq_hctx_mark_pending()
85 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) in blk_mq_hctx_mark_pending()
86 sbitmap_set_bit(&hctx->ctx_map, bit); in blk_mq_hctx_mark_pending()
89 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, in blk_mq_hctx_clear_pending() argument
92 const int bit = ctx->index_hw[hctx->type]; in blk_mq_hctx_clear_pending()
[all …]
Dblk-mq.h43 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
47 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
48 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
69 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
73 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
78 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
130 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
161 struct blk_mq_hw_ctx *hctx; member
172 return data->hctx->sched_tags; in blk_mq_tags_from_data()
174 return data->hctx->tags; in blk_mq_tags_from_data()
[all …]
Dblk-mq-sysfs.c36 struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, in blk_mq_hw_sysfs_release() local
39 if (hctx->flags & BLK_MQ_F_BLOCKING) in blk_mq_hw_sysfs_release()
40 cleanup_srcu_struct(hctx->srcu); in blk_mq_hw_sysfs_release()
41 blk_free_flush_queue(hctx->fq); in blk_mq_hw_sysfs_release()
42 sbitmap_free(&hctx->ctx_map); in blk_mq_hw_sysfs_release()
43 free_cpumask_var(hctx->cpumask); in blk_mq_hw_sysfs_release()
44 kfree(hctx->ctxs); in blk_mq_hw_sysfs_release()
45 kfree(hctx); in blk_mq_hw_sysfs_release()
106 struct blk_mq_hw_ctx *hctx; in blk_mq_hw_sysfs_show() local
111 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); in blk_mq_hw_sysfs_show()
[all …]
Dblk-mq-debugfs.c228 struct blk_mq_hw_ctx *hctx = data; in hctx_state_show() local
230 blk_flags_show(m, hctx->state, hctx_state_name, in hctx_state_show()
256 struct blk_mq_hw_ctx *hctx = data; in hctx_flags_show() local
257 const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags); in hctx_flags_show()
267 hctx->flags ^ BLK_ALLOC_POLICY_TO_MQ_FLAG(alloc_policy), in hctx_flags_show()
365 __acquires(&hctx->lock) in hctx_dispatch_start()
367 struct blk_mq_hw_ctx *hctx = m->private; in hctx_dispatch_start() local
369 spin_lock(&hctx->lock); in hctx_dispatch_start()
370 return seq_list_start(&hctx->dispatch, *pos); in hctx_dispatch_start()
375 struct blk_mq_hw_ctx *hctx = m->private; in hctx_dispatch_next() local
[all …]
Dblk-mq-tag.c24 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) in __blk_mq_tag_busy() argument
26 if (blk_mq_is_sbitmap_shared(hctx->flags)) { in __blk_mq_tag_busy()
27 struct request_queue *q = hctx->queue; in __blk_mq_tag_busy()
34 if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) && in __blk_mq_tag_busy()
35 !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) in __blk_mq_tag_busy()
36 atomic_inc(&hctx->tags->active_queues); in __blk_mq_tag_busy()
56 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) in __blk_mq_tag_idle() argument
58 struct blk_mq_tags *tags = hctx->tags; in __blk_mq_tag_idle()
59 struct request_queue *q = hctx->queue; in __blk_mq_tag_idle()
62 if (blk_mq_is_sbitmap_shared(hctx->flags)) { in __blk_mq_tag_idle()
[all …]
Dblk-mq-tag.h43 extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
56 struct blk_mq_hw_ctx *hctx) in bt_wait_ptr() argument
58 if (!hctx) in bt_wait_ptr()
60 return sbq_wait_ptr(bt, &hctx->wait_index); in bt_wait_ptr()
72 static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) in blk_mq_tag_busy() argument
74 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) in blk_mq_tag_busy()
77 return __blk_mq_tag_busy(hctx); in blk_mq_tag_busy()
80 static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) in blk_mq_tag_idle() argument
82 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) in blk_mq_tag_idle()
85 __blk_mq_tag_idle(hctx); in blk_mq_tag_idle()
Dkyber-iosched.c448 static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx) in kyber_depth_updated() argument
450 struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data; in kyber_depth_updated()
451 struct blk_mq_tags *tags = hctx->sched_tags; in kyber_depth_updated()
459 static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) in kyber_init_hctx() argument
464 khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node); in kyber_init_hctx()
468 khd->kcqs = kmalloc_array_node(hctx->nr_ctx, in kyber_init_hctx()
470 GFP_KERNEL, hctx->numa_node); in kyber_init_hctx()
474 for (i = 0; i < hctx->nr_ctx; i++) in kyber_init_hctx()
478 if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx, in kyber_init_hctx()
479 ilog2(8), GFP_KERNEL, hctx->numa_node)) { in kyber_init_hctx()
[all …]
Dblk-mq-debugfs.h24 struct blk_mq_hw_ctx *hctx);
25 void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx);
32 struct blk_mq_hw_ctx *hctx);
33 void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx);
48 struct blk_mq_hw_ctx *hctx) in blk_mq_debugfs_register_hctx() argument
52 static inline void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx) in blk_mq_debugfs_unregister_hctx() argument
73 struct blk_mq_hw_ctx *hctx) in blk_mq_debugfs_register_sched_hctx() argument
77 static inline void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx) in blk_mq_debugfs_unregister_sched_hctx() argument
Dblk-mq-sched.h17 void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx);
18 void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
22 void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx,
26 void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
71 static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx) in blk_mq_sched_has_work() argument
73 struct elevator_queue *e = hctx->queue->elevator; in blk_mq_sched_has_work()
76 return e->type->ops.has_work(hctx); in blk_mq_sched_has_work()
81 static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx) in blk_mq_sched_needs_restart() argument
83 return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); in blk_mq_sched_needs_restart()
Dmq-deadline.c381 static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) in dd_dispatch_request() argument
383 struct deadline_data *dd = hctx->queue->elevator->elevator_data; in dd_dispatch_request()
486 static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, in dd_insert_request() argument
489 struct request_queue *q = hctx->queue; in dd_insert_request()
529 static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, in dd_insert_requests() argument
532 struct request_queue *q = hctx->queue; in dd_insert_requests()
541 dd_insert_request(hctx, rq, at_head); in dd_insert_requests()
542 atomic_inc(&hctx->elevator_queued); in dd_insert_requests()
585 static bool dd_has_work(struct blk_mq_hw_ctx *hctx) in dd_has_work() argument
587 struct deadline_data *dd = hctx->queue->elevator->elevator_data; in dd_has_work()
[all …]
Dblk-flush.c354 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; in mq_flush_data_end_io() local
372 blk_mq_sched_restart(hctx); in mq_flush_data_end_io()
526 void blk_mq_hctx_set_fq_lock_class(struct blk_mq_hw_ctx *hctx, in blk_mq_hctx_set_fq_lock_class() argument
529 lockdep_set_class(&hctx->fq->mq_flush_lock, key); in blk_mq_hctx_set_fq_lock_class()
Dbfq-iosched.c4650 static bool bfq_has_work(struct blk_mq_hw_ctx *hctx) in bfq_has_work() argument
4652 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_has_work()
4654 if (!atomic_read(&hctx->elevator_queued)) in bfq_has_work()
4665 static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) in __bfq_dispatch_request() argument
4667 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in __bfq_dispatch_request()
4805 static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) in bfq_dispatch_request() argument
4807 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_dispatch_request()
4817 rq = __bfq_dispatch_request(hctx); in bfq_dispatch_request()
4824 bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue, in bfq_dispatch_request()
5503 static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, in bfq_insert_request() argument
[all …]
Dbsg-lib.c264 static blk_status_t bsg_queue_rq(struct blk_mq_hw_ctx *hctx, in bsg_queue_rq() argument
267 struct request_queue *q = hctx->queue; in bsg_queue_rq()
/kernel/linux/linux-5.10/include/linux/
Dblk-mq.h513 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
514 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
517 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
521 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
522 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
579 #define queue_for_each_hw_ctx(q, hctx, i) \ argument
581 ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++)
583 #define hctx_for_each_ctx(hctx, ctx, i) \ argument
584 for ((i) = 0; (i) < (hctx)->nr_ctx && \
585 ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++)
[all …]
/kernel/linux/linux-5.10/net/dccp/ccids/
Dccid3.h104 struct ccid3_hc_tx_sock *hctx = ccid_priv(dccp_sk(sk)->dccps_hc_tx_ccid); in ccid3_hc_tx_sk() local
105 BUG_ON(hctx == NULL); in ccid3_hc_tx_sk()
106 return hctx; in ccid3_hc_tx_sk()
/kernel/linux/linux-5.10/drivers/s390/block/
Dscm_blk.c283 static blk_status_t scm_blk_request(struct blk_mq_hw_ctx *hctx, in scm_blk_request() argument
286 struct scm_device *scmdev = hctx->queue->queuedata; in scm_blk_request()
288 struct scm_queue *sq = hctx->driver_data; in scm_blk_request()
332 static int scm_blk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in scm_blk_init_hctx() argument
341 hctx->driver_data = qd; in scm_blk_init_hctx()
346 static void scm_blk_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx) in scm_blk_exit_hctx() argument
348 struct scm_queue *qd = hctx->driver_data; in scm_blk_exit_hctx()
351 kfree(hctx->driver_data); in scm_blk_exit_hctx()
352 hctx->driver_data = NULL; in scm_blk_exit_hctx()
/kernel/linux/linux-5.10/drivers/block/rnbd/
Drnbd-clt.c163 if (WARN_ON(!q->hctx)) in rnbd_clt_dev_requeue()
167 blk_mq_run_hw_queue(q->hctx, true); in rnbd_clt_dev_requeue()
1106 struct blk_mq_hw_ctx *hctx, in rnbd_clt_dev_kick_mq_queue() argument
1109 struct rnbd_queue *q = hctx->driver_data; in rnbd_clt_dev_kick_mq_queue()
1112 blk_mq_delay_run_hw_queue(hctx, delay); in rnbd_clt_dev_kick_mq_queue()
1118 blk_mq_delay_run_hw_queue(hctx, 10/*ms*/); in rnbd_clt_dev_kick_mq_queue()
1121 static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx, in rnbd_queue_rq() argument
1135 rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY); in rnbd_queue_rq()
1144 rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/); in rnbd_queue_rq()
1252 struct blk_mq_hw_ctx *hctx) in rnbd_init_hw_queue() argument
[all …]
Drnbd-clt.h99 struct blk_mq_hw_ctx *hctx; member
/kernel/linux/linux-5.10/drivers/nvme/target/
Dloop.c131 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, in nvme_loop_queue_rq() argument
134 struct nvme_ns *ns = hctx->queue->queuedata; in nvme_loop_queue_rq()
135 struct nvme_loop_queue *queue = hctx->driver_data; in nvme_loop_queue_rq()
216 static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_loop_init_hctx() argument
230 blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key); in nvme_loop_init_hctx()
232 hctx->driver_data = queue; in nvme_loop_init_hctx()
236 static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_loop_init_admin_hctx() argument
244 hctx->driver_data = queue; in nvme_loop_init_admin_hctx()
/kernel/linux/linux-5.10/drivers/block/
Dvirtio_blk.c203 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) in virtio_commit_rqs() argument
205 struct virtio_blk *vblk = hctx->queue->queuedata; in virtio_commit_rqs()
206 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; in virtio_commit_rqs()
217 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, in virtio_queue_rq() argument
220 struct virtio_blk *vblk = hctx->queue->queuedata; in virtio_queue_rq()
225 int qid = hctx->queue_num; in virtio_queue_rq()
269 num = blk_rq_map_sg(hctx->queue, req, vbr->sg); in virtio_queue_rq()
285 blk_mq_stop_hw_queue(hctx); in virtio_queue_rq()
Dnull_blk_main.c1472 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, in null_queue_rq() argument
1476 struct nullb_queue *nq = hctx->driver_data; in null_queue_rq()
1480 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); in null_queue_rq()
1528 static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) in null_exit_hctx() argument
1530 struct nullb_queue *nq = hctx->driver_data; in null_exit_hctx()
1543 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data, in null_init_hctx() argument
1546 struct nullb *nullb = hctx->queue->queuedata; in null_init_hctx()
1555 hctx->driver_data = nq; in null_init_hctx()
/kernel/linux/linux-5.10/drivers/nvme/host/
Dpci.c396 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_admin_init_hctx() argument
403 WARN_ON(dev->admin_tagset.tags[0] != hctx->tags); in nvme_admin_init_hctx()
405 hctx->driver_data = nvmeq; in nvme_admin_init_hctx()
409 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_init_hctx() argument
415 WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags); in nvme_init_hctx()
416 hctx->driver_data = nvmeq; in nvme_init_hctx()
513 static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx) in nvme_commit_rqs() argument
515 struct nvme_queue *nvmeq = hctx->driver_data; in nvme_commit_rqs()
912 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, in nvme_queue_rq() argument
915 struct nvme_ns *ns = hctx->queue->queuedata; in nvme_queue_rq()
[all …]
Dtcp.c443 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_tcp_init_hctx() argument
449 hctx->driver_data = queue; in nvme_tcp_init_hctx()
453 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_tcp_init_admin_hctx() argument
459 hctx->driver_data = queue; in nvme_tcp_init_admin_hctx()
2329 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) in nvme_tcp_commit_rqs() argument
2331 struct nvme_tcp_queue *queue = hctx->driver_data; in nvme_tcp_commit_rqs()
2337 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, in nvme_tcp_queue_rq() argument
2340 struct nvme_ns *ns = hctx->queue->queuedata; in nvme_tcp_queue_rq()
2341 struct nvme_tcp_queue *queue = hctx->driver_data; in nvme_tcp_queue_rq()
2406 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) in nvme_tcp_poll() argument
[all …]
Dfc.c37 struct blk_mq_hw_ctx *hctx; member
2190 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl, in __nvme_fc_init_hctx() argument
2195 hctx->driver_data = queue; in __nvme_fc_init_hctx()
2196 queue->hctx = hctx; in __nvme_fc_init_hctx()
2200 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_fc_init_hctx() argument
2205 __nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1); in nvme_fc_init_hctx()
2211 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, in nvme_fc_init_admin_hctx() argument
2216 __nvme_fc_init_hctx(hctx, ctrl, hctx_idx); in nvme_fc_init_admin_hctx()
2768 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx, in nvme_fc_queue_rq() argument
2771 struct nvme_ns *ns = hctx->queue->queuedata; in nvme_fc_queue_rq()
[all …]

123